← Back to branch summary

~jcsackett/charmworld/bac-tag-constraints

« back to all changes in this revision

Viewing changes to charmworld/jobs/scan.py

  • Committer: kapil.foss at gmail
  • Date: 2011-10-10 05:03:21 UTC
  • Revision ID: kapil.foss@gmail.com-20111010050321-0ky828632m92nx2o
work in progress on multi namespace support

Show diffs side-by-side

added added

removed removed

Lines of Context:
charmworld/jobs/scan.py
9
9
log = logging.getLogger("charm.scan")
10
10
 
11
11
 
12
 
def scan_charm(db, name, charm_dir, **md):
13
 
    revision_path = os.path.join(charm_dir, "revision")
14
 
    metadata_path = os.path.join(charm_dir, "metadata.yaml")
15
 
    config_path = os.path.join(charm_dir, "config.yaml")
 
12
def scan_charm(db, charm_data):
 
13
    revision_path = os.path.join(
 
14
        charm_data["branch_dir"], "revision")
 
15
    metadata_path = os.path.join(
 
16
        charm_data["branch_dir"], "metadata.yaml")
 
17
    config_path = os.path.join(
 
18
        charm_data["branch_dir"], "config.yaml")
16
19
 
17
20
    if not os.path.exists(metadata_path):
18
 
        log.info("Charm has no metadata: %s", name)
 
21
        log.info("Charm has no metadata: %s", charm_data["branch_spec"])
19
22
        return
20
23
 
21
24
    with open(metadata_path) as fh:
22
25
        try:
23
26
            metadata = yaml.load(fh.read())
24
27
        except Exception, e:
25
 
            log.info("Invalid charm metadata %s: %s", name, e)
 
28
            log.info(
 
29
                "Invalid charm metadata %s: %s", charm_data["branch_spec"], e)
26
30
            return
27
31
 
28
32
    if os.path.exists(config_path):
30
34
            config_raw = open(config_path).read()
31
35
            config = yaml.load(config_raw)
32
36
        except:
33
 
            log.info("Invalid config metadata %s", name)
 
37
            log.info("Invalid config metadata %s", charm_data["branch_spec"])
34
38
            return
35
 
 
36
39
        metadata["config"] = config
37
40
        metadata["config_raw"] = config_raw
38
41
 
40
43
        try:
41
44
            rev_id = int(open(revision_path).read().strip())
42
45
        except:
43
 
            log.info("Invalid revision %s", name)
 
46
            log.info("Invalid revision %s", charm_data["branch_spec"])
44
47
        metadata["revision"] = rev_id
 
48
    elif not "revision" in metadata:
 
49
        log.info("Invalid revision %s", charm_data["branch_spec"])
 
50
        metadata["revision"] = 0
45
51
 
46
 
    hook_dir = os.path.join(charm_dir, "hooks")
 
52
    hook_dir = os.path.join(charm_data["branch_dir"], "hooks")
47
53
    hooks = []
48
54
    if os.path.exists(hook_dir):
49
55
        hooks = os.listdir(hook_dir)
51
57
    metadata["hooks"] = hooks
52
58
 
53
59
    # Stuff into the db
54
 
    metadata.update(md)
55
 
    metadata["_id"] = metadata["repo"]
 
60
    metadata.update(charm_data)
 
61
    metadata["_id"] = metadata["branch_spec"]
56
62
    item = db.find_one({"_id": metadata["_id"]})
57
63
    if item is None:
58
64
        item = metadata
59
65
    else:
60
 
        log.debug("Updating %s", name)
 
66
        log.debug("Updating %s", metadata["branch_spec"])
61
67
        item.update(metadata)
62
68
    item = process_charm(item)
63
69
    db.update({"_id": item["_id"]}, item, upsert=True)
64
70
 
65
71
 
66
72
def process_charm(charm):
67
 
    owner, _ = charm["repo"].split("/", 1)
68
 
    owner = owner[1:]
69
 
    charm["owner"] = owner
 
73
    # Enrich charm metadata for webapp.
 
74
 
 
75
    # Charm url
 
76
    if charm["owner"] == "charmers":
 
77
        charm["short_url"] = "/charms/%s/%s" % (
 
78
            charm["series"], charm["name"])
 
79
    else:
 
80
        charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
 
81
                                             charm["series"],
 
82
                                             charm["name"])
 
83
 
 
84
    # Charm label
 
85
    if charm["owner"] == "charmers":
 
86
        charm["label"] = "%s/%s" % (charm["series"], charm["name"])
 
87
    else:
 
88
        charm["label"] = "~%s:%s/%s" % (charm["owner"],
 
89
                                        charm["series"],
 
90
                                        charm["name"])
70
91
 
71
92
    # Flatten the interfaces provided
72
93
    i_provides = []
93
114
 
94
115
 
95
116
def scan_repo(db, root_dir):
96
 
 
97
117
    charms = os.listdir(root_dir)
98
118
    for c in charms:
99
119
        charm_dir = os.path.join(root_dir, c)
117
137
        if not item:
118
138
            return
119
139
        try:
120
 
            repo, commit, series = json.loads(item)
121
 
            name = repo.rsplit("/", 2)[1]
122
 
            charm_dir = os.path.join(root_dir, name)
123
 
            log.info("Processing %s", name)
124
 
            scan_charm(db, name, charm_dir,
125
 
                       commit=commit, series=series, repo=repo)
126
 
 
 
140
            charm_data = json.loads(item)
 
141
            log.info("Processing %s", charm_data["branch_spec"])
 
142
            scan_charm(db, charm_data)
127
143
        except Exception, e:
128
144
            log.exception("scan error %s: %s", item, e)
 
145
            break
 
146
        else:
 
147
            server.lpush(index_queue, item)
129
148
 
130
149
 
131
150
def main():
135
154
 
136
155
    conn = pymongo.Connection()
137
156
    db = conn.juju.charms
138
 
    #server = redis.Redis()
139
 
    #scan_charms(db, server, root_dir)
140
 
    scan_repo(db, root_dir)
 
157
    server = redis.Redis()
 
158
    scan_charms(db, server, root_dir)
 
159
    #scan_repo(db, root_dir)
141
160
 
142
161
if __name__ == '__main__':
143
162
    main()