~jcsackett/charmworld/bac-tag-constraints

« back to all changes in this revision

Viewing changes to charmworld/jobs/scan.py

  • Committer: Aaron Bentley
  • Date: 2013-02-12 18:57:02 UTC
  • mfrom: (149 charmworld)
  • mto: This revision was merged to the branch mainline in revision 150.
  • Revision ID: aaron@canonical.com-20130212185702-9gnf40ao17a7uw7v
Merged trunk into mongo-urls.

Show diffs side-by-side

added added

removed removed

Lines of Context:
3
3
# the file LICENSE).
4
4
 
5
5
import logging
6
 
import os
 
6
 
7
7
import pymongo
8
 
import yaml
 
8
 
 
9
from charmworld.models import getdb
9
10
 
10
11
from config import DB_NAME
11
12
from config import MONGO_URL
12
13
from config import SCAN_IN_QUEUE
13
14
from config import SCAN_OUT_QUEUE
14
15
from utils import get_queues
15
 
 
16
 
from charmworld.models import getfs
17
 
from charmworld.models import getdb
18
 
from charmworld.models import CharmFileSet
19
 
from charmworld.utils import quote_key
20
 
from charmworld.utils import quote_yaml
21
 
 
22
 
 
23
 
log = logging.getLogger("charm.scan")
24
 
 
25
 
 
26
 
def scan_charm(db, charm_data):
27
 
    fs = getfs(db)
28
 
    files = charm_data['files']
29
 
    # Some files have bad characters in them since they are used as mongo
30
 
    # keys. Use their escaped forms instead.
31
 
    metadata_file = quote_key('metadata.yaml') 
32
 
    config_file = quote_key('config.yaml')
33
 
 
34
 
    if metadata_file not in files:
35
 
        log.info("Charm has no metadata: %s", charm_data["branch_spec"])
36
 
        return
37
 
    else:
38
 
        cfile = CharmFileSet.get_by_id(fs, files[metadata_file]['fileid'])
39
 
        try:
40
 
            metadata = quote_yaml(yaml.load(cfile.read()))
41
 
        except Exception, exc:
42
 
            log.info(
43
 
                'Invalid charm metadata %s: %s' % (
44
 
                    charm_data['branch_spec'],
45
 
                    exc)
46
 
            )
47
 
 
48
 
    if config_file in files:
49
 
        cfile = CharmFileSet.get_by_id(fs, files[config_file]['fileid'])
50
 
        config_raw = cfile.read()
51
 
 
52
 
        try:
53
 
            config = quote_yaml(yaml.load(config_raw))
54
 
        except Exception, exc:
55
 
            log.info(
56
 
                'Invalid charm config yaml. %s: %s' % (
57
 
                    charm_data['branch_spec'],
58
 
                    exc)
59
 
            )
60
 
 
61
 
        metadata["config"] = config
62
 
        metadata["config_raw"] = config_raw
63
 
 
64
 
    if 'revision' in files:
65
 
        cfile = CharmFileSet.get_by_id(fs, files['revision']['fileid'])
66
 
        rev_raw = cfile.read()
67
 
        rev_id = int(rev_raw.strip())
68
 
        metadata["revision"] = rev_id
69
 
    elif not "revision" in metadata:
70
 
        log.info("Invalid revision %s", charm_data["branch_spec"])
71
 
        metadata["revision"] = 0
72
 
 
73
 
    hooks = []
74
 
    for filedata in files.values():
75
 
        if filedata['subdir'] == 'hooks':
76
 
            hooks.append(filedata['filename'])
77
 
    hooks.sort()
78
 
    metadata["hooks"] = hooks
79
 
 
80
 
    # Stuff into the db
81
 
    metadata.update(charm_data)
82
 
    metadata["_id"] = metadata["branch_spec"]
83
 
    item = db.charms.find_one({"_id": metadata["_id"]})
84
 
    if item is None:
85
 
        item = metadata
86
 
    else:
87
 
        #log.debug("Updating %s", metadata["branch_spec"])
88
 
        item.update(metadata)
89
 
    item = process_charm(item)
90
 
    db.charms.update({"_id": item["_id"]}, item, upsert=True)
91
 
 
92
 
 
93
 
def process_charm(charm):
94
 
    # Enrich charm metadata for webapp.
95
 
 
96
 
    # Charm url
97
 
    if charm["owner"] == "charmers":
98
 
        charm["short_url"] = "/charms/%s/%s" % (
99
 
            charm["series"], charm["name"])
100
 
    else:
101
 
        charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
102
 
                                             charm["series"],
103
 
                                             charm["name"])
104
 
 
105
 
    # Charm label
106
 
    if charm["owner"] == "charmers":
107
 
        charm["label"] = "%s/%s" % (charm["series"], charm["name"])
108
 
    else:
109
 
        charm["label"] = "~%s:%s/%s" % (charm["owner"],
110
 
                                        charm["series"],
111
 
                                        charm["name"])
112
 
 
113
 
    # Flatten the interfaces provided
114
 
    i_provides = []
115
 
    provides = charm.get("provides")
116
 
    if provides:
117
 
        for v in provides.values():
118
 
            if not isinstance(v, dict):
119
 
                continue
120
 
            i = v.get("interface")
121
 
            if not i:
122
 
                continue
123
 
            i_provides.append(i)
124
 
    charm["i_provides"] = i_provides
125
 
 
126
 
    # Flatten the interfaces required
127
 
    i_requires = []
128
 
    requires = charm.get("requires")
129
 
    if requires:
130
 
        for v in requires.values():
131
 
            i = v.get("interface")
132
 
            if not i:
133
 
                continue
134
 
            i_requires.append(i)
135
 
    charm["i_requires"] = i_requires
136
 
    return charm
137
 
 
138
 
 
139
 
def scan_repo(db, root_dir):
140
 
    charms = os.listdir(root_dir)
141
 
    for c in charms:
142
 
        charm_dir = os.path.join(root_dir, c)
143
 
        if not os.path.isdir(charm_dir):
144
 
            continue
145
 
        #log.info("Processing %s", c)
146
 
        try:
147
 
            scan_charm(db, c, charm_dir, repo="~charmers/charm/oneiric/%s" % c)
148
 
        except:
149
 
            log.exception("Unknown scan error")
150
 
            raise
151
 
            import pdb
152
 
            import sys
153
 
            import traceback
154
 
            traceback.print_exc()
155
 
            pdb.post_mortem(sys.exc_info()[-1])
156
 
            raise
157
 
 
158
 
 
159
 
def scan_charms(db, scan_queue, index_queue):
160
 
 
161
 
    while 1:
162
 
        item = scan_queue.next()
163
 
        if not item:
164
 
            return
165
 
        try:
166
 
            charm_data = item.payload
167
 
            scan_charm(db, charm_data)
168
 
            # XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for
169
 
            # scan_charm to reindex what's on disk; we should probably just
170
 
            # have a different job for this that's not part of ingest.
171
 
            #scan_repo(db, CHARM_DIR)
172
 
        except Exception, e:
173
 
            log.exception("scan error %s: %s", charm_data, e)
174
 
            break
175
 
        else:
176
 
            index_queue.put(charm_data)
177
 
 
178
 
 
179
 
def main():
 
16
from worker import scan_charms
 
17
 
 
18
 
 
19
if __name__ == '__main__':
180
20
    logging.basicConfig(
181
21
        level=logging.DEBUG,
182
22
        format="%(asctime)s: %(name)s@%(levelname)s: %(message)s")
185
25
    db = getdb(conn, DB_NAME)
186
26
    in_queue, out_queue = get_queues(SCAN_IN_QUEUE, SCAN_OUT_QUEUE)
187
27
    scan_charms(db, in_queue, out_queue)
188
 
 
189
 
if __name__ == '__main__':
190
 
    main()