9
from charmworld.models import getdb
10
11
from config import DB_NAME
11
12
from config import MONGO_URL
12
13
from config import SCAN_IN_QUEUE
13
14
from config import SCAN_OUT_QUEUE
14
15
from utils import get_queues
16
from charmworld.models import getfs
17
from charmworld.models import getdb
18
from charmworld.models import CharmFileSet
19
from charmworld.utils import quote_key
20
from charmworld.utils import quote_yaml
23
log = logging.getLogger("charm.scan")
26
def scan_charm(db, charm_data):
28
files = charm_data['files']
29
# Some files have bad characters in them since they are used as mongo
30
# keys. Use their escaped forms instead.
31
metadata_file = quote_key('metadata.yaml')
32
config_file = quote_key('config.yaml')
34
if metadata_file not in files:
35
log.info("Charm has no metadata: %s", charm_data["branch_spec"])
38
cfile = CharmFileSet.get_by_id(fs, files[metadata_file]['fileid'])
40
metadata = quote_yaml(yaml.load(cfile.read()))
41
except Exception, exc:
43
'Invalid charm metadata %s: %s' % (
44
charm_data['branch_spec'],
48
if config_file in files:
49
cfile = CharmFileSet.get_by_id(fs, files[config_file]['fileid'])
50
config_raw = cfile.read()
53
config = quote_yaml(yaml.load(config_raw))
54
except Exception, exc:
56
'Invalid charm config yaml. %s: %s' % (
57
charm_data['branch_spec'],
61
metadata["config"] = config
62
metadata["config_raw"] = config_raw
64
if 'revision' in files:
65
cfile = CharmFileSet.get_by_id(fs, files['revision']['fileid'])
66
rev_raw = cfile.read()
67
rev_id = int(rev_raw.strip())
68
metadata["revision"] = rev_id
69
elif not "revision" in metadata:
70
log.info("Invalid revision %s", charm_data["branch_spec"])
71
metadata["revision"] = 0
74
for filedata in files.values():
75
if filedata['subdir'] == 'hooks':
76
hooks.append(filedata['filename'])
78
metadata["hooks"] = hooks
81
metadata.update(charm_data)
82
metadata["_id"] = metadata["branch_spec"]
83
item = db.charms.find_one({"_id": metadata["_id"]})
87
#log.debug("Updating %s", metadata["branch_spec"])
89
item = process_charm(item)
90
db.charms.update({"_id": item["_id"]}, item, upsert=True)
93
def process_charm(charm):
94
# Enrich charm metadata for webapp.
97
if charm["owner"] == "charmers":
98
charm["short_url"] = "/charms/%s/%s" % (
99
charm["series"], charm["name"])
101
charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
106
if charm["owner"] == "charmers":
107
charm["label"] = "%s/%s" % (charm["series"], charm["name"])
109
charm["label"] = "~%s:%s/%s" % (charm["owner"],
113
# Flatten the interfaces provided
115
provides = charm.get("provides")
117
for v in provides.values():
118
if not isinstance(v, dict):
120
i = v.get("interface")
124
charm["i_provides"] = i_provides
126
# Flatten the interfaces required
128
requires = charm.get("requires")
130
for v in requires.values():
131
i = v.get("interface")
135
charm["i_requires"] = i_requires
139
def scan_repo(db, root_dir):
140
charms = os.listdir(root_dir)
142
charm_dir = os.path.join(root_dir, c)
143
if not os.path.isdir(charm_dir):
145
#log.info("Processing %s", c)
147
scan_charm(db, c, charm_dir, repo="~charmers/charm/oneiric/%s" % c)
149
log.exception("Unknown scan error")
154
traceback.print_exc()
155
pdb.post_mortem(sys.exc_info()[-1])
159
def scan_charms(db, scan_queue, index_queue):
162
item = scan_queue.next()
166
charm_data = item.payload
167
scan_charm(db, charm_data)
168
# XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for
169
# scan_charm to reindex what's on disk; we should probably just
170
# have a different job for this that's not part of ingest.
171
#scan_repo(db, CHARM_DIR)
173
log.exception("scan error %s: %s", charm_data, e)
176
index_queue.put(charm_data)
16
from worker import scan_charms
19
if __name__ == '__main__':
180
20
logging.basicConfig(
181
21
level=logging.DEBUG,
182
22
format="%(asctime)s: %(name)s@%(levelname)s: %(message)s")