« back to all changes in this revision
Viewing changes to charmworld/jobs/scan.py
- browse files at revision 149
- compare with another revision
- download diff
- download tarball
- view history from revision 149
- Committer: Tarmac
- Author(s): j.c.sackett
- Date: 2013-02-12 18:15:28 UTC
- mfrom: (142.3.15 workify-jobs-2)
- Revision ID: tarmac-20130212181528-wf7ipovofsbwkc9w
[r=jcsackett][bug=][author=jcsackett] Split jobs into worker code and ingest code, in preparation for worker refactor.
- files added:
- charmworld/jobs/ingest.py
- files modified:
- charmworld/jobs/bzr.py
added
removed
charmworld/jobs/scan.py
3
3
# the file LICENSE).
4
4
5
5
import logging
6
import os
6
7
7
import pymongo
8
import yaml
8
9
from charmworld.models import getdb
9
10
10
11
from config import DB_NAME
11
12
from config import MONGO_HOST
13
14
from config import SCAN_IN_QUEUE
14
15
from config import SCAN_OUT_QUEUE
15
16
from utils import get_queues
16
17
from charmworld.models import getfs
18
from charmworld.models import getdb
19
from charmworld.models import CharmFileSet
20
from charmworld.utils import quote_key
21
from charmworld.utils import quote_yaml
22
23
24
log = logging.getLogger("charm.scan")
25
26
27
def scan_charm(db, charm_data):
28
fs = getfs(db)
29
files = charm_data['files']
30
# Some files have bad characters in them since they are used as mongo
31
# keys. Use their escaped forms instead.
32
metadata_file = quote_key('metadata.yaml')
33
config_file = quote_key('config.yaml')
34
35
if metadata_file not in files:
36
log.info("Charm has no metadata: %s", charm_data["branch_spec"])
37
return
38
else:
39
cfile = CharmFileSet.get_by_id(fs, files[metadata_file]['fileid'])
40
try:
41
metadata = quote_yaml(yaml.load(cfile.read()))
42
except Exception, exc:
43
log.info(
44
'Invalid charm metadata %s: %s' % (
45
charm_data['branch_spec'],
46
exc)
47
)
48
49
if config_file in files:
50
cfile = CharmFileSet.get_by_id(fs, files[config_file]['fileid'])
51
config_raw = cfile.read()
52
53
try:
54
config = quote_yaml(yaml.load(config_raw))
55
except Exception, exc:
56
log.info(
57
'Invalid charm config yaml. %s: %s' % (
58
charm_data['branch_spec'],
59
exc)
60
)
61
62
metadata["config"] = config
63
metadata["config_raw"] = config_raw
64
65
if 'revision' in files:
66
cfile = CharmFileSet.get_by_id(fs, files['revision']['fileid'])
67
rev_raw = cfile.read()
68
rev_id = int(rev_raw.strip())
69
metadata["revision"] = rev_id
70
elif not "revision" in metadata:
71
log.info("Invalid revision %s", charm_data["branch_spec"])
72
metadata["revision"] = 0
73
74
hooks = []
75
for filedata in files.values():
76
if filedata['subdir'] == 'hooks':
77
hooks.append(filedata['filename'])
78
hooks.sort()
79
metadata["hooks"] = hooks
80
81
# Stuff into the db
82
metadata.update(charm_data)
83
metadata["_id"] = metadata["branch_spec"]
84
item = db.charms.find_one({"_id": metadata["_id"]})
85
if item is None:
86
item = metadata
87
else:
88
#log.debug("Updating %s", metadata["branch_spec"])
89
item.update(metadata)
90
item = process_charm(item)
91
db.charms.update({"_id": item["_id"]}, item, upsert=True)
92
93
94
def process_charm(charm):
95
# Enrich charm metadata for webapp.
96
97
# Charm url
98
if charm["owner"] == "charmers":
99
charm["short_url"] = "/charms/%s/%s" % (
100
charm["series"], charm["name"])
101
else:
102
charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
103
charm["series"],
104
charm["name"])
105
106
# Charm label
107
if charm["owner"] == "charmers":
108
charm["label"] = "%s/%s" % (charm["series"], charm["name"])
109
else:
110
charm["label"] = "~%s:%s/%s" % (charm["owner"],
111
charm["series"],
112
charm["name"])
113
114
# Flatten the interfaces provided
115
i_provides = []
116
provides = charm.get("provides")
117
if provides:
118
for v in provides.values():
119
if not isinstance(v, dict):
120
continue
121
i = v.get("interface")
122
if not i:
123
continue
124
i_provides.append(i)
125
charm["i_provides"] = i_provides
126
127
# Flatten the interfaces required
128
i_requires = []
129
requires = charm.get("requires")
130
if requires:
131
for v in requires.values():
132
i = v.get("interface")
133
if not i:
134
continue
135
i_requires.append(i)
136
charm["i_requires"] = i_requires
137
return charm
138
139
140
def scan_repo(db, root_dir):
141
charms = os.listdir(root_dir)
142
for c in charms:
143
charm_dir = os.path.join(root_dir, c)
144
if not os.path.isdir(charm_dir):
145
continue
146
#log.info("Processing %s", c)
147
try:
148
scan_charm(db, c, charm_dir, repo="~charmers/charm/oneiric/%s" % c)
149
except:
150
log.exception("Unknown scan error")
151
raise
152
import pdb
153
import sys
154
import traceback
155
traceback.print_exc()
156
pdb.post_mortem(sys.exc_info()[-1])
157
raise
158
159
160
def scan_charms(db, scan_queue, index_queue):
161
162
while 1:
163
item = scan_queue.next()
164
if not item:
165
return
166
try:
167
charm_data = item.payload
168
scan_charm(db, charm_data)
169
# XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for
170
# scan_charm to reindex what's on disk; we should probably just
171
# have a different job for this that's not part of ingest.
172
#scan_repo(db, CHARM_DIR)
173
except Exception, e:
174
log.exception("scan error %s: %s", charm_data, e)
175
break
176
else:
177
index_queue.put(charm_data)
178
179
180
def main():
17
from worker import scan_charms
18
19
20
if __name__ == '__main__':
181
21
logging.basicConfig(
182
22
level=logging.DEBUG,
183
23
format="%(asctime)s: %(name)s@%(levelname)s: %(message)s")
186
26
db = getdb(conn, DB_NAME)
187
27
in_queue, out_queue = get_queues(SCAN_IN_QUEUE, SCAN_OUT_QUEUE)
188
28
scan_charms(db, in_queue, out_queue)
189
190
if __name__ == '__main__':
191
main()
Loggerhead is a web-based interface for Breezy
Version: 2.0.1