1
from datetime import datetime
9
from bzrlib.branch import Branch
10
from bzrlib.revisionspec import RevisionSpec
11
from bzrlib.transport import get_transport
15
from charmworld.models import getconnection
16
from charmworld.models import getdb
17
from charmworld.models import getfs
18
from charmworld.models import CharmFileSet
19
from charmworld.utils import quote_key
20
from charmworld.utils import quote_yaml
22
from config import settings
23
from config import STORE_URL
25
JENKINS_PROVIDERS = ['ec2', 'openstack', 'local']
26
JENKINS_ARTIFACT_URL = (
27
"https://jenkins.qa.ubuntu.com/job/"
28
"%(series)s-%(provider)s-charm-%(charm)s/%(build)d/artifact/%(artifact)s")
30
"https://jenkins.qa.ubuntu.com/job/"
31
"%(series)s-%(provider)s-charm-%(charm)s/lastBuild/api/json")
34
def add_files(charm_data):
35
charm_data['files'] = dict([
36
(quote_key(cfile.filename), dict(cfile)) for cfile in
37
store_branch_files(charm_data)
42
def fetch_branch(root_dir, charm_data, retry=True):
43
"""Fetch a branch from bzr, and augment charm data."""
44
log = logging.getLogger("charm.bzr")
45
branch_dir = os.path.abspath(
46
str(os.path.join(root_dir,
50
charm_data["bname"])))
52
if not os.path.exists(os.path.dirname(branch_dir)):
53
os.makedirs(os.path.dirname(branch_dir))
55
# Store the branch directory
56
charm_data["branch_dir"] = branch_dir
58
if not os.path.exists(branch_dir):
59
# The branch has never been seen before. Original branch.
60
log.info("Branching charm lp:%s", charm_data["branch_spec"])
61
subprocess.check_output(
62
["/usr/bin/bzr", "co", "-q",
63
"lp:%s" % charm_data["branch_spec"], branch_dir])
64
charm_data = add_files(charm_data)
67
# It exists and check if it's the latest revision already.
68
log.debug("Existing charm from lp:%s", charm_data["branch_spec"])
69
transport = get_transport(branch_dir)
70
branch = Branch.open_from_transport(transport)
71
cur_rev_id = branch.last_revision()
72
if cur_rev_id == charm_data['commit']:
73
charm_data = add_files(charm_data)
74
log.debug("Already up to date lp:%s", charm_data["branch_spec"])
77
log.debug("Updating branch lp:%s", charm_data["branch_spec"])
80
# It exists, but it's not up to date, so update it.
81
subprocess.check_output(
82
["/usr/bin/bzr", "update", "-q"],
84
stderr=subprocess.STDOUT)
85
charm_data = add_files(charm_data)
86
except subprocess.CalledProcessError:
87
# It existed but the update failed for some reason. Just strip the
88
# whole tree and start over with the above.
90
shutil.rmtree(branch_dir)
91
return fetch_branch(root_dir, charm_data, retry=False)
95
def store_branch_files(charm_data, db=None):
96
"""Process the bzr branch for files that need to be stored in gridfs."""
97
log = logging.getLogger("charm.bzr")
98
log.info('Storing files of branch into gridfs')
100
connection = getconnection(settings)
101
db = getdb(connection, settings.get('mongo.database'))
103
filestore = CharmFileSet.save_files(
104
fs, charm_data, charm_data['branch_dir'])
106
log.info('Completed gridfs storage.')
109
def _rev_info(r, branch):
111
"revno": branch.revision_id_to_revno(r.revision_id),
112
"committer": r.committer,
113
"created": r.timestamp,
119
def fetch_changes(charm_data):
120
branch_dir = charm_data["branch_dir"]
121
transport = get_transport(branch_dir)
122
branch = Branch.open_from_transport(transport)
124
# We only want the last 10 changes, in descending order.
125
_, cur_rev_id = branch.last_revision_info()
126
spec = RevisionSpec.from_string("revno:-10")
127
last_rev_id = spec.as_revision_id(branch)
131
graph = branch.repository.get_graph()
133
graph.iter_lefthand_ancestry(cur_rev_id, (last_rev_id,)))
134
if not cur_rev_id in revs:
135
revs.insert(0, cur_rev_id)
138
revisions = map(branch.repository.get_revision, revs)
140
charm_data["changes"] = changes = []
143
changes.append(_rev_info(r, branch))
144
charm_data["last_change"] = changes[0]
145
charm_data["first_change"] = _rev_info(
146
branch.repository.get_revision(
147
RevisionSpec.from_string("revno:1").as_revision_id(branch)),
151
def index_charm(indexer, charm):
152
log = logging.getLogger("charm.index")
153
doc = xappy.UnprocessedDocument()
155
# Weight critical fields higher for official charms.
156
if charm['owner'] == 'charmers':
160
doc.fields.append(xappy.Field(
161
"name", charm["name"], weight=weight))
162
doc.fields.append(xappy.Field(
163
"summary", charm["summary"], weight=weight))
164
doc.fields.append(xappy.Field(
165
"description", charm["description"], weight=weight))
167
doc.fields.append(xappy.Field("owner", charm["owner"]))
168
doc.fields.append(xappy.Field("short_url", charm["short_url"]))
169
doc.fields.append(xappy.Field("label", charm["label"]))
170
doc.fields.append(xappy.Field("series", charm["series"]))
172
if charm.get('subordinate'):
173
doc.fields.append(xappy.Field('subordinate', 'true'))
175
if "config" in charm \
176
and charm['config'] \
177
and 'options' in charm['config'] \
178
and charm['config']['options']:
181
for key, option in charm["config"]["options"].items():
182
config_text.append(key)
183
config_text.append(option.get("description", ""))
184
doc.fields.append(xappy.Field("config", " ".join(config_text)))
188
if "requires" in charm and charm["requires"]:
189
for key, option in charm["requires"].items():
190
relation_text.append(key)
191
relation_text.append(option["interface"])
192
doc.fields.append(xappy.Field('requires', option["interface"]))
193
if "provides" in charm and charm["provides"]:
194
for key, option in charm["provides"].items():
195
if not isinstance(option, dict):
196
log.warning("invalid charm provides %s", charm['branch_spec'])
198
relation_text.append(key)
199
relation_text.append(option["interface"])
200
doc.fields.append(xappy.Field("provides", option["interface"]))
203
doc.fields.append(xappy.Field("relations", " ".join(relation_text)))
206
for change in charm.get("changes", ()):
207
change_text.append(change['message'])
208
change_text.append(change['committer'])
211
doc.fields.append(xappy.Field("changes", " ".join(change_text)))
213
if 'store_url' in charm:
214
doc.fields.append(xappy.Field('store_url', charm['store_url']))
216
log.warning("No store url found for %s", charm["_id"])
217
doc.id = charm["_id"]
222
def check_jenkins(db, fs, charm):
223
log = logging.getLogger("charm.jenkins")
224
if not charm['branch_spec'].startswith('~charmers'):
228
charm['test_results'] = {}
230
for p in JENKINS_PROVIDERS:
232
result_id, status = store_provider_results(db, fs, p, charm)
233
if result_id is None:
235
charm['tests'][p] = status
236
charm['test_results'][p] = result_id
238
log.exception("Unknown error while processing %s %s",
239
charm['branch_spec'], p)
242
def _fetch_artifacts(provider, charm, result, fs):
244
for artifact in result['artifacts']:
245
a_path = "%s/%s/%s/%s" % (
246
charm['branch_spec'],
249
artifact['displayPath'])
250
a_url = JENKINS_ARTIFACT_URL % (dict(
251
series=charm['series'],
254
build=result['number'],
255
artifact=artifact['relativePath']))
259
# Load up the charm revision as a result property.
260
if artifact['displayPath'] == 'charm-revision':
261
charm_revision = urllib2.urlopen(a_url).read().strip()
262
if not charm_revision:
264
result['revno'] = int(charm_revision)
267
# Mark the test result as graph runner enabled.
268
if "graph-tests" in artifact['displayPath']:
269
result['charmrunner'] = True
271
# Skip the actual charm content
272
if "charm-%s.zip" % charm['name'] == artifact['displayPath']:
275
# XXX Short circuit before actual fetching.
278
a_file = urllib2.urlopen(a_url)
279
file_id = fs.put(a_file, path=a_path)
280
artifact['file_id'] = file_id
281
artifacts.append(artifact)
286
def store_provider_results(db, fs, provider, charm):
287
log = logging.getLogger("charm.jenkins")
288
charm_result_url = JENKINS_QA_URL % (
289
dict(series=charm['series'],
291
charm=charm['name']))
293
log.debug("Loading %s from %s", charm['name'], charm_result_url)
296
contents = urllib2.urlopen(charm_result_url).read()
297
except urllib2.URLError:
298
log.debug("No test result for %s @ %s", charm['branch_spec'], provider)
301
result = json.loads(contents)
303
# If we already have results no pointing in refetching.
304
result_id = "%s::%s-%s" % (
305
charm['branch_spec'], provider, result['number'])
306
db_result = db.jenkins.find_one({'_id': result_id})
307
if db_result is not None:
308
return result_id, db_result['result']
310
# Fetch test artifacts.
311
artifacts = _fetch_artifacts(provider, charm, result, fs)
313
# Inject test metadata.
314
result['branch_spec'] = charm['branch_spec']
315
result['provider'] = provider
316
result['artifacts'] = artifacts
317
result['_id'] = result_id
318
db.jenkins.insert(result)
319
return (result_id, result['result'])
322
def proof_charm(charm, prooflib):
324
lint, exit_code = prooflib.run(charm['branch_dir'])
328
level, msg = line.split(':', 1)
329
if level == "W" and 'name' in msg:
331
proof.setdefault(level.lower(), []).append(msg)
332
charm['proof'] = proof
335
def process_charm(charm):
336
# Enrich charm metadata for webapp.
339
if charm["owner"] == "charmers":
340
charm["short_url"] = "/charms/%s/%s" % (
341
charm["series"], charm["name"])
343
charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
348
if charm["owner"] == "charmers":
349
charm["label"] = "%s/%s" % (charm["series"], charm["name"])
351
charm["label"] = "~%s:%s/%s" % (charm["owner"],
355
# Flatten the interfaces provided
357
provides = charm.get("provides")
359
for v in provides.values():
360
if not isinstance(v, dict):
362
i = v.get("interface")
366
charm["i_provides"] = i_provides
368
# Flatten the interfaces required
370
requires = charm.get("requires")
372
for v in requires.values():
373
i = v.get("interface")
377
charm["i_requires"] = i_requires
381
def scan_repo(db, root_dir):
382
log = logging.getLogger("charm.scan")
383
charms = os.listdir(root_dir)
385
charm_dir = os.path.join(root_dir, c)
386
if not os.path.isdir(charm_dir):
388
#log.info("Processing %s", c)
390
scan_charm(db, c, charm_dir, repo="~charmers/charm/oneiric/%s" % c)
392
log.exception("Unknown scan error")
397
traceback.print_exc()
398
pdb.post_mortem(sys.exc_info()[-1])
402
def scan_charm(db, charm_data):
403
log = logging.getLogger("charm.scan")
405
files = charm_data['files']
406
# Some files have bad characters in them since they are used as mongo
407
# keys. Use their escaped forms instead.
408
metadata_file = quote_key('metadata.yaml')
409
config_file = quote_key('config.yaml')
411
if metadata_file not in files:
412
log.info("Charm has no metadata: %s", charm_data["branch_spec"])
415
cfile = CharmFileSet.get_by_id(fs, files[metadata_file]['fileid'])
417
metadata = quote_yaml(yaml.load(cfile.read()))
418
except Exception, exc:
420
'Invalid charm metadata %s: %s' % (
421
charm_data['branch_spec'],
425
if config_file in files:
426
cfile = CharmFileSet.get_by_id(fs, files[config_file]['fileid'])
427
config_raw = cfile.read()
430
config = quote_yaml(yaml.load(config_raw))
431
except Exception, exc:
433
'Invalid charm config yaml. %s: %s' % (
434
charm_data['branch_spec'],
438
metadata["config"] = config
439
metadata["config_raw"] = config_raw
441
if 'revision' in files:
442
cfile = CharmFileSet.get_by_id(fs, files['revision']['fileid'])
443
rev_raw = cfile.read()
444
rev_id = int(rev_raw.strip())
445
metadata["revision"] = rev_id
446
elif not "revision" in metadata:
447
log.info("Invalid revision %s", charm_data["branch_spec"])
448
metadata["revision"] = 0
451
for filedata in files.values():
452
if filedata['subdir'] == 'hooks':
453
hooks.append(filedata['filename'])
455
metadata["hooks"] = hooks
458
metadata.update(charm_data)
459
metadata["_id"] = metadata["branch_spec"]
460
item = db.charms.find_one({"_id": metadata["_id"]})
464
#log.debug("Updating %s", metadata["branch_spec"])
465
item.update(metadata)
466
item = process_charm(item)
467
db.charms.update({"_id": item["_id"]}, item, upsert=True)
470
def check_store(charm):
471
log = logging.getLogger("charm.store")
472
if charm['owner'] == 'charmers':
473
address = "cs:%s/%s" % (charm["series"], charm["name"])
475
address = "cs:~%s/%s/%s" % (
476
charm["owner"], charm["series"], charm["name"])
478
data = _store_get(address)
480
if 'errors' in data or 'warnings' in data:
481
if charm['owner'] == 'charmers':
482
log.info("rechecking %s with ~charmers", address)
483
retry_address = "cs:~%s/%s/%s" % (
484
charm["owner"], charm["series"], charm["name"])
485
retry_data = _store_get(retry_address)
487
# Update the url to the user qualified name.
488
# if not 'errors' in retry_data \
489
# and not 'warnings' in retry_data:
491
address = retry_address
493
if 'errors' in data or 'warnings' in data:
494
log.warning("store error on %s %s" % (address, data))
496
data["store_checked"] = datetime.now().ctime()
498
charm['store_data'] = data
499
charm['store_url'] = address + "-%d" % data['revision']
502
def _store_get(address):
503
url = STORE_URL + "/charm-info?charms=%s&stats=0" % address
504
contents = urllib2.urlopen(url).read()
505
data = json.loads(contents)
507
data['address'] = address