8
from bzrlib.errors import InvalidRevisionSpec
12
from config import CHARM_IMPORT_FILTER
13
from config import CHARM_IMPORT_LIMIT
14
from ingest import check_jenkins
15
from ingest import fetch_branch
16
from ingest import fetch_changes
17
from ingest import index_charm
18
from ingest import proof_charm
19
from ingest import scan_charm
20
from ingest import scan_repo
21
from ingest import check_store
22
from utils import parse_branch
25
BRANCH_TIPS = "https://api.launchpad.net/devel/charm?ws.op=getBranchTips"
29
def fetch_branches(root_dir, queue, scan_queue):
30
log = logging.getLogger("charm.bzr")
36
charm_data = item.payload
37
fetch_branch(root_dir, charm_data)
38
except Exception as e:
39
log.exception("bzr error error on %s: %s", charm_data, str(e))
40
# XXX j.c.sackett Jan 30, 2012 Bug:1110539 The charm here is being
41
# annotated with error data, but since the charm falls out of
42
# scope when this iteration of the loop ends, nothing is being
43
# down with it--it's not stored, or put on the out queue.
44
charm_data["error_stage"] = "bzr"
45
if hasattr(e, 'output'):
46
charm_data["error"] = str(e.output)
49
scan_queue.put(charm_data)
51
# Remove the job from the input queue.
55
def scan_changes(in_queue, out_queue):
56
log = logging.getLogger("charm.changelog")
58
item = in_queue.next()
62
charm_data = item.payload
63
#log.info("Fetching changes for %s", charm_data["branch_spec"])
64
fetch_changes(charm_data)
65
except InvalidRevisionSpec, e:
66
log.warning("bzr invalid rev on %s %s" % (
67
charm_data['branch_spec'], charm_data['branch_dir']))
69
log.exception("bzr error on %s: %s", charm_data, str(e))
71
out_queue.put(charm_data)
73
# Remove the job from the input queue.
77
def setup_indexer(indexer):
78
indexer.add_field_action('owner', xappy.FieldActions.INDEX_EXACT)
79
indexer.add_field_action('series', xappy.FieldActions.INDEX_EXACT)
80
indexer.add_field_action('subordinate', xappy.FieldActions.INDEX_EXACT)
81
indexer.add_field_action('provides', xappy.FieldActions.INDEX_EXACT)
82
indexer.add_field_action('requires', xappy.FieldActions.INDEX_EXACT)
85
indexer.add_field_action('name', xappy.FieldActions.STORE_CONTENT)
86
indexer.add_field_action('store_url', xappy.FieldActions.STORE_CONTENT)
87
indexer.add_field_action('summary', xappy.FieldActions.STORE_CONTENT)
88
indexer.add_field_action('short_url', xappy.FieldActions.STORE_CONTENT)
89
indexer.add_field_action('label', xappy.FieldActions.STORE_CONTENT)
90
indexer.add_field_action('series', xappy.FieldActions.STORE_CONTENT)
91
indexer.add_field_action('owner', xappy.FieldActions.STORE_CONTENT)
92
indexer.add_field_action('subordinate', xappy.FieldActions.STORE_CONTENT)
94
# Full text search fields
95
indexer.add_field_action(
96
'name', xappy.FieldActions.INDEX_FREETEXT, weight=10,
99
indexer.add_field_action(
100
'summary', xappy.FieldActions.INDEX_FREETEXT, weight=5,
103
indexer.add_field_action(
104
'description', xappy.FieldActions.INDEX_FREETEXT, weight=3,
107
indexer.add_field_action(
108
'config', xappy.FieldActions.INDEX_FREETEXT, language='en')
110
indexer.add_field_action(
111
'relations', xappy.FieldActions.INDEX_FREETEXT, language='en')
113
indexer.add_field_action(
114
'changes', xappy.FieldActions.INDEX_FREETEXT, language='en')
117
def index_queue(db, indexer, index_queue):
118
log = logging.getLogger("charm.index")
122
item = index_queue.next()
126
charm_data = item.payload
127
charm = db.find_one({"_id": charm_data["branch_spec"]})
129
log.debug("Indexing charm %s", charm_data["branch_spec"])
131
index_charm(indexer, charm)
134
"Skipping unknown charm %s", charm_data["branch_spec"])
135
except KeyboardInterrupt:
138
log.exception("Error indexing %s", charm_data)
141
# Remove the job from the input queue.
143
log.info("Indexed %d Charms" % count)
146
def reindex(db, indexer):
147
log = logging.getLogger("charm.index")
149
for charm in db.find():
150
log.debug("Indexing %s", charm['branch_spec'])
152
index_charm(indexer, charm)
154
log.error("Indexing charm %s", charm['branch_spec'])
157
log.info("Indexed %d Charms" % count)
160
def test_charms(db, fs, in_queue, out_queue):
161
log = logging.getLogger("charm.jenkins")
163
item = in_queue.next()
167
charm_data = item.payload
168
log.debug("Checking tests for %s", charm_data["branch_spec"])
169
check_jenkins(db, fs, charm_data)
171
log.exception("jenkins error on %s: %s", charm_data, str(e))
174
out_queue.put(charm_data)
176
# Remove the job from the input queue.
180
def available_charms(charm_data=None):
181
log = logging.getLogger("charm.launchpad")
183
charm_data = urllib.urlopen(BRANCH_TIPS).read()
184
charm_data = json.loads(charm_data)
188
for repo, commit, series in charm_data:
189
_, branch_name = repo.rsplit("/", 1)
192
data = parse_branch(repo, series, commit)
194
log.warning("Unable to parse repo %s", repo)
197
if CHARM_IMPORT_FILTER and not data['branch_spec'].startswith(
198
CHARM_IMPORT_FILTER):
201
if data["bname"] not in ("trunk", "trunk-1"):
202
log.debug("Skipped branch %s", repo)
205
log.info("Queueing %s", data['branch_spec'])
210
if CHARM_IMPORT_LIMIT and count >= CHARM_IMPORT_LIMIT:
211
log.info("Import limit reached (%d)... stopping",
216
def queue_charms(out_queue):
217
log = logging.getLogger("charm.launchpad")
218
for charm in available_charms():
219
added = out_queue.put(charm)
221
log.info("Queued %s", charm)
224
@contextlib.contextmanager
225
def get_proof_lib(new_path):
226
if new_path not in sys.path:
227
sys.path.append(new_path)
229
import lib.proof as prooflib
234
if new_path in sys.path:
235
sys.path.remove(new_path)
238
def proof_charms(in_queue, out_queue):
239
log = logging.getLogger("charm.proof")
240
if not os.path.isdir(config.CHARM_PROOF_PATH):
241
err_msg = ("proof error before processing began: could not find "
243
log.exception(err_msg)
244
log.exception("CHARM_PROOF_PATH: %s", config.CHARM_PROOF_PATH)
245
log.exception("proof aborted.")
247
with get_proof_lib(config.CHARM_PROOF_PATH) as prooflib:
249
err_msg = ("proof error before processing began: could not "
250
"import charm proof lib.")
251
log.exception(err_msg)
252
log.exception("CHARM_PROOF_PATH: %s", config.CHARM_PROOF_PATH)
253
log.exception("proof aborted.")
256
item = in_queue.next()
260
charm_data = item.payload
261
log.info("Proofing charm %s", charm_data["branch_spec"])
262
proof_charm(charm_data, prooflib)
263
except Exception as e:
264
err = "proof error on %s: %s" % (
265
charm_data['branch_spec'], str(e))
266
charm_data['proof'] = {'e': [err]}
269
# Remove the job from the input queue.
271
out_queue.put(charm_data)
274
def scan_charms(db, scan_queue, index_queue):
275
log = logging.getLogger("charm.scan")
277
item = scan_queue.next()
281
charm_data = item.payload
282
scan_charm(db, charm_data)
283
# XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for
284
# scan_charm to reindex what's on disk; we should probably just
285
# have a different job for this that's not part of ingest.
286
#scan_repo(db, CHARM_DIR)
288
log.exception("scan error %s: %s", charm_data, e)
291
index_queue.put(charm_data)
294
def check_charms_store(in_queue, out_queue):
295
log = logging.getLogger("charm.store")
297
item = in_queue.next()
301
charm_data = item.payload
302
log.debug("Checking store for %s", charm_data["branch_spec"])
303
check_store(charm_data)
305
log.exception("store error on %s: %s", charm_data, str(e))
308
out_queue.put(charm_data)
310
# Remove the job from the input queue.