← Back to branch summary

~jcsackett/charmworld/bac-tag-constraints

« back to all changes in this revision

Viewing changes to charmworld/jobs/ingest.py

  • Committer: Aaron Bentley
  • Date: 2013-03-28 16:55:07 UTC
  • mto: This revision was merged to the branch mainline in revision 182.
  • Revision ID: aaron@canonical.com-20130328165507-yoiwb2pdby5tz9n8
Remove all mention of Xapian.

Show diffs side-by-side

added added

removed removed

Lines of Context:
charmworld/jobs/ingest.py
16
16
from bzrlib.revisionspec import RevisionSpec
17
17
from bzrlib.transport import get_transport
18
18
import yaml
19
 
import xappy
20
19
 
21
20
from charmworld.models import getconnection
22
21
from charmworld.models import getdb
27
26
from charmworld.utils import quote_yaml
28
27
 
29
28
from config import CHARM_DIR
30
 
from config import CHARM_INDEX_DIR
31
29
from config import CHARM_PROOF_PATH
32
30
from config import settings
33
31
from config import STORE_URL
219
217
            branch)
220
218
 
221
219
 
222
 
class XappyClient:
223
 
 
224
 
    def __init__(self, indexer=None):
225
 
        if indexer is None:
226
 
            indexer = self._make_indexer()
227
 
        self.indexer = indexer
228
 
 
229
 
    @staticmethod
230
 
    def _make_indexer(self):
231
 
        indexer = xappy.IndexerConnection(CHARM_INDEX_DIR)
232
 
        indexer.add_field_action('owner', xappy.FieldActions.INDEX_EXACT)
233
 
        indexer.add_field_action('series', xappy.FieldActions.INDEX_EXACT)
234
 
        indexer.add_field_action('subordinate', xappy.FieldActions.INDEX_EXACT)
235
 
        indexer.add_field_action('provides', xappy.FieldActions.INDEX_EXACT)
236
 
        indexer.add_field_action('requires', xappy.FieldActions.INDEX_EXACT)
237
 
 
238
 
        # Store content
239
 
        indexer.add_field_action('name', xappy.FieldActions.STORE_CONTENT)
240
 
        indexer.add_field_action('store_url', xappy.FieldActions.STORE_CONTENT)
241
 
        indexer.add_field_action('summary', xappy.FieldActions.STORE_CONTENT)
242
 
        indexer.add_field_action('short_url', xappy.FieldActions.STORE_CONTENT)
243
 
        indexer.add_field_action('label', xappy.FieldActions.STORE_CONTENT)
244
 
        indexer.add_field_action('series', xappy.FieldActions.STORE_CONTENT)
245
 
        indexer.add_field_action('owner', xappy.FieldActions.STORE_CONTENT)
246
 
        indexer.add_field_action(
247
 
            'subordinate', xappy.FieldActions.STORE_CONTENT)
248
 
 
249
 
        # Full text search fields
250
 
        indexer.add_field_action(
251
 
            'name', xappy.FieldActions.INDEX_FREETEXT, weight=10,
252
 
            language='en')
253
 
 
254
 
        indexer.add_field_action(
255
 
            'summary', xappy.FieldActions.INDEX_FREETEXT, weight=5,
256
 
            language='en')
257
 
 
258
 
        indexer.add_field_action(
259
 
            'description', xappy.FieldActions.INDEX_FREETEXT, weight=3,
260
 
            language='en')
261
 
 
262
 
        indexer.add_field_action(
263
 
            'config', xappy.FieldActions.INDEX_FREETEXT, language='en')
264
 
 
265
 
        indexer.add_field_action(
266
 
            'relations', xappy.FieldActions.INDEX_FREETEXT, language='en')
267
 
 
268
 
        indexer.add_field_action(
269
 
            'changes', xappy.FieldActions.INDEX_FREETEXT, language='en')
270
 
 
271
 
    def index_charm(self, charm):
272
 
        doc = xappy.UnprocessedDocument()
273
 
 
274
 
        # Weight critical fields higher for official charms.
275
 
        if charm['owner'] == 'charmers':
276
 
            weight = 10
277
 
        else:
278
 
            weight = 1
279
 
        doc.fields.append(xappy.Field(
280
 
            "name", charm["name"], weight=weight))
281
 
        doc.fields.append(xappy.Field(
282
 
            "summary", charm["summary"], weight=weight))
283
 
        doc.fields.append(xappy.Field(
284
 
            "description", charm["description"], weight=weight))
285
 
 
286
 
        doc.fields.append(xappy.Field("owner", charm["owner"]))
287
 
        doc.fields.append(xappy.Field("short_url", charm["short_url"]))
288
 
        doc.fields.append(xappy.Field("label", charm["label"]))
289
 
        doc.fields.append(xappy.Field("series", charm["series"]))
290
 
 
291
 
        if charm.get('subordinate'):
292
 
            doc.fields.append(xappy.Field('subordinate', 'true'))
293
 
 
294
 
        if "config" in charm \
295
 
           and charm['config'] \
296
 
           and 'options' in charm['config'] \
297
 
           and charm['config']['options']:
298
 
            config_text = []
299
 
 
300
 
            for key, option in charm["config"]["options"].items():
301
 
                config_text.append(key)
302
 
                config_text.append(option.get("description", ""))
303
 
            doc.fields.append(xappy.Field("config", " ".join(config_text)))
304
 
 
305
 
        relation_text = []
306
 
 
307
 
        if "requires" in charm and charm["requires"]:
308
 
            for key, option in charm["requires"].items():
309
 
                relation_text.append(key)
310
 
                relation_text.append(option["interface"])
311
 
                doc.fields.append(xappy.Field('requires', option["interface"]))
312
 
        if "provides" in charm and charm["provides"]:
313
 
            for key, option in charm["provides"].items():
314
 
                if not isinstance(option, dict):
315
 
                    self.log.warning(
316
 
                        "invalid charm provides %s", charm['branch_spec'])
317
 
                    continue
318
 
                relation_text.append(key)
319
 
                relation_text.append(option["interface"])
320
 
                doc.fields.append(xappy.Field("provides", option["interface"]))
321
 
 
322
 
        if relation_text:
323
 
            doc.fields.append(
324
 
                xappy.Field("relations", " ".join(relation_text)))
325
 
 
326
 
        change_text = []
327
 
        for change in charm.get("changes", ()):
328
 
            change_text.append(change['message'])
329
 
            change_text.append(change['committer'])
330
 
 
331
 
        if change_text:
332
 
            doc.fields.append(xappy.Field("changes", " ".join(change_text)))
333
 
 
334
 
        if 'store_url' in charm:
335
 
            doc.fields.append(xappy.Field('store_url', charm['store_url']))
336
 
        else:
337
 
            self.log.warning("No store url found for %s", charm["_id"])
338
 
        doc.id = charm["_id"]
339
 
        self.indexer.replace(doc)
340
 
        self.indexer.flush()
341
 
 
342
 
 
343
220
class IndexIngestJob(IngestJob):
344
221
 
345
222
    name = 'index'