~jcsackett/charmworld/bac-tag-constraints

« back to all changes in this revision

Viewing changes to charmworld/jobs/tests/test_ingest.py

  • Committer: Tarmac
  • Author(s): Brad Crittenden
  • Date: 2014-04-14 20:07:28 UTC
  • mfrom: (500.1.9 es-missing-data)
  • Revision ID: tarmac-20140414200728-ew38mtkdrsuprcff
Do not index charm annotations in bundles.

Charms in bundles have "annotations" which include information relating to the
widget layout when placed in the GUI ('gui-x' and 'gui-y').  This information
is ancilliary to the actual bundle and should not be searchable.

The presence of such data has also caused operational problems.  In
production, some bundles are not being indexed due to parsing errors of the x
and y coordinates.

An example failure is shown here:
2014-04-11 19:12:30,096 DEBUG [pyelasticsearch][MainThread] got response {u'items': [{u'index': {u'_type': u'bundle', u'_id': u'~charmers/muletrain/wiki', u'error': u'MapperParsingException[failed to parse [data.data.services.mediawiki.annotations.gui-y]]; nested: NumberFormatException[For input string: "-117.5"]; ', u'_index': u'charms-78512'}}], u'took': 4}

Unfortunately, this ingestion failure cannot be reproduced on staging or
locally.  Further, it appears pyelasticsearch is not returning an error in
this situation but is instead silently failing to index the bundle.

To QA the branch, in one window run 'make run'.  In another, ingest some
bundles:

% bin/ingest-queued --prefix="~charmers/charms/bundles"

Afterwards, go to 'localhost:2464' in your browser.  Search for 'bundles' and
see the ones that got ingested.  Next search for 'gui-x' and see no results.

https://codereview.appspot.com/87710043/

R=jcsackett.

Approved by Juju Gui Bot.

Show diffs side-by-side

added added

removed removed

Lines of Context:
143
143
        charm_data = factory.get_charm_json()
144
144
        with charm_update_environment(charm_data, self.use_index_client()):
145
145
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
146
 
        self.assertIsNot(None, self.index_client.get(charm_data['_id']))
 
146
        self.assertIsNotNone(self.index_client.get(charm_data['_id']))
147
147
 
148
148
    def test_run_updates_mongo(self):
149
149
        charm_data = factory.get_charm_json()
150
150
        with charm_update_environment(charm_data, self.use_index_client()):
151
151
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
152
 
        self.assertIsNot(None, self.db.charms.find_one(charm_data['_id']))
 
152
        self.assertIsNotNone(self.db.charms.find_one(charm_data['_id']))
153
153
 
154
154
    def test_run_includes_old_mongo_data_in_index(self):
155
155
        charm_data = factory.get_charm_json()
178
178
        update_handler = self.get_handler('charm.update')
179
179
        with charm_update_environment(charm_data, self.use_index_client()):
180
180
            run_job(UpdateCharmJob(), self.payload(charm_data, 2), db=self.db)
181
 
        logs = ''.join(r.getMessage() for r in update_handler.buffer)
182
 
        self.assertEqual('Skipping %s' % charm_data['_id'], logs)
 
181
        logs = tuple(r.getMessage() for r in update_handler.buffer)
 
182
        _id = charm_data['_id']
 
183
        expected = (
 
184
            u'Updating charm {}'.format(_id),
 
185
            u'Skipping {}'.format(_id),
 
186
        )
 
187
        self.assertEqual(expected, logs)
183
188
 
184
189
    def test_old_revision_of_charm_saved_if_not_in_db(self):
185
190
        charm_data = factory.get_charm_json(revision=1)
188
193
            with patch('charmworld.jobs.ingest.trigger_tests'):
189
194
                run_job(
190
195
                    UpdateCharmJob(), self.payload(charm_data, 2), db=self.db)
191
 
        logs = ''.join(r.getMessage() for r in update_handler.buffer)
192
 
        self.assertEqual('Saving %s' % charm_data['_id'], logs)
 
196
        logs = tuple(r.getMessage() for r in update_handler.buffer)
 
197
        _id = charm_data['_id']
 
198
        expected = ('Updating charm {}'.format(_id),
 
199
                    'Saving {}'.format(_id),
 
200
                    )
 
201
        self.assertEqual(expected, logs)
193
202
 
194
203
    def test_trigger_tests_builds_correct_url(self):
195
204
        with patch('requests.get') as patched_get:
241
250
            with patch('charmworld.jobs.ingest.trigger_tests'):
242
251
                run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
243
252
        self.assertIsNotNone(self.db.charms.find_one(charm_data['_id']))
244
 
        logs = ''.join(r.getMessage() for r in update_handler.buffer)
245
 
        update_charm_logs = ''.join(r.getMessage()
246
 
                                    for r in update_charm_handler.buffer)
247
 
        self.assertEqual('Saving %s' % charm_data['_id'], logs)
 
253
        logs = tuple(r.getMessage() for r in update_handler.buffer)
 
254
        update_charm_logs = ' '.join(
 
255
            r.getMessage() for r in update_charm_handler.buffer)
 
256
        _id = charm_data['_id']
 
257
        expected = ('Updating charm {}'.format(_id),
 
258
                    'Saving {}'.format(_id),
 
259
                    )
 
260
        self.assertEqual(expected, logs)
248
261
        self.assertIn("'NoneType' object has no attribute 'startswith'",
249
262
                      update_charm_logs)
250
263
 
364
377
 
365
378
    with patch(
366
379
        'charmworld.jobs.ingest.checkout_branch',
367
 
        side_effect=checkout_branch_mock
 
380
        side_effect=checkout_branch_mock,
368
381
    ) as checkout_branch:
369
382
        with bzr_isolation() as working_dir:
370
383
            with patch('charmworld.jobs.ingest.UpdateBundleJob.store_bundles'):
388
401
        basket_data = factory.get_payload_json(name=NAME)
389
402
        with bundle_update_environment():
390
403
            run_job(UpdateBundleJob(), basket_data, db=self.db)
391
 
            self.assertIsNotNone(self.db.baskets.find_one(basket_data['_id']))
 
404
        self.assertIsNotNone(self.db.baskets.find_one(basket_data['_id']))
392
405
 
393
406
    def test_run_creates_branch(self):
394
407
        # The ingest job fetches the target branch.
539
552
            with bundle_update_environment(source_branch_dir):
540
553
                job.decorate_basket(bundle_data, fs)
541
554
        bundles_hash = bundle_data['file_hashes'].get('bundles%2Eyaml')
542
 
        self.assertIsNot(None, bundles_hash)
 
555
        self.assertIsNotNone(bundles_hash)
543
556
        fs_yaml = fs.get(bundles_hash).read()
544
557
        self.assertEqual(yaml.load(original_yaml), yaml.load(fs_yaml))
545
558
 
752
765
 
753
766
    def test_run_job_defaults(self):
754
767
        job = TestJob()
755
 
        result = run_job(job, charm_data='foo')
 
768
        result = run_job(job, data='foo')
756
769
        self.assertTrue(result)
757
770
        self.assertTrue(job.setup_called)
758
771
        self.assertTrue(job.run_called)