~bac/charmworld/tag-constraints

# Copyright 2013-2014 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

__metaclass__ = type


from contextlib import contextmanager
import hashlib
import json
import logging
import os.path
import shutil
import subprocess
from textwrap import dedent
import time

import bzrlib
from bzrlib.bzrdir import BzrDir
from httmock import (
    HTTMock,
    urlmatch,
)
from mock import patch
import yaml

from charmworld.charmstore import (
    CharmStore,
)
from charmworld.jobs.config import CHARM_DIR
from charmworld.jobs.ingest import (
    fetch_branch,
    IngestError,
    IngestJob,
    run_job,
    should_update_charm,
    slurp_files,
    trigger_tests,
    update_branch,
    update_charm,
    update_hash,
    UpdateCharmJob,
    UpdateBundleJob,
    update_date_created,
)
from charmworld.models import (
    CharmSource,
    getfs,
)
from charmworld.testing import (
    bzr_isolation,
    factory,
    force_index_client,
    MongoTestBase,
    response404,
    TestCase,
)
from charmworld.utils import (
    get_ini,
    LAST_INGEST_JOB_FINISHED,
    quote_key,
)


class TestUpdateDateCreated(TestCase):

    def test_update_date_created(self):
        result = json.dumps(
            {'date_created': '2012-03-06T17:59:55.818912+00:00'})
        charm = {'branch_spec': 'steve'}
        with HTTMock(lambda x, y: result):
            update_date_created(charm, logging.getLogger('foo'))
        self.assertEqual({
            'date_created': '2012-03-06T17:59:55Z',
            'branch_spec': 'steve',
        }, charm)

    def test_update_date_created_ignores_existing(self):
        charm = {'date_created': None, 'branch_spec': 'steve'}
        update_date_created(charm, logging.getLogger('foo'))
        self.assertEqual({'date_created': None, 'branch_spec': 'steve'}, charm)

    def test_update_date_created_no_branch(self):
        with HTTMock(response404):
            charm = {'branch_spec': 'steve'}
            update_date_created(charm, logging.getLogger('foo'))
        self.assertNotIn('date_created', charm)


@contextmanager
def charm_update_environment(real_charm_data, index_client):
    """Mock many things that are used when updating charms."""

    def checkout_branch(charm_data, branch_dir, log, revisionId):
        wt = BzrDir.create_standalone_workingtree(branch_dir)
        wt.bzrdir.root_transport.mkdir('hooks')
        wt.bzrdir.root_transport.put_bytes('metadata.yaml', json.dumps({
            'name': charm_data['name'],
            'summary': real_charm_data['summary'],
        }))
        wt.add(['hooks', 'metadata.yaml'])
        charm_data['store_data']['digest'] = wt.commit('commit 1')
    checkout_patch = patch(
        'charmworld.jobs.ingest.checkout_branch', checkout_branch)

    @urlmatch(path='/stats/counter/charm-bundle:.*')
    def mock_counts(url, request):
        return json.dumps([[0]])

    @urlmatch(netloc='api.launchpad.net')
    def mock_date_created(url, request):
        return json.dumps({'date_created': '2005-06-07T12:13:14.15+00:00'})

    test_results = urlmatch(netloc='jenkins.qa.ubuntu.com')(response404)
    httmock = HTTMock(mock_counts, test_results, mock_date_created)
    with bzr_isolation() as cwd, checkout_patch:
        root_dir = os.path.join(cwd, 'root_dir')
        dir_patch = patch('charmworld.jobs.ingest.CHARM_DIR', root_dir)
        with dir_patch, httmock, force_index_client(index_client):
            yield


class TestUpdateCharmJob(MongoTestBase):

    @staticmethod
    def payload(charm_data, newest_revision=1):
        payload = {
            'series': charm_data['series'],
            'owner': charm_data['owner'],
            'name': charm_data['name'],
            'bname': charm_data['bname'],
            'commit': charm_data['commit'],
            'branch_spec': charm_data['branch_spec'],
            'branch_deleted': charm_data['branch_deleted'],
            'promulgated': charm_data['promulgated'],
            'store_data': charm_data['store_data'],
            'store_url': charm_data['store_url'],
        }
        if newest_revision is not None:
            payload['newest_revision'] = newest_revision
        return payload

    def test_run_indexes(self):
        charm_data = factory.get_charm_json()
        with charm_update_environment(charm_data, self.use_index_client()):
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
        self.assertIsNotNone(self.index_client.get(charm_data['_id']))

    def test_run_updates_mongo(self):
        charm_data = factory.get_charm_json()
        with charm_update_environment(charm_data, self.use_index_client()):
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
        self.assertIsNotNone(self.db.charms.find_one(charm_data['_id']))

    def test_run_includes_old_mongo_data_in_index(self):
        charm_data = factory.get_charm_json()
        charm_data['foo'] = 'bar'
        self.db.charms.save(charm_data)
        del charm_data['foo']
        with charm_update_environment(charm_data, self.use_index_client()):
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
        indexed_charm = self.index_client.get(charm_data['_id'])
        self.assertEqual('bar', indexed_charm.get('foo'))

    def test_mongo_and_index_store_identical_charm(self):
        charm_data = factory.get_charm_json(revision=1)
        charm_data['foo'] = 'bar'
        self.db.charms.save(charm_data)
        del charm_data['foo']
        with charm_update_environment(charm_data, self.use_index_client()):
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
        indexed_charm = self.index_client.get(charm_data['_id'])
        mongo_charm = self.db.charms.find_one(charm_data['_id'])
        self.assertEqual(indexed_charm, mongo_charm)

    def test_old_revision_of_charm_skipped(self):
        charm_data = factory.get_charm_json(revision=1)
        self.db.charms.save(charm_data)
        update_handler = self.get_handler('charm.update')
        with charm_update_environment(charm_data, self.use_index_client()):
            run_job(UpdateCharmJob(), self.payload(charm_data, 2), db=self.db)
        logs = tuple(r.getMessage() for r in update_handler.buffer)
        _id = charm_data['_id']
        expected = (
            u'Updating charm {}'.format(_id),
            u'Skipping {}'.format(_id),
        )
        self.assertEqual(expected, logs)

    def test_old_revision_of_charm_saved_if_not_in_db(self):
        charm_data = factory.get_charm_json(revision=1)
        update_handler = self.get_handler('charm.update')
        with charm_update_environment(charm_data, self.use_index_client()):
            with patch('charmworld.jobs.ingest.trigger_tests'):
                run_job(
                    UpdateCharmJob(), self.payload(charm_data, 2), db=self.db)
        logs = tuple(r.getMessage() for r in update_handler.buffer)
        _id = charm_data['_id']
        expected = ('Updating charm {}'.format(_id),
                    'Saving {}'.format(_id),
                    )
        self.assertEqual(expected, logs)

    def test_trigger_tests_builds_correct_url(self):
        with patch('requests.get') as patched_get:
            trigger_tests('~foo', 1)
        settings = get_ini()
        charm_test_url = settings['charm_test_url']
        charm_test_token = settings['charm_test_token']
        charm_test_job = settings['charm_test_job']
        expected_url = ('%s/job/%s/buildWithParameters?'
                        'token=%s&branch=lp:~foo&revno=1') % (
                            charm_test_url, charm_test_job, charm_test_token)
        patched_get.assert_called_once_with(expected_url)

    def test_run_does_not_trigger_tests_for_revisions_in_db(self):
        charm_data = factory.get_charm_json(store_revision=2)
        charm_data['promulgated'] = True
        self.db.charms.save(charm_data)
        with charm_update_environment(charm_data, self.use_index_client()):
            with patch('charmworld.jobs.ingest.trigger_tests') as trigger:
                run_job(
                    UpdateCharmJob(), self.payload(charm_data, 2), db=self.db)
        self.assertEqual(0, trigger.call_count)

    def test_run_does_not_trigger_tests_for_old_revision(self):
        charm_data = factory.get_charm_json(store_revision=2)
        self.db.charms.save(charm_data)
        with charm_update_environment(charm_data, self.use_index_client()):
            with patch('charmworld.jobs.ingest.trigger_tests') as trigger:
                run_job(
                    UpdateCharmJob(), self.payload(charm_data, 1), db=self.db)
        self.assertEqual(0, trigger.call_count)

    def test_run_triggers_tests_with_new_revision_not_in_db(self):
        charm_data = factory.get_charm_json(store_revision=2)
        charm_data['promulgated'] = True
        with charm_update_environment(charm_data, self.use_index_client()):
            with patch('charmworld.jobs.ingest.trigger_tests') as trigger:
                run_job(
                    UpdateCharmJob(), self.payload(charm_data, 2), db=self.db)
        self.assertEqual(1, trigger.call_count)

    def test_saves_early_error(self):
        charm_data = factory.get_charm_json()
        # Induce improbable error -- bname must be a string.
        charm_data['bname'] = None
        update_handler = self.get_handler('charm.update')
        update_charm_handler = self.get_handler('charm.update_charm')
        with patch('charmworld.jobs.ingest.update_from_store'):
            with patch('charmworld.jobs.ingest.trigger_tests'):
                run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
        self.assertIsNotNone(self.db.charms.find_one(charm_data['_id']))
        logs = tuple(r.getMessage() for r in update_handler.buffer)
        update_charm_logs = ' '.join(
            r.getMessage() for r in update_charm_handler.buffer)
        _id = charm_data['_id']
        expected = ('Updating charm {}'.format(_id),
                    'Saving {}'.format(_id),
                    )
        self.assertEqual(expected, logs)
        self.assertIn("'NoneType' object has no attribute 'startswith'",
                      update_charm_logs)

    def test_updates_promulgated(self):
        charm_data = factory.get_charm_json(promulgated=False)
        self.use_index_client()
        CharmSource.from_request(self).save(charm_data)
        charm_data['promulgated'] = True
        with charm_update_environment(charm_data, self.index_client):
            run_job(UpdateCharmJob(), self.payload(charm_data), db=self.db)
        self.assertTrue(
            self.db.charms.find_one(charm_data['_id'])['promulgated'])

    def test_should_update_if_newest(self):
        self.assertTrue(should_update_charm(7, 7, True))

    def test_should_update_if_not_in_db(self):
        self.assertTrue(should_update_charm(7, 9, False))

    def test_should_not_update_if_neither(self):
        self.assertFalse(should_update_charm(7, 9, True))


class TestUpdateBranch(TestCase):

    def setUp(self):
        self.use_context(bzr_isolation())

    def test_update_branch_passes_on_subprocess_errors_if_not_retrying(self):
        def check_output(command, *args, **kws):
            self.assertEqual(command[:2], ['/usr/bin/bzr', 'update'])
            raise subprocess.CalledProcessError('foo', 'bar')

        with patch('subprocess.check_output', check_output):
            with self.assertRaises(subprocess.CalledProcessError):
                update_branch(
                    'ROOT-DIR', {'branch_spec': 'BRANCH-SPEC'}, '',
                    logging.getLogger('foo'), 'REVISION-ID')

    def test_update_branch_passes_retries(self):
        # If the bzr update fails, the branch is deleted and checked out anew.

        first_time = [True]

        def check_output(command, *args, **kws):
            if first_time:
                del first_time[0]
                self.assertEqual(command[:2], ['/usr/bin/bzr', 'update'])
                raise subprocess.CalledProcessError('foo', 'bar')
            else:
                self.assertEqual(command[:2], ['/usr/bin/bzr', 'checkout'])

        with patch('subprocess.check_output', check_output):
            with patch('shutil.rmtree') as rmtree:
                update_branch(
                    'ROOT-DIR', {'branch_spec': 'BRANCH-SPEC'}, '',
                    logging.getLogger('foo'), 'REVISION-ID', retry=True)
                self.assertTrue(rmtree.called)

    def test_update_branch_specifies_the_revision_id(self):
        # If a revision ID is specified in the call to update_branch, it will
        # construct a revision spec that includes the given ID.
        def check_output(command, *args, **kws):
            self.assertIn('revid:REVISION-ID', command)
            raise subprocess.CalledProcessError('foo', 'bar')

        with patch('subprocess.check_output', check_output):
            with self.assertRaises(subprocess.CalledProcessError):
                update_branch(
                    'ROOT-DIR', {'branch_spec': 'BRANCH-SPEC'}, '',
                    logging.getLogger('foo'), 'REVISION-ID')


class TestFetchBranch(TestCase):
    def setUp(self):
        self.use_context(bzr_isolation())
        self.logger = logging.getLogger('test_fetch_branch')
        self.construct_branch_dir_patch = patch(
            'charmworld.jobs.ingest.construct_branch_dir',
            self.construct_branch_dir_mock)
        self.makedirs_patch = patch('os.makedirs')
        self.branch_data = {'branch_spec': 'Fake branch'}

    @staticmethod
    def construct_branch_dir_mock(root_dir, branch_data):
        return '/tmp/{}'.format(root_dir)

    def test_charm_is_new(self):
        with self.construct_branch_dir_patch, self.makedirs_patch:
            with patch('os.path.exists', lambda x: False):
                with patch('charmworld.jobs.ingest.checkout_branch') as mock:
                    fetch_branch('root_dir', self.branch_data, self.logger)
        mock.assert_called_with(
            self.branch_data, '/tmp/root_dir', self.logger, None)

    def test_charm_needs_updating(self):
        with self.construct_branch_dir_patch, self.makedirs_patch:
            with patch('os.path.exists', lambda x: True):
                with patch('charmworld.jobs.ingest.branch_is_current',
                           lambda x, y, z: False):
                    with patch('charmworld.jobs.ingest.update_branch') as mock:
                        fetch_branch('root_dir', self.branch_data,
                                     self.logger)
        mock.assert_called_with(
            'root_dir', self.branch_data, '/tmp/root_dir', self.logger, None,
            True)


@contextmanager
def bundle_update_environment(source_branch_dir=None):
    def checkout_branch_mock(charm_data, branch_dir, log, revisionId):
        if source_branch_dir is None:
            wt = BzrDir.create_standalone_workingtree(branch_dir)
            wt.commit('commit message', rev_id=charm_data['commit'])
        else:
            shutil.copytree(source_branch_dir, branch_dir)

    with patch(
        'charmworld.jobs.ingest.checkout_branch',
        side_effect=checkout_branch_mock,
    ) as checkout_branch:
        with bzr_isolation() as working_dir:
            with patch('charmworld.jobs.ingest.UpdateBundleJob.store_bundles'):
                try:
                    yield checkout_branch, working_dir
                finally:
                    # Clean up despite exceptions raised by the context
                    # manager client.
                    shutil.rmtree(CHARM_DIR, ignore_errors=True)


def mock_proof(path):
    return [], {}, 0


class TestUpdateBundleJob(MongoTestBase):

    def test_run_updates_mongo(self):
        # Running a bundle job creates a bundle document in mongo.
        NAME = 'our-source-package'
        basket_data = factory.get_payload_json(name=NAME)
        with bundle_update_environment():
            run_job(UpdateBundleJob(), basket_data, db=self.db)
        self.assertIsNotNone(self.db.baskets.find_one(basket_data['_id']))

    def test_run_creates_branch(self):
        # The ingest job fetches the target branch.
        NAME = 'our-source-package'
        bundle_data = factory.get_payload_json(name=NAME)
        with bundle_update_environment() as (checkout_branch, working_dir):
            run_job(
                UpdateBundleJob(working_dir=working_dir), bundle_data,
                db=self.db)
            self.assertTrue(checkout_branch.called)

    def test_slurp_files_puts_files_in_gridfs(self):
        with bundle_update_environment() as (checkout_branch, working_dir):
            job = UpdateBundleJob()
            job.setup(self.db)
            with factory.bzr_branch() as (cwd, _, tree):
                bytes_ = factory.random_string()
                tree.bzrdir.root_transport.put_bytes('foo', bytes_)
                tree.add(['foo'])
                tree.commit(message='add the file')
                hashes = slurp_files(getfs(self.db, 'hashed-files'), tree)

            expected_hash = hashlib.sha256(bytes_).hexdigest()
            self.assertEqual(hashes, {'foo': expected_hash})
            fs = getfs(self.db, 'hashed-files')
            self.assertEqual(fs.get(expected_hash).read(), bytes_)

    def test_dots_in_file_paths_do_not_break_things(self):
        # Since mongodb doesn't like keys with dots in them and we want to
        # store a mapping from file name (potentially with dots) to file hash,
        # we need to quote the file path.
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(job, 'proof', mock_proof):
            with patch.object(job, 'get_deployer_config'):
                with bundle_update_environment():
                    with patch(
                            'charmworld.jobs.ingest.slurp_files',
                            side_effect=lambda *_: {'a.b': 'X'}) as slurp:
                        bundle_data = factory.get_payload_json()
                        job.run(bundle_data)
                        self.assertTrue(slurp.called)

    def test_file_paths_with_dots_can_be_retrieved(self):
        # Since we have to escape file paths for mongodb, it is important that
        # we check that file paths containing dots can be retrieved.
        job = UpdateBundleJob()
        basket_data = {'file_hashes': {quote_key('bundles.yaml'): 'HASH'}}

        class FauxFS:
            @staticmethod
            def get(hash):
                self.assertEqual(hash, 'HASH')
                self.get_called = True
                return ''
        job.get_deployer_config(FauxFS(), basket_data)
        self.assertTrue(self.get_called)

    def test_file_missing_yaml(self):
        # If the file 'bundles.yaml' is not found, get_deployer_config returns
        # None.  Note we're using 'bundle.yaml' for the test.
        job = UpdateBundleJob()
        basket_data = {'file_hashes': {quote_key('bundle.yaml'): 'HASH'}}
        self.get_called = False

        class FauxFS:
            @staticmethod
            def get(hash):
                self.assertEqual(hash, 'HASH')
                self.get_called = True
                return ''

        config = job.get_deployer_config(FauxFS(), basket_data)
        self.assertIsNone(config)
        self.assertFalse(self.get_called)

    def test_run_pulls_in_all_branch_files(self):
        # The ingest job puts all files in the branch into the database.
        NAME = 'our-source-package'
        bundle_data = factory.get_payload_json(name=NAME)
        with bundle_update_environment() as (checkout_branch, working_dir):
            run_job(
                UpdateBundleJob(working_dir=working_dir), bundle_data,
                db=self.db)
            self.assertTrue(checkout_branch.called)

    def test_run_slurps(self):
        # The run method calls slurp_files to actually read the files into the
        # database.
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(job, 'proof', mock_proof):
            with patch.object(job, 'get_deployer_config'):
                with bundle_update_environment():
                    with patch(
                            'charmworld.jobs.ingest.slurp_files',
                            side_effect=lambda *_: {}) as slurp_files:
                        bundle_data = factory.get_payload_json()
                        job.run(bundle_data)
                        self.assertTrue(slurp_files.called)

    def test_bundle_knows_where_its_files_are(self):
        # The mapping of bundle file paths to hashes (keys in gridfs) is
        # stored in the bundle document.
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(job, 'proof', mock_proof):
            with patch.object(job, 'get_deployer_config'):
                with bundle_update_environment():
                    def slurp_files_side_effect(*args):
                        return {'path1': 'hash1', 'path2': 'hash2'}
                    with patch(
                            'charmworld.jobs.ingest.slurp_files',
                            side_effect=slurp_files_side_effect):
                        bundle_data = factory.get_payload_json()
                        job.run(bundle_data)
                        self.assertEqual(
                            bundle_data['file_hashes'],
                            {'path1': 'hash1', 'path2': 'hash2'})

    def test_bundle_ids_include_their_revno(self):
        job = UpdateBundleJob()
        job.setup(self.db)
        bundle_data = factory.get_payload_json()
        # We use multiple commits to show that the latest revno is used.
        with bzr_isolation() as source_branch_dir:
            wt = BzrDir.create_standalone_workingtree(source_branch_dir)
            for x in range(7):
                wt.commit('commit %d' % x)
            wt.commit('commit message', rev_id=bundle_data['commit'])

            with bundle_update_environment(source_branch_dir):
                job.decorate_basket(bundle_data, None)

        self.assertTrue(bundle_data['_id'].endswith('/8'))

    def test_decorate_basket_stores_files(self):
        job = UpdateBundleJob()
        fs = getfs(self.db, collection='hashed-files')
        bundle_data = factory.get_payload_json()
        original_yaml = '{foo: {}}'
        with bzr_isolation() as source_branch_dir:
            wt = BzrDir.create_standalone_workingtree(source_branch_dir)
            wt.bzrdir.root_transport.put_bytes(
                'bundles.yaml', original_yaml)
            wt.add('bundles.yaml')
            bundle_data['commit'] = wt.commit('commit message')
            with bundle_update_environment(source_branch_dir):
                job.decorate_basket(bundle_data, fs)
        bundles_hash = bundle_data['file_hashes'].get('bundles%2Eyaml')
        self.assertIsNotNone(bundles_hash)
        fs_yaml = fs.get(bundles_hash).read()
        self.assertEqual(yaml.load(original_yaml), yaml.load(fs_yaml))

    def test_missing_revisions_generate_errors(self):
        job = UpdateBundleJob()
        job.setup(self.db)
        bundle_data = factory.get_payload_json()
        # We use multiple commits to show that the latest revno is used.
        with bzr_isolation() as source_branch_dir:
            wt = BzrDir.create_standalone_workingtree(source_branch_dir)
            base = wt.commit('commit message')
            wt.commit('commit message', rev_id=bundle_data['commit'])
            wt.pull(wt.branch, stop_revision=base, overwrite=True)
            wt.merge_from_branch(wt.branch, to_revision=bundle_data['commit'])
            wt.commit('do a merge which hides the second commit')

            with bundle_update_environment(source_branch_dir):
                with self.assertRaises(bzrlib.errors.NoSuchRevision):
                    job.decorate_basket(bundle_data, None)

    def test_set_basket_info_includes_owner_if_promulgated(self):
        payload = factory.get_payload_json(name='mysql', promulgated=True)
        UpdateBundleJob.set_basket_info(payload, 5)
        self.assertEqual('mysql/5', payload['name_revno'])
        self.assertEqual('~charmers/mysql/5', payload['_id'])

    def test_set_basket_info_includes_owner_if_not_promulgated(self):
        payload = factory.get_payload_json(name='mysql', promulgated=False)
        UpdateBundleJob.set_basket_info(payload, 5)
        self.assertEqual('mysql/5', payload['name_revno'])
        self.assertEqual('~charmers/mysql/5', payload['_id'])

    def test_job_can_store_bundles_in_the_db(self):
        basket_data = factory.get_payload_json(name='wordpress')
        UpdateBundleJob.set_basket_info(basket_data, 5)
        bundle_id = '%s/%s' % (basket_data['_id'], 'stage')
        job = UpdateBundleJob()
        job.setup(self.db)
        DEPLOYER_CONFIG_HASH = '31337'
        basket_data['file_hashes'] = {
            'bundles.yaml': DEPLOYER_CONFIG_HASH,
        }
        fs = getfs(self.db)
        deployer_config = dedent("""\
            stage:
                series: precise
                services:
                    blog:
                        charm: cs:precise/wordpress
                        constraints: mem=2
                        options:
                            tuning: optimized
                            engine: apache
        """)
        fs.put(deployer_config, _id=DEPLOYER_CONFIG_HASH)
        job.store_bundles(
            yaml.safe_load(deployer_config), basket_data['owner'],
            basket_data['name_revno'], None, None, None, None, None, False)
        self.assertIsNotNone(self.db.bundles.find_one(bundle_id))

    def test_job_run_stores_bundles_in_the_db(self):
        basket_data = factory.get_payload_json(name='wordpress')
        basket_data['name_revno'] = 'dummy'
        basket_data['first_change'] = None
        basket_data['last_change'] = None
        basket_data['changes'] = None
        basket_data['branch_spec'] = None
        basket_data['branch_dir'] = 'dummy-dir'
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(job, 'proof', mock_proof):
            with patch.object(job, 'get_deployer_config'):
                with patch.object(job, 'decorate_basket'):
                    with patch.object(job, 'store_bundles') as store_bundles:
                        job.run(basket_data)
                        self.assertTrue(store_bundles.called)

    def test_job_run_does_not_alter_bundle_data(self):
        basket_data = factory.get_payload_json(name='wordpress')
        basket_data['name_revno'] = 'dummy'
        basket_data['first_change'] = None
        basket_data['last_change'] = None
        basket_data['changes'] = None
        basket_data['branch_spec'] = None
        basket_data['branch_dir'] = 'dummy-dir'
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(job, 'proof', mock_proof):
            with patch.object(job, 'get_deployer_config',
                              side_effect=lambda x, y: {'foo': {}}):
                with patch.object(job, 'decorate_basket'):
                    with patch.object(job, 'store_bundles') as store_bundles:
                        job.run(basket_data)
                        store_bundles.assert_called_with(
                            {'foo': {}}, 'charmers', 'dummy', None, None, None,
                            None, {}, False)

    def test_proof_is_called_with_branch_dir(self):
        basket_data = factory.get_payload_json(name='wordpress')
        basket_data['name_revno'] = 'dummy'
        basket_data['first_change'] = None
        basket_data['last_change'] = None
        basket_data['changes'] = None
        basket_data['branch_dir'] = branch_dir = 'dummy-dir'
        job = UpdateBundleJob()
        job.setup(self.db)
        called_with = []

        def mock_proof_redux(arg):
            called_with.append(arg)
            return [], {}, 0

        with patch.object(job, 'get_deployer_config',
                          side_effect=lambda x, y: {'foo': {}}):
            with patch.object(job, 'decorate_basket'):
                with patch.object(job, 'store_bundles'):
                    with patch.object(job, 'proof', mock_proof_redux):
                        job.run(basket_data)
                        self.assertEqual(called_with, [branch_dir])

    def test_proof_rejects_with_errors(self):
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(UpdateBundleJob, '_proof',
                          side_effect=lambda x: ([], {}, 200)):
            lint, services_info, err = job.proof('dummy')
            self.assertEqual(True, err)

    def test_proof_accepts_with_warnings(self):
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(UpdateBundleJob, '_proof',
                          side_effect=lambda x: ([], {}, 100)):
            lint, services_info, err = job.proof('dummy')
            self.assertEqual(False, err)

    def test_proof_accepts_with_no_problems(self):
        job = UpdateBundleJob()
        job.setup(self.db)
        with patch.object(UpdateBundleJob, '_proof',
                          side_effect=lambda x: ([], {}, 0)):
            lint, services_info, err = job.proof('dummy')
            self.assertEqual(False, err)


class TestUpdateCharm(MongoTestBase):

    def test_update_charm_does_not_index(self):
        charm_data = factory.get_charm_json()
        with charm_update_environment(charm_data, self.use_index_client()):
            update_charm(charm_data, self.db, CharmStore())
        self.index_client.wait_for_green_status()
        self.assertIs(None, self.index_client.get(charm_data['_id']))

    def test_update_charm_does_not_update_mongo(self):
        charm_data = factory.get_charm_json()
        with charm_update_environment(charm_data, self.use_index_client()):
            update_charm(charm_data, self.db, CharmStore())
        self.assertIs(None, self.db.charms.find_one(charm_data['_id']))

    def test_forgets_past_error(self):
        charm_data = factory.get_charm_json(promulgated=True)
        charm_data['error'] = {'error_stage': 'foo', 'error': 'bar'}
        with charm_update_environment(charm_data, self.use_index_client()):
            update_charm(charm_data, self.db, CharmStore())
        self.assertNotIn('error', charm_data)

    def test_updates_download_info(self):
        charm_data = factory.get_charm_json()
        del charm_data['downloads']

        class FakeCharmStore:
            @staticmethod
            def count_downloads_in_days(charm, days, end):
                return 0

            @staticmethod
            def get_download_counts(charm, start=None, end=None):
                return [[5]]

        with charm_update_environment(charm_data, self.use_index_client()):
            update_charm(charm_data, self.db, FakeCharmStore)
        self.assertEqual(5, charm_data['downloads'])


class TestJob(IngestJob):
    name = 'test'

    def __init__(self, exc=None):
        super(TestJob, self).setup()
        self.setup_called = False
        self.run_called = False
        self.exc = exc
        self.charm_data = None

    def setup(self, db=None):
        self.setup_called = True
        self.db = db

    def run(self, charm_data):
        self.charm_data = charm_data
        if self.exc is not None:
            raise self.exc('test error')
        self.run_called = True
        if getattr(self, 'db', None) is not None:
            self.update_ingest_status()


class TestRunJob(MongoTestBase):

    def test_run_job_defaults(self):
        job = TestJob()
        result = run_job(job, data='foo')
        self.assertTrue(result)
        self.assertTrue(job.setup_called)
        self.assertTrue(job.run_called)
        self.assertEqual('foo', job.charm_data)
        self.assertIs(None, job.db)

    def test_run_job_no_setup(self):
        job = TestJob()
        run_job(job, 'foo', needs_setup=False)
        self.assertFalse(job.setup_called)

    def test_run_job_db_specified(self):
        job = TestJob()
        before = time.time()
        run_job(job, 'foo', db=self.db)
        after = time.time()
        self.assertEqual(self.db, job.db)
        job_finished = self.db.server_status.find_one(LAST_INGEST_JOB_FINISHED)
        job_finished = job_finished['time']
        self.assertTrue(before <= job_finished <= after)
        job = TestJob()
        run_job(job, 'foo', needs_setup=False, db=self.db)
        self.assertIs(None, getattr(job, 'db', None))

    def test_run_job_general_exception(self):
        handler = self.get_handler('charm.test')
        job = TestJob(ValueError)
        charm_data = {'foo': 'bar'}
        result = run_job(job, charm_data)
        self.assertFalse(result)
        expected_error = {
            'error': 'test error',
            'error_stage': 'test',
        }
        self.assertEqual(expected_error, charm_data['error'])
        log_messages = [record.getMessage() for record in handler.buffer]
        expected = "test error on {'foo': 'bar'}: test error\nTraceback"
        self.assertEqual(1, len(log_messages))
        self.assertTrue(log_messages[0].startswith(expected))
        log_level = handler.buffer[0].levelname
        self.assertEqual('ERROR', log_level)

    def test_run_job_IngestError(self):
        handler = self.get_handler('charm.test')
        job = TestJob(IngestError)
        charm_data = {'foo': 'bar'}
        result = run_job(job, charm_data)
        self.assertFalse(result)
        expected_error = {
            'error': 'test error',
            'error_stage': 'test',
        }
        self.assertEqual(expected_error, charm_data['error'])
        log_messages = [record.getMessage() for record in handler.buffer]
        expected = ["test error on {'foo': 'bar'}: test error"]
        self.assertEqual(expected, log_messages)
        log_level = handler.buffer[0].levelname
        self.assertEqual('INFO', log_level)


class TestUpdateHash(TestCase):

    def test_update_hash(self):
        # update_hash() adds a hash to a charm.
        charm_data = factory.get_charm_json()
        self.assertFalse('hash' in charm_data)
        update_hash(charm_data)
        old_hash = charm_data['hash']
        # Changing any value of the charm changes the hash.
        charm_data['name'] = charm_data['name'] + 'changed'
        update_hash(charm_data)
        self.assertNotEqual(old_hash, charm_data['hash'])

    def test_hash_value_not_included_in_calculation(self):
        # The hash value itself is not included in a new calculation of the
        # hash.
        charm_data = factory.get_charm_json()
        update_hash(charm_data)
        old_hash = charm_data['hash']
        charm_data['hash'] = 'some other value'
        update_hash(charm_data)
        self.assertEqual(old_hash, charm_data['hash'])