~bac/charmworld/tag-constraints

# Copyright 2012-2014 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

from argparse import Namespace
import json
import logging
import os.path
from requests.exceptions import ConnectionError

from httmock import HTTMock
from mock import (
    ANY,
    patch,
)
import shutil

from charmworld.lp import BASKET_SERIES
from charmworld.models import getfs
from charmworld.testing import (
    factory,
    JobTestBase,
    MONGO_DATABASE,
    MongoTestBase,
    redirect_stderr,
    substitute_argv,
    temp_dir,
    TestCase,
)

from charmworld.jobs.lp import (
    all_charms,
    all_branch_data,
    CHARM_QUEUE,
    db_charms,
    dequeue,
    local_branch_data,
    main,
    run,
    queue_from_branches,
    unspecified,
)
from charmworld.jobs.utils import get_queue
from charmworld.tests.test_charmstore import FakeCharmStore


def all_branch_data_mock(charm_data=None):
    return ([{'branch_dir': 'foo', 'branch_spec': 'bar', 'owner': 'baz',
             'series': 'qux', 'name': 'quxx', 'promulgated': False}],
            [{'foo': 'bar', 'branch_spec': '~baz/bar/1/bundle'}])


def local_branch_data_mock(local_repo, charm_data=None):
    return ([{'branch_dir': 'foo', 'branch_spec': 'bar', 'owner': 'baz',
             'series': 'qux', 'name': 'quxx', 'promulgated': True}],
            [])


def failing_requests_get(*args, **kwargs):
    raise ConnectionError


class TestLPJob(JobTestBase):

    def test_request(self):
        charm_data = [[u'~charmers/charms/precise/someproject/trunk',
                       u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
                       []]]

        def all_branch_data_mock(url, request):
            return json.dumps(charm_data)

        with HTTMock(all_branch_data_mock):
            charms, baskets = all_branch_data()

        self.assertEqual([{
            'branch_spec': u'~charmers/charms/precise/someproject/trunk',
            'branch_deleted': False,
            'owner': u'charmers',
            'series': u'precise',
            'name': u'someproject',
            'bname': u'trunk',
            'commit': u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
            'distro_series': [],
            'promulgated': False}], charms)

    def test_request_promulgated_charm(self):
        charm_data = [[u'~charmers/charms/precise/someproject/trunk',
                       u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
                       ['precise']]]

        def available_mock(url, request):
            return json.dumps(charm_data)

        with HTTMock(available_mock):
            charms, baskets = all_branch_data()

        self.assertEqual([{
            'branch_spec': u'~charmers/charms/precise/someproject/trunk',
            'branch_deleted': False,
            'owner': u'charmers',
            'series': u'precise',
            'name': u'someproject',
            'bname': u'trunk',
            'commit': u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
            'distro_series': ['precise'],
            'promulgated': True}], charms)

    def test_all_charms_explicit_filter(self):
        # If the branch doesn't start with the defined filter (i.e.
        # ~charmers/charms/precise) it is not returned
        lp_data = [
            [u'~not-charmers/charms/precise/someproject/trunk',
             u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
             []],
            [u'~charmers/charms/precise/someproject/trunk',
             u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
             []],
        ]
        factory.makeCharm(
            self.db, owner='charmers', name='only-in-db')
        factory.makeCharm(
            self.db, owner='not-charmers', name='only-in-db')
        charms = all_charms(
            self.db, all_branch_data(lp_data)[0],
            import_filter='~charmers/charms/precise')
        self.assertEqual(['~charmers/charms/precise/only-in-db/trunk',
                          '~charmers/charms/precise/someproject/trunk'],
                         [charm['branch_spec'] for charm in charms])

    def test_all_branch_data_not_trunk(self):
        # If the branch is not trunk, the charm is not returned by
        # all_branch_data.
        charm_data = [[u'~charmers/charms/precise/someproject/foo',
                       u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc',
                       []]]
        charms = all_branch_data(charm_data)[0]
        self.assertEqual([], charms)

    def test_all_charms_limit(self):
        handler = self.get_handler("charm.launchpad")
        limit = 75
        charm_data = [
            [u'~charmers/charms/precise/someproject-%s/trunk' % count,
             u'ja@appflower.com-20120329093714-s2m9e28dwotmijqc', []]
            for count in xrange(limit - 10)]
        for count in range(20):
            factory.makeCharm(self.db)
        charms = [charm for charm in all_charms(
            self.db, all_branch_data(charm_data)[0], limit=limit)]
        log_messages = [record.getMessage() for record in handler.buffer]
        err_msg = "Import limit reached (%d)... stopping" % limit
        self.assertIn(err_msg, log_messages)
        self.assertEqual(limit, len(charms))

    def test_all_branch_data_uses_series(self):
        branch = '~foo/charms/not-bundle/bar/trunk'
        revision_id = 'jrandom@example.org-asdf'
        charms, baskets = all_branch_data([(branch, revision_id, [])])
        self.assertEqual([], baskets)
        self.assertEqual(1, len(charms))
        branch = '~foo/charms/' + BASKET_SERIES + '/bar/bundle'
        charms, baskets = all_branch_data([(branch, revision_id, [])])
        self.assertEqual([], charms)
        self.assertEqual(1, len(baskets))
        branch = '~foo/charms/' + BASKET_SERIES + '/bar/trunk'
        charms, baskets = all_branch_data([(branch, revision_id, [])])
        self.assertEqual([], charms)
        self.assertEqual([], baskets)

    def test_promulgated_not_set_for_bundles_on_series(self):
        # The data from Launchpad does not have the series for bundles, so it
        # is not used to determine whether a bundle is promulgated.
        branch = '~foo/charms/' + BASKET_SERIES + '/bar/bundle'
        charms, baskets = all_branch_data([(branch, 'asdf', [BASKET_SERIES])])
        self.assertFalse(baskets[0]['promulgated'])

    def test_promulgated_set_for_bundles_for_charmers(self):
        # The branch is owned by ~charmers so it is marked as promulgated.
        branch = '~charmers/charms/' + BASKET_SERIES + '/bar/bundle'
        charms, baskets = all_branch_data([(branch, 'asdf', [])])
        self.assertTrue(baskets[0]['promulgated'])

    def test_db_charms(self):
        charm_1 = factory.makeCharm(
            self.db, owner='o2', series='s1', name='c1')[1]
        charm_2 = factory.makeCharm(
            self.db, owner='o1', series='s2', name='c1')[1]
        charm_3 = factory.makeCharm(
            self.db, owner='o1', series='s1', name='c2')[1]
        charm_4 = factory.makeCharm(
            self.db, owner='o1', series='s1', name='c1')[1]
        result = sorted([c['branch_spec'] for c in db_charms(self.db)])
        expected = [
            c['branch_spec'] for c in (charm_4, charm_3, charm_2, charm_1)]
        self.assertEqual(expected, result)

    def makeLPData(self, owner, series, name, promulgated=False):
        if promulgated:
            linked_series = [series]
        else:
            linked_series = []
        return ['~%s/charms/%s/%s/trunk' % (owner, series, name),
                '%s@example.com-%s-%s' % (owner, series, name), linked_series]

    def makeCharmAndLPData(self, owner, series, name, promulgated=False):
        charm = factory.makeCharm(
            self.db, owner=owner, series=series, name=name,
            promulgated=promulgated)[1]
        return charm, self.makeLPData(owner, series, name, promulgated)

    def test_all_charms(self):
        lp_data = []
        charm_1, lp_1 = self.makeCharmAndLPData(
            'owner', 'series', 'c1', promulgated=True)
        lp_data.append(lp_1)
        charm_2, lp_2 = self.makeCharmAndLPData(
            'owner', 'series', 'c2', promulgated=True)
        lp_3 = self.makeLPData('owner', 'series', 'c3')
        lp_data.append(lp_3)
        result = list(all_charms(self.db, all_branch_data(lp_data)[0]))

        # Note that data for the promulgated charm_2 does not appear
        # in lp_data; db_charms() changed its promulgated status to
        # false.
        expected = [
            {
                'branch_spec': '~owner/charms/series/c1/trunk',
                'branch_deleted': False,
                'bname': 'trunk',
                'commit': 'owner@example.com-series-c1',
                'distro_series': ['series'],
                'name': 'c1',
                'owner': 'owner',
                'promulgated': True,
                'series': 'series',
            },
            {
                'branch_spec': '~owner/charms/series/c2/trunk',
                'branch_deleted': True,
                'bname': 'trunk',
                'distro_series': [],
                'name': 'c2',
                'owner': 'owner',
                'series': 'series',
                'promulgated': False,
                '_id': '~owner/series/c2/1',
            },
            {
                'branch_spec': '~owner/charms/series/c3/trunk',
                'branch_deleted': False,
                'bname': 'trunk',
                'commit': 'owner@example.com-series-c3',
                'distro_series': [],
                'name': 'c3',
                'owner': 'owner',
                'promulgated': False,
                'series': 'series',
            },
        ]
        self.assertEqual(expected, result)

    def test_dequeue(self):
        queue = get_queue(self.db, 'test_queue')
        self.addCleanup(queue.clear)
        charm_data = factory.get_charm_json()
        queue.put(charm_data)
        self.assertEqual(1, queue.size())
        with patch('charmworld.jobs.lp.configure_logging'):
            lp_getdb = 'charmworld.jobs.lp.getdb'
            with patch(lp_getdb, return_value=self.db) as getdb:
                lp_get_queue = 'charmworld.jobs.lp.get_queue'
                with patch(lp_get_queue, return_value=queue) as gq:
                    dequeue()
        getdb.assert_called_with(ANY, MONGO_DATABASE)
        gq.assert_called_with(self.db, CHARM_QUEUE)
        self.assertEqual(0, queue.size())

    @patch('requests.get', failing_requests_get)
    def test_nothing_queued_when_lp_unreachable(self):
        # When all_branch_data() cannot return the complete set of
        # charm branches, no ingest jobs are queued. Otherwise, the
        # promulgated status may be incorrectly reset for promulgated
        # charms that are not returned by all_branch_data().
        out_queue = get_queue(self.db, 'test_queue')
        factory.makeCharm(self.db)
        self.assertRaises(
            ConnectionError, queue_from_branches, self.db, FakeCharmStore(),
            out_queue, out_queue)
        self.assertEqual(0, out_queue.size())


class TestQueueFromBranches(MongoTestBase):

    CHARM_PAYLOAD = {
        'address': 'cs:~baz/qux/quxx',
        'branch_dir': 'foo',
        'branch_spec': 'bar',
        'name': 'quxx',
        'owner': 'baz',
        'commit': None,
        'promulgated': False,
        'series': 'qux',
        'store_data': {
            'errors': ['entry not found'],
            'revision': 0,
            'store_checked': 'lambada',
        },
        'store_url': 'cs:~baz/qux/quxx-0',
    }

    BASKET_PAYLOAD = {
        'foo': 'bar',
        'branch_spec': '~baz/bar/1/bundle',
        'branch_deleted': False,
    }

    def setUp(self):
        super(TestQueueFromBranches, self).setUp()
        self.charm_queue = get_queue(self.db, 'test_charm_queue')
        self.basket_queue = get_queue(self.db, 'test_basket_queue')

    def queue_from_branches(self, import_filter=unspecified, check_time='foo'):
        queue_from_branches(self.db, FakeCharmStore(), self.charm_queue,
                            self.basket_queue,
                            import_filter=import_filter,
                            check_time=check_time)

    @patch('charmworld.jobs.lp.all_branch_data', all_branch_data_mock)
    def test_queue_from_branches(self):
        self.addCleanup(self.basket_queue.clear)
        self.queue_from_branches(check_time='lambada')
        item = self.charm_queue.next()
        newest_revision = item.payload.pop('newest_revision', None)
        self.assertIsNotNone(newest_revision)
        self.assertEqual(self.CHARM_PAYLOAD, item.payload)
        item = self.basket_queue.next()
        self.assertEqual(self.BASKET_PAYLOAD, item.payload)

    def test_queue_from_branches_import_filter_charms(self):
        self.use_context(patch('charmworld.jobs.lp.all_branch_data',
                               all_branch_data_mock))
        self.queue_from_branches(import_filter='bi', check_time='lambada')
        self.assertIs(None, self.charm_queue.next())
        self.queue_from_branches(import_filter='ba', check_time='lambada')
        item = self.charm_queue.next()
        newest_revision = item.payload.pop('newest_revision', None)
        self.assertIsNotNone(newest_revision)
        self.assertEqual(self.CHARM_PAYLOAD, item.payload)

    def test_queue_from_branches_import_filter_baskets(self):
        self.use_context(patch('charmworld.jobs.lp.all_branch_data',
                               all_branch_data_mock))
        self.queue_from_branches(import_filter='~baz/bar/1/bundr')
        self.assertIs(None, self.basket_queue.next())
        self.queue_from_branches(import_filter='~baz/bar/1/bundl')
        self.assertEqual(self.BASKET_PAYLOAD, self.basket_queue.next().payload)

    def test_filter_from_config(self):
        # If the branch doesn't start with the defined filter (i.e.
        # ~charmers/charms/precise) it is not returned
        factory.makeCharm(self.db, owner='not-charmers')
        self.use_context(patch(
            'charmworld.jobs.lp.all_branch_data', side_effect=lambda: ([{
                'branch_spec':
                u'~not-charmers/charms/precise/someproject/trunk',
            }], [])
        ))
        with patch('charmworld.jobs.lp.CHARM_IMPORT_FILTER',
                   '~charmers/charms/precise'):
            self.queue_from_branches()
        self.assertIs(None, self.charm_queue.next())


class TestLocalBranchData(MongoTestBase):
    """Test the local_branch_data function."""

    def setUp(self):
        super(TestLocalBranchData, self).setUp()
        self.log = logging.getLogger('charm.launchpad')
        self.fs = getfs(self.db)

    def test_finds_charms(self):
        with factory.charm_branch() as c1:
            with factory.charm_branch() as c2:
                with temp_dir() as root_dir:
                    parts = c1[0].split(os.sep)
                    src = os.sep.join(parts[0:3])
                    branch_one = os.sep.join(parts[3:])
                    dst = os.path.join(root_dir, 'charm1')
                    shutil.copytree(src, dst)
                    parts = c2[0].split(os.sep)
                    src = os.sep.join(parts[0:3])
                    branch_two = os.sep.join(parts[3:])
                    dst = os.path.join(root_dir, 'charm2')
                    shutil.copytree(src, dst)

                    # Now run the test.
                    charms, bundles = local_branch_data(root_dir)

        self.assertEqual([], bundles)
        self.assertEqual(2, len(charms))
        expected = 'local:' + os.path.join(root_dir, 'charm1', branch_one)
        branch_specs = sorted(charm['branch_spec'] for charm in charms)
        self.assertEqual(expected, branch_specs[0])
        expected = 'local:' + os.path.join(root_dir, 'charm2', branch_two)
        self.assertEqual(expected, branch_specs[1])
        for charm in charms:
            self.assertTrue(charm['promulgated'])
            self.assertEqual('mark.mims', charm['owner'])


class TestMain(TestCase):

    def run_main(self, extra_args=None):
        with patch('charmworld.jobs.lp.configure_logging') as mock_cl:
            with patch('charmworld.jobs.lp.run') as mock_run:
                fake_argv = None
                if extra_args is not None:
                   # Listify and add.
                    fake_argv = extra_args.split()
                if fake_argv is not None:
                    main(fake_argv)
                else:
                    # We must be able to call main with no arguments.
                    main()
        self.assertTrue(mock_cl.called)
        return mock_run

    def test_main_args(self):
        # Show arguments are passed and used by main.
        args = ('--limit 10 --prefix ~sinzui --clear '
                '--local-repo /home/godzilla')
        mock = self.run_main(args)
        expected = Namespace()
        expected.prefix = '~sinzui'
        expected.limit = 10
        expected.clear = True
        expected.local_repo = '/home/godzilla'
        expected.debug = False
        mock.assert_called_with(expected)

    def test_main_bad_args(self):
        # Show bad arguments passed to main are caught.
        with redirect_stderr() as stderr:
            with self.assertRaises(SystemExit) as e:
                self.run_main('--nolimit 10 --prefix ~sinzui')
                self.assertEqual(2, e.exception.code)
        self.assertIn(
            'error: unrecognized arguments: --nolimit 10', stderr.read())

    def test_main_no_args(self):
        # Show main can be called with no arguments and in that case uses
        # sys.argv.
        test_args = ['ignore', '--prefix', '~benji', '--limit', '101']
        with substitute_argv(test_args):
            mock = self.run_main()
        expected = Namespace()
        expected.prefix = '~benji'
        expected.limit = 101
        expected.clear = False
        expected.local_repo = None
        expected.debug = False
        mock.assert_called_with(expected)

    def mock_get_queue(db, name):
        return name

    @staticmethod
    def get_default_args():
        fake_args = Namespace()
        fake_args.limit = None
        fake_args.clear = False
        fake_args.prefix = unspecified
        fake_args.debug = False
        fake_args.local_repo = None
        return fake_args

    @patch('charmworld.jobs.lp.getconnection')
    @patch('charmworld.jobs.lp.getdb')
    @patch('charmworld.jobs.lp.get_queue', mock_get_queue)
    @patch('charmworld.jobs.lp.lock')
    @patch('charmworld.jobs.lp.queue_from_branches')
    def test_clear_called(self, *args):
        # Show that clear_queues is called with the two (fake) queues when the
        # 'clear' argument is True.
        fake_args = self.get_default_args()
        fake_args.limit = 100
        fake_args.clear = True
        fake_args.prefix = '~kiko'
        with patch('charmworld.jobs.lp.clear_queues') as mock_clear_queues:
            run(fake_args)
        mock_clear_queues.assert_called_with(('charm-queue', 'basket-queue'))

    @patch('charmworld.jobs.lp.getconnection')
    @patch('charmworld.jobs.lp.getdb')
    @patch('charmworld.jobs.lp.get_queue', mock_get_queue)
    @patch('charmworld.jobs.lp.lock')
    @patch('charmworld.jobs.lp.queue_from_branches')
    def test_clear_not_called(self, *args):
        # Show that clear_queues is not called when the 'clear' argument is
        # False.
        fake_args = self.get_default_args()
        fake_args.limit = 100
        fake_args.prefix = '~kiko'
        with patch('charmworld.jobs.lp.clear_queues') as mock_clear_queues:
            run(fake_args)
        self.assertFalse(mock_clear_queues.called)

    @patch('charmworld.jobs.lp.getconnection')
    @patch('charmworld.jobs.lp.getdb')
    @patch('charmworld.jobs.lp.get_store_charm_info')
    @patch('charmworld.jobs.lp.get_queue', mock_get_queue)
    @patch('charmworld.jobs.lp.lock')
    def test_local_branch_data_called(self, *args):
        # Show that local_branch_data is called when the --local-repo option
        # is set.
        fake_args = self.get_default_args()
        fake_args.local_repo = '/home/boozoo'
        method = 'charmworld.jobs.lp.local_branch_data'
        with patch(method, side_effect=local_branch_data_mock) as mocker:
            run(fake_args)
        mocker.assert_called_with(('/home/boozoo'))