~jcsackett/charmworld/bac-tag-constraints

# Copyright 2012, 2013 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

__metaclass__ = type

from datetime import (
    date,
    datetime,
)
import hashlib
import json

from pyramid.httpexceptions import HTTPNotFound
from webob.etag import ETagMatcher
from webob.multidict import MultiDict

from charmworld.charmstore import (
    get_address,
    make_store_url,
)
from charmworld.models import (
    Charm,
    CharmSource,
    QADataSource,
)
from charmworld.views.api import (
    API2,
    json_response,
)
from charmworld.search import (
    exact_fields,
    free_text_fields,
)
from charmworld.testing import factory
from charmworld.testing import (
    ViewTestBase,
    WebTestBase,
)
from charmworld.utils import quote_key


class TestObsoletedApis(WebTestBase):
    """Verify that our obsolete apis are erroring correctly."""
    obsolete_message = 'no longer supported'

    def test_api0_obsolete(self):
        """API0 is obsolete."""
        resp = self.app.get('/api/0/charm/precise/apache2-1', status=410)
        self.assertIn(self.obsolete_message, resp.body)

    def test_api1_obsolete(self):
        """API1 is obsolete."""
        resp = self.app.get('/api/1/charm/precise/apache2-1', status=410)
        self.assertIn(self.obsolete_message, resp.body)


class TestJSONResponse(ViewTestBase):

    def test_json_response(self):
        """json_response produces expected response."""
        error = json_response(200, {'type': 'not_an_error'})
        self.assertEqual(200, error.status_code)
        self.assertEqual({'type': 'not_an_error'}, error.json_body)
        self.assertEqual('application/json', error.content_type)
        self.assertIn(("Access-Control-Allow-Origin", "*"), error.headerlist)
        self.assertIn(
            ("Access-Control-Allow-Headers", "X-Requested-With"),
            error.headerlist)

    def test_json_response_none(self):
        response = json_response(201, None)
        self.assertEqual(201, response.status_code)
        self.assertEqual(0, response.content_length)
        self.assertEqual('application/json', response.content_type)
        self.assertIn(("Access-Control-Allow-Origin", "*"),
                      response.headerlist)
        self.assertIn(
            ("Access-Control-Allow-Headers", "X-Requested-With"),
            response.headerlist)


class APITestBase(ViewTestBase):

    def make_request(self, endpoint, remainder, **kwargs):
        """Make a request for a given endpoint, remainder and params."""
        request = self.getRequest()
        request.matchdict['endpoint'] = endpoint
        if remainder is not None:
            request.matchdict['remainder'] = remainder.lstrip('/').split('/')
        request.GET = MultiDict()
        for key, values in kwargs.iteritems():
            key = {'type_': 'type'}.get(key, key)
            if isinstance(values, basestring):
                values = [values]
            for value in values:
                request.GET.add(key, value)
        return request

    def get_response(self, endpoint, remainder='', **kwargs):
        """Get a Response for a given endpoint, remainder and params."""
        request = self.make_request(endpoint, remainder, **kwargs)
        return self.api_class(request)()

    @staticmethod
    def weightless(charm):
        """Remove weight from charm because we cannot predict its value."""
        charm = dict(charm)
        del charm['weight']
        return charm

    @classmethod
    def weightless_relation(cls, relation):
        new_relation = {}
        for interface, charms in relation.items():
            charms = [cls.weightless(charm) for charm in charms]
            new_relation[interface] = charms
        return new_relation

    @classmethod
    def related_weightless(cls, charm):
        return cls.weightless(cls.api_class._format_related(charm, 0))


class API2Mixin(APITestBase):

    api_class = API2


class TestAPICharms:

    def setUp(self):
        super(TestAPICharms, self).setUp()
        self.use_index_client()

    def makeCharm(self, *args, **kwargs):
        charm_id, charm = factory.makeCharm(self.db, *args, **kwargs)
        self.index_client.index_charm(charm)
        return charm_id, charm

    def test_remote_friendly_headers(self):
        """The Juju-Gui needs to be able to query the api via diff origin."""
        self.index_client.wait_for_startup()
        resp = self.get_response('charms')
        self.assertIn(("Access-Control-Allow-Origin", "*"), resp.headerlist)
        self.assertIn(
            ("Access-Control-Allow-Headers", "X-Requested-With"),
            resp.headerlist)

    def get_charms(self, *args, **kwargs):
        return self.get_response('charms', *args, **kwargs).json_body['result']

    def _test_filter(self, filter_str, make_charm=None):
        if make_charm is None:
            def make_charm(value, warty=False):
                kwargs = {filter_str: value}
                if filter_str != 'series' and warty:
                    kwargs['series'] = 'warty'
                return self.makeCharm(**kwargs)
        one_id, one = make_charm('common')
        self.index_client.index_charm(one)
        two_id, two = make_charm('common', warty=True)
        self.index_client.index_charm(two)
        three_id, three = make_charm('different')
        self.index_client.index_charm(three)
        four_id, four = make_charm('different-again')
        self.index_client.index_charm(four)
        result = self.get_charms(**{filter_str: 'common'})
        self.assertEqual(2, len(result))
        res = lambda x: self.api_class._charm_result(Charm(x))
        self.assertItemsEqual([res(one), res(two)], result)
        result = self.get_charms(**{filter_str: 'different'})
        self.assertItemsEqual([res(three)], result)
        self.assertEqual(1, len(result))
        result = self.get_charms(**{filter_str: ['different', 'common']})
        self.assertEqual(3, len(result))

    def test_charms_name_filter(self):
        """Charms can be filtered by name."""
        self._test_filter('name')

    def test_charms_series_filter(self):
        """Charms can be filtered by series."""
        self._test_filter('series')

    def test_charms_owner_filter(self):
        """Charms can be filtered by owner."""
        self._test_filter('owner')

    def test_charms_categories_filter(self):
        """Charms can be filtered by owner."""
        def make_charm(categories, warty=False):
            return self.makeCharm(categories=[categories])
        self._test_filter('categories', make_charm)

    def test_charms_provides_filter(self):
        """Charms can be filtered by provided interfaces."""
        def make_charm(value, warty=False):
            provides = {'foo': {'interface': value}}
            return self.makeCharm(provides=provides)
        self._test_filter('provides', make_charm)

    def test_charms_requires_filter(self):
        """Charms can be filtered by required interfaces."""
        def make_charm(value, warty=False):
            requires = {'foo': {'interface': value}}
            return self.makeCharm(requires=requires)
        self._test_filter('requires', make_charm)

    def test_charms_type_filter(self):
        """Filtering by type works as well as possible.

        Charms are considered 'approved' if owned by charmers (and
        promulgated), 'community' otherwise.  'environment' causes no charms
        to be returned.
        """
        charmers = self.api_class._charm_result(
            Charm(self.makeCharm(promulgated=True)[1]))
        steve = self.api_class._charm_result(
            Charm(self.makeCharm(owner='steve')[1]))
        result = self.get_charms(type_='community')
        self.assertEqual([steve], [charm for charm in result])
        result = self.get_charms(type_='approved')
        self.assertEqual([charmers], [charm for charm in result])
        result = self.get_charms(type_='environment')
        self.assertEqual([], [charm['owner'] for charm in result])
        response = self.get_response('charms', type_='foo')
        self.assertEqual(406, response.status_code)
        self.assertEqual({'type': 'unsupported_value', 'parameter': 'type',
                          'value': 'foo'}, response.json_body)
        self.assertEqual([], [charm['owner'] for charm in result])
        result = self.get_charms(type_=['approved', 'community'])
        self.assertEqual([charmers, steve],
                         [charm for charm in result])

    def test_charms_type_filter_considers_promulgation(self):
        """Promulgation controls type."""
        reviewed = self.api_class._charm_result(
            Charm(self.makeCharm(promulgated=True)[1]))
        unreviewed = self.api_class._charm_result(
            Charm(self.makeCharm(promulgated=False)[1]))
        result = self.get_charms(type_='community')
        self.assertEqual([unreviewed], [charm for charm in result])
        result = self.get_charms(type_='approved')
        self.assertEqual([reviewed], [charm for charm in result])

    def test_charms_accepts_provider(self):
        """Provider is accepted (but ignored)."""
        response = self.get_response('charms', provider='asdf')
        self.assertEqual(200, response.status_code)

    def test_charms_accepts_scope(self):
        """Scope is accepted (but ignored)."""
        response = self.get_response('charms', scope='asdf')
        self.assertEqual(200, response.status_code)

    def test_charms_text_search(self):
        abc_charm = self.makeCharm(summary='abc def')[1]
        self.makeCharm(summary='ghi jkl')[1]
        response = self.get_response('charms', text='def')
        self.assertEqual(200, response.status_code)
        charms = response.json_body['result']
        self.assertEqual(1, len(charms))
        self.assertEqual(
            self.api_class._charm_result(Charm(abc_charm)), charms[0])

    def test_charms_text_search_respects_filters(self):
        official = self.makeCharm(summary='abc def', promulgated=True)[1]
        self.makeCharm()
        self.makeCharm(summary='abc def', owner='steve')
        self.makeCharm(owner='steve')
        result = self.get_charms(type_=['approved'], text='def')
        self.assertEqual(
            [self.api_class._charm_result(Charm(official))],
            [charm for charm in result])

    def test_charms_limit(self):
        """Limit provides a maximum for charm numbers."""
        response = self.get_response('charms', limit='6')
        self.assertEqual(0, len(response.json_body['result']))
        self.makeCharm()
        self.makeCharm(owner='jrandom')
        response = self.get_response('charms', limit='6')
        self.assertEqual(2, len(response.json_body['result']))
        response = self.get_response('charms', limit='1')
        self.assertEqual(1, len(response.json_body['result']))
        response = self.get_response('charms', limit='0')
        self.assertEqual(0, len(response.json_body['result']))
        response = self.get_response('charms', limit='-1')
        self.assertEqual(406, response.status_code)
        self.assertEqual({
                         'type': 'negative_value',
                         'parameter': 'limit',
                         }, response.json_body)

    def test_charms_interesting(self):
        """A json dump of interesting charms is available."""
        for num in range(15):
            dt = datetime.utcfromtimestamp(num * 7200 + 7200)
            last_new_charm = self.makeCharm(date_created=dt,
                                            name='new%d' % num)[1]
        for num in range(15):
            dt = datetime.utcfromtimestamp(0)
            last_popular_charm = self.makeCharm(
                downloads=num, name='pop%d' % num,
                date_created=dt)[1]
        for num in range(15):
            dt = datetime.utcfromtimestamp(0)
            last_featured_charm = self.makeCharm(
                name='featured%d' % num, date_created=dt, is_featured=True)[1]
        res = self.get_response('charms', remainder='/interesting')
        data = res.json_body
        self.assertIn('result', data)
        self.assertIn('new', data['result'])
        self.assertIn('featured', data['result'])
        self.assertEqual(['pop%d' % num for num in range(14, 4, -1)],
                         [r['charm']['name']
                          for r in data['result']['popular']])
        self.assertEqual(
            self.api_class._charm_result(Charm(last_popular_charm)),
            data['result']['popular'][0])
        self.assertEqual(['new%d' % num for num in range(14, 4, -1)],
                         [r['charm']['name'] for r in data['result']['new']])
        self.assertEqual(
            self.api_class._charm_result(Charm(last_new_charm)),
            data['result']['new'][0])
        self.assertItemsEqual(
            ['featured%d' % num for num in range(14, -1, -1)],
            [r['charm']['name'] for r in data['result']['featured']])
        featured14, = [charm for charm in data['result']['featured'] if
                       charm['charm']['name'] == 'featured14']
        self.assertEqual(
            self.api_class._charm_result(Charm(last_featured_charm)),
            featured14)

    def test_charms_accepts_categories(self):
        """Categories are accepted (but ignored)."""
        response = self.get_response('charms', categories='asdf')
        self.assertEqual(200, response.status_code)

    provides2 = {'cookie-factory': {
        'interface': 'imap',
    }}

    requires2 = {'postal-service': {
        'interface': 'envelope',
    }}

    options2 = {
        'foobar': {
            'type': 'str',
            'default': 'baz',
            'description': 'quxx',
        },
    }

    def test_charms_denies_category(self):
        """Category is not accepted."""
        response = self.get_response('charms', category='asdf')
        self.assertEqual(406, response.status_code)

    def test_charms(self):
        """Charms endpoint produces expected result."""
        one_changes = [factory.makeChange(created=datetime(1999, 12, 31),
                                          revno=37, message='Hello')]
        one_id, one = self.makeCharm(name='name1', changes=one_changes,
                                     date_created=datetime(1997, 10, 29,
                                                           12, 10, 9),
                                     promulgated=True)
        self.index_client.index_charm(one)
        maintainer2 = 'jrandom@example.com (J. Random Hacker)'
        files2 = {
            'readme': {
                'filename': 'README.md',
                'subdir': '',
            },
            'config-changed': {
                'filename': 'config-changed',
                'subdir': 'hooks',
            },
        }
        two_changes = [factory.makeChange(created=datetime(1998, 11, 30),
                                          revno=32, message='Hi',
                                          authors=['Baz <baz@example.org>'])]
        two_id, two = self.makeCharm(owner='jrandom', revision=3,
                                     series='warty', summary='Charm two',
                                     name='name2', revno=13,
                                     maintainer=maintainer2,
                                     commit_message='message2',
                                     provides=self.provides2,
                                     requires=self.requires2,
                                     options=self.options2, files=files2,
                                     subordinate=True, changes=two_changes,
                                     date_created=datetime(1996, 9, 28,
                                                           11, 9, 8),
                                     categories=['cache-proxy'])
        response = self.get_response('charms')
        self.assertEqual(200, response.status_code)
        self.assertEqual('application/json', response.content_type)
        result = response.json_body['result']
        self.assertEqual(2, len(result))
        one_url = make_store_url(1, get_address(one, short=True))
        self.assertEqual(self.expected_one(one), result[0])
        two_url = make_store_url(1, get_address(two, short=False))
        self.assertEqual(self.expected_two(two), result[1])
        self.assertNotEqual(one_id, two_id)
        self.assertNotEqual(one_url, two_url)

    def test_charms_path_notfound(self):
        """Charms endpoint raises HTTPNotFound for unsupported paths."""
        with self.assertRaises(HTTPNotFound):
            self.get_response('charms', 'precise/bong-2')

    def test_autocomplete_matches_on_names(self):
        matching_charm = Charm(self.makeCharm(name='foo')[1])
        matching_id = self.api_class._get_api_id(matching_charm)
        differing_charm = self.makeCharm(name='bar')[1]
        for field in free_text_fields:
            if '.' in field or field == 'name':
                continue
            differing_charm[field] = 'foo'
        for field in exact_fields:
            differing_charm[field] = 'foo'
        self.index_client.index_charm(differing_charm)
        response = self.get_response(
            'charms', autocomplete='true', text='foo')
        ids = [result['charm']['id'] for result in
               response.json_body['result']]
        self.assertEqual([matching_id], ids)

    def expected_one(self, one):
        one_url = make_store_url(1, get_address(one, short=True))
        return {
            'charm': {
                'owner': 'charmers',
                'maintainer': {
                    'name': 'John Doe',
                    'email': 'jdoe@example.com',
                },
                'id': one_url[3:],
                'categories': [],
                'date_created': '1997-10-29T12:10:09',
                'name': 'name1',
                'url': one_url,
                'description': one['description'],
                'revision': 1,
                'summary': "Remote terminal classroom over https",
                'distro_series': 'precise',
                'is_approved': True,
                'rating_numerator': 0,
                'rating_denominator': 0,
                'code_source': {
                    'type': 'bzr',
                    'location':
                        'lp:~charmers/charms/precise/name1/trunk',
                    'revision': '12',
                    'last_log': 'maintainer',
                    'bugs_link':
                        'https://bugs.launchpad.net/charms/'
                        '+source/name1',
                    'revisions': [{
                        'revno': 37,
                        'message': 'Hello',
                        'date': '1999-12-31T00:00:00Z',
                        'authors': [
                            {
                                'name': 'Foo',
                                'email': 'foo@example.com'
                            },
                            {
                                'name': 'Bar',
                                'email': 'bar@example.com'
                            },
                        ],
                    }],
                },
                'relations': {
                    'provides': {
                        'website': {
                            'interface': 'https',
                        },
                    },
                    'requires': {
                        'database': {
                            'interface': 'mongodb',
                        },
                    },
                },
                'options': {
                    'script-interval': {
                        'type': 'int',
                        'default': 5,
                        'description':
                        'The interval between script runs.'
                    },
                },
                'files': ['README', 'hooks/install'],
                'tested_providers': {},
                'downloads': 0,
                'downloads_in_past_30_days': 0,
                'is_subordinate': False,
            },
            'metadata': {
            }
        }

    def expected_two(self, two):
        two_url = make_store_url(1, get_address(two, short=False))
        return{
            'charm': {
                'owner': 'jrandom',
                'maintainer': {
                    'name': 'J. Random Hacker',
                    'email': 'jrandom@example.com',
                },
                'id': two_url[3:],
                'categories': ['cache-proxy'],
                'date_created': '1996-09-28T11:09:08',
                'name': 'name2',
                'url': two_url,
                'description': two['description'],
                'revision': 3,
                'summary': 'Charm two',
                'distro_series': 'warty',
                'is_approved': False,
                'rating_numerator': 0,
                'rating_denominator': 0,
                'code_source': {
                    'type': 'bzr',
                    'location':
                        'lp:~jrandom/charms/warty/name2/trunk',
                    'revision': '13',
                    'revisions': [{
                        'revno': 32,
                        'message': 'Hi',
                        'date': '1998-11-30T00:00:00Z',
                        'authors': [
                            {
                                'name': 'Baz',
                                'email': 'baz@example.org'
                            },
                        ],
                    }],
                    'last_log': 'message2',
                    'bugs_link':
                    'https://bugs.launchpad.net/charms/+source/name2'
                },
                'relations': {
                    'provides': self.provides2,
                    'requires': self.requires2,

                },
                'options': self.options2,
                'files': ['README.md', 'hooks/config-changed'],
                'tested_providers': {},
                'downloads': 0,
                'downloads_in_past_30_days': 0,
                'is_subordinate': True,
            },
            'metadata': {
            }
        }

    def test_charms_interesting_metadata_excludes_related(self):
        older = datetime.utcfromtimestamp(0)
        newer = datetime.utcfromtimestamp(7200)
        self.makeCharm(
            date_created=newer, name='new',
            provides={'asdf': {'interface': 'new-p'}},
            requires={'asdf': {'interface': 'new-r'}},
        )[1]
        self.makeCharm(
            date_created=older, name='req-new', requires={
                'asdf': {'interface': 'new-p'}
            })[1]
        self.makeCharm(
            date_created=older, name='provides-new', provides={
                'asdf': {'interface': 'new-r'}
            })[1]
        self.makeCharm(
            date_created=older, name='pop',
            provides={'asdf': {'interface': 'pop-p'}},
            requires={'asdf': {'interface': 'pop-r'}},
            downloads_in_past_30_days=10,
        )[1]
        self.makeCharm(
            date_created=older, name='req-pop', requires={
                'asdf': {'interface': 'pop-p'}
            })[1]
        self.makeCharm(
            date_created=older, name='provides-pop', provides={
                'asdf': {'interface': 'pop-r'}
            })[1]
        self.makeCharm(
            date_created=older, name='feat',
            provides={'asdf': {'interface': 'feat-p'}},
            requires={'asdf': {'interface': 'feat-r'}},
            is_featured=True,
        )[1]
        self.makeCharm(
            date_created=older, name='req-feat', requires={
                'asdf': {'interface': 'feat-p'}
            })[1]
        self.makeCharm(
            date_created=older, name='provides-feat', provides={
                'asdf': {'interface': 'feat-r'}
            })[1]
        data = self.get_response('charms', remainder='/interesting').json_body
        self.assertNotIn('related', data['result']['new'][0]['metadata'])
        self.assertNotIn('related', data['result']['popular'][0]['metadata'])
        self.assertNotIn('related', data['result']['featured'][0]['metadata'])

    def test_charm_with_empty_branch(self):
        # Empty branches do not have real data for the attributes
        # first_change and last_change. The view is nevertheless properly
        # rendered.
        self.makeCharm(empty_branch=True)
        result = json.loads(self.get_response('charms').body)['result'][0]
        # But the returned data does contain only dummy revision related
        # data.
        self.assertEqual('0', result['charm']['code_source']['revision'])
        self.assertEqual('', result['charm']['code_source']['last_log'])


class TestAPI2Charms(TestAPICharms, API2Mixin):
    """Test API 2's "charms" endpoint."""


class TestAPIBundles:

    def makeBundle(self, *args, **kwargs):
        bundle, _ = factory.makeBundle(self.db, *args, **kwargs)
        return bundle

    def test_default_bundle(self):
        bundle = self.makeBundle()
        expected = dict(
            name=bundle.name,
            owner=bundle.owner,
            basket=bundle.basket,
            inherits=None,
            title='',
            data=dict(services=dict(),
                      relations=dict(),
                      series='precise',
                      ),
            description='',
            promulgated=False,
            branch_deleted=False,
        )
        self.assertEqual(expected, bundle._representation)

    def test_default_model_object(self):
        bundle = self.makeBundle(name='jojo')
        self.assertEqual('jojo', bundle.name)

    def test_db_insert_and_retrieval(self):
        bundle = self.makeBundle(name='ant')
        bundle_data = self.db.bundles.find_one({'name': 'ant'})
        self.assertEqual(bundle_data['_id'], bundle.id)
        del bundle_data['_id']
        self.assertEqual(bundle_data, bundle._representation)

    def test_invalid_path(self):
        self.makeBundle(name='bat')
        with self.assertRaises(HTTPNotFound):
            self.get_response('bundle', 'foo')

    def test_not_found(self):
        self.makeBundle(name='bat')
        response = self.get_response('bundle', 'foo/4/bar')
        self.assertEqual(404, response.status_code)

    def test_no_path(self):
        with self.assertRaises(HTTPNotFound):
            self.get_response('bundle', remainder=None)

    def test_valid_lookup_unpromulgated(self):
        self.makeBundle(name='bat', owner='bac', basket='byobu/4')
        response = self.get_response('bundle', '~bac/byobu/4/bat')
        self.assertEqual(200, response.status_code)
        response = self.get_response('bundle', 'byobu/4/bat')
        self.assertEqual(404, response.status_code)

    def test_valid_lookup_promulgated(self):
        self.makeBundle(
            name='bat', owner='bac', basket='byobu/4',
            promulgated=True)
        # Can look up via owner-based id and short id.
        response = self.get_response('bundle', '~bac/byobu/4/bat')
        self.assertEqual(200, response.status_code)
        response = self.get_response('bundle', 'byobu/4/bat')
        self.assertEqual(200, response.status_code)

    def test_results_match(self):
        bundle = self.makeBundle(name='bat', owner='bac', basket='byobu/4')
        response = self.get_response('bundle', '~bac/byobu/4/bat')
        self.assertEqual(200, response.status_code)
        json = dict(response.json)
        self.assertEqual(json['bat'], bundle.data)


class TestAPI2Bundles(TestAPIBundles, API2Mixin):
    """Test API 2 bundle endpoint."""


class TestAPI:

    def make_charm(self, *args, **kwargs):
        """Make a charm formatted for api."""
        charm_data = factory.makeCharm(self.db, *args, **kwargs)[1]
        charm = Charm(charm_data)
        return (self.api_class._get_api_id(charm),
                self.api_class._charm_result(charm))

    def test_unknown_endpoint(self):
        """Unknown endpoints produce a 404."""
        with self.assertRaises(HTTPNotFound):
            self.get_response('tharms')

    def test_underscore_endpoint(self):
        """Underscores in endpoints do not match."""
        with self.assertRaises(HTTPNotFound):
            self.get_response('__call__')

    def test_unsupported_parameter(self):
        """Unsupported parameters cause a 406."""
        response = self.get_response('charms', kermit='6')
        self.assertEqual(406, response.status_code)
        self.assertEqual({
                         'type': 'parameter_not_supported',
                         'parameter': 'kermit',
                         }, response.json_body)

    def test_format_charm_with_charm_model(self):
        charmid, charm_data = factory.makeCharm(self.db)
        charm = Charm(charm_data)
        formatted = self.api_class._format_charm(charm)
        self.assertEqual(charm.name, formatted['name'])
        self.assertEqual(charm.store_url, formatted['url'])

    def test_format_charm_no_maintainer(self):
        """A missing 'maintainer' field is handled gracefully."""
        charmid, charm = factory.makeCharm(self.db)
        del charm['maintainer']
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertEqual({'name': '', 'email': ''}, formatted['maintainer'])

    def test_format_charm_includes_revisions(self):
        changes = [factory.makeChange(message='Ahoy!', revno=25,
                   created=1365416730.25)]
        charm = factory.get_charm_json(changes=changes)
        formatted = self.api_class._format_charm(Charm(charm))
        revisions = formatted['code_source']['revisions']
        self.assertItemsEqual(['message', 'revno', 'date', 'authors'],
                              revisions[0].keys())
        self.assertEqual('Ahoy!', revisions[0]['message'])
        self.assertEqual(25, revisions[0]['revno'])
        self.assertEqual('2013-04-08T10:25:30Z', revisions[0]['date'])
        self.assertEqual([
            {'name': 'Foo', 'email': 'foo@example.com'},
            {'name': 'Bar', 'email': 'bar@example.com'},
        ], revisions[0]['authors'])

    def test_format_charm_respects_download_count(self):
        charm = factory.get_charm_json()
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertEqual(0, formatted['downloads'])
        self.assertEqual(0, formatted['downloads_in_past_30_days'])
        charm['downloads'] = 30
        charm['downloads_in_past_30_days'] = 20
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertEqual(30, formatted['downloads'])
        self.assertEqual(20, formatted['downloads_in_past_30_days'])

    def test_format_charm_tested_providers(self):
        charm = factory.get_charm_json()
        self.assertNotIn('tests', charm)
        self.assertEqual(
            {}, self.api_class._format_charm(Charm(charm))['tested_providers'])
        charm['tests'] = {}
        self.assertEqual(
            {}, self.api_class._format_charm(Charm(charm))['tested_providers'])
        charm['tests'] = {'ec2': 'SUCCESS', 'openstack': 'FAILURE'}
        self.assertEqual({
            'ec2': 'SUCCESS',
            'openstack': 'FAILURE',
        }, self.api_class._format_charm(Charm(charm))['tested_providers'])
        charm['tests'] = {'ec2': 'RED'}
        with self.assertRaises(ValueError) as e:
            self.api_class._format_charm(Charm(charm))
        self.assertEqual('Unsupported test status: RED', str(e.exception))

    def test_format_charm_tested_providers_unstable(self):
        """UNSTABLE is an acceptable status."""
        charm = factory.get_charm_json()
        charm['tests'] = {'ec2': 'UNSTABLE'}
        self.assertEqual(
            {'ec2': 'UNSTABLE'},
            self.api_class._format_charm(Charm(charm))['tested_providers'])

    def test_format_charm_tested_providers_aborted(self):
        """ABORTED is an acceptable status."""
        charm = factory.get_charm_json()
        charm['tests'] = {'ec2': 'ABORTED'}
        self.assertEqual(
            {'ec2': 'ABORTED'},
            self.api_class._format_charm(Charm(charm))['tested_providers'])

    def test_format_charm_no_date_created(self):
        charm = factory.get_charm_json()
        del charm['date_created']
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertIs(None, formatted['date_created'])

    def test_format_is_approved_follows_promulgation(self):
        charm = factory.get_charm_json(promulgated=True)
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertTrue(formatted['is_approved'])
        charm = factory.get_charm_json(promulgated=False)
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertFalse(formatted['is_approved'])

    def test_format_invalid_category(self):
        charm = factory.get_charm_json(categories=['applications', 'bong'])
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertEqual(['applications'], formatted['categories'])

    def test_charm(self):
        """Retrieving a charm works."""
        self.use_index_client()
        charmid, charm = self.make_charm()
        response = self.get_response(
            'charm', remainder=charmid)
        self.assertEqual(200, response.status_code)
        self.assertEqual(charm, response.json_body)
        response = self.get_response('charm', remainder='foo')
        self.assertEqual(404, response.status_code)
        self.assertEqual({'type': 'no_such_charm', 'charm_id': 'foo'},
                         response.json_body)

    def test_no_path(self):
        with self.assertRaises(HTTPNotFound):
            self.get_response('charm', remainder=None)

    @staticmethod
    def get_id(charm):
        return charm['charm']['id']

    def test_charm_wrong_revision(self):
        """Retrieving a charm works even if revision is wrong."""
        self.use_index_client()
        charm = self.make_charm()[1]
        stale_charm_id = self.get_id(charm) + '37'
        response = self.get_response('charm', remainder=stale_charm_id)
        self.assertEqual(200, response.status_code)
        self.assertEqual(charm, response.json_body)

    def test_charm_invalid_charmid(self):
        """Retrieving a charm 404s if tilde is missing."""
        charmid, charm = self.make_charm(owner='steve')
        self.assertEqual('~', self.get_id(charm)[0])
        response = self.get_response('charm', remainder=self.get_id(charm)[1:])
        self.assertEqual(404, response.status_code)

    def test_charm_url_form(self):
        """The form of the URL is not permitted to vary."""
        charmid, charm = self.make_charm(promulgated=True)
        self.assertEqual(1, self.get_id(charm).count('/'))
        wrong_id = '~charmers/' + self.get_id(charm)
        response = self.get_response('charm', remainder=wrong_id)
        self.assertEqual(404, response.status_code)

    def test_parse_charm_id(self):
        result = self.api_class._parse_charm_id('~jrandom/foo/bar-23')
        self.assertEqual(('jrandom', 'foo', 'bar', '23'), result)
        result = self.api_class._parse_charm_id('~jdoe/bar/foo-33-33')
        self.assertEqual(('jdoe', 'bar', 'foo-33', '33'), result)
        with self.assertRaises(ValueError):
            self.api_class._parse_charm_id('jdoe/bar/foo-33-33')
        result = self.api_class._parse_charm_id('bar/foo-33-33')
        self.assertEqual((None, 'bar', 'foo-33', '33'), result)

    def test_parse_charm_id_unquotes(self):
        result = self.api_class._parse_charm_id(
            '~jra%3cndom/f%2eoo/ba%2fr%2d23%35')
        self.assertEqual(('jra<ndom', 'f.oo', 'ba/r', '235'), result)

    def test_find_charm_promulgated(self):
        p_id, promulgated = factory.makeCharm(
            self.db, promulgated=True, name='magic-charm', series='precise',
            owner='foo')
        u_id, unpromulgated = factory.makeCharm(
            self.db, promulgated=False, name='magic-charm', series='precise',
            owner='bar')
        self.assertEqual(
            ('precise/magic-charm-1', None, promulgated),
            self.api_class(self.getRequest())._find_charm(
                'precise/magic-charm-1'.split('/')))
        self.assertEqual(
            ('~bar/precise/magic-charm-1', None, unpromulgated),
            self.api_class(self.getRequest())._find_charm(
                '~bar/precise/magic-charm-1'.split('/')))

    def test_extract_charm_id(self):
        charm_id, trailing = self.api_class._extract_charm_id(
            '~steve/foo/bar'.split('/'))
        self.assertEqual('~steve/foo/bar', charm_id)
        self.assertIs(None, trailing)
        charm_id, trailing = self.api_class._extract_charm_id(
            '~steve/foo/bar/file/baz'.split('/'))
        self.assertEqual('~steve/foo/bar', charm_id)
        self.assertEqual('file/baz', trailing)
        charm_id, trailing = self.api_class._extract_charm_id(
            'foo/bar/file/baz'.split('/'))
        self.assertEqual('foo/bar', charm_id)
        self.assertEqual('file/baz', trailing)

    def test_charm_response_includes_etag(self):
        charm_id, charm = self.make_charm(hash_='fake-hash')
        response = self.get_response(
            'charm', remainder='/' + charm_id)
        self.assertIn(('Etag', 'fake-hash'), response.headerlist)

        # If no hash value for a charm exists, an Etag header is not included.
        charm_id, charm = self.make_charm(hash_=None)
        response = self.get_response(
            'charm', remainder='/' + charm_id)
        header_names = [header[0] for header in response.headerlist]
        self.assertNotIn('Etag', header_names)

    def test_charm_304_response(self):
        charm_id, charm = self.make_charm(hash_='fake-hash')
        request = self.make_request('charm', remainder='/' + charm_id)
        request.if_none_match = ETagMatcher(['wrong-value'])
        response = self.api_class(request)()
        self.assertEqual(200, response.status_code)
        request.if_none_match = ETagMatcher(['fake-hash'])
        response = self.api_class(request)()
        self.assertEqual(304, response.status_code)

    def test_file(self):
        charm = factory.makeCharm(self.db)[1]
        charm_file = factory.make_charm_file(self.db, charm, 'README')
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/file/README')
        self.assertEqual(200, response.status_code)
        self.assertEqual(charm_file.read(), response.body)
        self.assertIn(
            ("Access-Control-Allow-Origin", "*"),
            response.headerlist)
        self.assertIn(
            ("Access-Control-Allow-Headers", "X-Requested-With"),
            response.headerlist)

    def test_file_missing(self):
        charm = factory.makeCharm(self.db)[1]
        charm_id = self.api_class._get_api_id(Charm(charm))
        path = '/' + charm_id + '/file/subdir/README'
        response = self.get_response('charm', path)
        self.assertEqual(404, response.status_code)
        self.assertEqual({'type': 'no_such_file', 'path': 'subdir/README'},
                         response.json_body)

    def test_file_includes_content_type(self):
        charm = factory.makeCharm(self.db)[1]
        factory.make_charm_file(self.db, charm, 'readme.txt')
        factory.make_charm_file(self.db, charm, 'readme')
        factory.make_charm_file(self.db, charm, 'readme.svg')
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/file/readme.txt')
        self.assertEqual('text/plain', response.content_type)
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/file/readme')
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/file/readme.svg')
        self.assertEqual('image/svg+xml', response.content_type)

    def test_file_includes_cache_instructions(self):
        # The files can be cached a long time because they rarely change, and
        # when they do, their default URL will also change.
        charm = factory.makeCharm(self.db)[1]
        factory.make_charm_file(self.db, charm, 'readme.txt')
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/file/readme.txt')
        self.assertIn(("Cache-Control", "max-age=86400, public"),
                      response.headerlist)

    def test_file_includes_etag(self):
        charm = factory.makeCharm(self.db)[1]
        charm_file = factory.make_charm_file(self.db, charm, 'readme.txt')
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/file/readme.txt')
        self.assertIn(("Etag", '"%s"' % charm_file.md5),
                      response.headerlist)

    def test_file_304(self):
        charm = factory.makeCharm(self.db)[1]
        charm_file = factory.make_charm_file(self.db, charm, 'readme.txt')
        charm['files'][quote_key(charm_file.filename)] = dict(charm_file)
        self.db.charms.save(charm)
        charm_id = self.api_class._get_api_id(Charm(charm))
        request = self.make_request(
            'charm', remainder='/' + charm_id + '/file/readme.txt')
        request.if_none_match = ETagMatcher(['"asdf"'])
        response = self.api_class(request)()
        self.assertEqual(200, response.status_code)
        request.if_none_match = ETagMatcher([charm_file.md5])
        response = self.api_class(request)()
        self.assertEqual(304, response.status_code)
        self.assertEqual('', response.body)

    def test_charm_with_icon(self):
        charm = factory.makeCharm(self.db, promulgated=True)[1]
        charm_file = factory.make_charm_file(
            self.db, charm, 'icon.svg', '<?xml version="1.0"?><svg></svg>')
        charm['files'][quote_key(charm_file.filename)] = dict(charm_file)
        self.db.charms.save(charm)
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/icon.svg')
        self.assertEqual(200, response.status_code)
        self.assertEqual(charm_file.read(), response.body)

    def test_charm_with_category_icon(self):
        charm = factory.makeCharm(self.db)[1]
        charm['categories'] = ['app-servers']
        self.db.charms.save(charm)
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/icon.svg')
        self.assertEqual(302, response.status_code)

    def test_charm_with_default_icon(self):
        charm = factory.makeCharm(self.db)[1]
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response(
            'charm', remainder='/' + charm_id + '/icon.svg')
        self.assertEqual(302, response.status_code)

    def test_unknown_sub_endpoint(self):
        charm = factory.makeCharm(self.db)[1]
        charm_id = self.api_class._get_api_id(Charm(charm))
        with self.assertRaises(HTTPNotFound):
            self.get_response('charm', '/' + charm_id + '/random')

    def test_format_category(self):
        """Formatting should return the data in the expected format.

        The expected format happens to be the current db format minus the _id,
        so formatting is trivial.
        """
        category = factory.makeQAQuestionGroup()
        category['questions'] = category['questions'][:1]
        # formatting should omit the Mongo ID.
        category['_id'] = object()
        self.assertEqual({
            'name': u'reliable',
            'description': u'Reliable',
            'questions': [{
                'id': hashlib.md5('AWS').hexdigest(),
                'description': u'AWS',
                'points': 1,
                'extended_description': u'',
            }],
        }, self.api_class._format_category(category))

    def test_charm_qa(self):
        """"Ensure expected values are returned."""
        category = factory.makeQAQuestionGroup()
        self.db.qa.insert(category)
        charm = factory.makeCharm(self.db)[1]
        qa_data_source = QADataSource.from_db(self.db)
        qa = {
            'reliable': {
                '4847e034bb0a55fcbc8a3380d6a3ab80': '1',
                'a60b40bce263c47559d32435a89756fe': '0',
                '7ff6cc997b141399bcdf2c338fb4031c': '1',
                '9a736d9ab5ca1e16ddd942e7a62b8820': '0',
                '51dceb2909262166283ac17f2af504e7': '1',
                '9659919b570c6ef9a25df7261375e6bb': '',
            },
        }
        qa_data_source.save_qa_data(Charm(charm), qa)
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response('charm', '/' + charm_id + '/qa')
        self.assertEqual(200, response.status_code)
        self.assertEqual({
            'result': {'questions': [
                self.api_class._format_category(category)
            ]},
            'scores': qa,
        }, response.json_body)

    def test_filter_scores(self):
        """Scores for unknown categories and questions are filtered out."""
        self.db.qa.insert(factory.makeQAQuestionGroup())
        category2 = factory.makeQAQuestionGroup()
        category2['name'] = 'useful'
        self.db.qa.insert(category2)
        charm = factory.makeCharm(self.db)[1]
        qa = {
            'reliable': {
                '4847e034bb0a55fcbc8a3380d6a3ab80': '1',
                'bar': '1',
            },
            'foo': {},
        }
        QADataSource.from_db(self.db).save_qa_data(Charm(charm), qa)
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response('charm', '/' + charm_id + '/qa')
        self.assertEqual({
            'reliable': {'4847e034bb0a55fcbc8a3380d6a3ab80': '1'}},
            response.json_body['scores'])

    def test_charm_qa_no_answers(self):
        """"Handle charms that have no answers."""
        category = factory.makeQAQuestionGroup()
        self.db.qa.insert(category)
        charm = factory.makeCharm(self.db)[1]
        charm_id = self.api_class._get_api_id(Charm(charm))
        response = self.get_response('charm', '/' + charm_id + '/qa')
        self.assertEqual(200, response.status_code)
        self.assertEqual({
            'result': {'questions': [
                self.api_class._format_category(category)
            ]},
            'scores': None,
        }, response.json_body)

    def test_options_request_charms(self):

        class FailIndexClient:

            @staticmethod
            def api_search(*args):
                self.fail('Index client was called.')

        # OPTIONS should return as fast as possible-- it should not search.
        self.index_client = FailIndexClient
        request = self.make_request('charms', '')
        request.method = 'OPTIONS'
        response = self.api_class(request)()
        # Only the headers matter for a CORS preflight response.
        self.assertEqual(0, response.content_length)

    def test_charm_result_includes_charm_and_metadata(self):
        charm = factory.get_charm_json(promulgated=False)
        result = self.api_class._charm_result(Charm(charm))
        self.assertIn('metadata', result)
        self.assertIn('charm', result)

    def test_format_related(self):
        charm = factory.get_charm_json(
            downloads=100,
            downloads_in_past_30_days=57)
        formatted = self.api_class._format_related(charm, 0)
        self.assertItemsEqual([
            'id', 'name', 'categories', 'code_source', 'weight', 'has_icon',
            'downloads', 'downloads_in_past_30_days',
            'commits_in_past_30_days', 'is_approved'
        ], formatted.keys())
        self.assertEqual(
            self.api_class._get_api_id(Charm(charm)), formatted['id'])
        self.assertEqual(charm['name'], formatted['name'])
        self.assertEqual(charm['categories'], formatted['categories'])
        self.assertEqual(charm['downloads'], 100)
        self.assertEqual(charm['downloads_in_past_30_days'], 57)

    def test_format_related_is_approved(self):
        charm = factory.get_charm_json(promulgated=True)
        formatted = self.api_class._format_related(charm, 0)
        self.assertTrue(formatted['is_approved'])
        charm = factory.get_charm_json(promulgated=False)
        formatted = self.api_class._format_related(charm, 0)
        self.assertFalse(formatted['is_approved'])

    def test_format_related_downloads_in_past_30_days(self):
        charm = factory.get_charm_json(downloads_in_past_30_days=42)
        formatted = self.api_class._format_related(charm, 0)
        self.assertEqual(42, formatted['downloads_in_past_30_days'])
        charm = factory.get_charm_json(downloads_in_past_30_days=52)
        formatted = self.api_class._format_related(charm, 0)
        self.assertEqual(52, formatted['downloads_in_past_30_days'])

    def test_format_related_commits_in_past_30_days(self):
        now = datetime(2013, 06, 17)
        charm = factory.get_charm_json(
            changes=[factory.makeChange(created=now.date())
                     for x in range(27)])
        formatted = self.api_class._format_related(charm, 0, _now=now)
        self.assertEqual(27, formatted['commits_in_past_30_days'])
        charm = factory.get_charm_json(
            changes=[factory.makeChange(created=date(2013, 05, 18))
                     for x in range(32)] +
                    [factory.makeChange(created=date(2013, 05, 17))])
        formatted = self.api_class._format_related(
            charm, 0, _now=now)
        self.assertEqual(32, formatted['commits_in_past_30_days'])

    def test_format_related_has_icon(self):
        charm = factory.get_charm_json(files={})
        formatted = self.api_class._format_related(charm, 0)
        self.assertFalse(formatted['has_icon'])
        charm = factory.get_charm_json(files={quote_key('icon.svg'): {}})
        formatted = self.api_class._format_related(charm, 0)
        self.assertTrue(formatted['has_icon'])

    def test_format_related_no_categories(self):
        charm = factory.get_charm_json()
        del charm['categories']
        formatted = self.api_class._format_related(charm, 0)
        self.assertEqual([], formatted['categories'])

    def test_format_related_icon(self):
        charm = factory.get_charm_json()
        charm['icon'] = 'hello_world'
        formatted = self.api_class._format_related(charm, 0)
        self.assertNotIn('icon', formatted)

    def test_format_related_code_source(self):
        """Verify that we get the code source data for related charms."""
        charm = factory.get_charm_json()
        formatted = self.api_class._format_related(charm, 0)
        self.assertIn('code_source', formatted)
        self.assertIn('revision', formatted['code_source'])

    def test_format_charm_omits_icon(self):
        """If a charm provides an icon, it is added to the data returned
        by the API.
        """
        charmid, charm = factory.makeCharm(self.db)
        # The test data does not contain an icon.
        charm['icon'] = 'fake'
        formatted = self.api_class._format_charm(Charm(charm))
        self.assertNotIn('icon', formatted)

    def test_charm_result_excludes_related(self):
        charm = factory.get_charm_json(promulgated=False, provides={
            'cookie-factory': {
            'interface': 'imap',
            }
        }, requires={
            'cookie-factory': {
            'interface': 'ssl',
            }
        })
        result = self.api_class._charm_result(Charm(charm))
        self.assertNotIn('related', result['metadata'])

    def get_charm_response(self, charm, suffix):
        remainder = self.api_class._get_api_id(Charm(charm)) + '/' + suffix
        return self.get_response('charm', remainder=remainder).json

    def test_related(self):
        self.use_index_client()
        charm_source = CharmSource.from_request(self)
        charm = factory.get_charm_json(promulgated=False, provides={
            'cookie-factory': {
            'interface': 'imap',
            }
        }, requires={
            'cookie-factory': {
            'interface': 'ssl',
            }
        })
        charm_source.save(charm)
        imap_user = factory.get_charm_json(requires={
            'foo': {'interface': 'imap'}
        })
        self.index_client.index_charm(imap_user)
        ssl_user = factory.get_charm_json(requires={
            'bar': {'interface': 'ssl'}
        })
        self.index_client.index_charm(ssl_user)
        ssl_provider = factory.get_charm_json(provides={
            'bar': {'interface': 'ssl'}
        })
        self.index_client.index_charm(ssl_provider)
        imap_provider = factory.get_charm_json(provides={
            'bar': {'interface': 'imap'}
        })
        self.index_client.index_charm(imap_provider)
        result_json = self.get_charm_response(charm, 'related')
        self.assertEqual(
            self.weightless_relation(result_json['result']['requires']),
            {'imap': [self.related_weightless(imap_user)]})
        self.assertEqual(
            self.weightless_relation(result_json['result']['provides']),
            {'ssl': [self.related_weightless(ssl_provider)]})

    def test_charm_related_excludes_different_series(self):
        self.use_index_client()
        charm_source = CharmSource.from_request(self)
        charm_one = factory.get_charm_json(promulgated=False, provides={
            'cookie-factory': {
            'interface': 'imap',
            }
        })
        charm_source.save(charm_one)
        charm_two = factory.get_charm_json(promulgated=False, requires={
            'cookie-factory': {
            'interface': 'imap',
            }
        }, series=charm_one['series'])
        charm_source.save(charm_two)
        response = self.get_charm_response(charm_one, 'related')
        charm_two_id = self.api_class._get_api_id(Charm(charm_two))
        self.assertEqual(
            [charm_two_id], [charm['id'] for charm
                             in response['result']['requires']['imap']])
        charm_source.index_client.delete_charm(charm_two)
        charm_two['series'] = factory.random_string(10)
        charm_source.save(charm_two)
        response = self.get_charm_response(charm_one, 'related')
        self.assertEqual({}, response['result']['requires'])

    def test_charm_related_excludes_same_name(self):
        self.use_index_client()
        charm_source = CharmSource.from_request(self)
        charm_one = factory.get_charm_json(provides={
            'cookie-factory': {
            'interface': 'imap',
            }
        }, owner='charmers')
        charm_source.save(charm_one)
        charm_two = factory.get_charm_json(
            requires=charm_one['provides'], name=charm_one['name'],
            series=charm_one['series'], owner='jrandom')
        charm_source.save(charm_two)
        response = self.get_charm_response(charm_one, 'related')
        self.assertEqual({}, response['result']['requires'])


class TestAPI2(TestAPI, API2Mixin):
    """Test API 2."""