~bac/charmworld/tag-constraints

# Copyright 2013-2014 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

import calendar
from datetime import (
    date,
    datetime,
    timedelta,
)
import hashlib
import json
import logging
import os
import shutil
import subprocess
import traceback
from types import StringTypes
import urllib2

from bzrlib.branch import Branch
from bzrlib.revision import NULL_REVISION
from bzrlib.transport import get_transport
from charmtools import proof
from charmtools.bundles import (
    Bundle as CharmtoolBundle,
    BundleLinter,
)
from charmworldlib import bundle as cw_bundle
import requests
import yaml

from charmworld.charmstore import (
    CharmStore,
    get_address,
    make_store_url,
)
from charmworld.lp import (
    get_branch_info,
    parse_date,
)
from charmworld.models import (
    Bundle,
    CharmFileSet,
    CharmSource,
    construct_charm_id,
    getconnection,
    getdb,
    getfs,
    get_basket_info,
    options_to_storage,
    slurp_files,
    store_bundles,
)
from charmworld.search import (
    ElasticSearchClient,
    SearchServiceNotAvailable,
)
from charmworld.utils import (
    get_ini,
    LAST_INGEST_JOB_FINISHED,
    quote_key,
    quote_yaml,
    read_locked,
    set_status_timestamp,
    timestamp,
    unquote_yaml,
)

# XXX Why not "from charmworld.jobs.config import..."? (Benji)
from config import CHARM_DIR
from config import settings

ICON_FILENAME = 'icon.svg'
JENKINS_PROVIDERS = ['ec2', 'openstack', 'local']
JENKINS_ARTIFACT_URL = (
    "https://jenkins.qa.ubuntu.com/job/"
    "%(series)s-%(provider)s-charm-%(charm)s/%(build)d/artifact/%(artifact)s")
JENKINS_QA_URL = (
    "https://jenkins.qa.ubuntu.com/job/"
    "%(series)s-%(provider)s-charm-%(charm)s/lastBuild/api/json")
# The charmtool 'proof' returns three codes:
# 0:   no errors or warnings.
# 100: one or more warnings but no errors.
# 200: one or more errors.
# We are only interested in the last one.
CHARMTOOL_ERROR_CODE = 200


class _BundleLinter(BundleLinter):
    # This extension of BundleLinter is required so we can specify the server
    # and port charmworldib will use for proofing.  For ingesting we must
    # specify 'localhost' since the process is running within the production
    # data center and can't use external addressing.
    def remote_proof(self, bundle, server, port):
        deployer_file = bundle.bundle_file(parse=True)
        bundles = cw_bundle.Bundles(server=server, port=port, secure=False)
        proof_output = bundles.proof(deployer_file)

        # If additional information about services comes through, add it to
        # the linter for access down the road.
        self.services_info = proof_output.get('services_info', None)

        if 'error_messages' in proof_output:
            for message in proof_output['error_messages']:
                self.err(message)


class IngestError(Exception):
    pass


class IngestJob(object):

    name = 'default'

    def __init__(self, debug=False):
        self.log = logging.getLogger("charm.%s" % self.name)
        if debug:
            self.log.setLevel(logging.DEBUG)

    def setup(self):
        pass

    def run(self, charm_data):
        raise NotImplementedError

    def update_ingest_status(self):
        set_status_timestamp(self.db, LAST_INGEST_JOB_FINISHED)


class DBIngestJob(IngestJob):

    def setup(self, db=None):
        if not db:
            connection = getconnection(settings)
            db = getdb(connection, settings.get('mongo.database'))
        self.db = db


def do_bzr_update(charm_data, fs, log, root_dir=None, revisionId=None):
    """Fetch a branch from bzr, and augment charm data."""
    if not root_dir:
        root_dir = CHARM_DIR
    if not os.path.exists(root_dir):
        os.makedirs(root_dir)
    update_charm_files(root_dir, fs, charm_data, log, revisionId=revisionId)


def update_branch(root_dir, branch_data, branch_dir, log, revisionId,
                  retry=False):
    log.debug('Updating branch lp:{} revid {}'.format(
        branch_data['branch_spec'],
        revisionId))
    try:
        subprocess.check_output(
            ['/usr/bin/bzr', 'update', '-q', '-r', 'revid:' + revisionId],
            cwd=branch_dir,
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError:
        # Update failed for some reason; destroy it and start over.
        if retry:
            shutil.rmtree(branch_dir)
            return checkout_branch(branch_data, branch_dir, log, revisionId)
        raise


def branch_is_current(branch_data, branch_dir, log):
    # It exists and check if it's the latest revision already.
    log.debug(
        "Existing checkout from lp:%s", branch_data['branch_spec'])
    transport = get_transport(branch_dir)
    branch = Branch.open_from_transport(transport)
    cur_rev_id = branch.last_revision()
    return cur_rev_id == branch_data['commit']


def checkout_branch(branch_data, branch_dir, log, revisionId):
    # The branch has never been seen before. Original branch.
    branch_spec = branch_data['branch_spec']
    log.info(
        'Checking out lp:%s, revision ID %s', branch_spec,
        revisionId)

    if branch_spec.startswith('local:'):
        branch_spec = branch_spec.replace('local:', '')
    else:
        branch_spec = 'lp:{}'.format(branch_spec)
    command = ['/usr/bin/bzr', 'checkout', '-q', '-r', 'revid:' + revisionId,
               branch_spec, branch_dir]
    subprocess.check_output(command)


def store_charm_files(fs, branch_dir, charm_data, log):
    """Process the bzr branch for files that need to be stored in gridfs.
    """
    log.info('Storing files of branch %s into gridfs', branch_dir)
    filestore = CharmFileSet.save_files(
        fs, charm_data, branch_dir, log)
    log.info('Completed gridfs storage.')
    return filestore


def add_files(fs, branch_dir, charm_data, log):
    charm_data['files'] = dict([
        (quote_key(cfile.filename), dict(cfile)) for cfile in
        store_charm_files(fs, branch_dir, charm_data, log)
    ])
    return charm_data


def construct_branch_dir(root_dir, branch_data):
    return os.path.abspath(
        str(os.path.join(root_dir,
                         branch_data["series"],
                         branch_data["owner"],
                         branch_data["name"],
                         branch_data["bname"])))


def fetch_branch(root_dir, branch_data, log, revisionId=None, retry=True):
    branch_dir = construct_branch_dir(root_dir, branch_data)
    if not os.path.exists(os.path.dirname(branch_dir)):
        os.makedirs(os.path.dirname(branch_dir))

    if not os.path.exists(branch_dir):
        # Branch doesn't exist; check it out.
        checkout_branch(branch_data, branch_dir, log, revisionId)
    else:
        # Branch exists, but needs updating; update it.
        update_branch(
            root_dir, branch_data, branch_dir, log, revisionId,
            retry)
    return branch_dir


def update_charm_files(root_dir, fs, charm_data, log, retry=True,
                       revisionId=None):
    if charm_data['branch_deleted']:
        return
    # Store the branch directory
    branch_dir = fetch_branch(root_dir, charm_data, log, revisionId, retry)
    charm_data['branch_dir'] = branch_dir
    charm_data = add_files(fs, branch_dir, charm_data, log)


def log(stage, level, exc, charm_data, tb=None):
    logger = logging.getLogger("charm.%s" % stage)
    err_msg = "%s error on %s: %s" % (
        stage, charm_data, str(exc))
    if tb is not None:
        err_msg = '%s\n%s' % (err_msg, tb)
    logger.log(level, err_msg)
    charm_error = {'error_stage': stage, 'error': str(exc)}
    if hasattr(exc, 'output'):
        charm_error['output'] = str(exc.output)
    charm_data['error'] = charm_error


def run_job(job, data, needs_setup=True, db=None):
    if needs_setup:
        if db is not None:
            job.setup(db=db)
        else:
            job.setup()
    try:
        job.run(data)
    except IngestError as e:
        log(job.name, logging.INFO, e, data)
        return False
    except SearchServiceNotAvailable:
        raise
    except Exception as e:
        tb = traceback.format_exc()
        log(job.name, logging.ERROR, e, data, tb)
        return False
    return True


def update_hash(charm_data):
    hashable_data = charm_data.copy()
    hashable_data.pop('hash', None)
    h = hashlib.sha256()
    h.update(json.dumps(hashable_data, sort_keys=True))
    charm_data['hash'] = h.hexdigest()


def trigger_tests(charm_branch, revision, log=None):
    settings = get_ini()
    charm_test_url = settings.get('charm_test_url', None)
    token = settings['charm_test_token']
    job = settings['charm_test_job']
    charm_branch = "lp:%s" % charm_branch
    # If tesitng parameters aren't set, don't bother trying to trigger a test.
    if '' not in [charm_test_url, token, job]:
        url = ('%s/job/%s/buildWithParameters?'
               'token=%s&branch=%s&revno=%s') % (
                   charm_test_url, job, token, charm_branch, revision)
        # Having assembled the url, all we need to do to trigger the build is
        # call get on the url.
        if log:
            log.info('Triggering tests with url: %s' % url)
        try:
            requests.get(url)
        except Exception as e:
            if log:
                log.error('Failed to trigger test build: %s' % e)
    else:
        err_msg = ('Charm test configuration not provided, skipping '
                   'triggering tests.')
        if log:
            log.info(err_msg)


def should_trigger_tests(this_rev, newest_rev, charm_in_db, is_promulgated):
    '''Determine if we should run tests for this charm. We only want to run
    tests the first time we see a new revision of a charm--that is, when we're
    looking at the most up-to-date version, and that version does not yet exist
    in the database. Additionally, we only run tests for promulgated charms.
    @param this_rev: The store revision of the currently ingesting version of
        the charm.
    @type this_rev: int
    @param newest_rev: The store revision of the most up-to-date version of
        this charm.
    @type this_rev: int
    @param charm_in_db: Whether this revision of this charm is in the db.
    @type charm_in_db: bool
    @param is_promulgated: Whether this charm is promulgated.
    @type is_promulgated: bool
    '''
    return (
        this_rev == newest_rev and
        not charm_in_db and
        is_promulgated
    )


def should_update_charm(this_rev, newest_rev, charm_in_db):
    '''Determine whether a charm's data should be updated.
    Since the last 10 versions of a charm are ingested per run, we only want to
    update data for the most current version of the charm or if the charm is
    new (i.e. not in the db).
    @param this_rev: The store revision of the currently ingesting version of
        the charm.
    @type this_rev: int
    @param newest_rev: The store revision of the most up-to-date version of
        this charm.
    @type this_rev: int
    @param charm_in_db: Whether this revision of this charm is in the db.
    @type charm_in_db: bool
    '''
    return this_rev == newest_rev or not charm_in_db


def update_charm(charm_data, db, store):
    # Drop existing error data so that charms can lose their error status.
    charm_data.pop('error', None)
    if 'errors' in charm_data.get('store_data', {}):
        return
    log = logging.getLogger('charm.update_charm')
    fs = getfs(db)
    try:
        revision = charm_data.get('commit')
        do_bzr_update(charm_data, fs, log, revisionId=revision)
        update_download_count(store, charm_data)
        update_proof_data(charm_data, log)
        update_jenkins_data(db, charm_data, log)
        update_revision_details(charm_data, newest_revision=revision)
        update_date_created(charm_data, log)
        scan_charm(charm_data, fs, log)
        update_hash(charm_data)
    except Exception as e:
        err_msg = "%s error: %s" % (charm_data, str(e))
        log.exception(err_msg)
        charm_error = {'error': str(e)}
        if hasattr(e, 'output'):
            charm_error['output'] = str(e.output)
        charm_data['error'] = charm_error


class UpdateCharmJob(DBIngestJob):

    name = 'update'

    def run(self, payload):
        charm_id = construct_charm_id(payload)
        self.log.info("Updating charm {}".format(charm_id))
        charm_data = self.db.charms.find_one(charm_id)
        if charm_data is None:
            charm_in_db = False
            charm_data = {}
        else:
            charm_in_db = True
        newest_revision = payload.pop('newest_revision', None)
        charm_data.update(payload)
        charm_data['_id'] = charm_id
        this_rev = charm_data['store_data']['revision']
        if should_update_charm(this_rev, newest_revision, charm_in_db):
            update_charm(charm_data, self.db, CharmStore())
            index_client = ElasticSearchClient.from_settings(settings)
            self.log.info('Saving %s' % charm_data['_id'])
            CharmSource(self.db, index_client).save(charm_data)
        else:
            self.log.info('Skipping %s' % charm_data['_id'])
        if should_trigger_tests(
            this_rev, newest_revision, charm_in_db, charm_data['promulgated']):
            trigger_tests(
                charm_data['branch_spec'], newest_revision, self.log)
        self.update_ingest_status()


class UpdateBundleJob(DBIngestJob):

    name = 'update-bundle'

    def __init__(self, debug=False, working_dir=None):
        if working_dir is None:
            working_dir = CHARM_DIR
        self.working_dir = working_dir
        super(UpdateBundleJob, self).__init__(debug)

    def store_bundles(self, deployer_config, owner, basket_id, first_change,
                      last_change, changes, branch_spec, services_info,
                      promulgated):
        store_bundles(
            self.db.bundles, deployer_config, owner, basket_id, first_change,
            last_change, changes, branch_spec, services_info,
            promulgated=promulgated)

    @staticmethod
    def set_basket_info(data, revno):
        data.update(get_basket_info(revno, **data))

    def decorate_basket(self, basket_data, fs):
        branch_dir = fetch_branch(self.working_dir, basket_data, self.log,
                                  revisionId=basket_data['commit'])
        basket_data['branch_dir'] = branch_dir
        update_revision_details(
            basket_data, newest_revision=basket_data['commit'])
        branch = Branch.open(branch_dir)
        revno = branch.revision_id_to_revno(basket_data['commit'])
        self.set_basket_info(basket_data, revno)
        with read_locked(branch):
            tree = branch.repository.revision_tree(basket_data['commit'])
            basket_data['file_hashes'] = quote_yaml(
                slurp_files(fs, tree, self.log))

    @staticmethod
    def get_deployer_config(fs, basket_data):
        hashes = unquote_yaml(basket_data['file_hashes'])
        yaml_hash = hashes.get('bundles.yaml')
        if yaml_hash is None:
            return None
        deployer_config_bytes = fs.get(yaml_hash)
        return yaml.safe_load(deployer_config_bytes)

    @staticmethod
    def _proof(basket_file):
        bundle = CharmtoolBundle(basket_file)
        linter = _BundleLinter()
        linter.local_proof(bundle)
        linter.remote_proof(
            bundle, server='localhost', port=settings['proof.port'])
        return linter.lint, linter.services_info, linter.exit_code

    @staticmethod
    def proof(basket_file):
        lint, services_info, err = UpdateBundleJob._proof(basket_file)
        return lint, services_info, err >= CHARMTOOL_ERROR_CODE

    def run(self, basket_data):
        self.log.info('Saving %s' % basket_data['branch_spec'])
        fs = Bundle.getfs(self.db)
        self.decorate_basket(basket_data, fs)
        self.db.baskets.save(basket_data)
        deployer_config = self.get_deployer_config(fs, basket_data)
        if deployer_config is None:
            self.log.error(
                'No deployer config (bundles.yaml) file found for {}.'.format(
                    basket_data['branch_spec']))
        else:
            lint, services_info, err = self.proof(basket_data['branch_dir'])
            if err:
                # Reject the entire basket.
                self.log.error('Rejecting {} due to proof errors.'.format(
                    basket_data['branch_spec']))
                for l in lint:
                    self.log.error('\t{}'.format(l))
            else:
                self.store_bundles(
                    deployer_config, basket_data['owner'],
                    basket_data['name_revno'],
                    basket_data['first_change'],
                    basket_data['last_change'],
                    basket_data['changes'],
                    basket_data['branch_spec'],
                    services_info,
                    basket_data['promulgated'])
        self.update_ingest_status()


def _rev_info(r, branch):
    d = {
        'authors': r.get_apparent_authors(),
        'revno': branch.revision_id_to_revno(r.revision_id),
        'committer': r.committer,
        'created': r.timestamp,
        'message': r.message,
    }
    return d


def find_revision_cutoff(now=None, settings=settings):
    days_of_revisions = settings.get('days_of_revisions')
    if days_of_revisions is None:
        days_of_revisions = 30
    if now is None:
        now = datetime.utcnow()
    cutoff = now - timedelta(int(days_of_revisions))
    return calendar.timegm(cutoff.timetuple())


def update_revision_details(data, limit=10, newest_revision=None):
    # If the branch has been deleted, there can be no new revisions to pull in.
    if data['branch_deleted']:
        return
    since = find_revision_cutoff()
    changes, first_change, last_change = get_changes(
        data['branch_dir'], limit, since, newest_revision)
    data['changes'] = changes
    data['first_change'] = first_change
    data['last_change'] = last_change


def get_changes(branch_dir, limit, since, newest_revision=None):
    branch = Branch.open(branch_dir)
    branch.lock_read()
    try:
        revisions = get_revisions(branch, limit, since)
        changes = []
        # If there is no limit on how new a revision we can include, then
        # consider the starting revision to be already found.
        found_start = newest_revision is None
        for revision in revisions:
            if not found_start and revision.revision_id != newest_revision:
                continue
            else:
                found_start = True
            changes.append(_rev_info(revision, branch))
        if len(changes) == 0:
            last_change = None
            first_change = None
        else:
            last_change = changes[0]
            first = branch.repository.get_revision(branch.get_rev_id(1))
            first_change = _rev_info(first, branch)
        return changes, first_change, last_change
    finally:
        branch.unlock()


def get_revisions(branch, limit, since):
    # We only want the last 10 changes, in descending order.
    graph = branch.repository.get_graph()
    cur_rev_id = branch.last_revision()
    ancestry_iter = graph.iter_lefthand_ancestry(cur_rev_id)
    revs = []
    for num, revision_id in enumerate(ancestry_iter):
        if revision_id == NULL_REVISION:
            break
        revision = branch.repository.get_revision(revision_id)
        if num >= limit:
            if revision.timestamp < since:
                break
        revs.append(revision)
    return revs


def update_jenkins_data(db, charm, log):
    if not charm['promulgated']:
        return

    charm.setdefault('tests', {})
    charm.setdefault('test_results', {})

    for p in JENKINS_PROVIDERS:
        try:
            result_id, status = store_provider_results(db, p, charm, log)
            if result_id is None:
                continue
            charm['tests'][p] = status
            charm['test_results'][p] = result_id
        except:
            log.exception("Unknown error while processing %s %s",
                          charm['branch_spec'], p)


def scan_artifacts(provider, charm, result):
    artifacts = []
    for artifact in result['artifacts']:
        a_url = JENKINS_ARTIFACT_URL % (dict(
            series=charm['series'],
            provider=provider,
            charm=charm['name'],
            build=result['number'],
            artifact=artifact['relativePath']))

        # Load up the charm revision as a result property.
        if artifact['displayPath'] == 'charm-revision':
            charm_revision = urllib2.urlopen(a_url).read().strip()
            if not charm_revision:
                continue
            revision_info = charm_revision.split(' ', 1)
            result['revno'] = int(revision_info[0])
            if len(revision_info) > 1:
                result['revision-id'] = revision_info[1].rstrip('\n')
            continue

        # Mark the test result as graph runner enabled.
        if "graph-tests" in artifact['displayPath']:
            result['charmrunner'] = True

    return artifacts


def store_provider_results(db, provider, charm, log):
    charm_result_url = JENKINS_QA_URL % (
        dict(series=charm['series'],
             provider=provider,
             charm=charm['name']))

    log.debug("Loading %s from %s", charm['name'], charm_result_url)

    response = requests.get(charm_result_url)
    if response.status_code is not None and response.status_code != 200:
        log.info(
            "No test result for %s @ %s", charm['branch_spec'], provider)
        return None, None

    result = response.json()

    # If we already have results no pointing in refetching.
    result_id = "%s::%s-%s" % (
        charm['branch_spec'], provider, result['number'])
    db_result = db.jenkins.find_one({'_id': result_id})
    if db_result is not None:
        return result_id, db_result['result']

    # Fetch test artifacts.
    artifacts = scan_artifacts(provider, charm, result)

    # Inject test metadata.
    result['branch_spec'] = charm['branch_spec']
    result['provider'] = provider
    result['artifacts'] = artifacts
    result['_id'] = result_id
    db.jenkins.insert(result)
    return (result_id, result['result'])


def update_proof_data(charm, log):
    if charm['branch_deleted']:
        return
    proof_result = {}
    lint, exit_code = proof.proof(charm['branch_dir'])
    for line in lint:
        if not ':' in line:
            continue
        level, msg = line.split(':', 1)
        if level == "W" and 'name' in msg:
            continue
        proof_result.setdefault(level.lower(), []).append(msg)
    charm['proof'] = proof_result


def scan_charm(charm_data, fs, log):
    # Note: charm_data is modified in-place.  IndexIngestJob requires
    # these modifications.
    files = charm_data.get('files')
    if files is None:
        raise IngestError(
            'Charm contains no files: {}'.format(
                charm_data['branch_spec']))

    # Some files have bad characters in them since they are used as mongo
    # keys. Use their escaped forms instead.
    metadata_file = quote_key('metadata.yaml')
    config_file = quote_key('config.yaml')

    if metadata_file not in files:
        log.info(
            'Charm has no metadata: %s', charm_data['branch_spec'])
        return
    else:
        cfile = CharmFileSet.get_by_id(
            fs, files[metadata_file]['fileid'])
        try:
            metadata = quote_yaml(yaml.safe_load(cfile.read()))
        except Exception, exc:
            raise IngestError(
                'Invalid charm metadata %s: %s' % (
                charm_data['branch_spec'],
                exc)
            )

    if config_file in files:
        cfile = CharmFileSet.get_by_id(
            fs, files[config_file]['fileid'])
        config_raw = cfile.read()

        try:
            config_yaml = yaml.safe_load(config_raw)
            if 'options' in config_yaml:
                config_yaml['options'] = options_to_storage(
                    config_yaml['options'])
            config = quote_yaml(config_yaml)
        except Exception, exc:
            raise IngestError(
                'Invalid charm config yaml. %s: %s' % (
                    charm_data['branch_spec'],
                    exc)
            )

        metadata['config'] = config
        metadata['config_raw'] = config_raw

    if 'revision' in files:
        cfile = CharmFileSet.get_by_id(
            fs, files['revision']['fileid'])
        rev_raw = cfile.read()
        try:
            rev_id = int(rev_raw.strip())
        except ValueError:
            rev_id = 0
            log.info('Invalid revision %s', rev_raw)
        metadata['revision'] = rev_id
    elif not 'revision' in metadata:
        log.info('No revision file for %s', charm_data['branch_spec'])
        metadata['revision'] = 0

    hooks = []
    for filedata in files.values():
        if filedata['subdir'] == 'hooks':
            hooks.append(filedata['filename'])
    hooks.sort()
    metadata['hooks'] = hooks
    # The charm name specified in the metadata may differ from the
    # LP source package name and the name used by the main charm store.
    # Using the name from the metadata can cause problems like described
    # in bug 1202665.
    if 'name' in metadata:
        del metadata['name']

    # Update the historic or incomplete charm_data with the new metadata.
    # Stuff into the db.
    charm_data.update(metadata)
    complete_charm_data = process_charm(charm_data)
    # Modify charm_data itself.
    charm_data.clear()
    charm_data.update(complete_charm_data)


def normalize_interfaces(interfaces):
    # Transform the short defintion of relations into the
    # long form. Quoted from
    # https://juju.ubuntu.com/docs/charm.html#the-metadata-file :
    #
    #     As a shortcut, if these properties are not defined, and
    #     instead a single string value is provided next to the
    #     relation name, the string is taken as the interface
    #     value, as seen in this example:
    #
    #         requires:
    #             db: mysql
    for relation_name, relation_data in interfaces.items():
        if isinstance(relation_data, StringTypes):
            interfaces[relation_name] = {'interface': relation_data}


def process_charm(base_charm):
    # Enrich charm metadata for webapp.
    charm = dict(base_charm)
    # Charm url
    if charm["promulgated"]:
        charm["short_url"] = "/charms/%s/%s" % (
            charm["series"], charm["name"])
    else:
        charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
                                             charm["series"],
                                             charm["name"])

    # Charm label
    if charm["promulgated"]:
        charm["label"] = "%s/%s" % (charm["series"], charm["name"])
    else:
        charm["label"] = "~%s:%s/%s" % (charm["owner"],
                                        charm["series"],
                                        charm["name"])

    # Flatten the interfaces provided
    i_provides = []
    provides = charm.get("provides")
    if provides:
        normalize_interfaces(provides)
        for v in provides.values():
            if not isinstance(v, dict):
                continue
            i = v.get("interface")
            if not i:
                continue
            i_provides.append(i)
    charm["i_provides"] = i_provides

    # Flatten the interfaces required
    i_requires = []
    requires = charm.get("requires")
    if requires:
        normalize_interfaces(requires)
        for v in requires.values():
            i = v.get("interface")
            if not i:
                continue
            i_requires.append(i)
    charm["i_requires"] = i_requires
    return charm


def addresses(charm):
    """Return an iterator of potential store addresses."""
    if charm['promulgated']:
        yield get_address(charm, short=True)
    yield get_address(charm, short=False)


def update_download_count(store, charm):
    count = store.count_downloads_in_days(charm, 30, date.today())
    charm['downloads_in_past_30_days'] = count
    count = store.get_download_counts(charm)
    charm['downloads'] = count[0][0] if count else 0
    count = store.count_downloads_in_days(charm, 7, date.today())
    charm['downloads_in_past_7_days'] = count
    count = store.count_downloads_in_days(charm, 182, date.today())
    charm['downloads_in_past_half_year'] = count


def update_date_created(charm, log):
    """Use Launchpad to update the charm's date_created field.

    Field will only be set if it is missing AND a branch with the correct
    branch_spec exists in Launchpad.
    """
    if 'date_created' in charm:
        log.debug('Skipping %s which already has date_created',
                  charm['branch_spec'])
        return
    log.info('Retrieving date_created for %s', charm['branch_spec'])
    info = get_branch_info(charm['branch_spec'])
    if info is None:
        log.warning('No branch for %s', charm['branch_spec'])
        return
    date_created = parse_date(info['date_created'])
    charm['date_created'] = timestamp(date_created.replace(microsecond=0))


def update_from_store(charm, address, data, check_time, log):
    if 'errors' in data or 'warnings' in data:
        log.warning('store error on %s %s' % (address, data))
    data['store_checked'] = check_time
    charm['address'] = address
    charm['commit'] = data.get('digest')
    charm['store_data'] = data
    charm['store_url'] = make_store_url(data['revision'], address)