~jcsackett/charmworld/bac-tag-constraints

# Copyright 2013 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

from base64 import b64encode
import calendar
import contextlib
from datetime import (
    datetime,
    timedelta,
)
import json
import logging
import os
import shutil
import subprocess
import sys
import urllib2

from bzrlib.branch import Branch
from bzrlib.revision import NULL_REVISION
from bzrlib.transport import get_transport
import yaml

from charmworld.models import getconnection
from charmworld.models import getdb
from charmworld.models import getfs
from charmworld.models import CharmFileSet
from charmworld.search import ElasticSearchClient
from charmworld.utils import quote_key
from charmworld.utils import quote_yaml

from config import CHARM_DIR
from config import CHARM_PROOF_PATH
from config import settings
from config import STORE_URL

ICON_FILENAME = 'icon.svg'
JENKINS_PROVIDERS = ['ec2', 'openstack', 'local']
JENKINS_ARTIFACT_URL = (
    "https://jenkins.qa.ubuntu.com/job/"
    "%(series)s-%(provider)s-charm-%(charm)s/%(build)d/artifact/%(artifact)s")
JENKINS_QA_URL = (
    "https://jenkins.qa.ubuntu.com/job/"
    "%(series)s-%(provider)s-charm-%(charm)s/lastBuild/api/json")


class IngestJob(object):

    name = 'default'

    def __init__(self):
        self.log = logging.getLogger("charm.%s" % self.name)

    def setup(self):
        pass

    def run(self, charm_data):
        raise NotImplemented


class DBIngestJob(IngestJob):

    def setup(self, db=None):
        if not db:
            connection = getconnection(settings)
            db = getdb(connection, settings.get('mongo.database'))
        self.db = db


class FSIngestJob(DBIngestJob):

    def setup(self, db=None, fs=None):
        super(FSIngestJob, self).setup(db)
        if not fs:
            fs = getfs(self.db)
        self.fs = fs


class BzrIngestJob(FSIngestJob):

    name = 'bzr'

    def setup(self, root_dir=None, db=None, fs=None):
        super(BzrIngestJob, self).setup(db, fs)
        if not root_dir:
            root_dir = CHARM_DIR
        self.root_dir = root_dir
        if not os.path.exists(self.root_dir):
            os.makedirs(self.root_dir)

    def store_branch_files(self, charm_data):
        """Process the bzr branch for files that need to be stored in gridfs.
        """
        self.log.info('Storing files of branch into gridfs')
        filestore = CharmFileSet.save_files(
            self.fs, charm_data, charm_data['branch_dir'])
        self.log.info('Completed gridfs storage.')
        return filestore

    def add_files(self, charm_data):
        charm_data['files'] = dict([
            (quote_key(cfile.filename), dict(cfile)) for cfile in
            self.store_branch_files(charm_data)
        ])
        if ICON_FILENAME in os.listdir(charm_data['branch_dir']):
            file_path = os.path.join(charm_data['branch_dir'], ICON_FILENAME)
            with open(file_path, 'rb') as icon_file:
                charm_data['icon'] = b64encode(icon_file.read())
        else:
            # It might happen that an existing icon is removed from the
            # charm. We should assume that this is a deliberate decision
            # by the charm's maintainer, be it for an aesthetic, legal
            # or any other reason.
            if 'icon' in charm_data:
                del charm_data['icon']
        return charm_data

    def checkout_charm(self, charm_data, branch_dir):
        # The branch has never been seen before. Original branch.
        self.log.info("Branching charm lp:%s", charm_data["branch_spec"])
        subprocess.check_output(
            ["/usr/bin/bzr", "co", "-q",
             "lp:%s" % charm_data["branch_spec"], branch_dir])
        charm_data = self.add_files(charm_data)

    def charm_is_current(self, charm_data, branch_dir):
        # It exists and check if it's the latest revision already.
        self.log.debug(
            "Existing charm from lp:%s", charm_data["branch_spec"])
        transport = get_transport(branch_dir)
        branch = Branch.open_from_transport(transport)
        cur_rev_id = branch.last_revision()
        return cur_rev_id == charm_data['commit']

    def update_charm(self, charm_data, branch_dir, retry=False):
        self.log.debug("Updating branch lp:%s", charm_data["branch_spec"])
        try:
            subprocess.check_output(
                ["/usr/bin/bzr", "update", "-q"],
                cwd=branch_dir,
                stderr=subprocess.STDOUT)
            charm_data = self.add_files(charm_data)
        except subprocess.CalledProcessError:
            # Update failed for some reason; destroy it and start over.
            if retry:
                shutil.rmtree(branch_dir)
                return self.run(charm_data, retry=False)
            raise

    def run(self, charm_data, retry=True):
        """Fetch a branch from bzr, and augment charm data."""
        branch_dir = os.path.abspath(
            str(os.path.join(self.root_dir,
                             charm_data["series"],
                             charm_data["owner"],
                             charm_data["name"],
                             charm_data["bname"])))

        if not os.path.exists(os.path.dirname(branch_dir)):
            os.makedirs(os.path.dirname(branch_dir))
        # Store the branch directory
        charm_data["branch_dir"] = branch_dir

        if not os.path.exists(branch_dir):
            # Charm doesn't exist; check it out.
            self.checkout_charm(charm_data, branch_dir)
            return
        elif self.charm_is_current(charm_data, branch_dir):
            # Charm exists, and is current; log and finish.
            charm_data = self.add_files(charm_data)
            self.log.debug(
                "Already up to date lp:%s", charm_data["branch_spec"])
            return
        else:
            # Charm exists, but needs updating; update it.
            self.update_charm(charm_data, branch_dir, retry)


class ChangelogIngestJob(IngestJob):

    name = 'changelog'

    def __init__(self, limit=10, since=None):
        self.limit = limit
        self.since = since

    def setup(self):
        super(ChangelogIngestJob, self).setup()
        if self.since is None:
            days_of_revisions = settings.get('days_of_revisions')
            if days_of_revisions is not None:
                cutoff = datetime.utcnow() - timedelta(int(days_of_revisions))
                self.since = calendar.timegm(cutoff.timetuple())

    @staticmethod
    def _rev_info(r, branch):
        d = {
            'authors': r.get_apparent_authors(),
            "revno": branch.revision_id_to_revno(r.revision_id),
            "committer": r.committer,
            "created": r.timestamp,
            "message": r.message
        }
        return d

    def run(self, charm_data):
        branch_dir = charm_data["branch_dir"]
        charm_data.update(self.get_changes(branch_dir))

    def get_changes(self, branch_dir):
        charm_data = {}
        branch = Branch.open(branch_dir)
        branch.lock_read()
        try:
            revisions = self.get_revisions(branch)
            charm_data["changes"] = changes = []
            for r in revisions:
                changes.append(self._rev_info(r, branch))
            if len(revisions) == 0:
                last_change = None
                first_change = None
            else:
                last_change = changes[0]
                first = branch.repository.get_revision(branch.get_rev_id(1))
                first_change = self._rev_info(first, branch)
            charm_data.update({
                'last_change': last_change,
                'first_change': first_change,
            })
            return charm_data
        finally:
            branch.unlock()

    def get_revisions(self, branch):
        # We only want the last 10 changes, in descending order.
        graph = branch.repository.get_graph()
        cur_rev_id = branch.last_revision()
        ancestry_iter = graph.iter_lefthand_ancestry(cur_rev_id)
        revs = []
        for num, revision_id in enumerate(ancestry_iter):
            if revision_id == NULL_REVISION:
                break
            revision = branch.repository.get_revision(revision_id)
            if num >= self.limit:
                if self.since is None or revision.timestamp < self.since:
                    break
            revs.append(revision)
        return revs


class IndexIngestJob(IngestJob):

    name = 'index'

    def setup(self, index_client=None):
        if not index_client:
            index_client = ElasticSearchClient.from_settings(settings)
        self.index_client = index_client

    def run(self, charm):
        self.log.info('Indexing %s' % charm['store_url'])
        self.index_client.index_charm(charm)


class JenkinsIngestJob(FSIngestJob):

    name = 'jenkins'

    def run(self, charm):
        if not charm['branch_spec'].startswith('~charmers'):
            return

        charm['tests'] = {}
        charm['test_results'] = {}

        for p in JENKINS_PROVIDERS:
            try:
                result_id, status = self.store_provider_results(p, charm)
                if result_id is None:
                    continue
                charm['tests'][p] = status
                charm['test_results'][p] = result_id
            except:
                self.log.exception("Unknown error while processing %s %s",
                                   charm['branch_spec'], p)

    def _fetch_artifacts(self, provider, charm, result):
        artifacts = []
        for artifact in result['artifacts']:
            a_path = "%s/%s/%s/%s" % (
                charm['branch_spec'],
                provider,
                result['number'],
                artifact['displayPath'])
            a_url = JENKINS_ARTIFACT_URL % (dict(
                series=charm['series'],
                provider=provider,
                charm=charm['name'],
                build=result['number'],
                artifact=artifact['relativePath']))

            ## File sniffing

            # Load up the charm revision as a result property.
            if artifact['displayPath'] == 'charm-revision':
                charm_revision = urllib2.urlopen(a_url).read().strip()
                if not charm_revision:
                    continue
                result['revno'] = int(charm_revision)
                continue

            # Mark the test result as graph runner enabled.
            if "graph-tests" in artifact['displayPath']:
                result['charmrunner'] = True

            # Skip the actual charm content
            if "charm-%s.zip" % charm['name'] == artifact['displayPath']:
                continue

            # XXX Short circuit before actual fetching.
            continue

            a_file = urllib2.urlopen(a_url)
            file_id = self.fs.put(a_file, path=a_path)
            artifact['file_id'] = file_id
            artifacts.append(artifact)

        return artifacts

    def store_provider_results(self, provider, charm):
        charm_result_url = JENKINS_QA_URL % (
            dict(series=charm['series'],
                 provider=provider,
                 charm=charm['name']))

        self.log.debug("Loading %s from %s", charm['name'], charm_result_url)

        try:
            contents = urllib2.urlopen(charm_result_url).read()
        except urllib2.URLError:
            self.log.debug(
                "No test result for %s @ %s", charm['branch_spec'], provider)
            return None, None

        result = json.loads(contents)

        # If we already have results no pointing in refetching.
        result_id = "%s::%s-%s" % (
            charm['branch_spec'], provider, result['number'])
        db_result = self.db.jenkins.find_one({'_id': result_id})
        if db_result is not None:
            return result_id, db_result['result']

        # Fetch test artifacts.
        artifacts = self._fetch_artifacts(provider, charm, result)

        # Inject test metadata.
        result['branch_spec'] = charm['branch_spec']
        result['provider'] = provider
        result['artifacts'] = artifacts
        result['_id'] = result_id
        self.db.jenkins.insert(result)
        return (result_id, result['result'])


class ProofIngestJob(IngestJob):

    name = 'proof'

    @contextlib.contextmanager
    def _get_proof_lib(self, new_path):
        if new_path not in sys.path:
            sys.path.append(new_path)
        try:
            import lib.proof as prooflib
            yield prooflib
        except ImportError:
            yield None
        finally:
            if new_path in sys.path:
                sys.path.remove(new_path)

    def setup(self, proofer=None, proof_path=None):
        if not proofer:
            proofer = self.get_proofer(proof_path)
        self.proofer = proofer

    def get_proofer(self, proof_path=None):
        proofer = None
        if not proof_path:
            proof_path = CHARM_PROOF_PATH

        # Use config.CHARM_PROOF_PATH for testing. Monkeypatching is scary.
        if not os.path.isdir(proof_path):
            err_msg = ("proof error before processing began: could not find "
                       "charm proof path.")
            self.log.exception(err_msg)
            self.log.exception("CHARM_PROOF_PATH: %s", proof_path)
        with self._get_proof_lib(proof_path) as prooflib:
            if not prooflib:
                err_msg = ("proof error before processing began: could not "
                           "import charm proof lib.")
                self.log.exception(err_msg)
                self.log.exception(
                    "CHARM_PROOF_PATH: %s", proof_path)
            else:
                proofer = prooflib.run
        return proofer

    def run(self, charm):
        if not self.proofer:
            self.log.exception("proof aborted.")
            raise Exception("No proofer")
        proof = {}
        lint, exit_code = self.proofer(charm['branch_dir'])
        for line in lint:
            if not ':' in line:
                continue
            level, msg = line.split(':', 1)
            if level == "W" and 'name' in msg:
                continue
            proof.setdefault(level.lower(), []).append(msg)
        charm['proof'] = proof


class ScanIngestJob(FSIngestJob):

    name = 'scan'

    def process_charm(self, charm):
        # Enrich charm metadata for webapp.

        # Charm url
        if charm["owner"] == "charmers":
            charm["short_url"] = "/charms/%s/%s" % (
                charm["series"], charm["name"])
        else:
            charm["short_url"] = "/~%s/%s/%s" % (charm["owner"],
                                                 charm["series"],
                                                 charm["name"])

        # Charm label
        if charm["owner"] == "charmers":
            charm["label"] = "%s/%s" % (charm["series"], charm["name"])
        else:
            charm["label"] = "~%s:%s/%s" % (charm["owner"],
                                            charm["series"],
                                            charm["name"])

        # Flatten the interfaces provided
        i_provides = []
        provides = charm.get("provides")
        if provides:
            for v in provides.values():
                if not isinstance(v, dict):
                    continue
                i = v.get("interface")
                if not i:
                    continue
                i_provides.append(i)
        charm["i_provides"] = i_provides

        # Flatten the interfaces required
        i_requires = []
        requires = charm.get("requires")
        if requires:
            for v in requires.values():
                i = v.get("interface")
                if not i:
                    continue
                i_requires.append(i)
        charm["i_requires"] = i_requires
        return charm

    def run(self, charm_data):
        # Note: charm_data is modified in-place.  IndexIngestJob requires
        # these modifications.
        files = charm_data['files']
        # Some files have bad characters in them since they are used as mongo
        # keys. Use their escaped forms instead.
        metadata_file = quote_key('metadata.yaml')
        config_file = quote_key('config.yaml')

        if metadata_file not in files:
            self.log.info(
                "Charm has no metadata: %s", charm_data["branch_spec"])
            return
        else:
            cfile = CharmFileSet.get_by_id(
                self.fs, files[metadata_file]['fileid'])
            try:
                metadata = quote_yaml(yaml.load(cfile.read()))
            except Exception, exc:
                self.log.info('Invalid charm metadata %s: %s' % (
                    charm_data['branch_spec'],
                    exc)
                )

        if config_file in files:
            cfile = CharmFileSet.get_by_id(
                self.fs, files[config_file]['fileid'])
            config_raw = cfile.read()

            try:
                config = quote_yaml(yaml.load(config_raw))
            except Exception, exc:
                self.log.info(
                    'Invalid charm config yaml. %s: %s' % (
                        charm_data['branch_spec'],
                        exc)
                )

            metadata["config"] = config
            metadata["config_raw"] = config_raw

        if 'revision' in files:
            cfile = CharmFileSet.get_by_id(
                self.fs, files['revision']['fileid'])
            rev_raw = cfile.read()
            rev_id = int(rev_raw.strip())
            metadata["revision"] = rev_id
        elif not "revision" in metadata:
            self.log.info("Invalid revision %s", charm_data["branch_spec"])
            metadata["revision"] = 0

        hooks = []
        for filedata in files.values():
            if filedata['subdir'] == 'hooks':
                hooks.append(filedata['filename'])
        hooks.sort()
        metadata["hooks"] = hooks

        # Stuff into the db
        metadata.update(charm_data)
        metadata["_id"] = metadata["branch_spec"]
        item = self.db.charms.find_one({"_id": metadata["_id"]})
        if item is None:
            item = metadata
        else:
            #log.debug("Updating %s", metadata["branch_spec"])
            item.update(metadata)
        item = self.process_charm(item)
        self.db.charms.update({"_id": item["_id"]}, item, upsert=True)
        # Modify charm_data so that IndexIngestJob gets the modified version.
        charm_data.clear()
        charm_data.update(item)


def addresses(charm):
    """Return an iterator of potential store addresses."""
    if charm['owner'] == 'charmers':
        yield "cs:%s/%s" % (charm["series"], charm["name"])
    yield "cs:~%s/%s/%s" % (
        charm["owner"], charm["series"], charm["name"])


def make_store_url(revision, address=None, charm=None):
    """Make a store URL given a revision, store address or charm."""
    if address is None:
        for address in addresses(charm):
            break
    return address + "-%d" % revision


class StoreIngestJob(IngestJob):

    name = 'store'

    def run(self, charm):
        old_address = None
        for address in addresses(charm):
            if old_address is not None:
                self.log.info("rechecking %s with ~charmers", old_address)
            old_address = address
            data = self._store_get(address)
            if 'errors' not in data and 'warnings' not in data:
                break

        if 'errors' in data or 'warnings' in data:
            self.log.warning("store error on %s %s" % (address, data))

        data["store_checked"] = datetime.now().ctime()

        charm['store_data'] = data
        charm['store_url'] = make_store_url(data['revision'], address)

    def _store_get(self, address):
        url = STORE_URL + "/charm-info?charms=%s&stats=0" % address
        contents = urllib2.urlopen(url).read()
        data = json.loads(contents)
        data = data[address]
        data['address'] = address
        return data


# XXX j.c.sackett Jan 31 2013 Bug:1111708 scan_repo is swapped for
# scan_charm to reindex what's on disk; we should probably just
# have a different job for this that's not part of ingest.
#def scan_repo(db, root_dir):
    #log = logging.getLogger("charm.scan")
    #charms = os.listdir(root_dir)
    #for c in charms:
        #charm_dir = os.path.join(root_dir, c)
        #if not os.path.isdir(charm_dir):
            #continue
        ##log.info("Processing %s", c)
        #try:
            #scan_charm(
                #db, c, charm_dir, repo="~charmers/charm/oneiric/%s" % c)
        #except:
            #log.exception("Unknown scan error")
            #raise
            #import pdb
            #import sys
            #import traceback
            #traceback.print_exc()
            #pdb.post_mortem(sys.exc_info()[-1])
            #raise