~fabricematrat/charmworld/redirect-charm

# Copyright 2012, 2013 Marco Ceppi, Canonical Ltd.  This software is
# licensed under the GNU Affero General Public License version 3 (see
# the file LICENSE).

from calendar import timegm
import errno
import logging
from mimetypes import guess_type
from os import listdir
from os.path import dirname
from os.path import exists
from os.path import isdir
from os.path import join
from os.path import realpath
import random
import re

import gridfs
import pymongo

from charmworld.utils import (
    configure_logging,
    get_ini,
    quote_key,
)
from charmworld.search import ElasticSearchClient

OBSOLETE_SERIES = set(['oneiric'])


def getconnection(settings, fsync=False, safe=False):
    """Initiliaze the PyMongo database Connection object.

    :param settings: dict of application settings including the mongo.X info.

    """
    url_or_host = settings.get('mongo.url')
    if not url_or_host:
        url_or_host = settings.get('mongo.host')
        port = int(settings.get('mongo.port'))
    else:
        port = None
    connection = pymongo.Connection(
        url_or_host, port,
        fsync=fsync,
        safe=safe,
    )
    return connection


def getdb(connection, database_name):
    """Helper to access the db instance.

    This is needed because the test runner nose loads views which tried to
    import db directly, before the main() is run and initialize_db is run with
    the connection information.

    :param connection: pymongo connection object.
    :param database_name: mongo db name to connect to.
    """
    return getattr(connection, database_name)


def getfs(database):
    """Helper to get the gridfs handle used to store files."""
    return gridfs.GridFS(database)


def parse_sso_fields(payload):
    """Process the data form a sso request to a normal clean dict."""
    user_sso_data = set([
        'openid.sreg.nickname',
        'openid.sreg.fullname',
        'openid.sreg.email',
        'openid.sreg.timezone',
    ])
    common = user_sso_data.intersection(payload)
    # We're using the last bit of the key in our User object.
    user_data = dict((key.split('.')[2], payload[key]) for key in common)
    groups = []
    if payload['openid.lp.is_member'] != '':
        groups = payload['openid.lp.is_member'].split(',')
    user_data['groups'] = groups
    return user_data


class UserMgr(object):
    """Handle non-instance user actions."""

    @staticmethod
    def auth_principalfinder(userid, request):
        """Pyramid wants to know what principals are associated with a user.

        This is a helper provided for the RequestWithUser factory.
        """
        user = request.user
        if user is None:
            return None
        group_principals = ['group %s' % group for group in user.groups]
        return ['user %s' % user.userid] + group_principals

    @staticmethod
    def find_one(db, userid):
        found = db.users.find_one({'userid': userid})

        if not found:
            return None
        else:
            # Make sure we return a instantiated object vs the raw dictionary
            # from mongo.
            userid = found['userid']
            del(found['userid'])
            return User(userid, **found)

    @staticmethod
    def from_sso(payload):
        """Generate a User object instance from the SSO payload.

        We take the SSO dict and turn it into something that a User object,
        without being SSO aware, can easily handle and process.

        """
        user_data = parse_sso_fields(payload)
        user_id = payload['openid.claimed_id']
        return User(user_id, **user_data)


class User(object):
    """A user in our system."""

    # Attributes are defined as a list so we can easily create a
    # dict(someuser) helper to dump to mongodb.
    attributes = [
        '_id',
        'userid',
        'nickname',
        'fullname',
        'email',
        'groups',
        'timezone',
    ]

    def __init__(self, userid, **kwargs):
        self.userid = userid

        # Set some sane defaults since this depends on the users providing us
        # with the data.
        self.nickname = kwargs.get('nickname', 'Not Provided')
        self.fullname = kwargs.get('fullname', 'Not Provided')
        self.email = kwargs.get('email', None)
        self.groups = kwargs['groups']
        self.timezone = kwargs.get('timezone', 'GMT')

        # We need to worry about if this is a mongodb record or just a new
        # user object. A new object has no mongo id.
        if '_id' in kwargs:
            self._id = kwargs['_id']
        else:
            self._id = None

    def __iter__(self):
        """Allow the User to be dumped to a dict for mongo."""
        for key in self.attributes:
            yield(key, getattr(self, key))

    def in_group(self, group):
        return True if group in self.groups else False

    def save(self, db):
        """Actual storing of the instance into mongodb."""
        mongo_data = dict(self)
        if not self._id:
            db.users.save(mongo_data)
        else:
            self._id = db.users.insert(mongo_data)

    def update_from_sso(self, payload):
        """Update the stored user data from a fresh sso payload dump."""
        user_data = parse_sso_fields(payload)
        for key, val in user_data.items():
            setattr(self, key, val)


class Charm(object):
    """A model to access a charm representation.

    A charm representation is constructed from several processes which
    add keys and values to the charm dict. This model represents the
    the properties that all charms will possess when they are processed
    without error.
    """

    _COMMON_REPRESENTATION = {
        # Provided by Lp charms queue.
        'branch_spec': '',
        'bzr_branch': '',
        'branch_deleted': False,
        'bname': '',
        'commit': '',
        'distro_series': [],
        'name': '',
        'owner': '',
        'promulgated': False,
        'series': '',

        # Provided by the update charm job.
        'date_created': None,
        'error': '',
        '_id': '',

        # Provided from the bazaar job.
        'branch_dir': '',
        'files': {},

        # Provided by the charm store job.
        'downloads': 0,
        'downloads_in_past_30_days': 0,
        'store_data': {},
        'store_url': '',

        # Provided by the proof job.
        'proof': {},

        # Provided by the Jenkins job.
        'test_results': {},
        'tests': {},

        # Provided by the changelog job.
        'changes': [],
        'first_change': None,
        'last_change': None,

        # Provided by the scan ingest job.
        'categories': [],
        'config': {'options': {}},
        'config_raw': '',
        'description': '',
        'hooks': [],
        'i_provides': [],
        'i_requires': [],
        'label': '',
        'maintainer': '',
        'provides': {},
        'requires': {},
        'revision': 0,
        'short_url': '',
        'subordinate': False,
        'summary': '',

        # Provided by forms used by ~charmers.
        'qa': None,
        'is_featured': False,
    }

    def __init__(self, charm_data):
        """Init the charm from a charm representation.

        :param charm_data: A dict that represents the properties of a charm.
        """
        self._raw_representation = charm_data
        self._representation = dict(self._COMMON_REPRESENTATION)
        self._representation.update(charm_data)

    def __eq__(self, other):
        if (isinstance(other, self.__class__)
                and self.owner == other.owner
                and self.series == other.series
                and self.name == other.name):
            # The store thinks these two charms are the same.
            return True
        return False

    @property
    def branch_spec(self):
        """The location of the charm's branch.

        This is based on the Launchpad URI. It is used to locate the branch
        Locally as well as on Lp. For example, ~owner/charms/series/name/trunk.
        """
        return self._representation['branch_spec']

    @property
    def bzr_branch(self):
        """The Lp location of the charm's branch."""
        return "lp:%s" % self.branch_spec

    @property
    def branch_deleted(self):
        """True if the Launchpad branch of this charm is deleted, else False.
        """
        return self._representation['branch_deleted']

    @property
    def owner(self):
        """The owner of the charm's branch.'

        The branch owner is not the maintainer from the charm metadata. The
        branch owner is the person who published the charm.
        """
        return self._representation['owner']

    @property
    def series(self):
        """The series the charm is intended to deploy on.

        The series corresponds to the charms series in Launchpad, which mirrors
        the Ubuntu series names in Launchpad.
        """
        return self._representation['series']

    @property
    def name(self):
        """The charm name.

        The name corresponds to the charms distribution source package in
        Launchpad. For example, 'apache2' is the name cs:precise/apache2-33.
        """
        return self._representation['name']

    @property
    def bname(self):
        """The charm's branch name.

        Only branches named trunk or trunk-1 are ingested from Launchpad.
        """
        return self._representation['bname']

    @property
    def commit(self):
        """The revid of the charm branch's last commit.

        The string can be empty if the branch is not found or it is empty.
        """
        return self._representation['commit']

    @property
    def distro_series(self):
        """The official series the branch is linked to.

        A branch linked to a series package is synonymous with
        promulgated to a series charm after review.
        """
        return self._representation['distro_series']

    @property
    def promulgated(self):
        """Is the charm promulgated?

        Is the charm reviewed and recommended to be the default charm
        for a series? The charm's URI will not include the owner so
        that it can be deployed by specifying just the charm's series
        and name.
        """
        return self._representation['promulgated']

    @property
    def _id(self):
        """The _id used to find the charm in the database.

        The value comes from the branch_spec. The database _id for a charm
        looks like ~owner/charms/series/name/trunk
        """
        return self._representation['_id']

    @property
    def date_created(self):
        """The ISO timestamp the charm was created.

        This value is created from the date the branch was created, which
        can disagree with the intent to show the age of the charm. For example,
        when the source branch for a charm is changed, the date_created value
        will change to the value from the new branch. The value can be an
        empty string if the branch can not be found.
        """
        return self._representation['date_created']

    @property
    def error(self):
        """The error seen during ingest, or an empty string when successful."""
        return self._representation['error']

    @property
    def branch_dir(self):
        """The path to the charm's branch directory in the local repository.

        The path takes the form of /var/charms/<series>/<owner>/<name>/<bname>.
        """
        return self._representation['branch_dir']

    @property
    def files(self):
        """The dict `CharmFile` dicts representing the files in the charm."""
        return self._representation['files']

    @property
    def store_url(self):
        """The charm's deployment URL."""
        return self._representation['store_url']

    @property
    def store_revision(self):
        """The charm's revision in the store."""
        revision = self.store_data.get("revision")
        if revision is None:
            revision = 1
        return revision

    @property
    def store_data(self):
        """The charm data provided by the store.

        A dict of information including address, store_checked, digest,
        revision, and sha256. The error key may replace the digest, revision,
        and sha256 keys if the store found errors. The dict will be empty
        if there were ingest errors"""
        return self._representation['store_data']

    @property
    def downloads(self):
        """The number of times the charm was downloaded.

        The number comes from the store. It does not equate to deploys."""
        return self._representation['downloads']

    @property
    def downloads_in_past_30_days(self):
        """The number of times the charm was downloaded in the past 30 days.

        The number comes from the store. It does not equate to deploys."""
        return self._representation['downloads_in_past_30_days']

    @property
    def proof(self):
        """The dict charm's proof warnings and errors."""
        return self._representation['proof']

    @property
    def tests(self):
        """The dict charm's status of tested providers."""
        return self._representation['tests']

    @property
    def test_results(self):
        """The dict charm's test results for each provider."""
        return self._representation['test_results']

    @property
    def changes(self):
        """The list of revision info dicts from the charm's branch."""
        return self._representation['changes']

    @property
    def first_change(self):
        """The revision dict of the first change to the charm's branch."""
        return self._representation['first_change']

    @property
    def last_change(self):
        """The revision dict of the last change to the charm's branch."""
        return self._representation['last_change']

    @property
    def revision(self):
        """The charm revision number.

        The charm revision comes from the revision file in the charm's branch.
        It is a manually incremented version number that is independent of
        the charm's branch revisions.
        """
        return self._representation['revision']

    @property
    def summary(self):
        """The charms summary.

        The summary comes from the charm's metadata.
        """
        return self._representation['summary']

    @property
    def description(self):
        """The charms description.

        The description comes from the charm's metadata.
        """
        return self._representation['description']

    @property
    def categories(self):
        """The list of categories the charm relates to.

        The categories come from the charm's metadata.
        """
        return self._representation['categories']

    @property
    def maintainer(self):
        """The maintainer of this charm.

        The maintainer is a string or a list of strings in the RFC2822 format.
        """
        return self._representation['maintainer']

    @property
    def subordinate(self):
        """Is this charm a subordinate of another charm?"""
        return self._representation['subordinate']

    @property
    def provides(self):
        """The dict of interfaces that the charm provides.

        The provides interfaces comes from the charm's metadata.
        """
        return self._representation['provides']

    @property
    def i_provides(self):
        """The list of interfaces that the charm provides.

        The list is a flattened representation of the provides property.
        """
        return self._representation['i_provides']

    @property
    def requires(self):
        """The dict of interfaces that the charm requires.

        The requires interfaces comes from the charm's metadata.
        """
        return self._representation['requires']

    @property
    def i_requires(self):
        """The list of interfaces that the charm requires.

        The list is a flattened representation of the requires property.
        """
        return self._representation['i_requires']

    @property
    def config(self):
        """The dict of the charm's configuration options.

        The dict is parsed from the config_raw property's YAML.
        """
        return self._representation['config']

    @property
    def config_raw(self):
        """The charm's configuration YAML.

        The unparsed content of the charm's config.yaml file.
        """
        return self._representation['config_raw']

    @property
    def options(self):
        """The charm's configuration options."""
        config = self._representation['config']
        if config is not None and 'options' in config:
            return config['options']
        else:
            return {}

    @property
    def hooks(self):
        """The list of life cycle and relation hooks the charm supports.

        The list is created from the filenames found in the hooks directory.
        """
        return self._representation['hooks']

    @property
    def label(self):
        """The charm name qualified by series and owner.

        XXX sinzui 2013-06-04 bug=1118351:
        Nb. This property overlaps with branch_spec and short_url.
        It is the cause of ambiguous paths that users misconstrue to
        be cs: urls. In general, users believe cs:urls to be a proper
        label.
        """
        return self._representation['label']

    @property
    def short_url(self):
        """The short URL the path to the charm within charmworld's routing.

        The short_url property is based on the branch_spec. It is either
        ~owner/series/name or charms/series/name for promulgated charms.
        """
        return self._representation['short_url']

    @property
    def qa(self):
        """The charm's dict of QA scores."""
        return self._representation['qa']

    @property
    def is_featured(self):
        """Is this charm featured in the interesting view?"""
        return self._representation['is_featured']

    def changes_since(self, date):
        """The changes that have been introduced since a given datetime."""
        date = timegm(date.timetuple())
        return [change for change in self.changes if change['created'] >= date]


class QAData(object):
    """Build a usable object out of the QA questions and instance data."""

    def _load_questionset(self):
        """Load the qa questions we want to have data about."""
        stored_questions = self.db.qa.find()
        # We index them for easy fetching of each categories data later.
        self.qa_categories = dict([(q['name'], q) for q in stored_questions])

    def __init__(self, db, charm_data=None):
        self.db = db
        self._load_questionset()
        self.charm_data = charm_data

    @property
    def categories(self):
        return sorted(self.qa_categories)

    @property
    def score(self):
        """Calculate the score for whole qa set across categories."""

        # If we have no qa data then we need to respond with not available
        if not self.charm_data:
            return None

        score = 0
        for category in self.categories:
            category_score = self.category_score(category)
            if category_score is not None:
                score = score + category_score

        return score

    def category_score(self, category):
        """Given a category retrieve the scores for it."""
        score = 0
        if self.charm_data is None or category not in self.charm_data:
            return None

        category_questions = self.qa_categories[category]['questions']
        charm_category_scores = self.charm_data[category]

        for qnum, question in enumerate(category_questions):
            score_key = "{0}_{1}".format(category, qnum)
            q_score = charm_category_scores.get(score_key, 0)
            # The score could be '' if it's unknown vs yes/no
            if q_score:
                score = score + int(q_score)
        return score

    def category_max_score(self, category):
        """How many possible points does the category have?"""
        return len(self.qa_categories[category]['questions'])

    def filtered_scores(self):
        """Return only scores that match the categories."""
        if self.charm_data is None:
            return None
        score_categories = {}
        for category in self.qa_categories:
            if category not in self.charm_data:
                continue
            score_categories[category] = {}
            for question in self.qa_categories[category]['questions']:
                qid = question['id']
                score = self.charm_data[category].get(qid)
                if score is not None:
                    score_categories[category][qid] = score
        return score_categories


class CharmFileSet(object):
    """Handle Charm file storage and access.

    Charm key - series/owner/name/bname [branch name: trunk]

    """
    store_files = [
        'config.yaml',
        'metadata.yaml',
        'readme.*',
        'revision',
        'icon.svg',
    ]

    store_directories = [
        'hooks',
    ]

    @classmethod
    def make_charm_file(cls, fs, charm_data, file_path, filename):
        charmid = cls.gen_charmid(charm_data)
        fileid = cls.gen_fileid(charm_data, file_path)
        content_type = guess_type(filename)[0]
        return CharmFile(
            fs,
            fileid,
            charmid,
            filename=filename,
            contentType=content_type,
        )

    @classmethod
    def save_file(cls, fs, logger, charm_data, filename, local_directory,
                  subdir=None):
        """Do the actual storage of the file contents.

        :param fs: The mongo fs connection to use.
        :param fileid: The id of the file which is a full path and escaped.
        :param filename: The original short filename of the file.
        :param directory: The directory is this file located in on disk.

        """
        charm_root = realpath(local_directory)
        if subdir is not None:
            file_path = join(local_directory, subdir, filename)
        else:
            file_path = join(local_directory, filename)
        real_file_path = realpath(file_path)
        if not real_file_path.startswith(charm_root):
            logger.warn(
                "Invalid file path: Path %s resolves to %s, which is not "
                "inside the charm's root directory %s" % (
                    file_path, real_file_path, charm_root))
            return
        if not exists(real_file_path):
            logger.warn(
                "File does not exist: %s (resolving to %s)." % (
                    file_path, real_file_path))
            return
        if subdir:
            complete_filename = join(subdir, filename)
        else:
            complete_filename = filename
        charm_file = cls.make_charm_file(
            fs, charm_data, complete_filename, filename)

        file_path = join(local_directory, complete_filename)
        with open(file_path) as handle:
            charm_file.save(handle.read())
        return charm_file

    @classmethod
    def gen_fileid(cls, charm_data, filename):
        """Generate our own mongo _id for the files.

        In keeping with the bzr structure files are stored:
        series-owner-name-bname-filename
        """
        return quote_key("/".join([
            cls.gen_charmid(charm_data),
            filename
        ]))

    @staticmethod
    def gen_charmid(charm_data):
        """Generate the charm id to aid in searching."""
        return "/".join([
            charm_data['owner'],
            charm_data['series'],
            charm_data['name'],
            charm_data['bname'],
        ])

    @staticmethod
    def get_by_id(fs, fileid):
        """Fetch the contents of the file

        :param fileid: The stored _id of the file.

        """
        if '.' in fileid:
            fileid = quote_key(fileid)
        found = fs.get(fileid)
        cf = CharmFile(fs, fileid, found.charmid, instance=found)
        return cf

    @classmethod
    def save_files(cls, fs, charm_data, charm_bzr_path, logger):
        """Given a bzr branch, load the files we care about."""
        root_files = listdir(charm_bzr_path)
        stored = []

        for pattern in cls.store_files:
            # Make sure our file exists and find the right filename for it.
            found_filename = None
            for filename in root_files:
                m = re.match(pattern, filename, flags=re.I)
                if m:
                    found_filename = filename

            if found_filename:
                charm_file = cls.save_file(
                    fs,
                    logger,
                    charm_data,
                    found_filename,
                    charm_bzr_path)
                if charm_file is not None:
                    stored.append(charm_file)

        for directory in cls.store_directories:
            dir_path = join(charm_bzr_path, directory)
            try:
                dir_files = listdir(dir_path)
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
                continue

            for filename in dir_files:
                if not isdir(join(dir_path, filename)):
                    charm_file = cls.save_file(
                        fs, logger, charm_data, filename, charm_bzr_path,
                        subdir=directory)
                    if charm_file is not None:
                        stored.append(charm_file)
        return stored


class CharmFile(object):
    """Representation of the GridFS stored document.

    Wraps a PyMongo GridOut object instance.

    """
    # Attributes are defined as a list so we can easily create a
    # dict(charmfile) helper to dump to mongodb.
    attributes = [
        'fileid',
        'charmid',
        'contentType',
        'subdir',
        'filename',
        'length',
        'md5',
    ]

    def __iter__(self):
        """Allow the Object to be dumped to a dict for mongo."""
        for key in self.attributes:
            if hasattr(self, key):
                yield(key, getattr(self, key))
            else:
                yield(key, getattr(self._gridout, key))

    @property
    def subdir(self):
        """From a file id drop the file and return a charmid"""
        filename = self.fileid.replace(quote_key(self.charmid), '').lstrip('/')
        # If there's no subdir in the filename, there's no subdir to return.
        return dirname(filename)

    @property
    def fileid(self):
        """fileid is more clear in code, but we're mapping this to mongo _id

        fileid cannot have . characters; it's used as a key in mongo.
        """
        if '.' in self._id:
            return quote_key(self._id)
        else:
            return self._id

    @property
    def filename(self):
        """The filename is in the fileid so auto pull it out."""
        if self._filename:
            return self._filename
        elif self._gridout:
            return self._gridout.filename
        else:
            return self._id.split('/')[-1]

    @property
    def md5(self):
        if self._gridout is None:
            return None
        return self._gridout.md5

    # value to use as default to distinguish between None/null and unset.
    undefined = object()

    def __init__(self, fs, fileid, charmid, filename=None,
                 instance=None, metadata=None, contentType=undefined):
        """Get a file object started.

        :param fs: the gridfs client to use.
        :param fileid: The full mongo fileid to use for this file.
        :param charmid: The path of the charm identification through branch
            name.
        :param filename: Override the name of the file from being determined
            by the fileid.
        :param instance: The GridOut instance to be associated with the file.
        :param metadata: dict of extra data to store with the file in GridFS.

        """
        self.fs = fs
        self._id = fileid
        self.charmid = charmid
        self._filename = None
        if contentType is not self.undefined:
            self.contentType = contentType
        elif instance is not None:
            self.contentType = getattr(instance, 'contentType', None)
        else:
            self.contentType = None

        if instance:
            self._gridout = instance
            # The filename might have been written out previously.
            self._filename = instance.filename
        else:
            self._gridout = None

        if filename:
            self._filename = filename

        if metadata:
            self.metadata = metadata
        else:
            self.metadata = {}

    def read(self):
        """Provide a read interface for the file."""
        if not self._gridout:
            self._gridout = self.fs.get(self.fileid)
        return self._gridout.read()

    def save(self, contents, data=None):
        """Store the contents of the file.

        :param contents: String/readable value of file content.
        :param data: dict of metadata to override defaults. See GridOut

        """
        if data:
            self.metadata.update(data)

        # Delete any existing charm file with this id, including orphaned
        # chunks.
        self.fs.delete(self.fileid)
        self.metadata['_id'] = self.fileid
        self.metadata['charmid'] = self.charmid

        self.fs.put(
            contents,
            filename=self.filename,
            contentType=self.contentType,
            **self.metadata
        )

        # We then have to pull the file we just saved back out so that we can
        # load the instance data into our object.
        self._gridout = self.fs.get(self.fileid)
        return self.fileid


class CharmSource:

    def __init__(self, db, index_client):
        self.db = db
        self.index_client = index_client

    @classmethod
    def from_settings(cls, settings):
        db = getdb(getconnection(settings), settings['mongo.database'])
        index_client = ElasticSearchClient.from_settings(settings)
        return cls(db, index_client)

    @classmethod
    def from_request(cls, request):
        return cls(request.db, request.index_client)

    def sync_index(self):
        for charm in self.db.charms.find():
            yield charm['_id']
            self.index_client.index_charm(charm)

    def save(self, charm):
        self.db.charms.save(charm)
        self.index_client.index_charm(charm)

    def get_store_charms(self):
        return self.db.charms.find({'store_url': {'$exists': True}})

    @staticmethod
    def exclude_obsolete(query):
        return {'$and': [query, {'series': {'$nin': list(OBSOLETE_SERIES)}}]}

    def find_mia_charms(self):
        query = self.exclude_obsolete(
            {"store_data.errors": {"$exists": True}})
        return (Charm(data) for data in self.db.charms.find(query))

    def find_proof_error_charms(self, sub_only):
        query = {"proof.e": {"$exists": True}, "promulgated": True}
        if sub_only:
            query['subordinate'] = True
        query = self.exclude_obsolete(query)
        sort = [("series", pymongo.DESCENDING), ("name", pymongo.ASCENDING)]
        return (Charm(data) for data in self.db.charms.find(query, sort=sort))

    def _get_all(self, charm_id):
        """"Return all stored values for a given charm."""
        yield self.db.charms.find_one({'_id': charm_id})
        yield self.index_client.get(charm_id)


def acquire_session_secret(database):
    """Return a secret to use for AuthTkt sessions.

    If a secret is already in the database, use that.  Otherwise, generate a
    new secret.
    """
    secret = str(random.getrandbits(32))
    try:
        database.app_config.insert({'_id': 'session-secret', 'secret': secret})
    except pymongo.errors.DuplicateKeyError:
        # If there was a secret already, just use the existing one.
        pass
    unicode_secret = database.app_config.find_one('session-secret')['secret']
    return unicode_secret.encode('ascii')


def find_charms(db, spec=None, fields=None, skip=0, limit=0, timeout=True,
                snapshot=False, tailable=False, sort=None, max_scan=None,
                as_class=None, valid_charms_only=True, only_promulgated=False,
                **kwargs):
    """Return items from the charms collection.

    If the parameter valid_charms_only is True, a filter is added to spec
    so that only those charms are returned where the field store_data.errors
    does not exist.
    """
    collection = db.charms
    if valid_charms_only:
        if spec is None:
            spec = {}
        spec['store_data.errors'] = {'$exists': False}
    if only_promulgated:
        if spec is None:
            spec = {}
        spec['promulgated'] = True
    return collection.find(
        spec, fields, skip, limit, timeout, snapshot, tailable, sort,
        max_scan, as_class, **kwargs)


def sync_index():
    """"Entry point for sync-index script."""
    configure_logging()
    logger = logging.getLogger('sync-index')
    charm_source = CharmSource.from_settings(get_ini())
    charm_ids = charm_source.sync_index()
    while True:
        try:
            charm_id = next(charm_ids)
        except StopIteration:
            break
        except Exception:
            logger.exception('Unable to index charm.')
        else:
            logger.info('Syncing %s to Elasticsearch' % charm_id)