~launchpad-pqm/launchpad/devel

# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the

# GNU Affero General Public License version 3 (see the file LICENSE).

"""Launchpad bug-related database table classes."""

__metaclass__ = type

__all__ = [

    'Bug',

    'BugAffectsPerson',

    'BugBecameQuestionEvent',

    'BugMute',

    'BugSet',

    'BugTag',

    'FileBugData',

    'generate_subscription_with',

    'get_also_notified_subscribers',

    'get_bug_tags_open_count',

    ]

from cStringIO import StringIO

from email.utils import make_msgid

from functools import wraps

from itertools import chain

import operator

import re

from lazr.lifecycle.event import (

    ObjectCreatedEvent,

    ObjectModifiedEvent,

    )

from lazr.lifecycle.snapshot import Snapshot

import pytz

from sqlobject import (

    BoolCol,

    ForeignKey,

    IntCol,

    SQLMultipleJoin,

    SQLObjectNotFound,

    SQLRelatedJoin,

    StringCol,

    )

from storm.expr import (

    And,

    Coalesce,

    Desc,

    In,

    Join,

    LeftJoin,

    Max,

    Not,

    Or,

    Select,

    SQL,

    Sum,

    Union,

    With,

    )

from storm.info import ClassAlias

from storm.locals import (

    DateTime,

    Int,

    Reference,

    )

from storm.store import (

    EmptyResultSet,

    Store,

    )

from zope.component import getUtility

from zope.contenttype import guess_content_type

from zope.event import notify

from zope.interface import (

    implementer,

    providedBy,

    )

from zope.security.interfaces import Unauthorized

from zope.security.proxy import (

    ProxyFactory,

    removeSecurityProxy,

    )

from lp.answers.interfaces.questiontarget import IQuestionTarget

from lp.app.enums import (

    InformationType,

    PRIVATE_INFORMATION_TYPES,

    PROPRIETARY_INFORMATION_TYPES,

    SECURITY_INFORMATION_TYPES,

    ServiceUsage,

    )

from lp.app.errors import (

    NotFoundError,

    SubscriptionPrivacyViolation,

    UserCannotUnsubscribePerson,

    )

from lp.app.interfaces.informationtype import IInformationType

from lp.app.interfaces.launchpad import ILaunchpadCelebrities

from lp.app.interfaces.services import IService

from lp.app.model.launchpad import InformationTypeMixin

from lp.app.validators import LaunchpadValidationError

from lp.bugs.adapters.bug import convert_to_information_type

from lp.bugs.adapters.bugchange import (

    BranchLinkedToBug,

    BranchUnlinkedFromBug,

    BugConvertedToQuestion,

    BugDuplicateChange,

    BugWatchAdded,

    BugWatchRemoved,

    SeriesNominated,

    UnsubscribedFromBug,

    )

from lp.bugs.enums import BugNotificationLevel

from lp.bugs.errors import InvalidDuplicateValue

from lp.bugs.interfaces.bug import (

    IBug,

    IBugBecameQuestionEvent,

    IBugMute,

    IBugSet,

    IFileBugData,

    )

from lp.bugs.interfaces.bugactivity import IBugActivitySet

from lp.bugs.interfaces.bugattachment import (

    BugAttachmentType,

    IBugAttachmentSet,

    )

from lp.bugs.interfaces.bugmessage import IBugMessageSet

from lp.bugs.interfaces.bugnomination import (

    BugNominationStatus,

    NominationError,

    NominationSeriesObsoleteError,

    )

from lp.bugs.interfaces.bugnotification import IBugNotificationSet

from lp.bugs.interfaces.bugtarget import ISeriesBugTarget

from lp.bugs.interfaces.bugtask import (

    BugTaskStatus,

    BugTaskStatusSearch,

    IBugTask,

    IBugTaskSet,

    IllegalTarget,

    UNRESOLVED_BUGTASK_STATUSES,

    )

from lp.bugs.interfaces.bugtracker import BugTrackerType

from lp.bugs.interfaces.bugwatch import IBugWatchSet

from lp.bugs.interfaces.cve import ICveSet

from lp.bugs.interfaces.hasbug import IHasBug

from lp.bugs.mail.bugnotificationrecipients import BugNotificationRecipients

from lp.bugs.model.bugactivity import BugActivity

from lp.bugs.model.bugattachment import BugAttachment

from lp.bugs.model.bugbranch import BugBranch

from lp.bugs.model.buglinktarget import (

    ObjectLinkedEvent,

    ObjectUnlinkedEvent,

    )

from lp.bugs.model.bugmessage import BugMessage

from lp.bugs.model.bugnomination import BugNomination

from lp.bugs.model.bugnotification import BugNotification

from lp.bugs.model.bugsubscription import BugSubscription

from lp.bugs.model.bugtarget import OfficialBugTag

from lp.bugs.model.bugtask import (

    BugTask,

    bugtask_sort_key,

    )

from lp.bugs.model.bugwatch import BugWatch

from lp.bugs.model.structuralsubscription import (

    get_structural_subscribers,

    get_structural_subscriptions,

    )

from lp.code.interfaces.branchcollection import IAllBranches

from lp.code.interfaces.gitcollection import IAllGitRepositories

from lp.hardwaredb.interfaces.hwdb import IHWSubmissionBugSet

from lp.registry.errors import CannotChangeInformationType

from lp.registry.interfaces.accesspolicy import (

    IAccessArtifactGrantSource,

    IAccessArtifactSource,

    )

from lp.registry.interfaces.distribution import IDistribution

from lp.registry.interfaces.distroseries import IDistroSeries

from lp.registry.interfaces.person import (

    IPersonSet,

    validate_person,

    validate_public_person,

    )

from lp.registry.interfaces.product import IProduct

from lp.registry.interfaces.productseries import IProductSeries

from lp.registry.interfaces.role import IPersonRoles

from lp.registry.interfaces.series import SeriesStatus

from lp.registry.interfaces.sharingjob import (

    IRemoveArtifactSubscriptionsJobSource,

    )

from lp.registry.interfaces.sourcepackage import ISourcePackage

from lp.registry.model.accesspolicy import reconcile_access_for_artifact

from lp.registry.model.person import (

    Person,

    person_sort_key,

    PersonSet,

    )

from lp.registry.model.pillar import pillar_sort_key

from lp.registry.model.teammembership import TeamParticipation

from lp.services.config import config

from lp.services.database import bulk

from lp.services.database.constants import UTC_NOW

from lp.services.database.datetimecol import UtcDateTimeCol

from lp.services.database.decoratedresultset import DecoratedResultSet

from lp.services.database.enumcol import EnumCol

from lp.services.database.interfaces import IStore

from lp.services.database.sqlbase import (

    SQLBase,

    sqlvalues,

    )

from lp.services.database.stormbase import StormBase

from lp.services.fields import DuplicateBug

from lp.services.helpers import shortlist

from lp.services.librarian.interfaces import ILibraryFileAliasSet

from lp.services.librarian.model import (

    LibraryFileAlias,

    LibraryFileContent,

    )

from lp.services.messages.interfaces.message import (

    IMessage,

    IndexedMessage,

    )

from lp.services.messages.model.message import (

    Message,

    MessageChunk,

    MessageSet,

    )

from lp.services.propertycache import (

    cachedproperty,

    clear_property_cache,

    get_property_cache,

    )

from lp.services.webapp.authorization import check_permission

from lp.services.webapp.interfaces import ILaunchBag

from lp.services.webapp.publisher import (

    get_raw_form_value_from_current_request,

    )

from lp.services.xref.interfaces import IXRefSet

def snapshot_bug_params(bug_params):

    """Return a snapshot of a `CreateBugParams` object."""

    return Snapshot(

        bug_params, names=[

            "owner", "title", "comment", "description", "msg",

            "datecreated", "information_type", "target", "status",

            "subscribers", "tags", "subscribe_owner", "filed_by",

            "importance", "milestone", "assignee", "cve"])

class BugTag(SQLBase):

    """A tag belonging to a bug."""

    bug = ForeignKey(dbName='bug', foreignKey='Bug', notNull=True)

    tag = StringCol(notNull=True)

def get_bug_tags_open_count(context_condition, user, tag_limit=0,

    include_tags=None):

    """Worker for IBugTarget.getUsedBugTagsWithOpenCounts.

    See `IBugTarget` for details.

    The only change is that this function takes a SQL expression for limiting

    the found tags.

    :param context_condition: A Storm SQL expression, limiting the

        used tags to a specific context. Only the BugSummary table may be

        used to choose the context. If False then no query will be performed

        (and {} returned).

    """

    # Circular fail.

    from lp.bugs.model.bugsummary import (

        BugSummary,

        get_bugsummary_filter_for_user,

        )

    tags = {}

    if include_tags:

        tags = dict((tag, 0) for tag in include_tags)

    where_conditions = [

        BugSummary.status.is_in(UNRESOLVED_BUGTASK_STATUSES),

        BugSummary.tag != None,

        context_condition,

        ]

    # Apply the privacy filter.

    store = IStore(BugSummary)

    user_with, user_where = get_bugsummary_filter_for_user(user)

    if user_with:

        store = store.with_(user_with)

    where_conditions.extend(user_where)

    sum_count = Sum(BugSummary.count)

    tag_count_columns = (BugSummary.tag, sum_count)

    # Always query for used

    def _query(*args):

        return store.find(tag_count_columns, *(where_conditions + list(args))

            ).group_by(BugSummary.tag).having(sum_count != 0).order_by(

            Desc(Sum(BugSummary.count)), BugSummary.tag)

    used = _query()

    if tag_limit:

        used = used[:tag_limit]

    if include_tags:

        # Union in a query for just include_tags.

        used = used.union(_query(BugSummary.tag.is_in(include_tags)))

    tags.update(dict(used))

    return tags

@implementer(IBugBecameQuestionEvent)

class BugBecameQuestionEvent:

    """See `IBugBecameQuestionEvent`."""

    def __init__(self, bug, question, user):

        self.bug = bug

        self.question = question

        self.user = user

def update_bug_heat(bug_ids):

    """Update the heat for the specified bugs."""

    # We need to flush the store first to ensure that changes are

    # reflected in the new bug heat total.

    if not bug_ids:

        return

    store = IStore(Bug)

    store.find(

        Bug, Bug.id.is_in(bug_ids)).set(

            heat=SQL('calculate_bug_heat(Bug.id)'),

            heat_last_updated=UTC_NOW)

@implementer(IBug, IInformationType)

class Bug(SQLBase, InformationTypeMixin):

    """A bug."""

    _defaultOrder = '-id'

    # db field names

    name = StringCol(unique=True, default=None)

    title = StringCol(notNull=True)

    description = StringCol(notNull=False, default=None)

    owner = ForeignKey(

        dbName='owner', foreignKey='Person',

        storm_validator=validate_public_person, notNull=True)

    duplicateof = ForeignKey(

        dbName='duplicateof', foreignKey='Bug', default=None)

    datecreated = UtcDateTimeCol(notNull=True, default=UTC_NOW)

    date_last_updated = UtcDateTimeCol(notNull=True, default=UTC_NOW)

    date_made_private = UtcDateTimeCol(notNull=False, default=None)

    who_made_private = ForeignKey(

        dbName='who_made_private', foreignKey='Person',

        storm_validator=validate_public_person, default=None)

    information_type = EnumCol(

        enum=InformationType, notNull=True, default=InformationType.PUBLIC)

    # useful Joins

    activity = SQLMultipleJoin('BugActivity', joinColumn='bug', orderBy='id')

    messages = SQLRelatedJoin('Message', joinColumn='bug',

                           otherColumn='message',

                           intermediateTable='BugMessage',

                           prejoins=['owner'],

                           orderBy=['datecreated', 'id'])

    bug_messages = SQLMultipleJoin(

        'BugMessage', joinColumn='bug', orderBy='index')

    watches = SQLMultipleJoin(

        'BugWatch', joinColumn='bug', orderBy=['bugtracker', 'remotebug'])

    duplicates = SQLMultipleJoin('Bug', joinColumn='duplicateof', orderBy='id')

    linked_bugbranches = SQLMultipleJoin(

        'BugBranch', joinColumn='bug', orderBy='id')

    date_last_message = UtcDateTimeCol(default=None)

    number_of_duplicates = IntCol(notNull=True, default=0)

    message_count = IntCol(notNull=True, default=0)

    users_affected_count = IntCol(notNull=True, default=0)

    users_unaffected_count = IntCol(notNull=True, default=0)

    heat = IntCol(notNull=True, default=0)

    heat_last_updated = UtcDateTimeCol(default=None)

    latest_patch_uploaded = UtcDateTimeCol(default=None)

    @property

    def linked_branches(self):

        return [link.branch for link in self.linked_bugbranches]

    @property

    def cves(self):

        from lp.bugs.model.cve import Cve

        xref_cve_sequences = [

            sequence for _, sequence in getUtility(IXRefSet).findFrom(

                (u'bug', unicode(self.id)), types=[u'cve'])]

        expr = Cve.sequence.is_in(xref_cve_sequences)

        return list(sorted(

            IStore(Cve).find(Cve, expr), key=operator.attrgetter('sequence')))

    @property

    def questions(self):

        from lp.answers.model.question import Question

        question_ids = [

            int(id) for _, id in getUtility(IXRefSet).findFrom(

                (u'bug', unicode(self.id)), types=[u'question'])]

        return list(sorted(

            bulk.load(Question, question_ids), key=operator.attrgetter('id')))

    @property

    def specifications(self):

        from lp.blueprints.model.specification import Specification

        spec_ids = [

            int(id) for _, id in getUtility(IXRefSet).findFrom(

                (u'bug', unicode(self.id)), types=[u'specification'])]

        return list(sorted(

            bulk.load(Specification, spec_ids), key=operator.attrgetter('id')))

    def getSpecifications(self, user):

        """See `IBug`."""

        from lp.blueprints.model.specification import Specification

        from lp.blueprints.model.specificationsearch import (

            get_specification_privacy_filter,

            )

        return IStore(Specification).find(

            Specification,

            Specification.id.is_in(spec.id for spec in self.specifications),

            *get_specification_privacy_filter(user))

    @property

    def security_related(self):

        return self.information_type in SECURITY_INFORMATION_TYPES

    @cachedproperty

    def _subscriber_cache(self):

        """Caches known subscribers."""

        return set()

    @cachedproperty

    def _subscriber_dups_cache(self):

        """Caches known subscribers to dupes."""

        return set()

    @cachedproperty

    def _unsubscribed_cache(self):

        """Cache known non-subscribers."""

        return set()

    @property

    def latest_patch(self):

        """See `IBug`."""

        # We want to retrieve the most recently added bug attachment

        # that is of type BugAttachmentType.PATCH. In order to find

        # this attachment, we should in theory sort by

        # BugAttachment.message.datecreated. Since we don't have

        # an index for Message.datecreated, such a query would be

        # quite slow. We search instead for the BugAttachment with

        # the largest ID for a given bug. This is "nearly" equivalent

        # to searching the record with the maximum value of

        # message.datecreated: The only exception is the rare case when

        # two BugAttachment records are simultaneuosly added to the same

        # bug, where bug_attachment_1.id < bug_attachment_2.id, while

        # the Message record for bug_attachment_2 is created before

        # the Message record for bug_attachment_1. The difference of

        # the datecreated values of the Message records is in this case

        # probably smaller than one second and the selection of the

        # "most recent" patch anyway somewhat arbitrary.

        return Store.of(self).find(

            BugAttachment, BugAttachment.id == Select(

                Max(BugAttachment.id),

                And(BugAttachment.bug == self.id,

                    BugAttachment.type == BugAttachmentType.PATCH))).one()

    @property

    def comment_count(self):

        """See `IBug`."""

        return self.message_count - 1

    @property

    def users_affected(self):

        """See `IBug`."""

        return Store.of(self).find(

            Person, BugAffectsPerson.person == Person.id,

            BugAffectsPerson.affected, BugAffectsPerson.bug == self)

    @property

    def users_unaffected(self):

        """See `IBug`."""

        return Store.of(self).find(

            Person, BugAffectsPerson.person == Person.id,

            Not(BugAffectsPerson.affected), BugAffectsPerson.bug == self)

    @property

    def user_ids_affected_with_dupes(self):

        """Return all IDs of Persons affected by this bug and its dupes.

        The return value is a Storm expression.  Running a query with

        this expression returns a result that may contain the same ID

        multiple times, for example if that person is affected via

        more than one duplicate."""

        return Union(

            Select(Person.id,

                   And(BugAffectsPerson.person == Person.id,

                       BugAffectsPerson.affected,

                       BugAffectsPerson.bug == self)),

            Select(Person.id,

                   And(BugAffectsPerson.person == Person.id,

                       BugAffectsPerson.bug == Bug.id,

                       BugAffectsPerson.affected,

                       Bug.duplicateof == self.id)))

    @property

    def users_affected_with_dupes(self):

        """See `IBug`."""

        return Store.of(self).find(

            Person, Person.id.is_in(self.user_ids_affected_with_dupes))

    @property

    def users_affected_count_with_dupes(self):

        """See `IBug`."""

        return self.users_affected_with_dupes.count()

    @property

    def other_users_affected_count_with_dupes(self):

        """See `IBug`."""

        current_user = getUtility(ILaunchBag).user

        if not current_user:

            return self.users_affected_count_with_dupes

        return self.users_affected_with_dupes.find(

            Person.id != current_user.id).count()

    @property

    def indexed_messages(self):

        """See `IMessageTarget`."""

        # Note that this is a decorated result set, so will cache its

        # value (in the absence of slices)

        return self._indexed_messages(include_content=True)

    def _indexed_messages(self, include_content=False, include_parents=True):

        """Get the bugs messages, indexed.

        :param include_content: If True retrieve the content for the messages

            too.

        :param include_parents: If True retrieve the object for parent

            messages too. If False the parent attribute will be *forced* to

            None to reduce database lookups.

        """

        # Make all messages be 'in' the main bugtask.

        inside = self.default_bugtask

        store = Store.of(self)

        message_by_id = {}

        to_messages = lambda rows: [row[0] for row in rows]

        def eager_load_owners(messages):

            # Because we may have multiple owners, we spend less time

            # in storm with very large bugs by not joining and instead

            # querying a second time. If this starts to show high db

            # time, we can left outer join instead.

            owner_ids = set(message.ownerID for message in messages)

            owner_ids.discard(None)

            if not owner_ids:

                return

            list(store.find(Person, Person.id.is_in(owner_ids)))

        def eager_load_content(messages):

            # To avoid the complexity of having multiple rows per

            # message, or joining in the database (though perhaps in

            # future we should do that), we do a single separate query

            # for the message content.

            message_ids = set(message.id for message in messages)

            chunks = store.find(

                MessageChunk, MessageChunk.messageID.is_in(message_ids))

            chunks.order_by(MessageChunk.id)

            chunk_map = {}

            for chunk in chunks:

                message_chunks = chunk_map.setdefault(chunk.messageID, [])

                message_chunks.append(chunk)

            for message in messages:

                if message.id not in chunk_map:

                    continue

                cache = get_property_cache(message)

                cache.text_contents = Message.chunks_text(

                    chunk_map[message.id])

        def eager_load(rows):

            messages = to_messages(rows)

            eager_load_owners(messages)

            if include_content:

                eager_load_content(messages)

        def index_message(row):

            # convert row to an IndexedMessage

            if include_parents:

                message, parent, bugmessage = row

                if parent is not None:

                    # If there is an IndexedMessage available as parent, use

                    # that to reduce on-demand parent lookups.

                    parent = message_by_id.get(parent.id, parent)

            else:

                message, bugmessage = row

                parent = None  # parent attribute is not going to be accessed.

            index = bugmessage.index

            result = IndexedMessage(message, inside, index, parent)

            if include_parents:

                # This message may be the parent for another: stash it to

                # permit use.

                message_by_id[message.id] = result

            return result

        if include_parents:

            ParentMessage = ClassAlias(Message)

            ParentBugMessage = ClassAlias(BugMessage)

            tables = [

                Message,

                Join(

                    BugMessage,

                    BugMessage.messageID == Message.id),

                LeftJoin(

                    Join(

                        ParentMessage,

                        ParentBugMessage,

                        ParentMessage.id == ParentBugMessage.messageID),

                    And(

                        Message.parent == ParentMessage.id,

                        ParentBugMessage.bugID == self.id)),

                ]

            results = store.using(*tables).find(

                (Message, ParentMessage, BugMessage),

                BugMessage.bugID == self.id,

                )

        else:

            lookup = Message, BugMessage

            results = store.find(lookup,

                BugMessage.bugID == self.id,

                BugMessage.messageID == Message.id,

                )

        results.order_by(BugMessage.index)

        return DecoratedResultSet(results, index_message,

            pre_iter_hook=eager_load)

    @property

    def displayname(self):

        """See `IBug`."""

        dn = 'Bug #%d' % self.id

        if self.name:

            dn += ' (' + self.name + ')'

        return dn

    @cachedproperty

    def bugtasks(self):

        """See `IBug`."""

        # \o/ circular imports.

        from lp.registry.model.distribution import Distribution

        from lp.registry.model.distroseries import DistroSeries

        from lp.registry.model.product import Product

        from lp.registry.model.productseries import ProductSeries

        from lp.registry.model.sourcepackagename import SourcePackageName