~jcsackett/charmworld/bac-tag-constraints

« back to all changes in this revision

Viewing changes to charmworld/jobs/bzr.py

  • Committer: Aaron Bentley
  • Date: 2013-02-12 18:57:02 UTC
  • mfrom: (149 charmworld)
  • mto: This revision was merged to the branch mainline in revision 150.
  • Revision ID: aaron@canonical.com-20130212185702-9gnf40ao17a7uw7v
Merged trunk into mongo-urls.

Show diffs side-by-side

added added

removed removed

Lines of Context:
3
3
# the file LICENSE).
4
4
 
5
5
import logging
6
 
import os
7
 
import subprocess
8
 
import shutil
9
 
 
10
 
from bzrlib.transport import get_transport
11
 
from bzrlib.branch import Branch
12
6
 
13
7
from config import BZR_IN_QUEUE
14
8
from config import BZR_OUT_QUEUE
15
9
from config import CHARM_DIR
16
 
from config import settings
 
10
from worker import fetch_branches
17
11
from utils import get_queues
18
12
 
19
 
from charmworld.models import getconnection
20
 
from charmworld.models import getdb
21
 
from charmworld.models import getfs
22
 
from charmworld.models import CharmFileSet
23
 
from charmworld.utils import quote_key
24
 
 
25
 
log = logging.getLogger("charm.bzr")
26
 
 
27
 
 
28
 
def add_files(charm_data):
29
 
    charm_data['files'] = dict([
30
 
        (quote_key(cfile.filename), dict(cfile)) for cfile in
31
 
        store_branch_files(charm_data)
32
 
    ])
33
 
    return charm_data
34
 
 
35
 
def fetch_branch(root_dir, charm_data, retry=True):
36
 
    """Fetch a branch from bzr, and augment charm data."""
37
 
    branch_dir = os.path.abspath(
38
 
        str(os.path.join(root_dir,
39
 
                         charm_data["series"],
40
 
                         charm_data["owner"],
41
 
                         charm_data["name"],
42
 
                         charm_data["bname"])))
43
 
 
44
 
    if not os.path.exists(os.path.dirname(branch_dir)):
45
 
        os.makedirs(os.path.dirname(branch_dir))
46
 
 
47
 
    # Store the branch directory
48
 
    charm_data["branch_dir"] = branch_dir
49
 
 
50
 
    if not os.path.exists(branch_dir):
51
 
        # The branch has never been seen before. Original branch.
52
 
        log.info("Branching charm lp:%s", charm_data["branch_spec"])
53
 
        subprocess.check_output(
54
 
            ["/usr/bin/bzr", "co", "-q",
55
 
             "lp:%s" % charm_data["branch_spec"], branch_dir])
56
 
        charm_data = add_files(charm_data)
57
 
        return
58
 
    else:
59
 
        # It exists and check if it's the latest revision already.
60
 
        log.debug("Existing charm from lp:%s", charm_data["branch_spec"])
61
 
        transport = get_transport(branch_dir)
62
 
        branch = Branch.open_from_transport(transport)
63
 
        cur_rev_id = branch.last_revision()
64
 
        if cur_rev_id == charm_data['commit']:
65
 
            charm_data = add_files(charm_data)
66
 
            log.debug("Already up to date lp:%s", charm_data["branch_spec"])
67
 
            return
68
 
 
69
 
    log.debug("Updating branch lp:%s", charm_data["branch_spec"])
70
 
 
71
 
    try:
72
 
        # It exists, but it's not up to date, so update it.
73
 
        subprocess.check_output(
74
 
            ["/usr/bin/bzr", "update", "-q"],
75
 
            cwd=branch_dir,
76
 
            stderr=subprocess.STDOUT)
77
 
        charm_data = add_files(charm_data)
78
 
    except subprocess.CalledProcessError:
79
 
        # It existed but the update failed for some reason. Just strip the
80
 
        # whole tree and start over with the above.
81
 
        if retry:
82
 
            shutil.rmtree(branch_dir)
83
 
            return fetch_branch(root_dir, charm_data, retry=False)
84
 
        raise
85
 
 
86
 
 
87
 
def store_branch_files(charm_data, db=None):
88
 
    """Process the bzr branch for files that need to be stored in gridfs."""
89
 
    log.info('Storing files of branch into gridfs')
90
 
    if db is None:
91
 
        connection = getconnection(settings)
92
 
        db = getdb(connection, settings.get('mongo.database'))
93
 
    fs = getfs(db)
94
 
    filestore = CharmFileSet.save_files(
95
 
        fs, charm_data, charm_data['branch_dir'])
96
 
    return filestore
97
 
    log.info('Completed gridfs storage.')
98
 
 
99
 
 
100
 
def fetch_branches(root_dir, queue, scan_queue):
101
 
    while 1:
102
 
        item = queue.next()
103
 
        if not item:
104
 
            return
105
 
        try:
106
 
            charm_data = item.payload
107
 
            fetch_branch(root_dir, charm_data)
108
 
        except Exception as e:
109
 
            log.exception("bzr error error on %s: %s", charm_data, str(e))
110
 
            # XXX j.c.sackett Jan 30, 2012 Bug:1110539 The charm here is being
111
 
            # annotated with error data, but since the charm falls out of
112
 
            # scope when this iteration of the loop ends, nothing is being
113
 
            # down with it--it's not stored, or put on the out queue.
114
 
            charm_data["error_stage"] = "bzr"
115
 
            if hasattr(e, 'output'):
116
 
                charm_data["error"] = str(e.output)
117
 
            continue
118
 
        else:
119
 
            scan_queue.put(charm_data)
120
 
        finally:
121
 
            # Remove the job from the input queue.
122
 
            item.complete()
123
 
 
124
 
 
125
 
def main():
 
13
 
 
14
if __name__ == '__main__':
126
15
    logging.basicConfig(
127
16
        level=logging.WARNING,
128
17
        format="%(asctime)s: %(name)s@%(levelname)s: %(message)s")
129
18
    in_queue, out_queue = get_queues(BZR_IN_QUEUE, BZR_OUT_QUEUE)
130
19
    fetch_branches(CHARM_DIR, in_queue, out_queue)
131
 
 
132
 
 
133
 
if __name__ == '__main__':
134
 
    main()