10
from bzrlib.transport import get_transport
11
from bzrlib.branch import Branch
13
7
from config import BZR_IN_QUEUE
14
8
from config import BZR_OUT_QUEUE
15
9
from config import CHARM_DIR
16
from config import settings
10
from worker import fetch_branches
17
11
from utils import get_queues
19
from charmworld.models import getconnection
20
from charmworld.models import getdb
21
from charmworld.models import getfs
22
from charmworld.models import CharmFileSet
23
from charmworld.utils import quote_key
25
log = logging.getLogger("charm.bzr")
28
def add_files(charm_data):
29
charm_data['files'] = dict([
30
(quote_key(cfile.filename), dict(cfile)) for cfile in
31
store_branch_files(charm_data)
35
def fetch_branch(root_dir, charm_data, retry=True):
36
"""Fetch a branch from bzr, and augment charm data."""
37
branch_dir = os.path.abspath(
38
str(os.path.join(root_dir,
42
charm_data["bname"])))
44
if not os.path.exists(os.path.dirname(branch_dir)):
45
os.makedirs(os.path.dirname(branch_dir))
47
# Store the branch directory
48
charm_data["branch_dir"] = branch_dir
50
if not os.path.exists(branch_dir):
51
# The branch has never been seen before. Original branch.
52
log.info("Branching charm lp:%s", charm_data["branch_spec"])
53
subprocess.check_output(
54
["/usr/bin/bzr", "co", "-q",
55
"lp:%s" % charm_data["branch_spec"], branch_dir])
56
charm_data = add_files(charm_data)
59
# It exists and check if it's the latest revision already.
60
log.debug("Existing charm from lp:%s", charm_data["branch_spec"])
61
transport = get_transport(branch_dir)
62
branch = Branch.open_from_transport(transport)
63
cur_rev_id = branch.last_revision()
64
if cur_rev_id == charm_data['commit']:
65
charm_data = add_files(charm_data)
66
log.debug("Already up to date lp:%s", charm_data["branch_spec"])
69
log.debug("Updating branch lp:%s", charm_data["branch_spec"])
72
# It exists, but it's not up to date, so update it.
73
subprocess.check_output(
74
["/usr/bin/bzr", "update", "-q"],
76
stderr=subprocess.STDOUT)
77
charm_data = add_files(charm_data)
78
except subprocess.CalledProcessError:
79
# It existed but the update failed for some reason. Just strip the
80
# whole tree and start over with the above.
82
shutil.rmtree(branch_dir)
83
return fetch_branch(root_dir, charm_data, retry=False)
87
def store_branch_files(charm_data, db=None):
88
"""Process the bzr branch for files that need to be stored in gridfs."""
89
log.info('Storing files of branch into gridfs')
91
connection = getconnection(settings)
92
db = getdb(connection, settings.get('mongo.database'))
94
filestore = CharmFileSet.save_files(
95
fs, charm_data, charm_data['branch_dir'])
97
log.info('Completed gridfs storage.')
100
def fetch_branches(root_dir, queue, scan_queue):
106
charm_data = item.payload
107
fetch_branch(root_dir, charm_data)
108
except Exception as e:
109
log.exception("bzr error error on %s: %s", charm_data, str(e))
110
# XXX j.c.sackett Jan 30, 2012 Bug:1110539 The charm here is being
111
# annotated with error data, but since the charm falls out of
112
# scope when this iteration of the loop ends, nothing is being
113
# down with it--it's not stored, or put on the out queue.
114
charm_data["error_stage"] = "bzr"
115
if hasattr(e, 'output'):
116
charm_data["error"] = str(e.output)
119
scan_queue.put(charm_data)
121
# Remove the job from the input queue.
14
if __name__ == '__main__':
126
15
logging.basicConfig(
127
16
level=logging.WARNING,
128
17
format="%(asctime)s: %(name)s@%(levelname)s: %(message)s")
129
18
in_queue, out_queue = get_queues(BZR_IN_QUEUE, BZR_OUT_QUEUE)
130
19
fetch_branches(CHARM_DIR, in_queue, out_queue)
133
if __name__ == '__main__':