40
from udd.paths import paths
42
# Not ideal to declare this here but this is used by several local functions
43
# below. Some way to share the config is needed -- vila 2011-12-07
44
conf = iconfig.ImporterStack()
43
46
sqlite_timeout = 30
44
47
running_sentinel = "Apparently the supervisor died\n"
132
def _lock_path(path):
133
ensure_directory(os.path.dirname(path))
135
def lock_path(dir_name, file_name):
136
ensure_directory(dir_name)
137
path = os.path.join(dir_name, file_name)
134
138
f = open(path, "wb")
136
140
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
143
def lock_package(package):
144
path = os.path.join(paths.locks_dir, package)
145
return _lock_path(path)
149
path = os.path.join(paths.base_dir, "main_lock")
150
return _lock_path(path)
153
def lock_list_packages():
154
path = os.path.join(paths.base_dir, "list_packages")
155
return _lock_path(path)
158
def lock_add_import_jobs():
159
path = os.path.join(paths.base_dir, "add_import_jobs")
160
return _lock_path(path)
163
def lock_categorise_failures():
164
path = os.path.join(paths.base_dir, "categorise_failures")
165
return _lock_path(path)
168
def lock_email_failures():
169
path = os.path.join(paths.base_dir, "email_failures")
170
return _lock_path(path)
173
def lock_update_lists():
174
path = os.path.join(paths.base_dir, "update_lists")
175
return _lock_path(path)
178
148
class PackageToImport(object):
179
149
"""This object is analogous to (in Soyuz terminology) a source package
338
308
status_db, earliest_date=datetime.now() - timedelta(days=10))
339
309
# We want all the new Published records since the last run
340
310
publications = set()
341
config = iconfig.Iconfig()
342
distributions = config.get('pkgimport.distributions')
311
distributions = conf.get('pi.distributions')
343
312
for distro in distributions:
344
313
for spph in iter_published_sources(lp, lp.distributions[distro],
345
314
status='Published',
346
since=last_known_published.isoformat()):
316
last_known_published.isoformat()):
347
317
publications.add(spph)
348
318
import_from_publications(status_db, publications, last_known_published)
1161
1131
self.suffix = suffix
1162
1132
self._series_cache = {}
1163
1133
self._sp_cache = {}
1164
self._config_stack = iconfig.Iconfig()
1134
self._config_stack = iconfig.PackageStack(package)
1166
# TODO: When udd requires bzr 2.4, _get_host_name and _get_branch_owner
1167
# can be registered as standard options and then can have the defaults
1169
1136
def _get_host_name(self):
1170
"""Return the configured or default branch hostname"""
1171
return self._config_stack.get('udd.branch_hostname') or 'bazaar.launchpad.net'
1137
"""Return the configured hostname"""
1138
return self._config_stack.get('udd.branch_hostname')
1173
1140
def _get_branch_owner(self):
1174
"""Return the configured or default branch owner, as a Launchpad or unix username."""
1175
return self._config_stack.get('udd.branch_owner') or 'ubuntu-branches'
1141
"""Return the configured owner as a Launchpad or unix username."""
1142
return self._config_stack.get('udd.branch_owner')
1177
1144
def _branch_location(self, distro, release, pocket, readonly=False):
1178
1145
"""Returns a tuple (branch_url, is_marked_official)"""
1191
1158
return ("%s%s/~%s/%s/"
1192
1159
"%s/%s/%s%s" % (
1193
1160
scheme, host, self._get_branch_owner(),
1194
distro, release, self.package, suite, self.suffix), False)
1161
distro, release, self.package, suite, self.suffix),
1196
1164
path = urlparse.urlparse(lp_branch.self_link).path
1197
1165
lp_branch_path = path[path[1:].index('/')+1:]
1264
1232
def _set_reviewer(self, branch, _retry_count=0):
1265
ubuntu_dev = lpapi.lp_call(operator.getitem, self.lp.people, 'ubuntu-dev')
1233
ubuntu_dev = lpapi.lp_call(operator.getitem, self.lp.people,
1266
1235
branch.reviewer = ubuntu_dev
1268
1237
lpapi.lp_call(branch.lp_save)
1269
1238
except HTTPError, e:
1270
1239
if e.response.status == 412 and _retry_count < 3:
1272
self._set_reviewer(lpapi.lp_call(self.lp.load, branch.self_link),
1273
_retry_count=_retry_count+1)
1241
self._set_reviewer(lpapi.lp_call(self.lp.load,
1243
_retry_count=_retry_count+1)
1289
1259
lp_series = None
1291
1261
lp_distro = self.lp._udd_distributions[distro]
1292
lp_series = lpapi.lp_call(lp_distro.getSeries, name_or_version=release)
1262
lp_series = lpapi.lp_call(lp_distro.getSeries,
1263
name_or_version=release)
1293
1264
self._series_cache[(distro, release)] = lp_series
1294
1265
return lp_series
1342
1313
def __init__(self, package, lp, suffix):
1343
1314
BranchStore.__init__(self, package, lp, suffix)
1344
self.branches_dir = os.path.join(paths.localbranches_dir, package)
1315
self.branches_dir = os.path.join(conf.get('pi.localbranches_dir'),
1345
1317
if not os.path.exists(self.branches_dir):
1346
1318
os.makedirs(self.branches_dir)
1347
1319
format = bzrdir.format_registry.make_bzrdir('2a')
1401
1373
distro, suite = self._translate_suite(distro, suite)
1402
1374
now = datetime.utcnow()
1403
1375
now_str = now.strftime("%Y%m%d%H%M")
1404
return os.path.join(paths.updates_dir, self.package,
1376
return os.path.join(conf.get('pi.updates_dir'), self.package,
1405
1377
suite + "-%s" % now_str)
1464
1436
def generate_debian_diffs(lp, temp_dir, package, bstore,
1465
possible_transports=None):
1466
ensure_directory(paths.debian_diffs_dir)
1467
target_path = os.path.join(paths.debian_diffs_dir, package)
1437
possible_transports=None):
1438
debian_diffs_dir = conf.get('pi.debian_diffs_dir')
1439
ensure_directory(debian_diffs_dir)
1440
target_path = os.path.join(debian_diffs_dir, package)
1468
1441
debian_b, ubuntu_b, _ = get_debian_ubuntu_branches(lp, temp_dir, package,
1469
1442
bstore, possible_transports=possible_transports)
1470
1443
if debian_b is None:
1487
1460
def possibly_generate_debian_diffs(lp, temp_dir, package, bstore,
1488
possible_transports=None):
1489
if os.path.exists(os.path.join(paths.debian_diffs_dir, package)):
1461
possible_transports=None):
1462
debian_diffs_dir = conf.get('pi.debian_diffs_dir')
1463
if os.path.exists(os.path.join(debian_diffs_dir, package)):
1491
1465
generate_debian_diffs(lp, temp_dir, package, bstore,
1492
possible_transports=possible_transports)
1466
possible_transports=possible_transports)
1495
1469
def generate_ubuntu_merges(lp, temp_dir, package, bstore,
1496
possible_transports=None):
1497
debian_b, ubuntu_b, ubuntu_tree = get_debian_ubuntu_branches(lp,
1498
temp_dir, package, bstore, ensure_ubuntu_local=True,
1499
possible_transports=possible_transports)
1500
ensure_directory(paths.ubuntu_merge_dir)
1501
target_path = os.path.join(paths.ubuntu_merge_dir, package)
1470
possible_transports=None):
1471
debian_b, ubuntu_b, ubuntu_tree = get_debian_ubuntu_branches(
1472
lp, temp_dir, package, bstore, ensure_ubuntu_local=True,
1473
possible_transports=possible_transports)
1474
ubuntu_merges_dir = conf.get('pi.ubuntu_merges_dir')
1475
ensure_directory(ubuntu_merges_dir)
1476
target_path = os.path.join(ubuntu_merges_dir, package)
1502
1477
if debian_b is None:
1503
1478
if os.path.exists(target_path):
1504
1479
os.unlink(target_path)
1557
1532
def possibly_generate_ubuntu_merges(lp, temp_dir, package, bstore,
1558
possible_transports=None):
1559
if os.path.exists(os.path.join(paths.ubuntu_merge_dir, package)):
1533
possible_transports=None):
1534
ubuntu_merges_dir = conf.get('pi.ubuntu_merges_dir')
1535
if os.path.exists(os.path.join(ubuntu_merges_dir, package)):
1561
1537
generate_ubuntu_merges(lp, temp_dir, package, bstore,
1562
possible_transports=possible_transports)
1538
possible_transports=possible_transports)
1565
1541
def find_earliest_merge(b, revid):
1583
1560
for reason in reasons:
1584
1561
explanations[reason] = explanation
1586
with open(paths.explanations_file) as f:
1563
with open(explanations_file) as f:
1588
1565
# Each line is either a reason (the exception followed by a
1589
1566
# traceback signature) or part of an explanation
1611
1588
def update_list(release, component, source_url):
1612
target_dir = os.path.join(paths.lists_dir, "dists", release, component,
1589
lists_dir = conf.get('pi.lists_dir')
1590
target_dir = os.path.join(lists_dir, 'dists', release, component, "source")
1614
1591
ensure_directory(target_dir)
1615
1592
target_path = os.path.join(target_dir, "Sources.gz")
1616
1593
target_lcfile = target_path + ".lc"