~james-w/udd/management-commands

« back to all changes in this revision

Viewing changes to udd/icommon.py

Committer: James Westby
Date: 2011-12-13 21:09:23 UTC
mfrom: (557.1.1 drop_email_failures)
Revision ID: james.westby@canonical.com-20111213210923-tfrirlx3xbwmi70u

Merged drop_email_failures into management-commands.

files added:
udd/tests/__init__.py

udd/tests/test_categorise_failures.py

udd/tests/test_count_outstanding_jobs.py

udd/tests/test_graph_failures.py

udd/tests/test_list_packages.py

udd/tests/test_logrotate.py

udd/tests/test_mass_import.py

files removed:
udd/paths.py

files renamed:
udd/tests/__init__.py => udd/tests/test_icommon.py

files modified:
README

bin/import-package

pkgimport.conf

selftest.py

udd/icommon.py

udd/iconfig.py

udd/lpapi.py

udd/scripts/add_import_jobs.py

udd/scripts/categorise_failures.py

udd/scripts/count_outstanding_jobs.py

udd/scripts/graph_failures.py

udd/scripts/import_package.py

udd/scripts/list_packages.py

udd/scripts/logrotate.py

udd/scripts/mass_import.py

udd/scripts/requeue_package.py

udd/scripts/show_failure.py

udd/tests/test_config.py

Show diffs side-by-side

added added

removed removed

udd/icommon.py

iconfig,

lpapi,

)

from udd.paths import paths

# Not ideal to declare this here but this is used by several local functions

# below. Some way to share the config is needed -- vila 2011-12-07

conf = iconfig.ImporterStack()

sqlite_timeout = 30

running_sentinel = "Apparently the supervisor died\n"

129

132

return suite

130

133

131

134

132

def _lock_path(path):

133

ensure_directory(os.path.dirname(path))

135

def lock_path(dir_name, file_name):

136

ensure_directory(dir_name)

137

path = os.path.join(dir_name, file_name)

134

138

f = open(path, "wb")

135

139

try:

136

140

fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)

140

144

return None

141

145

raise

142

146

143

def lock_package(package):

144

path = os.path.join(paths.locks_dir, package)

145

return _lock_path(path)

146

147

148

def lock_main():

149

path = os.path.join(paths.base_dir, "main_lock")

150

return _lock_path(path)

151

152

153

def lock_list_packages():

154

path = os.path.join(paths.base_dir, "list_packages")

155

return _lock_path(path)

156

157

158

def lock_add_import_jobs():

159

path = os.path.join(paths.base_dir, "add_import_jobs")

160

return _lock_path(path)

161

162

163

def lock_categorise_failures():

164

path = os.path.join(paths.base_dir, "categorise_failures")

165

return _lock_path(path)

166

167

168

def lock_email_failures():

169

path = os.path.join(paths.base_dir, "email_failures")

170

return _lock_path(path)

171

172

173

def lock_update_lists():

174

path = os.path.join(paths.base_dir, "update_lists")

175

return _lock_path(path)

176

177

147

178

148

class PackageToImport(object):

179

149

"""This object is analogous to (in Soyuz terminology) a source package

338

308

status_db, earliest_date=datetime.now() - timedelta(days=10))

339

309

# We want all the new Published records since the last run

340

310

publications = set()

341

config = iconfig.Iconfig()

342

distributions = config.get('pkgimport.distributions')

311

distributions = conf.get('pi.distributions')

343

312

for distro in distributions:

344

313

for spph in iter_published_sources(lp, lp.distributions[distro],

345

314

status='Published',

346

since=last_known_published.isoformat()):

315

since=

316

last_known_published.isoformat()):

347

317

publications.add(spph)

348

318

import_from_publications(status_db, publications, last_known_published)

349

319

1161

1131

self.suffix = suffix

1162

1132

self._series_cache = {}

1163

1133

self._sp_cache = {}

1164

self._config_stack = iconfig.Iconfig()

1134

self._config_stack = iconfig.PackageStack(package)

1165

1135

1166

# TODO: When udd requires bzr 2.4, _get_host_name and _get_branch_owner

1167

# can be registered as standard options and then can have the defaults

1168

# declared there.

1169

1136

def _get_host_name(self):

1170

"""Return the configured or default branch hostname"""

1171

return self._config_stack.get('udd.branch_hostname') or 'bazaar.launchpad.net'

1137

"""Return the configured hostname"""

1138

return self._config_stack.get('udd.branch_hostname')

1172

1139

1173

1140

def _get_branch_owner(self):

1174

"""Return the configured or default branch owner, as a Launchpad or unix username."""

1175

return self._config_stack.get('udd.branch_owner') or 'ubuntu-branches'

1141

"""Return the configured owner as a Launchpad or unix username."""

1142

return self._config_stack.get('udd.branch_owner')

1176

1143

1177

1144

def _branch_location(self, distro, release, pocket, readonly=False):

1178

1145

"""Returns a tuple (branch_url, is_marked_official)"""

1191

1158

return ("%s%s/~%s/%s/"

1192

1159

"%s/%s/%s%s" % (

1193

1160

scheme, host, self._get_branch_owner(),

1194

distro, release, self.package, suite, self.suffix), False)

1161

distro, release, self.package, suite, self.suffix),

1162

False)

1195

1163

else:

1196

1164

path = urlparse.urlparse(lp_branch.self_link).path

1197

1165

lp_branch_path = path[path[1:].index('/')+1:]

1262

1230

raise

1263

1231

1264

1232

def _set_reviewer(self, branch, _retry_count=0):

1265

ubuntu_dev = lpapi.lp_call(operator.getitem, self.lp.people, 'ubuntu-dev')

1233

ubuntu_dev = lpapi.lp_call(operator.getitem, self.lp.people,

1234

'ubuntu-dev')

1266

1235

branch.reviewer = ubuntu_dev

1267

1236

try:

1268

1237

lpapi.lp_call(branch.lp_save)

1269

1238

except HTTPError, e:

1270

1239

if e.response.status == 412 and _retry_count < 3:

1271

1240

time.sleep(1)

1272

self._set_reviewer(lpapi.lp_call(self.lp.load, branch.self_link),

1273

_retry_count=_retry_count+1)

1241

self._set_reviewer(lpapi.lp_call(self.lp.load,

1242

branch.self_link),

1243

_retry_count=_retry_count+1)

1274

1244

else:

1275

1245

raise

1276

1246

1289

1259

lp_series = None

1290

1260

else:

1291

1261

lp_distro = self.lp._udd_distributions[distro]

1292

lp_series = lpapi.lp_call(lp_distro.getSeries, name_or_version=release)

1262

lp_series = lpapi.lp_call(lp_distro.getSeries,

1263

name_or_version=release)

1293

1264

self._series_cache[(distro, release)] = lp_series

1294

1265

return lp_series

1295

1266

1341

1312

1342

1313

def __init__(self, package, lp, suffix):

1343

1314

BranchStore.__init__(self, package, lp, suffix)

1344

self.branches_dir = os.path.join(paths.localbranches_dir, package)

1315

self.branches_dir = os.path.join(conf.get('pi.localbranches_dir'),

1316

package)

1345

1317

if not os.path.exists(self.branches_dir):

1346

1318

os.makedirs(self.branches_dir)

1347

1319

format = bzrdir.format_registry.make_bzrdir('2a')

1401

1373

distro, suite = self._translate_suite(distro, suite)

1402

1374

now = datetime.utcnow()

1403

1375

now_str = now.strftime("%Y%m%d%H%M")

1404

return os.path.join(paths.updates_dir, self.package,

1376

return os.path.join(conf.get('pi.updates_dir'), self.package,

1405

1377

suite + "-%s" % now_str)

1406

1378

1407

1379

1462

1434

1463

1435

1464

1436

def generate_debian_diffs(lp, temp_dir, package, bstore,

1465

possible_transports=None):

1466

ensure_directory(paths.debian_diffs_dir)

1467

target_path = os.path.join(paths.debian_diffs_dir, package)

1437

possible_transports=None):

1438

debian_diffs_dir = conf.get('pi.debian_diffs_dir')

1439

ensure_directory(debian_diffs_dir)

1440

target_path = os.path.join(debian_diffs_dir, package)

1468

1441

debian_b, ubuntu_b, _ = get_debian_ubuntu_branches(lp, temp_dir, package,

1469

1442

bstore, possible_transports=possible_transports)

1470

1443

if debian_b is None:

1485

1458

1486

1459

1487

1460

def possibly_generate_debian_diffs(lp, temp_dir, package, bstore,

1488

possible_transports=None):

1489

if os.path.exists(os.path.join(paths.debian_diffs_dir, package)):

1461

possible_transports=None):

1462

debian_diffs_dir = conf.get('pi.debian_diffs_dir')

1463

if os.path.exists(os.path.join(debian_diffs_dir, package)):

1490

1464

return

1491

1465

generate_debian_diffs(lp, temp_dir, package, bstore,

1492

possible_transports=possible_transports)

1466

possible_transports=possible_transports)

1493

1467

1494

1468

1495

1469

def generate_ubuntu_merges(lp, temp_dir, package, bstore,

1496

possible_transports=None):

1497

debian_b, ubuntu_b, ubuntu_tree = get_debian_ubuntu_branches(lp,

1498

temp_dir, package, bstore, ensure_ubuntu_local=True,

1499

possible_transports=possible_transports)

1500

ensure_directory(paths.ubuntu_merge_dir)

1501

target_path = os.path.join(paths.ubuntu_merge_dir, package)

1470

possible_transports=None):

1471

debian_b, ubuntu_b, ubuntu_tree = get_debian_ubuntu_branches(

1472

lp, temp_dir, package, bstore, ensure_ubuntu_local=True,

1473

possible_transports=possible_transports)

1474

ubuntu_merges_dir = conf.get('pi.ubuntu_merges_dir')

1475

ensure_directory(ubuntu_merges_dir)

1476

target_path = os.path.join(ubuntu_merges_dir, package)

1502

1477

if debian_b is None:

1503

1478

if os.path.exists(target_path):

1504

1479

os.unlink(target_path)

1555

1530

1556

1531

1557

1532

def possibly_generate_ubuntu_merges(lp, temp_dir, package, bstore,

1558

possible_transports=None):

1559

if os.path.exists(os.path.join(paths.ubuntu_merge_dir, package)):

1533

possible_transports=None):

1534

ubuntu_merges_dir = conf.get('pi.ubuntu_merges_dir')

1535

if os.path.exists(os.path.join(ubuntu_merges_dir, package)):

1560

1536

return

1561

1537

generate_ubuntu_merges(lp, temp_dir, package, bstore,

1562

possible_transports=possible_transports)

1538

possible_transports=possible_transports)

1563

1539

1564

1540

1565

1541

def find_earliest_merge(b, revid):

1572

1548

1573

1549

1574

1550

def load_explanations():

1551

explanations_file = conf.get('pi.explanations_file')

1575

1552

explanations = {}

1576

if not os.path.exists(paths.explanations_file):

1553

if not os.path.exists(explanations_file):

1577

1554

return explanations

1578

1555

1579

1556

reasons = []

1583

1560

for reason in reasons:

1584

1561

explanations[reason] = explanation

1585

1562

1586

with open(paths.explanations_file) as f:

1563

with open(explanations_file) as f:

1587

1564

for line in f:

1588

1565

# Each line is either a reason (the exception followed by a

1589

1566

# traceback signature) or part of an explanation

1609

1586

1610

1587

1611

1588

def update_list(release, component, source_url):

1612

target_dir = os.path.join(paths.lists_dir, "dists", release, component,

1613

"source")

1589

lists_dir = conf.get('pi.lists_dir')

1590

target_dir = os.path.join(lists_dir, 'dists', release, component, "source")

1614

1591

ensure_directory(target_dir)

1615

1592

target_path = os.path.join(target_dir, "Sources.gz")

1616

1593

target_lcfile = target_path + ".lc"

Older »