~maxb/udd/misc-tidying : contents of list_packages.py at revision 365

~maxb/udd/misc-tidying : (revision 365)

69 by James Westby list_packages is a python script	1	#!/usr/bin/python
69 by James Westby list_packages is a python script	2
237.1.50 by James Westby Move from a thread to list packges to the DB.	3	import datetime
66 by James Westby Move the package listing in to another process as LP API isn't thread safe	4	import os
85 by James Westby Catch and quieten KeyboardInterrupt in list_packages.	5	import sys
66 by James Westby Move the package listing in to another process as LP API isn't thread safe	6
	7	from launchpadlib.errors import HTTPError
	8
155 by James Westby Start adding backoff retry for LP calls.	9	sys.path.insert(0, os.path.dirname(__file__))
	10	import icommon
66 by James Westby Move the package listing in to another process as LP API isn't thread safe	11
333 by James Westby Add locking to cron jobs to prevent runaway problems when they are slow.	12	lock = icommon.lock_list_packages()
	13	if lock is None:
	14	print "Another instance of list_packages is already running."
	15	sys.exit(0)
233 by James Westby Make list_packages use get_lp() as well.	16	try:
	17	lp = icommon.get_lp()
72 by James Westby Catch more HTTPErrors	18	debian = lp.distributions['ubuntu']
	19	ubuntu = lp.distributions['ubuntu']
	20	d_archive = debian.main_archive
	21	u_archive = ubuntu.main_archive
235 by James Westby Save an HTTP round trip on every iteration of list_packages loop	22	current_u_series_name = ubuntu.current_series.name
72 by James Westby Catch more HTTPErrors	23
237.1.50 by James Westby Move from a thread to list packges to the DB.	24
	25	db = icommon.PackageDatabase(icommon.sqlite_package_file)
	26
	27	last_update = db.last_update()
	28	if last_update is not None:
	29	last_update = last_update - datetime.timedelta(0, 86400)
	30	last_update = last_update.isoformat()
72 by James Westby Catch more HTTPErrors	31
175 by James Westby No need to do the current ubuntu series first, it's the first anyway.	32	for s in ubuntu.series:
237.1.97 by James Westby Fix calling convention for lp_call	33	collection = icommon.lp_call(icommon.call_with_limited_size,
237.1.96 by James Westby Limit size of all calls to getPublishedSources	34	u_archive.getPublishedSources,
155 by James Westby Start adding backoff retry for LP calls.	35	status="Published",
237.1.97 by James Westby Fix calling convention for lp_call	36	distro_series=s, created_since_date=last_update)
231 by James Westby Fix iteration of large collections to not timeout LP.	37	for publication in icommon.iterate_collection(collection):
274 by James Westby Make list_packages do something even when it doesn't complete a full run	38	packages = {}
237.1.50 by James Westby Move from a thread to list packges to the DB.	39	if publication.source_package_name in packages:
231 by James Westby Fix iteration of large collections to not timeout LP.	40	continue
235 by James Westby Save an HTTP round trip on every iteration of list_packages loop	41	if (s.name == current_u_series_name and
231 by James Westby Fix iteration of large collections to not timeout LP.	42	publication.component_name == "main"):
237.1.50 by James Westby Move from a thread to list packges to the DB.	43	packages[publication.source_package_name] = 1
231 by James Westby Fix iteration of large collections to not timeout LP.	44	else:
237.1.50 by James Westby Move from a thread to list packges to the DB.	45	packages[publication.source_package_name] = 0
274 by James Westby Make list_packages do something even when it doesn't complete a full run	46	db.update_packages(packages)
72 by James Westby Catch more HTTPErrors	47	for s in debian.series:
237.1.97 by James Westby Fix calling convention for lp_call	48	collection = icommon.lp_call(icommon.call_with_limited_size,
237.1.96 by James Westby Limit size of all calls to getPublishedSources	49	d_archive.getPublishedSources,
155 by James Westby Start adding backoff retry for LP calls.	50	status="Pending",
237.1.97 by James Westby Fix calling convention for lp_call	51	distro_series=s, created_since_date=last_update)
231 by James Westby Fix iteration of large collections to not timeout LP.	52	for publication in icommon.iterate_collection(collection):
274 by James Westby Make list_packages do something even when it doesn't complete a full run	53	packages = {}
237.1.50 by James Westby Move from a thread to list packges to the DB.	54	if publication.source_package_name in packages:
231 by James Westby Fix iteration of large collections to not timeout LP.	55	continue
237.1.50 by James Westby Move from a thread to list packges to the DB.	56	packages[publication.source_package_name] = 0
274 by James Westby Make list_packages do something even when it doesn't complete a full run	57	db.update_packages(packages)
237.1.50 by James Westby Move from a thread to list packges to the DB.	58
72 by James Westby Catch more HTTPErrors	59	except HTTPError, e:
108 by James Westby Write to stderr from list_packages as stderr is redirected.	60	sys.stderr.write(e.content + "\n")
72 by James Westby Catch more HTTPErrors	61	raise
85 by James Westby Catch and quieten KeyboardInterrupt in list_packages.	62	except KeyboardInterrupt, e:
108 by James Westby Write to stderr from list_packages as stderr is redirected.	63	sys.stderr.write("KeyboardInterrupt\n")
85 by James Westby Catch and quieten KeyboardInterrupt in list_packages.	64	sys.exit(1)
333 by James Westby Add locking to cron jobs to prevent runaway problems when they are slow.	65	finally:
	66	lock.close()