|
69
by James Westby
list_packages is a python script |
1 |
#!/usr/bin/python
|
2 |
||
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
3 |
import datetime |
|
66
by James Westby
Move the package listing in to another process as LP API isn't thread safe |
4 |
import os |
|
85
by James Westby
Catch and quieten KeyboardInterrupt in list_packages. |
5 |
import sys |
|
66
by James Westby
Move the package listing in to another process as LP API isn't thread safe |
6 |
|
7 |
from launchpadlib.errors import HTTPError |
|
8 |
||
|
155
by James Westby
Start adding backoff retry for LP calls. |
9 |
sys.path.insert(0, os.path.dirname(__file__)) |
10 |
import icommon |
|
|
66
by James Westby
Move the package listing in to another process as LP API isn't thread safe |
11 |
|
|
233
by James Westby
Make list_packages use get_lp() as well. |
12 |
try: |
13 |
lp = icommon.get_lp() |
|
|
72
by James Westby
Catch more HTTPErrors |
14 |
debian = lp.distributions['ubuntu'] |
15 |
ubuntu = lp.distributions['ubuntu'] |
|
16 |
d_archive = debian.main_archive |
|
17 |
u_archive = ubuntu.main_archive |
|
|
235
by James Westby
Save an HTTP round trip on every iteration of list_packages loop |
18 |
current_u_series_name = ubuntu.current_series.name |
|
72
by James Westby
Catch more HTTPErrors |
19 |
|
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
20 |
|
21 |
db = icommon.PackageDatabase(icommon.sqlite_package_file) |
|
22 |
||
23 |
last_update = db.last_update() |
|
24 |
if last_update is not None: |
|
25 |
last_update = last_update - datetime.timedelta(0, 86400) |
|
26 |
last_update = last_update.isoformat() |
|
|
72
by James Westby
Catch more HTTPErrors |
27 |
|
|
175
by James Westby
No need to do the current ubuntu series first, it's the first anyway. |
28 |
for s in ubuntu.series: |
|
237.1.97
by James Westby
Fix calling convention for lp_call |
29 |
collection = icommon.lp_call(icommon.call_with_limited_size, |
|
237.1.96
by James Westby
Limit size of all calls to getPublishedSources |
30 |
u_archive.getPublishedSources, |
|
155
by James Westby
Start adding backoff retry for LP calls. |
31 |
status="Published", |
|
237.1.97
by James Westby
Fix calling convention for lp_call |
32 |
distro_series=s, created_since_date=last_update) |
|
231
by James Westby
Fix iteration of large collections to not timeout LP. |
33 |
for publication in icommon.iterate_collection(collection): |
|
274
by James Westby
Make list_packages do something even when it doesn't complete a full run |
34 |
packages = {} |
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
35 |
if publication.source_package_name in packages: |
|
231
by James Westby
Fix iteration of large collections to not timeout LP. |
36 |
continue
|
|
235
by James Westby
Save an HTTP round trip on every iteration of list_packages loop |
37 |
if (s.name == current_u_series_name and |
|
231
by James Westby
Fix iteration of large collections to not timeout LP. |
38 |
publication.component_name == "main"): |
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
39 |
packages[publication.source_package_name] = 1 |
|
231
by James Westby
Fix iteration of large collections to not timeout LP. |
40 |
else: |
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
41 |
packages[publication.source_package_name] = 0 |
|
274
by James Westby
Make list_packages do something even when it doesn't complete a full run |
42 |
db.update_packages(packages) |
|
72
by James Westby
Catch more HTTPErrors |
43 |
for s in debian.series: |
|
237.1.97
by James Westby
Fix calling convention for lp_call |
44 |
collection = icommon.lp_call(icommon.call_with_limited_size, |
|
237.1.96
by James Westby
Limit size of all calls to getPublishedSources |
45 |
d_archive.getPublishedSources, |
|
155
by James Westby
Start adding backoff retry for LP calls. |
46 |
status="Pending", |
|
237.1.97
by James Westby
Fix calling convention for lp_call |
47 |
distro_series=s, created_since_date=last_update) |
|
231
by James Westby
Fix iteration of large collections to not timeout LP. |
48 |
for publication in icommon.iterate_collection(collection): |
|
274
by James Westby
Make list_packages do something even when it doesn't complete a full run |
49 |
packages = {} |
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
50 |
if publication.source_package_name in packages: |
|
231
by James Westby
Fix iteration of large collections to not timeout LP. |
51 |
continue
|
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
52 |
packages[publication.source_package_name] = 0 |
|
274
by James Westby
Make list_packages do something even when it doesn't complete a full run |
53 |
db.update_packages(packages) |
|
237.1.50
by James Westby
Move from a thread to list packges to the DB. |
54 |
|
|
72
by James Westby
Catch more HTTPErrors |
55 |
except HTTPError, e: |
|
108
by James Westby
Write to stderr from list_packages as stderr is redirected. |
56 |
sys.stderr.write(e.content + "\n") |
|
72
by James Westby
Catch more HTTPErrors |
57 |
raise
|
|
85
by James Westby
Catch and quieten KeyboardInterrupt in list_packages. |
58 |
except KeyboardInterrupt, e: |
|
108
by James Westby
Write to stderr from list_packages as stderr is redirected. |
59 |
sys.stderr.write("KeyboardInterrupt\n") |
|
85
by James Westby
Catch and quieten KeyboardInterrupt in list_packages. |
60 |
sys.exit(1) |