1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
|
#!/usr/bin/python
import cgi
import datetime
import hashlib
import os
import shutil
import sys
import time
from udd import icommon
from udd import paths
html_head = '''<html>
<head>
<title>%(title)s</title>
<meta http-equiv="refresh" content="600">
<script src="jquery.min.js"></script>
<script src="jquery.flot.js"></script>
</head>
<body>
<p><a href="http://package-import.ubuntu.com/status/">All Ubuntu Overview</a>,
<a href="http://package-import.ubuntu.com/status/main.html">Ubuntu Main Overview</a></p>
<p>You are looking at information on the Bazaar importer system that serves <a
href="https://wiki.ubuntu.com/DistributedDevelopment">Ubuntu Distributed
Development</a>. It's hoped that you won't have to care about the existence of
this, but things don't always work out that way.</p>
<p>It is expected that uploads appear in the branches in less than 30 minutes,
if it takes longer then something is up and this page should tell you why. To
discuss issues or to ask for a package to be looked at please use our mailing
list <a
href="mailto:ubuntu-distributed-devel@lists.ubuntu.com">ubuntu-distributed-devel@lists.ubuntu.com</a>.</p>
<p>The scripts used to import the packages are available from a <a
href="https://code.launchpad.net/~udd/udd/import-scripts"> bzr branch. </a> If you encounter issues, please
file <a href="https://launchpad.net/udd/+filebug"> bugs</a>. If you're curious, a detailed explanation about how it works is documented in the <a
href="https://wiki.ubuntu.com/DistributedDevelopment/UnderTheHood">
UnderTheHood </a> part of the <a
href="https://wiki.ubuntu.com/DistributedDevelopment">
wiki. </a> </p>
<p>This page was last updated at <tt>''' + datetime.datetime.utcnow().isoformat() + ''' UTC.</tt></p>
'''
html_contents = '''<h3>Contents of this page</h3>
<ul>
<li><a href="#stats">Summary</a></li>
<li><a href="#latest">Latest failures</a></li>
<li><a href="#analysis">All failures</a></li>
</ul>
'''
html_tail = '''</body>
</html>
'''
def process_failures():
db = icommon.StatusDatabase(paths.sqlite_file)
return db.summarise_failures()
def package_link(package):
return '<a href="%s.html#%s">%s</a>' % (cgi.escape(package.name, True),
package.timestamp, cgi.escape(package.name))
def error_page(reason):
checksum = hashlib.md5()
checksum.update(reason)
return "%s.html" % cgi.escape(checksum.hexdigest(), True)
def error_link(reason):
return '<a href="%s">Packages with this issue</a>' % error_page(reason)
def list_old_pages():
old_pages = set()
for page in os.listdir(paths.web_status_dir):
path = os.path.join(paths.web_status_dir, page)
if not os.path.isdir(path):
old_pages.add(path)
return old_pages
def remove_obsolete_pages(old_pages, new_pages):
obsolete_pages = old_pages - new_pages
for path in obsolete_pages:
if os.path.exists(path) and not os.path.isdir(path):
os.unlink(path)
def write_individual_pages(reasons, explanations):
written_paths = set()
for reason in reasons:
for info in reasons[reason][:]:
assert not "/" in info.name
path = os.path.join(paths.web_status_dir, cgi.escape(info.name,
True) + ".html")
f = open(path, "wb")
try:
f.write(html_head % {"title":
"Failure reason for %s" % cgi.escape(info.name)})
f.write("<p>Failed at %s</p>\n" % info.timestamp)
if reason in explanations:
f.write("<p>%s</p>\n" % cgi.escape(explanations[reason]))
f.write("<p><pre>\n")
f.write(cgi.escape(info.raw_reason))
f.write("\n</pre></p>\n")
f.write("<div>%s</div>" % error_link(reason))
f.write(html_tail)
finally:
f.close()
written_paths.add(path)
path = os.path.join(paths.web_status_dir, error_page(reason))
f = open(path, "wb")
try:
f.write(html_head % {"title": "Failures of type"})
f.write("<ul>\n")
for package in reasons[reason][:]:
f.write("<li>%s</li>\n" % package_link(package))
f.write("</ul>\n")
f.write(html_tail)
finally:
f.close()
written_paths.add(path)
return written_paths
def find_info_where(package_info, clause):
ret = []
for info in package_info:
if clause(info):
ret.append(info)
return ret
def plural_form(format_str, quantity, singular, plural):
"""Take a format string which must contain one %d and one %s in that order,
a numeric quantity, and singular and plural forms relating to it. Compose
them appropriately."""
return format_str % (quantity, singular if quantity == 1 else plural)
def write_stats(f, package_info):
running = find_info_where(package_info, lambda x: x.running)
running.sort(key=lambda x: x.timestamp)
outstanding = find_info_where(package_info, lambda x: x.queued)
outstanding.sort(key=lambda x: x.name)
failures = find_info_where(package_info,
lambda x: not x.running and not x.queued)
fail_normal = find_info_where(failures, lambda x: not x.auto_retry)
fail_auto_retry = find_info_where(failures,
lambda x: x.auto_retry and not x.auto_retry_masked)
fail_masked = find_info_where(failures,
lambda x: x.auto_retry and x.auto_retry_masked)
f.write("<ul>\n")
f.write("<li>%d currently running\n<ul>\n" % len(running))
for package in running:
f.write("<li>%s (since %s)</li>" % (cgi.escape(package.name),
package.timestamp))
f.write("</ul>\n</li>\n")
f.write(plural_form("<li>%d outstanding %s\n<ul>\n<li>",
len(outstanding), "job", "jobs"))
for package in outstanding:
f.write(cgi.escape(package.name) + " ")
f.write("</li>\n</ul>\n</li>\n")
f.write(plural_form("<li>%d %s - of which:\n",
len(failures), "failure", "failures"))
f.write("<ul>\n")
f.write(plural_form("<li>%d normal %s</li>\n",
len(fail_normal), "failure", "failures"))
f.write(plural_form("<li>%d spurious %s pending auto-retry</li>\n",
len(fail_auto_retry), "failure", "failures"))
f.write(plural_form("<li>%d spurious %s repeated too many times</li>\n",
len(fail_masked), "failure", "failures"))
f.write("</ul>\n")
f.write("</li>\n")
f.write("</ul>\n")
def write_analyis_partial(f, keys, reasons, explanations, clause, text):
for reason in keys:
packages = reasons[reason]
subset = find_info_where(packages, clause)
if len(subset) > 0:
f.write("<li>%d packages %s with key %s\n<ul>\n"
% (len(subset), text, cgi.escape(reason)))
if reason in explanations:
f.write('<li><pre>%s</pre></li>\n'
% cgi.escape(explanations[reason]))
f.write('<li>')
subset.sort(key=lambda x: x.name)
for package in subset:
f.write(package_link(package) + " ")
f.write("</li>\n</ul>\n</li>\n")
def write_analysis(f, reasons, explanations):
keys = reasons.keys()
def compare(a, b):
return len(reasons[b]) - len(reasons[a])
keys.sort(cmp=compare)
write_analyis_partial(f, keys, reasons, explanations,
lambda x: x.auto_retry_masked, "failed too many times to retry")
write_analyis_partial(f, keys, reasons, explanations,
lambda x: not x.auto_retry, "failed")
write_analyis_partial(f, keys, reasons, explanations,
lambda x: x.auto_retry and not x.auto_retry_masked,
"spuriously failed")
def write_latest(f, package_info):
failed = find_info_where(package_info, lambda x: x.failure_count > 0)
failed.sort(key=lambda x: x.timestamp, reverse=True)
i = 0
for package in failed:
i += 1
if i > 50:
return
f.write("<li>%s\n<ul>\n" % package_link(package))
f.write("<li>failed at %s</li>\n" % package.timestamp)
f.write("<li>with key %s</li>\n" % cgi.escape(package.signature))
if package.auto_retry_masked:
f.write("<li>and failed too many times to retry</li>\n")
elif package.auto_retry:
f.write("<li>and will be retried automatically around %s</li>\n" % package.auto_retry_time)
f.write("</ul>\n</li>\n")
def write_overview_page(reasons, package_info, explanations):
written_paths = set()
path = os.path.join(paths.web_status_dir, "index.html")
f = open(path, "wb")
try:
f.write(html_head % {"title": "bzr import failures"})
f.write(html_contents)
f.write('<div id="stats">\n')
write_stats(f, package_info)
f.write('</div>\n<div id="latest">\n<h3>Latest 50 Failures:</h3>\n<ul>\n')
write_latest(f, package_info)
f.write('</ul>\n</div>\n')
f.write('</div>\n<h3>All Failures by Category</h3>\n<div id="analysis">\n<ul>\n')
write_analysis(f, reasons, explanations)
f.write('</ul>\n</div>\n')
db = icommon.HistoryDatabase(paths.sqlite_history_file)
write_graphs(f, db.get_counts())
f.write(html_tail)
finally:
f.close()
written_paths.add(path)
return written_paths
def get_main_packages():
db = icommon.PackageDatabase(paths.sqlite_package_file)
return db.list_packages_in_main()
def filter_packages(filter_set, reasons, package_info):
filtered_package_info = find_info_where(package_info, lambda x: x.name in filter_set)
filtered_reasons = {}
for sig in reasons:
for info in reasons[sig]:
if info in filtered_package_info:
filtered_reasons.setdefault(sig, [])
filtered_reasons[sig].append(info)
return filtered_reasons, filtered_package_info
def write_graph(f, counts, title, ident):
f.write('<div>\n%s\n' % title)
f.write('<div id="%s_graph" style="width:600px;height:300px"></div>\n' % ident)
f.write('<script type="text/javascript">\n')
f.write("""$(document).ready(function () {
$.plot($("#%s_graph"), [[
""" % ident)
first = True
for ts, val in counts:
if not first:
f.write(",")
first = False
f.write("[%s, %d]" % (str(time.mktime(ts.utctimetuple())*1000), val))
f.write("""]], {xaxis: {mode: "time", min: %s}, yaxis: {min: 0, transform: function (v) { if (v == 0) { return 0; } return Math.log(v); }, inverseTransform: function (v) { if (v == 0) { return 0; } return Math.exp(v); }}});
});
""" % (str(time.mktime(datetime.datetime(2010, 10, 04, 0, 0, 0).utctimetuple())*1000), ))
f.write('</script>\n')
f.write('</div>\n')
def write_graphs(f, counts):
write_graph(f, [(a[0], a[1]) for a in counts], "Number of packages waiting to be imported", "queue")
write_graph(f, [(a[0], a[2]) for a in counts], "Number of packages that failed to import", "failed")
def write_main_page(reasons, package_info, explanations):
main_packages = get_main_packages()
reasons, package_info = filter_packages(main_packages, reasons, package_info)
written_paths = set()
path = os.path.join(paths.web_status_dir, "main.html")
f = open(path, "wb")
try:
f.write(html_head % {"title": "bzr import failures for main"})
f.write(html_contents)
f.write('<h3>Summary</h3>\n<div id="stats">\n')
write_stats(f, package_info)
f.write('</div>\n<div id="latest">\n<h3>Latest 50 Failures:</h3>\n<ul>\n')
write_latest(f, package_info)
f.write('</ul>\n</p>\n</div>\n')
f.write('</div>\n<h3>All Failures by Category</h3>\n<div id="analysis">\n<ul>\n')
write_analysis(f, reasons, explanations)
f.write('</ul>\n</div>\n')
db = icommon.HistoryDatabase(paths.sqlite_history_file)
write_graphs(f, db.get_main_counts())
f.write(html_tail)
finally:
f.close()
written_paths.add(path)
return written_paths
def copy_js_files():
source_paths = set(["jquery.min.js", "jquery.flot.js"])
copied_paths = set()
for fn in source_paths:
shutil.copy(os.path.join(os.path.dirname(__file__), fn),
paths.web_status_dir)
copied_paths.add(os.path.join(paths.web_status_dir, fn))
return copied_paths
def get_info():
new_output = None
reasons, package_info = process_failures()
explanations = icommon.load_explanations()
old_pages = list_old_pages()
new_pages = write_individual_pages(reasons, explanations)
new_pages |= write_overview_page(reasons, package_info, explanations)
new_pages |= write_main_page(reasons, package_info, explanations)
new_pages |= copy_js_files()
remove_obsolete_pages(old_pages, new_pages)
def install_index():
content = '''<html>
<head>
<title>Ubuntu Bazaar package importer</title>
<meta http-equiv="refresh" content="0;url=status/">
</head>
<body>
Hi!
See <a href="status">the status page.</a>
</body>
</html>
'''
# FIXME: 'status' in content should stay in sync with 'status' in
# web_status_dir, we could use os.path.basename(paths.web_status_dir)
# -- vila 2011-03-23
index_path = os.path.join(paths.web_base_dir, 'index.html')
f = open(index_path, 'w')
try:
f.write(content)
finally:
f.close()
def main():
lock = icommon.lock_categorise_failures()
if lock is None:
print "Another instance of categorise_failures is already running."
sys.exit(0)
try:
icommon.ensure_directory(paths.web_status_dir)
install_index()
get_info()
finally:
lock.close()
if __name__ == '__main__':
main()
|