1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
|
#!/usr/bin/python
import cgi
import datetime
import hashlib
import os
import shutil
import sys
import time
sys.path.insert(0, os.path.dirname(__file__))
import icommon
output_dir = "/srv/package-import.canonical.com/new/logs/status/"
html_head = '''<html>
<head>
<title>%(title)s</title>
<meta http-equiv="refresh" content="600">
<script src="jquery.min.js"></script>
<script src="jquery.flot.js"></script>
</head>
<body>
<a href="http://package-import.ubuntu.com/status/">Overview</a>
<p>You are looking at information on the Bazaar importer system that serves <a href="https://wiki.ubuntu.com/DistributedDevelopment">Ubuntu Distributed Development</a>. It's hoped that you won't have to care about the existence of this, but things don't always work out that way.</p>
<p>It is expected that uploads appear in the branches in less than 30 minutes, if it takes longer then something is up and this page should tell you why. To discuss issues or to ask for a package to be looked at please email James Westby.</p>
<p>This page was last updated at <tt>''' + datetime.datetime.utcnow().isoformat() + ''' UTC.</tt></p>
'''
html_tail = '''</body>
</html>
'''
def process_failures():
db = icommon.StatusDatabase(icommon.sqlite_file)
return db.summarise_failures()
def package_link(package):
return '<a href="%s.html#%s">%s</a>' % (cgi.escape(package.name, True),
package.timestamp, cgi.escape(package.name))
def error_page(reason):
checksum = hashlib.md5()
checksum.update(reason)
return "%s.html" % cgi.escape(checksum.hexdigest(), True)
def error_link(reason):
return '<a href="%s">Packages with this issue</a>' % error_page(reason)
def list_old_pages():
old_pages = set()
for page in os.listdir(output_dir):
path = os.path.join(output_dir, page)
if not os.path.isdir(path):
old_pages.add(path)
return old_pages
def remove_obsolete_pages(old_pages, new_pages):
obsolete_pages = old_pages - new_pages
for path in obsolete_pages:
if os.path.exists(path) and not os.path.isdir(path):
os.unlink(path)
def write_individual_pages(reasons, explanations):
paths = set()
for reason in reasons:
for info in reasons[reason][:]:
assert not "/" in info.name
path = os.path.join(output_dir, cgi.escape(info.name,
True) + ".html")
f = open(path, "wb")
try:
f.write(html_head % {"title":
"Failure reason for %s" % cgi.escape(info.name)})
f.write("<p>Failed at %s</p>\n" % info.timestamp)
if reason in explanations:
f.write("<p>%s</p>\n" % cgi.escape(explanations[reason]))
f.write("<p><pre>\n")
f.write(cgi.escape(info.raw_reason))
f.write("\n</pre></p>\n")
f.write("<div>%s</div>" % error_link(reason))
f.write(html_tail)
finally:
f.close()
paths.add(path)
path = os.path.join(output_dir, error_page(reason))
f = open(path, "wb")
try:
f.write(html_head % {"title": "Failures of type"})
f.write("<ul>\n")
for package in reasons[reason][:]:
f.write("<li>%s</li>\n" % package_link(package))
f.write("</ul>\n")
f.write(html_tail)
finally:
f.close()
paths.add(path)
return paths
def find_info_where(package_info, clause):
ret = []
for info in package_info:
if clause(info):
ret.append(info)
return ret
def write_stats(f, package_info):
f.write("<ul>\n")
running = find_info_where(package_info, lambda x: x.running)
f.write("<li>%d currently running\n<ul>\n" % len(running))
running.sort(key=lambda x: x.timestamp)
for package in running:
f.write("<li>%s (since %s)</li>" % (cgi.escape(package.name),
package.timestamp))
f.write("</ul>\n</li>\n")
outstanding = find_info_where(package_info, lambda x: x.queued)
f.write("<li>%d outstanding jobs\n<ul>\n<li>" % len(outstanding))
outstanding.sort(key=lambda x: x.name)
for package in outstanding:
f.write(cgi.escape(package.name) + " ")
f.write("</li>\n</ul>\n</li>\n")
f.write("<li>%d failures\n</li>"
% (len(package_info) - len(outstanding) - len(running)))
f.write("</ul>\n")
def write_analyis_partial(f, keys, reasons, explanations, clause, text):
for reason in keys:
packages = reasons[reason]
subset = find_info_where(packages, clause)
if len(subset) > 0:
f.write("<li>%d packages %s with key %s\n<ul>\n"
% (len(subset), text, cgi.escape(reason)))
if reason in explanations:
f.write('<li><pre>%s</pre></li>\n'
% cgi.escape(explanations[reason]))
f.write('<li>')
subset.sort(key=lambda x: x.name)
for package in subset:
f.write(package_link(package) + " ")
f.write("</li>\n</ul>\n</li>\n")
def write_analysis(f, reasons, explanations):
keys = reasons.keys()
def compare(a, b):
return len(reasons[b]) - len(reasons[a])
keys.sort(cmp=compare)
write_analyis_partial(f, keys, reasons, explanations,
lambda x: x.auto_retry_masked, "failed to many times to retry")
write_analyis_partial(f, keys, reasons, explanations,
lambda x: not x.auto_retry, "failed")
write_analyis_partial(f, keys, reasons, explanations,
lambda x: x.auto_retry and not x.auto_retry_masked,
"spuriously failed")
def write_latest(f, package_info):
failed = find_info_where(package_info, lambda x: x.failure_count > 0)
failed.sort(key=lambda x: x.timestamp, reverse=True)
i = 0
for package in failed:
i += 1
if i > 50:
return
f.write("<li>%s\n<ul>\n" % package_link(package))
f.write("<li>failed at %s</li>\n" % package.timestamp)
f.write("<li>with key %s</li>\n" % cgi.escape(package.signature))
if package.auto_retry_masked:
f.write("<li>and failed too many times to retry</li>\n")
elif package.auto_retry:
f.write("<li>and will be retried automatically around %s</li>\n" % package.auto_retry_time)
f.write("</ul>\n</li>\n")
def write_overview_page(reasons, package_info, explanations):
paths = set()
path = os.path.join(output_dir, "index.html")
f = open(path, "wb")
try:
f.write(html_head % {"title": "bzr import failures"})
f.write("""<h3>Contents of this page</h3>
<ul>
<li><a href="#stats">Summary</a></li>
<li><a href="#latest">Latest failures</a></li>
<li><a href="#analysis">All failures</a></li>
</ul>
""")
f.write('<div id="stats">\n')
write_stats(f, package_info)
f.write('</div>\n<div id="latest">\nLatest 50 Failures:\n<ul>\n')
write_latest(f, package_info)
f.write('</ul>\n</div>\n')
f.write('</div>\n<div id="analysis">\n<ul>\n')
write_analysis(f, reasons, explanations)
f.write('</ul>\n</div>\n')
db = icommon.HistoryDatabase(icommon.sqlite_history_file)
write_graphs(f, db.get_counts())
f.write(html_tail)
finally:
f.close()
paths.add(path)
return paths
def get_main_packages():
db = icommon.PackageDatabase(icommon.sqlite_package_file)
return db.list_packages_in_main()
def filter_packages(filter_set, reasons, package_info):
filtered_package_info = find_info_where(package_info, lambda x: x.name in filter_set)
filtered_reasons = {}
for sig in reasons:
for info in reasons[sig]:
if info in filtered_package_info:
filtered_reasons.setdefault(sig, [])
filtered_reasons[sig].append(info)
return filtered_reasons, filtered_package_info
def write_graph(f, counts, title, ident):
f.write('<div>\n%s\n' % title)
f.write('<div id="%s_graph" style="width:600px;height:300px"></div>\n' % ident)
f.write('<script type="text/javascript">\n')
f.write("""$(document).ready(function () {
$.plot($("#%s_graph"), [[
""" % ident)
first = True
for ts, val in counts:
if not first:
f.write(",")
first = False
f.write("[%s, %d]" % (str(time.mktime(ts.utctimetuple())*1000), val))
f.write("""]], {xaxis: {mode: "time", min: %s}, yaxis: {min: 0, transform: function (v) { if (v == 0) { return 0; } return Math.log(v); }, inverseTransform: function (v) { if (v == 0) { return 0; } return Math.exp(v); }}});
});
""" % (str(time.mktime(datetime.datetime(2010, 10, 04, 0, 0, 0).utctimetuple())*1000), ))
f.write('</script>\n')
f.write('</div>\n')
def write_graphs(f, counts):
write_graph(f, [(a[0], a[1]) for a in counts], "Number of packages waiting to be imported", "queue")
write_graph(f, [(a[0], a[2]) for a in counts], "Number of packages that failed to import", "failed")
def write_main_page(reasons, package_info, explanations):
main_packages = get_main_packages()
reasons, package_info = filter_packages(main_packages, reasons, package_info)
paths = set()
path = os.path.join(output_dir, "main.html")
f = open(path, "wb")
try:
f.write(html_head % {"title": "bzr import failures for main"})
f.write("""<p><h3>Contents of this page</h3>
<ul>
<li><a href="#stats">Summary</a></li>
<li><a href="#latest">Latest failures</a></li>
<li><a href="#analysis">All failures</a></li>
</ul>
</p>
""")
f.write('<div id="stats">\n')
write_stats(f, package_info)
f.write('</div>\n<div id="latest">\n<p>Latest 50 Failures:\n<ul>\n')
write_latest(f, package_info)
f.write('</ul>\n</p>\n</div>\n')
f.write('</div>\n<div id="analysis">\n<ul>\n')
write_analysis(f, reasons, explanations)
f.write('</ul>\n</div>\n')
db = icommon.HistoryDatabase(icommon.sqlite_history_file)
write_graphs(f, db.get_main_counts())
f.write(html_tail)
finally:
f.close()
paths.add(path)
return paths
def copy_js_files():
source_paths = set(["jquery.min.js", "jquery.flot.js"])
copied_paths = set()
for fn in source_paths:
shutil.copy(os.path.join(os.path.dirname(__file__), fn),
output_dir)
copied_paths.add(os.path.join(output_dir, fn))
return copied_paths
def get_info():
new_output = None
reasons, package_info = process_failures()
explanations = icommon.load_explanations()
old_pages = list_old_pages()
new_pages = write_individual_pages(reasons, explanations)
new_pages |= write_overview_page(reasons, package_info, explanations)
new_pages |= write_main_page(reasons, package_info, explanations)
new_pages |= copy_js_files()
remove_obsolete_pages(old_pages, new_pages)
def main():
lock = icommon.lock_categorise_failures()
if lock is None:
print "Another instance of categorise_failures is already running."
sys.exit(0)
try:
get_info()
finally:
lock.close()
if __name__ == '__main__':
main()
|