198
by Martin Pitt
add nbs-report |
1 |
#!/usr/bin/python
|
355
by Colin Watson
Apply GPLv3 to anything not already licensed; ok slangasek, broder, laney, kitterman, geser |
2 |
|
3 |
# Copyright (C) 2011, 2012 Canonical Ltd.
|
|
4 |
# Author: Martin Pitt <martin.pitt@ubuntu.com>
|
|
5 |
||
6 |
# This program is free software: you can redistribute it and/or modify
|
|
7 |
# it under the terms of the GNU General Public License as published by
|
|
8 |
# the Free Software Foundation; version 3 of the License.
|
|
9 |
#
|
|
10 |
# This program is distributed in the hope that it will be useful,
|
|
11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13 |
# GNU General Public License for more details.
|
|
14 |
#
|
|
15 |
# You should have received a copy of the GNU General Public License
|
|
16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17 |
||
545
by Colin Watson
Make all scripts pass pep8(1). |
18 |
# Generate a HTML report of current NBS binary packages from a checkrdepends
|
19 |
# output directory
|
|
198
by Martin Pitt
add nbs-report |
20 |
|
359
by Colin Watson
Use Python 3-style print functions. |
21 |
from __future__ import print_function |
22 |
||
555
by Colin Watson
Simplify using collections.defaultdict. |
23 |
from collections import defaultdict |
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
24 |
import csv |
847
by Colin Watson
nbs-report: Convert option parsing to optparse. |
25 |
from optparse import OptionParser |
356
by Colin Watson
PEP-8 import ordering |
26 |
import os |
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
27 |
import sys |
356
by Colin Watson
PEP-8 import ordering |
28 |
import time |
198
by Martin Pitt
add nbs-report |
29 |
|
849
by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time. |
30 |
from charts import make_chart, make_chart_header |
31 |
||
492
by Colin Watson
sort imports |
32 |
|
198
by Martin Pitt
add nbs-report |
33 |
def parse_checkrdepends_file(path, pkgmap): |
34 |
'''Parse one checkrdepends file into the NBS map'''
|
|
35 |
||
36 |
cur_component = None |
|
37 |
cur_arch = None |
|
38 |
||
427
by Colin Watson
Use context managers in a few more places. |
39 |
with open(path) as f: |
40 |
for line in f: |
|
41 |
if line.startswith('-- '): |
|
42 |
(cur_component, cur_arch) = line.split('/', 1)[1].split()[:2] |
|
43 |
continue
|
|
44 |
assert cur_component |
|
45 |
assert cur_arch |
|
198
by Martin Pitt
add nbs-report |
46 |
|
733
by Colin Watson
nbs-report: tolerate extra words in checkrdepends output |
47 |
rdep = line.strip().split()[0] |
427
by Colin Watson
Use context managers in a few more places. |
48 |
pkgmap.setdefault(rdep, (cur_component, []))[1].append(cur_arch) |
198
by Martin Pitt
add nbs-report |
49 |
|
545
by Colin Watson
Make all scripts pass pep8(1). |
50 |
|
205
by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed |
51 |
def _pkg_removable(pkg, nbs, checked_v): |
52 |
'''Recursively check if pakcage is removable.
|
|
339
by Colin Watson
remove trailing whitespace |
53 |
|
205
by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed |
54 |
checked_v is the working set of already checked vertices, to avoid infinite
|
55 |
loops.
|
|
56 |
'''
|
|
57 |
checked_v.add(pkg) |
|
58 |
for rdep in nbs.get(pkg, []): |
|
59 |
if rdep in checked_v: |
|
60 |
continue
|
|
61 |
#checked_v.add(rdep)
|
|
62 |
if not rdep in nbs: |
|
63 |
try: |
|
64 |
checked_v.remove(rdep) |
|
65 |
except KeyError: |
|
66 |
pass
|
|
67 |
return False |
|
68 |
if not _pkg_removable(rdep, nbs, checked_v): |
|
69 |
try: |
|
70 |
checked_v.remove(rdep) |
|
71 |
except KeyError: |
|
72 |
pass
|
|
73 |
return False |
|
74 |
return True |
|
75 |
||
545
by Colin Watson
Make all scripts pass pep8(1). |
76 |
|
198
by Martin Pitt
add nbs-report |
77 |
def get_removables(nbs): |
78 |
'''Get set of removable packages.
|
|
79 |
||
205
by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed |
80 |
This includes packages with no rdepends and disconnected subgraphs, i. e.
|
81 |
clusters of NBS packages which only depend on each other.
|
|
198
by Martin Pitt
add nbs-report |
82 |
'''
|
83 |
removable = set() |
|
205
by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed |
84 |
|
198
by Martin Pitt
add nbs-report |
85 |
for p in nbs: |
205
by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed |
86 |
if p in removable: |
87 |
continue
|
|
88 |
checked_v = set() |
|
89 |
if _pkg_removable(p, nbs, checked_v): |
|
90 |
# we can add the entire cluster here, not just p; avoids
|
|
91 |
# re-checking the other vertices in that cluster
|
|
92 |
removable.update(checked_v) |
|
198
by Martin Pitt
add nbs-report |
93 |
|
94 |
return removable |
|
95 |
||
545
by Colin Watson
Make all scripts pass pep8(1). |
96 |
|
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
97 |
def html_report(options, nbs, removables): |
198
by Martin Pitt
add nbs-report |
98 |
'''Generate HTML report from NBS map.'''
|
99 |
||
545
by Colin Watson
Make all scripts pass pep8(1). |
100 |
print('''\ |
101 |
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
|
|
102 |
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
|
198
by Martin Pitt
add nbs-report |
103 |
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
|
104 |
<head>
|
|
105 |
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
106 |
<title>NBS packages</title>
|
|
107 |
<style type="text/css">
|
|
108 |
body { background: #CCCCB0; color: black; }
|
|
109 |
a { text-decoration: none; }
|
|
339
by Colin Watson
remove trailing whitespace |
110 |
table { border-collapse: collapse; border-style: none none;
|
198
by Martin Pitt
add nbs-report |
111 |
margin-bottom: 3ex; empty-cells: show; }
|
339
by Colin Watson
remove trailing whitespace |
112 |
table th { text-align: left; border-style: solid none none none;
|
198
by Martin Pitt
add nbs-report |
113 |
border-width: 3px; padding-right: 10px; }
|
114 |
table td { vertical-align:top; text-align: left; border-style: dotted none;
|
|
115 |
border-width: 1px; padding-right: 10px; }
|
|
116 |
.normal { }
|
|
117 |
.removable { color: green; font-weight: bold; }
|
|
118 |
.nbs { color: blue; }
|
|
850
by Colin Watson
nbs-report: fix head generation |
119 |
.componentsup { font-size: 70%%; color: red; font-weight: bold; } |
120 |
.componentunsup { font-size: 70%%; color: darkred; } |
|
198
by Martin Pitt
add nbs-report |
121 |
</style>
|
849
by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time. |
122 |
%s |
198
by Martin Pitt
add nbs-report |
123 |
</head>
|
124 |
<body>
|
|
125 |
<h1>NBS: Binary packages not built from any source</h1>
|
|
126 |
||
127 |
<h2>Archive Administrator commands</h2>
|
|
128 |
<p>Run this command to remove NBS packages which are not required any more:</p>
|
|
849
by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time. |
129 |
''' % make_chart_header()) |
359
by Colin Watson
Use Python 3-style print functions. |
130 |
|
897
by Colin Watson
Parameterise the NBS report so that it can be run for ubuntu-rtm too. |
131 |
print('<p style="font-family: monospace">remove-package -m NBS ' |
132 |
'-d %s -s %s -b -y %s</p>' % |
|
133 |
(options.distribution, options.suite, ' '.join(sorted(removables)))) |
|
359
by Colin Watson
Use Python 3-style print functions. |
134 |
|
135 |
print(''' |
|
198
by Martin Pitt
add nbs-report |
136 |
<h2>Reverse dependencies</h2>
|
339
by Colin Watson
remove trailing whitespace |
137 |
|
545
by Colin Watson
Make all scripts pass pep8(1). |
138 |
<p><span class="nbs">Reverse dependencies which are NBS themselves</span><br/>
|
339
by Colin Watson
remove trailing whitespace |
139 |
<span class="removable">NBS package which can be removed safely</span></p>
|
198
by Martin Pitt
add nbs-report |
140 |
<table>
|
359
by Colin Watson
Use Python 3-style print functions. |
141 |
''') |
555
by Colin Watson
Simplify using collections.defaultdict. |
142 |
reverse_nbs = defaultdict(list) # non_nbs_pkg -> [nbspkg1, ...] |
545
by Colin Watson
Make all scripts pass pep8(1). |
143 |
pkg_component = {} # non_nbs_pkg -> (component, component_class) |
198
by Martin Pitt
add nbs-report |
144 |
|
145 |
for pkg in sorted(nbs): |
|
146 |
nbsmap = nbs[pkg] |
|
147 |
if pkg in removables: |
|
148 |
cls = 'removable' |
|
149 |
else: |
|
150 |
cls = 'normal' |
|
359
by Colin Watson
Use Python 3-style print functions. |
151 |
print('<tr><th colspan="4"><span class="%s">%s</span></td></tr>' % |
152 |
(cls, pkg), end="") |
|
198
by Martin Pitt
add nbs-report |
153 |
for rdep in sorted(nbsmap): |
154 |
(component, arches) = nbsmap[rdep] |
|
202
by Martin Pitt
nbs-report: add component to reverse dep list |
155 |
|
156 |
if component in ('main', 'restricted'): |
|
157 |
component_cls = 'sup' |
|
158 |
else: |
|
159 |
component_cls = 'unsup' |
|
160 |
||
198
by Martin Pitt
add nbs-report |
161 |
if rdep in nbs: |
162 |
if rdep in removables: |
|
163 |
cls = 'removable' |
|
164 |
else: |
|
165 |
cls = 'nbs' |
|
166 |
else: |
|
167 |
cls = 'normal' |
|
555
by Colin Watson
Simplify using collections.defaultdict. |
168 |
reverse_nbs[rdep].append(pkg) |
202
by Martin Pitt
nbs-report: add component to reverse dep list |
169 |
pkg_component[rdep] = (component, component_cls) |
200
by Martin Pitt
nbs-report: colorize components |
170 |
|
545
by Colin Watson
Make all scripts pass pep8(1). |
171 |
print('<tr><td> </td>', end='') |
172 |
print('<td><span class="%s">%s</span></td> ' % (cls, rdep), end='') |
|
173 |
print('<td><span class="component%s">%s</span></td>' % |
|
174 |
(component_cls, component), end='') |
|
175 |
print('<td>%s</td></tr>' % ' '.join(arches)) |
|
198
by Martin Pitt
add nbs-report |
176 |
|
359
by Colin Watson
Use Python 3-style print functions. |
177 |
print('''</table> |
198
by Martin Pitt
add nbs-report |
178 |
<h2>Packages which depend on NBS packages</h2>
|
359
by Colin Watson
Use Python 3-style print functions. |
179 |
<table>''') |
198
by Martin Pitt
add nbs-report |
180 |
|
181 |
def sort_rev_nbs(k1, k2): |
|
182 |
len_cmp = cmp(len(reverse_nbs[k1]), len(reverse_nbs[k2])) |
|
183 |
if len_cmp == 0: |
|
184 |
return cmp(k1, k2) |
|
185 |
else: |
|
186 |
return -len_cmp |
|
187 |
||
188 |
for pkg in sorted(reverse_nbs, cmp=sort_rev_nbs): |
|
545
by Colin Watson
Make all scripts pass pep8(1). |
189 |
print('<tr><td>%s</td> ' |
190 |
'<td><span class="component%s">%s</span></td><td>' % ( |
|
677
by Colin Watson
make all scripts pass current stricter pep8(1) in raring |
191 |
pkg, pkg_component[pkg][1], pkg_component[pkg][0]), end="") |
369
by Colin Watson
fix a few incorrect uses of print(end="") |
192 |
print(" ".join(sorted(reverse_nbs[pkg])), end="") |
359
by Colin Watson
Use Python 3-style print functions. |
193 |
print('</td></tr>') |
198
by Martin Pitt
add nbs-report |
194 |
|
359
by Colin Watson
Use Python 3-style print functions. |
195 |
print('</table>') |
849
by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time. |
196 |
|
197 |
print("<h2>Over time</h2>") |
|
198 |
print(make_chart("nbs.csv", ["removable", "total"])) |
|
199 |
||
359
by Colin Watson
Use Python 3-style print functions. |
200 |
print('<p><small>Generated at %s.</small></p>' % |
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
201 |
time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime(options.time))) |
359
by Colin Watson
Use Python 3-style print functions. |
202 |
print('</body></html>') |
545
by Colin Watson
Make all scripts pass pep8(1). |
203 |
|
204 |
||
847
by Colin Watson
nbs-report: Convert option parsing to optparse. |
205 |
def main(): |
206 |
parser = OptionParser( |
|
207 |
usage="%prog <checkrdepends output directory>", |
|
208 |
description="Generate an HTML report of current NBS binary packages.") |
|
897
by Colin Watson
Parameterise the NBS report so that it can be run for ubuntu-rtm too. |
209 |
parser.add_option('-d', '--distribution', default='ubuntu') |
911
by Colin Watson
utopic → vivid |
210 |
parser.add_option('-s', '--suite', default='vivid') |
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
211 |
parser.add_option( |
212 |
'--csv-file', help='record CSV time series data in this file') |
|
213 |
options, args = parser.parse_args() |
|
847
by Colin Watson
nbs-report: Convert option parsing to optparse. |
214 |
if len(args) != 1: |
215 |
parser.error("need a checkrdepends output directory") |
|
216 |
||
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
217 |
options.time = time.time() |
218 |
||
847
by Colin Watson
nbs-report: Convert option parsing to optparse. |
219 |
# pkg -> rdep_pkg -> (component, [arch1, arch2, ...])
|
220 |
nbs = defaultdict(dict) |
|
221 |
||
222 |
for f in os.listdir(args[0]): |
|
223 |
if f.startswith('.') or f.endswith('.html'): |
|
224 |
continue
|
|
225 |
parse_checkrdepends_file(os.path.join(args[0], f), nbs[f]) |
|
226 |
||
227 |
#with open('/tmp/dot', 'w') as dot:
|
|
228 |
# print('digraph {', file=dot)
|
|
229 |
# print(' ratio 0.1', file=dot)
|
|
230 |
# pkgnames = set(nbs)
|
|
231 |
# for m in nbs.itervalues():
|
|
232 |
# pkgnames.update(m)
|
|
233 |
# for n in pkgnames:
|
|
234 |
# print(' %s [label="%s"' % (n.replace('-', '').replace('.', ''), n),
|
|
235 |
# end="", file=dot)
|
|
236 |
# if n in nbs:
|
|
237 |
# print(', style="filled", fillcolor="lightblue"', end="", file=dot)
|
|
238 |
# print(']', file=dot)
|
|
239 |
# print(file=dot)
|
|
240 |
# for pkg, map in nbs.iteritems():
|
|
241 |
# for rd in map:
|
|
242 |
# print(' %s -> %s' % (
|
|
243 |
# pkg.replace('-', '').replace('.', ''),
|
|
244 |
# rd.replace('-', '').replace('.', '')), file=dot)
|
|
245 |
# print('}', file=dot)
|
|
246 |
||
247 |
removables = get_removables(nbs) |
|
248 |
||
848
by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series. |
249 |
html_report(options, nbs, removables) |
250 |
||
251 |
if options.csv_file is not None: |
|
252 |
if sys.version < "3": |
|
253 |
open_mode = "ab" |
|
254 |
open_kwargs = {} |
|
255 |
else: |
|
256 |
open_mode = "a" |
|
257 |
open_kwargs = {"newline": ""} |
|
258 |
csv_is_new = not os.path.exists(options.csv_file) |
|
259 |
with open(options.csv_file, open_mode, **open_kwargs) as csv_file: |
|
260 |
# Field names deliberately hardcoded; any changes require
|
|
261 |
# manually rewriting the output file.
|
|
262 |
fieldnames = [ |
|
263 |
"time", |
|
264 |
"removable", |
|
265 |
"total", |
|
266 |
]
|
|
267 |
csv_writer = csv.DictWriter(csv_file, fieldnames) |
|
268 |
if csv_is_new: |
|
269 |
csv_writer.writeheader() |
|
270 |
csv_writer.writerow({ |
|
271 |
"time": int(options.time * 1000), |
|
272 |
"removable": len(removables), |
|
273 |
"total": len(nbs), |
|
274 |
})
|
|
847
by Colin Watson
nbs-report: Convert option parsing to optparse. |
275 |
|
276 |
||
277 |
if __name__ == '__main__': |
|
278 |
main() |