~xnox/ubuntu-archive-tools/sru-report-autopkgtest-vomit

198 by Martin Pitt
add nbs-report
1
#!/usr/bin/python
355 by Colin Watson
Apply GPLv3 to anything not already licensed; ok slangasek, broder, laney, kitterman, geser
2
3
# Copyright (C) 2011, 2012  Canonical Ltd.
4
# Author: Martin Pitt <martin.pitt@ubuntu.com>
5
6
# This program is free software: you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; version 3 of the License.
9
#
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17
545 by Colin Watson
Make all scripts pass pep8(1).
18
# Generate a HTML report of current NBS binary packages from a checkrdepends
19
# output directory
198 by Martin Pitt
add nbs-report
20
359 by Colin Watson
Use Python 3-style print functions.
21
from __future__ import print_function
22
555 by Colin Watson
Simplify using collections.defaultdict.
23
from collections import defaultdict
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
24
import csv
847 by Colin Watson
nbs-report: Convert option parsing to optparse.
25
from optparse import OptionParser
356 by Colin Watson
PEP-8 import ordering
26
import os
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
27
import sys
356 by Colin Watson
PEP-8 import ordering
28
import time
198 by Martin Pitt
add nbs-report
29
849 by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time.
30
from charts import make_chart, make_chart_header
31
492 by Colin Watson
sort imports
32
198 by Martin Pitt
add nbs-report
33
def parse_checkrdepends_file(path, pkgmap):
34
    '''Parse one checkrdepends file into the NBS map'''
35
36
    cur_component = None
37
    cur_arch = None
38
427 by Colin Watson
Use context managers in a few more places.
39
    with open(path) as f:
40
        for line in f:
41
            if line.startswith('-- '):
42
                (cur_component, cur_arch) = line.split('/', 1)[1].split()[:2]
43
                continue
44
            assert cur_component
45
            assert cur_arch
198 by Martin Pitt
add nbs-report
46
733 by Colin Watson
nbs-report: tolerate extra words in checkrdepends output
47
            rdep = line.strip().split()[0]
427 by Colin Watson
Use context managers in a few more places.
48
            pkgmap.setdefault(rdep, (cur_component, []))[1].append(cur_arch)
198 by Martin Pitt
add nbs-report
49
545 by Colin Watson
Make all scripts pass pep8(1).
50
205 by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed
51
def _pkg_removable(pkg, nbs, checked_v):
52
    '''Recursively check if pakcage is removable.
339 by Colin Watson
remove trailing whitespace
53
205 by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed
54
    checked_v is the working set of already checked vertices, to avoid infinite
55
    loops.
56
    '''
57
    checked_v.add(pkg)
58
    for rdep in nbs.get(pkg, []):
59
        if rdep in checked_v:
60
            continue
61
        #checked_v.add(rdep)
62
        if not rdep in nbs:
63
            try:
64
                checked_v.remove(rdep)
65
            except KeyError:
66
                pass
67
            return False
68
        if not _pkg_removable(rdep, nbs, checked_v):
69
            try:
70
                checked_v.remove(rdep)
71
            except KeyError:
72
                pass
73
            return False
74
    return True
75
545 by Colin Watson
Make all scripts pass pep8(1).
76
198 by Martin Pitt
add nbs-report
77
def get_removables(nbs):
78
    '''Get set of removable packages.
79
205 by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed
80
    This includes packages with no rdepends and disconnected subgraphs, i. e.
81
    clusters of NBS packages which only depend on each other.
198 by Martin Pitt
add nbs-report
82
    '''
83
    removable = set()
205 by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed
84
198 by Martin Pitt
add nbs-report
85
    for p in nbs:
205 by Martin Pitt
nbs-report: Add detection of closed NBS subgraphs, to find out more packages which can be safely removed
86
        if p in removable:
87
            continue
88
        checked_v = set()
89
        if _pkg_removable(p, nbs, checked_v):
90
            # we can add the entire cluster here, not just p; avoids
91
            # re-checking the other vertices in that cluster
92
            removable.update(checked_v)
198 by Martin Pitt
add nbs-report
93
94
    return removable
95
545 by Colin Watson
Make all scripts pass pep8(1).
96
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
97
def html_report(options, nbs, removables):
198 by Martin Pitt
add nbs-report
98
    '''Generate HTML report from NBS map.'''
99
545 by Colin Watson
Make all scripts pass pep8(1).
100
    print('''\
101
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
102
 "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
198 by Martin Pitt
add nbs-report
103
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
104
<head>
105
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
106
  <title>NBS packages</title>
107
  <style type="text/css">
108
    body { background: #CCCCB0; color: black; }
109
    a { text-decoration: none; }
339 by Colin Watson
remove trailing whitespace
110
    table { border-collapse: collapse; border-style: none none;
198 by Martin Pitt
add nbs-report
111
            margin-bottom: 3ex; empty-cells: show; }
339 by Colin Watson
remove trailing whitespace
112
    table th { text-align: left; border-style: solid none none none;
198 by Martin Pitt
add nbs-report
113
               border-width: 3px; padding-right: 10px; }
114
    table td { vertical-align:top; text-align: left; border-style: dotted none;
115
               border-width: 1px; padding-right: 10px; }
116
    .normal { }
117
    .removable { color: green; font-weight: bold; }
118
    .nbs { color: blue; }
850 by Colin Watson
nbs-report: fix head generation
119
    .componentsup { font-size: 70%%; color: red; font-weight: bold; }
120
    .componentunsup { font-size: 70%%; color: darkred; }
198 by Martin Pitt
add nbs-report
121
  </style>
849 by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time.
122
  %s
198 by Martin Pitt
add nbs-report
123
</head>
124
<body>
125
<h1>NBS: Binary packages not built from any source</h1>
126
127
<h2>Archive Administrator commands</h2>
128
<p>Run this command to remove NBS packages which are not required any more:</p>
849 by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time.
129
''' % make_chart_header())
359 by Colin Watson
Use Python 3-style print functions.
130
897 by Colin Watson
Parameterise the NBS report so that it can be run for ubuntu-rtm too.
131
    print('<p style="font-family: monospace">remove-package -m NBS '
132
          '-d %s -s %s -b -y %s</p>' %
133
          (options.distribution, options.suite, ' '.join(sorted(removables))))
359 by Colin Watson
Use Python 3-style print functions.
134
135
    print('''
198 by Martin Pitt
add nbs-report
136
<h2>Reverse dependencies</h2>
339 by Colin Watson
remove trailing whitespace
137
545 by Colin Watson
Make all scripts pass pep8(1).
138
<p><span class="nbs">Reverse dependencies which are NBS themselves</span><br/>
339 by Colin Watson
remove trailing whitespace
139
<span class="removable">NBS package which can be removed safely</span></p>
198 by Martin Pitt
add nbs-report
140
<table>
359 by Colin Watson
Use Python 3-style print functions.
141
''')
555 by Colin Watson
Simplify using collections.defaultdict.
142
    reverse_nbs = defaultdict(list)  # non_nbs_pkg -> [nbspkg1, ...]
545 by Colin Watson
Make all scripts pass pep8(1).
143
    pkg_component = {}  # non_nbs_pkg -> (component, component_class)
198 by Martin Pitt
add nbs-report
144
145
    for pkg in sorted(nbs):
146
        nbsmap = nbs[pkg]
147
        if pkg in removables:
148
            cls = 'removable'
149
        else:
150
            cls = 'normal'
359 by Colin Watson
Use Python 3-style print functions.
151
        print('<tr><th colspan="4"><span class="%s">%s</span></td></tr>' %
152
              (cls, pkg), end="")
198 by Martin Pitt
add nbs-report
153
        for rdep in sorted(nbsmap):
154
            (component, arches) = nbsmap[rdep]
202 by Martin Pitt
nbs-report: add component to reverse dep list
155
156
            if component in ('main', 'restricted'):
157
                component_cls = 'sup'
158
            else:
159
                component_cls = 'unsup'
160
198 by Martin Pitt
add nbs-report
161
            if rdep in nbs:
162
                if rdep in removables:
163
                    cls = 'removable'
164
                else:
165
                    cls = 'nbs'
166
            else:
167
                cls = 'normal'
555 by Colin Watson
Simplify using collections.defaultdict.
168
                reverse_nbs[rdep].append(pkg)
202 by Martin Pitt
nbs-report: add component to reverse dep list
169
                pkg_component[rdep] = (component, component_cls)
200 by Martin Pitt
nbs-report: colorize components
170
545 by Colin Watson
Make all scripts pass pep8(1).
171
            print('<tr><td>&nbsp; &nbsp; </td>', end='')
172
            print('<td><span class="%s">%s</span></td> ' % (cls, rdep), end='')
173
            print('<td><span class="component%s">%s</span></td>' %
174
                  (component_cls, component), end='')
175
            print('<td>%s</td></tr>' % ' '.join(arches))
198 by Martin Pitt
add nbs-report
176
359 by Colin Watson
Use Python 3-style print functions.
177
    print('''</table>
198 by Martin Pitt
add nbs-report
178
<h2>Packages which depend on NBS packages</h2>
359 by Colin Watson
Use Python 3-style print functions.
179
<table>''')
198 by Martin Pitt
add nbs-report
180
181
    def sort_rev_nbs(k1, k2):
182
        len_cmp = cmp(len(reverse_nbs[k1]), len(reverse_nbs[k2]))
183
        if len_cmp == 0:
184
            return cmp(k1, k2)
185
        else:
186
            return -len_cmp
187
188
    for pkg in sorted(reverse_nbs, cmp=sort_rev_nbs):
545 by Colin Watson
Make all scripts pass pep8(1).
189
        print('<tr><td>%s</td> '
190
              '<td><span class="component%s">%s</span></td><td>' % (
677 by Colin Watson
make all scripts pass current stricter pep8(1) in raring
191
                  pkg, pkg_component[pkg][1], pkg_component[pkg][0]), end="")
369 by Colin Watson
fix a few incorrect uses of print(end="")
192
        print(" ".join(sorted(reverse_nbs[pkg])), end="")
359 by Colin Watson
Use Python 3-style print functions.
193
        print('</td></tr>')
198 by Martin Pitt
add nbs-report
194
359 by Colin Watson
Use Python 3-style print functions.
195
    print('</table>')
849 by Colin Watson
architecture-mismatches, component-mismatches, priority-mismatches, nbs-report: Use YUI to generate charts of progress over time.
196
197
    print("<h2>Over time</h2>")
198
    print(make_chart("nbs.csv", ["removable", "total"]))
199
359 by Colin Watson
Use Python 3-style print functions.
200
    print('<p><small>Generated at %s.</small></p>' %
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
201
          time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime(options.time)))
359 by Colin Watson
Use Python 3-style print functions.
202
    print('</body></html>')
545 by Colin Watson
Make all scripts pass pep8(1).
203
204
847 by Colin Watson
nbs-report: Convert option parsing to optparse.
205
def main():
206
    parser = OptionParser(
207
        usage="%prog <checkrdepends output directory>",
208
        description="Generate an HTML report of current NBS binary packages.")
897 by Colin Watson
Parameterise the NBS report so that it can be run for ubuntu-rtm too.
209
    parser.add_option('-d', '--distribution', default='ubuntu')
911 by Colin Watson
utopic → vivid
210
    parser.add_option('-s', '--suite', default='vivid')
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
211
    parser.add_option(
212
        '--csv-file', help='record CSV time series data in this file')
213
    options, args = parser.parse_args()
847 by Colin Watson
nbs-report: Convert option parsing to optparse.
214
    if len(args) != 1:
215
        parser.error("need a checkrdepends output directory")
216
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
217
    options.time = time.time()
218
847 by Colin Watson
nbs-report: Convert option parsing to optparse.
219
    # pkg -> rdep_pkg -> (component, [arch1, arch2, ...])
220
    nbs = defaultdict(dict)
221
222
    for f in os.listdir(args[0]):
223
        if f.startswith('.') or f.endswith('.html'):
224
            continue
225
        parse_checkrdepends_file(os.path.join(args[0], f), nbs[f])
226
227
    #with open('/tmp/dot', 'w') as dot:
228
    #    print('digraph {', file=dot)
229
    #    print('   ratio 0.1', file=dot)
230
    #    pkgnames = set(nbs)
231
    #    for m in nbs.itervalues():
232
    #        pkgnames.update(m)
233
    #    for n in pkgnames:
234
    #        print('  %s [label="%s"' % (n.replace('-', '').replace('.', ''), n),
235
    #              end="", file=dot)
236
    #        if n in nbs:
237
    #            print(', style="filled", fillcolor="lightblue"', end="", file=dot)
238
    #        print(']', file=dot)
239
    #    print(file=dot)
240
    #    for pkg, map in nbs.iteritems():
241
    #        for rd in map:
242
    #            print('  %s -> %s' % (
243
    #                    pkg.replace('-', '').replace('.', ''),
244
    #                    rd.replace('-', '').replace('.', '')), file=dot)
245
    #    print('}', file=dot)
246
247
    removables = get_removables(nbs)
248
848 by Colin Watson
nbs-report, cron.NBS: Record results as a CSV time series.
249
    html_report(options, nbs, removables)
250
251
    if options.csv_file is not None:
252
        if sys.version < "3":
253
            open_mode = "ab"
254
            open_kwargs = {}
255
        else:
256
            open_mode = "a"
257
            open_kwargs = {"newline": ""}
258
        csv_is_new = not os.path.exists(options.csv_file)
259
        with open(options.csv_file, open_mode, **open_kwargs) as csv_file:
260
            # Field names deliberately hardcoded; any changes require
261
            # manually rewriting the output file.
262
            fieldnames = [
263
                "time",
264
                "removable",
265
                "total",
266
                ]
267
            csv_writer = csv.DictWriter(csv_file, fieldnames)
268
            if csv_is_new:
269
                csv_writer.writeheader()
270
            csv_writer.writerow({
271
                "time": int(options.time * 1000),
272
                "removable": len(removables),
273
                "total": len(nbs),
274
                })
847 by Colin Watson
nbs-report: Convert option parsing to optparse.
275
276
277
if __name__ == '__main__':
278
    main()