1065.1.1
by Kees Cook
skip atime checks when mounted noatime; fix vim encoding line |
1 |
# vim: set encoding=UTF-8 fileencoding=UTF-8 :
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
2 |
|
1369.1.310
by Martin Pitt
Update all copyright and description headers and consistently format them. |
3 |
'''Store, load, and handle problem reports.'''
|
4 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
5 |
# Copyright (C) 2006 - 2012 Canonical Ltd.
|
1369.1.310
by Martin Pitt
Update all copyright and description headers and consistently format them. |
6 |
# Author: Martin Pitt <martin.pitt@ubuntu.com>
|
1369.34.404
by Martin Pitt
remove trailing whitespace |
7 |
#
|
1369.1.310
by Martin Pitt
Update all copyright and description headers and consistently format them. |
8 |
# This program is free software; you can redistribute it and/or modify it
|
9 |
# under the terms of the GNU General Public License as published by the
|
|
10 |
# Free Software Foundation; either version 2 of the License, or (at your
|
|
11 |
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
|
|
12 |
# the full text of the license.
|
|
28
by martin at piware
create initial library problem_report.py, move write_debcontrol() into it |
13 |
|
1369.34.376
by Martin Pitt
Move all test suites out of the code modules into test/test_<module>.py. This avoids having to load it every time the program runs, and also allows running the tests against the installed version of Apport. |
14 |
import zlib, base64, time, sys, gzip, struct, os |
1369.34.42
by Martin Pitt
Python 3 compatible "print" (not everything yet) |
15 |
from email.encoders import encode_base64 |
16 |
from email.mime.multipart import MIMEMultipart |
|
17 |
from email.mime.base import MIMEBase |
|
18 |
from email.mime.text import MIMEText |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
19 |
from io import BytesIO |
1369.34.42
by Martin Pitt
Python 3 compatible "print" (not everything yet) |
20 |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
21 |
if sys.version[0] < '3': |
22 |
from UserDict import IterableUserDict as UserDict |
|
1369.34.552
by Martin Pitt
* Clean up module imports. * test/run: Run pyflakes, if available. |
23 |
UserDict # pyflakes |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
24 |
_python2 = True |
25 |
else: |
|
1369.34.42
by Martin Pitt
Python 3 compatible "print" (not everything yet) |
26 |
from collections import UserDict |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
27 |
_python2 = False |
28
by martin at piware
create initial library problem_report.py, move write_debcontrol() into it |
28 |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
29 |
|
991
by Martin Pitt
problem_report.py: fix syntax for older Python versions |
30 |
class CompressedValue: |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
31 |
'''Represent a ProblemReport value which is gzip compressed.'''
|
32 |
||
973
by Martin Pitt
problem_report.py, CompressedValue: add default value ctor argument |
33 |
def __init__(self, value=None, name=None): |
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
34 |
'''Initialize an empty CompressedValue object with an optional name.'''
|
1369.34.404
by Martin Pitt
remove trailing whitespace |
35 |
|
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
36 |
self.gzipvalue = None |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
37 |
self.name = name |
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
38 |
# By default, compressed values are in gzip format. Earlier versions of
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
39 |
# problem_report used zlib format (without gzip header). If you have
|
40 |
# such a case, set legacy_zlib to True.
|
|
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
41 |
self.legacy_zlib = False |
42 |
||
973
by Martin Pitt
problem_report.py, CompressedValue: add default value ctor argument |
43 |
if value: |
44 |
self.set_value(value) |
|
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
45 |
|
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
46 |
def set_value(self, value): |
47 |
'''Set uncompressed value.'''
|
|
48 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
49 |
out = BytesIO() |
1369.34.1127
by Martin Pitt
* ProblemReport: Set a timestamp of 0 in gzip compressed fields; they are meaningless and cause unnecessary jitter in the output. |
50 |
gzip.GzipFile(self.name, mode='wb', fileobj=out, mtime=0).write(value) |
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
51 |
self.gzipvalue = out.getvalue() |
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
52 |
self.legacy_zlib = False |
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
53 |
|
54 |
def get_value(self): |
|
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
55 |
'''Return uncompressed value.'''
|
56 |
||
976
by Martin Pitt
problem_report.py: remove property, they are broken; add CompressedValue.write() |
57 |
if not self.gzipvalue: |
58 |
return None |
|
59 |
||
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
60 |
if self.legacy_zlib: |
61 |
return zlib.decompress(self.gzipvalue) |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
62 |
return gzip.GzipFile(fileobj=BytesIO(self.gzipvalue)).read() |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
63 |
|
976
by Martin Pitt
problem_report.py: remove property, they are broken; add CompressedValue.write() |
64 |
def write(self, file): |
65 |
'''Write uncompressed value into given file-like object.'''
|
|
66 |
||
67 |
assert self.gzipvalue |
|
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
68 |
|
69 |
if self.legacy_zlib: |
|
70 |
file.write(zlib.decompress(self.gzipvalue)) |
|
71 |
return
|
|
72 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
73 |
gz = gzip.GzipFile(fileobj=BytesIO(self.gzipvalue)) |
976
by Martin Pitt
problem_report.py: remove property, they are broken; add CompressedValue.write() |
74 |
while True: |
75 |
block = gz.read(1048576) |
|
76 |
if not block: |
|
77 |
break
|
|
78 |
file.write(block) |
|
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
79 |
|
977
by Martin Pitt
problem_report.py: Add CompressedValue.__len__ |
80 |
def __len__(self): |
81 |
'''Return length of uncompressed value.'''
|
|
82 |
||
83 |
assert self.gzipvalue |
|
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
84 |
if self.legacy_zlib: |
85 |
return len(self.get_value()) |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
86 |
return int(struct.unpack('<L', self.gzipvalue[-4:])[0]) |
977
by Martin Pitt
problem_report.py: Add CompressedValue.__len__ |
87 |
|
1027
by martin at piware
* apport/report.py testsuite: Check that our methods get along with binary |
88 |
def splitlines(self): |
89 |
'''Behaves like splitlines() for a normal string.'''
|
|
90 |
||
91 |
return self.get_value().splitlines() |
|
92 |
||
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
93 |
|
1369.34.42
by Martin Pitt
Python 3 compatible "print" (not everything yet) |
94 |
class ProblemReport(UserDict): |
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
95 |
def __init__(self, type='Crash', date=None): |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
96 |
'''Initialize a fresh problem report.
|
688
by Martin Pitt
* Remove trailing white space in all Python files. |
97 |
|
1161.1.1
by Matt Zimmerman
Add bin/kernel_oops hook to capture a kernel oops (eg. via kerneloops) |
98 |
type can be 'Crash', 'Packaging', 'KernelCrash' or 'KernelOops'.
|
99 |
date is the desired date/time string; if None (default), the
|
|
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
100 |
current local time is used.
|
101 |
'''
|
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
102 |
if date is None: |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
103 |
date = time.asctime() |
104 |
self.data = {'ProblemType': type, 'Date': date} |
|
105 |
||
691
by Martin Pitt
* problem_report.py: Add new method get_new() which returns a set of all |
106 |
# keeps track of keys which were added since the last ctor or load()
|
107 |
self.old_keys = set() |
|
108 |
||
1369.34.1244
by Martin Pitt
* Add key filtering to ProblemReport.load(). |
109 |
def load(self, file, binary=True, key_filter=None): |
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
110 |
'''Initialize problem report from a file-like object.
|
1369.34.404
by Martin Pitt
remove trailing whitespace |
111 |
|
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
112 |
If binary is False, binary data is not loaded; the dictionary key is
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
113 |
created, but its value will be an empty string. If it is True, it is
|
114 |
transparently uncompressed and available as dictionary byte array values.
|
|
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
115 |
If binary is 'compressed', the compressed value is retained, and the
|
116 |
dictionary value will be a CompressedValue object. This is useful if
|
|
117 |
the compressed value is still useful (to avoid recompression if the
|
|
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
118 |
file needs to be written back).
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
119 |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
120 |
file needs to be opened in binary mode.
|
121 |
||
1369.34.1244
by Martin Pitt
* Add key filtering to ProblemReport.load(). |
122 |
If key_filter is given, only those keys will be loaded.
|
123 |
||
1369.34.1230
by Martin Pitt
* doc/data-format.tex: Clarify that key names are being treated as case sensitive (unlike RFC822). |
124 |
Files are in RFC822 format, but with case sensitive keys.
|
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
125 |
'''
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
126 |
self._assert_bin_mode(file) |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
127 |
self.data.clear() |
128 |
key = None |
|
129 |
value = None |
|
130 |
b64_block = False |
|
131 |
bd = None |
|
1369.34.1244
by Martin Pitt
* Add key filtering to ProblemReport.load(). |
132 |
if key_filter: |
133 |
remaining_keys = set(key_filter) |
|
134 |
else: |
|
135 |
remaining_keys = None |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
136 |
for line in file: |
137 |
# continuation line
|
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
138 |
if line.startswith(b' '): |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
139 |
if b64_block and not binary: |
140 |
continue
|
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
141 |
assert (key is not None and value is not None) |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
142 |
if b64_block: |
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
143 |
block = base64.b64decode(line) |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
144 |
if bd: |
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
145 |
value += bd.decompress(block) |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
146 |
else: |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
147 |
if binary == 'compressed': |
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
148 |
# check gzip header; if absent, we have legacy zlib
|
149 |
# data
|
|
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
150 |
if value.gzipvalue == b'' and not block.startswith(b'\037\213\010'): |
1041
by Martin Pitt
* problem_report.py: Support reading reports with legacy zlib |
151 |
value.legacy_zlib = True |
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
152 |
value.gzipvalue += block |
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
153 |
else: |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
154 |
# lazy initialization of bd
|
155 |
# skip gzip header, if present
|
|
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
156 |
if block.startswith(b'\037\213\010'): |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
157 |
bd = zlib.decompressobj(-zlib.MAX_WBITS) |
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
158 |
value = bd.decompress(self._strip_gzip_header(block)) |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
159 |
else: |
160 |
# legacy zlib-only format used default block
|
|
161 |
# size
|
|
162 |
bd = zlib.decompressobj() |
|
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
163 |
value += bd.decompress(block) |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
164 |
else: |
165 |
if len(value) > 0: |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
166 |
value += b'\n' |
167 |
if line.endswith(b'\n'): |
|
1369.34.165
by Martin Pitt
problem_report.py, load(): Fix missing last character if the last line in a multi-line field is not terminated with a newline. |
168 |
value += line[1:-1] |
169 |
else: |
|
170 |
value += line[1:] |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
171 |
else: |
172 |
if b64_block: |
|
967
by Martin Pitt
* problem_report.py: Remove support for reading bz2 compressed binary data. |
173 |
if bd: |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
174 |
value += bd.flush() |
175 |
b64_block = False |
|
176 |
bd = None |
|
177 |
if key: |
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
178 |
assert value is not None |
1369.34.1244
by Martin Pitt
* Add key filtering to ProblemReport.load(). |
179 |
if remaining_keys is not None: |
180 |
try: |
|
181 |
remaining_keys.remove(key) |
|
182 |
self.data[key] = self._try_unicode(value) |
|
183 |
if not remaining_keys: |
|
184 |
key = None |
|
185 |
break
|
|
186 |
except KeyError: |
|
187 |
pass
|
|
188 |
else: |
|
189 |
self.data[key] = self._try_unicode(value) |
|
190 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
191 |
(key, value) = line.split(b':', 1) |
1369.82.2
by Martin Pitt
More Python 3 fixes |
192 |
if not _python2: |
193 |
key = key.decode('ASCII') |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
194 |
value = value.strip() |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
195 |
if value == b'base64': |
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
196 |
if binary == 'compressed': |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
197 |
value = CompressedValue(key.encode()) |
198 |
value.gzipvalue = b'' |
|
971
by Martin Pitt
* problem_report.py, write(): Add new permitted 'binary' argument value |
199 |
else: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
200 |
value = b'' |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
201 |
b64_block = True |
202 |
||
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
203 |
if key is not None: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
204 |
self.data[key] = self._try_unicode(value) |
36
by martin at piware
fix reading of last field in ProblemReport.load() |
205 |
|
691
by Martin Pitt
* problem_report.py: Add new method get_new() which returns a set of all |
206 |
self.old_keys = set(self.data.keys()) |
207 |
||
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
208 |
def extract_keys(self, file, bin_keys, dir): |
1369.141.10
by Louis Bouchard
Rename method to extract_key and fix description |
209 |
'''Extract only one binary element from the problem_report
|
210 |
||
1369.34.1111
by Martin Pitt
* Add a new method ProblemReport.extract_keys() which writes binary keys (which can be very large) directly to files without loading them all into memory first. Use that in apport-unpack. Thanks Louis Bouchard! (LP: #1307413) |
211 |
Binary elements like kernel crash dumps can be very big. This method
|
212 |
extracts directly files without loading the report into memory.
|
|
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
213 |
'''
|
214 |
self._assert_bin_mode(file) |
|
1369.34.1111
by Martin Pitt
* Add a new method ProblemReport.extract_keys() which writes binary keys (which can be very large) directly to files without loading them all into memory first. Use that in apport-unpack. Thanks Louis Bouchard! (LP: #1307413) |
215 |
# support singe key and collection of keys
|
216 |
if isinstance(bin_keys, str): |
|
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
217 |
bin_keys = [bin_keys] |
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
218 |
key = None |
219 |
value = None |
|
1369.34.1111
by Martin Pitt
* Add a new method ProblemReport.extract_keys() which writes binary keys (which can be very large) directly to files without loading them all into memory first. Use that in apport-unpack. Thanks Louis Bouchard! (LP: #1307413) |
220 |
missing_keys = list(bin_keys) |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
221 |
b64_block = {} |
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
222 |
bd = None |
1369.141.8
by Louis Bouchard
Fix pylint complains |
223 |
out = None |
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
224 |
for line in file: |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
225 |
# Identify the bin_keys we're looking for
|
226 |
while not line.startswith(b' '): |
|
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
227 |
(key, value) = line.split(b':', 1) |
228 |
if not _python2: |
|
229 |
key = key.decode('ASCII') |
|
1369.34.1111
by Martin Pitt
* Add a new method ProblemReport.extract_keys() which writes binary keys (which can be very large) directly to files without loading them all into memory first. Use that in apport-unpack. Thanks Louis Bouchard! (LP: #1307413) |
230 |
if key not in missing_keys: |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
231 |
break
|
232 |
b64_block[key] = False |
|
1369.34.1111
by Martin Pitt
* Add a new method ProblemReport.extract_keys() which writes binary keys (which can be very large) directly to files without loading them all into memory first. Use that in apport-unpack. Thanks Louis Bouchard! (LP: #1307413) |
233 |
missing_keys.remove(key) |
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
234 |
value = value.strip() |
235 |
if value == b'base64': |
|
236 |
value = b'' |
|
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
237 |
b64_block[key] = True |
238 |
try: |
|
239 |
bd = None |
|
240 |
with open(os.path.join(dir, key), 'wb') as out: |
|
241 |
for line in file: |
|
242 |
# continuation line
|
|
243 |
if line.startswith(b' '): |
|
244 |
assert (key is not None and value is not None) |
|
245 |
if b64_block[key]: |
|
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
246 |
block = base64.b64decode(line) |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
247 |
if bd: |
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
248 |
out.write(bd.decompress(block)) |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
249 |
else: |
250 |
# lazy initialization of bd
|
|
251 |
# skip gzip header, if present
|
|
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
252 |
if block.startswith(b'\037\213\010'): |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
253 |
bd = zlib.decompressobj(-zlib.MAX_WBITS) |
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
254 |
out.write(bd.decompress(self._strip_gzip_header(block))) |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
255 |
else: |
256 |
# legacy zlib-only format used default block
|
|
257 |
# size
|
|
258 |
bd = zlib.decompressobj() |
|
1369.34.1325
by Martin Pitt
Fix PEP-8 errors with latest pycodestyle |
259 |
out.write(bd.decompress(block)) |
1369.141.17
by Louis Bouchard
Rework the extract_key logic to simplify |
260 |
else: |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
261 |
break
|
262 |
except IOError: |
|
1369.141.21
by Louis Bouchard
Replace format() by %s % syntax |
263 |
raise IOError('unable to open %s' % (os.path.join(dir, key))) |
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
264 |
else: |
265 |
break
|
|
1369.34.1111
by Martin Pitt
* Add a new method ProblemReport.extract_keys() which writes binary keys (which can be very large) directly to files without loading them all into memory first. Use that in apport-unpack. Thanks Louis Bouchard! (LP: #1307413) |
266 |
if missing_keys: |
267 |
raise KeyError('Cannot find %s in report' % ', '.join(missing_keys)) |
|
1369.141.20
by Louis Bouchard
Implement multiple key extract : rename to extract_keys |
268 |
if False in b64_block.values(): |
1369.141.21
by Louis Bouchard
Replace format() by %s % syntax |
269 |
raise ValueError('%s has no binary content' % |
270 |
[item for item, element in b64_block.items() if element is False]) |
|
1369.141.1
by Louis Bouchard
Add ProblemReport.extract() method : writes binary element to disk |
271 |
|
229
by martin at piware
* problem_report.py: Add method has_removed_fields() to check whether load() |
272 |
def has_removed_fields(self): |
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
273 |
'''Check if the report has any keys which were not loaded.
|
1369.34.404
by Martin Pitt
remove trailing whitespace |
274 |
|
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
275 |
This could happen when using binary=False in load().
|
276 |
'''
|
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
277 |
return ('' in self.values()) |
229
by martin at piware
* problem_report.py: Add method has_removed_fields() to check whether load() |
278 |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
279 |
@classmethod
|
280 |
def _is_binary(klass, string): |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
281 |
'''Check if the given strings contains binary data.'''
|
146
by martin at piware
problem_report.py: Fix writing back binary data, adapt test suite to check it |
282 |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
283 |
if type(string) == bytes: |
284 |
for c in string: |
|
285 |
if c < 32 and not chr(c).isspace(): |
|
286 |
return True |
|
287 |
return False |
|
288 |
||
289 |
@classmethod
|
|
290 |
def _try_unicode(klass, value): |
|
291 |
'''Try to convert bytearray value to unicode'''
|
|
292 |
||
293 |
if type(value) == bytes and not klass._is_binary(value): |
|
294 |
try: |
|
295 |
return value.decode('UTF-8') |
|
296 |
except UnicodeDecodeError: |
|
297 |
return value |
|
298 |
return value |
|
299 |
||
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
300 |
def write(self, file, only_new=False): |
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
301 |
'''Write information into the given file-like object.
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
302 |
|
692
by Martin Pitt
* problem_report.py: Add optional parameter only_new to write(), which |
303 |
If only_new is True, only keys which have been added since the last
|
304 |
load() are written (i. e. those returned by new_keys()).
|
|
305 |
||
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
306 |
If a value is a string, it is written directly. Otherwise it must be a
|
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
307 |
tuple of the form (file, encode=True, limit=None, fail_on_empty=False).
|
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
308 |
The first argument can be a file name or a file-like object,
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
309 |
which will be read and its content will become the value of this key.
|
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
310 |
'encode' specifies whether the contents will be
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
311 |
gzip compressed and base64-encoded (this defaults to True). If limit is
|
1369.1.370
by Martin Pitt
problem_report.py: Fix documentation of write() |
312 |
set to a positive integer, the file is not attached if it's larger
|
313 |
than the given limit, and the entire key will be removed. If
|
|
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
314 |
fail_on_empty is True, reading zero bytes will cause an IOError.
|
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
315 |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
316 |
file needs to be opened in binary mode.
|
317 |
||
1357
by Martin Pitt
problem_report.py, man/apport-unpack.1: Fix description of .crash file |
318 |
Files are written in RFC822 format.
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
319 |
'''
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
320 |
self._assert_bin_mode(file) |
321 |
||
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
322 |
# sort keys into ASCII non-ASCII/binary attachment ones, so that
|
323 |
# the base64 ones appear last in the report
|
|
324 |
asckeys = [] |
|
325 |
binkeys = [] |
|
326 |
for k in self.data.keys(): |
|
692
by Martin Pitt
* problem_report.py: Add optional parameter only_new to write(), which |
327 |
if only_new and k in self.old_keys: |
328 |
continue
|
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
329 |
v = self.data[k] |
330 |
if hasattr(v, 'find'): |
|
331 |
if self._is_binary(v): |
|
332 |
binkeys.append(k) |
|
333 |
else: |
|
334 |
asckeys.append(k) |
|
335 |
else: |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
336 |
if not isinstance(v, CompressedValue) and len(v) >= 2 and not v[1]: |
337 |
# force uncompressed
|
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
338 |
asckeys.append(k) |
339 |
else: |
|
340 |
binkeys.append(k) |
|
341 |
||
342 |
asckeys.sort() |
|
343 |
if 'ProblemType' in asckeys: |
|
344 |
asckeys.remove('ProblemType') |
|
345 |
asckeys.insert(0, 'ProblemType') |
|
346 |
binkeys.sort() |
|
347 |
||
348 |
# write the ASCII keys first
|
|
349 |
for k in asckeys: |
|
350 |
v = self.data[k] |
|
351 |
||
352 |
# if it's a tuple, we have a file reference; read the contents
|
|
353 |
if not hasattr(v, 'find'): |
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
354 |
if len(v) >= 3 and v[2] is not None: |
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
355 |
limit = v[2] |
356 |
else: |
|
357 |
limit = None |
|
358 |
||
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
359 |
fail_on_empty = len(v) >= 4 and v[3] |
360 |
||
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
361 |
if hasattr(v[0], 'read'): |
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
362 |
v = v[0].read() # file-like object |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
363 |
else: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
364 |
with open(v[0], 'rb') as f: # file name |
365 |
v = f.read() |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
366 |
|
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
367 |
if fail_on_empty and len(v) == 0: |
1369.34.40
by Martin Pitt
Python 3 compatible exception handling |
368 |
raise IOError('did not get any data for field ' + k) |
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
369 |
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
370 |
if limit is not None and len(v) > limit: |
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
371 |
del self.data[k] |
372 |
continue
|
|
373 |
||
1369.174.44
by Brian Murray
switch from pyflakes to pyflakes3, drop some python2 code |
374 |
if isinstance(v, str): |
375 |
# unicode → str
|
|
376 |
v = v.encode('UTF-8') |
|
1272
by Martin Pitt
problem_report.py, test_write(): Add test cases for single-line |
377 |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
378 |
file.write(k.encode('ASCII')) |
379 |
if b'\n' in v: |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
380 |
# multiline value
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
381 |
file.write(b':\n ') |
382 |
file.write(v.replace(b'\n', b'\n ')) |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
383 |
else: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
384 |
file.write(b': ') |
385 |
file.write(v) |
|
386 |
file.write(b'\n') |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
387 |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
388 |
# now write the binary keys with gzip compression and base64 encoding
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
389 |
for k in binkeys: |
390 |
v = self.data[k] |
|
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
391 |
limit = None |
392 |
size = 0 |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
393 |
|
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
394 |
curr_pos = file.tell() |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
395 |
file.write(k.encode('ASCII')) |
396 |
file.write(b': base64\n ') |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
397 |
|
1061
by Martin Pitt
* problem_report.py(): Make write() work for reports with CompressedValues. |
398 |
# CompressedValue
|
399 |
if isinstance(v, CompressedValue): |
|
400 |
file.write(base64.b64encode(v.gzipvalue)) |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
401 |
file.write(b'\n') |
1061
by Martin Pitt
* problem_report.py(): Make write() work for reports with CompressedValues. |
402 |
continue
|
403 |
||
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
404 |
# write gzip header
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
405 |
gzip_header = b'\037\213\010\010\000\000\000\000\002\377' + k.encode('UTF-8') + b'\000' |
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
406 |
file.write(base64.b64encode(gzip_header)) |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
407 |
file.write(b'\n ') |
408 |
crc = zlib.crc32(b'') |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
409 |
|
1369.174.53
by Brian Murray
problem_report.py: Decrease zlib compression level from 9 to 6. |
410 |
bc = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS, |
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
411 |
zlib.DEF_MEM_LEVEL, 0) |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
412 |
# direct value
|
413 |
if hasattr(v, 'find'): |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
414 |
size += len(v) |
415 |
crc = zlib.crc32(v, crc) |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
416 |
outblock = bc.compress(v) |
417 |
if outblock: |
|
418 |
file.write(base64.b64encode(outblock)) |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
419 |
file.write(b'\n ') |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
420 |
# file reference
|
421 |
else: |
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
422 |
if len(v) >= 3 and v[2] is not None: |
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
423 |
limit = v[2] |
424 |
||
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
425 |
if hasattr(v[0], 'read'): |
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
426 |
f = v[0] # file-like object |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
427 |
else: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
428 |
f = open(v[0], 'rb') # file name |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
429 |
while True: |
430 |
block = f.read(1048576) |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
431 |
size += len(block) |
432 |
crc = zlib.crc32(block, crc) |
|
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
433 |
if limit is not None: |
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
434 |
if size > limit: |
435 |
# roll back
|
|
436 |
file.seek(curr_pos) |
|
437 |
file.truncate(curr_pos) |
|
438 |
del self.data[k] |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
439 |
crc = None |
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
440 |
break
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
441 |
if block: |
442 |
outblock = bc.compress(block) |
|
443 |
if outblock: |
|
444 |
file.write(base64.b64encode(outblock)) |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
445 |
file.write(b'\n ') |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
446 |
else: |
447 |
break
|
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
448 |
if not hasattr(v[0], 'read'): |
449 |
f.close() |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
450 |
|
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
451 |
if len(v) >= 4 and v[3]: |
452 |
if size == 0: |
|
1369.34.40
by Martin Pitt
Python 3 compatible exception handling |
453 |
raise IOError('did not get any data for field %s from %s' % (k, str(v[0]))) |
859
by Martin Pitt
* problem_report.py: Introduce a fourth optional parameter "fail_on_empty" |
454 |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
455 |
# flush compressor and write the rest
|
728
by Martin Pitt
* problem_report.py, write(): Allow a third optional argument in tuple |
456 |
if not limit or size <= limit: |
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
457 |
block = bc.flush() |
458 |
# append gzip trailer: crc (32 bit) and size (32 bit)
|
|
459 |
if crc: |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
460 |
block += struct.pack('<L', crc & 0xFFFFFFFF) |
461 |
block += struct.pack('<L', size & 0xFFFFFFFF) |
|
970
by Martin Pitt
* problem_report.py: Switch encoding of binary values from bare zlib to |
462 |
|
463 |
file.write(base64.b64encode(block)) |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
464 |
file.write(b'\n') |
28
by martin at piware
create initial library problem_report.py, move write_debcontrol() into it |
465 |
|
297
by martin at piware
* problem_report.py: Add new method ProblemReport.add_to_existing() to |
466 |
def add_to_existing(self, reportfile, keep_times=False): |
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
467 |
'''Add this report's data to an already existing report file.
|
688
by Martin Pitt
* Remove trailing white space in all Python files. |
468 |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
469 |
The file will be temporarily chmod'ed to 000 to prevent frontends
|
470 |
from picking up a hal-updated report file. If keep_times
|
|
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
471 |
is True, then the file's atime and mtime restored after updating.
|
472 |
'''
|
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
473 |
st = os.stat(reportfile) |
474 |
try: |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
475 |
f = open(reportfile, 'ab') |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
476 |
os.chmod(reportfile, 0) |
477 |
self.write(f) |
|
478 |
f.close() |
|
479 |
finally: |
|
480 |
if keep_times: |
|
481 |
os.utime(reportfile, (st.st_atime, st.st_mtime)) |
|
482 |
os.chmod(reportfile, st.st_mode) |
|
297
by martin at piware
* problem_report.py: Add new method ProblemReport.add_to_existing() to |
483 |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
484 |
def write_mime(self, file, attach_treshold=5, extra_headers={}, |
1369.34.622
by Martin Pitt
* Fix PEP-8 violations picked up by latest pep8 checker. |
485 |
skip_keys=None, priority_fields=None): |
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
486 |
'''Write MIME/Multipart RFC 2822 formatted data into file.
|
487 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
488 |
file must be a file-like object, not a path. It needs to be opened in
|
489 |
binary mode.
|
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
490 |
|
975
by Martin Pitt
* problem_report, write_mime(): Make function work for compressed binary |
491 |
If a value is a string or a CompressedValue, it is written directly.
|
492 |
Otherwise it must be a tuple containing the source file and an optional
|
|
493 |
boolean value (in that order); the first argument can be a file name or
|
|
494 |
a file-like object, which will be read and its content will become the
|
|
495 |
value of this key. The file will be gzip compressed, unless the key
|
|
496 |
already ends in .gz.
|
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
497 |
|
498 |
attach_treshold specifies the maximum number of lines for a value to be
|
|
499 |
included into the first inline text part. All bigger values (as well as
|
|
1369.34.453
by Martin Pitt
problem_report.py, write_mime(): Do not put a key inline if it is bigger than 1 kB, to guard against very long lines. (LP: #957326) |
500 |
all non-ASCII ones) will become an attachment, as well as text
|
501 |
values bigger than 1 kB.
|
|
708
by Martin Pitt
* problem_report.py, write_mime(): Add optional 'preamble' parameter. Add |
502 |
|
767
by martin at piware
* problem_report.py, write_mime(): Drop preamble argument, replace it with |
503 |
Extra MIME preamble headers can be specified, too, as a dictionary.
|
1127
by Martin Pitt
problem_report.py, write_mime(): Add new "skip_keys" argument to filter |
504 |
|
505 |
skip_keys is a set/list specifying keys which are filtered out and not
|
|
506 |
written to the destination file.
|
|
1369.28.1
by Brian Murray
add priority_fields for ordering report keys (a test too) and add ordering for launchpad bug reports |
507 |
|
508 |
priority_fields is a set/list specifying the order in which keys should
|
|
509 |
appear in the destination file.
|
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
510 |
'''
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
511 |
self._assert_bin_mode(file) |
512 |
||
1369.34.44
by Martin Pitt
problem_report.py: More Python 3 friendliness |
513 |
keys = sorted(self.data.keys()) |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
514 |
|
1369.34.919
by Martin Pitt
* ProblemReport.write_mime(): Adjust MIMEText handling to latest Python 3.3 upstream changes which now don't tolerate passing bytes any more. (LP: #1227381) |
515 |
text = '' |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
516 |
attachments = [] |
517 |
||
518 |
if 'ProblemType' in keys: |
|
519 |
keys.remove('ProblemType') |
|
520 |
keys.insert(0, 'ProblemType') |
|
521 |
||
1369.28.1
by Brian Murray
add priority_fields for ordering report keys (a test too) and add ordering for launchpad bug reports |
522 |
if priority_fields: |
523 |
counter = 0 |
|
524 |
for priority_field in priority_fields: |
|
525 |
if priority_field in keys: |
|
526 |
keys.remove(priority_field) |
|
527 |
keys.insert(counter, priority_field) |
|
528 |
counter += 1 |
|
529 |
||
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
530 |
for k in keys: |
1127
by Martin Pitt
problem_report.py, write_mime(): Add new "skip_keys" argument to filter |
531 |
if skip_keys and k in skip_keys: |
532 |
continue
|
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
533 |
v = self.data[k] |
534 |
attach_value = None |
|
535 |
||
975
by Martin Pitt
* problem_report, write_mime(): Make function work for compressed binary |
536 |
# compressed values are ready for attaching in gzip form
|
537 |
if isinstance(v, CompressedValue): |
|
538 |
attach_value = v.gzipvalue |
|
539 |
||
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
540 |
# if it's a tuple, we have a file reference; read the contents
|
541 |
# and gzip it
|
|
975
by Martin Pitt
* problem_report, write_mime(): Make function work for compressed binary |
542 |
elif not hasattr(v, 'find'): |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
543 |
attach_value = '' |
544 |
if hasattr(v[0], 'read'): |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
545 |
f = v[0] # file-like object |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
546 |
else: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
547 |
f = open(v[0], 'rb') # file name |
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
548 |
if k.endswith('.gz'): |
974
by Martin Pitt
* problem_report, write_mime(): Eliminate unnecessary usage of StringIO. |
549 |
attach_value = f.read() |
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
550 |
else: |
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
551 |
io = BytesIO() |
1369.34.1127
by Martin Pitt
* ProblemReport: Set a timestamp of 0 in gzip compressed fields; they are meaningless and cause unnecessary jitter in the output. |
552 |
gf = gzip.GzipFile(k, mode='wb', fileobj=io, mtime=0) |
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
553 |
while True: |
554 |
block = f.read(1048576) |
|
555 |
if block: |
|
556 |
gf.write(block) |
|
557 |
else: |
|
558 |
gf.close() |
|
559 |
break
|
|
974
by Martin Pitt
* problem_report, write_mime(): Eliminate unnecessary usage of StringIO. |
560 |
attach_value = io.getvalue() |
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
561 |
f.close() |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
562 |
|
563 |
# binary value
|
|
564 |
elif self._is_binary(v): |
|
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
565 |
if k.endswith('.gz'): |
974
by Martin Pitt
* problem_report, write_mime(): Eliminate unnecessary usage of StringIO. |
566 |
attach_value = v |
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
567 |
else: |
974
by Martin Pitt
* problem_report, write_mime(): Eliminate unnecessary usage of StringIO. |
568 |
attach_value = CompressedValue(v, k).gzipvalue |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
569 |
|
570 |
# if we have an attachment value, create an attachment
|
|
571 |
if attach_value: |
|
572 |
att = MIMEBase('application', 'x-gzip') |
|
733
by Martin Pitt
* problem_report.py, write_mime(): Do not re-compress keys which already end |
573 |
if k.endswith('.gz'): |
574 |
att.add_header('Content-Disposition', 'attachment', filename=k) |
|
575 |
else: |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
576 |
att.add_header('Content-Disposition', 'attachment', filename=k + '.gz') |
974
by Martin Pitt
* problem_report, write_mime(): Eliminate unnecessary usage of StringIO. |
577 |
att.set_payload(attach_value) |
992
by Martin Pitt
* problem_report.py, write_mime(): Use base64 encoding for gzipped |
578 |
encode_base64(att) |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
579 |
attachments.append(att) |
580 |
else: |
|
581 |
# plain text value
|
|
1369.34.453
by Martin Pitt
problem_report.py, write_mime(): Do not put a key inline if it is bigger than 1 kB, to guard against very long lines. (LP: #957326) |
582 |
size = len(v) |
1369.1.279
by Martin Pitt
launchpad.py: Ensure that text attachments on initial bug filing are valid UTF-8. (LP: #453203) |
583 |
|
584 |
# ensure that byte arrays are valid UTF-8
|
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
585 |
if type(v) == bytes: |
1369.1.279
by Martin Pitt
launchpad.py: Ensure that text attachments on initial bug filing are valid UTF-8. (LP: #453203) |
586 |
v = v.decode('UTF-8', 'replace') |
587 |
# convert unicode to UTF-8 str
|
|
1369.174.44
by Brian Murray
switch from pyflakes to pyflakes3, drop some python2 code |
588 |
assert isinstance(v, str) |
1269
by Martin Pitt
problem_report.py, test_write_mime_text(): Add test cases for |
589 |
|
451
by Martin Pitt
* problem_report.py, write_mime(): Make sure that multi-line values that go |
590 |
lines = len(v.splitlines()) |
1369.34.453
by Martin Pitt
problem_report.py, write_mime(): Do not put a key inline if it is bigger than 1 kB, to guard against very long lines. (LP: #957326) |
591 |
if size <= 1000 and lines == 1: |
1269
by Martin Pitt
problem_report.py, test_write_mime_text(): Add test cases for |
592 |
v = v.rstrip() |
1369.34.919
by Martin Pitt
* ProblemReport.write_mime(): Adjust MIMEText handling to latest Python 3.3 upstream changes which now don't tolerate passing bytes any more. (LP: #1227381) |
593 |
text += k + ': ' + v + '\n' |
1369.34.488
by Martin Pitt
* problem_report.py, write_mime(): Fix regression from version 1.95: Add a value as attachment if it is bigger than 1000 bytes, not if it is bigger than 100. (LP: #977882) |
594 |
elif size <= 1000 and lines <= attach_treshold: |
1369.34.919
by Martin Pitt
* ProblemReport.write_mime(): Adjust MIMEText handling to latest Python 3.3 upstream changes which now don't tolerate passing bytes any more. (LP: #1227381) |
595 |
text += k + ':\n ' |
596 |
if not v.endswith('\n'): |
|
597 |
v += '\n' |
|
598 |
text += v.strip().replace('\n', '\n ') + '\n' |
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
599 |
else: |
600 |
# too large, separate attachment
|
|
601 |
att = MIMEText(v, _charset='UTF-8') |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
602 |
att.add_header('Content-Disposition', 'attachment', filename=k + '.txt') |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
603 |
attachments.append(att) |
604 |
||
605 |
# create initial text attachment
|
|
606 |
att = MIMEText(text, _charset='UTF-8') |
|
607 |
att.add_header('Content-Disposition', 'inline') |
|
608 |
attachments.insert(0, att) |
|
609 |
||
610 |
msg = MIMEMultipart() |
|
1369.34.44
by Martin Pitt
problem_report.py: More Python 3 friendliness |
611 |
for k, v in extra_headers.items(): |
767
by martin at piware
* problem_report.py, write_mime(): Drop preamble argument, replace it with |
612 |
msg.add_header(k, v) |
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
613 |
for a in attachments: |
614 |
msg.attach(a) |
|
615 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
616 |
file.write(msg.as_string().encode('UTF-8')) |
617 |
file.write(b'\n') |
|
392
by martin at piware
* problem_report.py: Add new method write_mime() to encode a problem report |
618 |
|
28
by martin at piware
create initial library problem_report.py, move write_debcontrol() into it |
619 |
def __setitem__(self, k, v): |
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
620 |
assert hasattr(k, 'isalnum') |
1369.34.1288
by Martin Pitt
* problem_report.py: Instead of AssertionError, raise a ValueError for invalid key names and TypeError for invalid kinds of values. Thanks Barry Warsaw. |
621 |
if not k.replace('.', '').replace('-', '').replace('_', '').isalnum(): |
622 |
raise ValueError("key '%s' contains invalid characters (only numbers, letters, '.', '_', and '-' are allowed)" % k) |
|
972
by Martin Pitt
problem_report.py: Enrich CompressedValue interface a bit, add test cases |
623 |
# value must be a string or a CompressedValue or a file reference
|
1369.34.1304
by Martin Pitt
* problem_report.py: Fail with proper exception when trying to assign a list to a report key, or when trying to assing a tuple with more than 4 entries. (LP: #1596713) |
624 |
# (tuple (string|file [, bool, [, max_size [, fail_on_empty]]]))
|
1369.34.1288
by Martin Pitt
* problem_report.py: Instead of AssertionError, raise a ValueError for invalid key names and TypeError for invalid kinds of values. Thanks Barry Warsaw. |
625 |
if not (isinstance(v, CompressedValue) or hasattr(v, 'isalnum') or |
1369.34.1304
by Martin Pitt
* problem_report.py: Fail with proper exception when trying to assign a list to a report key, or when trying to assing a tuple with more than 4 entries. (LP: #1596713) |
626 |
(isinstance(v, tuple) and ( |
627 |
len(v) == 1 or (len(v) >= 2 and len(v) <= 4 and v[1] in (True, False))) and |
|
1369.34.1288
by Martin Pitt
* problem_report.py: Instead of AssertionError, raise a ValueError for invalid key names and TypeError for invalid kinds of values. Thanks Barry Warsaw. |
628 |
(hasattr(v[0], 'isalnum') or hasattr(v[0], 'read')))): |
629 |
raise TypeError("value for key %s must be a string, CompressedValue, or a file reference" % k) |
|
33
by martin at piware
add sanity checks to ProblemReport.__setitem__ |
630 |
|
353
by martin at piware
* Convert all tabs in Python source code files to spaces to comply to PEP 8. |
631 |
return self.data.__setitem__(k, v) |
148
by martin at piware
ProblemReport: Restructure class to inherit from IterableUserDict and |
632 |
|
691
by Martin Pitt
* problem_report.py: Add new method get_new() which returns a set of all |
633 |
def new_keys(self): |
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
634 |
'''Return newly added keys.
|
691
by Martin Pitt
* problem_report.py: Add new method get_new() which returns a set of all |
635 |
|
1369.1.143
by Martin Pitt
PEP-8 compatible docstrings |
636 |
Return the set of keys which have been added to the report since it
|
637 |
was constructed or loaded.
|
|
638 |
'''
|
|
691
by Martin Pitt
* problem_report.py: Add new method get_new() which returns a set of all |
639 |
return set(self.data.keys()) - self.old_keys |
640 |
||
1061
by Martin Pitt
* problem_report.py(): Make write() work for reports with CompressedValues. |
641 |
@classmethod
|
642 |
def _strip_gzip_header(klass, line): |
|
643 |
'''Strip gzip header from line and return the rest.'''
|
|
644 |
||
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
645 |
if _python2: |
646 |
return klass._strip_gzip_header_py2(line) |
|
647 |
||
648 |
flags = line[3] |
|
649 |
offset = 10 |
|
650 |
if flags & 4: # FLG.FEXTRA |
|
651 |
offset += line[offset] + 1 |
|
652 |
if flags & 8: # FLG.FNAME |
|
653 |
while line[offset] != 0: |
|
654 |
offset += 1 |
|
655 |
offset += 1 |
|
656 |
if flags & 16: # FLG.FCOMMENT |
|
657 |
while line[offset] != 0: |
|
658 |
offset += 1 |
|
659 |
offset += 1 |
|
660 |
if flags & 2: # FLG.FHCRC |
|
661 |
offset += 2 |
|
662 |
||
663 |
return line[offset:] |
|
664 |
||
665 |
@classmethod
|
|
666 |
def _strip_gzip_header_py2(klass, line): |
|
667 |
'''Strip gzip header from line and return the rest. (Python 2)'''
|
|
668 |
||
1061
by Martin Pitt
* problem_report.py(): Make write() work for reports with CompressedValues. |
669 |
flags = ord(line[3]) |
670 |
offset = 10 |
|
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
671 |
if flags & 4: # FLG.FEXTRA |
1061
by Martin Pitt
* problem_report.py(): Make write() work for reports with CompressedValues. |
672 |
offset += line[offset] + 1 |
1369.34.492
by Martin Pitt
* Fix the whole code to be PEP-8 compatible, and enforce this in test/run by running the "pep8" tool. |
673 |
if flags & 8: # FLG.FNAME |
674 |
while ord(line[offset]) != 0: |
|
675 |
offset += 1 |
|
676 |
offset += 1 |
|
677 |
if flags & 16: # FLG.FCOMMENT |
|
678 |
while ord(line[offset]) != 0: |
|
679 |
offset += 1 |
|
680 |
offset += 1 |
|
681 |
if flags & 2: # FLG.FHCRC |
|
1061
by Martin Pitt
* problem_report.py(): Make write() work for reports with CompressedValues. |
682 |
offset += 2 |
683 |
||
684 |
return line[offset:] |
|
1369.34.517
by Martin Pitt
problem_report.py: Fix for Python 3 |
685 |
|
686 |
@classmethod
|
|
687 |
def _assert_bin_mode(klass, file): |
|
688 |
'''Assert that given file object is in binary mode'''
|
|
689 |
||
690 |
if _python2: |
|
691 |
assert (type(file) == BytesIO or 'b' in file.mode), 'file stream must be in binary mode' |
|
692 |
else: |
|
693 |
assert not hasattr(file, 'encoding'), 'file stream must be in binary mode' |