« back to all changes in this revision
Viewing changes to util.py
- browse files at revision 7
- compare with another revision
- download diff
- download tarball
- view history from revision 7
- Committer: Bazaar Package Importer
- Author(s): Jelmer Vernooij
- Date: 2010-01-18 19:15:26 UTC
- mfrom: (5.1.5 karmic)
- Revision ID: james.westby@ubuntu.com-20100118191526-fzyw0n60z6vrhhhn
Tags: 2.2
* Upload to unstable.
* Bump standards version to 3.8.3.
* Bump standards version to 3.8.3.
- files added:
- cmds.py
- files removed:
- doc/user_manual/export_upstream.rst
- files modified:
- README
added
removed
util.py
1
1
# util.py -- Utility functions
2
2
# Copyright (C) 2006 James Westby <jw+debian@jameswestby.net>
3
#
3
#
4
4
# This file is part of bzr-builddeb.
5
5
#
6
6
# bzr-builddeb is free software; you can redistribute it and/or modify
18
18
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19
19
#
20
20
21
try:
22
import hashlib as md5
23
except ImportError:
24
import md5
25
import signal
21
26
import shutil
27
import tempfile
22
28
import os
23
29
import re
24
30
25
from bzrlib.trace import info, mutter
26
27
from debian_bundle.changelog import Changelog
28
29
from bzrlib.plugins.builddeb.errors import (MissingChangelogError,
31
from bzrlib.trace import mutter
32
33
from debian_bundle import deb822
34
from debian_bundle.changelog import Changelog, ChangelogParseError
35
36
from bzrlib import (
37
bugtracker,
38
errors,
39
urlutils,
40
)
41
from bzrlib.transport import (
42
do_catching_redirections,
43
get_transport,
44
)
45
from bzrlib.plugins.builddeb.errors import (
46
MissingChangelogError,
30
47
AddChangelogError,
48
UnparseableChangelog,
31
49
)
32
50
33
51
34
52
def recursive_copy(fromdir, todir):
35
"""Copy the contents of fromdir to todir. Like shutil.copytree, but the
36
destination directory must already exist with this method, rather than
37
not exists for shutil."""
38
mutter("Copying %s to %s", fromdir, todir)
39
for entry in os.listdir(fromdir):
40
path = os.path.join(fromdir, entry)
41
if os.path.isdir(path):
42
tosubdir = os.path.join(todir, entry)
43
if not os.path.exists(tosubdir):
44
os.mkdir(tosubdir)
45
recursive_copy(path, tosubdir)
46
else:
47
shutil.copy(path, todir)
48
49
50
def find_changelog(t, merge):
53
"""Copy the contents of fromdir to todir.
54
55
Like shutil.copytree, but the destination directory must already exist
56
with this method, rather than not exists for shutil.
57
"""
58
mutter("Copying %s to %s", fromdir, todir)
59
for entry in os.listdir(fromdir):
60
path = os.path.join(fromdir, entry)
61
if os.path.isdir(path):
62
tosubdir = os.path.join(todir, entry)
63
if not os.path.exists(tosubdir):
64
os.mkdir(tosubdir)
65
recursive_copy(path, tosubdir)
66
else:
67
shutil.copy(path, todir)
68
69
70
def find_changelog(t, merge, max_blocks=1):
71
"""Find the changelog in the given tree.
72
73
First looks for 'debian/changelog'. If "merge" is true will also
74
look for 'changelog'.
75
76
The returned changelog is created with 'allow_empty_author=True'
77
as some people do this but still want to build.
78
'max_blocks' defaults to 1 to try and prevent old broken
79
changelog entries from causing the command to fail,
80
81
"larstiq" is a subset of "merge" mode. It indicates that the
82
'.bzr' dir is at the same level as 'changelog' etc., rather
83
than being at the same level as 'debian/'.
84
85
:param t: the Tree to look in.
86
:param merge: whether this is a "merge" package.
87
:param max_blocks: Number of max_blocks to parse (defaults to 1)
88
:return: (changelog, larstiq) where changelog is the Changelog,
89
and larstiq is a boolean indicating whether the file is at
90
'changelog' if merge was given, False otherwise.
91
"""
51
92
changelog_file = 'debian/changelog'
52
93
larstiq = False
53
94
t.lock_read()
54
95
try:
55
if not t.has_filename(changelog_file):
56
if merge:
57
#Assume LarstiQ's layout (.bzr in debian/)
58
changelog_file = 'changelog'
59
larstiq = True
60
if not t.has_filename(changelog_file):
61
raise MissingChangelogError("debian/changelog or changelog")
62
else:
63
raise MissingChangelogError("debian/changelog")
64
else:
65
if merge and t.has_filename('changelog'):
66
if (t.kind(t.path2id('debian')) == 'symlink' and
67
t.get_symlink_target(t.path2id('debian')) == '.'):
68
changelog_file = 'changelog'
69
larstiq = True
70
mutter("Using '%s' to get package information", changelog_file)
71
changelog_id = t.path2id(changelog_file)
72
if changelog_id is None:
73
raise AddChangelogError(changelog_file)
74
contents = t.get_file_text(changelog_id)
96
if not t.has_filename(changelog_file):
97
if merge:
98
#Assume LarstiQ's layout (.bzr in debian/)
99
changelog_file = 'changelog'
100
larstiq = True
101
if not t.has_filename(changelog_file):
102
raise MissingChangelogError('"debian/changelog" or '
103
'"changelog"')
104
else:
105
raise MissingChangelogError('"debian/changelog"')
106
elif merge and t.has_filename('changelog'):
107
# If it is a "larstiq" pacakge and debian is a symlink to
108
# "." then it will have found debian/changelog. Try and detect
109
# this.
110
if (t.kind(t.path2id('debian')) == 'symlink' and
111
t.get_symlink_target(t.path2id('debian')) == '.'):
112
changelog_file = 'changelog'
113
larstiq = True
114
mutter("Using '%s' to get package information", changelog_file)
115
changelog_id = t.path2id(changelog_file)
116
if changelog_id is None:
117
raise AddChangelogError(changelog_file)
118
contents = t.get_file_text(changelog_id)
75
119
finally:
76
t.unlock()
120
t.unlock()
77
121
changelog = Changelog()
78
changelog.parse_changelog(contents, max_blocks=1, allow_empty_author=True)
122
try:
123
changelog.parse_changelog(contents, max_blocks=max_blocks, allow_empty_author=True)
124
except ChangelogParseError, e:
125
raise UnparseableChangelog(str(e))
79
126
return changelog, larstiq
80
127
128
129
def strip_changelog_message(changes):
130
"""Strip a changelog message like debcommit does.
131
132
Takes a list of changes from a changelog entry and applies a transformation
133
so the message is well formatted for a commit message.
134
135
:param changes: a list of lines from the changelog entry
136
:return: another list of lines with blank lines stripped from the start
137
and the spaces the start of the lines split if there is only one logical
138
entry.
139
"""
140
if not changes:
141
return changes
142
while changes and changes[-1] == '':
143
changes.pop()
144
while changes and changes[0] == '':
145
changes.pop(0)
146
147
whitespace_column_re = re.compile(r' |\t')
148
changes = map(lambda line: whitespace_column_re.sub('', line, 1), changes)
149
150
leader_re = re.compile(r'[ \t]*[*+-] ')
151
count = len(filter(leader_re.match, changes))
152
if count == 1:
153
return map(lambda line: leader_re.sub('', line, 1).lstrip(), changes)
154
else:
155
return changes
156
157
81
158
def tarball_name(package, version):
82
"""Return the name of the .orig.tar.gz for the given package and version."""
83
84
return "%s_%s.orig.tar.gz" % (package, str(version))
159
"""Return the name of the .orig.tar.gz for the given package and version.
160
161
:param package: the name of the source package.
162
:param version: the upstream version of the package.
163
:return: a string that is the name of the upstream tarball to use.
164
"""
165
return "%s_%s.orig.tar.gz" % (package, str(version))
166
85
167
86
168
def get_snapshot_revision(upstream_version):
87
"""Return the upstream revision specifier if specified in the upstream version or None. """
88
match = re.search("~bzr([0-9]+)$", upstream_version)
89
if match is not None:
90
return match.groups()[0]
91
match = re.search("(?:~|\\+)svn([0-9]+)$", upstream_version)
92
if match is not None:
93
return "svn:%s" % match.groups()[0]
94
return None
95
96
# vim: ts=2 sts=2 sw=2
169
"""Return the upstream revision specifier if specified in the upstream version.
170
171
When packaging an upstream snapshot some people use +vcsnn or ~vcsnn to indicate
172
what revision number of the upstream VCS was taken for the snapshot. This given
173
an upstream version number this function will return an identifier of the
174
upstream revision if it appears to be a snapshot. The identifier is a string
175
containing a bzr revision spec, so it can be transformed in to a revision.
176
177
:param upstream_version: a string containing the upstream version number.
178
:return: a string containing a revision specifier for the revision of the
179
upstream branch that the snapshot was taken from, or None if it doesn't
180
appear to be a snapshot.
181
"""
182
match = re.search("(?:~|\\+)bzr([0-9]+)$", upstream_version)
183
if match is not None:
184
return match.groups()[0]
185
match = re.search("(?:~|\\+)svn([0-9]+)$", upstream_version)
186
if match is not None:
187
return "svn:%s" % match.groups()[0]
188
return None
189
190
191
def get_export_upstream_revision(config, version=None):
192
rev = None
193
if version is not None:
194
rev = get_snapshot_revision(str(version.upstream_version))
195
if rev is None:
196
rev = config._get_best_opt('export-upstream-revision')
197
if rev is not None and version is not None:
198
rev = rev.replace('$UPSTREAM_VERSION',
199
str(version.upstream_version))
200
return rev
201
202
203
def suite_to_distribution(suite):
204
"""Infer the distribution from a suite.
205
206
When passed the name of a suite (anything in the distributions field of
207
a changelog) it will infer the distribution from that (i.e. Debian or
208
Ubuntu).
209
210
:param suite: the string containing the suite
211
:return: "debian", "ubuntu", or None if the distribution couldn't be inferred.
212
"""
213
debian_releases = ('woody', 'sarge', 'etch', 'lenny', 'squeeze', 'stable',
214
'testing', 'unstable', 'experimental', 'frozen', 'sid')
215
debian_targets = ('', '-security', '-proposed-updates', '-backports')
216
ubuntu_releases = ('warty', 'hoary', 'breezy', 'dapper', 'edgy',
217
'feisty', 'gutsy', 'hardy', 'intrepid', 'jaunty', 'karmic',
218
'lucid')
219
ubuntu_targets = ('', '-proposed', '-updates', '-security', '-backports')
220
all_debian = [r + t for r in debian_releases for t in debian_targets]
221
all_ubuntu = [r + t for r in ubuntu_releases for t in ubuntu_targets]
222
if suite in all_debian:
223
return "debian"
224
if suite in all_ubuntu:
225
return "ubuntu"
226
return None
227
228
229
def lookup_distribution(distribution_or_suite):
230
"""Get the distribution name based on a distribtion or suite name.
231
232
:param distribution_or_suite: a string that is either the name of
233
a distribution or a suite.
234
:return: a string with a distribution name or None.
235
"""
236
if distribution_or_suite.lower() in ("debian", "ubuntu"):
237
return distribution_or_suite.lower()
238
return suite_to_distribution(distribution_or_suite)
239
240
241
def move_file_if_different(source, target, md5sum):
242
if os.path.exists(target):
243
if os.path.samefile(source, target):
244
return
245
t_md5sum = md5.md5()
246
target_f = open(target)
247
try:
248
for line in target_f:
249
t_md5sum.update(line)
250
finally:
251
target_f.close()
252
if t_md5sum.hexdigest() == md5sum:
253
return
254
shutil.move(source, target)
255
256
257
def write_if_different(contents, target):
258
md5sum = md5.md5()
259
md5sum.update(contents)
260
fd, temp_path = tempfile.mkstemp("builddeb-rename-")
261
fobj = os.fdopen(fd, "wd")
262
try:
263
try:
264
fobj.write(contents)
265
finally:
266
fobj.close()
267
move_file_if_different(temp_path, target, md5sum.hexdigest())
268
finally:
269
if os.path.exists(temp_path):
270
os.unlink(temp_path)
271
272
273
def _download_part(name, base_transport, target_dir, md5sum):
274
part_base_dir, part_path = urlutils.split(name)
275
f_t = base_transport
276
if part_base_dir != '':
277
f_t = base_transport.clone(part_base_dir)
278
f_f = f_t.get(part_path)
279
try:
280
target_path = os.path.join(target_dir, part_path)
281
fd, temp_path = tempfile.mkstemp(prefix="builldeb-")
282
fobj = os.fdopen(fd, "wb")
283
try:
284
try:
285
shutil.copyfileobj(f_f, fobj)
286
finally:
287
fobj.close()
288
move_file_if_different(temp_path, target_path, md5sum)
289
finally:
290
if os.path.exists(temp_path):
291
os.unlink(temp_path)
292
finally:
293
f_f.close()
294
295
296
def open_file(path):
297
filename, transport = open_transport(path)
298
return open_file_via_transport(filename, transport)
299
300
301
def open_transport(path):
302
"""Obtain an appropriate transport instance for the given path."""
303
base_dir, path = urlutils.split(path)
304
transport = get_transport(base_dir)
305
return (path, transport)
306
307
308
def open_file_via_transport(filename, transport):
309
"""Open a file using the transport, follow redirects as necessary."""
310
def open_file(transport):
311
return transport.get(filename)
312
def follow_redirection(transport, e, redirection_notice):
313
mutter(redirection_notice)
314
_filename, redirected_transport = open_transport(e.target)
315
return redirected_transport
316
317
result = do_catching_redirections(open_file, transport, follow_redirection)
318
return result
319
320
321
def _dget(cls, dsc_location, target_dir):
322
if not os.path.isdir(target_dir):
323
raise errors.NotADirectory(target_dir)
324
path, dsc_t = open_transport(dsc_location)
325
dsc_contents = open_file_via_transport(path, dsc_t).read()
326
dsc = cls(dsc_contents)
327
for file_details in dsc['files']:
328
name = file_details['name']
329
_download_part(name, dsc_t, target_dir, file_details['md5sum'])
330
target_file = os.path.join(target_dir, path)
331
write_if_different(dsc_contents, target_file)
332
return target_file
333
334
335
def dget(dsc_location, target_dir):
336
return _dget(deb822.Dsc, dsc_location, target_dir)
337
338
339
def dget_changes(changes_location, target_dir):
340
return _dget(deb822.Changes, changes_location, target_dir)
341
342
343
def get_parent_dir(target):
344
parent = os.path.dirname(target)
345
if os.path.basename(target) == '':
346
parent = os.path.dirname(parent)
347
return parent
348
349
350
def find_bugs_fixed(changes, branch, _lplib=None):
351
if _lplib is None:
352
from bzrlib.plugins.builddeb import launchpad as _lplib
353
bugs = []
354
for change in changes:
355
for match in re.finditer("closes:\s*(?:bug)?\#?\s?\d+"
356
"(?:,\s*(?:bug)?\#?\s?\d+)*", change,
357
re.IGNORECASE):
358
closes_list = match.group(0)
359
for match in re.finditer("\d+", closes_list):
360
bug_url = bugtracker.get_bug_url("deb", branch,
361
match.group(0))
362
bugs.append(bug_url + " fixed")
363
lp_bugs = _lplib.ubuntu_bugs_for_debian_bug(match.group(0))
364
if len(lp_bugs) == 1:
365
bug_url = bugtracker.get_bug_url("lp", branch,
366
lp_bugs[0])
367
bugs.append(bug_url + " fixed")
368
for match in re.finditer("lp:\s+\#\d+(?:,\s*\#\d+)*",
369
change, re.IGNORECASE):
370
closes_list = match.group(0)
371
for match in re.finditer("\d+", closes_list):
372
bug_url = bugtracker.get_bug_url("lp", branch,
373
match.group(0))
374
bugs.append(bug_url + " fixed")
375
deb_bugs = _lplib.debian_bugs_for_ubuntu_bug(match.group(0))
376
if len(deb_bugs) == 1:
377
bug_url = bugtracker.get_bug_url("deb", branch,
378
deb_bugs[0])
379
bugs.append(bug_url + " fixed")
380
return bugs
381
382
383
def find_extra_authors(changes):
384
extra_author_re = re.compile(r"\s*\[([^\]]+)]\s*", re.UNICODE)
385
authors = []
386
for change in changes:
387
# Parse out any extra authors.
388
match = extra_author_re.match(change.decode("utf-8"))
389
if match is not None:
390
new_author = match.group(1).strip()
391
already_included = False
392
for author in authors:
393
if author.startswith(new_author):
394
already_included = True
395
break
396
if not already_included:
397
authors.append(new_author)
398
return authors
399
400
401
def find_thanks(changes):
402
thanks_re = re.compile(r"[tT]hank(?:(?:s)|(?:you))(?:\s*to)?"
403
"((?:\s+(?:(?:[A-Z]\.)|(?:[A-Z]\w+(?:-[A-Z]\w+)*)))+"
404
"(?:\s+<[^@>]+@[^@>]+>)?)",
405
re.UNICODE)
406
thanks = []
407
changes_str = " ".join(changes).decode("utf-8")
408
for match in thanks_re.finditer(changes_str):
409
if thanks is None:
410
thanks = []
411
thanks_str = match.group(1).strip()
412
thanks_str = re.sub(r"\s+", " ", thanks_str)
413
thanks.append(thanks_str)
414
return thanks
415
416
417
def get_commit_info_from_changelog(changelog, branch, _lplib=None):
418
"""Retrieves the messages from the last section of debian/changelog.
419
420
Reads the latest stanza of debian/changelog and returns the
421
text of the changes in that section. It also returns other
422
information about the change, including the authors of the change,
423
anyone that is thanked, and the bugs that are declared fixed by it.
424
425
:return: a tuple (message, authors, thanks, bugs). message is the
426
commit message that should be used. authors is a list of strings,
427
with those that contributed to the change, thanks is a list
428
of string, with those who were thanked in the changelog entry.
429
bugs is a list of bug URLs like for --fixes.
430
If the information is not available then any can be None.
431
"""
432
message = None
433
authors = []
434
thanks = []
435
bugs = []
436
if changelog._blocks:
437
block = changelog._blocks[0]
438
authors = [block.author.decode("utf-8")]
439
changes = strip_changelog_message(block.changes())
440
authors += find_extra_authors(changes)
441
bugs = find_bugs_fixed(changes, branch, _lplib=_lplib)
442
thanks = find_thanks(changes)
443
message = "\n".join(changes).replace("\r", "")
444
return (message, authors, thanks, bugs)
445
446
447
def find_last_distribution(changelog):
448
"""Find the last changelog that was used in a changelog.
449
450
This will skip stanzas with the 'UNRELEASED' distribution.
451
452
:param changelog: Changelog to analyze
453
"""
454
for block in changelog._blocks:
455
distribution = block.distributions.split(" ")[0]
456
if distribution != "UNRELEASED":
457
return distribution
458
return None
459
460
461
def subprocess_setup():
462
# Python installs a SIGPIPE handler by default. This is usually not what
463
# non-Python subprocesses expect.
464
# Many, many thanks to Colin Watson
465
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1