« back to all changes in this revision
Viewing changes to parser.py
- browse files at revision 9
- compare with another revision
- download diff
- download tarball
- view history from revision 9
- Committer: Bazaar Package Importer
- Author(s): Jelmer Vernooij
- Date: 2010-11-06 18:40:27 UTC
- mfrom: (1.1.6 upstream)
- Revision ID: james.westby@ubuntu.com-20101106184027-iclo8iim9equ6i8b
Tags: 0.9.0+bzr279-1
* New upstream snapshot.
* Bump standards version to 3.9.1 (no changes).
* Run testsuite during package build.
* Bump standards version to 3.9.1 (no changes).
* Run testsuite during package build.
- files added:
- .pc
- .pc/debian-changes-0.9.0+bzr279-1
- .pc/debian-changes-0.9.0+bzr279-1/tests
- debian/patches
- files removed:
- commands.py
- files modified:
- NEWS
added
removed
parser.py
1
# Copyright (C) 2008 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
17
"""Parser of import data into command objects.
18
19
In order to reuse existing front-ends, the stream format is a subset of
20
the one used by git-fast-import (as of the 1.5.4 release of git at least).
21
The grammar is:
22
23
stream ::= cmd*;
24
25
cmd ::= new_blob
26
| new_commit
27
| new_tag
28
| reset_branch
29
| checkpoint
30
| progress
31
;
32
33
new_blob ::= 'blob' lf
34
mark?
35
file_content;
36
file_content ::= data;
37
38
new_commit ::= 'commit' sp ref_str lf
39
mark?
40
('author' sp name '<' email '>' when lf)?
41
'committer' sp name '<' email '>' when lf
42
commit_msg
43
('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
44
('merge' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)*
45
file_change*
46
lf?;
47
commit_msg ::= data;
48
49
file_change ::= file_clr
50
| file_del
51
| file_rnm
52
| file_cpy
53
| file_obm
54
| file_inm;
55
file_clr ::= 'deleteall' lf;
56
file_del ::= 'D' sp path_str lf;
57
file_rnm ::= 'R' sp path_str sp path_str lf;
58
file_cpy ::= 'C' sp path_str sp path_str lf;
59
file_obm ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf;
60
file_inm ::= 'M' sp mode sp 'inline' sp path_str lf
61
data;
62
63
new_tag ::= 'tag' sp tag_str lf
64
'from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf
65
'tagger' sp name '<' email '>' when lf
66
tag_msg;
67
tag_msg ::= data;
68
69
reset_branch ::= 'reset' sp ref_str lf
70
('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
71
lf?;
72
73
checkpoint ::= 'checkpoint' lf
74
lf?;
75
76
progress ::= 'progress' sp not_lf* lf
77
lf?;
78
79
# note: the first idnum in a stream should be 1 and subsequent
80
# idnums should not have gaps between values as this will cause
81
# the stream parser to reserve space for the gapped values. An
82
# idnum can be updated in the future to a new object by issuing
83
# a new mark directive with the old idnum.
84
#
85
mark ::= 'mark' sp idnum lf;
86
data ::= (delimited_data | exact_data)
87
lf?;
88
89
# note: delim may be any string but must not contain lf.
90
# data_line may contain any data but must not be exactly
91
# delim. The lf after the final data_line is included in
92
# the data.
93
delimited_data ::= 'data' sp '<<' delim lf
94
(data_line lf)*
95
delim lf;
96
97
# note: declen indicates the length of binary_data in bytes.
98
# declen does not include the lf preceeding the binary data.
99
#
100
exact_data ::= 'data' sp declen lf
101
binary_data;
102
103
# note: quoted strings are C-style quoting supporting \c for
104
# common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
105
# is the signed byte value in octal. Note that the only
106
# characters which must actually be escaped to protect the
107
# stream formatting is: \, " and LF. Otherwise these values
108
# are UTF8.
109
#
110
ref_str ::= ref;
111
sha1exp_str ::= sha1exp;
112
tag_str ::= tag;
113
path_str ::= path | '"' quoted(path) '"' ;
114
mode ::= '100644' | '644'
115
| '100755' | '755'
116
| '120000'
117
;
118
119
declen ::= # unsigned 32 bit value, ascii base10 notation;
120
bigint ::= # unsigned integer value, ascii base10 notation;
121
binary_data ::= # file content, not interpreted;
122
123
when ::= raw_when | rfc2822_when;
124
raw_when ::= ts sp tz;
125
rfc2822_when ::= # Valid RFC 2822 date and time;
126
127
sp ::= # ASCII space character;
128
lf ::= # ASCII newline (LF) character;
129
130
# note: a colon (':') must precede the numerical value assigned to
131
# an idnum. This is to distinguish it from a ref or tag name as
132
# GIT does not permit ':' in ref or tag strings.
133
#
134
idnum ::= ':' bigint;
135
path ::= # GIT style file path, e.g. "a/b/c";
136
ref ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
137
tag ::= # GIT tag name, e.g. "FIREFOX_1_5";
138
sha1exp ::= # Any valid GIT SHA1 expression;
139
hexsha1 ::= # SHA1 in hexadecimal format;
140
141
# note: name and email are UTF8 strings, however name must not
142
# contain '<' or lf and email must not contain any of the
143
# following: '<', '>', lf.
144
#
145
name ::= # valid GIT author/committer name;
146
email ::= # valid GIT author/committer email;
147
ts ::= # time since the epoch in seconds, ascii base10 notation;
148
tz ::= # GIT style timezone;
149
150
# note: comments may appear anywhere in the input, except
151
# within a data command. Any form of the data command
152
# always escapes the related input from comment processing.
153
#
154
# In case it is not clear, the '#' that starts the comment
155
# must be the first character on that the line (an lf have
156
# preceeded it).
157
#
158
comment ::= '#' not_lf* lf;
159
not_lf ::= # Any byte that is not ASCII newline (LF);
160
"""
161
162
163
import re
164
import sys
165
166
import commands
167
import dates
168
import errors
169
170
171
## Stream parsing ##
172
173
class LineBasedParser(object):
174
175
def __init__(self, input):
176
"""A Parser that keeps track of line numbers.
177
178
:param input: the file-like object to read from
179
"""
180
self.input = input
181
self.lineno = 0
182
# Lines pushed back onto the input stream
183
self._buffer = []
184
185
def abort(self, exception, *args):
186
"""Raise an exception providing line number information."""
187
raise exception(self.lineno, *args)
188
189
def readline(self):
190
"""Get the next line including the newline or '' on EOF."""
191
self.lineno += 1
192
if self._buffer:
193
return self._buffer.pop()
194
else:
195
return self.input.readline()
196
197
def next_line(self):
198
"""Get the next line without the newline or None on EOF."""
199
line = self.readline()
200
if line:
201
return line[:-1]
202
else:
203
return None
204
205
def push_line(self, line):
206
"""Push line back onto the line buffer.
207
208
:param line: the line with no trailing newline
209
"""
210
self.lineno -= 1
211
self._buffer.append(line + "\n")
212
213
def read_bytes(self, count):
214
"""Read a given number of bytes from the input stream.
215
216
Throws MissingBytes if the bytes are not found.
217
218
Note: This method does not read from the line buffer.
219
220
:return: a string
221
"""
222
result = self.input.read(count)
223
found = len(result)
224
self.lineno += result.count("\n")
225
if found != count:
226
self.abort(errors.MissingBytes, count, found)
227
return result
228
229
def read_until(self, terminator):
230
"""Read the input stream until the terminator is found.
231
232
Throws MissingTerminator if the terminator is not found.
233
234
Note: This method does not read from the line buffer.
235
236
:return: the bytes read up to but excluding the terminator.
237
"""
238
239
lines = []
240
term = terminator + '\n'
241
while True:
242
line = self.input.readline()
243
if line == term:
244
break
245
else:
246
lines.append(line)
247
return ''.join(lines)
248
249
250
# Regular expression used for parsing. (Note: The spec states that the name
251
# part should be non-empty but git-fast-export doesn't always do that so
252
# the first bit is \w*, not \w+.) Also git-fast-import code says the
253
# space before the email is optional.
254
_WHO_AND_WHEN_RE = re.compile(r'([^<]*)<(.*)> (.+)')
255
_WHO_RE = re.compile(r'([^<]*)<(.*)>')
256
257
258
class ImportParser(LineBasedParser):
259
260
def __init__(self, input, verbose=False, output=sys.stdout,
261
user_mapper=None):
262
"""A Parser of import commands.
263
264
:param input: the file-like object to read from
265
:param verbose: display extra information of not
266
:param output: the file-like object to write messages to (YAGNI?)
267
:param user_mapper: if not None, the UserMapper used to adjust
268
user-ids for authors, committers and taggers.
269
"""
270
LineBasedParser.__init__(self, input)
271
self.verbose = verbose
272
self.output = output
273
self.user_mapper = user_mapper
274
# We auto-detect the date format when a date is first encountered
275
self.date_parser = None
276
277
def warning(self, msg):
278
sys.stderr.write("warning line %d: %s\n" % (self.lineno, msg))
279
280
def iter_commands(self):
281
"""Iterator returning ImportCommand objects."""
282
while True:
283
line = self.next_line()
284
if line is None:
285
break
286
elif len(line) == 0 or line.startswith('#'):
287
continue
288
# Search for commands in order of likelihood
289
elif line.startswith('commit '):
290
yield self._parse_commit(line[len('commit '):])
291
elif line.startswith('blob'):
292
yield self._parse_blob()
293
elif line.startswith('progress '):
294
yield commands.ProgressCommand(line[len('progress '):])
295
elif line.startswith('reset '):
296
yield self._parse_reset(line[len('reset '):])
297
elif line.startswith('tag '):
298
yield self._parse_tag(line[len('tag '):])
299
elif line.startswith('checkpoint'):
300
yield commands.CheckpointCommand()
301
elif line.startswith('feature'):
302
yield self._parse_feature(line[len('feature '):])
303
else:
304
self.abort(errors.InvalidCommand, line)
305
306
def iter_file_commands(self):
307
"""Iterator returning FileCommand objects.
308
309
If an invalid file command is found, the line is silently
310
pushed back and iteration ends.
311
"""
312
while True:
313
line = self.next_line()
314
if line is None:
315
break
316
elif len(line) == 0 or line.startswith('#'):
317
continue
318
# Search for file commands in order of likelihood
319
elif line.startswith('M '):
320
yield self._parse_file_modify(line[2:])
321
elif line.startswith('D '):
322
path = self._path(line[2:])
323
yield commands.FileDeleteCommand(path)
324
elif line.startswith('R '):
325
old, new = self._path_pair(line[2:])
326
yield commands.FileRenameCommand(old, new)
327
elif line.startswith('C '):
328
src, dest = self._path_pair(line[2:])
329
yield commands.FileCopyCommand(src, dest)
330
elif line.startswith('deleteall'):
331
yield commands.FileDeleteAllCommand()
332
else:
333
self.push_line(line)
334
break
335
336
def _parse_blob(self):
337
"""Parse a blob command."""
338
lineno = self.lineno
339
mark = self._get_mark_if_any()
340
data = self._get_data('blob')
341
return commands.BlobCommand(mark, data, lineno)
342
343
def _parse_commit(self, ref):
344
"""Parse a commit command."""
345
lineno = self.lineno
346
mark = self._get_mark_if_any()
347
author = self._get_user_info('commit', 'author', False)
348
more_authors = []
349
while True:
350
another_author = self._get_user_info('commit', 'author', False)
351
if another_author is not None:
352
more_authors.append(another_author)
353
else:
354
break
355
committer = self._get_user_info('commit', 'committer')
356
message = self._get_data('commit', 'message')
357
try:
358
message = message.decode('utf_8')
359
except UnicodeDecodeError:
360
self.warning(
361
"commit message not in utf8 - replacing unknown characters")
362
message = message.decode('utf_8', 'replace')
363
from_ = self._get_from()
364
merges = []
365
while True:
366
merge = self._get_merge()
367
if merge is not None:
368
# while the spec suggests it's illegal, git-fast-export
369
# outputs multiple merges on the one line, e.g.
370
# merge :x :y :z
371
these_merges = merge.split(" ")
372
merges.extend(these_merges)
373
else:
374
break
375
properties = {}
376
while True:
377
name_value = self._get_property()
378
if name_value is not None:
379
name, value = name_value
380
properties[name] = value
381
else:
382
break
383
return commands.CommitCommand(ref, mark, author, committer, message,
384
from_, merges, self.iter_file_commands, lineno=lineno,
385
more_authors=more_authors, properties=properties)
386
387
def _parse_feature(self, info):
388
"""Parse a feature command."""
389
parts = info.split("=", 1)
390
name = parts[0]
391
if len(parts) > 1:
392
value = self._path(parts[1])
393
else:
394
value = None
395
return commands.FeatureCommand(name, value, lineno=self.lineno)
396
397
def _parse_file_modify(self, info):
398
"""Parse a filemodify command within a commit.
399
400
:param info: a string in the format "mode dataref path"
401
(where dataref might be the hard-coded literal 'inline').
402
"""
403
params = info.split(' ', 2)
404
path = self._path(params[2])
405
is_executable, kind = self._mode(params[0])
406
if params[1] == 'inline':
407
dataref = None
408
data = self._get_data('filemodify')
409
else:
410
dataref = params[1]
411
data = None
412
return commands.FileModifyCommand(path, kind, is_executable, dataref,
413
data)
414
415
def _parse_reset(self, ref):
416
"""Parse a reset command."""
417
from_ = self._get_from()
418
return commands.ResetCommand(ref, from_)
419
420
def _parse_tag(self, name):
421
"""Parse a tag command."""
422
from_ = self._get_from('tag')
423
tagger = self._get_user_info('tag', 'tagger', accept_just_who=True)
424
message = self._get_data('tag', 'message').decode('utf_8')
425
return commands.TagCommand(name, from_, tagger, message)
426
427
def _get_mark_if_any(self):
428
"""Parse a mark section."""
429
line = self.next_line()
430
if line.startswith('mark :'):
431
return line[len('mark :'):]
432
else:
433
self.push_line(line)
434
return None
435
436
def _get_from(self, required_for=None):
437
"""Parse a from section."""
438
line = self.next_line()
439
if line is None:
440
return None
441
elif line.startswith('from '):
442
return line[len('from '):]
443
elif required_for:
444
self.abort(errors.MissingSection, required_for, 'from')
445
else:
446
self.push_line(line)
447
return None
448
449
def _get_merge(self):
450
"""Parse a merge section."""
451
line = self.next_line()
452
if line is None:
453
return None
454
elif line.startswith('merge '):
455
return line[len('merge '):]
456
else:
457
self.push_line(line)
458
return None
459
460
def _get_property(self):
461
"""Parse a property section."""
462
line = self.next_line()
463
if line is None:
464
return None
465
elif line.startswith('property '):
466
return self._name_value(line[len('property '):])
467
else:
468
self.push_line(line)
469
return None
470
471
def _get_user_info(self, cmd, section, required=True,
472
accept_just_who=False):
473
"""Parse a user section."""
474
line = self.next_line()
475
if line.startswith(section + ' '):
476
return self._who_when(line[len(section + ' '):], cmd, section,
477
accept_just_who=accept_just_who)
478
elif required:
479
self.abort(errors.MissingSection, cmd, section)
480
else:
481
self.push_line(line)
482
return None
483
484
def _get_data(self, required_for, section='data'):
485
"""Parse a data section."""
486
line = self.next_line()
487
if line.startswith('data '):
488
rest = line[len('data '):]
489
if rest.startswith('<<'):
490
return self.read_until(rest[2:])
491
else:
492
size = int(rest)
493
read_bytes = self.read_bytes(size)
494
# optional LF after data.
495
next = self.input.readline()
496
self.lineno += 1
497
if len(next) > 1 or next != "\n":
498
self.push_line(next[:-1])
499
return read_bytes
500
else:
501
self.abort(errors.MissingSection, required_for, section)
502
503
def _who_when(self, s, cmd, section, accept_just_who=False):
504
"""Parse who and when information from a string.
505
506
:return: a tuple of (name,email,timestamp,timezone). name may be
507
the empty string if only an email address was given.
508
"""
509
match = _WHO_AND_WHEN_RE.search(s)
510
if match:
511
datestr = match.group(3).lstrip()
512
if self.date_parser is None:
513
# auto-detect the date format
514
if len(datestr.split(' ')) == 2:
515
format = 'raw'
516
elif datestr == 'now':
517
format = 'now'
518
else:
519
format = 'rfc2822'
520
self.date_parser = dates.DATE_PARSERS_BY_NAME[format]
521
try:
522
when = self.date_parser(datestr, self.lineno)
523
except ValueError:
524
print "failed to parse datestr '%s'" % (datestr,)
525
raise
526
else:
527
match = _WHO_RE.search(s)
528
if accept_just_who and match:
529
# HACK around missing time
530
# TODO: output a warning here
531
when = dates.DATE_PARSERS_BY_NAME['now']('now')
532
else:
533
self.abort(errors.BadFormat, cmd, section, s)
534
name = match.group(1)
535
if len(name) > 0:
536
if name[-1] == " ":
537
try:
538
name = name[:-1].decode('utf_8')
539
except UnicodeDecodeError:
540
# The spec says names are *typically* utf8 encoded
541
# but that isn't enforced by git-fast-export (at least)
542
self.warning("%s name not in utf8 - replacing unknown "
543
"characters" % (section,))
544
name = name[:-1].decode('utf_8', 'replace')
545
email = match.group(2)
546
# While it shouldn't happen, some datasets have email addresses
547
# which contain unicode characters. See bug 338186. We sanitize
548
# the data at this level just in case.
549
try:
550
email = email.decode('utf_8')
551
except UnicodeDecodeError:
552
self.warning("%s email not in utf8 - replacing unknown characters"
553
% (section,))
554
email = email.decode('utf_8', 'replace')
555
if self.user_mapper:
556
name, email = self.user_mapper.map_name_and_email(name, email)
557
return (name, email, when[0], when[1])
558
559
def _name_value(self, s):
560
"""Parse a (name,value) tuple from 'name value-length value'."""
561
parts = s.split(' ', 2)
562
name = parts[0]
563
if len(parts) == 1:
564
value = None
565
else:
566
size = int(parts[1])
567
value = parts[2]
568
still_to_read = size - len(value)
569
if still_to_read > 0:
570
read_bytes = self.read_bytes(still_to_read)
571
value += "\n" + read_bytes[:still_to_read - 1]
572
value = value.decode('utf8')
573
return (name, value)
574
575
def _path(self, s):
576
"""Parse a path."""
577
if s.startswith('"'):
578
if s[-1] != '"':
579
self.abort(errors.BadFormat, '?', '?', s)
580
else:
581
return _unquote_c_string(s[1:-1])
582
try:
583
return s.decode('utf_8')
584
except UnicodeDecodeError:
585
# The spec recommends utf8 encoding but that isn't enforced
586
return s
587
588
def _path_pair(self, s):
589
"""Parse two paths separated by a space."""
590
# TODO: handle a space in the first path
591
if s.startswith('"'):
592
parts = s[1:].split('" ', 1)
593
else:
594
parts = s.split(' ', 1)
595
if len(parts) != 2:
596
self.abort(errors.BadFormat, '?', '?', s)
597
elif parts[1].startswith('"') and parts[1].endswith('"'):
598
parts[1] = parts[1][1:-1]
599
elif parts[1].startswith('"') or parts[1].endswith('"'):
600
self.abort(errors.BadFormat, '?', '?', s)
601
return map(_unquote_c_string, parts)
602
603
def _mode(self, s):
604
"""Parse a file mode into executable and kind.
605
606
:return (is_executable, kind)
607
"""
608
# Note: Output from git-fast-export slightly different to spec
609
if s in ['644', '100644', '0100644']:
610
return False, commands.FILE_KIND
611
elif s in ['755', '100755', '0100755']:
612
return True, commands.FILE_KIND
613
elif s in ['040000', '0040000']:
614
return False, commands.DIRECTORY_KIND
615
elif s in ['120000', '0120000']:
616
return False, commands.SYMLINK_KIND
617
elif s in ['160000', '0160000']:
618
return False, commands.TREE_REFERENCE_KIND
619
else:
620
self.abort(errors.BadFormat, 'filemodify', 'mode', s)
621
622
623
def _unquote_c_string(s):
624
"""replace C-style escape sequences (\n, \", etc.) with real chars."""
625
# HACK: Python strings are close enough
626
return s.decode('string_escape', 'replace')
Loggerhead is a web-based interface for Breezy
Version: 2.0.1