« back to all changes in this revision
Viewing changes to duplicity/selection.py.old
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: bescoto
- Date: 2002-10-29 01:49:46 UTC
- Revision ID: vcs-imports@canonical.com-20021029014946-3m4rmm5plom7pl6q
Initial checkin
- files added:
- dist
- duplicity
- testing
added
removed
duplicity/selection.py.old
1
# Copyright 2002 Ben Escoto
2
#
3
# This file is part of rdiff-backup.
4
#
5
# rdiff-backup is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
8
# 02139, USA; either version 2 of the License, or (at your option) any
9
# later version; incorporated herein by reference.
10
11
"""Iterate exactly the requested files in a directory
12
13
Parses includes and excludes to yield correct files. More
14
documentation on what this code does can be found on the man page.
15
16
"""
17
18
from __future__ import generators
19
import re
20
from path import *
21
import robust, log, globals
22
23
24
class SelectError(Exception):
25
"""Some error dealing with the Select class"""
26
pass
27
28
class FilePrefixError(SelectError):
29
"""Signals that a specified file doesn't start with correct prefix"""
30
pass
31
32
class GlobbingError(SelectError):
33
"""Something has gone wrong when parsing a glob string"""
34
pass
35
36
37
class Select:
38
"""Iterate appropriate Paths in given directory
39
40
This class acts as an iterator on account of its next() method.
41
Basically, it just goes through all the files in a directory in
42
order (depth-first) and subjects each file to a bunch of tests
43
(selection functions) in order. The first test that includes or
44
excludes the file means that the file gets included (iterated) or
45
excluded. The default is include, so with no tests we would just
46
iterate all the files in the directory in order.
47
48
The one complication to this is that sometimes we don't know
49
whether or not to include a directory until we examine its
50
contents. For instance, if we want to include all the **.py
51
files. If /home/ben/foo.py exists, we should also include /home
52
and /home/ben, but if these directories contain no **.py files,
53
they shouldn't be included. For this reason, a test may not
54
include or exclude a directory, but merely "scan" it. If later a
55
file in the directory gets included, so does the directory.
56
57
As mentioned above, each test takes the form of a selection
58
function. The selection function takes a path, and returns:
59
60
None - means the test has nothing to say about the related file
61
0 - the file is excluded by the test
62
1 - the file is included
63
2 - the test says the file (must be directory) should be scanned
64
65
Also, a selection function f has a variable f.exclude which should
66
be true iff f could potentially exclude some file. This is used
67
to signal an error if the last function only includes, which would
68
be redundant and presumably isn't what the user intends.
69
70
"""
71
# This re should not match normal filenames, but usually just globs
72
glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S)
73
74
def __init__(self, path):
75
"""Initializer, called with Path of root directory"""
76
assert isinstance(path, Path), path
77
self.selection_functions = []
78
self.rootpath = path
79
self.prefix = self.rootpath.name
80
81
def set_iter(self):
82
"""Initialize generator, prepare to iterate."""
83
self.rootpath.setdata() # this may have changed since Select init
84
self.iter = self.Iterate(self.rootpath)
85
self.next = self.iter.next
86
self.__iter__ = lambda: self
87
return self
88
89
def Iterate(self, path):
90
"""Return iterator yielding paths in path
91
92
This function looks a bit more complicated than it needs to be
93
because it avoids extra recursion (and no extra function calls
94
for non-directory files) while still doing the "directory
95
scanning" bit.
96
97
"""
98
def error_handler(exc, path, filename):
99
log.Log("Error initializing file %s/%s" % (path.name, filename), 2)
100
return None
101
102
def diryield(path):
103
"""Generate relevant files in directory path
104
105
Returns (path, num) where num == 0 means path should be
106
generated normally, num == 1 means the path is a directory
107
and should be included iff something inside is included.
108
109
"""
110
for filename in robust.listpath(path):
111
new_path = robust.check_common_error(
112
error_handler, Path.append, (path, filename))
113
if new_path:
114
s = self.Select(new_path)
115
if s == 1: yield (new_path, 0)
116
elif s == 2 and new_path.isdir(): yield (new_path, 1)
117
118
if not path.type: # base doesn't exist
119
log.Log("Warning: base %s doesn't exist, continuing" %
120
path.name, 2)
121
return
122
log.Log("Selecting %s" % path.name, 7)
123
yield path
124
if not path.isdir(): return
125
diryield_stack = [diryield(path)]
126
delayed_path_stack = []
127
128
while diryield_stack:
129
try: subpath, val = diryield_stack[-1].next()
130
except StopIteration:
131
diryield_stack.pop()
132
if delayed_path_stack: delayed_path_stack.pop()
133
continue
134
if val == 0:
135
if delayed_path_stack:
136
for delayed_path in delayed_path_stack:
137
log.Log("Selecting %s" % delayed_path.name, 7)
138
yield delayed_path
139
del delayed_path_stack[:]
140
log.Log("Selecting %s" % subpath.name, 7)
141
yield subpath
142
if subpath.isdir(): diryield_stack.append(diryield(subpath))
143
elif val == 1:
144
delayed_path_stack.append(subpath)
145
diryield_stack.append(diryield(subpath))
146
147
def Select(self, path):
148
"""Run through the selection functions and return dominant val 0/1/2"""
149
for sf in self.selection_functions:
150
result = sf(path)
151
if result is not None: return result
152
return 1
153
154
def ParseArgs(self, argtuples):
155
"""Create selection functions based on list of tuples
156
157
The tuples are created when the initial commandline arguments
158
are read. They have the form (option string, additional
159
argument) except for the filelist tuples, which should be
160
(option-string, (additional argument, filelist_fp)).
161
162
"""
163
try:
164
for opt, arg in argtuples:
165
if opt == "--exclude":
166
self.add_selection_func(self.glob_get_sf(arg, 0))
167
elif opt == "--exclude-device-files":
168
self.add_selection_func(self.devfiles_get_sf())
169
elif opt == "--exclude-filelist":
170
self.add_selection_func(self.filelist_get_sf(
171
arg[1], 0, arg[0]))
172
elif opt == "--exclude-other-filesystems":
173
self.add_selection_func(self.other_filesystems_get_sf(0))
174
elif opt == "--exclude-regexp":
175
self.add_selection_func(self.regexp_get_sf(arg, 0))
176
elif opt == "--include":
177
self.add_selection_func(self.glob_get_sf(arg, 1))
178
elif opt == "--include-filelist":
179
self.add_selection_func(self.filelist_get_sf(
180
arg[1], 1, arg[0]))
181
elif opt == "--include-regexp":
182
self.add_selection_func(self.regexp_get_sf(arg, 1))
183
else: assert 0, "Bad selection option %s" % opt
184
except SelectError, e: self.parse_catch_error(e)
185
self.parse_last_excludes()
186
187
def parse_catch_error(self, exc):
188
"""Deal with selection error exc"""
189
if isinstance(exc, FilePrefixError):
190
log.FatalError(
191
"""Fatal Error: The file specification
192
%s
193
cannot match any files in the base directory
194
%s
195
Useful file specifications begin with the base directory or some
196
pattern (such as '**') which matches the base directory.""" %
197
(exc, self.prefix))
198
elif isinstance(e, GlobbingError):
199
log.FatalError("Fatal Error while processing expression\n"
200
"%s" % exc)
201
else: raise
202
203
def parse_last_excludes(self):
204
"""Exit with error if last selection function isn't an exclude"""
205
if (self.selection_functions and
206
not self.selection_functions[-1].exclude):
207
log.FatalError(
208
"""Last selection expression:
209
%s
210
only specifies that files be included. Because the default is to
211
include all files, the expression is redundant. Exiting because this
212
probably isn't what you meant.""" %
213
(self.selection_functions[-1].name,))
214
215
def add_selection_func(self, sel_func, add_to_start = None):
216
"""Add another selection function at the end or beginning"""
217
if add_to_start: self.selection_functions.insert(0, sel_func)
218
else: self.selection_functions.append(sel_func)
219
220
def filelist_get_sf(self, filelist_fp, inc_default, filelist_name):
221
"""Return selection function by reading list of files
222
223
The format of the filelist is documented in the man page.
224
filelist_fp should be an (open) file object.
225
inc_default should be true if this is an include list,
226
false for an exclude list.
227
filelist_name is just a string used for logging.
228
229
"""
230
log.Log("Reading filelist %s" % filelist_name, 4)
231
tuple_list, something_excluded = \
232
self.filelist_read(filelist_fp, inc_default, filelist_name)
233
log.Log("Sorting filelist %s" % filelist_name, 4)
234
tuple_list.sort()
235
i = [0] # We have to put index in list because of stupid scoping rules
236
237
def selection_function(path):
238
while 1:
239
if i[0] >= len(tuple_list): return None
240
include, move_on = \
241
self.filelist_pair_match(path, tuple_list[i[0]])
242
if move_on:
243
i[0] += 1
244
if include is None: continue # later line may match
245
return include
246
247
selection_function.exclude = something_excluded or inc_default == 0
248
selection_function.name = "Filelist: " + filelist_name
249
return selection_function
250
251
def filelist_read(self, filelist_fp, include, filelist_name):
252
"""Read filelist from fp, return (tuplelist, something_excluded)"""
253
prefix_warnings = [0]
254
def incr_warnings(exc):
255
"""Warn if prefix is incorrect"""
256
prefix_warnings[0] += 1
257
if prefix_warnings[0] < 6:
258
log.Log("Warning: file specification '%s' in filelist %s\n"
259
"doesn't start with correct prefix %s. Ignoring." %
260
(exc, filelist_name, self.prefix), 2)
261
if prefix_warnings[0] == 5:
262
log.Log("Future prefix errors will not be logged.", 2)
263
264
something_excluded, tuple_list = None, []
265
separator = globals.null_separator and "\0" or "\n"
266
for line in filelist_fp.read().split(separator):
267
if not line: continue # skip blanks
268
try: tuple = self.filelist_parse_line(line, include)
269
except FilePrefixError, exc:
270
incr_warnings(exc)
271
continue
272
tuple_list.append(tuple)
273
if not tuple[1]: something_excluded = 1
274
if filelist_fp.close():
275
log.Log("Error closing filelist %s" % filelist_name, 2)
276
return (tuple_list, something_excluded)
277
278
def filelist_parse_line(self, line, include):
279
"""Parse a single line of a filelist, returning a pair
280
281
pair will be of form (index, include), where index is another
282
tuple, and include is 1 if the line specifies that we are
283
including a file. The default is given as an argument.
284
prefix is the string that the index is relative to.
285
286
"""
287
line = line.strip()
288
if line[:2] == "+ ": # Check for "+ "/"- " syntax
289
include = 1
290
line = line[2:]
291
elif line[:2] == "- ":
292
include = 0
293
line = line[2:]
294
295
if not line.startswith(self.prefix): raise FilePrefixError(line)
296
line = line[len(self.prefix):] # Discard prefix
297
index = tuple(filter(lambda x: x, line.split("/"))) # remove empties
298
return (index, include)
299
300
def filelist_pair_match(self, path, pair):
301
"""Matches a filelist tuple against a path
302
303
Returns a pair (include, move_on). include is None if the
304
tuple doesn't match either way, and 0/1 if the tuple excludes
305
or includes the path.
306
307
move_on is true if the tuple cannot match a later index, and
308
so we should move on to the next tuple in the index.
309
310
"""
311
index, include = pair
312
if include == 1:
313
if index < path.index: return (None, 1)
314
if index == path.index: return (1, 1)
315
elif index[:len(path.index)] == path.index:
316
return (1, None) # /foo/bar implicitly includes /foo
317
else: return (None, None) # path greater, not initial sequence
318
elif include == 0:
319
if path.index[:len(index)] == index:
320
return (0, None) # /foo implicitly excludes /foo/bar
321
elif index < path.index: return (None, 1)
322
else: return (None, None) # path greater, not initial sequence
323
else: assert 0, "Include is %s, should be 0 or 1" % (include,)
324
325
def other_filesystems_get_sf(self, include):
326
"""Return selection function matching files on other filesystems"""
327
assert include == 0 or include == 1
328
root_devloc = self.rootpath.getdevloc()
329
def sel_func(path):
330
if path.getdevloc() == root_devloc: return None
331
else: return include
332
sel_func.exclude = not include
333
sel_func.name = "Match other filesystems"
334
return sel_func
335
336
def regexp_get_sf(self, regexp_string, include):
337
"""Return selection function given by regexp_string"""
338
assert include == 0 or include == 1
339
try: regexp = re.compile(regexp_string)
340
except:
341
log.Log("Error compiling regular expression %s" % regexp_string, 1)
342
raise
343
344
def sel_func(path):
345
if regexp.search(path.name): return include
346
else: return None
347
348
sel_func.exclude = not include
349
sel_func.name = "Regular expression: %s" % regexp_string
350
return sel_func
351
352
def devfiles_get_sf(self):
353
"""Return a selection function to exclude all dev files"""
354
if self.selection_functions:
355
log.Log("Warning: exclude-device-files is not the first "
356
"selector.\nThis may not be what you intended", 3)
357
def sel_func(path):
358
if path.isdev(): return 0
359
else: return None
360
sel_func.exclude = 1
361
sel_func.name = "Exclude device files"
362
return sel_func
363
364
def glob_get_sf(self, glob_str, include):
365
"""Return selection function given by glob string"""
366
assert include == 0 or include == 1
367
if glob_str == "**": sel_func = lambda path: include
368
elif not self.glob_re.match(glob_str): # normal file
369
sel_func = self.glob_get_filename_sf(glob_str, include)
370
else: sel_func = self.glob_get_normal_sf(glob_str, include)
371
372
sel_func.exclude = not include
373
sel_func.name = "Command-line %s glob: %s" % \
374
(include and "include" or "exclude", glob_str)
375
return sel_func
376
377
def glob_get_filename_sf(self, filename, include):
378
"""Get a selection function given a normal filename
379
380
Some of the parsing is better explained in
381
filelist_parse_line. The reason this is split from normal
382
globbing is things are a lot less complicated if no special
383
globbing characters are used.
384
385
"""
386
if not filename.startswith(self.prefix):
387
raise FilePrefixError(filename)
388
index = tuple(filter(lambda x: x,
389
filename[len(self.prefix):].split("/")))
390
return self.glob_get_tuple_sf(index, include)
391
392
def glob_get_tuple_sf(self, tuple, include):
393
"""Return selection function based on tuple"""
394
def include_sel_func(path):
395
if (path.index == tuple[:len(path.index)] or
396
path.index[:len(tuple)] == tuple):
397
return 1 # /foo/bar implicitly matches /foo, vice-versa
398
else: return None
399
400
def exclude_sel_func(path):
401
if path.index[:len(tuple)] == tuple:
402
return 0 # /foo excludes /foo/bar, not vice-versa
403
else: return None
404
405
if include == 1: sel_func = include_sel_func
406
elif include == 0: sel_func = exclude_sel_func
407
sel_func.exclude = not include
408
sel_func.name = "Tuple select %s" % (tuple,)
409
return sel_func
410
411
def glob_get_normal_sf(self, glob_str, include):
412
"""Return selection function based on glob_str
413
414
The basic idea is to turn glob_str into a regular expression,
415
and just use the normal regular expression. There is a
416
complication because the selection function should return '2'
417
(scan) for directories which may contain a file which matches
418
the glob_str. So we break up the glob string into parts, and
419
any file which matches an initial sequence of glob parts gets
420
scanned.
421
422
Thanks to Donovan Baarda who provided some code which did some
423
things similar to this.
424
425
"""
426
if glob_str.lower().startswith("ignorecase:"):
427
re_comp = lambda r: re.compile(r, re.I | re.S)
428
glob_str = glob_str[len("ignorecase:"):]
429
else: re_comp = lambda r: re.compile(r, re.S)
430
431
# matches what glob matches and any files in directory
432
glob_comp_re = re_comp("^%s($|/)" % self.glob_to_re(glob_str))
433
434
if glob_str.find("**") != -1:
435
glob_str = glob_str[:glob_str.find("**")+2] # truncate after **
436
437
scan_comp_re = re_comp("^(%s)$" %
438
"|".join(self.glob_get_prefix_res(glob_str)))
439
440
def include_sel_func(path):
441
if glob_comp_re.match(path.name): return 1
442
elif scan_comp_re.match(path.name): return 2
443
else: return None
444
445
def exclude_sel_func(path):
446
if glob_comp_re.match(path.name): return 0
447
else: return None
448
449
# Check to make sure prefix is ok
450
if not include_sel_func(self.rootpath): raise FilePrefixError(glob_str)
451
452
if include: return include_sel_func
453
else: return exclude_sel_func
454
455
def glob_get_prefix_res(self, glob_str):
456
"""Return list of regexps equivalent to prefixes of glob_str"""
457
glob_parts = glob_str.split("/")
458
if "" in glob_parts[1:-1]: # "" OK if comes first or last, as in /foo/
459
raise GlobbingError("Consecutive '/'s found in globbing string "
460
+ glob_str)
461
462
prefixes = map(lambda i: "/".join(glob_parts[:i+1]),
463
range(len(glob_parts)))
464
# we must make exception for root "/", only dir to end in slash
465
if prefixes[0] == "": prefixes[0] = "/"
466
return map(self.glob_to_re, prefixes)
467
468
def glob_to_re(self, pat):
469
"""Returned regular expression equivalent to shell glob pat
470
471
Currently only the ?, *, [], and ** expressions are supported.
472
Ranges like [a-z] are also currently unsupported. There is no
473
way to quote these special characters.
474
475
This function taken with minor modifications from efnmatch.py
476
by Donovan Baarda.
477
478
"""
479
i, n, res = 0, len(pat), ''
480
while i < n:
481
c, s = pat[i], pat[i:i+2]
482
i = i+1
483
if s == '**':
484
res = res + '.*'
485
i = i + 1
486
elif c == '*': res = res + '[^/]*'
487
elif c == '?': res = res + '[^/]'
488
elif c == '[':
489
j = i
490
if j < n and pat[j] in '!^': j = j+1
491
if j < n and pat[j] == ']': j = j+1
492
while j < n and pat[j] != ']': j = j+1
493
if j >= n: res = res + '\\[' # interpret the [ literally
494
else: # Deal with inside of [..]
495
stuff = pat[i:j].replace('\\','\\\\')
496
i = j+1
497
if stuff[0] in '!^': stuff = '^' + stuff[1:]
498
res = res + '[' + stuff + ']'
499
else: res = res + re.escape(c)
500
return res
501
502
Loggerhead is a web-based interface for Breezy
Version: 2.0.1