1
# ubuntuone.syncdaemon.filesystem_manager - FSM
3
# Author: Facundo Batista <facundo@canonical.com>
5
# Copyright 2009 Canonical Ltd.
7
# This program is free software: you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License version 3, as published
9
# by the Free Software Foundation.
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranties of
13
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
14
# PURPOSE. See the GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License along
17
# with this program. If not, see <http://www.gnu.org/licenses/>.
18
'''Module that implements the File System Manager.'''
19
from __future__ import with_statement
29
from ubuntuone.syncdaemon import file_shelf
32
METADATA_VERSION = "2"
35
# File System Manager (FSM)
36
# --------------------------
38
# The FileSystemManager is the one that interacts with the filesystem, and
39
# keeps a storage with metadata. This storage is verterok's FileShelf.
41
# The metadata, in disk, is a dictionary, where the keys are 'mdid's (metadata
42
# ids), and the values are different parameters, some of them can be modified
43
# from outside, and some can not.
45
# There're two very important values: path and node_id. The path is the
46
# pointer to where the file or directory is located in the filesystem, and
47
# the node_id is the unique identifier from the server. When a new file is
48
# created (with the .create() method), a mdid is assigned to the path and the
49
# share, but no node_id yet. When the server assigns the node_id, it needs to
50
# be set here with the .set_node_id() method.
52
# All the data can be retrieved generally using this three values (mdid, path,
53
# and node_id/share) using specific get_by_*() methods. For this to be fast,
54
# two indexes are created at init time, two dictionaries that hold the
55
# relationships path->mdid, and (share,node_id)->mdid. In any case, KeyError
56
# is raised if an incorrect value is passed to the getters. Note that a mdid
57
# identifies uniquely the MD Object, like also the path; but for the node_id it
58
# also needs to have the share, as the same "server file" can live in different
59
# directories in the "client disk".
61
# Once assigned, the path, share and node_id values can not be changed. For any
62
# other value (except another special one, 'info', see below), three methods
63
# are provided to set them: set_by_*() (symmetric to the getters). These
64
# methods receive a first argument to indicate what is modified, and then
65
# several keyword arguments with all the values to be set.
67
# The 'info' is a special value set by the FileSystemManager itself, that
68
# records operations and changes made to each node, and as I said before,
69
# it can only be accesses from outside, not modified.
71
# Another method is provided to retrieve the created objects:
72
# get_mdobjs_by_share_id, that returns all the objects in that share.
74
# When asked for data, the FSM returns an object that is a thin wrapper to the
75
# info, only to be easily accessible, like using "mdobj.path", or
76
# "mdobj.info.is_partial". This object is not alive: it does not get updated
77
# if something changes in the metadata, and any change in the object is not
78
# written back to the metadata (this is by design, because of how the
79
# information flows in the system).
81
# As I said before, the FileSystemManager not only keeps the metadata, but also
82
# interacts with the filesystem itself. As such, it provides several operations
83
# on files and directories.
85
# In the process of downloading a file from the server, FSM handles the
86
# .partial files. With the .create_partial() method the system creates this
87
# special file where the new content will be downloaded. When it finishes ok,
88
# the .commit_partial() is called, and that content is moved into the old file.
89
# If the download fails for any reason, .remove_partial() is called and all is
92
# Other services are provided:
94
# .move_to_conflict(): moves a file or dir in problem to the same name but
95
# adding a .conflict to the name (if .conflict already exists, it will try with
96
# .conflict.1, .conflict.2, and so on).
98
# .upload_finished(): sets a new hash in the metadata, marking that the
99
# new content was uploaded to the server.
101
# .move_file(): moves a file or directory from one pathname to other.
103
# .delete_file(): removes a file or directory from disk.
105
# Finally, the FSM has three methods that provides high level information,
106
# in some cases synthesising their values using some internal values:
108
# .has_metadata(): returns True if the system has metadata for that path,
109
# node_id or mdid (note that we may don't have metadata even to an old mdid,
110
# because it was deleted in the middle)
112
# .changed(): returns 'local', 'server', or 'none', depending of what
113
# changed for that node
115
# .is_dir: returns if the node is a directory.
121
fsm_logger = logging.getLogger('ubuntuone.SyncDaemon.fsm')
122
logger = functools.partial(fsm_logger.log, logging.INFO)
123
log_warning = functools.partial(fsm_logger.log, logging.WARNING)
125
is_forbidden = set("info path node_id share_id is_dir stat".split()
128
class InconsistencyError(Exception):
129
'''Inconsistency between internal records and filesystem itself.'''
131
class Despair(Exception):
132
'''This should never happen, we're in an impossible condition!'''
135
class _MDObject(object):
136
'''Wrapper around MD dict.'''
137
def __init__(self, **mdobj):
138
self.__dict__.update(mdobj)
140
# info is a special one
142
self.info = _MDObject(**mdobj["info"])
144
def __eq__(self, other):
145
return self.__dict__ == other.__dict__
148
class ShareNodeDict(dict):
149
'''Cache for node_id and share.'''
150
def __getitem__(self, key):
151
share_id, node_id = key
153
raise ValueError("The node_id can not be None")
154
return dict.__getitem__(self, key)
156
def __setitem__(self, key, value):
157
share_id, node_id = key
159
raise ValueError("The node_id can not be None")
160
return dict.__setitem__(self, key, value)
162
def __contains__(self, key):
163
share_id, node_id = key
165
raise ValueError("The node_id can not be None")
166
return dict.__contains__(self, key)
171
class FileSystemManager(object):
172
'''Keeps the files/dirs metadata and interacts with the filesystem.
174
It has a FileShelf where all the metadata is stored, using 'mdid's as
175
keys. 'mdid' is 'metadata id'... it's actually an uuid, but we call it
176
mdid to don't get confused about names, as we also have the node_id is
177
the one assigned by the server.
179
At init time two indexes are built in memory:
181
- idx_path: relationship path -> mdid
182
- idx_node_id: relationship (share_id, node_id) -> mdid
184
def __init__(self, data_dir, vm):
185
if not isinstance(data_dir, basestring):
186
raise TypeError("data_dir should be a string instead of %s" % \
188
fsmdir = os.path.join(data_dir, 'fsm')
189
self.fs = file_shelf.FileShelf(fsmdir)
195
self._idx_node_id = ShareNodeDict()
197
# get the metadata version
198
self._version_file = os.path.join(data_dir, "metadata_version")
199
if os.path.exists(self._version_file):
200
with open(self._version_file) as fh:
201
md_version = fh.read().strip()
205
# load the info from the metadata
206
if md_version == METADATA_VERSION:
207
self._load_metadata_updated()
209
load_method = getattr(self, "_load_metadata_%s" % md_version)
210
load_method(md_version)
212
logger("initialized: idx_path: %s, idx_node_id: %s, shares: %s" %
213
(len(self._idx_path), len(self._idx_node_id), len(self.shares)))
215
def _load_metadata_None(self, old_version):
216
'''Loads metadata from when it wasn't even versioned.'''
217
logger("loading metadata from old version %r", old_version)
219
for mdid in self.fs.keys():
220
mdobj = self.fs[mdid]
222
# assure path are bytes (new to version 2)
224
mdobj["path"] = mdobj["path"].encode("utf8")
225
except UnicodeDecodeError:
226
# this is an invalid path, we shouldn't have it
230
abspath = self.get_abspath(mdobj["share_id"], mdobj["path"])
231
self._idx_path[abspath] = mdid
232
if mdobj["node_id"] is not None:
233
self._idx_node_id[(mdobj["share_id"], mdobj["node_id"])] = mdid
235
# assure we have stat info (new to version 1)
236
if os.path.exists(abspath):
237
newstat = os.stat(abspath)
240
mdobj["stat"] = newstat
241
self.fs[mdid] = mdobj
244
with open(self._version_file, "w") as fh:
245
fh.write(METADATA_VERSION)
247
def _load_metadata_1(self, old_version):
248
'''Loads metadata from when it wasn't even versioned.'''
249
logger("loading metadata from old version %r", old_version)
251
for mdid in self.fs.keys():
252
mdobj = self.fs[mdid]
254
# assure path are bytes (new to version 2)
256
mdobj["path"] = mdobj["path"].encode("utf8")
257
except UnicodeDecodeError:
258
# this is an invalid path, we shouldn't have it
262
abspath = self.get_abspath(mdobj["share_id"], mdobj["path"])
263
self._idx_path[abspath] = mdid
264
if mdobj["node_id"] is not None:
265
self._idx_node_id[(mdobj["share_id"], mdobj["node_id"])] = mdid
266
self.fs[mdid] = mdobj
269
with open(self._version_file, "w") as fh:
270
fh.write(METADATA_VERSION)
272
def _load_metadata_updated(self):
273
'''Loads metadata of last version.'''
274
logger("loading updated metadata")
275
for mdid in self.fs.keys():
276
mdobj = self.fs[mdid]
277
abspath = self.get_abspath(mdobj["share_id"], mdobj["path"])
278
self._idx_path[abspath] = mdid
279
if mdobj["node_id"] is not None:
280
self._idx_node_id[(mdobj["share_id"], mdobj["node_id"])] = mdid
282
def create(self, path, share_id, is_dir=False):
283
'''Creates a new md object.'''
285
raise ValueError("Empty paths are not allowed (got %r)" % path)
287
path = os.path.normpath(path)
288
if path in self._idx_path:
289
raise ValueError("The path %r is already created!" % path)
292
mdid = str(uuid.uuid4())
293
# make path relative to the share_id
294
relpath = self._share_relative_path(share_id, path)
295
newobj = dict(path=relpath, node_id=None, share_id=share_id,
296
is_dir=is_dir, local_hash=None,
297
server_hash=None, mdid=mdid)
298
newobj["info"] = dict(created=time.time(), is_partial=False)
299
if os.path.exists(path):
300
newobj["stat"] = os.stat(path)
302
newobj["stat"] = None
304
logger("create: path=%r mdid=%r share_id=%r node_id=%r is_dir=%r" % (
305
path, mdid, share_id, None, is_dir))
306
self.fs[mdid] = newobj
309
self._idx_path[path] = mdid
313
def set_node_id(self, path, node_id):
314
'''Sets the node_id to a md object.'''
315
path = os.path.normpath(path)
316
mdid = self._idx_path[path]
317
mdobj = self.fs[mdid]
318
if mdobj["node_id"] is not None:
319
msg = "The path %r already has the node_id %r" % (path, node_id)
320
raise ValueError(msg)
323
share_id = mdobj["share_id"]
324
self._idx_node_id[(share_id, node_id)] = mdid
327
mdobj["node_id"] = node_id
328
mdobj["info"]["node_id_assigned"] = time.time()
329
self.fs[mdid] = mdobj
331
logger("set_node_id: path=%r mdid=%r share_id=%r node_id=%r" % (
332
path, mdid, share_id, node_id))
334
def get_mdobjs_by_share_id(self, share_id):
335
'''Returns all the mdids that belongs to a share.'''
337
for mdid in self.fs.keys():
338
mdobj = self.fs[mdid]
339
if mdobj["share_id"] == share_id:
340
all_mdobjs.append(_MDObject(**mdobj))
343
def get_data_for_server_rescan(self):
344
'''Generates all the (share, node, hash) tuples needed for rescan'''
346
for _, v in self.fs.items():
349
(v['share_id'], v['node_id'], v['server_hash'] or ''))
352
def get_by_mdid(self, mdid):
353
'''Returns the md object according to the mdid.'''
354
mdobj = self.fs[mdid]
355
return _MDObject(**mdobj)
357
def get_by_path(self, path):
358
'''Returns the md object according to the path.'''
359
path = os.path.normpath(path)
360
mdid = self._idx_path[path]
361
mdobj = self.fs[mdid]
362
return _MDObject(**mdobj)
364
def get_by_node_id(self, share_id, node_id):
365
'''Returns the md object according to the node_id and share_id.'''
366
mdid = self._idx_node_id[(share_id, node_id)]
367
mdobj = self.fs[mdid]
368
return _MDObject(**mdobj)
370
def set_by_mdid(self, mdid, **kwargs):
371
'''Set some values to the md object with that mdid.'''
372
forbidden = is_forbidden(set(kwargs))
374
raise ValueError("The following attributes can not be set "
375
"externally: %s" % forbidden)
377
logger("set mdid=%r: %s" % (mdid, kwargs))
378
mdobj = self.fs[mdid]
379
for k, v in kwargs.items():
381
self.fs[mdid] = mdobj
383
def set_by_path(self, path, **kwargs):
384
'''Set some values to the md object with that path.'''
386
raise ValueError("The mdid is forbidden to set externally")
387
path = os.path.normpath(path)
388
mdid = self._idx_path[path]
389
self.set_by_mdid(mdid, **kwargs)
391
def set_by_node_id(self, node_id, share_id, **kwargs):
392
'''Set some values to the md object with that node_id/share_id.'''
394
raise ValueError("The mdid is forbidden to set externally")
395
mdid = self._idx_node_id[(share_id, node_id)]
396
self.set_by_mdid(mdid, **kwargs)
398
def refresh_stat(self, path):
399
'''Refreshes the stat of a md object.'''
400
logger("refresh stat to path=%r", path)
401
mdid = self._idx_path[path]
402
mdobj = self.fs[mdid]
403
mdobj["stat"] = os.stat(path)
404
self.fs[mdid] = mdobj
406
def update_stat(self, mdid, stat):
407
'''Updates the stat of a md object.'''
408
logger("update stat of mdid=%r", mdid)
409
mdobj = self.fs[mdid]
411
self.fs[mdid] = mdobj
414
def move_file(self, new_share_id, path_from, path_to):
415
'''Moves a file/dir from one point to the other.'''
416
path_from = os.path.normpath(path_from)
417
path_to = os.path.normpath(path_to)
418
mdid = self._idx_path[path_from]
419
mdobj = self.fs[mdid]
421
# move the file in the fs
422
from_context = self._enable_share_write(mdobj['share_id'], path_from)
423
to_context = self._enable_share_write(new_share_id, path_to)
425
# pylint: disable-msg=W0704
427
with contextlib.nested(from_context, to_context):
428
shutil.move(path_from, path_to)
430
# file was not yet created
431
m = "IOError %s when trying to move file/dir %r"
432
log_warning(m, e, path_from)
433
self.moved(new_share_id, path_from, path_to)
435
def moved(self, new_share_id, path_from, path_to):
436
"""change the metadata of a moved file"""
437
path_from = os.path.normpath(path_from)
438
path_to = os.path.normpath(path_to)
440
# if the move overwrites other file, adjust *that* metadata
441
if path_to in self._idx_path:
442
mdid = self._idx_path.pop(path_to)
443
mdobj = self.fs[mdid]
444
if mdobj["node_id"] is not None:
445
del self._idx_node_id[(mdobj["share_id"], mdobj["node_id"])]
448
# adjust the metadata of "from" file
449
mdid = self._idx_path.pop(path_from)
450
logger("move_file: mdid=%r path_from=%r path_to=%r",
451
mdid, path_from, path_to)
452
mdobj = self.fs[mdid]
454
self._idx_path[path_to] = mdid
456
# change the path, make it relative to the share_id
457
relpath = self._share_relative_path(new_share_id, path_to)
458
mdobj["path"] = relpath
459
mdobj['share_id'] = new_share_id
460
mdobj["info"]["last_moved_from"] = path_from
461
mdobj["info"]["last_moved_time"] = time.time()
462
# we try to stat, if we fail, so what?
463
#pylint: disable-msg=W0704
465
mdobj["stat"] = os.stat(path_to) # needed if not the same FS
467
logger("Got an OSError while getting the stat of %r", path_to)
468
self.fs[mdid] = mdobj
471
# change the path for all the children of that node
472
path_from = path_from + os.path.sep
473
len_from = len(path_from)
474
for path in (x for x in self._idx_path if x.startswith(path_from)):
475
newpath = os.path.join(path_to, path[len_from:])
477
# change in the index
478
mdid = self._idx_path.pop(path)
479
self._idx_path[newpath] = mdid
481
# and in the object itself
482
mdobj = self.fs[mdid]
483
relpath = self._share_relative_path(new_share_id, newpath)
484
mdobj["path"] = relpath
485
self.fs[mdid] = mdobj
488
def delete_metadata(self, path):
489
'''Deletes the metadata.'''
490
path = os.path.normpath(path)
491
mdid = self._idx_path[path]
492
mdobj = self.fs[mdid]
493
logger("delete metadata: path=%r mdid=%r" % (path, mdid))
496
del self._idx_path[path]
497
if mdobj["node_id"] is not None:
498
del self._idx_node_id[(mdobj["share_id"], mdobj["node_id"])]
501
def delete_file(self, path):
502
'''Deletes a file/dir and the metadata.'''
503
# adjust the metadata
504
path = os.path.normpath(path)
505
mdid = self._idx_path[path]
506
mdobj = self.fs[mdid]
507
logger("delete: path=%r mdid=%r" % (path, mdid))
511
with self._enable_share_write(mdobj['share_id'], path):
512
if self.is_dir(path=path):
517
if e.errno == errno.ENOTEMPTY:
519
m = "OSError %s when trying to remove file/dir %r"
520
log_warning(m, e, path)
521
self.delete_metadata(path)
523
def move_to_conflict(self, mdid):
524
'''Moves a file/dir to its .conflict.'''
525
mdobj = self.fs[mdid]
526
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
527
logger("move_to_conflict: path=%r mdid=%r" % (path, mdid))
528
base_to_path = to_path = path + ".conflict"
530
while os.path.exists(to_path):
532
to_path = base_to_path + "." + str(ind)
533
with self._enable_share_write(mdobj['share_id'], path):
535
os.rename(path, to_path)
537
if e.errno == errno.ENOENT:
538
m = "Already removed when trying to move to conflict: %r"
543
for p, is_dir in self.get_paths_starting_with(path):
547
# remove inotify watch
549
self.vm.m.event_q.inotify_rm_watch(p)
551
# pyinotify has an ugly error management, if we can call
552
# it that, :(. We handle this here because it's possible
553
# and correct that the path is not there anymore
554
m = "Error %s when trying to remove the watch on %r"
555
log_warning(m, e, path)
557
self.delete_metadata(p)
558
mdobj["info"]["last_conflicted"] = time.time()
559
self.fs[mdid] = mdobj
561
def _check_partial(self, mdid):
562
'''Checks consistency between internal flag and FS regarding partial'''
564
mdobj = self.fs[mdid]
565
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
566
partial_in_md = mdobj["info"]["is_partial"]
568
partial_path = os.path.join(path, ".partial")
570
partial_path = path + ".partial"
571
partial_in_disk = os.path.exists(partial_path)
574
if partial_in_md != partial_in_disk:
575
msg = "'partial' inconsistency for object with mdid %r! In disk:"\
576
" %s, In MD: %s" % (mdid, partial_in_disk, partial_in_md)
577
raise InconsistencyError(msg)
580
def get_partial_path(self, mdobj):
581
'''Gets the path of the .partial file for a given mdobj'''
582
is_dir = mdobj["is_dir"]
583
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
585
return os.path.join(path, ".partial")
587
return path + ".partial"
589
def create_partial(self, node_id, share_id):
590
'''Creates a .partial in disk and set the flag in metadata.'''
591
mdid = self._idx_node_id[(share_id, node_id)]
592
if self._check_partial(mdid):
593
raise ValueError("The object with share_id %r and node_id %r is "
594
"already partial!" % (share_id, node_id))
596
# create an empty partial and set the flag
597
mdobj = self.fs[mdid]
598
is_dir = mdobj["is_dir"]
599
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
600
partial_path = self.get_partial_path(mdobj)
601
with self._enable_share_write(share_id, partial_path):
602
# if we don't have the dir yet, create it
603
if is_dir and not os.path.exists(path):
605
open(partial_path, "w").close()
607
mdobj["info"]["last_partial_created"] = time.time()
608
mdobj["info"]["is_partial"] = True
609
self.fs[mdid] = mdobj
611
def get_partial_for_writing(self, node_id, share_id):
612
'''Get a write-only fd to a partial file'''
613
mdid = self._idx_node_id[(share_id, node_id)]
614
if not self._check_partial(mdid):
615
raise ValueError("The object with share_id %r and node_id %r is "
616
"not partial!" % (share_id, node_id))
618
mdobj = self.fs[mdid]
619
partial_path = self.get_partial_path(mdobj)
620
with self._enable_share_write(share_id, partial_path):
621
fh = open(partial_path, "w")
624
def get_partial(self, node_id, share_id):
625
'''Gets a read-only fd to a partial file.'''
626
mdid = self._idx_node_id[(share_id, node_id)]
627
if not self._check_partial(mdid):
628
raise ValueError("The object with share_id %r and node_id %r is "
629
"not partial!" % (share_id, node_id))
631
partial_path = self.get_partial_path(self.fs[mdid])
632
fd = open(partial_path, "r")
635
def commit_partial(self, node_id, share_id, local_hash):
636
'''Creates a .partial in disk and set the flag in metadata.'''
637
mdid = self._idx_node_id[(share_id, node_id)]
638
mdobj = self.fs[mdid]
640
raise ValueError("Directory partials can not be commited!")
641
if not self._check_partial(mdid):
642
raise ValueError("The object with share_id %r and node_id %r is "
643
"not partial!" % (share_id, node_id))
645
# move the .partial to the real path, and set the md info
646
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
647
logger("commit_partial: path=%r mdid=%r share_id=%r node_id=%r" %
648
(path, mdid, share_id, node_id))
650
partial_path = path + ".partial"
651
partial_context = self._enable_share_write(share_id, partial_path)
652
path_context = self._enable_share_write(share_id, path)
653
with contextlib.nested(partial_context, path_context):
654
shutil.move(path + ".partial", path)
655
mdobj["local_hash"] = local_hash
656
mdobj["info"]["last_downloaded"] = time.time()
657
mdobj["info"]["is_partial"] = False
658
mdobj["stat"] = os.stat(path)
659
self.fs[mdid] = mdobj
661
def remove_partial(self, node_id, share_id):
662
'''Removes a .partial in disk and set the flag in metadata.'''
663
mdid = self._idx_node_id[(share_id, node_id)]
665
# delete the .partial, and set the md info
666
mdobj = self.fs[mdid]
667
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
668
logger("remove_partial: path=%r mdid=%r share_id=%r node_id=%r" %
669
(path, mdid, share_id, node_id))
670
if self.is_dir(path=path):
671
partial_path = os.path.join(path, ".partial")
673
partial_path = path + ".partial"
674
with self._enable_share_write(share_id, partial_path):
675
#pylint: disable-msg=W0704
677
os.remove(partial_path)
679
# we only remove it if its there.
680
m = "OSError %s when trying to remove partial_path %r"
681
log_warning(m, e, partial_path)
682
mdobj["info"]["last_partial_removed"] = time.time()
683
mdobj["info"]["is_partial"] = False
684
self.fs[mdid] = mdobj
686
def upload_finished(self, mdid, server_hash):
687
'''Sets the metadata with timestamp and server hash.'''
688
mdobj = self.fs[mdid]
689
mdobj["info"]["last_uploaded"] = time.time()
690
mdobj["server_hash"] = server_hash
691
self.fs[mdid] = mdobj
693
def _get_mdid_from_args(self, kwargs, parent):
694
'''Parses the kwargs and gets the mdid.'''
695
if len(kwargs) == 1 and "path" in kwargs:
696
path = os.path.normpath(kwargs["path"])
697
return self._idx_path[path]
698
if len(kwargs) == 1 and "mdid" in kwargs:
699
return kwargs["mdid"]
700
if len(kwargs) == 2 and "node_id" in kwargs and "share_id" in kwargs:
701
return self._idx_node_id[(kwargs["share_id"], kwargs["node_id"])]
702
raise TypeError("Incorrect arguments for %r: %r" % (parent, kwargs))
704
def is_dir(self, **kwargs):
705
'''Return True if the path of a given object is a directory.'''
706
mdid = self._get_mdid_from_args(kwargs, "is_dir")
707
mdobj = self.fs[mdid]
708
return mdobj["is_dir"]
710
def has_metadata(self, **kwargs):
711
'''Return True if there's metadata for a given object.'''
712
if len(kwargs) == 1 and "path" in kwargs:
713
path = os.path.normpath(kwargs["path"])
714
return path in self._idx_path
715
if len(kwargs) == 1 and "mdid" in kwargs:
716
return kwargs["mdid"] in self.fs
717
if len(kwargs) == 2 and "node_id" in kwargs and "share_id" in kwargs:
718
return (kwargs["share_id"], kwargs["node_id"]) in self._idx_node_id
719
raise TypeError("Incorrect arguments for 'has_metadata': %r" % kwargs)
721
def changed(self, **kwargs):
722
'''Return True if there's metadata for a given object.'''
724
mdid = self._get_mdid_from_args(kwargs, "changed")
725
mdobj = self.fs[mdid]
726
is_partial = mdobj["info"]["is_partial"]
727
local_hash = mdobj.get("local_hash", False)
728
server_hash = mdobj.get("server_hash", False)
731
if local_hash == server_hash:
733
return "We broke the Universe! local_hash %r, server_hash %r,"\
734
" is_partial %r" % (local_hash, server_hash, is_partial)
744
def dir_content(self, path):
745
'''Returns the content of the directory in a server-comparable way.'''
746
path = os.path.normpath(path)
747
mdid = self._idx_path[path]
748
mdobj = self.fs[mdid]
749
if not mdobj["is_dir"]:
750
raise ValueError("You can ask dir_content only on a directory.")
753
'''find the mdids that match'''
754
for p, m in self._idx_path.iteritems():
755
if os.path.dirname(p) == path and p != path:
757
yield (os.path.basename(p), mdobj["is_dir"],
760
return sorted(_get_all())
762
def open_file(self, mdid):
763
'''returns a file like object for reading the contents of the file.'''
764
mdobj = self.fs[mdid]
766
raise ValueError("You can only open files, not directories.")
768
return open(self.get_abspath(mdobj['share_id'], mdobj['path']))
770
def create_file(self, mdid):
771
'''create the file.'''
772
mdobj = self.fs[mdid]
773
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
774
with self._enable_share_write(mdobj['share_id'], path):
778
# use os.open so this wont raise IN_CLOSE_WRITE
779
fd = os.open(path, os.O_CREAT)
782
def _share_relative_path(self, share_id, path):
783
""" returns the relative path from the share_id. """
784
share = self._get_share(share_id)
785
if path == share.path:
786
# the relaitve path is the fullpath
788
head, sep, tail = path.rpartition(share.path)
790
raise ValueError("'%s' isn't a child of '%s'" % (path, share.path))
791
relpath = tail.lstrip(os.path.sep)
792
# remove the initial os.path.sep
793
return relpath.lstrip(os.path.sep)
795
def _get_share(self, share_id):
796
""" returns the share with id: share_id. """
797
share = self.shares.get(share_id, None)
799
share = self.vm.shares.get(share_id)
800
self.shares[share_id] = share
803
def get_abspath(self, share_id, path):
804
""" return the absolute path: share.path + path"""
805
share_path = self._get_share(share_id).path
806
if share_path == path:
807
# the relaitve path is the fullpath
810
return os.path.join(share_path, path)
812
def _enable_share_write(self, share_id, path):
813
""" helper to create a EnableShareWrite context manager. """
814
share = self._get_share(share_id)
815
return EnableShareWrite(share, path)
817
def get_paths_starting_with(self, base_path):
818
""" return a list of paths that starts with: path. """
820
for path in self._idx_path:
821
if path.startswith(base_path):
822
mdid = self._idx_path[path]
823
mdobj = self.fs[mdid]
824
all_paths.append((path, mdobj['is_dir']))
828
class EnableShareWrite(object):
829
""" Context manager to allow write in ro-shares. """
830
#pylint: disable-msg=W0201
832
def __init__(self, share, path):
833
""" create the instance """
838
""" ContextManager API """
839
self.ro = not self.share.can_write()
840
if os.path.isdir(self.path) and \
841
os.path.normpath(self.path) == os.path.normpath(self.share.path):
842
self.parent = self.path
844
self.parent = os.path.dirname(self.path)
847
if not os.path.exists(self.parent):
848
# if we don't have the parent yet, create it
849
with EnableShareWrite(self.share, self.parent):
850
os.mkdir(self.parent)
851
os.chmod(self.parent, 0755)
852
if os.path.exists(self.path) and not os.path.isdir(self.path):
853
os.chmod(self.path, 0744)
855
def __exit__(self, *exc_info):
856
""" ContextManager API """
858
if os.path.exists(self.path):
859
if os.path.isdir(self.path):
860
os.chmod(self.path, 0555)
862
os.chmod(self.path, 0444)
863
if os.path.exists(self.parent):
864
os.chmod(self.parent, 0555)
added
removed
ubuntuone/syncdaemon/filesystem_manager.py
