186
186
- idx_path: relationship path -> mdid
187
187
- idx_node_id: relationship (share_id, node_id) -> mdid
189
def __init__(self, data_dir, vm):
189
def __init__(self, data_dir, partials_dir, vm):
190
190
if not isinstance(data_dir, basestring):
191
191
raise TypeError("data_dir should be a string instead of %s" % \
193
193
fsmdir = os.path.join(data_dir, 'fsm')
194
self.partials_dir = partials_dir
195
if not os.path.exists(self.partials_dir):
196
os.makedirs(self.partials_dir)
198
# ensure that we can write in the partials_dir
199
os.chmod(self.partials_dir, 0755)
194
200
self.fs = file_shelf.CachedFileShelf(fsmdir, cache_size=1500,
195
201
cache_compact_threshold=4)
668
674
'''Checks consistency between internal flag and FS regarding partial'''
670
676
mdobj = self.fs[mdid]
671
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
672
677
partial_in_md = mdobj["info"]["is_partial"]
674
partial_path = os.path.join(path, ".u1partial")
676
dirname, filename = os.path.split(path)
677
partial_path = os.path.join(dirname, ".u1partial." + filename)
678
partial_in_disk = os.path.exists(partial_path)
678
partial_in_disk = os.path.exists(self._get_partial_path(mdobj))
680
680
# check and return
681
681
if partial_in_md != partial_in_disk:
684
684
raise InconsistencyError(msg)
685
685
return partial_in_md
687
def get_partial_path(self, mdobj):
687
def _get_partial_path(self, mdobj):
688
688
'''Gets the path of the .partial file for a given mdobj'''
689
689
is_dir = mdobj["is_dir"]
690
690
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
692
return os.path.join(path, ".u1partial")
694
dirname, filename = os.path.split(path)
695
return os.path.join(dirname, ".u1partial." + filename)
691
partial_path = os.path.join(self.partials_dir, mdobj['mdid'] + '.u1partial')
692
dirname, filename = os.path.split(path)
693
return partial_path + '.' + filename
697
695
def create_partial(self, node_id, share_id):
698
696
'''Creates a .partial in disk and set the flag in metadata.'''
699
697
mdid = self._idx_node_id[(share_id, node_id)]
698
log_debug("create_partial: mdid=%r share_id=%r node_id=%r" %
699
(mdid, share_id, node_id))
700
700
if self._check_partial(mdid):
701
701
raise ValueError("The object with share_id %r and node_id %r is "
702
702
"already partial!" % (share_id, node_id))
705
705
mdobj = self.fs[mdid]
706
706
is_dir = mdobj["is_dir"]
707
707
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
708
partial_path = self.get_partial_path(mdobj)
709
with self._enable_share_write(share_id, partial_path):
708
partial_path = self._get_partial_path(mdobj)
709
with self._enable_share_write(share_id, os.path.dirname(path)):
710
710
# if we don't have the dir yet, create it
711
711
if is_dir and not os.path.exists(path):
713
open(partial_path, "w").close()
713
open(partial_path, "w").close()
715
714
mdobj["info"]["last_partial_created"] = time.time()
716
715
mdobj["info"]["is_partial"] = True
717
716
self.fs[mdid] = mdobj
719
718
def get_partial_for_writing(self, node_id, share_id):
720
719
'''Get a write-only fd to a partial file'''
721
720
mdid = self._idx_node_id[(share_id, node_id)]
721
log_debug("get_partial_for_writing: mdid=%r share_id=%r node_id=%r" %
722
(mdid, share_id, node_id))
722
723
if not self._check_partial(mdid):
723
724
raise ValueError("The object with share_id %r and node_id %r is "
724
725
"not partial!" % (share_id, node_id))
726
727
mdobj = self.fs[mdid]
727
partial_path = self.get_partial_path(mdobj)
728
with self._enable_share_write(share_id, partial_path):
729
fh = open(partial_path, "w")
728
partial_path = self._get_partial_path(mdobj)
729
return open(partial_path, "w")
732
731
def get_partial(self, node_id, share_id):
733
732
'''Gets a read-only fd to a partial file.'''
736
735
raise ValueError("The object with share_id %r and node_id %r is "
737
736
"not partial!" % (share_id, node_id))
739
partial_path = self.get_partial_path(self.fs[mdid])
738
partial_path = self._get_partial_path(self.fs[mdid])
740
739
fd = open(partial_path, "r")
755
754
log_debug("commit_partial: path=%r mdid=%r share_id=%r node_id=%r" %
756
755
(path, mdid, share_id, node_id))
758
dirname, filename = os.path.split(path)
759
partial_path = os.path.join(dirname, ".u1partial." + filename)
760
partial_context = self._enable_share_write(share_id, partial_path)
761
path_context = self._enable_share_write(share_id, path)
762
with contextlib.nested(partial_context, path_context):
757
partial_path = self._get_partial_path(mdobj)
758
with self._enable_share_write(share_id, path):
763
759
shutil.move(partial_path, path)
764
760
mdobj["local_hash"] = local_hash
765
761
mdobj["info"]["last_downloaded"] = time.time()
776
772
path = self.get_abspath(mdobj['share_id'], mdobj['path'])
777
773
log_debug("remove_partial: path=%r mdid=%r share_id=%r node_id=%r" %
778
774
(path, mdid, share_id, node_id))
779
if self.is_dir(path=path):
780
partial_path = os.path.join(path, ".u1partial")
782
dirname, filename = os.path.split(path)
783
partial_path = os.path.join(dirname, ".u1partial." + filename)
784
with self._enable_share_write(share_id, partial_path):
785
#pylint: disable-msg=W0704
787
os.remove(partial_path)
789
# we only remove it if its there.
790
m = "OSError %s when trying to remove partial_path %r"
791
log_warning(m, e, partial_path)
775
partial_path = self._get_partial_path(mdobj)
776
#pylint: disable-msg=W0704
778
os.remove(partial_path)
780
# we only remove it if its there.
781
m = "OSError %s when trying to remove partial_path %r"
782
log_warning(m, e, partial_path)
792
783
mdobj["info"]["last_partial_removed"] = time.time()
793
784
mdobj["info"]["is_partial"] = False
794
785
self.fs[mdid] = mdobj
added
removed
ubuntuone/syncdaemon/filesystem_manager.py
