← Back to branch summary

~ubuntu-branches/ubuntu/precise/dulwich/precise-security

« back to all changes in this revision

Viewing changes to dulwich/object_store.py

  • Committer: Bazaar Package Importer
  • Author(s): Jelmer Vernooij
  • Date: 2011-08-07 15:03:44 UTC
  • mto: This revision was merged to the branch mainline in revision 25.
  • Revision ID: james.westby@ubuntu.com-20110807150344-94xw3o3hnh47y1m8
Tags: upstream-0.8.0
Import upstream version 0.8.0

Show diffs side-by-side

added added

removed removed

Lines of Context:
dulwich/object_store.py
25
25
import os
26
26
import stat
27
27
import tempfile
28
 
import urllib2
29
28
 
30
29
from dulwich.diff_tree import (
31
30
    tree_changes,
50
49
from dulwich.pack import (
51
50
    Pack,
52
51
    PackData,
53
 
    ThinPackData,
54
52
    iter_sha1,
55
 
    load_pack_index,
56
 
    write_pack,
57
 
    write_pack_data,
 
53
    write_pack_header,
58
54
    write_pack_index_v2,
 
55
    write_pack_object,
 
56
    write_pack_objects,
 
57
    compute_file_sha,
 
58
    PackIndexer,
 
59
    PackStreamCopier,
59
60
    )
60
61
 
61
62
INFODIR = 'info'
271
272
        objects = set()
272
273
        for sha in self._iter_loose_objects():
273
274
            objects.add((self._get_loose_object(sha), None))
274
 
        self.add_objects(objects)
 
275
        self.add_objects(list(objects))
275
276
        for obj, path in objects:
276
277
            self._remove_loose_object(obj.id)
277
278
        return len(objects)
321
322
            # Don't bother writing an empty pack file
322
323
            return
323
324
        f, commit = self.add_pack()
324
 
        write_pack_data(f, objects, len(objects))
 
325
        write_pack_objects(f, objects)
325
326
        return commit()
326
327
 
327
328
 
387
388
    def _remove_loose_object(self, sha):
388
389
        os.remove(self._get_shafile_path(sha))
389
390
 
390
 
    def move_in_thin_pack(self, path):
 
391
    def _complete_thin_pack(self, f, path, copier, indexer):
391
392
        """Move a specific file containing a pack into the pack directory.
392
393
 
393
394
        :note: The file should be on the same file system as the
394
395
            packs directory.
395
396
 
 
397
        :param f: Open file object for the pack.
396
398
        :param path: Path to the pack file.
 
399
        :param copier: A PackStreamCopier to use for writing pack data.
 
400
        :param indexer: A PackIndexer for indexing the pack.
397
401
        """
398
 
        data = ThinPackData(self.get_raw, path)
399
 
 
400
 
        # Write index for the thin pack (do we really need this?)
401
 
        temppath = os.path.join(self.pack_dir,
402
 
            sha_to_hex(urllib2.randombytes(20))+".tempidx")
403
 
        data.create_index_v2(temppath)
404
 
        p = Pack.from_objects(data, load_pack_index(temppath))
405
 
 
 
402
        entries = list(indexer)
 
403
 
 
404
        # Update the header with the new number of objects.
 
405
        f.seek(0)
 
406
        write_pack_header(f, len(entries) + len(indexer.ext_refs()))
 
407
 
 
408
        # Rescan the rest of the pack, computing the SHA with the new header.
 
409
        new_sha = compute_file_sha(f, end_ofs=-20)
 
410
 
 
411
        # Complete the pack.
 
412
        for ext_sha in indexer.ext_refs():
 
413
            type_num, data = self.get_raw(ext_sha)
 
414
            offset = f.tell()
 
415
            crc32 = write_pack_object(f, type_num, data, sha=new_sha)
 
416
            entries.append((ext_sha, offset, crc32))
 
417
        pack_sha = new_sha.digest()
 
418
        f.write(pack_sha)
 
419
        f.close()
 
420
 
 
421
        # Move the pack in.
 
422
        entries.sort()
 
423
        pack_base_name = os.path.join(
 
424
          self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries))
 
425
        os.rename(path, pack_base_name + '.pack')
 
426
 
 
427
        # Write the index.
 
428
        index_file = GitFile(pack_base_name + '.idx', 'wb')
406
429
        try:
407
 
            # Write a full pack version
408
 
            temppath = os.path.join(self.pack_dir,
409
 
                sha_to_hex(urllib2.randombytes(20))+".temppack")
410
 
            write_pack(temppath, ((o, None) for o in p.iterobjects()), len(p))
 
430
            write_pack_index_v2(index_file, entries, pack_sha)
 
431
            index_file.close()
411
432
        finally:
412
 
            p.close()
 
433
            index_file.abort()
413
434
 
414
 
        pack_sha = load_pack_index(temppath+".idx").objects_sha1()
415
 
        newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
416
 
        os.rename(temppath+".pack", newbasename+".pack")
417
 
        os.rename(temppath+".idx", newbasename+".idx")
418
 
        final_pack = Pack(newbasename)
 
435
        # Add the pack to the store and return it.
 
436
        final_pack = Pack(pack_base_name)
 
437
        final_pack.check_length_and_checksum()
419
438
        self._add_known_pack(final_pack)
420
439
        return final_pack
421
440
 
 
441
    def add_thin_pack(self, read_all, read_some):
 
442
        """Add a new thin pack to this object store.
 
443
 
 
444
        Thin packs are packs that contain deltas with parents that exist outside
 
445
        the pack. They should never be placed in the object store directly, and
 
446
        always indexed and completed as they are copied.
 
447
 
 
448
        :param read_all: Read function that blocks until the number of requested
 
449
            bytes are read.
 
450
        :param read_some: Read function that returns at least one byte, but may
 
451
            not return the number of bytes requested.
 
452
        :return: A Pack object pointing at the now-completed thin pack in the
 
453
            objects/pack directory.
 
454
        """
 
455
        fd, path = tempfile.mkstemp(dir=self.path, prefix='tmp_pack_')
 
456
        f = os.fdopen(fd, 'w+b')
 
457
 
 
458
        try:
 
459
            indexer = PackIndexer(f, resolve_ext_ref=self.get_raw)
 
460
            copier = PackStreamCopier(read_all, read_some, f,
 
461
                                      delta_iter=indexer)
 
462
            copier.verify()
 
463
            return self._complete_thin_pack(f, path, copier, indexer)
 
464
        finally:
 
465
            f.close()
 
466
 
422
467
    def move_in_pack(self, path):
423
468
        """Move a specific file containing a pack into the pack directory.
424
469
 
442
487
        self._add_known_pack(final_pack)
443
488
        return final_pack
444
489
 
445
 
    def add_thin_pack(self):
446
 
        """Add a new thin pack to this object store.
447
 
 
448
 
        Thin packs are packs that contain deltas with parents that exist
449
 
        in a different pack.
450
 
        """
451
 
        fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
452
 
        f = os.fdopen(fd, 'wb')
453
 
        def commit():
454
 
            os.fsync(fd)
455
 
            f.close()
456
 
            if os.path.getsize(path) > 0:
457
 
                return self.move_in_thin_pack(path)
458
 
            else:
459
 
                os.remove(path)
460
 
                return None
461
 
        return f, commit
462
 
 
463
490
    def add_pack(self):
464
491
        """Add a new pack to this object store.
465
492
 
540
567
        :param name: sha for the object.
541
568
        :return: tuple with numeric type and object contents.
542
569
        """
543
 
        return self[name].as_raw_string()
 
570
        obj = self[name]
 
571
        return obj.type_num, obj.as_raw_string()
544
572
 
545
573
    def __getitem__(self, name):
546
574
        return self._data[name]
647
675
 
648
676
 
649
677
def tree_lookup_path(lookup_obj, root_sha, path):
650
 
    """Lookup an object in a Git tree.
 
678
    """Look up an object in a Git tree.
651
679
 
652
680
    :param lookup_obj: Callback for retrieving object by SHA1
653
681
    :param root_sha: SHA1 of the root tree
654
682
    :param path: Path to lookup
 
683
    :return: A tuple of (mode, SHA) of the resulting path.
655
684
    """
656
 
    parts = path.split("/")
657
 
    sha = root_sha
658
 
    mode = None
659
 
    for p in parts:
660
 
        obj = lookup_obj(sha)
661
 
        if not isinstance(obj, Tree):
662
 
            raise NotTreeError(sha)
663
 
        if p == '':
664
 
            continue
665
 
        mode, sha = obj[p]
666
 
    return mode, sha
 
685
    tree = lookup_obj(root_sha)
 
686
    if not isinstance(tree, Tree):
 
687
        raise NotTreeError(root_sha)
 
688
    return tree.lookup_path(lookup_obj, path)
667
689
 
668
690
 
669
691
class MissingObjectFinder(object):
709
731
        self.add_todo([(tag.object[1], None, False)])
710
732
 
711
733
    def next(self):
712
 
        if not self.objects_to_send:
713
 
            return None
714
 
        (sha, name, leaf) = self.objects_to_send.pop()
 
734
        while True:
 
735
            if not self.objects_to_send:
 
736
                return None
 
737
            (sha, name, leaf) = self.objects_to_send.pop()
 
738
            if sha not in self.sha_done:
 
739
                break
715
740
        if not leaf:
716
741
            o = self.object_store[sha]
717
742
            if isinstance(o, Commit):
757
782
            # collect all ancestors
758
783
            new_ancestors = set()
759
784
            for a in ancestors:
760
 
                if a in self.parents:
761
 
                    new_ancestors.update(self.parents[a])
 
785
                ps = self.parents.get(a)
 
786
                if ps is not None:
 
787
                    new_ancestors.update(ps)
 
788
                self.parents[a] = None
762
789
 
763
790
            # no more ancestors; stop
764
791
            if not new_ancestors:
772
799
            ret = self.heads.pop()
773
800
            ps = self.get_parents(ret)
774
801
            self.parents[ret] = ps
775
 
            self.heads.update(ps)
 
802
            self.heads.update([p for p in ps if not p in self.parents])
776
803
            return ret
777
804
        return None