~ubuntu-branches/ubuntu/precise/dulwich/precise-updates : revision 1.3.1

1

# object_store.py -- Object store for git objects

2

3

#

4

# This program is free software; you can redistribute it and/or

5

# modify it under the terms of the GNU General Public License

16

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

17

# MA 02110-1301, USA.

18

19

20

"""Git object store interfaces and implementation."""

21

22

23

import itertools

19

24

import os

25

import stat

20

26

import tempfile

21

27

import urllib2

22

28

24

30

NotTreeError,

25

31

)

26

32

from dulwich.objects import (

33

Commit,

27

34

ShaFile,

35

Tag,

28

36

Tree,

29

37

hex_to_sha,

30

38

sha_to_hex,

33

41

Pack,

34

42

PackData,

35

43

iter_sha1,

36

load_packs,

37

44

load_pack_index,

38

45

write_pack,

39

46

write_pack_data,

42

49

43

50

PACKDIR = 'pack'

44

51

45

class ObjectStore(object):

46

"""Object store."""

52

53

class BaseObjectStore(object):

54

"""Object store interface."""

55

56

def determine_wants_all(self, refs):

57

return [sha for (ref, sha) in refs.iteritems() if not sha in self and not ref.endswith("^{}")]

58

59

def iter_shas(self, shas):

60

"""Iterate over the objects for the specified shas.

61

62

:param shas: Iterable object with SHAs

63

:return: Object iterator

64

"""

65

return ObjectStoreIterator(self, shas)

66

67

def __contains__(self, sha):

68

"""Check if a particular object is present by SHA1."""

69

raise NotImplementedError(self.__contains__)

70

71

def get_raw(self, name):

72

"""Obtain the raw text for an object.

73

74

:param name: sha for the object.

75

:return: tuple with object type and object contents.

76

"""

77

raise NotImplementedError(self.get_raw)

78

79

def __getitem__(self, sha):

80

"""Obtain an object by SHA1."""

81

type, uncomp = self.get_raw(sha)

82

return ShaFile.from_raw_string(type, uncomp)

83

84

def __iter__(self):

85

"""Iterate over the SHAs that are present in this store."""

86

raise NotImplementedError(self.__iter__)

87

88

def add_object(self, obj):

89

"""Add a single object to this object store.

90

91

"""

92

raise NotImplementedError(self.add_object)

93

94

def add_objects(self, objects):

95

"""Add a set of objects to this object store.

96

97

:param objects: Iterable over a list of objects.

98

"""

99

raise NotImplementedError(self.add_objects)

100

101

def find_missing_objects(self, haves, wants, progress=None):

102

"""Find the missing objects required for a set of revisions.

103

104

:param haves: Iterable over SHAs already in common.

105

:param wants: Iterable over SHAs of objects to fetch.

106

:param progress: Simple progress function that will be called with

107

updated progress strings.

108

:return: Iterator over (sha, path) pairs.

109

"""

110

return iter(MissingObjectFinder(self, haves, wants, progress).next, None)

111

112

def find_common_revisions(self, graphwalker):

113

"""Find which revisions this store has in common using graphwalker.

114

115

:param graphwalker: A graphwalker object.

116

:return: List of SHAs that are in common

117

"""

118

haves = []

119

sha = graphwalker.next()

120

while sha:

121

if sha in self:

122

haves.append(sha)

123

graphwalker.ack(sha)

124

sha = graphwalker.next()

125

return haves

126

127

def get_graph_walker(self, heads):

128

"""Obtain a graph walker for this object store.

129

130

:param heads: Local heads to start search with

131

:return: GraphWalker object

132

"""

133

return ObjectStoreGraphWalker(heads, lambda sha: self[sha].parents)

134

135

136

def generate_pack_contents(self, have, want):

137

return self.iter_shas(self.find_missing_objects(have, want))

138

139

140

class DiskObjectStore(BaseObjectStore):

141

"""Git-style object store that exists on disk."""

47

142

48

143

def __init__(self, path):

49

144

"""Open an object store.

54

149

self._pack_cache = None

55

150

self.pack_dir = os.path.join(self.path, PACKDIR)

56

151

57

def determine_wants_all(self, refs):

58

return [sha for (ref, sha) in refs.iteritems() if not sha in self and not ref.endswith("^{}")]

59

60

def iter_shas(self, shas):

61

"""Iterate over the objects for the specified shas.

62

63

:param shas: Iterable object with SHAs

64

"""

65

return ObjectStoreIterator(self, shas)

66

67

152

def __contains__(self, sha):

153

"""Check if a particular object is present by SHA1."""

68

154

for pack in self.packs:

69

155

if sha in pack:

70

156

return True

73

159

return True

74

160

return False

75

161

162

def __iter__(self):

163

"""Iterate over the SHAs that are present in this store."""

164

iterables = self.packs + [self._iter_shafile_shas()]

165

return itertools.chain(*iterables)

166

76

167

@property

77

168

def packs(self):

78

169

"""List with pack objects."""

79

170

if self._pack_cache is None:

80

self._pack_cache = list(load_packs(self.pack_dir))

171

self._pack_cache = list(self._load_packs())

81

172

return self._pack_cache

82

173

174

def _load_packs(self):

175

if not os.path.exists(self.pack_dir):

176

return

177

for name in os.listdir(self.pack_dir):

178

if name.startswith("pack-") and name.endswith(".pack"):

179

yield Pack(os.path.join(self.pack_dir, name[:-len(".pack")]))

180

83

181

def _add_known_pack(self, path):

84

182

"""Add a newly appeared pack to the cache by path.

85

183

93

191

# Check from object dir

94

192

return os.path.join(self.path, dir, file)

95

193

194

def _iter_shafile_shas(self):

195

for base in os.listdir(self.path):

196

if len(base) != 2:

197

continue

198

for rest in os.listdir(os.path.join(self.path, base)):

199

yield base+rest

200

96

201

def _get_shafile(self, sha):

97

202

path = self._get_shafile_path(sha)

98

203

if os.path.exists(path):

100

205

return None

101

206

102

207

def _add_shafile(self, sha, o):

103

path = self._get_shafile_path(sha)

104

f = os.path.open(path, 'w')

208

dir = os.path.join(self.path, sha[:2])

209

if not os.path.isdir(dir):

210

os.mkdir(dir)

211

path = os.path.join(dir, sha[2:])

212

if os.path.exists(path):

213

return # Already there, no need to write again

214

f = open(path, 'w+')

105

215

try:

106

f.write(o._header())

107

f.write(o._text)

216

f.write(o.as_legacy_object())

108

217

finally:

109

218

f.close()

110

219

131

240

hexsha = sha_to_hex(name)

132

241

ret = self._get_shafile(hexsha)

133

242

if ret is not None:

134

return ret.as_raw_string()

243

return ret.type, ret.as_raw_string()

135

244

raise KeyError(hexsha)

136

245

137

def __getitem__(self, sha):

138

type, uncomp = self.get_raw(sha)

139

return ShaFile.from_raw_string(type, uncomp)

140

141

246

def move_in_thin_pack(self, path):

142

247

"""Move a specific file containing a pack into the pack directory.

143

248

146

251

147

252

:param path: Path to the pack file.

148

253

"""

149

p = PackData(path)

254

data = PackData(path)

255

256

# Write index for the thin pack (do we really need this?)

257

temppath = os.path.join(self.pack_dir,

258

sha_to_hex(urllib2.randombytes(20))+".tempidx")

259

data.create_index_v2(temppath, self.get_raw)

260

p = Pack.from_objects(data, load_pack_index(temppath))

261

262

# Write a full pack version

150

263

temppath = os.path.join(self.pack_dir,

151

264

sha_to_hex(urllib2.randombytes(20))+".temppack")

152

write_pack(temppath, p.iterobjects(self.get_raw), len(p))

265

write_pack(temppath, ((o, None) for o in p.iterobjects(self.get_raw)),

266

len(p))

153

267

pack_sha = load_pack_index(temppath+".idx").objects_sha1()

154

268

newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)

155

269

os.rename(temppath+".pack", newbasename+".pack")

179

293

in a different pack.

180

294

"""

181

295

fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")

182

f = os.fdopen(fd, 'w')

296

f = os.fdopen(fd, 'wb')

183

297

def commit():

184

298

os.fsync(fd)

185

299

f.close()

194

308

call when the pack is finished.

195

309

"""

196

310

fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")

197

f = os.fdopen(fd, 'w')

311

f = os.fdopen(fd, 'wb')

198

312

def commit():

199

313

os.fsync(fd)

200

314

f.close()

202

316

self.move_in_pack(path)

203

317

return f, commit

204

318

319

def add_object(self, obj):

320

"""Add a single object to this object store.

321

322

"""

323

self._add_shafile(obj.id, obj)

324

205

325

def add_objects(self, objects):

206

326

"""Add a set of objects to this object store.

207

327

214

334

commit()

215

335

216

336

337

class MemoryObjectStore(BaseObjectStore):

338

339

def __init__(self):

340

super(MemoryObjectStore, self).__init__()

341

self._data = {}

342

343

def __contains__(self, sha):

344

return sha in self._data

345

346

def __iter__(self):

347

"""Iterate over the SHAs that are present in this store."""

348

return self._data.iterkeys()

349

350

def get_raw(self, name):

351

"""Obtain the raw text for an object.

352

353

:param name: sha for the object.

354

:return: tuple with object type and object contents.

355

"""

356

return self[name].as_raw_string()

357

358

def __getitem__(self, name):

359

return self._data[name]

360

361

def add_object(self, obj):

362

"""Add a single object to this object store.

363

364

"""

365

self._data[obj.id] = obj

366

367

def add_objects(self, objects):

368

"""Add a set of objects to this object store.

369

370

:param objects: Iterable over a list of objects.

371

"""

372

for obj in objects:

373

self._data[obj.id] = obj

374

375

217

376

class ObjectImporter(object):

218

377

"""Interface for importing objects."""

219

378

279

438

return len(list(self.itershas()))

280

439

281

440

282

def tree_lookup_path(object_store, root_sha, path):

441

def tree_lookup_path(lookup_obj, root_sha, path):

442

"""Lookup an object in a Git tree.

443

444

:param lookup_obj: Callback for retrieving object by SHA1

445

:param root_sha: SHA1 of the root tree

446

:param path: Path to lookup

447

"""

283

448

parts = path.split("/")

284

449

sha = root_sha

285

450

for p in parts:

286

obj = object_store[sha]

451

obj = lookup_obj(sha)

287

452

if type(obj) is not Tree:

288

453

raise NotTreeError(sha)

454

if p == '':

455

continue

289

456

mode, sha = obj[p]

290

return object_store[sha]

457

return lookup_obj(sha)

458

459

460

class MissingObjectFinder(object):

461

"""Find the objects missing from another object store.

462

463

:param object_store: Object store containing at least all objects to be

464

sent

465

:param haves: SHA1s of commits not to send (already present in target)

466

:param wants: SHA1s of commits to send

467

:param progress: Optional function to report progress to.

468

"""

469

470

def __init__(self, object_store, haves, wants, progress=None):

471

self.sha_done = set(haves)

472

self.objects_to_send = set([(w, None, False) for w in wants if w not in haves])

473

self.object_store = object_store

474

if progress is None:

475

self.progress = lambda x: None

476

else:

477

self.progress = progress

478

479

def add_todo(self, entries):

480

self.objects_to_send.update([e for e in entries if not e[0] in self.sha_done])

481

482

def parse_tree(self, tree):

483

self.add_todo([(sha, name, not stat.S_ISDIR(mode)) for (mode, name, sha) in tree.entries()])

484

485

def parse_commit(self, commit):

486

self.add_todo([(commit.tree, "", False)])

487

self.add_todo([(p, None, False) for p in commit.parents])

488

489

def parse_tag(self, tag):

490

self.add_todo([(tag.object[1], None, False)])

491

492

def next(self):

493

if not self.objects_to_send:

494

return None

495

(sha, name, leaf) = self.objects_to_send.pop()

496

if not leaf:

497

o = self.object_store[sha]

498

if isinstance(o, Commit):

499

self.parse_commit(o)

500

elif isinstance(o, Tree):

501

self.parse_tree(o)

502

elif isinstance(o, Tag):

503

self.parse_tag(o)

504

self.sha_done.add(sha)

505

self.progress("counting objects: %d\r" % len(self.sha_done))

506

return (sha, name)

507

508

509

class ObjectStoreGraphWalker(object):

510

"""Graph walker that finds out what commits are missing from an object store."""

511

512

def __init__(self, local_heads, get_parents):

513

"""Create a new instance.

514

515

:param local_heads: Heads to start search with

516

:param get_parents: Function for finding the parents of a SHA1.

517

"""

518

self.heads = set(local_heads)

519

self.get_parents = get_parents

520

self.parents = {}

521

522

def ack(self, sha):

523

"""Ack that a particular revision and its ancestors are present in the source."""

524

if sha in self.heads:

525

self.heads.remove(sha)

526

if sha in self.parents:

527

for p in self.parents[sha]:

528

self.ack(p)

529

530

def next(self):

531

"""Iterate over ancestors of heads in the target."""

532

if self.heads:

533

ret = self.heads.pop()

534

ps = self.get_parents(ret)

535

self.parents[ret] = ps

536

self.heads.update(ps)

537

return ret

538

return None