17.41.690
by Jelmer Vernooij
Handle encoding better in working tree iter changes. |
1 |
# Copyright (C) 2010-2012 Jelmer Vernooij <jelmer@samba.org>
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
2 |
#
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
||
17 |
"""A Git repository implementation that uses a Bazaar transport."""
|
|
18 |
||
17.41.671
by Jelmer Vernooij
Use absolute_import everywhere. |
19 |
from __future__ import absolute_import |
20 |
||
17.41.29
by Jelmer Vernooij
Write git pack files rather than loose objects. |
21 |
from cStringIO import StringIO |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
22 |
|
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
23 |
import os |
17.69.17
by Jelmer Vernooij
Implement TransportRepo._determine_file_mode. |
24 |
import sys |
17.41.653
by Jelmer Vernooij
Merge a bunch of fixes from store-roundtrip-info. |
25 |
import urllib |
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
26 |
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
27 |
from dulwich.errors import ( |
28 |
NotGitRepository, |
|
17.34.7
by Jelmer Vernooij
more work. |
29 |
NoIndexPresent, |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
30 |
)
|
17.34.4
by Jelmer Vernooij
more work on transportgit. |
31 |
from dulwich.objects import ( |
32 |
ShaFile, |
|
33 |
)
|
|
34 |
from dulwich.object_store import ( |
|
35 |
PackBasedObjectStore, |
|
36 |
PACKDIR, |
|
37 |
)
|
|
38 |
from dulwich.pack import ( |
|
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
39 |
MemoryPackIndex, |
17.34.7
by Jelmer Vernooij
more work. |
40 |
PackData, |
17.34.4
by Jelmer Vernooij
more work on transportgit. |
41 |
Pack, |
17.41.13
by Jelmer Vernooij
Test transportgit. |
42 |
iter_sha1, |
17.34.7
by Jelmer Vernooij
more work. |
43 |
load_pack_index_file, |
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
44 |
write_pack_data, |
17.41.13
by Jelmer Vernooij
Test transportgit. |
45 |
write_pack_index_v2, |
17.34.4
by Jelmer Vernooij
more work on transportgit. |
46 |
)
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
47 |
from dulwich.repo import ( |
48 |
BaseRepo, |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
49 |
RefsContainer, |
17.41.196
by Jelmer Vernooij
Refactor repository initialization. |
50 |
BASE_DIRECTORIES, |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
51 |
INDEX_FILENAME, |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
52 |
OBJECTDIR, |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
53 |
REFSDIR, |
54 |
SYMREF, |
|
55 |
check_ref_format, |
|
56 |
read_packed_refs_with_peeled, |
|
57 |
read_packed_refs, |
|
58 |
write_packed_refs, |
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
59 |
)
|
60 |
||
17.73.1
by Jelmer Vernooij
Use relative imports where possible. |
61 |
from ... import ( |
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
62 |
transport as _mod_transport, |
63 |
)
|
|
17.73.1
by Jelmer Vernooij
Use relative imports where possible. |
64 |
from ...errors import ( |
17.41.10
by Jelmer Vernooij
Allow directories to already exist :-) |
65 |
FileExists, |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
66 |
NoSuchFile, |
17.41.23
by Jelmer Vernooij
Fix reading pack files over http. |
67 |
TransportNotPossible, |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
68 |
)
|
69 |
||
70 |
||
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
71 |
class TransportRefsContainer(RefsContainer): |
72 |
"""Refs container that reads refs from a transport."""
|
|
73 |
||
74 |
def __init__(self, transport): |
|
75 |
self.transport = transport |
|
76 |
self._packed_refs = None |
|
77 |
self._peeled_refs = None |
|
78 |
||
79 |
def __repr__(self): |
|
80 |
return "%s(%r)" % (self.__class__.__name__, self.transport) |
|
81 |
||
17.46.2
by Jelmer Vernooij
Fix transportgit. |
82 |
def _ensure_dir_exists(self, path): |
83 |
for n in range(path.count("/")): |
|
84 |
dirname = "/".join(path.split("/")[:n+1]) |
|
85 |
try: |
|
86 |
self.transport.mkdir(dirname) |
|
87 |
except FileExists: |
|
88 |
pass
|
|
89 |
||
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
90 |
def subkeys(self, base): |
91 |
keys = set() |
|
92 |
try: |
|
93 |
iter_files = self.transport.clone(base).iter_files_recursive() |
|
17.69.8
by Jelmer Vernooij
Some formatting fixes. |
94 |
keys.update(("%s/%s" % (base, urllib.unquote(refname))).strip("/") for |
17.46.2
by Jelmer Vernooij
Fix transportgit. |
95 |
refname in iter_files if check_ref_format("%s/%s" % (base, refname))) |
96 |
except (TransportNotPossible, NoSuchFile): |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
97 |
pass
|
98 |
for key in self.get_packed_refs(): |
|
99 |
if key.startswith(base): |
|
100 |
keys.add(key[len(base):].strip("/")) |
|
101 |
return keys |
|
102 |
||
103 |
def allkeys(self): |
|
104 |
keys = set() |
|
17.41.464
by Jelmer Vernooij
Avoid using HEAD. |
105 |
try: |
106 |
self.transport.get_bytes("HEAD") |
|
107 |
except NoSuchFile: |
|
108 |
pass
|
|
109 |
else: |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
110 |
keys.add("HEAD") |
111 |
try: |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
112 |
iter_files = list(self.transport.clone("refs").iter_files_recursive()) |
113 |
for filename in iter_files: |
|
17.41.653
by Jelmer Vernooij
Merge a bunch of fixes from store-roundtrip-info. |
114 |
refname = "refs/%s" % urllib.unquote(filename) |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
115 |
if check_ref_format(refname): |
116 |
keys.add(refname) |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
117 |
except (TransportNotPossible, NoSuchFile): |
118 |
pass
|
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
119 |
keys.update(self.get_packed_refs()) |
120 |
return keys |
|
121 |
||
122 |
def get_packed_refs(self): |
|
123 |
"""Get contents of the packed-refs file.
|
|
124 |
||
125 |
:return: Dictionary mapping ref names to SHA1s
|
|
126 |
||
127 |
:note: Will return an empty dictionary when no packed-refs file is
|
|
128 |
present.
|
|
129 |
"""
|
|
130 |
# TODO: invalidate the cache on repacking
|
|
131 |
if self._packed_refs is None: |
|
132 |
# set both to empty because we want _peeled_refs to be
|
|
133 |
# None if and only if _packed_refs is also None.
|
|
134 |
self._packed_refs = {} |
|
135 |
self._peeled_refs = {} |
|
136 |
try: |
|
137 |
f = self.transport.get("packed-refs") |
|
138 |
except NoSuchFile: |
|
139 |
return {} |
|
140 |
try: |
|
141 |
first_line = iter(f).next().rstrip() |
|
142 |
if (first_line.startswith("# pack-refs") and " peeled" in |
|
143 |
first_line): |
|
144 |
for sha, name, peeled in read_packed_refs_with_peeled(f): |
|
145 |
self._packed_refs[name] = sha |
|
146 |
if peeled: |
|
147 |
self._peeled_refs[name] = peeled |
|
148 |
else: |
|
149 |
f.seek(0) |
|
150 |
for sha, name in read_packed_refs(f): |
|
151 |
self._packed_refs[name] = sha |
|
152 |
finally: |
|
153 |
f.close() |
|
154 |
return self._packed_refs |
|
155 |
||
156 |
def get_peeled(self, name): |
|
157 |
"""Return the cached peeled value of a ref, if available.
|
|
158 |
||
159 |
:param name: Name of the ref to peel
|
|
160 |
:return: The peeled value of the ref. If the ref is known not point to a
|
|
161 |
tag, this will be the SHA the ref refers to. If the ref may point to
|
|
162 |
a tag, but no cached information is available, None is returned.
|
|
163 |
"""
|
|
164 |
self.get_packed_refs() |
|
165 |
if self._peeled_refs is None or name not in self._packed_refs: |
|
166 |
# No cache: no peeled refs were read, or this ref is loose
|
|
167 |
return None |
|
168 |
if name in self._peeled_refs: |
|
169 |
return self._peeled_refs[name] |
|
170 |
else: |
|
171 |
# Known not peelable
|
|
172 |
return self[name] |
|
173 |
||
174 |
def read_loose_ref(self, name): |
|
175 |
"""Read a reference file and return its contents.
|
|
176 |
||
177 |
If the reference file a symbolic reference, only read the first line of
|
|
178 |
the file. Otherwise, only read the first 40 bytes.
|
|
179 |
||
180 |
:param name: the refname to read, relative to refpath
|
|
181 |
:return: The contents of the ref file, or None if the file does not
|
|
182 |
exist.
|
|
183 |
:raises IOError: if any other error occurs
|
|
184 |
"""
|
|
185 |
try: |
|
186 |
f = self.transport.get(name) |
|
187 |
except NoSuchFile: |
|
188 |
return None |
|
17.41.649
by Jelmer Vernooij
Fix compatibility for fetching over dumb transport. |
189 |
f = StringIO(f.read()) |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
190 |
try: |
191 |
header = f.read(len(SYMREF)) |
|
192 |
if header == SYMREF: |
|
193 |
# Read only the first line
|
|
194 |
return header + iter(f).next().rstrip("\r\n") |
|
195 |
else: |
|
196 |
# Read only the first 40 bytes
|
|
197 |
return header + f.read(40-len(SYMREF)) |
|
198 |
finally: |
|
199 |
f.close() |
|
200 |
||
201 |
def _remove_packed_ref(self, name): |
|
202 |
if self._packed_refs is None: |
|
203 |
return
|
|
204 |
# reread cached refs from disk, while holding the lock
|
|
205 |
||
206 |
self._packed_refs = None |
|
207 |
self.get_packed_refs() |
|
208 |
||
209 |
if name not in self._packed_refs: |
|
210 |
return
|
|
211 |
||
212 |
del self._packed_refs[name] |
|
213 |
if name in self._peeled_refs: |
|
214 |
del self._peeled_refs[name] |
|
17.41.31
by Jelmer Vernooij
Avoid writing to memory first unless strictly necessary. |
215 |
f = self.transport.open_write_stream("packed-refs") |
216 |
try: |
|
217 |
write_packed_refs(f, self._packed_refs, self._peeled_refs) |
|
218 |
finally: |
|
219 |
f.close() |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
220 |
|
221 |
def set_symbolic_ref(self, name, other): |
|
222 |
"""Make a ref point at another ref.
|
|
223 |
||
224 |
:param name: Name of the ref to set
|
|
225 |
:param other: Name of the ref to point at
|
|
226 |
"""
|
|
227 |
self._check_refname(name) |
|
228 |
self._check_refname(other) |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
229 |
self._ensure_dir_exists(name) |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
230 |
self.transport.put_bytes(name, SYMREF + other + '\n') |
231 |
||
232 |
def set_if_equals(self, name, old_ref, new_ref): |
|
233 |
"""Set a refname to new_ref only if it currently equals old_ref.
|
|
234 |
||
235 |
This method follows all symbolic references, and can be used to perform
|
|
236 |
an atomic compare-and-swap operation.
|
|
237 |
||
238 |
:param name: The refname to set.
|
|
239 |
:param old_ref: The old sha the refname must refer to, or None to set
|
|
240 |
unconditionally.
|
|
241 |
:param new_ref: The new sha the refname will refer to.
|
|
242 |
:return: True if the set was successful, False otherwise.
|
|
243 |
"""
|
|
244 |
try: |
|
17.69.14
by Jelmer Vernooij
Use new RefsContainer.follow(). |
245 |
realnames, _ = self.follow(name) |
246 |
realname = realnames[-1] |
|
247 |
except (KeyError, IndexError): |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
248 |
realname = name |
17.46.2
by Jelmer Vernooij
Fix transportgit. |
249 |
self._ensure_dir_exists(realname) |
250 |
self.transport.put_bytes(realname, new_ref+"\n") |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
251 |
return True |
252 |
||
253 |
def add_if_new(self, name, ref): |
|
254 |
"""Add a new reference only if it does not already exist.
|
|
255 |
||
256 |
This method follows symrefs, and only ensures that the last ref in the
|
|
257 |
chain does not exist.
|
|
258 |
||
259 |
:param name: The refname to set.
|
|
260 |
:param ref: The new sha the refname will refer to.
|
|
261 |
:return: True if the add was successful, False otherwise.
|
|
262 |
"""
|
|
263 |
try: |
|
17.69.14
by Jelmer Vernooij
Use new RefsContainer.follow(). |
264 |
realnames, contents = self.follow(name) |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
265 |
if contents is not None: |
266 |
return False |
|
17.69.14
by Jelmer Vernooij
Use new RefsContainer.follow(). |
267 |
realname = realnames[-1] |
268 |
except (KeyError, IndexError): |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
269 |
realname = name |
270 |
self._check_refname(realname) |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
271 |
self._ensure_dir_exists(realname) |
272 |
self.transport.put_bytes(realname, ref+"\n") |
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
273 |
return True |
274 |
||
275 |
def remove_if_equals(self, name, old_ref): |
|
276 |
"""Remove a refname only if it currently equals old_ref.
|
|
277 |
||
278 |
This method does not follow symbolic references. It can be used to
|
|
279 |
perform an atomic compare-and-delete operation.
|
|
280 |
||
281 |
:param name: The refname to delete.
|
|
282 |
:param old_ref: The old sha the refname must refer to, or None to delete
|
|
283 |
unconditionally.
|
|
284 |
:return: True if the delete was successful, False otherwise.
|
|
285 |
"""
|
|
286 |
self._check_refname(name) |
|
287 |
# may only be packed
|
|
288 |
try: |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
289 |
self.transport.delete(name) |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
290 |
except NoSuchFile: |
291 |
pass
|
|
292 |
self._remove_packed_ref(name) |
|
293 |
return True |
|
294 |
||
17.41.536
by Jelmer Vernooij
Implement TransportRefsContainer.get. |
295 |
def get(self, name, default=None): |
296 |
try: |
|
297 |
return self[name] |
|
298 |
except KeyError: |
|
299 |
return default |
|
300 |
||
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
301 |
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
302 |
class TransportRepo(BaseRepo): |
303 |
||
17.41.562
by Jelmer Vernooij
Keep track of refs text when opening bare repository. |
304 |
def __init__(self, transport, bare, refs_text=None): |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
305 |
self.transport = transport |
17.41.464
by Jelmer Vernooij
Avoid using HEAD. |
306 |
self.bare = bare |
307 |
if self.bare: |
|
308 |
self._controltransport = self.transport |
|
309 |
else: |
|
310 |
self._controltransport = self.transport.clone('.git') |
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
311 |
object_store = TransportObjectStore( |
312 |
self._controltransport.clone(OBJECTDIR)) |
|
17.41.562
by Jelmer Vernooij
Keep track of refs text when opening bare repository. |
313 |
if refs_text is not None: |
314 |
from dulwich.repo import InfoRefsContainer # dulwich >= 0.8.2 |
|
315 |
refs_container = InfoRefsContainer(StringIO(refs_text)) |
|
17.41.649
by Jelmer Vernooij
Fix compatibility for fetching over dumb transport. |
316 |
try: |
317 |
head = TransportRefsContainer(self._controltransport).read_loose_ref("HEAD") |
|
318 |
except KeyError: |
|
319 |
pass
|
|
320 |
else: |
|
321 |
refs_container._refs["HEAD"] = head |
|
17.41.562
by Jelmer Vernooij
Keep track of refs text when opening bare repository. |
322 |
else: |
323 |
refs_container = TransportRefsContainer(self._controltransport) |
|
17.69.8
by Jelmer Vernooij
Some formatting fixes. |
324 |
super(TransportRepo, self).__init__(object_store, |
17.41.562
by Jelmer Vernooij
Keep track of refs text when opening bare repository. |
325 |
refs_container) |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
326 |
|
17.69.17
by Jelmer Vernooij
Implement TransportRepo._determine_file_mode. |
327 |
def _determine_file_mode(self): |
328 |
# Be consistent with bzr
|
|
329 |
if sys.platform == 'win32': |
|
330 |
return False |
|
331 |
return True |
|
332 |
||
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
333 |
def get_named_file(self, path): |
334 |
"""Get a file from the control dir with a specific name.
|
|
335 |
||
336 |
Although the filename should be interpreted as a filename relative to
|
|
337 |
the control dir in a disk-baked Repo, the object returned need not be
|
|
338 |
pointing to a file in that location.
|
|
339 |
||
340 |
:param path: The path to the file, relative to the control dir.
|
|
341 |
:return: An open file object, or None if the file does not exist.
|
|
342 |
"""
|
|
343 |
try: |
|
344 |
return self._controltransport.get(path.lstrip('/')) |
|
345 |
except NoSuchFile: |
|
346 |
return None |
|
347 |
||
17.41.196
by Jelmer Vernooij
Refactor repository initialization. |
348 |
def _put_named_file(self, relpath, contents): |
349 |
self._controltransport.put_bytes(relpath, contents) |
|
350 |
||
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
351 |
def index_path(self): |
352 |
"""Return the path to the index file."""
|
|
353 |
return self._controltransport.local_abspath(INDEX_FILENAME) |
|
354 |
||
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
355 |
def open_index(self): |
356 |
"""Open the index for this repository."""
|
|
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
357 |
from dulwich.index import Index |
358 |
if not self.has_index(): |
|
359 |
raise NoIndexPresent() |
|
360 |
return Index(self.index_path()) |
|
361 |
||
362 |
def has_index(self): |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
363 |
"""Check if an index is present."""
|
364 |
# Bare repos must never have index files; non-bare repos may have a
|
|
365 |
# missing index file, which is treated as empty.
|
|
366 |
return not self.bare |
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
367 |
|
17.41.623
by Jelmer Vernooij
Provide get_config. |
368 |
def get_config(self): |
17.41.624
by Jelmer Vernooij
Support setting branch nicks. |
369 |
from dulwich.config import ConfigFile |
370 |
try: |
|
371 |
return ConfigFile.from_file(self._controltransport.get('config')) |
|
372 |
except NoSuchFile: |
|
373 |
return ConfigFile() |
|
374 |
||
375 |
def get_config_stack(self): |
|
376 |
from dulwich.config import StackedConfig |
|
17.41.623
by Jelmer Vernooij
Provide get_config. |
377 |
backends = [] |
17.41.624
by Jelmer Vernooij
Support setting branch nicks. |
378 |
p = self.get_config() |
379 |
if p is not None: |
|
380 |
backends.append(p) |
|
381 |
writable = p |
|
382 |
else: |
|
383 |
writable = None |
|
17.41.623
by Jelmer Vernooij
Provide get_config. |
384 |
backends.extend(StackedConfig.default_backends()) |
17.41.624
by Jelmer Vernooij
Support setting branch nicks. |
385 |
return StackedConfig(backends, writable=writable) |
17.41.623
by Jelmer Vernooij
Provide get_config. |
386 |
|
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
387 |
def __repr__(self): |
17.41.108
by Jelmer Vernooij
Better __repr__. |
388 |
return "<%s for %r>" % (self.__class__.__name__, self.transport) |
17.34.1
by Jelmer Vernooij
Add TransportRepo class. |
389 |
|
17.41.196
by Jelmer Vernooij
Refactor repository initialization. |
390 |
@classmethod
|
391 |
def init(cls, transport, bare=False): |
|
392 |
if not bare: |
|
393 |
transport.mkdir(".git") |
|
394 |
control_transport = transport.clone(".git") |
|
395 |
else: |
|
396 |
control_transport = transport |
|
397 |
for d in BASE_DIRECTORIES: |
|
398 |
control_transport.mkdir("/".join(d)) |
|
399 |
control_transport.mkdir(OBJECTDIR) |
|
400 |
TransportObjectStore.init(control_transport.clone(OBJECTDIR)) |
|
17.41.464
by Jelmer Vernooij
Avoid using HEAD. |
401 |
ret = cls(transport, bare) |
17.41.196
by Jelmer Vernooij
Refactor repository initialization. |
402 |
ret.refs.set_symbolic_ref("HEAD", "refs/heads/master") |
403 |
ret._init_files(bare) |
|
404 |
return ret |
|
405 |
||
17.34.4
by Jelmer Vernooij
more work on transportgit. |
406 |
|
407 |
class TransportObjectStore(PackBasedObjectStore): |
|
408 |
"""Git-style object store that exists on disk."""
|
|
409 |
||
410 |
def __init__(self, transport): |
|
411 |
"""Open an object store.
|
|
412 |
||
413 |
:param transport: Transport to open data from
|
|
414 |
"""
|
|
415 |
super(TransportObjectStore, self).__init__() |
|
416 |
self.transport = transport |
|
417 |
self.pack_transport = self.transport.clone(PACKDIR) |
|
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
418 |
self._alternates = None |
17.41.108
by Jelmer Vernooij
Better __repr__. |
419 |
|
420 |
def __repr__(self): |
|
421 |
return "%s(%r)" % (self.__class__.__name__, self.transport) |
|
422 |
||
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
423 |
@property
|
424 |
def alternates(self): |
|
425 |
if self._alternates is not None: |
|
426 |
return self._alternates |
|
427 |
self._alternates = [] |
|
428 |
for path in self._read_alternate_paths(): |
|
429 |
# FIXME: Check path
|
|
17.41.419
by Jelmer Vernooij
Add basic support for alternates. |
430 |
t = _mod_transport.get_transport_from_path(path) |
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
431 |
self._alternates.append(self.__class__(t)) |
432 |
return self._alternates |
|
433 |
||
434 |
def _read_alternate_paths(self): |
|
435 |
try: |
|
436 |
f = self.transport.get("info/alternates") |
|
437 |
except NoSuchFile: |
|
438 |
return [] |
|
439 |
ret = [] |
|
440 |
try: |
|
17.41.649
by Jelmer Vernooij
Fix compatibility for fetching over dumb transport. |
441 |
for l in f.read().splitlines(): |
17.41.417
by Jelmer Vernooij
Add basic support for alternates. |
442 |
if l[0] == "#": |
443 |
continue
|
|
444 |
if os.path.isabs(l): |
|
445 |
continue
|
|
446 |
ret.append(l) |
|
447 |
return ret |
|
448 |
finally: |
|
449 |
f.close() |
|
450 |
||
17.70.2
by William Grant
Fix TransportObjectStore to actually have packs with the new cache dict. |
451 |
@property
|
452 |
def packs(self): |
|
453 |
# FIXME: Never invalidates.
|
|
454 |
if not self._pack_cache: |
|
455 |
self._update_pack_cache() |
|
456 |
return self._pack_cache.values() |
|
457 |
||
458 |
def _update_pack_cache(self): |
|
459 |
for pack in self._load_packs(): |
|
460 |
self._pack_cache[pack._basename] = pack |
|
461 |
||
17.45.1
by Jelmer Vernooij
Allow using manual listing for pack contents. |
462 |
def _pack_names(self): |
463 |
try: |
|
464 |
f = self.transport.get('info/packs') |
|
465 |
except NoSuchFile: |
|
466 |
return self.pack_transport.list_dir(".") |
|
467 |
else: |
|
17.41.12
by Jelmer Vernooij
Allow using manual listing for pack contents. |
468 |
ret = [] |
17.41.649
by Jelmer Vernooij
Fix compatibility for fetching over dumb transport. |
469 |
for line in f.read().splitlines(): |
17.45.1
by Jelmer Vernooij
Allow using manual listing for pack contents. |
470 |
if not line: |
471 |
continue
|
|
472 |
(kind, name) = line.split(" ", 1) |
|
473 |
if kind != "P": |
|
474 |
continue
|
|
17.41.12
by Jelmer Vernooij
Allow using manual listing for pack contents. |
475 |
ret.append(name) |
476 |
return ret |
|
17.45.1
by Jelmer Vernooij
Allow using manual listing for pack contents. |
477 |
|
17.34.4
by Jelmer Vernooij
more work on transportgit. |
478 |
def _load_packs(self): |
17.34.7
by Jelmer Vernooij
more work. |
479 |
ret = [] |
17.45.1
by Jelmer Vernooij
Allow using manual listing for pack contents. |
480 |
for name in self._pack_names(): |
17.34.9
by Jelmer Vernooij
Remove unused write code. |
481 |
if name.startswith("pack-") and name.endswith(".pack"): |
17.41.23
by Jelmer Vernooij
Fix reading pack files over http. |
482 |
try: |
483 |
size = self.pack_transport.stat(name).st_size |
|
484 |
except TransportNotPossible: |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
485 |
# FIXME: This reads the whole pack file at once
|
486 |
f = self.pack_transport.get(name) |
|
487 |
contents = f.read() |
|
488 |
pd = PackData(name, StringIO(contents), size=len(contents)) |
|
17.41.23
by Jelmer Vernooij
Fix reading pack files over http. |
489 |
else: |
17.46.2
by Jelmer Vernooij
Fix transportgit. |
490 |
pd = PackData(name, self.pack_transport.get(name), |
17.41.23
by Jelmer Vernooij
Fix reading pack files over http. |
491 |
size=size) |
17.34.9
by Jelmer Vernooij
Remove unused write code. |
492 |
idxname = name.replace(".pack", ".idx") |
17.46.2
by Jelmer Vernooij
Fix transportgit. |
493 |
idx = load_pack_index_file(idxname, self.pack_transport.get(idxname)) |
494 |
pack = Pack.from_objects(pd, idx) |
|
17.73.9
by Jelmer Vernooij
Provide TransportObjectStore._remove_pack. |
495 |
pack._basename = idxname[:-4] |
17.41.27
by Jelmer Vernooij
Load packs lazily. |
496 |
ret.append(pack) |
17.34.7
by Jelmer Vernooij
more work. |
497 |
return ret |
17.34.4
by Jelmer Vernooij
more work on transportgit. |
498 |
|
499 |
def _iter_loose_objects(self): |
|
500 |
for base in self.transport.list_dir('.'): |
|
501 |
if len(base) != 2: |
|
502 |
continue
|
|
503 |
for rest in self.transport.list_dir(base): |
|
504 |
yield base+rest |
|
505 |
||
506 |
def _split_loose_object(self, sha): |
|
507 |
return (sha[:2], sha[2:]) |
|
508 |
||
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
509 |
def _remove_loose_object(self, sha): |
510 |
path = '%s/%s' % self._split_loose_object(sha) |
|
17.46.2
by Jelmer Vernooij
Fix transportgit. |
511 |
self.transport.delete(path) |
17.46.1
by Jelmer Vernooij
use transport repo objects even for local access. |
512 |
|
17.73.9
by Jelmer Vernooij
Provide TransportObjectStore._remove_pack. |
513 |
def _remove_pack(self, pack): |
514 |
self.pack_transport.delete(pack.data.filename) |
|
515 |
self.pack_transport.delete(pack.index.filename) |
|
516 |
||
17.34.4
by Jelmer Vernooij
more work on transportgit. |
517 |
def _get_loose_object(self, sha): |
518 |
path = '%s/%s' % self._split_loose_object(sha) |
|
519 |
try: |
|
17.25.344
by Jelmer Vernooij
Fix compatibility with newer versions of Dulwich. |
520 |
return ShaFile.from_file(self.transport.get(path)) |
17.34.4
by Jelmer Vernooij
more work on transportgit. |
521 |
except NoSuchFile: |
522 |
return None |
|
523 |
||
524 |
def add_object(self, obj): |
|
525 |
"""Add a single object to this object store.
|
|
526 |
||
527 |
:param obj: Object to add
|
|
528 |
"""
|
|
529 |
(dir, file) = self._split_loose_object(obj.id) |
|
17.41.10
by Jelmer Vernooij
Allow directories to already exist :-) |
530 |
try: |
531 |
self.transport.mkdir(dir) |
|
532 |
except FileExists: |
|
533 |
pass
|
|
17.34.4
by Jelmer Vernooij
more work on transportgit. |
534 |
path = "%s/%s" % (dir, file) |
535 |
if self.transport.has(path): |
|
536 |
return # Already there, no need to write again |
|
537 |
self.transport.put_bytes(path, obj.as_legacy_object()) |
|
538 |
||
17.41.14
by Jelmer Vernooij
Avoid using os module in transportgit. |
539 |
def move_in_pack(self, f): |
17.41.13
by Jelmer Vernooij
Test transportgit. |
540 |
"""Move a specific file containing a pack into the pack directory.
|
541 |
||
542 |
:note: The file should be on the same file system as the
|
|
543 |
packs directory.
|
|
544 |
||
545 |
:param path: Path to the pack file.
|
|
546 |
"""
|
|
17.41.14
by Jelmer Vernooij
Avoid using os module in transportgit. |
547 |
f.seek(0) |
17.73.9
by Jelmer Vernooij
Provide TransportObjectStore._remove_pack. |
548 |
p = PackData("", f, len(f.getvalue())) |
17.41.13
by Jelmer Vernooij
Test transportgit. |
549 |
entries = p.sorted_entries() |
17.41.14
by Jelmer Vernooij
Avoid using os module in transportgit. |
550 |
basename = "pack-%s" % iter_sha1(entry[0] for entry in entries) |
551 |
f.seek(0) |
|
552 |
self.pack_transport.put_file(basename + ".pack", f) |
|
17.73.9
by Jelmer Vernooij
Provide TransportObjectStore._remove_pack. |
553 |
p._filename = basename + ".pack" |
17.41.31
by Jelmer Vernooij
Avoid writing to memory first unless strictly necessary. |
554 |
idxfile = self.pack_transport.open_write_stream(basename + ".idx") |
555 |
try: |
|
556 |
write_pack_index_v2(idxfile, entries, p.get_stored_checksum()) |
|
557 |
finally: |
|
558 |
idxfile.close() |
|
559 |
idxfile = self.pack_transport.get(basename + ".idx") |
|
17.41.29
by Jelmer Vernooij
Write git pack files rather than loose objects. |
560 |
idx = load_pack_index_file(basename+".idx", idxfile) |
17.41.14
by Jelmer Vernooij
Avoid using os module in transportgit. |
561 |
final_pack = Pack.from_objects(p, idx) |
17.73.9
by Jelmer Vernooij
Provide TransportObjectStore._remove_pack. |
562 |
final_pack._basename = basename |
17.72.2
by Jelmer Vernooij
Use dulwich 0.9.7. |
563 |
self._add_known_pack(basename, final_pack) |
17.41.13
by Jelmer Vernooij
Test transportgit. |
564 |
return final_pack |
565 |
||
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
566 |
def add_thin_pack(self): |
567 |
"""Add a new thin pack to this object store.
|
|
568 |
||
569 |
Thin packs are packs that contain deltas with parents that exist
|
|
570 |
in a different pack.
|
|
571 |
"""
|
|
572 |
from cStringIO import StringIO |
|
573 |
f = StringIO() |
|
574 |
def commit(): |
|
575 |
if len(f.getvalue()) > 0: |
|
576 |
return self.move_in_thin_pack(f) |
|
577 |
else: |
|
578 |
return None |
|
579 |
return f, commit |
|
580 |
||
581 |
def move_in_thin_pack(self, f): |
|
582 |
"""Move a specific file containing a pack into the pack directory.
|
|
583 |
||
584 |
:note: The file should be on the same file system as the
|
|
585 |
packs directory.
|
|
586 |
||
587 |
:param path: Path to the pack file.
|
|
588 |
"""
|
|
589 |
f.seek(0) |
|
17.41.375
by Jelmer Vernooij
Fix compatibility with newer versions of dulwich. |
590 |
data = PackData.from_file(self.get_raw, f, len(f.getvalue())) |
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
591 |
idx = MemoryPackIndex(data.sorted_entries(), data.get_stored_checksum()) |
592 |
p = Pack.from_objects(data, idx) |
|
593 |
||
594 |
pack_sha = idx.objects_sha1() |
|
595 |
||
17.41.366
by Jelmer Vernooij
Switch to dulwich 0.8.0. |
596 |
datafile = self.pack_transport.open_write_stream( |
597 |
"pack-%s.pack" % pack_sha) |
|
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
598 |
try: |
17.41.366
by Jelmer Vernooij
Switch to dulwich 0.8.0. |
599 |
entries, data_sum = write_pack_data(datafile, p.pack_tuples()) |
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
600 |
finally: |
601 |
datafile.close() |
|
602 |
entries.sort() |
|
17.41.366
by Jelmer Vernooij
Switch to dulwich 0.8.0. |
603 |
idxfile = self.pack_transport.open_write_stream( |
604 |
"pack-%s.idx" % pack_sha) |
|
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
605 |
try: |
606 |
write_pack_index_v2(idxfile, data.sorted_entries(), data_sum) |
|
607 |
finally: |
|
608 |
idxfile.close() |
|
17.70.4
by William Grant
Update the other _add_known_pack callsite. |
609 |
basename = "pack-%s" % pack_sha |
610 |
final_pack = Pack(basename) |
|
611 |
self._add_known_pack(basename, final_pack) |
|
17.41.80
by Jelmer Vernooij
Initial work on supporting move_in_thin_pack. |
612 |
return final_pack |
613 |
||
17.34.4
by Jelmer Vernooij
more work on transportgit. |
614 |
def add_pack(self): |
17.69.8
by Jelmer Vernooij
Some formatting fixes. |
615 |
"""Add a new pack to this object store.
|
17.34.4
by Jelmer Vernooij
more work on transportgit. |
616 |
|
17.69.8
by Jelmer Vernooij
Some formatting fixes. |
617 |
:return: Fileobject to write to and a commit function to
|
17.34.4
by Jelmer Vernooij
more work on transportgit. |
618 |
call when the pack is finished.
|
619 |
"""
|
|
17.41.14
by Jelmer Vernooij
Avoid using os module in transportgit. |
620 |
from cStringIO import StringIO |
621 |
f = StringIO() |
|
17.41.13
by Jelmer Vernooij
Test transportgit. |
622 |
def commit(): |
17.41.14
by Jelmer Vernooij
Avoid using os module in transportgit. |
623 |
if len(f.getvalue()) > 0: |
624 |
return self.move_in_pack(f) |
|
17.41.13
by Jelmer Vernooij
Test transportgit. |
625 |
else: |
626 |
return None |
|
17.41.703
by Jelmer Vernooij
Fix compatibility with dulwich 0.9.1. |
627 |
def abort(): |
628 |
return None |
|
629 |
return f, commit, abort |
|
17.41.11
by Jelmer Vernooij
Add init function. |
630 |
|
631 |
@classmethod
|
|
632 |
def init(cls, transport): |
|
633 |
transport.mkdir('info') |
|
634 |
transport.mkdir(PACKDIR) |
|
635 |
return cls(transport) |
|
17.73.9
by Jelmer Vernooij
Provide TransportObjectStore._remove_pack. |
636 |
|
637 |
def _remove_pack(self, pack): |
|
638 |
self.pack_transport.delete(pack.data.filename) |
|
639 |
self.pack_transport.delete(os.path.basename(pack.index.path)) |