26
25
from bzrlib import lru_cache, trace
27
from bzrlib.plugins.fastimport import branch_mapper, helpers
30
class _Cleanup(object):
31
"""This class makes sure we clean up when CacheManager goes away.
33
We use a helper class to ensure that we are never in a refcycle.
36
def __init__(self, disk_blobs):
37
self.disk_blobs = disk_blobs
39
self.small_blobs = None
45
if self.disk_blobs is not None:
46
for info in self.disk_blobs.itervalues():
47
if info[-1] is not None:
49
self.disk_blobs = None
50
if self.small_blobs is not None:
51
self.small_blobs.close()
52
self.small_blobs = None
53
if self.tempdir is not None:
54
shutils.rmtree(self.tempdir)
58
class _Cleanup(object):
59
"""This class makes sure we clean up when CacheManager goes away.
61
We use a helper class to ensure that we are never in a refcycle.
64
def __init__(self, disk_blobs):
65
self.disk_blobs = disk_blobs
67
self.small_blobs = None
73
if self.disk_blobs is not None:
74
for info in self.disk_blobs.itervalues():
75
if info[-1] is not None:
77
self.disk_blobs = None
78
if self.small_blobs is not None:
79
self.small_blobs.close()
80
self.small_blobs = None
81
if self.tempdir is not None:
82
shutils.rmtree(self.tempdir)
26
from bzrlib.plugins.fastimport import (
29
from fastimport.helpers import (
32
from fastimport.reftracker import (
37
class _Cleanup(object):
38
"""This class makes sure we clean up when CacheManager goes away.
40
We use a helper class to ensure that we are never in a refcycle.
43
def __init__(self, disk_blobs):
44
self.disk_blobs = disk_blobs
46
self.small_blobs = None
52
if self.disk_blobs is not None:
53
for info in self.disk_blobs.itervalues():
54
if info[-1] is not None:
56
self.disk_blobs = None
57
if self.small_blobs is not None:
58
self.small_blobs.close()
59
self.small_blobs = None
60
if self.tempdir is not None:
61
shutil.rmtree(self.tempdir)
64
class _Cleanup(object):
65
"""This class makes sure we clean up when CacheManager goes away.
67
We use a helper class to ensure that we are never in a refcycle.
70
def __init__(self, disk_blobs):
71
self.disk_blobs = disk_blobs
73
self.small_blobs = None
79
if self.disk_blobs is not None:
80
for info in self.disk_blobs.itervalues():
81
if info[-1] is not None:
83
self.disk_blobs = None
84
if self.small_blobs is not None:
85
self.small_blobs.close()
86
self.small_blobs = None
87
if self.tempdir is not None:
88
shutil.rmtree(self.tempdir)
85
91
class CacheManager(object):
87
93
_small_blob_threshold = 25*1024
88
94
_sticky_cache_size = 300*1024*1024
89
95
_sticky_flushed_size = 100*1024*1024
122
128
# (path, branch_ref) -> file-ids - as generated.
123
129
# (Use store_file_id/fetch_fileid methods rather than direct access.)
125
# Head tracking: last ref, last id per ref & map of commit ids to ref*s*
130
131
# Work out the blobs to make sticky - None means all
131
132
self._blob_ref_counts = {}
132
133
if info is not None:
145
146
# than reinstantiate on every usage
146
147
self.branch_mapper = branch_mapper.BranchMapper()
149
self.reftracker = RefTracker()
148
151
def dump_stats(self, note=trace.note):
149
152
"""Dump some statistics about what we cached."""
150
153
# TODO: add in inventory stastistics
153
156
self._show_stats_for(self.revision_ids, "revision-ids", note=note)
154
157
# These aren't interesting so omit from the output, at least for now
155
158
#self._show_stats_for(self._blobs, "other blobs", note=note)
156
#self._show_stats_for(self.last_ids, "last-ids", note=note)
157
#self._show_stats_for(self.heads, "heads", note=note)
159
#self.reftracker.dump_stats(note=note)
159
161
def _show_stats_for(self, dict, label, note=trace.note, tuple_key=False):
160
162
"""Dump statistics about a given dictionary.
176
178
size = size / 1024
178
180
note(" %-12s: %8.1f %s (%d %s)" % (label, size, unit, count,
179
helpers.single_plural(count, "item", "items")))
181
single_plural(count, "item", "items")))
181
183
def clear_all(self):
182
184
"""Free up any memory used by the caches."""
183
185
self._blobs.clear()
184
186
self._sticky_blobs.clear()
185
187
self.revision_ids.clear()
186
self.last_ids.clear()
188
self.reftracker.clear()
188
189
self.inventories.clear()
190
191
def _flush_blobs_to_disk(self):
193
194
total_blobs = len(sticky_blobs)
194
195
blobs.sort(key=lambda k:len(sticky_blobs[k]))
195
196
if self._tempdir is None:
196
tempdir = tempfile.mkdtemp(prefix='bzr_fastimport_blobs-')
197
tempdir = tempfile.mkdtemp(prefix='fastimport_blobs-')
197
198
self._tempdir = tempdir
198
199
self._cleanup.tempdir = self._tempdir
199
200
self._cleanup.small_blobs = tempfile.TemporaryFile(
234
235
trace.note('flushed %d/%d blobs w/ %.1fMB (%.1fMB small) to disk'
235
236
% (count, total_blobs, bytes / 1024. / 1024,
236
237
n_small_bytes / 1024. / 1024))
239
239
def store_blob(self, id, data):
240
240
"""Store a blob of data."""
289
289
self._sticky_memory_bytes -= len(content)
292
def track_heads(self, cmd):
293
"""Track the repository heads given a CommitCommand.
295
:param cmd: the CommitCommand
296
:return: the list of parents in terms of commit-ids
298
# Get the true set of parents
299
if cmd.from_ is not None:
300
parents = [cmd.from_]
302
last_id = self.last_ids.get(cmd.ref)
303
if last_id is not None:
307
parents.extend(cmd.merges)
310
self.track_heads_for_ref(cmd.ref, cmd.id, parents)
313
def track_heads_for_ref(self, cmd_ref, cmd_id, parents=None):
314
if parents is not None:
315
for parent in parents:
316
if parent in self.heads:
317
del self.heads[parent]
318
self.heads.setdefault(cmd_id, set()).add(cmd_ref)
319
self.last_ids[cmd_ref] = cmd_id
320
self.last_ref = cmd_ref
added
removed
cache_manager.py
