« back to all changes in this revision
Viewing changes to src/mark-compact.cc
- browse files at revision 26
- compare with another revision
- download diff
- download tarball
- view history from revision 26
- Committer: Package Import Robot
- Author(s): Jérémy Lal
- Date: 2012-02-20 14:08:17 UTC
- mfrom: (15.1.24 sid)
- Revision ID: package-import@ubuntu.com-20120220140817-bsvmeoa4sxsj5hbz
Tags: 3.7.12.22-3
Fix mipsel build, allow test debug-step-3 to fail (non-crucial)
- files added:
- .pc/0014_disable_cross_build.patch
- .pc/0015_enable_mips_tests.patch
- .pc/0015_enable_mips_tests.patch/test
- .pc/0015_enable_mips_tests.patch/test/mjsunit
- .pc/0015_enable_mips_tests.patch/tools
- benchmarks/spinning-balls
- files removed:
- .pc/0010_fix_arm_bug.patch
- .pc/0010_fix_arm_bug.patch/src
- .pc/0010_fix_arm_bug.patch/src/arm
- .pc/0012_make_check_testsuites.patch
- .pc/0013_enable_soname.patch
- .pc/0013_enable_soname.patch/tools
- .pc/0013_enable_soname.patch/tools/gyp
- .pc/0014_disable_armv7_defaults.patch
- .pc/0015_hash-collision-fix-v8-3.6.patch
- .pc/0015_hash-collision-fix-v8-3.6.patch/src
- .pc/0015_hash-collision-fix-v8-3.6.patch/src/arm
- .pc/0015_hash-collision-fix-v8-3.6.patch/src/ia32
- .pc/0015_hash-collision-fix-v8-3.6.patch/src/mips
- .pc/0015_hash-collision-fix-v8-3.6.patch/src/x64
- .pc/0015_hash-collision-fix-v8-3.6.patch/test
- .pc/0015_hash-collision-fix-v8-3.6.patch/test/cctest
- .pc/0015_hash-collision-fix-v8-3.6.patch/test/mjsunit
- files modified:
- .pc/0005-enable-i18n-extension.patch/tools/gyp/v8.gyp
added
removed
src/mark-compact.cc
27
27
28
28
#include "v8.h"
29
29
30
#include "code-stubs.h"
30
31
#include "compilation-cache.h"
32
#include "deoptimizer.h"
31
33
#include "execution.h"
32
#include "heap-profiler.h"
33
34
#include "gdb-jit.h"
34
35
#include "global-handles.h"
36
#include "heap-profiler.h"
35
37
#include "ic-inl.h"
38
#include "incremental-marking.h"
36
39
#include "liveobjectlist-inl.h"
37
40
#include "mark-compact.h"
38
41
#include "objects-visiting.h"
42
#include "objects-visiting-inl.h"
39
43
#include "stub-cache.h"
40
44
41
45
namespace v8 {
42
46
namespace internal {
43
47
48
49
const char* Marking::kWhiteBitPattern = "00";
50
const char* Marking::kBlackBitPattern = "10";
51
const char* Marking::kGreyBitPattern = "11";
52
const char* Marking::kImpossibleBitPattern = "01";
53
54
44
55
// -------------------------------------------------------------------------
45
56
// MarkCompactCollector
46
57
48
59
#ifdef DEBUG
49
60
state_(IDLE),
50
61
#endif
51
force_compaction_(false),
52
compacting_collection_(false),
53
compact_on_next_gc_(false),
54
previous_marked_count_(0),
62
sweep_precisely_(false),
63
compacting_(false),
64
was_marked_incrementally_(false),
65
collect_maps_(FLAG_collect_maps),
55
66
tracer_(NULL),
56
#ifdef DEBUG
57
live_young_objects_size_(0),
58
live_old_pointer_objects_size_(0),
59
live_old_data_objects_size_(0),
60
live_code_objects_size_(0),
61
live_map_objects_size_(0),
62
live_cell_objects_size_(0),
63
live_lo_objects_size_(0),
64
live_bytes_(0),
65
#endif
67
migration_slots_buffer_(NULL),
66
68
heap_(NULL),
67
69
code_flusher_(NULL),
68
70
encountered_weak_maps_(NULL) { }
69
71
70
72
73
#ifdef DEBUG
74
class VerifyMarkingVisitor: public ObjectVisitor {
75
public:
76
void VisitPointers(Object** start, Object** end) {
77
for (Object** current = start; current < end; current++) {
78
if ((*current)->IsHeapObject()) {
79
HeapObject* object = HeapObject::cast(*current);
80
ASSERT(HEAP->mark_compact_collector()->IsMarked(object));
81
}
82
}
83
}
84
};
85
86
87
static void VerifyMarking(Address bottom, Address top) {
88
VerifyMarkingVisitor visitor;
89
HeapObject* object;
90
Address next_object_must_be_here_or_later = bottom;
91
92
for (Address current = bottom;
93
current < top;
94
current += kPointerSize) {
95
object = HeapObject::FromAddress(current);
96
if (MarkCompactCollector::IsMarked(object)) {
97
ASSERT(current >= next_object_must_be_here_or_later);
98
object->Iterate(&visitor);
99
next_object_must_be_here_or_later = current + object->Size();
100
}
101
}
102
}
103
104
105
static void VerifyMarking(NewSpace* space) {
106
Address end = space->top();
107
NewSpacePageIterator it(space->bottom(), end);
108
// The bottom position is at the start of its page. Allows us to use
109
// page->body() as start of range on all pages.
110
ASSERT_EQ(space->bottom(),
111
NewSpacePage::FromAddress(space->bottom())->body());
112
while (it.has_next()) {
113
NewSpacePage* page = it.next();
114
Address limit = it.has_next() ? page->body_limit() : end;
115
ASSERT(limit == end || !page->Contains(end));
116
VerifyMarking(page->body(), limit);
117
}
118
}
119
120
121
static void VerifyMarking(PagedSpace* space) {
122
PageIterator it(space);
123
124
while (it.has_next()) {
125
Page* p = it.next();
126
VerifyMarking(p->ObjectAreaStart(), p->ObjectAreaEnd());
127
}
128
}
129
130
131
static void VerifyMarking(Heap* heap) {
132
VerifyMarking(heap->old_pointer_space());
133
VerifyMarking(heap->old_data_space());
134
VerifyMarking(heap->code_space());
135
VerifyMarking(heap->cell_space());
136
VerifyMarking(heap->map_space());
137
VerifyMarking(heap->new_space());
138
139
VerifyMarkingVisitor visitor;
140
141
LargeObjectIterator it(heap->lo_space());
142
for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
143
if (MarkCompactCollector::IsMarked(obj)) {
144
obj->Iterate(&visitor);
145
}
146
}
147
148
heap->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
149
}
150
151
152
class VerifyEvacuationVisitor: public ObjectVisitor {
153
public:
154
void VisitPointers(Object** start, Object** end) {
155
for (Object** current = start; current < end; current++) {
156
if ((*current)->IsHeapObject()) {
157
HeapObject* object = HeapObject::cast(*current);
158
CHECK(!MarkCompactCollector::IsOnEvacuationCandidate(object));
159
}
160
}
161
}
162
};
163
164
165
static void VerifyEvacuation(Address bottom, Address top) {
166
VerifyEvacuationVisitor visitor;
167
HeapObject* object;
168
Address next_object_must_be_here_or_later = bottom;
169
170
for (Address current = bottom;
171
current < top;
172
current += kPointerSize) {
173
object = HeapObject::FromAddress(current);
174
if (MarkCompactCollector::IsMarked(object)) {
175
ASSERT(current >= next_object_must_be_here_or_later);
176
object->Iterate(&visitor);
177
next_object_must_be_here_or_later = current + object->Size();
178
}
179
}
180
}
181
182
183
static void VerifyEvacuation(NewSpace* space) {
184
NewSpacePageIterator it(space->bottom(), space->top());
185
VerifyEvacuationVisitor visitor;
186
187
while (it.has_next()) {
188
NewSpacePage* page = it.next();
189
Address current = page->body();
190
Address limit = it.has_next() ? page->body_limit() : space->top();
191
ASSERT(limit == space->top() || !page->Contains(space->top()));
192
while (current < limit) {
193
HeapObject* object = HeapObject::FromAddress(current);
194
object->Iterate(&visitor);
195
current += object->Size();
196
}
197
}
198
}
199
200
201
static void VerifyEvacuation(PagedSpace* space) {
202
PageIterator it(space);
203
204
while (it.has_next()) {
205
Page* p = it.next();
206
if (p->IsEvacuationCandidate()) continue;
207
VerifyEvacuation(p->ObjectAreaStart(), p->ObjectAreaEnd());
208
}
209
}
210
211
212
static void VerifyEvacuation(Heap* heap) {
213
VerifyEvacuation(heap->old_pointer_space());
214
VerifyEvacuation(heap->old_data_space());
215
VerifyEvacuation(heap->code_space());
216
VerifyEvacuation(heap->cell_space());
217
VerifyEvacuation(heap->map_space());
218
VerifyEvacuation(heap->new_space());
219
220
VerifyEvacuationVisitor visitor;
221
heap->IterateStrongRoots(&visitor, VISIT_ALL);
222
}
223
#endif
224
225
226
void MarkCompactCollector::AddEvacuationCandidate(Page* p) {
227
p->MarkEvacuationCandidate();
228
evacuation_candidates_.Add(p);
229
}
230
231
232
bool MarkCompactCollector::StartCompaction() {
233
if (!compacting_) {
234
ASSERT(evacuation_candidates_.length() == 0);
235
236
CollectEvacuationCandidates(heap()->old_pointer_space());
237
CollectEvacuationCandidates(heap()->old_data_space());
238
239
if (FLAG_compact_code_space) {
240
CollectEvacuationCandidates(heap()->code_space());
241
}
242
243
heap()->old_pointer_space()->EvictEvacuationCandidatesFromFreeLists();
244
heap()->old_data_space()->EvictEvacuationCandidatesFromFreeLists();
245
heap()->code_space()->EvictEvacuationCandidatesFromFreeLists();
246
247
compacting_ = evacuation_candidates_.length() > 0;
248
}
249
250
return compacting_;
251
}
252
253
71
254
void MarkCompactCollector::CollectGarbage() {
72
255
// Make sure that Prepare() has been called. The individual steps below will
73
256
// update the state as they proceed.
74
257
ASSERT(state_ == PREPARE_GC);
75
258
ASSERT(encountered_weak_maps_ == Smi::FromInt(0));
76
259
77
// Prepare has selected whether to compact the old generation or not.
78
// Tell the tracer.
79
if (IsCompacting()) tracer_->set_is_compacting();
80
81
260
MarkLiveObjects();
261
ASSERT(heap_->incremental_marking()->IsStopped());
82
262
83
if (FLAG_collect_maps) ClearNonLiveTransitions();
263
if (collect_maps_) ClearNonLiveTransitions();
84
264
85
265
ClearWeakMaps();
86
266
87
SweepLargeObjectSpace();
88
89
if (IsCompacting()) {
90
GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_COMPACT);
91
EncodeForwardingAddresses();
92
93
heap()->MarkMapPointersAsEncoded(true);
94
UpdatePointers();
95
heap()->MarkMapPointersAsEncoded(false);
96
heap()->isolate()->pc_to_code_cache()->Flush();
97
98
RelocateObjects();
99
} else {
100
SweepSpaces();
101
heap()->isolate()->pc_to_code_cache()->Flush();
267
#ifdef DEBUG
268
if (FLAG_verify_heap) {
269
VerifyMarking(heap_);
102
270
}
271
#endif
272
273
SweepSpaces();
274
275
if (!collect_maps_) ReattachInitialMaps();
276
277
heap_->isolate()->inner_pointer_to_code_cache()->Flush();
103
278
104
279
Finish();
105
280
106
// Save the count of marked objects remaining after the collection and
107
// null out the GC tracer.
108
previous_marked_count_ = tracer_->marked_count();
109
ASSERT(previous_marked_count_ == 0);
110
281
tracer_ = NULL;
111
282
}
112
283
113
284
285
#ifdef DEBUG
286
void MarkCompactCollector::VerifyMarkbitsAreClean(PagedSpace* space) {
287
PageIterator it(space);
288
289
while (it.has_next()) {
290
Page* p = it.next();
291
CHECK(p->markbits()->IsClean());
292
CHECK_EQ(0, p->LiveBytes());
293
}
294
}
295
296
void MarkCompactCollector::VerifyMarkbitsAreClean(NewSpace* space) {
297
NewSpacePageIterator it(space->bottom(), space->top());
298
299
while (it.has_next()) {
300
NewSpacePage* p = it.next();
301
CHECK(p->markbits()->IsClean());
302
CHECK_EQ(0, p->LiveBytes());
303
}
304
}
305
306
void MarkCompactCollector::VerifyMarkbitsAreClean() {
307
VerifyMarkbitsAreClean(heap_->old_pointer_space());
308
VerifyMarkbitsAreClean(heap_->old_data_space());
309
VerifyMarkbitsAreClean(heap_->code_space());
310
VerifyMarkbitsAreClean(heap_->cell_space());
311
VerifyMarkbitsAreClean(heap_->map_space());
312
VerifyMarkbitsAreClean(heap_->new_space());
313
314
LargeObjectIterator it(heap_->lo_space());
315
for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
316
MarkBit mark_bit = Marking::MarkBitFrom(obj);
317
ASSERT(Marking::IsWhite(mark_bit));
318
}
319
}
320
#endif
321
322
323
static void ClearMarkbitsInPagedSpace(PagedSpace* space) {
324
PageIterator it(space);
325
326
while (it.has_next()) {
327
Bitmap::Clear(it.next());
328
}
329
}
330
331
332
static void ClearMarkbitsInNewSpace(NewSpace* space) {
333
NewSpacePageIterator it(space->ToSpaceStart(), space->ToSpaceEnd());
334
335
while (it.has_next()) {
336
Bitmap::Clear(it.next());
337
}
338
}
339
340
341
void MarkCompactCollector::ClearMarkbits() {
342
ClearMarkbitsInPagedSpace(heap_->code_space());
343
ClearMarkbitsInPagedSpace(heap_->map_space());
344
ClearMarkbitsInPagedSpace(heap_->old_pointer_space());
345
ClearMarkbitsInPagedSpace(heap_->old_data_space());
346
ClearMarkbitsInPagedSpace(heap_->cell_space());
347
ClearMarkbitsInNewSpace(heap_->new_space());
348
349
LargeObjectIterator it(heap_->lo_space());
350
for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
351
MarkBit mark_bit = Marking::MarkBitFrom(obj);
352
mark_bit.Clear();
353
mark_bit.Next().Clear();
354
}
355
}
356
357
358
bool Marking::TransferMark(Address old_start, Address new_start) {
359
// This is only used when resizing an object.
360
ASSERT(MemoryChunk::FromAddress(old_start) ==
361
MemoryChunk::FromAddress(new_start));
362
363
// If the mark doesn't move, we don't check the color of the object.
364
// It doesn't matter whether the object is black, since it hasn't changed
365
// size, so the adjustment to the live data count will be zero anyway.
366
if (old_start == new_start) return false;
367
368
MarkBit new_mark_bit = MarkBitFrom(new_start);
369
MarkBit old_mark_bit = MarkBitFrom(old_start);
370
371
#ifdef DEBUG
372
ObjectColor old_color = Color(old_mark_bit);
373
#endif
374
375
if (Marking::IsBlack(old_mark_bit)) {
376
old_mark_bit.Clear();
377
ASSERT(IsWhite(old_mark_bit));
378
Marking::MarkBlack(new_mark_bit);
379
return true;
380
} else if (Marking::IsGrey(old_mark_bit)) {
381
ASSERT(heap_->incremental_marking()->IsMarking());
382
old_mark_bit.Clear();
383
old_mark_bit.Next().Clear();
384
ASSERT(IsWhite(old_mark_bit));
385
heap_->incremental_marking()->WhiteToGreyAndPush(
386
HeapObject::FromAddress(new_start), new_mark_bit);
387
heap_->incremental_marking()->RestartIfNotMarking();
388
}
389
390
#ifdef DEBUG
391
ObjectColor new_color = Color(new_mark_bit);
392
ASSERT(new_color == old_color);
393
#endif
394
395
return false;
396
}
397
398
399
const char* AllocationSpaceName(AllocationSpace space) {
400
switch (space) {
401
case NEW_SPACE: return "NEW_SPACE";
402
case OLD_POINTER_SPACE: return "OLD_POINTER_SPACE";
403
case OLD_DATA_SPACE: return "OLD_DATA_SPACE";
404
case CODE_SPACE: return "CODE_SPACE";
405
case MAP_SPACE: return "MAP_SPACE";
406
case CELL_SPACE: return "CELL_SPACE";
407
case LO_SPACE: return "LO_SPACE";
408
default:
409
UNREACHABLE();
410
}
411
412
return NULL;
413
}
414
415
416
void MarkCompactCollector::CollectEvacuationCandidates(PagedSpace* space) {
417
ASSERT(space->identity() == OLD_POINTER_SPACE ||
418
space->identity() == OLD_DATA_SPACE ||
419
space->identity() == CODE_SPACE);
420
421
int number_of_pages = space->CountTotalPages();
422
423
PageIterator it(space);
424
const int kMaxMaxEvacuationCandidates = 1000;
425
int max_evacuation_candidates = Min(
426
kMaxMaxEvacuationCandidates,
427
static_cast<int>(sqrt(static_cast<double>(number_of_pages / 2)) + 1));
428
429
if (FLAG_stress_compaction || FLAG_always_compact) {
430
max_evacuation_candidates = kMaxMaxEvacuationCandidates;
431
}
432
433
class Candidate {
434
public:
435
Candidate() : fragmentation_(0), page_(NULL) { }
436
Candidate(int f, Page* p) : fragmentation_(f), page_(p) { }
437
438
int fragmentation() { return fragmentation_; }
439
Page* page() { return page_; }
440
441
private:
442
int fragmentation_;
443
Page* page_;
444
};
445
446
Candidate candidates[kMaxMaxEvacuationCandidates];
447
448
int count = 0;
449
if (it.has_next()) it.next(); // Never compact the first page.
450
int fragmentation = 0;
451
Candidate* least = NULL;
452
while (it.has_next()) {
453
Page* p = it.next();
454
p->ClearEvacuationCandidate();
455
if (FLAG_stress_compaction) {
456
int counter = space->heap()->ms_count();
457
uintptr_t page_number = reinterpret_cast<uintptr_t>(p) >> kPageSizeBits;
458
if ((counter & 1) == (page_number & 1)) fragmentation = 1;
459
} else {
460
fragmentation = space->Fragmentation(p);
461
}
462
if (fragmentation != 0) {
463
if (count < max_evacuation_candidates) {
464
candidates[count++] = Candidate(fragmentation, p);
465
} else {
466
if (least == NULL) {
467
for (int i = 0; i < max_evacuation_candidates; i++) {
468
if (least == NULL ||
469
candidates[i].fragmentation() < least->fragmentation()) {
470
least = candidates + i;
471
}
472
}
473
}
474
if (least->fragmentation() < fragmentation) {
475
*least = Candidate(fragmentation, p);
476
least = NULL;
477
}
478
}
479
}
480
}
481
for (int i = 0; i < count; i++) {
482
AddEvacuationCandidate(candidates[i].page());
483
}
484
485
if (count > 0 && FLAG_trace_fragmentation) {
486
PrintF("Collected %d evacuation candidates for space %s\n",
487
count,
488
AllocationSpaceName(space->identity()));
489
}
490
}
491
492
493
void MarkCompactCollector::AbortCompaction() {
494
if (compacting_) {
495
int npages = evacuation_candidates_.length();
496
for (int i = 0; i < npages; i++) {
497
Page* p = evacuation_candidates_[i];
498
slots_buffer_allocator_.DeallocateChain(p->slots_buffer_address());
499
p->ClearEvacuationCandidate();
500
p->ClearFlag(MemoryChunk::RESCAN_ON_EVACUATION);
501
}
502
compacting_ = false;
503
evacuation_candidates_.Rewind(0);
504
invalidated_code_.Rewind(0);
505
}
506
ASSERT_EQ(0, evacuation_candidates_.length());
507
}
508
509
114
510
void MarkCompactCollector::Prepare(GCTracer* tracer) {
511
was_marked_incrementally_ = heap()->incremental_marking()->IsMarking();
512
513
// Disable collection of maps if incremental marking is enabled.
514
// Map collection algorithm relies on a special map transition tree traversal
515
// order which is not implemented for incremental marking.
516
collect_maps_ = FLAG_collect_maps && !was_marked_incrementally_;
517
115
518
// Rather than passing the tracer around we stash it in a static member
116
519
// variable.
117
520
tracer_ = tracer;
120
523
ASSERT(state_ == IDLE);
121
524
state_ = PREPARE_GC;
122
525
#endif
123
ASSERT(!FLAG_always_compact || !FLAG_never_compact);
124
125
compacting_collection_ =
126
FLAG_always_compact || force_compaction_ || compact_on_next_gc_;
127
compact_on_next_gc_ = false;
128
129
if (FLAG_never_compact) compacting_collection_ = false;
130
if (!heap()->map_space()->MapPointersEncodable())
131
compacting_collection_ = false;
132
if (FLAG_collect_maps) CreateBackPointers();
526
527
ASSERT(!FLAG_never_compact || !FLAG_always_compact);
528
529
if (collect_maps_) CreateBackPointers();
133
530
#ifdef ENABLE_GDB_JIT_INTERFACE
134
531
if (FLAG_gdbjit) {
135
532
// If GDBJIT interface is active disable compaction.
137
534
}
138
535
#endif
139
536
537
// Clear marking bits for precise sweeping to collect all garbage.
538
if (was_marked_incrementally_ && PreciseSweepingRequired()) {
539
heap()->incremental_marking()->Abort();
540
ClearMarkbits();
541
AbortCompaction();
542
was_marked_incrementally_ = false;
543
}
544
545
// Don't start compaction if we are in the middle of incremental
546
// marking cycle. We did not collect any slots.
547
if (!FLAG_never_compact && !was_marked_incrementally_) {
548
StartCompaction();
549
}
550
140
551
PagedSpaces spaces;
141
552
for (PagedSpace* space = spaces.next();
142
space != NULL; space = spaces.next()) {
143
space->PrepareForMarkCompact(compacting_collection_);
553
space != NULL;
554
space = spaces.next()) {
555
space->PrepareForMarkCompact();
144
556
}
145
557
146
558
#ifdef DEBUG
147
live_bytes_ = 0;
148
live_young_objects_size_ = 0;
149
live_old_pointer_objects_size_ = 0;
150
live_old_data_objects_size_ = 0;
151
live_code_objects_size_ = 0;
152
live_map_objects_size_ = 0;
153
live_cell_objects_size_ = 0;
154
live_lo_objects_size_ = 0;
559
if (!was_marked_incrementally_ && FLAG_verify_heap) {
560
VerifyMarkbitsAreClean();
561
}
155
562
#endif
156
563
}
157
564
168
575
heap()->isolate()->stub_cache()->Clear();
169
576
170
577
heap()->external_string_table_.CleanUp();
171
172
// If we've just compacted old space there's no reason to check the
173
// fragmentation limit. Just return.
174
if (HasCompacted()) return;
175
176
// We compact the old generation on the next GC if it has gotten too
177
// fragmented (ie, we could recover an expected amount of space by
178
// reclaiming the waste and free list blocks).
179
static const int kFragmentationLimit = 15; // Percent.
180
static const int kFragmentationAllowed = 1 * MB; // Absolute.
181
intptr_t old_gen_recoverable = 0;
182
intptr_t old_gen_used = 0;
183
184
OldSpaces spaces;
185
for (OldSpace* space = spaces.next(); space != NULL; space = spaces.next()) {
186
old_gen_recoverable += space->Waste() + space->AvailableFree();
187
old_gen_used += space->Size();
188
}
189
190
int old_gen_fragmentation =
191
static_cast<int>((old_gen_recoverable * 100.0) / old_gen_used);
192
if (old_gen_fragmentation > kFragmentationLimit &&
193
old_gen_recoverable > kFragmentationAllowed) {
194
compact_on_next_gc_ = true;
195
}
196
578
}
197
579
198
580
237
619
}
238
620
239
621
void AddCandidate(JSFunction* function) {
240
ASSERT(function->unchecked_code() ==
241
function->unchecked_shared()->unchecked_code());
622
ASSERT(function->code() == function->shared()->code());
242
623
243
624
SetNextCandidate(function, jsfunction_candidates_head_);
244
625
jsfunction_candidates_head_ = function;
258
639
while (candidate != NULL) {
259
640
next_candidate = GetNextCandidate(candidate);
260
641
261
SharedFunctionInfo* shared = candidate->unchecked_shared();
642
SharedFunctionInfo* shared = candidate->shared();
262
643
263
Code* code = shared->unchecked_code();
264
if (!code->IsMarked()) {
644
Code* code = shared->code();
645
MarkBit code_mark = Marking::MarkBitFrom(code);
646
if (!code_mark.Get()) {
265
647
shared->set_code(lazy_compile);
266
648
candidate->set_code(lazy_compile);
267
649
} else {
268
candidate->set_code(shared->unchecked_code());
650
candidate->set_code(shared->code());
269
651
}
270
652
653
// We are in the middle of a GC cycle so the write barrier in the code
654
// setter did not record the slot update and we have to do that manually.
655
Address slot = candidate->address() + JSFunction::kCodeEntryOffset;
656
Code* target = Code::cast(Code::GetObjectFromEntryAddress(slot));
657
isolate_->heap()->mark_compact_collector()->
658
RecordCodeEntrySlot(slot, target);
659
271
660
candidate = next_candidate;
272
661
}
273
662
284
673
next_candidate = GetNextCandidate(candidate);
285
674
SetNextCandidate(candidate, NULL);
286
675
287
Code* code = candidate->unchecked_code();
288
if (!code->IsMarked()) {
676
Code* code = candidate->code();
677
MarkBit code_mark = Marking::MarkBitFrom(code);
678
if (!code_mark.Get()) {
289
679
candidate->set_code(lazy_compile);
290
680
}
291
681
311
701
312
702
static SharedFunctionInfo** GetNextCandidateField(
313
703
SharedFunctionInfo* candidate) {
314
Code* code = candidate->unchecked_code();
704
Code* code = candidate->code();
315
705
return reinterpret_cast<SharedFunctionInfo**>(
316
706
code->address() + Code::kNextCodeFlushingCandidateOffset);
317
707
}
355
745
// except the maps for the object and its possible substrings might be
356
746
// marked.
357
747
HeapObject* object = HeapObject::cast(*p);
358
MapWord map_word = object->map_word();
359
map_word.ClearMark();
360
InstanceType type = map_word.ToMap()->instance_type();
748
if (!FLAG_clever_optimizations) return object;
749
Map* map = object->map();
750
InstanceType type = map->instance_type();
361
751
if ((type & kShortcutTypeMask) != kShortcutTypeTag) return object;
362
752
363
753
Object* second = reinterpret_cast<ConsString*>(object)->unchecked_second();
364
Heap* heap = map_word.ToMap()->heap();
365
if (second != heap->raw_unchecked_empty_string()) {
754
Heap* heap = map->GetHeap();
755
if (second != heap->empty_string()) {
366
756
return object;
367
757
}
368
758
404
794
FixedArray::BodyDescriptor,
405
795
void>::Visit);
406
796
797
table_.Register(kVisitGlobalContext, &VisitGlobalContext);
798
407
799
table_.Register(kVisitFixedDoubleArray, DataObjectVisitor::Visit);
408
800
409
table_.Register(kVisitGlobalContext,
410
&FixedBodyVisitor<StaticMarkingVisitor,
411
Context::MarkCompactBodyDescriptor,
412
void>::Visit);
413
414
801
table_.Register(kVisitByteArray, &DataObjectVisitor::Visit);
802
table_.Register(kVisitFreeSpace, &DataObjectVisitor::Visit);
415
803
table_.Register(kVisitSeqAsciiString, &DataObjectVisitor::Visit);
416
804
table_.Register(kVisitSeqTwoByteString, &DataObjectVisitor::Visit);
417
805
456
844
}
457
845
458
846
INLINE(static void VisitPointer(Heap* heap, Object** p)) {
459
MarkObjectByPointer(heap, p);
847
MarkObjectByPointer(heap->mark_compact_collector(), p, p);
460
848
}
461
849
462
850
INLINE(static void VisitPointers(Heap* heap, Object** start, Object** end)) {
466
854
if (VisitUnmarkedObjects(heap, start, end)) return;
467
855
// We are close to a stack overflow, so just mark the objects.
468
856
}
469
for (Object** p = start; p < end; p++) MarkObjectByPointer(heap, p);
857
MarkCompactCollector* collector = heap->mark_compact_collector();
858
for (Object** p = start; p < end; p++) {
859
MarkObjectByPointer(collector, start, p);
860
}
861
}
862
863
static void VisitGlobalPropertyCell(Heap* heap, RelocInfo* rinfo) {
864
ASSERT(rinfo->rmode() == RelocInfo::GLOBAL_PROPERTY_CELL);
865
JSGlobalPropertyCell* cell =
866
JSGlobalPropertyCell::cast(rinfo->target_cell());
867
MarkBit mark = Marking::MarkBitFrom(cell);
868
heap->mark_compact_collector()->MarkObject(cell, mark);
869
}
870
871
static inline void VisitEmbeddedPointer(Heap* heap, RelocInfo* rinfo) {
872
ASSERT(rinfo->rmode() == RelocInfo::EMBEDDED_OBJECT);
873
// TODO(mstarzinger): We do not short-circuit cons strings here, verify
874
// that there can be no such embedded pointers and add assertion here.
875
HeapObject* object = HeapObject::cast(rinfo->target_object());
876
heap->mark_compact_collector()->RecordRelocSlot(rinfo, object);
877
MarkBit mark = Marking::MarkBitFrom(object);
878
heap->mark_compact_collector()->MarkObject(object, mark);
470
879
}
471
880
472
881
static inline void VisitCodeTarget(Heap* heap, RelocInfo* rinfo) {
473
882
ASSERT(RelocInfo::IsCodeTarget(rinfo->rmode()));
474
Code* code = Code::GetCodeFromTargetAddress(rinfo->target_address());
475
if (FLAG_cleanup_code_caches_at_gc && code->is_inline_cache_stub()) {
883
Code* target = Code::GetCodeFromTargetAddress(rinfo->target_address());
884
if (FLAG_cleanup_code_caches_at_gc && target->is_inline_cache_stub()) {
476
885
IC::Clear(rinfo->pc());
477
886
// Please note targets for cleared inline cached do not have to be
478
887
// marked since they are contained in HEAP->non_monomorphic_cache().
888
target = Code::GetCodeFromTargetAddress(rinfo->target_address());
479
889
} else {
480
heap->mark_compact_collector()->MarkObject(code);
481
}
482
}
483
484
static void VisitGlobalPropertyCell(Heap* heap, RelocInfo* rinfo) {
485
ASSERT(rinfo->rmode() == RelocInfo::GLOBAL_PROPERTY_CELL);
486
Object* cell = rinfo->target_cell();
487
Object* old_cell = cell;
488
VisitPointer(heap, &cell);
489
if (cell != old_cell) {
490
rinfo->set_target_cell(reinterpret_cast<JSGlobalPropertyCell*>(cell));
491
}
890
if (FLAG_cleanup_code_caches_at_gc &&
891
target->kind() == Code::STUB &&
892
target->major_key() == CodeStub::CallFunction &&
893
target->has_function_cache()) {
894
CallFunctionStub::Clear(heap, rinfo->pc());
895
}
896
MarkBit code_mark = Marking::MarkBitFrom(target);
897
heap->mark_compact_collector()->MarkObject(target, code_mark);
898
}
899
heap->mark_compact_collector()->RecordRelocSlot(rinfo, target);
492
900
}
493
901
494
902
static inline void VisitDebugTarget(Heap* heap, RelocInfo* rinfo) {
496
904
rinfo->IsPatchedReturnSequence()) ||
497
905
(RelocInfo::IsDebugBreakSlot(rinfo->rmode()) &&
498
906
rinfo->IsPatchedDebugBreakSlotSequence()));
499
HeapObject* code = Code::GetCodeFromTargetAddress(rinfo->call_address());
500
heap->mark_compact_collector()->MarkObject(code);
907
Code* target = Code::GetCodeFromTargetAddress(rinfo->call_address());
908
MarkBit code_mark = Marking::MarkBitFrom(target);
909
heap->mark_compact_collector()->MarkObject(target, code_mark);
910
heap->mark_compact_collector()->RecordRelocSlot(rinfo, target);
501
911
}
502
912
503
913
// Mark object pointed to by p.
504
INLINE(static void MarkObjectByPointer(Heap* heap, Object** p)) {
914
INLINE(static void MarkObjectByPointer(MarkCompactCollector* collector,
915
Object** anchor_slot,
916
Object** p)) {
505
917
if (!(*p)->IsHeapObject()) return;
506
918
HeapObject* object = ShortCircuitConsString(p);
507
if (!object->IsMarked()) {
508
heap->mark_compact_collector()->MarkUnmarkedObject(object);
509
}
919
collector->RecordSlot(anchor_slot, p, object);
920
MarkBit mark = Marking::MarkBitFrom(object);
921
collector->MarkObject(object, mark);
510
922
}
511
923
512
924
515
927
HeapObject* obj)) {
516
928
#ifdef DEBUG
517
929
ASSERT(Isolate::Current()->heap()->Contains(obj));
518
ASSERT(!obj->IsMarked());
930
ASSERT(!HEAP->mark_compact_collector()->IsMarked(obj));
519
931
#endif
520
932
Map* map = obj->map();
521
collector->SetMark(obj);
933
Heap* heap = obj->GetHeap();
934
MarkBit mark = Marking::MarkBitFrom(obj);
935
heap->mark_compact_collector()->SetMark(obj, mark);
522
936
// Mark the map pointer and the body.
523
if (!map->IsMarked()) collector->MarkUnmarkedObject(map);
937
MarkBit map_mark = Marking::MarkBitFrom(map);
938
heap->mark_compact_collector()->MarkObject(map, map_mark);
524
939
IterateBody(map, obj);
525
940
}
526
941
536
951
MarkCompactCollector* collector = heap->mark_compact_collector();
537
952
// Visit the unmarked objects.
538
953
for (Object** p = start; p < end; p++) {
539
if (!(*p)->IsHeapObject()) continue;
540
HeapObject* obj = HeapObject::cast(*p);
541
if (obj->IsMarked()) continue;
954
Object* o = *p;
955
if (!o->IsHeapObject()) continue;
956
collector->RecordSlot(start, p, o);
957
HeapObject* obj = HeapObject::cast(o);
958
MarkBit mark = Marking::MarkBitFrom(obj);
959
if (mark.Get()) continue;
542
960
VisitUnmarkedObject(collector, obj);
543
961
}
544
962
return true;
545
963
}
546
964
547
965
static inline void VisitExternalReference(Address* p) { }
966
static inline void VisitExternalReference(RelocInfo* rinfo) { }
548
967
static inline void VisitRuntimeEntry(RelocInfo* rinfo) { }
549
968
550
969
private:
567
986
void> StructObjectVisitor;
568
987
569
988
static void VisitJSWeakMap(Map* map, HeapObject* object) {
570
MarkCompactCollector* collector = map->heap()->mark_compact_collector();
989
MarkCompactCollector* collector = map->GetHeap()->mark_compact_collector();
571
990
JSWeakMap* weak_map = reinterpret_cast<JSWeakMap*>(object);
572
991
573
992
// Enqueue weak map in linked list of encountered weak maps.
578
997
// Skip visiting the backing hash table containing the mappings.
579
998
int object_size = JSWeakMap::BodyDescriptor::SizeOf(map, object);
580
999
BodyVisitorBase<StaticMarkingVisitor>::IteratePointers(
581
map->heap(),
1000
map->GetHeap(),
582
1001
object,
583
1002
JSWeakMap::BodyDescriptor::kStartOffset,
584
1003
JSWeakMap::kTableOffset);
585
1004
BodyVisitorBase<StaticMarkingVisitor>::IteratePointers(
586
map->heap(),
1005
map->GetHeap(),
587
1006
object,
588
1007
JSWeakMap::kTableOffset + kPointerSize,
589
1008
object_size);
590
1009
591
1010
// Mark the backing hash table without pushing it on the marking stack.
592
ASSERT(!weak_map->unchecked_table()->IsMarked());
593
ASSERT(weak_map->unchecked_table()->map()->IsMarked());
594
collector->SetMark(weak_map->unchecked_table());
1011
ObjectHashTable* table = ObjectHashTable::cast(weak_map->table());
1012
ASSERT(!MarkCompactCollector::IsMarked(table));
1013
collector->SetMark(table, Marking::MarkBitFrom(table));
1014
collector->MarkObject(table->map(), Marking::MarkBitFrom(table->map()));
1015
ASSERT(MarkCompactCollector::IsMarked(table->map()));
595
1016
}
596
1017
597
1018
static void VisitCode(Map* map, HeapObject* object) {
598
1019
reinterpret_cast<Code*>(object)->CodeIterateBody<StaticMarkingVisitor>(
599
map->heap());
1020
map->GetHeap());
600
1021
}
601
1022
602
1023
// Code flushing support.
608
1029
static const int kRegExpCodeThreshold = 5;
609
1030
610
1031
inline static bool HasSourceCode(Heap* heap, SharedFunctionInfo* info) {
611
Object* undefined = heap->raw_unchecked_undefined_value();
1032
Object* undefined = heap->undefined_value();
612
1033
return (info->script() != undefined) &&
613
1034
(reinterpret_cast<Script*>(info->script())->source() != undefined);
614
1035
}
615
1036
616
1037
617
1038
inline static bool IsCompiled(JSFunction* function) {
618
return function->unchecked_code() !=
1039
return function->code() !=
619
1040
function->GetIsolate()->builtins()->builtin(Builtins::kLazyCompile);
620
1041
}
621
1042
622
1043
inline static bool IsCompiled(SharedFunctionInfo* function) {
623
return function->unchecked_code() !=
1044
return function->code() !=
624
1045
function->GetIsolate()->builtins()->builtin(Builtins::kLazyCompile);
625
1046
}
626
1047
629
1050
630
1051
// Code is either on stack, in compilation cache or referenced
631
1052
// by optimized version of function.
632
if (function->unchecked_code()->IsMarked()) {
633
shared_info->set_code_age(0);
1053
MarkBit code_mark = Marking::MarkBitFrom(function->code());
1054
if (code_mark.Get()) {
1055
if (!Marking::MarkBitFrom(shared_info).Get()) {
1056
shared_info->set_code_age(0);
1057
}
634
1058
return false;
635
1059
}
636
1060
637
1061
// We do not flush code for optimized functions.
638
if (function->code() != shared_info->unchecked_code()) {
1062
if (function->code() != shared_info->code()) {
639
1063
return false;
640
1064
}
641
1065
645
1069
inline static bool IsFlushable(Heap* heap, SharedFunctionInfo* shared_info) {
646
1070
// Code is either on stack, in compilation cache or referenced
647
1071
// by optimized version of function.
648
if (shared_info->unchecked_code()->IsMarked()) {
649
shared_info->set_code_age(0);
1072
MarkBit code_mark =
1073
Marking::MarkBitFrom(shared_info->code());
1074
if (code_mark.Get()) {
650
1075
return false;
651
1076
}
652
1077
658
1083
659
1084
// We never flush code for Api functions.
660
1085
Object* function_data = shared_info->function_data();
661
if (function_data->IsHeapObject() &&
662
(SafeMap(function_data)->instance_type() ==
663
FUNCTION_TEMPLATE_INFO_TYPE)) {
1086
if (function_data->IsFunctionTemplateInfo()) {
664
1087
return false;
665
1088
}
666
1089
701
1124
return true;
702
1125
}
703
1126
704
705
static inline Map* SafeMap(Object* obj) {
706
MapWord map_word = HeapObject::cast(obj)->map_word();
707
map_word.ClearMark();
708
map_word.ClearOverflow();
709
return map_word.ToMap();
710
}
711
712
713
static inline bool IsJSBuiltinsObject(Object* obj) {
714
return obj->IsHeapObject() &&
715
(SafeMap(obj)->instance_type() == JS_BUILTINS_OBJECT_TYPE);
716
}
717
718
719
1127
static inline bool IsValidNotBuiltinContext(Object* ctx) {
720
if (!ctx->IsHeapObject()) return false;
721
722
Map* map = SafeMap(ctx);
723
Heap* heap = map->heap();
724
if (!(map == heap->raw_unchecked_function_context_map() ||
725
map == heap->raw_unchecked_catch_context_map() ||
726
map == heap->raw_unchecked_with_context_map() ||
727
map == heap->raw_unchecked_global_context_map())) {
728
return false;
729
}
730
731
Context* context = reinterpret_cast<Context*>(ctx);
732
733
if (IsJSBuiltinsObject(context->global())) {
734
return false;
735
}
736
737
return true;
1128
return ctx->IsContext() &&
1129
!Context::cast(ctx)->global()->IsJSBuiltinsObject();
738
1130
}
739
1131
740
1132
754
1146
bool is_ascii) {
755
1147
// Make sure that the fixed array is in fact initialized on the RegExp.
756
1148
// We could potentially trigger a GC when initializing the RegExp.
757
if (SafeMap(re->data())->instance_type() != FIXED_ARRAY_TYPE) return;
1149
if (HeapObject::cast(re->data())->map()->instance_type() !=
1150
FIXED_ARRAY_TYPE) return;
758
1151
759
1152
// Make sure this is a RegExp that actually contains code.
760
1153
if (re->TypeTagUnchecked() != JSRegExp::IRREGEXP) return;
761
1154
762
1155
Object* code = re->DataAtUnchecked(JSRegExp::code_index(is_ascii));
763
if (!code->IsSmi() && SafeMap(code)->instance_type() == CODE_TYPE) {
1156
if (!code->IsSmi() &&
1157
HeapObject::cast(code)->map()->instance_type() == CODE_TYPE) {
764
1158
// Save a copy that can be reinstated if we need the code again.
765
1159
re->SetDataAtUnchecked(JSRegExp::saved_code_index(is_ascii),
766
1160
code,
796
1190
// If we did not use the code for kRegExpCodeThreshold mark sweep GCs
797
1191
// we flush the code.
798
1192
static void VisitRegExpAndFlushCode(Map* map, HeapObject* object) {
799
Heap* heap = map->heap();
1193
Heap* heap = map->GetHeap();
800
1194
MarkCompactCollector* collector = heap->mark_compact_collector();
801
1195
if (!collector->is_code_flushing_enabled()) {
802
1196
VisitJSRegExpFields(map, object);
813
1207
814
1208
static void VisitSharedFunctionInfoAndFlushCode(Map* map,
815
1209
HeapObject* object) {
816
MarkCompactCollector* collector = map->heap()->mark_compact_collector();
1210
MarkCompactCollector* collector = map->GetHeap()->mark_compact_collector();
817
1211
if (!collector->is_code_flushing_enabled()) {
818
1212
VisitSharedFunctionInfoGeneric(map, object);
819
1213
return;
824
1218
825
1219
static void VisitSharedFunctionInfoAndFlushCodeGeneric(
826
1220
Map* map, HeapObject* object, bool known_flush_code_candidate) {
827
Heap* heap = map->heap();
1221
Heap* heap = map->GetHeap();
828
1222
SharedFunctionInfo* shared = reinterpret_cast<SharedFunctionInfo*>(object);
829
1223
830
1224
if (shared->IsInobjectSlackTrackingInProgress()) shared->DetachInitialMap();
841
1235
842
1236
843
1237
static void VisitCodeEntry(Heap* heap, Address entry_address) {
844
Object* code = Code::GetObjectFromEntryAddress(entry_address);
845
Object* old_code = code;
846
VisitPointer(heap, &code);
847
if (code != old_code) {
848
Memory::Address_at(entry_address) =
849
reinterpret_cast<Code*>(code)->entry();
1238
Code* code = Code::cast(Code::GetObjectFromEntryAddress(entry_address));
1239
MarkBit mark = Marking::MarkBitFrom(code);
1240
heap->mark_compact_collector()->MarkObject(code, mark);
1241
heap->mark_compact_collector()->
1242
RecordCodeEntrySlot(entry_address, code);
1243
}
1244
1245
static void VisitGlobalContext(Map* map, HeapObject* object) {
1246
FixedBodyVisitor<StaticMarkingVisitor,
1247
Context::MarkCompactBodyDescriptor,
1248
void>::Visit(map, object);
1249
1250
MarkCompactCollector* collector = map->GetHeap()->mark_compact_collector();
1251
for (int idx = Context::FIRST_WEAK_SLOT;
1252
idx < Context::GLOBAL_CONTEXT_SLOTS;
1253
++idx) {
1254
Object** slot =
1255
HeapObject::RawField(object, FixedArray::OffsetOfElementAt(idx));
1256
collector->RecordSlot(slot, slot, *slot);
850
1257
}
851
1258
}
852
1259
853
854
1260
static void VisitJSFunctionAndFlushCode(Map* map, HeapObject* object) {
855
Heap* heap = map->heap();
1261
Heap* heap = map->GetHeap();
856
1262
MarkCompactCollector* collector = heap->mark_compact_collector();
857
1263
if (!collector->is_code_flushing_enabled()) {
858
1264
VisitJSFunction(map, object);
867
1273
}
868
1274
869
1275
if (!flush_code_candidate) {
870
collector->MarkObject(jsfunction->unchecked_shared()->unchecked_code());
1276
Code* code = jsfunction->shared()->code();
1277
MarkBit code_mark = Marking::MarkBitFrom(code);
1278
collector->MarkObject(code, code_mark);
871
1279
872
if (jsfunction->unchecked_code()->kind() == Code::OPTIMIZED_FUNCTION) {
873
collector->MarkInlinedFunctionsCode(jsfunction->unchecked_code());
1280
if (jsfunction->code()->kind() == Code::OPTIMIZED_FUNCTION) {
1281
collector->MarkInlinedFunctionsCode(jsfunction->code());
874
1282
}
875
1283
}
876
1284
894
1302
static inline void VisitJSFunctionFields(Map* map,
895
1303
JSFunction* object,
896
1304
bool flush_code_candidate) {
897
Heap* heap = map->heap();
898
MarkCompactCollector* collector = heap->mark_compact_collector();
1305
Heap* heap = map->GetHeap();
899
1306
900
1307
VisitPointers(heap,
901
SLOT_ADDR(object, JSFunction::kPropertiesOffset),
902
SLOT_ADDR(object, JSFunction::kCodeEntryOffset));
1308
HeapObject::RawField(object, JSFunction::kPropertiesOffset),
1309
HeapObject::RawField(object, JSFunction::kCodeEntryOffset));
903
1310
904
1311
if (!flush_code_candidate) {
905
1312
VisitCodeEntry(heap, object->address() + JSFunction::kCodeEntryOffset);
909
1316
// Visit shared function info to avoid double checking of it's
910
1317
// flushability.
911
1318
SharedFunctionInfo* shared_info = object->unchecked_shared();
912
if (!shared_info->IsMarked()) {
1319
MarkBit shared_info_mark = Marking::MarkBitFrom(shared_info);
1320
if (!shared_info_mark.Get()) {
913
1321
Map* shared_info_map = shared_info->map();
914
collector->SetMark(shared_info);
915
collector->MarkObject(shared_info_map);
1322
MarkBit shared_info_map_mark =
1323
Marking::MarkBitFrom(shared_info_map);
1324
heap->mark_compact_collector()->SetMark(shared_info, shared_info_mark);
1325
heap->mark_compact_collector()->MarkObject(shared_info_map,
1326
shared_info_map_mark);
916
1327
VisitSharedFunctionInfoAndFlushCodeGeneric(shared_info_map,
917
1328
shared_info,
918
1329
true);
919
1330
}
920
1331
}
921
1332
922
VisitPointers(heap,
923
SLOT_ADDR(object,
924
JSFunction::kCodeEntryOffset + kPointerSize),
925
SLOT_ADDR(object, JSFunction::kNonWeakFieldsEndOffset));
1333
VisitPointers(
1334
heap,
1335
HeapObject::RawField(object,
1336
JSFunction::kCodeEntryOffset + kPointerSize),
1337
HeapObject::RawField(object,
1338
JSFunction::kNonWeakFieldsEndOffset));
926
1339
927
1340
// Don't visit the next function list field as it is a weak reference.
1341
Object** next_function =
1342
HeapObject::RawField(object, JSFunction::kNextFunctionLinkOffset);
1343
heap->mark_compact_collector()->RecordSlot(
1344
next_function, next_function, *next_function);
928
1345
}
929
1346
930
1347
static inline void VisitJSRegExpFields(Map* map,
931
1348
HeapObject* object) {
932
1349
int last_property_offset =
933
1350
JSRegExp::kSize + kPointerSize * map->inobject_properties();
934
VisitPointers(map->heap(),
1351
VisitPointers(map->GetHeap(),
935
1352
SLOT_ADDR(object, JSRegExp::kPropertiesOffset),
936
1353
SLOT_ADDR(object, last_property_offset));
937
1354
}
1007
1424
Object* obj = *slot;
1008
1425
if (obj->IsSharedFunctionInfo()) {
1009
1426
SharedFunctionInfo* shared = reinterpret_cast<SharedFunctionInfo*>(obj);
1010
collector_->MarkObject(shared->unchecked_code());
1011
collector_->MarkObject(shared);
1427
MarkBit shared_mark = Marking::MarkBitFrom(shared);
1428
MarkBit code_mark = Marking::MarkBitFrom(shared->code());
1429
collector_->MarkObject(shared->code(), code_mark);
1430
collector_->MarkObject(shared, shared_mark);
1012
1431
}
1013
1432
}
1014
1433
1022
1441
// of it's code and non-optimized version of all inlined functions.
1023
1442
// This is required to support bailing out from inlined code.
1024
1443
DeoptimizationInputData* data =
1025
reinterpret_cast<DeoptimizationInputData*>(
1026
code->unchecked_deoptimization_data());
1444
DeoptimizationInputData::cast(code->deoptimization_data());
1027
1445
1028
FixedArray* literals = data->UncheckedLiteralArray();
1446
FixedArray* literals = data->LiteralArray();
1029
1447
1030
1448
for (int i = 0, count = data->InlinedFunctionCount()->value();
1031
1449
i < count;
1032
1450
i++) {
1033
JSFunction* inlined = reinterpret_cast<JSFunction*>(literals->get(i));
1034
MarkObject(inlined->unchecked_shared()->unchecked_code());
1451
JSFunction* inlined = JSFunction::cast(literals->get(i));
1452
Code* inlined_code = inlined->shared()->code();
1453
MarkBit inlined_code_mark = Marking::MarkBitFrom(inlined_code);
1454
MarkObject(inlined_code, inlined_code_mark);
1035
1455
}
1036
1456
}
1037
1457
1045
1465
// actual optimized code object.
1046
1466
StackFrame* frame = it.frame();
1047
1467
Code* code = frame->unchecked_code();
1048
MarkObject(code);
1468
MarkBit code_mark = Marking::MarkBitFrom(code);
1469
MarkObject(code, code_mark);
1049
1470
if (frame->is_optimized()) {
1050
1471
MarkInlinedFunctionsCode(frame->LookupCode());
1051
1472
}
1056
1477
void MarkCompactCollector::PrepareForCodeFlushing() {
1057
1478
ASSERT(heap() == Isolate::Current()->heap());
1058
1479
1059
if (!FLAG_flush_code) {
1480
// TODO(1609) Currently incremental marker does not support code flushing.
1481
if (!FLAG_flush_code || was_marked_incrementally_) {
1060
1482
EnableCodeFlushing(false);
1061
1483
return;
1062
1484
}
1068
1490
return;
1069
1491
}
1070
1492
#endif
1493
1071
1494
EnableCodeFlushing(true);
1072
1495
1073
1496
// Ensure that empty descriptor array is marked. Method MarkDescriptorArray
1074
1497
// relies on it being marked before any other descriptor array.
1075
MarkObject(heap()->raw_unchecked_empty_descriptor_array());
1498
HeapObject* descriptor_array = heap()->empty_descriptor_array();
1499
MarkBit descriptor_array_mark = Marking::MarkBitFrom(descriptor_array);
1500
MarkObject(descriptor_array, descriptor_array_mark);
1076
1501
1077
1502
// Make sure we are not referencing the code from the stack.
1078
1503
ASSERT(this == heap()->mark_compact_collector());
1089
1514
heap()->isolate()->compilation_cache()->IterateFunctions(&visitor);
1090
1515
heap()->isolate()->handle_scope_implementer()->Iterate(&visitor);
1091
1516
1092
ProcessMarkingStack();
1517
ProcessMarkingDeque();
1093
1518
}
1094
1519
1095
1520
1113
1538
1114
1539
// Replace flat cons strings in place.
1115
1540
HeapObject* object = ShortCircuitConsString(p);
1116
if (object->IsMarked()) return;
1541
MarkBit mark_bit = Marking::MarkBitFrom(object);
1542
if (mark_bit.Get()) return;
1117
1543
1118
1544
Map* map = object->map();
1119
1545
// Mark the object.
1120
collector_->SetMark(object);
1546
collector_->SetMark(object, mark_bit);
1121
1547
1122
1548
// Mark the map pointer and body, and push them on the marking stack.
1123
collector_->MarkObject(map);
1549
MarkBit map_mark = Marking::MarkBitFrom(map);
1550
collector_->MarkObject(map, map_mark);
1124
1551
StaticMarkingVisitor::IterateBody(map, object);
1125
1552
1126
1553
// Mark all the objects reachable from the map and body. May leave
1127
1554
// overflowed objects in the heap.
1128
collector_->EmptyMarkingStack();
1555
collector_->EmptyMarkingDeque();
1129
1556
}
1130
1557
1131
1558
MarkCompactCollector* collector_;
1141
1568
virtual void VisitPointers(Object** start, Object** end) {
1142
1569
// Visit all HeapObject pointers in [start, end).
1143
1570
for (Object** p = start; p < end; p++) {
1144
if ((*p)->IsHeapObject() && !HeapObject::cast(*p)->IsMarked()) {
1571
Object* o = *p;
1572
if (o->IsHeapObject() &&
1573
!Marking::MarkBitFrom(HeapObject::cast(o)).Get()) {
1145
1574
// Check if the symbol being pruned is an external symbol. We need to
1146
1575
// delete the associated external data as this symbol is going away.
1147
1576
1148
1577
// Since no objects have yet been moved we can safely access the map of
1149
1578
// the object.
1150
if ((*p)->IsExternalString()) {
1579
if (o->IsExternalString()) {
1151
1580
heap_->FinalizeExternalString(String::cast(*p));
1152
1581
}
1153
// Set the entry to null_value (as deleted).
1154
*p = heap_->raw_unchecked_null_value();
1582
// Set the entry to the_hole_value (as deleted).
1583
*p = heap_->the_hole_value();
1155
1584
pointers_removed_++;
1156
1585
}
1157
1586
}
1172
1601
class MarkCompactWeakObjectRetainer : public WeakObjectRetainer {
1173
1602
public:
1174
1603
virtual Object* RetainAs(Object* object) {
1175
MapWord first_word = HeapObject::cast(object)->map_word();
1176
if (first_word.IsMarked()) {
1604
if (Marking::MarkBitFrom(HeapObject::cast(object)).Get()) {
1177
1605
return object;
1178
1606
} else {
1179
1607
return NULL;
1182
1610
};
1183
1611
1184
1612
1185
void MarkCompactCollector::MarkUnmarkedObject(HeapObject* object) {
1186
ASSERT(!object->IsMarked());
1613
void MarkCompactCollector::ProcessNewlyMarkedObject(HeapObject* object) {
1614
ASSERT(IsMarked(object));
1187
1615
ASSERT(HEAP->Contains(object));
1188
1616
if (object->IsMap()) {
1189
1617
Map* map = Map::cast(object);
1190
1618
if (FLAG_cleanup_code_caches_at_gc) {
1191
1619
map->ClearCodeCache(heap());
1192
1620
}
1193
SetMark(map);
1194
1621
1195
1622
// When map collection is enabled we have to mark through map's transitions
1196
1623
// in a special way to make transition links weak.
1197
1624
// Only maps for subclasses of JSReceiver can have transitions.
1198
1625
STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1199
if (FLAG_collect_maps && map->instance_type() >= FIRST_JS_RECEIVER_TYPE) {
1626
if (collect_maps_ && map->instance_type() >= FIRST_JS_RECEIVER_TYPE) {
1200
1627
MarkMapContents(map);
1201
1628
} else {
1202
marking_stack_.Push(map);
1629
marking_deque_.PushBlack(map);
1203
1630
}
1204
1631
} else {
1205
SetMark(object);
1206
marking_stack_.Push(object);
1632
marking_deque_.PushBlack(object);
1207
1633
}
1208
1634
}
1209
1635
1212
1638
// Mark prototype transitions array but don't push it into marking stack.
1213
1639
// This will make references from it weak. We will clean dead prototype
1214
1640
// transitions in ClearNonLiveTransitions.
1215
FixedArray* prototype_transitions = map->unchecked_prototype_transitions();
1216
if (!prototype_transitions->IsMarked()) SetMark(prototype_transitions);
1641
FixedArray* prototype_transitions = map->prototype_transitions();
1642
MarkBit mark = Marking::MarkBitFrom(prototype_transitions);
1643
if (!mark.Get()) {
1644
mark.Set();
1645
MemoryChunk::IncrementLiveBytes(prototype_transitions->address(),
1646
prototype_transitions->Size());
1647
}
1217
1648
1218
Object* raw_descriptor_array =
1219
*HeapObject::RawField(map,
1220
Map::kInstanceDescriptorsOrBitField3Offset);
1649
Object** raw_descriptor_array_slot =
1650
HeapObject::RawField(map, Map::kInstanceDescriptorsOrBitField3Offset);
1651
Object* raw_descriptor_array = *raw_descriptor_array_slot;
1221
1652
if (!raw_descriptor_array->IsSmi()) {
1222
1653
MarkDescriptorArray(
1223
1654
reinterpret_cast<DescriptorArray*>(raw_descriptor_array));
1231
1662
1232
1663
Object** end_slot = HeapObject::RawField(map, Map::kPointerFieldsEndOffset);
1233
1664
1234
StaticMarkingVisitor::VisitPointers(map->heap(), start_slot, end_slot);
1665
StaticMarkingVisitor::VisitPointers(map->GetHeap(), start_slot, end_slot);
1235
1666
}
1236
1667
1237
1668
1238
1669
void MarkCompactCollector::MarkDescriptorArray(
1239
1670
DescriptorArray* descriptors) {
1240
if (descriptors->IsMarked()) return;
1671
MarkBit descriptors_mark = Marking::MarkBitFrom(descriptors);
1672
if (descriptors_mark.Get()) return;
1241
1673
// Empty descriptor array is marked as a root before any maps are marked.
1242
ASSERT(descriptors != HEAP->raw_unchecked_empty_descriptor_array());
1243
SetMark(descriptors);
1674
ASSERT(descriptors != heap()->empty_descriptor_array());
1675
SetMark(descriptors, descriptors_mark);
1244
1676
1245
1677
FixedArray* contents = reinterpret_cast<FixedArray*>(
1246
1678
descriptors->get(DescriptorArray::kContentArrayIndex));
1247
1679
ASSERT(contents->IsHeapObject());
1248
ASSERT(!contents->IsMarked());
1680
ASSERT(!IsMarked(contents));
1249
1681
ASSERT(contents->IsFixedArray());
1250
1682
ASSERT(contents->length() >= 2);
1251
SetMark(contents);
1683
MarkBit contents_mark = Marking::MarkBitFrom(contents);
1684
SetMark(contents, contents_mark);
1252
1685
// Contents contains (value, details) pairs. If the details say that the type
1253
1686
// of descriptor is MAP_TRANSITION, CONSTANT_TRANSITION,
1254
1687
// EXTERNAL_ARRAY_TRANSITION or NULL_DESCRIPTOR, we don't mark the value as
1258
1691
// If the pair (value, details) at index i, i+1 is not
1259
1692
// a transition or null descriptor, mark the value.
1260
1693
PropertyDetails details(Smi::cast(contents->get(i + 1)));
1261
if (details.type() < FIRST_PHANTOM_PROPERTY_TYPE) {
1262
HeapObject* object = reinterpret_cast<HeapObject*>(contents->get(i));
1263
if (object->IsHeapObject() && !object->IsMarked()) {
1264
SetMark(object);
1265
marking_stack_.Push(object);
1694
1695
Object** slot = contents->data_start() + i;
1696
Object* value = *slot;
1697
if (!value->IsHeapObject()) continue;
1698
1699
RecordSlot(slot, slot, *slot);
1700
1701
if (details.IsProperty()) {
1702
HeapObject* object = HeapObject::cast(value);
1703
MarkBit mark = Marking::MarkBitFrom(HeapObject::cast(object));
1704
if (!mark.Get()) {
1705
SetMark(HeapObject::cast(object), mark);
1706
marking_deque_.PushBlack(object);
1707
}
1708
} else if (details.type() == ELEMENTS_TRANSITION && value->IsFixedArray()) {
1709
// For maps with multiple elements transitions, the transition maps are
1710
// stored in a FixedArray. Keep the fixed array alive but not the maps
1711
// that it refers to.
1712
HeapObject* object = HeapObject::cast(value);
1713
MarkBit mark = Marking::MarkBitFrom(HeapObject::cast(object));
1714
if (!mark.Get()) {
1715
SetMark(HeapObject::cast(object), mark);
1266
1716
}
1267
1717
}
1268
1718
}
1269
1719
// The DescriptorArray descriptors contains a pointer to its contents array,
1270
1720
// but the contents array is already marked.
1271
marking_stack_.Push(descriptors);
1721
marking_deque_.PushBlack(descriptors);
1272
1722
}
1273
1723
1274
1724
1275
1725
void MarkCompactCollector::CreateBackPointers() {
1276
1726
HeapObjectIterator iterator(heap()->map_space());
1277
for (HeapObject* next_object = iterator.next();
1278
next_object != NULL; next_object = iterator.next()) {
1279
if (next_object->IsMap()) { // Could also be ByteArray on free list.
1727
for (HeapObject* next_object = iterator.Next();
1728
next_object != NULL; next_object = iterator.Next()) {
1729
if (next_object->IsMap()) { // Could also be FreeSpace object on free list.
1280
1730
Map* map = Map::cast(next_object);
1281
STATIC_ASSERT(LAST_TYPE == LAST_CALLABLE_SPEC_OBJECT_TYPE);
1731
STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1282
1732
if (map->instance_type() >= FIRST_JS_RECEIVER_TYPE) {
1283
1733
map->CreateBackPointers();
1284
1734
} else {
1289
1739
}
1290
1740
1291
1741
1292
static int OverflowObjectSize(HeapObject* obj) {
1293
// Recover the normal map pointer, it might be marked as live and
1294
// overflowed.
1295
MapWord map_word = obj->map_word();
1296
map_word.ClearMark();
1297
map_word.ClearOverflow();
1298
return obj->SizeFromMap(map_word.ToMap());
1299
}
1300
1301
1302
class OverflowedObjectsScanner : public AllStatic {
1303
public:
1304
// Fill the marking stack with overflowed objects returned by the given
1305
// iterator. Stop when the marking stack is filled or the end of the space
1306
// is reached, whichever comes first.
1307
template<class T>
1308
static inline void ScanOverflowedObjects(MarkCompactCollector* collector,
1309
T* it) {
1310
// The caller should ensure that the marking stack is initially not full,
1311
// so that we don't waste effort pointlessly scanning for objects.
1312
ASSERT(!collector->marking_stack_.is_full());
1313
1314
for (HeapObject* object = it->next(); object != NULL; object = it->next()) {
1315
if (object->IsOverflowed()) {
1316
object->ClearOverflow();
1317
ASSERT(object->IsMarked());
1318
ASSERT(HEAP->Contains(object));
1319
collector->marking_stack_.Push(object);
1320
if (collector->marking_stack_.is_full()) return;
1321
}
1322
}
1323
}
1324
};
1742
// Fill the marking stack with overflowed objects returned by the given
1743
// iterator. Stop when the marking stack is filled or the end of the space
1744
// is reached, whichever comes first.
1745
template<class T>
1746
static void DiscoverGreyObjectsWithIterator(Heap* heap,
1747
MarkingDeque* marking_deque,
1748
T* it) {
1749
// The caller should ensure that the marking stack is initially not full,
1750
// so that we don't waste effort pointlessly scanning for objects.
1751
ASSERT(!marking_deque->IsFull());
1752
1753
Map* filler_map = heap->one_pointer_filler_map();
1754
for (HeapObject* object = it->Next();
1755
object != NULL;
1756
object = it->Next()) {
1757
MarkBit markbit = Marking::MarkBitFrom(object);
1758
if ((object->map() != filler_map) && Marking::IsGrey(markbit)) {
1759
Marking::GreyToBlack(markbit);
1760
MemoryChunk::IncrementLiveBytes(object->address(), object->Size());
1761
marking_deque->PushBlack(object);
1762
if (marking_deque->IsFull()) return;
1763
}
1764
}
1765
}
1766
1767
1768
static inline int MarkWordToObjectStarts(uint32_t mark_bits, int* starts);
1769
1770
1771
static void DiscoverGreyObjectsOnPage(MarkingDeque* marking_deque, Page* p) {
1772
ASSERT(strcmp(Marking::kWhiteBitPattern, "00") == 0);
1773
ASSERT(strcmp(Marking::kBlackBitPattern, "10") == 0);
1774
ASSERT(strcmp(Marking::kGreyBitPattern, "11") == 0);
1775
ASSERT(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
1776
1777
MarkBit::CellType* cells = p->markbits()->cells();
1778
1779
int last_cell_index =
1780
Bitmap::IndexToCell(
1781
Bitmap::CellAlignIndex(
1782
p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
1783
1784
int cell_index = Page::kFirstUsedCell;
1785
Address cell_base = p->ObjectAreaStart();
1786
1787
for (cell_index = Page::kFirstUsedCell;
1788
cell_index < last_cell_index;
1789
cell_index++, cell_base += 32 * kPointerSize) {
1790
ASSERT((unsigned)cell_index ==
1791
Bitmap::IndexToCell(
1792
Bitmap::CellAlignIndex(
1793
p->AddressToMarkbitIndex(cell_base))));
1794
1795
const MarkBit::CellType current_cell = cells[cell_index];
1796
if (current_cell == 0) continue;
1797
1798
const MarkBit::CellType next_cell = cells[cell_index + 1];
1799
MarkBit::CellType grey_objects = current_cell &
1800
((current_cell >> 1) | (next_cell << (Bitmap::kBitsPerCell - 1)));
1801
1802
int offset = 0;
1803
while (grey_objects != 0) {
1804
int trailing_zeros = CompilerIntrinsics::CountTrailingZeros(grey_objects);
1805
grey_objects >>= trailing_zeros;
1806
offset += trailing_zeros;
1807
MarkBit markbit(&cells[cell_index], 1 << offset, false);
1808
ASSERT(Marking::IsGrey(markbit));
1809
Marking::GreyToBlack(markbit);
1810
Address addr = cell_base + offset * kPointerSize;
1811
HeapObject* object = HeapObject::FromAddress(addr);
1812
MemoryChunk::IncrementLiveBytes(object->address(), object->Size());
1813
marking_deque->PushBlack(object);
1814
if (marking_deque->IsFull()) return;
1815
offset += 2;
1816
grey_objects >>= 2;
1817
}
1818
1819
grey_objects >>= (Bitmap::kBitsPerCell - 1);
1820
}
1821
}
1822
1823
1824
static void DiscoverGreyObjectsInSpace(Heap* heap,
1825
MarkingDeque* marking_deque,
1826
PagedSpace* space) {
1827
if (!space->was_swept_conservatively()) {
1828
HeapObjectIterator it(space);
1829
DiscoverGreyObjectsWithIterator(heap, marking_deque, &it);
1830
} else {
1831
PageIterator it(space);
1832
while (it.has_next()) {
1833
Page* p = it.next();
1834
DiscoverGreyObjectsOnPage(marking_deque, p);
1835
if (marking_deque->IsFull()) return;
1836
}
1837
}
1838
}
1325
1839
1326
1840
1327
1841
bool MarkCompactCollector::IsUnmarkedHeapObject(Object** p) {
1328
return (*p)->IsHeapObject() && !HeapObject::cast(*p)->IsMarked();
1842
Object* o = *p;
1843
if (!o->IsHeapObject()) return false;
1844
HeapObject* heap_object = HeapObject::cast(o);
1845
MarkBit mark = Marking::MarkBitFrom(heap_object);
1846
return !mark.Get();
1329
1847
}
1330
1848
1331
1849
1332
1850
void MarkCompactCollector::MarkSymbolTable() {
1333
SymbolTable* symbol_table = heap()->raw_unchecked_symbol_table();
1851
SymbolTable* symbol_table = heap()->symbol_table();
1334
1852
// Mark the symbol table itself.
1335
SetMark(symbol_table);
1853
MarkBit symbol_table_mark = Marking::MarkBitFrom(symbol_table);
1854
SetMark(symbol_table, symbol_table_mark);
1336
1855
// Explicitly mark the prefix.
1337
1856
MarkingVisitor marker(heap());
1338
1857
symbol_table->IteratePrefix(&marker);
1339
ProcessMarkingStack();
1858
ProcessMarkingDeque();
1340
1859
}
1341
1860
1342
1861
1349
1868
MarkSymbolTable();
1350
1869
1351
1870
// There may be overflowed objects in the heap. Visit them now.
1352
while (marking_stack_.overflowed()) {
1353
RefillMarkingStack();
1354
EmptyMarkingStack();
1871
while (marking_deque_.overflowed()) {
1872
RefillMarkingDeque();
1873
EmptyMarkingDeque();
1355
1874
}
1356
1875
}
1357
1876
1369
1888
bool group_marked = false;
1370
1889
for (size_t j = 0; j < entry->length_; j++) {
1371
1890
Object* object = *objects[j];
1372
if (object->IsHeapObject() && HeapObject::cast(object)->IsMarked()) {
1373
group_marked = true;
1374
break;
1891
if (object->IsHeapObject()) {
1892
HeapObject* heap_object = HeapObject::cast(object);
1893
MarkBit mark = Marking::MarkBitFrom(heap_object);
1894
if (mark.Get()) {
1895
group_marked = true;
1896
break;
1897
}
1375
1898
}
1376
1899
}
1377
1900
1380
1903
continue;
1381
1904
}
1382
1905
1383
// An object in the group is marked, so mark all heap objects in
1384
// the group.
1906
// An object in the group is marked, so mark as grey all white heap
1907
// objects in the group.
1385
1908
for (size_t j = 0; j < entry->length_; ++j) {
1386
if ((*objects[j])->IsHeapObject()) {
1387
MarkObject(HeapObject::cast(*objects[j]));
1909
Object* object = *objects[j];
1910
if (object->IsHeapObject()) {
1911
HeapObject* heap_object = HeapObject::cast(object);
1912
MarkBit mark = Marking::MarkBitFrom(heap_object);
1913
MarkObject(heap_object, mark);
1388
1914
}
1389
1915
}
1390
1916
1391
// Once the entire group has been marked, dispose it because it's
1392
// not needed anymore.
1917
// Once the entire group has been colored grey, set the object group
1918
// to NULL so it won't be processed again.
1393
1919
entry->Dispose();
1920
object_groups->at(i) = NULL;
1394
1921
}
1395
1922
object_groups->Rewind(last);
1396
1923
}
1405
1932
ImplicitRefGroup* entry = ref_groups->at(i);
1406
1933
ASSERT(entry != NULL);
1407
1934
1408
if (!(*entry->parent_)->IsMarked()) {
1935
if (!IsMarked(*entry->parent_)) {
1409
1936
(*ref_groups)[last++] = entry;
1410
1937
continue;
1411
1938
}
1414
1941
// A parent object is marked, so mark all child heap objects.
1415
1942
for (size_t j = 0; j < entry->length_; ++j) {
1416
1943
if ((*children[j])->IsHeapObject()) {
1417
MarkObject(HeapObject::cast(*children[j]));
1944
HeapObject* child = HeapObject::cast(*children[j]);
1945
MarkBit mark = Marking::MarkBitFrom(child);
1946
MarkObject(child, mark);
1418
1947
}
1419
1948
}
1420
1949
1430
1959
// Before: the marking stack contains zero or more heap object pointers.
1431
1960
// After: the marking stack is empty, and all objects reachable from the
1432
1961
// marking stack have been marked, or are overflowed in the heap.
1433
void MarkCompactCollector::EmptyMarkingStack() {
1434
while (!marking_stack_.is_empty()) {
1435
while (!marking_stack_.is_empty()) {
1436
HeapObject* object = marking_stack_.Pop();
1962
void MarkCompactCollector::EmptyMarkingDeque() {
1963
while (!marking_deque_.IsEmpty()) {
1964
while (!marking_deque_.IsEmpty()) {
1965
HeapObject* object = marking_deque_.Pop();
1437
1966
ASSERT(object->IsHeapObject());
1438
1967
ASSERT(heap()->Contains(object));
1439
ASSERT(object->IsMarked());
1440
ASSERT(!object->IsOverflowed());
1968
ASSERT(Marking::IsBlack(Marking::MarkBitFrom(object)));
1441
1969
1442
// Because the object is marked, we have to recover the original map
1443
// pointer and use it to mark the object's body.
1444
MapWord map_word = object->map_word();
1445
map_word.ClearMark();
1446
Map* map = map_word.ToMap();
1447
MarkObject(map);
1970
Map* map = object->map();
1971
MarkBit map_mark = Marking::MarkBitFrom(map);
1972
MarkObject(map, map_mark);
1448
1973
1449
1974
StaticMarkingVisitor::IterateBody(map, object);
1450
1975
}
1461
1986
// before sweeping completes. If sweeping completes, there are no remaining
1462
1987
// overflowed objects in the heap so the overflow flag on the markings stack
1463
1988
// is cleared.
1464
void MarkCompactCollector::RefillMarkingStack() {
1465
ASSERT(marking_stack_.overflowed());
1466
1467
SemiSpaceIterator new_it(heap()->new_space(), &OverflowObjectSize);
1468
OverflowedObjectsScanner::ScanOverflowedObjects(this, &new_it);
1469
if (marking_stack_.is_full()) return;
1470
1471
HeapObjectIterator old_pointer_it(heap()->old_pointer_space(),
1472
&OverflowObjectSize);
1473
OverflowedObjectsScanner::ScanOverflowedObjects(this, &old_pointer_it);
1474
if (marking_stack_.is_full()) return;
1475
1476
HeapObjectIterator old_data_it(heap()->old_data_space(), &OverflowObjectSize);
1477
OverflowedObjectsScanner::ScanOverflowedObjects(this, &old_data_it);
1478
if (marking_stack_.is_full()) return;
1479
1480
HeapObjectIterator code_it(heap()->code_space(), &OverflowObjectSize);
1481
OverflowedObjectsScanner::ScanOverflowedObjects(this, &code_it);
1482
if (marking_stack_.is_full()) return;
1483
1484
HeapObjectIterator map_it(heap()->map_space(), &OverflowObjectSize);
1485
OverflowedObjectsScanner::ScanOverflowedObjects(this, &map_it);
1486
if (marking_stack_.is_full()) return;
1487
1488
HeapObjectIterator cell_it(heap()->cell_space(), &OverflowObjectSize);
1489
OverflowedObjectsScanner::ScanOverflowedObjects(this, &cell_it);
1490
if (marking_stack_.is_full()) return;
1491
1492
LargeObjectIterator lo_it(heap()->lo_space(), &OverflowObjectSize);
1493
OverflowedObjectsScanner::ScanOverflowedObjects(this, &lo_it);
1494
if (marking_stack_.is_full()) return;
1495
1496
marking_stack_.clear_overflowed();
1989
void MarkCompactCollector::RefillMarkingDeque() {
1990
ASSERT(marking_deque_.overflowed());
1991
1992
SemiSpaceIterator new_it(heap()->new_space());
1993
DiscoverGreyObjectsWithIterator(heap(), &marking_deque_, &new_it);
1994
if (marking_deque_.IsFull()) return;
1995
1996
DiscoverGreyObjectsInSpace(heap(),
1997
&marking_deque_,
1998
heap()->old_pointer_space());
1999
if (marking_deque_.IsFull()) return;
2000
2001
DiscoverGreyObjectsInSpace(heap(),
2002
&marking_deque_,
2003
heap()->old_data_space());
2004
if (marking_deque_.IsFull()) return;
2005
2006
DiscoverGreyObjectsInSpace(heap(),
2007
&marking_deque_,
2008
heap()->code_space());
2009
if (marking_deque_.IsFull()) return;
2010
2011
DiscoverGreyObjectsInSpace(heap(),
2012
&marking_deque_,
2013
heap()->map_space());
2014
if (marking_deque_.IsFull()) return;
2015
2016
DiscoverGreyObjectsInSpace(heap(),
2017
&marking_deque_,
2018
heap()->cell_space());
2019
if (marking_deque_.IsFull()) return;
2020
2021
LargeObjectIterator lo_it(heap()->lo_space());
2022
DiscoverGreyObjectsWithIterator(heap(),
2023
&marking_deque_,
2024
&lo_it);
2025
if (marking_deque_.IsFull()) return;
2026
2027
marking_deque_.ClearOverflowed();
1497
2028
}
1498
2029
1499
2030
1501
2032
// stack. Before: the marking stack contains zero or more heap object
1502
2033
// pointers. After: the marking stack is empty and there are no overflowed
1503
2034
// objects in the heap.
1504
void MarkCompactCollector::ProcessMarkingStack() {
1505
EmptyMarkingStack();
1506
while (marking_stack_.overflowed()) {
1507
RefillMarkingStack();
1508
EmptyMarkingStack();
2035
void MarkCompactCollector::ProcessMarkingDeque() {
2036
EmptyMarkingDeque();
2037
while (marking_deque_.overflowed()) {
2038
RefillMarkingDeque();
2039
EmptyMarkingDeque();
1509
2040
}
1510
2041
}
1511
2042
1512
2043
1513
2044
void MarkCompactCollector::ProcessExternalMarking() {
1514
2045
bool work_to_do = true;
1515
ASSERT(marking_stack_.is_empty());
2046
ASSERT(marking_deque_.IsEmpty());
1516
2047
while (work_to_do) {
1517
2048
MarkObjectGroups();
1518
2049
MarkImplicitRefGroups();
1519
work_to_do = !marking_stack_.is_empty();
1520
ProcessMarkingStack();
2050
work_to_do = !marking_deque_.IsEmpty();
2051
ProcessMarkingDeque();
1521
2052
}
1522
2053
}
1523
2054
1529
2060
// with the C stack limit check.
1530
2061
PostponeInterruptsScope postpone(heap()->isolate());
1531
2062
2063
bool incremental_marking_overflowed = false;
2064
IncrementalMarking* incremental_marking = heap_->incremental_marking();
2065
if (was_marked_incrementally_) {
2066
// Finalize the incremental marking and check whether we had an overflow.
2067
// Both markers use grey color to mark overflowed objects so
2068
// non-incremental marker can deal with them as if overflow
2069
// occured during normal marking.
2070
// But incremental marker uses a separate marking deque
2071
// so we have to explicitly copy it's overflow state.
2072
incremental_marking->Finalize();
2073
incremental_marking_overflowed =
2074
incremental_marking->marking_deque()->overflowed();
2075
incremental_marking->marking_deque()->ClearOverflowed();
2076
} else {
2077
// Abort any pending incremental activities e.g. incremental sweeping.
2078
incremental_marking->Abort();
2079
}
2080
1532
2081
#ifdef DEBUG
1533
2082
ASSERT(state_ == PREPARE_GC);
1534
2083
state_ = MARK_LIVE_OBJECTS;
1535
2084
#endif
1536
// The to space contains live objects, the from space is used as a marking
1537
// stack.
1538
marking_stack_.Initialize(heap()->new_space()->FromSpaceLow(),
1539
heap()->new_space()->FromSpaceHigh());
2085
// The to space contains live objects, a page in from space is used as a
2086
// marking stack.
2087
Address marking_deque_start = heap()->new_space()->FromSpacePageLow();
2088
Address marking_deque_end = heap()->new_space()->FromSpacePageHigh();
2089
if (FLAG_force_marking_deque_overflows) {
2090
marking_deque_end = marking_deque_start + 64 * kPointerSize;
2091
}
2092
marking_deque_.Initialize(marking_deque_start,
2093
marking_deque_end);
2094
ASSERT(!marking_deque_.overflowed());
1540
2095
1541
ASSERT(!marking_stack_.overflowed());
2096
if (incremental_marking_overflowed) {
2097
// There are overflowed objects left in the heap after incremental marking.
2098
marking_deque_.SetOverflowed();
2099
}
1542
2100
1543
2101
PrepareForCodeFlushing();
1544
2102
2103
if (was_marked_incrementally_) {
2104
// There is no write barrier on cells so we have to scan them now at the end
2105
// of the incremental marking.
2106
{
2107
HeapObjectIterator cell_iterator(heap()->cell_space());
2108
HeapObject* cell;
2109
while ((cell = cell_iterator.Next()) != NULL) {
2110
ASSERT(cell->IsJSGlobalPropertyCell());
2111
if (IsMarked(cell)) {
2112
int offset = JSGlobalPropertyCell::kValueOffset;
2113
StaticMarkingVisitor::VisitPointer(
2114
heap(),
2115
reinterpret_cast<Object**>(cell->address() + offset));
2116
}
2117
}
2118
}
2119
}
2120
1545
2121
RootMarkingVisitor root_visitor(heap());
1546
2122
MarkRoots(&root_visitor);
1547
2123
1560
2136
&IsUnmarkedHeapObject);
1561
2137
// Then we mark the objects and process the transitive closure.
1562
2138
heap()->isolate()->global_handles()->IterateWeakRoots(&root_visitor);
1563
while (marking_stack_.overflowed()) {
1564
RefillMarkingStack();
1565
EmptyMarkingStack();
2139
while (marking_deque_.overflowed()) {
2140
RefillMarkingDeque();
2141
EmptyMarkingDeque();
1566
2142
}
1567
2143
1568
2144
// Repeat host application specific marking to mark unmarked objects
1569
2145
// reachable from the weak roots.
1570
2146
ProcessExternalMarking();
1571
2147
2148
AfterMarking();
2149
}
2150
2151
2152
void MarkCompactCollector::AfterMarking() {
1572
2153
// Object literal map caches reference symbols (cache keys) and maps
1573
2154
// (cache values). At this point still useful maps have already been
1574
2155
// marked. Mark the keys for the alive values before we process the
1578
2159
// Prune the symbol table removing all symbols only pointed to by the
1579
2160
// symbol table. Cannot use symbol_table() here because the symbol
1580
2161
// table is marked.
1581
SymbolTable* symbol_table = heap()->raw_unchecked_symbol_table();
2162
SymbolTable* symbol_table = heap()->symbol_table();
1582
2163
SymbolTableCleaner v(heap());
1583
2164
symbol_table->IterateElements(&v);
1584
2165
symbol_table->ElementsRemoved(v.PointersRemoved());
1607
2188
Object* raw_context = heap()->global_contexts_list_;
1608
2189
while (raw_context != heap()->undefined_value()) {
1609
2190
Context* context = reinterpret_cast<Context*>(raw_context);
1610
if (context->IsMarked()) {
2191
if (IsMarked(context)) {
1611
2192
HeapObject* raw_map_cache =
1612
2193
HeapObject::cast(context->get(Context::MAP_CACHE_INDEX));
1613
2194
// A map cache may be reachable from the stack. In this case
1614
2195
// it's already transitively marked and it's too late to clean
1615
2196
// up its parts.
1616
if (!raw_map_cache->IsMarked() &&
2197
if (!IsMarked(raw_map_cache) &&
1617
2198
raw_map_cache != heap()->undefined_value()) {
1618
2199
MapCache* map_cache = reinterpret_cast<MapCache*>(raw_map_cache);
1619
2200
int existing_elements = map_cache->NumberOfElements();
1623
2204
i += MapCache::kEntrySize) {
1624
2205
Object* raw_key = map_cache->get(i);
1625
2206
if (raw_key == heap()->undefined_value() ||
1626
raw_key == heap()->null_value()) continue;
2207
raw_key == heap()->the_hole_value()) continue;
1627
2208
STATIC_ASSERT(MapCache::kEntrySize == 2);
1628
2209
Object* raw_map = map_cache->get(i + 1);
1629
if (raw_map->IsHeapObject() &&
1630
HeapObject::cast(raw_map)->IsMarked()) {
2210
if (raw_map->IsHeapObject() && IsMarked(raw_map)) {
1631
2211
++used_elements;
1632
2212
} else {
1633
2213
// Delete useless entries with unmarked maps.
1634
2214
ASSERT(raw_map->IsMap());
1635
map_cache->set_null_unchecked(heap(), i);
1636
map_cache->set_null_unchecked(heap(), i + 1);
2215
map_cache->set_the_hole(i);
2216
map_cache->set_the_hole(i + 1);
1637
2217
}
1638
2218
}
1639
2219
if (used_elements == 0) {
1643
2223
// extra complexity during GC. We rely on subsequent cache
1644
2224
// usages (EnsureCapacity) to do this.
1645
2225
map_cache->ElementsRemoved(existing_elements - used_elements);
1646
MarkObject(map_cache);
2226
MarkBit map_cache_markbit = Marking::MarkBitFrom(map_cache);
2227
MarkObject(map_cache, map_cache_markbit);
1647
2228
}
1648
2229
}
1649
2230
}
1650
2231
// Move to next element in the list.
1651
2232
raw_context = context->get(Context::NEXT_CONTEXT_LINK);
1652
2233
}
1653
ProcessMarkingStack();
2234
ProcessMarkingDeque();
1654
2235
}
1655
2236
1656
2237
1657
#ifdef DEBUG
1658
void MarkCompactCollector::UpdateLiveObjectCount(HeapObject* obj) {
1659
live_bytes_ += obj->Size();
1660
if (heap()->new_space()->Contains(obj)) {
1661
live_young_objects_size_ += obj->Size();
1662
} else if (heap()->map_space()->Contains(obj)) {
1663
ASSERT(obj->IsMap());
1664
live_map_objects_size_ += obj->Size();
1665
} else if (heap()->cell_space()->Contains(obj)) {
1666
ASSERT(obj->IsJSGlobalPropertyCell());
1667
live_cell_objects_size_ += obj->Size();
1668
} else if (heap()->old_pointer_space()->Contains(obj)) {
1669
live_old_pointer_objects_size_ += obj->Size();
1670
} else if (heap()->old_data_space()->Contains(obj)) {
1671
live_old_data_objects_size_ += obj->Size();
1672
} else if (heap()->code_space()->Contains(obj)) {
1673
live_code_objects_size_ += obj->Size();
1674
} else if (heap()->lo_space()->Contains(obj)) {
1675
live_lo_objects_size_ += obj->Size();
1676
} else {
1677
UNREACHABLE();
2238
void MarkCompactCollector::ReattachInitialMaps() {
2239
HeapObjectIterator map_iterator(heap()->map_space());
2240
for (HeapObject* obj = map_iterator.Next();
2241
obj != NULL;
2242
obj = map_iterator.Next()) {
2243
if (obj->IsFreeSpace()) continue;
2244
Map* map = Map::cast(obj);
2245
2246
STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
2247
if (map->instance_type() < FIRST_JS_RECEIVER_TYPE) continue;
2248
2249
if (map->attached_to_shared_function_info()) {
2250
JSFunction::cast(map->constructor())->shared()->AttachInitialMap(map);
2251
}
1678
2252
}
1679
2253
}
1680
#endif // DEBUG
1681
1682
1683
void MarkCompactCollector::SweepLargeObjectSpace() {
1684
#ifdef DEBUG
1685
ASSERT(state_ == MARK_LIVE_OBJECTS);
1686
state_ =
1687
compacting_collection_ ? ENCODE_FORWARDING_ADDRESSES : SWEEP_SPACES;
1688
#endif
1689
// Deallocate unmarked objects and clear marked bits for marked objects.
1690
heap()->lo_space()->FreeUnmarkedObjects();
1691
}
1692
1693
1694
// Safe to use during marking phase only.
1695
bool MarkCompactCollector::SafeIsMap(HeapObject* object) {
1696
MapWord metamap = object->map_word();
1697
metamap.ClearMark();
1698
return metamap.ToMap()->instance_type() == MAP_TYPE;
1699
}
1700
2254
1701
2255
1702
2256
void MarkCompactCollector::ClearNonLiveTransitions() {
1703
HeapObjectIterator map_iterator(heap()->map_space(), &SizeOfMarkedObject);
2257
HeapObjectIterator map_iterator(heap()->map_space());
1704
2258
// Iterate over the map space, setting map transitions that go from
1705
2259
// a marked map to an unmarked map to null transitions. At the same time,
1706
2260
// set all the prototype fields of maps back to their original value,
1711
2265
// scan the descriptor arrays of those maps, not all maps.
1712
2266
// All of these actions are carried out only on maps of JSObjects
1713
2267
// and related subtypes.
1714
for (HeapObject* obj = map_iterator.next();
1715
obj != NULL; obj = map_iterator.next()) {
2268
for (HeapObject* obj = map_iterator.Next();
2269
obj != NULL; obj = map_iterator.Next()) {
1716
2270
Map* map = reinterpret_cast<Map*>(obj);
1717
if (!map->IsMarked() && map->IsByteArray()) continue;
2271
MarkBit map_mark = Marking::MarkBitFrom(map);
2272
if (map->IsFreeSpace()) continue;
1718
2273
1719
ASSERT(SafeIsMap(map));
2274
ASSERT(map->IsMap());
1720
2275
// Only JSObject and subtypes have map transitions and back pointers.
1721
STATIC_ASSERT(LAST_TYPE == LAST_CALLABLE_SPEC_OBJECT_TYPE);
1722
if (map->instance_type() < FIRST_JS_RECEIVER_TYPE) continue;
2276
STATIC_ASSERT(LAST_TYPE == LAST_JS_OBJECT_TYPE);
2277
if (map->instance_type() < FIRST_JS_OBJECT_TYPE) continue;
1723
2278
1724
if (map->IsMarked() && map->attached_to_shared_function_info()) {
2279
if (map_mark.Get() &&
2280
map->attached_to_shared_function_info()) {
1725
2281
// This map is used for inobject slack tracking and has been detached
1726
2282
// from SharedFunctionInfo during the mark phase.
1727
2283
// Since it survived the GC, reattach it now.
1730
2286
1731
2287
// Clear dead prototype transitions.
1732
2288
int number_of_transitions = map->NumberOfProtoTransitions();
1733
if (number_of_transitions > 0) {
1734
FixedArray* prototype_transitions =
1735
map->unchecked_prototype_transitions();
1736
int new_number_of_transitions = 0;
1737
const int header = Map::kProtoTransitionHeaderSize;
1738
const int proto_offset =
1739
header + Map::kProtoTransitionPrototypeOffset;
1740
const int map_offset = header + Map::kProtoTransitionMapOffset;
1741
const int step = Map::kProtoTransitionElementsPerEntry;
1742
for (int i = 0; i < number_of_transitions; i++) {
1743
Object* prototype = prototype_transitions->get(proto_offset + i * step);
1744
Object* cached_map = prototype_transitions->get(map_offset + i * step);
1745
if (HeapObject::cast(prototype)->IsMarked() &&
1746
HeapObject::cast(cached_map)->IsMarked()) {
1747
if (new_number_of_transitions != i) {
1748
prototype_transitions->set_unchecked(
1749
heap_,
1750
proto_offset + new_number_of_transitions * step,
1751
prototype,
1752
UPDATE_WRITE_BARRIER);
1753
prototype_transitions->set_unchecked(
1754
heap_,
1755
map_offset + new_number_of_transitions * step,
1756
cached_map,
1757
SKIP_WRITE_BARRIER);
1758
}
1759
new_number_of_transitions++;
2289
FixedArray* prototype_transitions = map->prototype_transitions();
2290
2291
int new_number_of_transitions = 0;
2292
const int header = Map::kProtoTransitionHeaderSize;
2293
const int proto_offset =
2294
header + Map::kProtoTransitionPrototypeOffset;
2295
const int map_offset = header + Map::kProtoTransitionMapOffset;
2296
const int step = Map::kProtoTransitionElementsPerEntry;
2297
for (int i = 0; i < number_of_transitions; i++) {
2298
Object* prototype = prototype_transitions->get(proto_offset + i * step);
2299
Object* cached_map = prototype_transitions->get(map_offset + i * step);
2300
if (IsMarked(prototype) && IsMarked(cached_map)) {
2301
if (new_number_of_transitions != i) {
2302
prototype_transitions->set_unchecked(
2303
heap_,
2304
proto_offset + new_number_of_transitions * step,
2305
prototype,
2306
UPDATE_WRITE_BARRIER);
2307
prototype_transitions->set_unchecked(
2308
heap_,
2309
map_offset + new_number_of_transitions * step,
2310
cached_map,
2311
SKIP_WRITE_BARRIER);
1760
2312
}
1761
2313
}
1762
2314
1763
2315
// Fill slots that became free with undefined value.
1764
Object* undefined = heap()->raw_unchecked_undefined_value();
2316
Object* undefined = heap()->undefined_value();
1765
2317
for (int i = new_number_of_transitions * step;
1766
2318
i < number_of_transitions * step;
1767
2319
i++) {
2320
// The undefined object is on a page that is never compacted and never
2321
// in new space so it is OK to skip the write barrier. Also it's a
2322
// root.
1768
2323
prototype_transitions->set_unchecked(heap_,
1769
2324
header + i,
1770
2325
undefined,
1771
2326
SKIP_WRITE_BARRIER);
2327
2328
Object** undefined_slot =
2329
prototype_transitions->data_start() + i;
2330
RecordSlot(undefined_slot, undefined_slot, undefined);
1772
2331
}
1773
2332
map->SetNumberOfProtoTransitions(new_number_of_transitions);
1774
2333
}
1775
2334
1776
2335
// Follow the chain of back pointers to find the prototype.
1777
2336
Map* current = map;
1778
while (SafeIsMap(current)) {
2337
while (current->IsMap()) {
1779
2338
current = reinterpret_cast<Map*>(current->prototype());
1780
2339
ASSERT(current->IsHeapObject());
1781
2340
}
1784
2343
// Follow back pointers, setting them to prototype,
1785
2344
// clearing map transitions when necessary.
1786
2345
current = map;
1787
bool on_dead_path = !current->IsMarked();
2346
bool on_dead_path = !map_mark.Get();
1788
2347
Object* next;
1789
while (SafeIsMap(current)) {
2348
while (current->IsMap()) {
1790
2349
next = current->prototype();
1791
2350
// There should never be a dead map above a live map.
1792
ASSERT(on_dead_path || current->IsMarked());
2351
MarkBit current_mark = Marking::MarkBitFrom(current);
2352
bool is_alive = current_mark.Get();
2353
ASSERT(on_dead_path || is_alive);
1793
2354
1794
2355
// A live map above a dead map indicates a dead transition.
1795
2356
// This test will always be false on the first iteration.
1796
if (on_dead_path && current->IsMarked()) {
2357
if (on_dead_path && is_alive) {
1797
2358
on_dead_path = false;
1798
2359
current->ClearNonLiveTransitions(heap(), real_prototype);
1799
2360
}
1800
2361
*HeapObject::RawField(current, Map::kPrototypeOffset) =
1801
2362
real_prototype;
2363
2364
if (is_alive) {
2365
Object** slot = HeapObject::RawField(current, Map::kPrototypeOffset);
2366
RecordSlot(slot, slot, real_prototype);
2367
}
1802
2368
current = reinterpret_cast<Map*>(next);
1803
2369
}
1804
2370
}
1808
2374
void MarkCompactCollector::ProcessWeakMaps() {
1809
2375
Object* weak_map_obj = encountered_weak_maps();
1810
2376
while (weak_map_obj != Smi::FromInt(0)) {
1811
ASSERT(HeapObject::cast(weak_map_obj)->IsMarked());
2377
ASSERT(MarkCompactCollector::IsMarked(HeapObject::cast(weak_map_obj)));
1812
2378
JSWeakMap* weak_map = reinterpret_cast<JSWeakMap*>(weak_map_obj);
1813
ObjectHashTable* table = weak_map->unchecked_table();
2379
ObjectHashTable* table = ObjectHashTable::cast(weak_map->table());
1814
2380
for (int i = 0; i < table->Capacity(); i++) {
1815
if (HeapObject::cast(table->KeyAt(i))->IsMarked()) {
2381
if (MarkCompactCollector::IsMarked(HeapObject::cast(table->KeyAt(i)))) {
1816
2382
Object* value = table->get(table->EntryToValueIndex(i));
1817
StaticMarkingVisitor::MarkObjectByPointer(heap(), &value);
2383
StaticMarkingVisitor::VisitPointer(heap(), &value);
1818
2384
table->set_unchecked(heap(),
1819
2385
table->EntryToValueIndex(i),
1820
2386
value,
1829
2395
void MarkCompactCollector::ClearWeakMaps() {
1830
2396
Object* weak_map_obj = encountered_weak_maps();
1831
2397
while (weak_map_obj != Smi::FromInt(0)) {
1832
ASSERT(HeapObject::cast(weak_map_obj)->IsMarked());
2398
ASSERT(MarkCompactCollector::IsMarked(HeapObject::cast(weak_map_obj)));
1833
2399
JSWeakMap* weak_map = reinterpret_cast<JSWeakMap*>(weak_map_obj);
1834
ObjectHashTable* table = weak_map->unchecked_table();
2400
ObjectHashTable* table = ObjectHashTable::cast(weak_map->table());
1835
2401
for (int i = 0; i < table->Capacity(); i++) {
1836
if (!HeapObject::cast(table->KeyAt(i))->IsMarked()) {
1837
table->RemoveEntry(i, heap());
2402
if (!MarkCompactCollector::IsMarked(HeapObject::cast(table->KeyAt(i)))) {
2403
table->RemoveEntry(i);
1838
2404
}
1839
2405
}
1840
2406
weak_map_obj = weak_map->next();
1843
2409
set_encountered_weak_maps(Smi::FromInt(0));
1844
2410
}
1845
2411
1846
// -------------------------------------------------------------------------
1847
// Phase 2: Encode forwarding addresses.
1848
// When compacting, forwarding addresses for objects in old space and map
1849
// space are encoded in their map pointer word (along with an encoding of
1850
// their map pointers).
1851
//
1852
// The excact encoding is described in the comments for class MapWord in
1853
// objects.h.
1854
//
1855
// An address range [start, end) can have both live and non-live objects.
1856
// Maximal non-live regions are marked so they can be skipped on subsequent
1857
// sweeps of the heap. A distinguished map-pointer encoding is used to mark
1858
// free regions of one-word size (in which case the next word is the start
1859
// of a live object). A second distinguished map-pointer encoding is used
1860
// to mark free regions larger than one word, and the size of the free
1861
// region (including the first word) is written to the second word of the
1862
// region.
1863
//
1864
// Any valid map page offset must lie in the object area of the page, so map
1865
// page offsets less than Page::kObjectStartOffset are invalid. We use a
1866
// pair of distinguished invalid map encodings (for single word and multiple
1867
// words) to indicate free regions in the page found during computation of
1868
// forwarding addresses and skipped over in subsequent sweeps.
1869
1870
1871
// Encode a free region, defined by the given start address and size, in the
1872
// first word or two of the region.
1873
void EncodeFreeRegion(Address free_start, int free_size) {
1874
ASSERT(free_size >= kIntSize);
1875
if (free_size == kIntSize) {
1876
Memory::uint32_at(free_start) = MarkCompactCollector::kSingleFreeEncoding;
1877
} else {
1878
ASSERT(free_size >= 2 * kIntSize);
1879
Memory::uint32_at(free_start) = MarkCompactCollector::kMultiFreeEncoding;
1880
Memory::int_at(free_start + kIntSize) = free_size;
1881
}
1882
1883
#ifdef DEBUG
1884
// Zap the body of the free region.
1885
if (FLAG_enable_slow_asserts) {
1886
for (int offset = 2 * kIntSize;
1887
offset < free_size;
1888
offset += kPointerSize) {
1889
Memory::Address_at(free_start + offset) = kZapValue;
1890
}
1891
}
1892
#endif
1893
}
1894
1895
1896
// Try to promote all objects in new space. Heap numbers and sequential
1897
// strings are promoted to the code space, large objects to large object space,
1898
// and all others to the old space.
1899
inline MaybeObject* MCAllocateFromNewSpace(Heap* heap,
1900
HeapObject* object,
1901
int object_size) {
1902
MaybeObject* forwarded;
1903
if (object_size > heap->MaxObjectSizeInPagedSpace()) {
1904
forwarded = Failure::Exception();
1905
} else {
1906
OldSpace* target_space = heap->TargetSpace(object);
1907
ASSERT(target_space == heap->old_pointer_space() ||
1908
target_space == heap->old_data_space());
1909
forwarded = target_space->MCAllocateRaw(object_size);
1910
}
1911
Object* result;
1912
if (!forwarded->ToObject(&result)) {
1913
result = heap->new_space()->MCAllocateRaw(object_size)->ToObjectUnchecked();
1914
}
1915
return result;
1916
}
1917
1918
1919
// Allocation functions for the paged spaces call the space's MCAllocateRaw.
1920
MUST_USE_RESULT inline MaybeObject* MCAllocateFromOldPointerSpace(
1921
Heap *heap,
1922
HeapObject* ignore,
1923
int object_size) {
1924
return heap->old_pointer_space()->MCAllocateRaw(object_size);
1925
}
1926
1927
1928
MUST_USE_RESULT inline MaybeObject* MCAllocateFromOldDataSpace(
1929
Heap* heap,
1930
HeapObject* ignore,
1931
int object_size) {
1932
return heap->old_data_space()->MCAllocateRaw(object_size);
1933
}
1934
1935
1936
MUST_USE_RESULT inline MaybeObject* MCAllocateFromCodeSpace(
1937
Heap* heap,
1938
HeapObject* ignore,
1939
int object_size) {
1940
return heap->code_space()->MCAllocateRaw(object_size);
1941
}
1942
1943
1944
MUST_USE_RESULT inline MaybeObject* MCAllocateFromMapSpace(
1945
Heap* heap,
1946
HeapObject* ignore,
1947
int object_size) {
1948
return heap->map_space()->MCAllocateRaw(object_size);
1949
}
1950
1951
1952
MUST_USE_RESULT inline MaybeObject* MCAllocateFromCellSpace(
1953
Heap* heap, HeapObject* ignore, int object_size) {
1954
return heap->cell_space()->MCAllocateRaw(object_size);
1955
}
1956
1957
1958
// The forwarding address is encoded at the same offset as the current
1959
// to-space object, but in from space.
1960
inline void EncodeForwardingAddressInNewSpace(Heap* heap,
1961
HeapObject* old_object,
1962
int object_size,
1963
Object* new_object,
1964
int* ignored) {
1965
int offset =
1966
heap->new_space()->ToSpaceOffsetForAddress(old_object->address());
1967
Memory::Address_at(heap->new_space()->FromSpaceLow() + offset) =
1968
HeapObject::cast(new_object)->address();
1969
}
1970
1971
1972
// The forwarding address is encoded in the map pointer of the object as an
1973
// offset (in terms of live bytes) from the address of the first live object
1974
// in the page.
1975
inline void EncodeForwardingAddressInPagedSpace(Heap* heap,
1976
HeapObject* old_object,
1977
int object_size,
1978
Object* new_object,
1979
int* offset) {
1980
// Record the forwarding address of the first live object if necessary.
1981
if (*offset == 0) {
1982
Page::FromAddress(old_object->address())->mc_first_forwarded =
1983
HeapObject::cast(new_object)->address();
1984
}
1985
1986
MapWord encoding =
1987
MapWord::EncodeAddress(old_object->map()->address(), *offset);
1988
old_object->set_map_word(encoding);
1989
*offset += object_size;
1990
ASSERT(*offset <= Page::kObjectAreaSize);
1991
}
1992
1993
1994
// Most non-live objects are ignored.
1995
inline void IgnoreNonLiveObject(HeapObject* object, Isolate* isolate) {}
1996
1997
1998
// Function template that, given a range of addresses (eg, a semispace or a
1999
// paged space page), iterates through the objects in the range to clear
2000
// mark bits and compute and encode forwarding addresses. As a side effect,
2001
// maximal free chunks are marked so that they can be skipped on subsequent
2002
// sweeps.
2003
//
2004
// The template parameters are an allocation function, a forwarding address
2005
// encoding function, and a function to process non-live objects.
2006
template<MarkCompactCollector::AllocationFunction Alloc,
2007
MarkCompactCollector::EncodingFunction Encode,
2008
MarkCompactCollector::ProcessNonLiveFunction ProcessNonLive>
2009
inline void EncodeForwardingAddressesInRange(MarkCompactCollector* collector,
2010
Address start,
2011
Address end,
2012
int* offset) {
2013
// The start address of the current free region while sweeping the space.
2014
// This address is set when a transition from live to non-live objects is
2015
// encountered. A value (an encoding of the 'next free region' pointer)
2016
// is written to memory at this address when a transition from non-live to
2017
// live objects is encountered.
2018
Address free_start = NULL;
2019
2020
// A flag giving the state of the previously swept object. Initially true
2021
// to ensure that free_start is initialized to a proper address before
2022
// trying to write to it.
2023
bool is_prev_alive = true;
2024
2025
int object_size; // Will be set on each iteration of the loop.
2026
for (Address current = start; current < end; current += object_size) {
2027
HeapObject* object = HeapObject::FromAddress(current);
2028
if (object->IsMarked()) {
2029
object->ClearMark();
2030
collector->tracer()->decrement_marked_count();
2031
object_size = object->Size();
2032
2033
Object* forwarded =
2034
Alloc(collector->heap(), object, object_size)->ToObjectUnchecked();
2035
Encode(collector->heap(), object, object_size, forwarded, offset);
2036
2037
#ifdef DEBUG
2038
if (FLAG_gc_verbose) {
2039
PrintF("forward %p -> %p.\n", object->address(),
2040
HeapObject::cast(forwarded)->address());
2041
}
2042
#endif
2043
if (!is_prev_alive) { // Transition from non-live to live.
2044
EncodeFreeRegion(free_start, static_cast<int>(current - free_start));
2045
is_prev_alive = true;
2046
}
2047
} else { // Non-live object.
2048
object_size = object->Size();
2049
ProcessNonLive(object, collector->heap()->isolate());
2050
if (is_prev_alive) { // Transition from live to non-live.
2051
free_start = current;
2052
is_prev_alive = false;
2053
}
2054
LiveObjectList::ProcessNonLive(object);
2055
}
2056
}
2057
2058
// If we ended on a free region, mark it.
2059
if (!is_prev_alive) {
2060
EncodeFreeRegion(free_start, static_cast<int>(end - free_start));
2061
}
2062
}
2063
2064
2065
// Functions to encode the forwarding pointers in each compactable space.
2066
void MarkCompactCollector::EncodeForwardingAddressesInNewSpace() {
2067
int ignored;
2068
EncodeForwardingAddressesInRange<MCAllocateFromNewSpace,
2069
EncodeForwardingAddressInNewSpace,
2070
IgnoreNonLiveObject>(
2071
this,
2072
heap()->new_space()->bottom(),
2073
heap()->new_space()->top(),
2074
&ignored);
2075
}
2076
2077
2078
template<MarkCompactCollector::AllocationFunction Alloc,
2079
MarkCompactCollector::ProcessNonLiveFunction ProcessNonLive>
2080
void MarkCompactCollector::EncodeForwardingAddressesInPagedSpace(
2081
PagedSpace* space) {
2082
PageIterator it(space, PageIterator::PAGES_IN_USE);
2083
while (it.has_next()) {
2084
Page* p = it.next();
2085
2086
// The offset of each live object in the page from the first live object
2087
// in the page.
2088
int offset = 0;
2089
EncodeForwardingAddressesInRange<Alloc,
2090
EncodeForwardingAddressInPagedSpace,
2091
ProcessNonLive>(
2092
this,
2093
p->ObjectAreaStart(),
2094
p->AllocationTop(),
2095
&offset);
2096
}
2097
}
2098
2099
2412
2100
2413
// We scavange new space simultaneously with sweeping. This is done in two
2101
2414
// passes.
2415
//
2102
2416
// The first pass migrates all alive objects from one semispace to another or
2103
// promotes them to old space. Forwading address is written directly into
2104
// first word of object without any encoding. If object is dead we are writing
2417
// promotes them to old space. Forwarding address is written directly into
2418
// first word of object without any encoding. If object is dead we write
2105
2419
// NULL as a forwarding address.
2106
// The second pass updates pointers to new space in all spaces. It is possible
2107
// to encounter pointers to dead objects during traversal of dirty regions we
2108
// should clear them to avoid encountering them during next dirty regions
2109
// iteration.
2110
static void MigrateObject(Heap* heap,
2111
Address dst,
2112
Address src,
2113
int size,
2114
bool to_old_space) {
2115
if (to_old_space) {
2116
heap->CopyBlockToOldSpaceAndUpdateRegionMarks(dst, src, size);
2420
//
2421
// The second pass updates pointers to new space in all spaces. It is possible
2422
// to encounter pointers to dead new space objects during traversal of pointers
2423
// to new space. We should clear them to avoid encountering them during next
2424
// pointer iteration. This is an issue if the store buffer overflows and we
2425
// have to scan the entire old space, including dead objects, looking for
2426
// pointers to new space.
2427
void MarkCompactCollector::MigrateObject(Address dst,
2428
Address src,
2429
int size,
2430
AllocationSpace dest) {
2431
HEAP_PROFILE(heap(), ObjectMoveEvent(src, dst));
2432
if (dest == OLD_POINTER_SPACE || dest == LO_SPACE) {
2433
Address src_slot = src;
2434
Address dst_slot = dst;
2435
ASSERT(IsAligned(size, kPointerSize));
2436
2437
for (int remaining = size / kPointerSize; remaining > 0; remaining--) {
2438
Object* value = Memory::Object_at(src_slot);
2439
2440
Memory::Object_at(dst_slot) = value;
2441
2442
if (heap_->InNewSpace(value)) {
2443
heap_->store_buffer()->Mark(dst_slot);
2444
} else if (value->IsHeapObject() && IsOnEvacuationCandidate(value)) {
2445
SlotsBuffer::AddTo(&slots_buffer_allocator_,
2446
&migration_slots_buffer_,
2447
reinterpret_cast<Object**>(dst_slot),
2448
SlotsBuffer::IGNORE_OVERFLOW);
2449
}
2450
2451
src_slot += kPointerSize;
2452
dst_slot += kPointerSize;
2453
}
2454
2455
if (compacting_ && HeapObject::FromAddress(dst)->IsJSFunction()) {
2456
Address code_entry_slot = dst + JSFunction::kCodeEntryOffset;
2457
Address code_entry = Memory::Address_at(code_entry_slot);
2458
2459
if (Page::FromAddress(code_entry)->IsEvacuationCandidate()) {
2460
SlotsBuffer::AddTo(&slots_buffer_allocator_,
2461
&migration_slots_buffer_,
2462
SlotsBuffer::CODE_ENTRY_SLOT,
2463
code_entry_slot,
2464
SlotsBuffer::IGNORE_OVERFLOW);
2465
}
2466
}
2467
} else if (dest == CODE_SPACE) {
2468
PROFILE(heap()->isolate(), CodeMoveEvent(src, dst));
2469
heap()->MoveBlock(dst, src, size);
2470
SlotsBuffer::AddTo(&slots_buffer_allocator_,
2471
&migration_slots_buffer_,
2472
SlotsBuffer::RELOCATED_CODE_OBJECT,
2473
dst,
2474
SlotsBuffer::IGNORE_OVERFLOW);
2475
Code::cast(HeapObject::FromAddress(dst))->Relocate(dst - src);
2117
2476
} else {
2118
heap->CopyBlock(dst, src, size);
2477
ASSERT(dest == OLD_DATA_SPACE || dest == NEW_SPACE);
2478
heap()->MoveBlock(dst, src, size);
2119
2479
}
2120
2121
2480
Memory::Address_at(src) = dst;
2122
2481
}
2123
2482
2124
2483
2125
class StaticPointersToNewGenUpdatingVisitor : public
2126
StaticNewSpaceVisitor<StaticPointersToNewGenUpdatingVisitor> {
2127
public:
2128
static inline void VisitPointer(Heap* heap, Object** p) {
2129
if (!(*p)->IsHeapObject()) return;
2130
2131
HeapObject* obj = HeapObject::cast(*p);
2132
Address old_addr = obj->address();
2133
2134
if (heap->new_space()->Contains(obj)) {
2135
ASSERT(heap->InFromSpace(*p));
2136
*p = HeapObject::FromAddress(Memory::Address_at(old_addr));
2137
}
2138
}
2139
};
2140
2141
2142
2484
// Visitor for updating pointers from live objects in old spaces to new space.
2143
2485
// It does not expect to encounter pointers to dead objects.
2144
class PointersToNewGenUpdatingVisitor: public ObjectVisitor {
2486
class PointersUpdatingVisitor: public ObjectVisitor {
2145
2487
public:
2146
explicit PointersToNewGenUpdatingVisitor(Heap* heap) : heap_(heap) { }
2488
explicit PointersUpdatingVisitor(Heap* heap) : heap_(heap) { }
2147
2489
2148
2490
void VisitPointer(Object** p) {
2149
StaticPointersToNewGenUpdatingVisitor::VisitPointer(heap_, p);
2491
UpdatePointer(p);
2150
2492
}
2151
2493
2152
2494
void VisitPointers(Object** start, Object** end) {
2153
for (Object** p = start; p < end; p++) {
2154
StaticPointersToNewGenUpdatingVisitor::VisitPointer(heap_, p);
2155
}
2495
for (Object** p = start; p < end; p++) UpdatePointer(p);
2496
}
2497
2498
void VisitEmbeddedPointer(RelocInfo* rinfo) {
2499
ASSERT(rinfo->rmode() == RelocInfo::EMBEDDED_OBJECT);
2500
Object* target = rinfo->target_object();
2501
VisitPointer(&target);
2502
rinfo->set_target_object(target);
2156
2503
}
2157
2504
2158
2505
void VisitCodeTarget(RelocInfo* rinfo) {
2172
2519
rinfo->set_call_address(Code::cast(target)->instruction_start());
2173
2520
}
2174
2521
2522
static inline void UpdateSlot(Heap* heap, Object** slot) {
2523
Object* obj = *slot;
2524
2525
if (!obj->IsHeapObject()) return;
2526
2527
HeapObject* heap_obj = HeapObject::cast(obj);
2528
2529
MapWord map_word = heap_obj->map_word();
2530
if (map_word.IsForwardingAddress()) {
2531
ASSERT(heap->InFromSpace(heap_obj) ||
2532
MarkCompactCollector::IsOnEvacuationCandidate(heap_obj));
2533
HeapObject* target = map_word.ToForwardingAddress();
2534
*slot = target;
2535
ASSERT(!heap->InFromSpace(target) &&
2536
!MarkCompactCollector::IsOnEvacuationCandidate(target));
2537
}
2538
}
2539
2175
2540
private:
2541
inline void UpdatePointer(Object** p) {
2542
UpdateSlot(heap_, p);
2543
}
2544
2176
2545
Heap* heap_;
2177
2546
};
2178
2547
2179
2548
2180
// Visitor for updating pointers from live objects in old spaces to new space.
2181
// It can encounter pointers to dead objects in new space when traversing map
2182
// space (see comment for MigrateObject).
2183
static void UpdatePointerToNewGen(HeapObject** p) {
2184
if (!(*p)->IsHeapObject()) return;
2549
static void UpdatePointer(HeapObject** p, HeapObject* object) {
2550
ASSERT(*p == object);
2185
2551
2186
Address old_addr = (*p)->address();
2187
ASSERT(HEAP->InFromSpace(*p));
2552
Address old_addr = object->address();
2188
2553
2189
2554
Address new_addr = Memory::Address_at(old_addr);
2190
2555
2191
if (new_addr == NULL) {
2192
// We encountered pointer to a dead object. Clear it so we will
2193
// not visit it again during next iteration of dirty regions.
2194
*p = NULL;
2556
// The new space sweep will overwrite the map word of dead objects
2557
// with NULL. In this case we do not need to transfer this entry to
2558
// the store buffer which we are rebuilding.
2559
if (new_addr != NULL) {
2560
*p = HeapObject::FromAddress(new_addr);
2195
2561
} else {
2196
*p = HeapObject::FromAddress(new_addr);
2197
}
2198
}
2199
2200
2201
static String* UpdateNewSpaceReferenceInExternalStringTableEntry(Heap* heap,
2202
Object** p) {
2203
Address old_addr = HeapObject::cast(*p)->address();
2204
Address new_addr = Memory::Address_at(old_addr);
2205
return String::cast(HeapObject::FromAddress(new_addr));
2206
}
2207
2208
2209
static bool TryPromoteObject(Heap* heap, HeapObject* object, int object_size) {
2562
// We have to zap this pointer, because the store buffer may overflow later,
2563
// and then we have to scan the entire heap and we don't want to find
2564
// spurious newspace pointers in the old space.
2565
*p = reinterpret_cast<HeapObject*>(Smi::FromInt(0));
2566
}
2567
}
2568
2569
2570
static String* UpdateReferenceInExternalStringTableEntry(Heap* heap,
2571
Object** p) {
2572
MapWord map_word = HeapObject::cast(*p)->map_word();
2573
2574
if (map_word.IsForwardingAddress()) {
2575
return String::cast(map_word.ToForwardingAddress());
2576
}
2577
2578
return String::cast(*p);
2579
}
2580
2581
2582
bool MarkCompactCollector::TryPromoteObject(HeapObject* object,
2583
int object_size) {
2210
2584
Object* result;
2211
2585
2212
if (object_size > heap->MaxObjectSizeInPagedSpace()) {
2586
if (object_size > heap()->MaxObjectSizeInPagedSpace()) {
2213
2587
MaybeObject* maybe_result =
2214
heap->lo_space()->AllocateRawFixedArray(object_size);
2588
heap()->lo_space()->AllocateRaw(object_size, NOT_EXECUTABLE);
2215
2589
if (maybe_result->ToObject(&result)) {
2216
2590
HeapObject* target = HeapObject::cast(result);
2217
MigrateObject(heap, target->address(), object->address(), object_size,
2218
true);
2219
heap->mark_compact_collector()->tracer()->
2591
MigrateObject(target->address(),
2592
object->address(),
2593
object_size,
2594
LO_SPACE);
2595
heap()->mark_compact_collector()->tracer()->
2220
2596
increment_promoted_objects_size(object_size);
2221
2597
return true;
2222
2598
}
2223
2599
} else {
2224
OldSpace* target_space = heap->TargetSpace(object);
2600
OldSpace* target_space = heap()->TargetSpace(object);
2225
2601
2226
ASSERT(target_space == heap->old_pointer_space() ||
2227
target_space == heap->old_data_space());
2602
ASSERT(target_space == heap()->old_pointer_space() ||
2603
target_space == heap()->old_data_space());
2228
2604
MaybeObject* maybe_result = target_space->AllocateRaw(object_size);
2229
2605
if (maybe_result->ToObject(&result)) {
2230
2606
HeapObject* target = HeapObject::cast(result);
2231
MigrateObject(heap,
2232
target->address(),
2607
MigrateObject(target->address(),
2233
2608
object->address(),
2234
2609
object_size,
2235
target_space == heap->old_pointer_space());
2236
heap->mark_compact_collector()->tracer()->
2610
target_space->identity());
2611
heap()->mark_compact_collector()->tracer()->
2237
2612
increment_promoted_objects_size(object_size);
2238
2613
return true;
2239
2614
}
2243
2618
}
2244
2619
2245
2620
2246
static void SweepNewSpace(Heap* heap, NewSpace* space) {
2247
heap->CheckNewSpaceExpansionCriteria();
2248
2249
Address from_bottom = space->bottom();
2250
Address from_top = space->top();
2621
void MarkCompactCollector::EvacuateNewSpace() {
2622
heap()->CheckNewSpaceExpansionCriteria();
2623
2624
NewSpace* new_space = heap()->new_space();
2625
2626
// Store allocation range before flipping semispaces.
2627
Address from_bottom = new_space->bottom();
2628
Address from_top = new_space->top();
2251
2629
2252
2630
// Flip the semispaces. After flipping, to space is empty, from space has
2253
2631
// live objects.
2254
space->Flip();
2255
space->ResetAllocationInfo();
2632
new_space->Flip();
2633
new_space->ResetAllocationInfo();
2256
2634
2257
int size = 0;
2258
2635
int survivors_size = 0;
2259
2636
2260
2637
// First pass: traverse all objects in inactive semispace, remove marks,
2261
// migrate live objects and write forwarding addresses.
2262
for (Address current = from_bottom; current < from_top; current += size) {
2263
HeapObject* object = HeapObject::FromAddress(current);
2264
2265
if (object->IsMarked()) {
2266
object->ClearMark();
2267
heap->mark_compact_collector()->tracer()->decrement_marked_count();
2268
2269
size = object->Size();
2638
// migrate live objects and write forwarding addresses. This stage puts
2639
// new entries in the store buffer and may cause some pages to be marked
2640
// scan-on-scavenge.
2641
SemiSpaceIterator from_it(from_bottom, from_top);
2642
for (HeapObject* object = from_it.Next();
2643
object != NULL;
2644
object = from_it.Next()) {
2645
MarkBit mark_bit = Marking::MarkBitFrom(object);
2646
if (mark_bit.Get()) {
2647
mark_bit.Clear();
2648
// Don't bother decrementing live bytes count. We'll discard the
2649
// entire page at the end.
2650
int size = object->Size();
2270
2651
survivors_size += size;
2271
2652
2272
2653
// Aggressively promote young survivors to the old space.
2273
if (TryPromoteObject(heap, object, size)) {
2654
if (TryPromoteObject(object, size)) {
2274
2655
continue;
2275
2656
}
2276
2657
2277
2658
// Promotion failed. Just migrate object to another semispace.
2278
// Allocation cannot fail at this point: semispaces are of equal size.
2279
Object* target = space->AllocateRaw(size)->ToObjectUnchecked();
2659
MaybeObject* allocation = new_space->AllocateRaw(size);
2660
if (allocation->IsFailure()) {
2661
if (!new_space->AddFreshPage()) {
2662
// Shouldn't happen. We are sweeping linearly, and to-space
2663
// has the same number of pages as from-space, so there is
2664
// always room.
2665
UNREACHABLE();
2666
}
2667
allocation = new_space->AllocateRaw(size);
2668
ASSERT(!allocation->IsFailure());
2669
}
2670
Object* target = allocation->ToObjectUnchecked();
2280
2671
2281
MigrateObject(heap,
2282
HeapObject::cast(target)->address(),
2283
current,
2672
MigrateObject(HeapObject::cast(target)->address(),
2673
object->address(),
2284
2674
size,
2285
false);
2675
NEW_SPACE);
2286
2676
} else {
2287
2677
// Process the dead object before we write a NULL into its header.
2288
2678
LiveObjectList::ProcessNonLive(object);
2289
2679
2290
size = object->Size();
2291
Memory::Address_at(current) = NULL;
2292
}
2680
// Mark dead objects in the new space with null in their map field.
2681
Memory::Address_at(object->address()) = NULL;
2682
}
2683
}
2684
2685
heap_->IncrementYoungSurvivorsCounter(survivors_size);
2686
new_space->set_age_mark(new_space->top());
2687
}
2688
2689
2690
void MarkCompactCollector::EvacuateLiveObjectsFromPage(Page* p) {
2691
AlwaysAllocateScope always_allocate;
2692
PagedSpace* space = static_cast<PagedSpace*>(p->owner());
2693
ASSERT(p->IsEvacuationCandidate() && !p->WasSwept());
2694
MarkBit::CellType* cells = p->markbits()->cells();
2695
p->MarkSweptPrecisely();
2696
2697
int last_cell_index =
2698
Bitmap::IndexToCell(
2699
Bitmap::CellAlignIndex(
2700
p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
2701
2702
int cell_index = Page::kFirstUsedCell;
2703
Address cell_base = p->ObjectAreaStart();
2704
int offsets[16];
2705
2706
for (cell_index = Page::kFirstUsedCell;
2707
cell_index < last_cell_index;
2708
cell_index++, cell_base += 32 * kPointerSize) {
2709
ASSERT((unsigned)cell_index ==
2710
Bitmap::IndexToCell(
2711
Bitmap::CellAlignIndex(
2712
p->AddressToMarkbitIndex(cell_base))));
2713
if (cells[cell_index] == 0) continue;
2714
2715
int live_objects = MarkWordToObjectStarts(cells[cell_index], offsets);
2716
for (int i = 0; i < live_objects; i++) {
2717
Address object_addr = cell_base + offsets[i] * kPointerSize;
2718
HeapObject* object = HeapObject::FromAddress(object_addr);
2719
ASSERT(Marking::IsBlack(Marking::MarkBitFrom(object)));
2720
2721
int size = object->Size();
2722
2723
MaybeObject* target = space->AllocateRaw(size);
2724
if (target->IsFailure()) {
2725
// OS refused to give us memory.
2726
V8::FatalProcessOutOfMemory("Evacuation");
2727
return;
2728
}
2729
2730
Object* target_object = target->ToObjectUnchecked();
2731
2732
MigrateObject(HeapObject::cast(target_object)->address(),
2733
object_addr,
2734
size,
2735
space->identity());
2736
ASSERT(object->map_word().IsForwardingAddress());
2737
}
2738
2739
// Clear marking bits for current cell.
2740
cells[cell_index] = 0;
2741
}
2742
p->ResetLiveBytes();
2743
}
2744
2745
2746
void MarkCompactCollector::EvacuatePages() {
2747
int npages = evacuation_candidates_.length();
2748
for (int i = 0; i < npages; i++) {
2749
Page* p = evacuation_candidates_[i];
2750
ASSERT(p->IsEvacuationCandidate() ||
2751
p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
2752
if (p->IsEvacuationCandidate()) {
2753
// During compaction we might have to request a new page.
2754
// Check that space still have room for that.
2755
if (static_cast<PagedSpace*>(p->owner())->CanExpand()) {
2756
EvacuateLiveObjectsFromPage(p);
2757
} else {
2758
// Without room for expansion evacuation is not guaranteed to succeed.
2759
// Pessimistically abandon unevacuated pages.
2760
for (int j = i; j < npages; j++) {
2761
Page* page = evacuation_candidates_[j];
2762
slots_buffer_allocator_.DeallocateChain(page->slots_buffer_address());
2763
page->ClearEvacuationCandidate();
2764
page->SetFlag(Page::RESCAN_ON_EVACUATION);
2765
}
2766
return;
2767
}
2768
}
2769
}
2770
}
2771
2772
2773
class EvacuationWeakObjectRetainer : public WeakObjectRetainer {
2774
public:
2775
virtual Object* RetainAs(Object* object) {
2776
if (object->IsHeapObject()) {
2777
HeapObject* heap_object = HeapObject::cast(object);
2778
MapWord map_word = heap_object->map_word();
2779
if (map_word.IsForwardingAddress()) {
2780
return map_word.ToForwardingAddress();
2781
}
2782
}
2783
return object;
2784
}
2785
};
2786
2787
2788
static inline void UpdateSlot(ObjectVisitor* v,
2789
SlotsBuffer::SlotType slot_type,
2790
Address addr) {
2791
switch (slot_type) {
2792
case SlotsBuffer::CODE_TARGET_SLOT: {
2793
RelocInfo rinfo(addr, RelocInfo::CODE_TARGET, 0, NULL);
2794
rinfo.Visit(v);
2795
break;
2796
}
2797
case SlotsBuffer::CODE_ENTRY_SLOT: {
2798
v->VisitCodeEntry(addr);
2799
break;
2800
}
2801
case SlotsBuffer::RELOCATED_CODE_OBJECT: {
2802
HeapObject* obj = HeapObject::FromAddress(addr);
2803
Code::cast(obj)->CodeIterateBody(v);
2804
break;
2805
}
2806
case SlotsBuffer::DEBUG_TARGET_SLOT: {
2807
RelocInfo rinfo(addr, RelocInfo::DEBUG_BREAK_SLOT, 0, NULL);
2808
if (rinfo.IsPatchedDebugBreakSlotSequence()) rinfo.Visit(v);
2809
break;
2810
}
2811
case SlotsBuffer::JS_RETURN_SLOT: {
2812
RelocInfo rinfo(addr, RelocInfo::JS_RETURN, 0, NULL);
2813
if (rinfo.IsPatchedReturnSequence()) rinfo.Visit(v);
2814
break;
2815
}
2816
case SlotsBuffer::EMBEDDED_OBJECT_SLOT: {
2817
RelocInfo rinfo(addr, RelocInfo::EMBEDDED_OBJECT, 0, NULL);
2818
rinfo.Visit(v);
2819
break;
2820
}
2821
default:
2822
UNREACHABLE();
2823
break;
2824
}
2825
}
2826
2827
2828
enum SweepingMode {
2829
SWEEP_ONLY,
2830
SWEEP_AND_VISIT_LIVE_OBJECTS
2831
};
2832
2833
2834
enum SkipListRebuildingMode {
2835
REBUILD_SKIP_LIST,
2836
IGNORE_SKIP_LIST
2837
};
2838
2839
2840
// Sweep a space precisely. After this has been done the space can
2841
// be iterated precisely, hitting only the live objects. Code space
2842
// is always swept precisely because we want to be able to iterate
2843
// over it. Map space is swept precisely, because it is not compacted.
2844
// Slots in live objects pointing into evacuation candidates are updated
2845
// if requested.
2846
template<SweepingMode sweeping_mode, SkipListRebuildingMode skip_list_mode>
2847
static void SweepPrecisely(PagedSpace* space,
2848
Page* p,
2849
ObjectVisitor* v) {
2850
ASSERT(!p->IsEvacuationCandidate() && !p->WasSwept());
2851
ASSERT_EQ(skip_list_mode == REBUILD_SKIP_LIST,
2852
space->identity() == CODE_SPACE);
2853
ASSERT((p->skip_list() == NULL) || (skip_list_mode == REBUILD_SKIP_LIST));
2854
2855
MarkBit::CellType* cells = p->markbits()->cells();
2856
p->MarkSweptPrecisely();
2857
2858
int last_cell_index =
2859
Bitmap::IndexToCell(
2860
Bitmap::CellAlignIndex(
2861
p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
2862
2863
int cell_index = Page::kFirstUsedCell;
2864
Address free_start = p->ObjectAreaStart();
2865
ASSERT(reinterpret_cast<intptr_t>(free_start) % (32 * kPointerSize) == 0);
2866
Address object_address = p->ObjectAreaStart();
2867
int offsets[16];
2868
2869
SkipList* skip_list = p->skip_list();
2870
int curr_region = -1;
2871
if ((skip_list_mode == REBUILD_SKIP_LIST) && skip_list) {
2872
skip_list->Clear();
2873
}
2874
2875
for (cell_index = Page::kFirstUsedCell;
2876
cell_index < last_cell_index;
2877
cell_index++, object_address += 32 * kPointerSize) {
2878
ASSERT((unsigned)cell_index ==
2879
Bitmap::IndexToCell(
2880
Bitmap::CellAlignIndex(
2881
p->AddressToMarkbitIndex(object_address))));
2882
int live_objects = MarkWordToObjectStarts(cells[cell_index], offsets);
2883
int live_index = 0;
2884
for ( ; live_objects != 0; live_objects--) {
2885
Address free_end = object_address + offsets[live_index++] * kPointerSize;
2886
if (free_end != free_start) {
2887
space->Free(free_start, static_cast<int>(free_end - free_start));
2888
}
2889
HeapObject* live_object = HeapObject::FromAddress(free_end);
2890
ASSERT(Marking::IsBlack(Marking::MarkBitFrom(live_object)));
2891
Map* map = live_object->map();
2892
int size = live_object->SizeFromMap(map);
2893
if (sweeping_mode == SWEEP_AND_VISIT_LIVE_OBJECTS) {
2894
live_object->IterateBody(map->instance_type(), size, v);
2895
}
2896
if ((skip_list_mode == REBUILD_SKIP_LIST) && skip_list != NULL) {
2897
int new_region_start =
2898
SkipList::RegionNumber(free_end);
2899
int new_region_end =
2900
SkipList::RegionNumber(free_end + size - kPointerSize);
2901
if (new_region_start != curr_region ||
2902
new_region_end != curr_region) {
2903
skip_list->AddObject(free_end, size);
2904
curr_region = new_region_end;
2905
}
2906
}
2907
free_start = free_end + size;
2908
}
2909
// Clear marking bits for current cell.
2910
cells[cell_index] = 0;
2911
}
2912
if (free_start != p->ObjectAreaEnd()) {
2913
space->Free(free_start, static_cast<int>(p->ObjectAreaEnd() - free_start));
2914
}
2915
p->ResetLiveBytes();
2916
}
2917
2918
2919
static bool SetMarkBitsUnderInvalidatedCode(Code* code, bool value) {
2920
Page* p = Page::FromAddress(code->address());
2921
2922
if (p->IsEvacuationCandidate() ||
2923
p->IsFlagSet(Page::RESCAN_ON_EVACUATION)) {
2924
return false;
2925
}
2926
2927
Address code_start = code->address();
2928
Address code_end = code_start + code->Size();
2929
2930
uint32_t start_index = MemoryChunk::FastAddressToMarkbitIndex(code_start);
2931
uint32_t end_index =
2932
MemoryChunk::FastAddressToMarkbitIndex(code_end - kPointerSize);
2933
2934
Bitmap* b = p->markbits();
2935
2936
MarkBit start_mark_bit = b->MarkBitFromIndex(start_index);
2937
MarkBit end_mark_bit = b->MarkBitFromIndex(end_index);
2938
2939
MarkBit::CellType* start_cell = start_mark_bit.cell();
2940
MarkBit::CellType* end_cell = end_mark_bit.cell();
2941
2942
if (value) {
2943
MarkBit::CellType start_mask = ~(start_mark_bit.mask() - 1);
2944
MarkBit::CellType end_mask = (end_mark_bit.mask() << 1) - 1;
2945
2946
if (start_cell == end_cell) {
2947
*start_cell |= start_mask & end_mask;
2948
} else {
2949
*start_cell |= start_mask;
2950
for (MarkBit::CellType* cell = start_cell + 1; cell < end_cell; cell++) {
2951
*cell = ~0;
2952
}
2953
*end_cell |= end_mask;
2954
}
2955
} else {
2956
for (MarkBit::CellType* cell = start_cell ; cell <= end_cell; cell++) {
2957
*cell = 0;
2958
}
2959
}
2960
2961
return true;
2962
}
2963
2964
2965
static bool IsOnInvalidatedCodeObject(Address addr) {
2966
// We did not record any slots in large objects thus
2967
// we can safely go to the page from the slot address.
2968
Page* p = Page::FromAddress(addr);
2969
2970
// First check owner's identity because old pointer and old data spaces
2971
// are swept lazily and might still have non-zero mark-bits on some
2972
// pages.
2973
if (p->owner()->identity() != CODE_SPACE) return false;
2974
2975
// In code space only bits on evacuation candidates (but we don't record
2976
// any slots on them) and under invalidated code objects are non-zero.
2977
MarkBit mark_bit =
2978
p->markbits()->MarkBitFromIndex(Page::FastAddressToMarkbitIndex(addr));
2979
2980
return mark_bit.Get();
2981
}
2982
2983
2984
void MarkCompactCollector::InvalidateCode(Code* code) {
2985
if (heap_->incremental_marking()->IsCompacting() &&
2986
!ShouldSkipEvacuationSlotRecording(code)) {
2987
ASSERT(compacting_);
2988
2989
// If the object is white than no slots were recorded on it yet.
2990
MarkBit mark_bit = Marking::MarkBitFrom(code);
2991
if (Marking::IsWhite(mark_bit)) return;
2992
2993
invalidated_code_.Add(code);
2994
}
2995
}
2996
2997
2998
bool MarkCompactCollector::MarkInvalidatedCode() {
2999
bool code_marked = false;
3000
3001
int length = invalidated_code_.length();
3002
for (int i = 0; i < length; i++) {
3003
Code* code = invalidated_code_[i];
3004
3005
if (SetMarkBitsUnderInvalidatedCode(code, true)) {
3006
code_marked = true;
3007
}
3008
}
3009
3010
return code_marked;
3011
}
3012
3013
3014
void MarkCompactCollector::RemoveDeadInvalidatedCode() {
3015
int length = invalidated_code_.length();
3016
for (int i = 0; i < length; i++) {
3017
if (!IsMarked(invalidated_code_[i])) invalidated_code_[i] = NULL;
3018
}
3019
}
3020
3021
3022
void MarkCompactCollector::ProcessInvalidatedCode(ObjectVisitor* visitor) {
3023
int length = invalidated_code_.length();
3024
for (int i = 0; i < length; i++) {
3025
Code* code = invalidated_code_[i];
3026
if (code != NULL) {
3027
code->Iterate(visitor);
3028
SetMarkBitsUnderInvalidatedCode(code, false);
3029
}
3030
}
3031
invalidated_code_.Rewind(0);
3032
}
3033
3034
3035
void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
3036
bool code_slots_filtering_required;
3037
{ GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_SWEEP_NEWSPACE);
3038
code_slots_filtering_required = MarkInvalidatedCode();
3039
3040
EvacuateNewSpace();
3041
}
3042
3043
3044
{ GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_EVACUATE_PAGES);
3045
EvacuatePages();
2293
3046
}
2294
3047
2295
3048
// Second pass: find pointers to new space and update them.
2296
PointersToNewGenUpdatingVisitor updating_visitor(heap);
2297
2298
// Update pointers in to space.
2299
Address current = space->bottom();
2300
while (current < space->top()) {
2301
HeapObject* object = HeapObject::FromAddress(current);
2302
current +=
2303
StaticPointersToNewGenUpdatingVisitor::IterateBody(object->map(),
2304
object);
2305
}
2306
2307
// Update roots.
2308
heap->IterateRoots(&updating_visitor, VISIT_ALL_IN_SWEEP_NEWSPACE);
2309
LiveObjectList::IterateElements(&updating_visitor);
2310
2311
// Update pointers in old spaces.
2312
heap->IterateDirtyRegions(heap->old_pointer_space(),
2313
&Heap::IteratePointersInDirtyRegion,
2314
&UpdatePointerToNewGen,
2315
heap->WATERMARK_SHOULD_BE_VALID);
2316
2317
heap->lo_space()->IterateDirtyRegions(&UpdatePointerToNewGen);
3049
PointersUpdatingVisitor updating_visitor(heap());
3050
3051
{ GCTracer::Scope gc_scope(tracer_,
3052
GCTracer::Scope::MC_UPDATE_NEW_TO_NEW_POINTERS);
3053
// Update pointers in to space.
3054
SemiSpaceIterator to_it(heap()->new_space()->bottom(),
3055
heap()->new_space()->top());
3056
for (HeapObject* object = to_it.Next();
3057
object != NULL;
3058
object = to_it.Next()) {
3059
Map* map = object->map();
3060
object->IterateBody(map->instance_type(),
3061
object->SizeFromMap(map),
3062
&updating_visitor);
3063
}
3064
}
3065
3066
{ GCTracer::Scope gc_scope(tracer_,
3067
GCTracer::Scope::MC_UPDATE_ROOT_TO_NEW_POINTERS);
3068
// Update roots.
3069
heap_->IterateRoots(&updating_visitor, VISIT_ALL_IN_SWEEP_NEWSPACE);
3070
LiveObjectList::IterateElements(&updating_visitor);
3071
}
3072
3073
{ GCTracer::Scope gc_scope(tracer_,
3074
GCTracer::Scope::MC_UPDATE_OLD_TO_NEW_POINTERS);
3075
StoreBufferRebuildScope scope(heap_,
3076
heap_->store_buffer(),
3077
&Heap::ScavengeStoreBufferCallback);
3078
heap_->store_buffer()->IteratePointersToNewSpace(&UpdatePointer);
3079
}
3080
3081
{ GCTracer::Scope gc_scope(tracer_,
3082
GCTracer::Scope::MC_UPDATE_POINTERS_TO_EVACUATED);
3083
SlotsBuffer::UpdateSlotsRecordedIn(heap_,
3084
migration_slots_buffer_,
3085
code_slots_filtering_required);
3086
if (FLAG_trace_fragmentation) {
3087
PrintF(" migration slots buffer: %d\n",
3088
SlotsBuffer::SizeOfChain(migration_slots_buffer_));
3089
}
3090
3091
if (compacting_ && was_marked_incrementally_) {
3092
// It's difficult to filter out slots recorded for large objects.
3093
LargeObjectIterator it(heap_->lo_space());
3094
for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
3095
// LargeObjectSpace is not swept yet thus we have to skip
3096
// dead objects explicitly.
3097
if (!IsMarked(obj)) continue;
3098
3099
Page* p = Page::FromAddress(obj->address());
3100
if (p->IsFlagSet(Page::RESCAN_ON_EVACUATION)) {
3101
obj->Iterate(&updating_visitor);
3102
p->ClearFlag(Page::RESCAN_ON_EVACUATION);
3103
}
3104
}
3105
}
3106
}
3107
3108
int npages = evacuation_candidates_.length();
3109
{ GCTracer::Scope gc_scope(
3110
tracer_, GCTracer::Scope::MC_UPDATE_POINTERS_BETWEEN_EVACUATED);
3111
for (int i = 0; i < npages; i++) {
3112
Page* p = evacuation_candidates_[i];
3113
ASSERT(p->IsEvacuationCandidate() ||
3114
p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
3115
3116
if (p->IsEvacuationCandidate()) {
3117
SlotsBuffer::UpdateSlotsRecordedIn(heap_,
3118
p->slots_buffer(),
3119
code_slots_filtering_required);
3120
if (FLAG_trace_fragmentation) {
3121
PrintF(" page %p slots buffer: %d\n",
3122
reinterpret_cast<void*>(p),
3123
SlotsBuffer::SizeOfChain(p->slots_buffer()));
3124
}
3125
3126
// Important: skip list should be cleared only after roots were updated
3127
// because root iteration traverses the stack and might have to find
3128
// code objects from non-updated pc pointing into evacuation candidate.
3129
SkipList* list = p->skip_list();
3130
if (list != NULL) list->Clear();
3131
} else {
3132
if (FLAG_gc_verbose) {
3133
PrintF("Sweeping 0x%" V8PRIxPTR " during evacuation.\n",
3134
reinterpret_cast<intptr_t>(p));
3135
}
3136
PagedSpace* space = static_cast<PagedSpace*>(p->owner());
3137
p->ClearFlag(MemoryChunk::RESCAN_ON_EVACUATION);
3138
3139
switch (space->identity()) {
3140
case OLD_DATA_SPACE:
3141
SweepConservatively(space, p);
3142
break;
3143
case OLD_POINTER_SPACE:
3144
SweepPrecisely<SWEEP_AND_VISIT_LIVE_OBJECTS, IGNORE_SKIP_LIST>(
3145
space, p, &updating_visitor);
3146
break;
3147
case CODE_SPACE:
3148
SweepPrecisely<SWEEP_AND_VISIT_LIVE_OBJECTS, REBUILD_SKIP_LIST>(
3149
space, p, &updating_visitor);
3150
break;
3151
default:
3152
UNREACHABLE();
3153
break;
3154
}
3155
}
3156
}
3157
}
3158
3159
GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_UPDATE_MISC_POINTERS);
2318
3160
2319
3161
// Update pointers from cells.
2320
HeapObjectIterator cell_iterator(heap->cell_space());
2321
for (HeapObject* cell = cell_iterator.next();
3162
HeapObjectIterator cell_iterator(heap_->cell_space());
3163
for (HeapObject* cell = cell_iterator.Next();
2322
3164
cell != NULL;
2323
cell = cell_iterator.next()) {
3165
cell = cell_iterator.Next()) {
2324
3166
if (cell->IsJSGlobalPropertyCell()) {
2325
3167
Address value_address =
2326
3168
reinterpret_cast<Address>(cell) +
2330
3172
}
2331
3173
2332
3174
// Update pointer from the global contexts list.
2333
updating_visitor.VisitPointer(heap->global_contexts_list_address());
3175
updating_visitor.VisitPointer(heap_->global_contexts_list_address());
3176
3177
heap_->symbol_table()->Iterate(&updating_visitor);
2334
3178
2335
3179
// Update pointers from external string table.
2336
heap->UpdateNewSpaceReferencesInExternalStringTable(
2337
&UpdateNewSpaceReferenceInExternalStringTableEntry);
2338
2339
// All pointers were updated. Update auxiliary allocation info.
2340
heap->IncrementYoungSurvivorsCounter(survivors_size);
2341
space->set_age_mark(space->top());
3180
heap_->UpdateReferencesInExternalStringTable(
3181
&UpdateReferenceInExternalStringTableEntry);
2342
3182
2343
3183
// Update JSFunction pointers from the runtime profiler.
2344
heap->isolate()->runtime_profiler()->UpdateSamplesAfterScavenge();
2345
}
2346
2347
2348
static void SweepSpace(Heap* heap, PagedSpace* space) {
2349
PageIterator it(space, PageIterator::PAGES_IN_USE);
2350
2351
// During sweeping of paged space we are trying to find longest sequences
2352
// of pages without live objects and free them (instead of putting them on
2353
// the free list).
2354
2355
// Page preceding current.
2356
Page* prev = Page::FromAddress(NULL);
2357
2358
// First empty page in a sequence.
2359
Page* first_empty_page = Page::FromAddress(NULL);
2360
2361
// Page preceding first empty page.
2362
Page* prec_first_empty_page = Page::FromAddress(NULL);
2363
2364
// If last used page of space ends with a sequence of dead objects
2365
// we can adjust allocation top instead of puting this free area into
2366
// the free list. Thus during sweeping we keep track of such areas
2367
// and defer their deallocation until the sweeping of the next page
2368
// is done: if one of the next pages contains live objects we have
2369
// to put such area into the free list.
2370
Address last_free_start = NULL;
2371
int last_free_size = 0;
3184
heap()->isolate()->runtime_profiler()->UpdateSamplesAfterCompact(
3185
&updating_visitor);
3186
3187
EvacuationWeakObjectRetainer evacuation_object_retainer;
3188
heap()->ProcessWeakReferences(&evacuation_object_retainer);
3189
3190
// Visit invalidated code (we ignored all slots on it) and clear mark-bits
3191
// under it.
3192
ProcessInvalidatedCode(&updating_visitor);
3193
3194
#ifdef DEBUG
3195
if (FLAG_verify_heap) {
3196
VerifyEvacuation(heap_);
3197
}
3198
#endif
3199
3200
slots_buffer_allocator_.DeallocateChain(&migration_slots_buffer_);
3201
ASSERT(migration_slots_buffer_ == NULL);
3202
for (int i = 0; i < npages; i++) {
3203
Page* p = evacuation_candidates_[i];
3204
if (!p->IsEvacuationCandidate()) continue;
3205
PagedSpace* space = static_cast<PagedSpace*>(p->owner());
3206
space->Free(p->ObjectAreaStart(), Page::kObjectAreaSize);
3207
p->set_scan_on_scavenge(false);
3208
slots_buffer_allocator_.DeallocateChain(p->slots_buffer_address());
3209
p->ClearEvacuationCandidate();
3210
}
3211
evacuation_candidates_.Rewind(0);
3212
compacting_ = false;
3213
}
3214
3215
3216
static const int kStartTableEntriesPerLine = 5;
3217
static const int kStartTableLines = 171;
3218
static const int kStartTableInvalidLine = 127;
3219
static const int kStartTableUnusedEntry = 126;
3220
3221
#define _ kStartTableUnusedEntry
3222
#define X kStartTableInvalidLine
3223
// Mark-bit to object start offset table.
3224
//
3225
// The line is indexed by the mark bits in a byte. The first number on
3226
// the line describes the number of live object starts for the line and the
3227
// other numbers on the line describe the offsets (in words) of the object
3228
// starts.
3229
//
3230
// Since objects are at least 2 words large we don't have entries for two
3231
// consecutive 1 bits. All entries after 170 have at least 2 consecutive bits.
3232
char kStartTable[kStartTableLines * kStartTableEntriesPerLine] = {
3233
0, _, _, _, _, // 0
3234
1, 0, _, _, _, // 1
3235
1, 1, _, _, _, // 2
3236
X, _, _, _, _, // 3
3237
1, 2, _, _, _, // 4
3238
2, 0, 2, _, _, // 5
3239
X, _, _, _, _, // 6
3240
X, _, _, _, _, // 7
3241
1, 3, _, _, _, // 8
3242
2, 0, 3, _, _, // 9
3243
2, 1, 3, _, _, // 10
3244
X, _, _, _, _, // 11
3245
X, _, _, _, _, // 12
3246
X, _, _, _, _, // 13
3247
X, _, _, _, _, // 14
3248
X, _, _, _, _, // 15
3249
1, 4, _, _, _, // 16
3250
2, 0, 4, _, _, // 17
3251
2, 1, 4, _, _, // 18
3252
X, _, _, _, _, // 19
3253
2, 2, 4, _, _, // 20
3254
3, 0, 2, 4, _, // 21
3255
X, _, _, _, _, // 22
3256
X, _, _, _, _, // 23
3257
X, _, _, _, _, // 24
3258
X, _, _, _, _, // 25
3259
X, _, _, _, _, // 26
3260
X, _, _, _, _, // 27
3261
X, _, _, _, _, // 28
3262
X, _, _, _, _, // 29
3263
X, _, _, _, _, // 30
3264
X, _, _, _, _, // 31
3265
1, 5, _, _, _, // 32
3266
2, 0, 5, _, _, // 33
3267
2, 1, 5, _, _, // 34
3268
X, _, _, _, _, // 35
3269
2, 2, 5, _, _, // 36
3270
3, 0, 2, 5, _, // 37
3271
X, _, _, _, _, // 38
3272
X, _, _, _, _, // 39
3273
2, 3, 5, _, _, // 40
3274
3, 0, 3, 5, _, // 41
3275
3, 1, 3, 5, _, // 42
3276
X, _, _, _, _, // 43
3277
X, _, _, _, _, // 44
3278
X, _, _, _, _, // 45
3279
X, _, _, _, _, // 46
3280
X, _, _, _, _, // 47
3281
X, _, _, _, _, // 48
3282
X, _, _, _, _, // 49
3283
X, _, _, _, _, // 50
3284
X, _, _, _, _, // 51
3285
X, _, _, _, _, // 52
3286
X, _, _, _, _, // 53
3287
X, _, _, _, _, // 54
3288
X, _, _, _, _, // 55
3289
X, _, _, _, _, // 56
3290
X, _, _, _, _, // 57
3291
X, _, _, _, _, // 58
3292
X, _, _, _, _, // 59
3293
X, _, _, _, _, // 60
3294
X, _, _, _, _, // 61
3295
X, _, _, _, _, // 62
3296
X, _, _, _, _, // 63
3297
1, 6, _, _, _, // 64
3298
2, 0, 6, _, _, // 65
3299
2, 1, 6, _, _, // 66
3300
X, _, _, _, _, // 67
3301
2, 2, 6, _, _, // 68
3302
3, 0, 2, 6, _, // 69
3303
X, _, _, _, _, // 70
3304
X, _, _, _, _, // 71
3305
2, 3, 6, _, _, // 72
3306
3, 0, 3, 6, _, // 73
3307
3, 1, 3, 6, _, // 74
3308
X, _, _, _, _, // 75
3309
X, _, _, _, _, // 76
3310
X, _, _, _, _, // 77
3311
X, _, _, _, _, // 78
3312
X, _, _, _, _, // 79
3313
2, 4, 6, _, _, // 80
3314
3, 0, 4, 6, _, // 81
3315
3, 1, 4, 6, _, // 82
3316
X, _, _, _, _, // 83
3317
3, 2, 4, 6, _, // 84
3318
4, 0, 2, 4, 6, // 85
3319
X, _, _, _, _, // 86
3320
X, _, _, _, _, // 87
3321
X, _, _, _, _, // 88
3322
X, _, _, _, _, // 89
3323
X, _, _, _, _, // 90
3324
X, _, _, _, _, // 91
3325
X, _, _, _, _, // 92
3326
X, _, _, _, _, // 93
3327
X, _, _, _, _, // 94
3328
X, _, _, _, _, // 95
3329
X, _, _, _, _, // 96
3330
X, _, _, _, _, // 97
3331
X, _, _, _, _, // 98
3332
X, _, _, _, _, // 99
3333
X, _, _, _, _, // 100
3334
X, _, _, _, _, // 101
3335
X, _, _, _, _, // 102
3336
X, _, _, _, _, // 103
3337
X, _, _, _, _, // 104
3338
X, _, _, _, _, // 105
3339
X, _, _, _, _, // 106
3340
X, _, _, _, _, // 107
3341
X, _, _, _, _, // 108
3342
X, _, _, _, _, // 109
3343
X, _, _, _, _, // 110
3344
X, _, _, _, _, // 111
3345
X, _, _, _, _, // 112
3346
X, _, _, _, _, // 113
3347
X, _, _, _, _, // 114
3348
X, _, _, _, _, // 115
3349
X, _, _, _, _, // 116
3350
X, _, _, _, _, // 117
3351
X, _, _, _, _, // 118
3352
X, _, _, _, _, // 119
3353
X, _, _, _, _, // 120
3354
X, _, _, _, _, // 121
3355
X, _, _, _, _, // 122
3356
X, _, _, _, _, // 123
3357
X, _, _, _, _, // 124
3358
X, _, _, _, _, // 125
3359
X, _, _, _, _, // 126
3360
X, _, _, _, _, // 127
3361
1, 7, _, _, _, // 128
3362
2, 0, 7, _, _, // 129
3363
2, 1, 7, _, _, // 130
3364
X, _, _, _, _, // 131
3365
2, 2, 7, _, _, // 132
3366
3, 0, 2, 7, _, // 133
3367
X, _, _, _, _, // 134
3368
X, _, _, _, _, // 135
3369
2, 3, 7, _, _, // 136
3370
3, 0, 3, 7, _, // 137
3371
3, 1, 3, 7, _, // 138
3372
X, _, _, _, _, // 139
3373
X, _, _, _, _, // 140
3374
X, _, _, _, _, // 141
3375
X, _, _, _, _, // 142
3376
X, _, _, _, _, // 143
3377
2, 4, 7, _, _, // 144
3378
3, 0, 4, 7, _, // 145
3379
3, 1, 4, 7, _, // 146
3380
X, _, _, _, _, // 147
3381
3, 2, 4, 7, _, // 148
3382
4, 0, 2, 4, 7, // 149
3383
X, _, _, _, _, // 150
3384
X, _, _, _, _, // 151
3385
X, _, _, _, _, // 152
3386
X, _, _, _, _, // 153
3387
X, _, _, _, _, // 154
3388
X, _, _, _, _, // 155
3389
X, _, _, _, _, // 156
3390
X, _, _, _, _, // 157
3391
X, _, _, _, _, // 158
3392
X, _, _, _, _, // 159
3393
2, 5, 7, _, _, // 160
3394
3, 0, 5, 7, _, // 161
3395
3, 1, 5, 7, _, // 162
3396
X, _, _, _, _, // 163
3397
3, 2, 5, 7, _, // 164
3398
4, 0, 2, 5, 7, // 165
3399
X, _, _, _, _, // 166
3400
X, _, _, _, _, // 167
3401
3, 3, 5, 7, _, // 168
3402
4, 0, 3, 5, 7, // 169
3403
4, 1, 3, 5, 7 // 170
3404
};
3405
#undef _
3406
#undef X
3407
3408
3409
// Takes a word of mark bits. Returns the number of objects that start in the
3410
// range. Puts the offsets of the words in the supplied array.
3411
static inline int MarkWordToObjectStarts(uint32_t mark_bits, int* starts) {
3412
int objects = 0;
3413
int offset = 0;
3414
3415
// No consecutive 1 bits.
3416
ASSERT((mark_bits & 0x180) != 0x180);
3417
ASSERT((mark_bits & 0x18000) != 0x18000);
3418
ASSERT((mark_bits & 0x1800000) != 0x1800000);
3419
3420
while (mark_bits != 0) {
3421
int byte = (mark_bits & 0xff);
3422
mark_bits >>= 8;
3423
if (byte != 0) {
3424
ASSERT(byte < kStartTableLines); // No consecutive 1 bits.
3425
char* table = kStartTable + byte * kStartTableEntriesPerLine;
3426
int objects_in_these_8_words = table[0];
3427
ASSERT(objects_in_these_8_words != kStartTableInvalidLine);
3428
ASSERT(objects_in_these_8_words < kStartTableEntriesPerLine);
3429
for (int i = 0; i < objects_in_these_8_words; i++) {
3430
starts[objects++] = offset + table[1 + i];
3431
}
3432
}
3433
offset += 8;
3434
}
3435
return objects;
3436
}
3437
3438
3439
static inline Address DigestFreeStart(Address approximate_free_start,
3440
uint32_t free_start_cell) {
3441
ASSERT(free_start_cell != 0);
3442
3443
// No consecutive 1 bits.
3444
ASSERT((free_start_cell & (free_start_cell << 1)) == 0);
3445
3446
int offsets[16];
3447
uint32_t cell = free_start_cell;
3448
int offset_of_last_live;
3449
if ((cell & 0x80000000u) != 0) {
3450
// This case would overflow below.
3451
offset_of_last_live = 31;
3452
} else {
3453
// Remove all but one bit, the most significant. This is an optimization
3454
// that may or may not be worthwhile.
3455
cell |= cell >> 16;
3456
cell |= cell >> 8;
3457
cell |= cell >> 4;
3458
cell |= cell >> 2;
3459
cell |= cell >> 1;
3460
cell = (cell + 1) >> 1;
3461
int live_objects = MarkWordToObjectStarts(cell, offsets);
3462
ASSERT(live_objects == 1);
3463
offset_of_last_live = offsets[live_objects - 1];
3464
}
3465
Address last_live_start =
3466
approximate_free_start + offset_of_last_live * kPointerSize;
3467
HeapObject* last_live = HeapObject::FromAddress(last_live_start);
3468
Address free_start = last_live_start + last_live->Size();
3469
return free_start;
3470
}
3471
3472
3473
static inline Address StartOfLiveObject(Address block_address, uint32_t cell) {
3474
ASSERT(cell != 0);
3475
3476
// No consecutive 1 bits.
3477
ASSERT((cell & (cell << 1)) == 0);
3478
3479
int offsets[16];
3480
if (cell == 0x80000000u) { // Avoid overflow below.
3481
return block_address + 31 * kPointerSize;
3482
}
3483
uint32_t first_set_bit = ((cell ^ (cell - 1)) + 1) >> 1;
3484
ASSERT((first_set_bit & cell) == first_set_bit);
3485
int live_objects = MarkWordToObjectStarts(first_set_bit, offsets);
3486
ASSERT(live_objects == 1);
3487
USE(live_objects);
3488
return block_address + offsets[0] * kPointerSize;
3489
}
3490
3491
3492
// Sweeps a space conservatively. After this has been done the larger free
3493
// spaces have been put on the free list and the smaller ones have been
3494
// ignored and left untouched. A free space is always either ignored or put
3495
// on the free list, never split up into two parts. This is important
3496
// because it means that any FreeSpace maps left actually describe a region of
3497
// memory that can be ignored when scanning. Dead objects other than free
3498
// spaces will not contain the free space map.
3499
intptr_t MarkCompactCollector::SweepConservatively(PagedSpace* space, Page* p) {
3500
ASSERT(!p->IsEvacuationCandidate() && !p->WasSwept());
3501
MarkBit::CellType* cells = p->markbits()->cells();
3502
p->MarkSweptConservatively();
3503
3504
int last_cell_index =
3505
Bitmap::IndexToCell(
3506
Bitmap::CellAlignIndex(
3507
p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
3508
3509
int cell_index = Page::kFirstUsedCell;
3510
intptr_t freed_bytes = 0;
3511
3512
// This is the start of the 32 word block that we are currently looking at.
3513
Address block_address = p->ObjectAreaStart();
3514
3515
// Skip over all the dead objects at the start of the page and mark them free.
3516
for (cell_index = Page::kFirstUsedCell;
3517
cell_index < last_cell_index;
3518
cell_index++, block_address += 32 * kPointerSize) {
3519
if (cells[cell_index] != 0) break;
3520
}
3521
size_t size = block_address - p->ObjectAreaStart();
3522
if (cell_index == last_cell_index) {
3523
freed_bytes += static_cast<int>(space->Free(p->ObjectAreaStart(),
3524
static_cast<int>(size)));
3525
ASSERT_EQ(0, p->LiveBytes());
3526
return freed_bytes;
3527
}
3528
// Grow the size of the start-of-page free space a little to get up to the
3529
// first live object.
3530
Address free_end = StartOfLiveObject(block_address, cells[cell_index]);
3531
// Free the first free space.
3532
size = free_end - p->ObjectAreaStart();
3533
freed_bytes += space->Free(p->ObjectAreaStart(),
3534
static_cast<int>(size));
3535
// The start of the current free area is represented in undigested form by
3536
// the address of the last 32-word section that contained a live object and
3537
// the marking bitmap for that cell, which describes where the live object
3538
// started. Unless we find a large free space in the bitmap we will not
3539
// digest this pair into a real address. We start the iteration here at the
3540
// first word in the marking bit map that indicates a live object.
3541
Address free_start = block_address;
3542
uint32_t free_start_cell = cells[cell_index];
3543
3544
for ( ;
3545
cell_index < last_cell_index;
3546
cell_index++, block_address += 32 * kPointerSize) {
3547
ASSERT((unsigned)cell_index ==
3548
Bitmap::IndexToCell(
3549
Bitmap::CellAlignIndex(
3550
p->AddressToMarkbitIndex(block_address))));
3551
uint32_t cell = cells[cell_index];
3552
if (cell != 0) {
3553
// We have a live object. Check approximately whether it is more than 32
3554
// words since the last live object.
3555
if (block_address - free_start > 32 * kPointerSize) {
3556
free_start = DigestFreeStart(free_start, free_start_cell);
3557
if (block_address - free_start > 32 * kPointerSize) {
3558
// Now that we know the exact start of the free space it still looks
3559
// like we have a large enough free space to be worth bothering with.
3560
// so now we need to find the start of the first live object at the
3561
// end of the free space.
3562
free_end = StartOfLiveObject(block_address, cell);
3563
freed_bytes += space->Free(free_start,
3564
static_cast<int>(free_end - free_start));
3565
}
3566
}
3567
// Update our undigested record of where the current free area started.
3568
free_start = block_address;
3569
free_start_cell = cell;
3570
// Clear marking bits for current cell.
3571
cells[cell_index] = 0;
3572
}
3573
}
3574
3575
// Handle the free space at the end of the page.
3576
if (block_address - free_start > 32 * kPointerSize) {
3577
free_start = DigestFreeStart(free_start, free_start_cell);
3578
freed_bytes += space->Free(free_start,
3579
static_cast<int>(block_address - free_start));
3580
}
3581
3582
p->ResetLiveBytes();
3583
return freed_bytes;
3584
}
3585
3586
3587
void MarkCompactCollector::SweepSpace(PagedSpace* space, SweeperType sweeper) {
3588
space->set_was_swept_conservatively(sweeper == CONSERVATIVE ||
3589
sweeper == LAZY_CONSERVATIVE);
3590
3591
space->ClearStats();
3592
3593
PageIterator it(space);
3594
3595
intptr_t freed_bytes = 0;
3596
int pages_swept = 0;
3597
intptr_t newspace_size = space->heap()->new_space()->Size();
3598
bool lazy_sweeping_active = false;
3599
bool unused_page_present = false;
3600
3601
intptr_t old_space_size = heap()->PromotedSpaceSize();
3602
intptr_t space_left =
3603
Min(heap()->OldGenPromotionLimit(old_space_size),
3604
heap()->OldGenAllocationLimit(old_space_size)) - old_space_size;
2372
3605
2373
3606
while (it.has_next()) {
2374
3607
Page* p = it.next();
2375
3608
2376
bool is_previous_alive = true;
2377
Address free_start = NULL;
2378
HeapObject* object;
2379
2380
for (Address current = p->ObjectAreaStart();
2381
current < p->AllocationTop();
2382
current += object->Size()) {
2383
object = HeapObject::FromAddress(current);
2384
if (object->IsMarked()) {
2385
object->ClearMark();
2386
heap->mark_compact_collector()->tracer()->decrement_marked_count();
2387
2388
if (!is_previous_alive) { // Transition from free to live.
2389
space->DeallocateBlock(free_start,
2390
static_cast<int>(current - free_start),
2391
true);
2392
is_previous_alive = true;
2393
}
2394
} else {
2395
heap->mark_compact_collector()->ReportDeleteIfNeeded(
2396
object, heap->isolate());
2397
if (is_previous_alive) { // Transition from live to free.
2398
free_start = current;
2399
is_previous_alive = false;
2400
}
2401
LiveObjectList::ProcessNonLive(object);
2402
}
2403
// The object is now unmarked for the call to Size() at the top of the
2404
// loop.
2405
}
2406
2407
bool page_is_empty = (p->ObjectAreaStart() == p->AllocationTop())
2408
|| (!is_previous_alive && free_start == p->ObjectAreaStart());
2409
2410
if (page_is_empty) {
2411
// This page is empty. Check whether we are in the middle of
2412
// sequence of empty pages and start one if not.
2413
if (!first_empty_page->is_valid()) {
2414
first_empty_page = p;
2415
prec_first_empty_page = prev;
2416
}
2417
2418
if (!is_previous_alive) {
2419
// There are dead objects on this page. Update space accounting stats
2420
// without putting anything into free list.
2421
int size_in_bytes = static_cast<int>(p->AllocationTop() - free_start);
2422
if (size_in_bytes > 0) {
2423
space->DeallocateBlock(free_start, size_in_bytes, false);
2424
}
2425
}
2426
} else {
2427
// This page is not empty. Sequence of empty pages ended on the previous
2428
// one.
2429
if (first_empty_page->is_valid()) {
2430
space->FreePages(prec_first_empty_page, prev);
2431
prec_first_empty_page = first_empty_page = Page::FromAddress(NULL);
2432
}
2433
2434
// If there is a free ending area on one of the previous pages we have
2435
// deallocate that area and put it on the free list.
2436
if (last_free_size > 0) {
2437
Page::FromAddress(last_free_start)->
2438
SetAllocationWatermark(last_free_start);
2439
space->DeallocateBlock(last_free_start, last_free_size, true);
2440
last_free_start = NULL;
2441
last_free_size = 0;
2442
}
2443
2444
// If the last region of this page was not live we remember it.
2445
if (!is_previous_alive) {
2446
ASSERT(last_free_size == 0);
2447
last_free_size = static_cast<int>(p->AllocationTop() - free_start);
2448
last_free_start = free_start;
2449
}
2450
}
2451
2452
prev = p;
2453
}
2454
2455
// We reached end of space. See if we need to adjust allocation top.
2456
Address new_allocation_top = NULL;
2457
2458
if (first_empty_page->is_valid()) {
2459
// Last used pages in space are empty. We can move allocation top backwards
2460
// to the beginning of first empty page.
2461
ASSERT(prev == space->AllocationTopPage());
2462
2463
new_allocation_top = first_empty_page->ObjectAreaStart();
2464
}
2465
2466
if (last_free_size > 0) {
2467
// There was a free ending area on the previous page.
2468
// Deallocate it without putting it into freelist and move allocation
2469
// top to the beginning of this free area.
2470
space->DeallocateBlock(last_free_start, last_free_size, false);
2471
new_allocation_top = last_free_start;
2472
}
2473
2474
if (new_allocation_top != NULL) {
2475
#ifdef DEBUG
2476
Page* new_allocation_top_page = Page::FromAllocationTop(new_allocation_top);
2477
if (!first_empty_page->is_valid()) {
2478
ASSERT(new_allocation_top_page == space->AllocationTopPage());
2479
} else if (last_free_size > 0) {
2480
ASSERT(new_allocation_top_page == prec_first_empty_page);
2481
} else {
2482
ASSERT(new_allocation_top_page == first_empty_page);
2483
}
2484
#endif
2485
2486
space->SetTop(new_allocation_top);
2487
}
2488
}
2489
2490
2491
void MarkCompactCollector::EncodeForwardingAddresses() {
2492
ASSERT(state_ == ENCODE_FORWARDING_ADDRESSES);
2493
// Objects in the active semispace of the young generation may be
2494
// relocated to the inactive semispace (if not promoted). Set the
2495
// relocation info to the beginning of the inactive semispace.
2496
heap()->new_space()->MCResetRelocationInfo();
2497
2498
// Compute the forwarding pointers in each space.
2499
EncodeForwardingAddressesInPagedSpace<MCAllocateFromOldPointerSpace,
2500
ReportDeleteIfNeeded>(
2501
heap()->old_pointer_space());
2502
2503
EncodeForwardingAddressesInPagedSpace<MCAllocateFromOldDataSpace,
2504
IgnoreNonLiveObject>(
2505
heap()->old_data_space());
2506
2507
EncodeForwardingAddressesInPagedSpace<MCAllocateFromCodeSpace,
2508
ReportDeleteIfNeeded>(
2509
heap()->code_space());
2510
2511
EncodeForwardingAddressesInPagedSpace<MCAllocateFromCellSpace,
2512
IgnoreNonLiveObject>(
2513
heap()->cell_space());
2514
2515
2516
// Compute new space next to last after the old and code spaces have been
2517
// compacted. Objects in new space can be promoted to old or code space.
2518
EncodeForwardingAddressesInNewSpace();
2519
2520
// Compute map space last because computing forwarding addresses
2521
// overwrites non-live objects. Objects in the other spaces rely on
2522
// non-live map pointers to get the sizes of non-live objects.
2523
EncodeForwardingAddressesInPagedSpace<MCAllocateFromMapSpace,
2524
IgnoreNonLiveObject>(
2525
heap()->map_space());
2526
2527
// Write relocation info to the top page, so we can use it later. This is
2528
// done after promoting objects from the new space so we get the correct
2529
// allocation top.
2530
heap()->old_pointer_space()->MCWriteRelocationInfoToPage();
2531
heap()->old_data_space()->MCWriteRelocationInfoToPage();
2532
heap()->code_space()->MCWriteRelocationInfoToPage();
2533
heap()->map_space()->MCWriteRelocationInfoToPage();
2534
heap()->cell_space()->MCWriteRelocationInfoToPage();
2535
}
2536
2537
2538
class MapIterator : public HeapObjectIterator {
2539
public:
2540
explicit MapIterator(Heap* heap)
2541
: HeapObjectIterator(heap->map_space(), &SizeCallback) { }
2542
2543
MapIterator(Heap* heap, Address start)
2544
: HeapObjectIterator(heap->map_space(), start, &SizeCallback) { }
2545
2546
private:
2547
static int SizeCallback(HeapObject* unused) {
2548
USE(unused);
2549
return Map::kSize;
2550
}
2551
};
2552
2553
2554
class MapCompact {
2555
public:
2556
explicit MapCompact(Heap* heap, int live_maps)
2557
: heap_(heap),
2558
live_maps_(live_maps),
2559
to_evacuate_start_(heap->map_space()->TopAfterCompaction(live_maps)),
2560
vacant_map_it_(heap),
2561
map_to_evacuate_it_(heap, to_evacuate_start_),
2562
first_map_to_evacuate_(
2563
reinterpret_cast<Map*>(HeapObject::FromAddress(to_evacuate_start_))) {
2564
}
2565
2566
void CompactMaps() {
2567
// As we know the number of maps to evacuate beforehand,
2568
// we stop then there is no more vacant maps.
2569
for (Map* next_vacant_map = NextVacantMap();
2570
next_vacant_map;
2571
next_vacant_map = NextVacantMap()) {
2572
EvacuateMap(next_vacant_map, NextMapToEvacuate());
2573
}
2574
2575
#ifdef DEBUG
2576
CheckNoMapsToEvacuate();
2577
#endif
2578
}
2579
2580
void UpdateMapPointersInRoots() {
2581
MapUpdatingVisitor map_updating_visitor;
2582
heap()->IterateRoots(&map_updating_visitor, VISIT_ONLY_STRONG);
2583
heap()->isolate()->global_handles()->IterateWeakRoots(
2584
&map_updating_visitor);
2585
LiveObjectList::IterateElements(&map_updating_visitor);
2586
}
2587
2588
void UpdateMapPointersInPagedSpace(PagedSpace* space) {
2589
ASSERT(space != heap()->map_space());
2590
2591
PageIterator it(space, PageIterator::PAGES_IN_USE);
2592
while (it.has_next()) {
2593
Page* p = it.next();
2594
UpdateMapPointersInRange(heap(),
2595
p->ObjectAreaStart(),
2596
p->AllocationTop());
2597
}
2598
}
2599
2600
void UpdateMapPointersInNewSpace() {
2601
NewSpace* space = heap()->new_space();
2602
UpdateMapPointersInRange(heap(), space->bottom(), space->top());
2603
}
2604
2605
void UpdateMapPointersInLargeObjectSpace() {
2606
LargeObjectIterator it(heap()->lo_space());
2607
for (HeapObject* obj = it.next(); obj != NULL; obj = it.next())
2608
UpdateMapPointersInObject(heap(), obj);
2609
}
2610
2611
void Finish() {
2612
heap()->map_space()->FinishCompaction(to_evacuate_start_, live_maps_);
2613
}
2614
2615
inline Heap* heap() const { return heap_; }
2616
2617
private:
2618
Heap* heap_;
2619
int live_maps_;
2620
Address to_evacuate_start_;
2621
MapIterator vacant_map_it_;
2622
MapIterator map_to_evacuate_it_;
2623
Map* first_map_to_evacuate_;
2624
2625
// Helper class for updating map pointers in HeapObjects.
2626
class MapUpdatingVisitor: public ObjectVisitor {
2627
public:
2628
MapUpdatingVisitor() {}
2629
2630
void VisitPointer(Object** p) {
2631
UpdateMapPointer(p);
2632
}
2633
2634
void VisitPointers(Object** start, Object** end) {
2635
for (Object** p = start; p < end; p++) UpdateMapPointer(p);
2636
}
2637
2638
private:
2639
void UpdateMapPointer(Object** p) {
2640
if (!(*p)->IsHeapObject()) return;
2641
HeapObject* old_map = reinterpret_cast<HeapObject*>(*p);
2642
2643
// Moved maps are tagged with overflowed map word. They are the only
2644
// objects those map word is overflowed as marking is already complete.
2645
MapWord map_word = old_map->map_word();
2646
if (!map_word.IsOverflowed()) return;
2647
2648
*p = GetForwardedMap(map_word);
2649
}
2650
};
2651
2652
static Map* NextMap(MapIterator* it, HeapObject* last, bool live) {
2653
while (true) {
2654
HeapObject* next = it->next();
2655
ASSERT(next != NULL);
2656
if (next == last)
2657
return NULL;
2658
ASSERT(!next->IsOverflowed());
2659
ASSERT(!next->IsMarked());
2660
ASSERT(next->IsMap() || FreeListNode::IsFreeListNode(next));
2661
if (next->IsMap() == live)
2662
return reinterpret_cast<Map*>(next);
2663
}
2664
}
2665
2666
Map* NextVacantMap() {
2667
Map* map = NextMap(&vacant_map_it_, first_map_to_evacuate_, false);
2668
ASSERT(map == NULL || FreeListNode::IsFreeListNode(map));
2669
return map;
2670
}
2671
2672
Map* NextMapToEvacuate() {
2673
Map* map = NextMap(&map_to_evacuate_it_, NULL, true);
2674
ASSERT(map != NULL);
2675
ASSERT(map->IsMap());
2676
return map;
2677
}
2678
2679
static void EvacuateMap(Map* vacant_map, Map* map_to_evacuate) {
2680
ASSERT(FreeListNode::IsFreeListNode(vacant_map));
2681
ASSERT(map_to_evacuate->IsMap());
2682
2683
ASSERT(Map::kSize % 4 == 0);
2684
2685
map_to_evacuate->heap()->CopyBlockToOldSpaceAndUpdateRegionMarks(
2686
vacant_map->address(), map_to_evacuate->address(), Map::kSize);
2687
2688
ASSERT(vacant_map->IsMap()); // Due to memcpy above.
2689
2690
MapWord forwarding_map_word = MapWord::FromMap(vacant_map);
2691
forwarding_map_word.SetOverflow();
2692
map_to_evacuate->set_map_word(forwarding_map_word);
2693
2694
ASSERT(map_to_evacuate->map_word().IsOverflowed());
2695
ASSERT(GetForwardedMap(map_to_evacuate->map_word()) == vacant_map);
2696
}
2697
2698
static Map* GetForwardedMap(MapWord map_word) {
2699
ASSERT(map_word.IsOverflowed());
2700
map_word.ClearOverflow();
2701
Map* new_map = map_word.ToMap();
2702
ASSERT_MAP_ALIGNED(new_map->address());
2703
return new_map;
2704
}
2705
2706
static int UpdateMapPointersInObject(Heap* heap, HeapObject* obj) {
2707
ASSERT(!obj->IsMarked());
2708
Map* map = obj->map();
2709
ASSERT(heap->map_space()->Contains(map));
2710
MapWord map_word = map->map_word();
2711
ASSERT(!map_word.IsMarked());
2712
if (map_word.IsOverflowed()) {
2713
Map* new_map = GetForwardedMap(map_word);
2714
ASSERT(heap->map_space()->Contains(new_map));
2715
obj->set_map(new_map);
2716
2717
#ifdef DEBUG
3609
// Clear sweeping flags indicating that marking bits are still intact.
3610
p->ClearSweptPrecisely();
3611
p->ClearSweptConservatively();
3612
3613
if (p->IsEvacuationCandidate()) {
3614
ASSERT(evacuation_candidates_.length() > 0);
3615
continue;
3616
}
3617
3618
if (p->IsFlagSet(Page::RESCAN_ON_EVACUATION)) {
3619
// Will be processed in EvacuateNewSpaceAndCandidates.
3620
continue;
3621
}
3622
3623
// One unused page is kept, all further are released before sweeping them.
3624
if (p->LiveBytes() == 0) {
3625
if (unused_page_present) {
3626
if (FLAG_gc_verbose) {
3627
PrintF("Sweeping 0x%" V8PRIxPTR " released page.\n",
3628
reinterpret_cast<intptr_t>(p));
3629
}
3630
space->ReleasePage(p);
3631
continue;
3632
}
3633
unused_page_present = true;
3634
}
3635
3636
if (lazy_sweeping_active) {
2718
3637
if (FLAG_gc_verbose) {
2719
PrintF("update %p : %p -> %p\n",
2720
obj->address(),
2721
reinterpret_cast<void*>(map),
2722
reinterpret_cast<void*>(new_map));
2723
}
2724
#endif
2725
}
2726
2727
int size = obj->SizeFromMap(map);
2728
MapUpdatingVisitor map_updating_visitor;
2729
obj->IterateBody(map->instance_type(), size, &map_updating_visitor);
2730
return size;
2731
}
2732
2733
static void UpdateMapPointersInRange(Heap* heap, Address start, Address end) {
2734
HeapObject* object;
2735
int size;
2736
for (Address current = start; current < end; current += size) {
2737
object = HeapObject::FromAddress(current);
2738
size = UpdateMapPointersInObject(heap, object);
2739
ASSERT(size > 0);
2740
}
2741
}
2742
2743
#ifdef DEBUG
2744
void CheckNoMapsToEvacuate() {
2745
if (!FLAG_enable_slow_asserts)
2746
return;
2747
2748
for (HeapObject* obj = map_to_evacuate_it_.next();
2749
obj != NULL; obj = map_to_evacuate_it_.next())
2750
ASSERT(FreeListNode::IsFreeListNode(obj));
2751
}
2752
#endif
2753
};
3638
PrintF("Sweeping 0x%" V8PRIxPTR " lazily postponed.\n",
3639
reinterpret_cast<intptr_t>(p));
3640
}
3641
continue;
3642
}
3643
3644
switch (sweeper) {
3645
case CONSERVATIVE: {
3646
if (FLAG_gc_verbose) {
3647
PrintF("Sweeping 0x%" V8PRIxPTR " conservatively.\n",
3648
reinterpret_cast<intptr_t>(p));
3649
}
3650
SweepConservatively(space, p);
3651
pages_swept++;
3652
break;
3653
}
3654
case LAZY_CONSERVATIVE: {
3655
if (FLAG_gc_verbose) {
3656
PrintF("Sweeping 0x%" V8PRIxPTR " conservatively as needed.\n",
3657
reinterpret_cast<intptr_t>(p));
3658
}
3659
freed_bytes += SweepConservatively(space, p);
3660
pages_swept++;
3661
if (space_left + freed_bytes > newspace_size) {
3662
space->SetPagesToSweep(p->next_page());
3663
lazy_sweeping_active = true;
3664
} else {
3665
if (FLAG_gc_verbose) {
3666
PrintF("Only %" V8PRIdPTR " bytes freed. Still sweeping.\n",
3667
freed_bytes);
3668
}
3669
}
3670
break;
3671
}
3672
case PRECISE: {
3673
if (FLAG_gc_verbose) {
3674
PrintF("Sweeping 0x%" V8PRIxPTR " precisely.\n",
3675
reinterpret_cast<intptr_t>(p));
3676
}
3677
if (space->identity() == CODE_SPACE) {
3678
SweepPrecisely<SWEEP_ONLY, REBUILD_SKIP_LIST>(space, p, NULL);
3679
} else {
3680
SweepPrecisely<SWEEP_ONLY, IGNORE_SKIP_LIST>(space, p, NULL);
3681
}
3682
pages_swept++;
3683
break;
3684
}
3685
default: {
3686
UNREACHABLE();
3687
}
3688
}
3689
}
3690
3691
if (FLAG_gc_verbose) {
3692
PrintF("SweepSpace: %s (%d pages swept)\n",
3693
AllocationSpaceName(space->identity()),
3694
pages_swept);
3695
}
3696
3697
// Give pages that are queued to be freed back to the OS.
3698
heap()->FreeQueuedChunks();
3699
}
2754
3700
2755
3701
2756
3702
void MarkCompactCollector::SweepSpaces() {
2757
3703
GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_SWEEP);
2758
2759
ASSERT(state_ == SWEEP_SPACES);
2760
ASSERT(!IsCompacting());
3704
#ifdef DEBUG
3705
state_ = SWEEP_SPACES;
3706
#endif
3707
SweeperType how_to_sweep =
3708
FLAG_lazy_sweeping ? LAZY_CONSERVATIVE : CONSERVATIVE;
3709
if (sweep_precisely_) how_to_sweep = PRECISE;
2761
3710
// Noncompacting collections simply sweep the spaces to clear the mark
2762
3711
// bits and free the nonlive blocks (for old and map spaces). We sweep
2763
3712
// the map space last because freeing non-live maps overwrites them and
2764
3713
// the other spaces rely on possibly non-live maps to get the sizes for
2765
3714
// non-live objects.
2766
SweepSpace(heap(), heap()->old_pointer_space());
2767
SweepSpace(heap(), heap()->old_data_space());
2768
SweepSpace(heap(), heap()->code_space());
2769
SweepSpace(heap(), heap()->cell_space());
2770
{ GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_SWEEP_NEWSPACE);
2771
SweepNewSpace(heap(), heap()->new_space());
2772
}
2773
SweepSpace(heap(), heap()->map_space());
2774
2775
heap()->IterateDirtyRegions(heap()->map_space(),
2776
&heap()->IteratePointersInDirtyMapsRegion,
2777
&UpdatePointerToNewGen,
2778
heap()->WATERMARK_SHOULD_BE_VALID);
2779
2780
intptr_t live_maps_size = heap()->map_space()->Size();
2781
int live_maps = static_cast<int>(live_maps_size / Map::kSize);
2782
ASSERT(live_map_objects_size_ == live_maps_size);
2783
2784
if (heap()->map_space()->NeedsCompaction(live_maps)) {
2785
MapCompact map_compact(heap(), live_maps);
2786
2787
map_compact.CompactMaps();
2788
map_compact.UpdateMapPointersInRoots();
2789
2790
PagedSpaces spaces;
2791
for (PagedSpace* space = spaces.next();
2792
space != NULL; space = spaces.next()) {
2793
if (space == heap()->map_space()) continue;
2794
map_compact.UpdateMapPointersInPagedSpace(space);
2795
}
2796
map_compact.UpdateMapPointersInNewSpace();
2797
map_compact.UpdateMapPointersInLargeObjectSpace();
2798
2799
map_compact.Finish();
2800
}
2801
}
2802
2803
2804
// Iterate the live objects in a range of addresses (eg, a page or a
2805
// semispace). The live regions of the range have been linked into a list.
2806
// The first live region is [first_live_start, first_live_end), and the last
2807
// address in the range is top. The callback function is used to get the
2808
// size of each live object.
2809
int MarkCompactCollector::IterateLiveObjectsInRange(
2810
Address start,
2811
Address end,
2812
LiveObjectCallback size_func) {
2813
int live_objects_size = 0;
2814
Address current = start;
2815
while (current < end) {
2816
uint32_t encoded_map = Memory::uint32_at(current);
2817
if (encoded_map == kSingleFreeEncoding) {
2818
current += kPointerSize;
2819
} else if (encoded_map == kMultiFreeEncoding) {
2820
current += Memory::int_at(current + kIntSize);
2821
} else {
2822
int size = (this->*size_func)(HeapObject::FromAddress(current));
2823
current += size;
2824
live_objects_size += size;
2825
}
2826
}
2827
return live_objects_size;
2828
}
2829
2830
2831
int MarkCompactCollector::IterateLiveObjects(
2832
NewSpace* space, LiveObjectCallback size_f) {
2833
ASSERT(MARK_LIVE_OBJECTS < state_ && state_ <= RELOCATE_OBJECTS);
2834
return IterateLiveObjectsInRange(space->bottom(), space->top(), size_f);
2835
}
2836
2837
2838
int MarkCompactCollector::IterateLiveObjects(
2839
PagedSpace* space, LiveObjectCallback size_f) {
2840
ASSERT(MARK_LIVE_OBJECTS < state_ && state_ <= RELOCATE_OBJECTS);
2841
int total = 0;
2842
PageIterator it(space, PageIterator::PAGES_IN_USE);
2843
while (it.has_next()) {
2844
Page* p = it.next();
2845
total += IterateLiveObjectsInRange(p->ObjectAreaStart(),
2846
p->AllocationTop(),
2847
size_f);
2848
}
2849
return total;
2850
}
2851
2852
2853
// -------------------------------------------------------------------------
2854
// Phase 3: Update pointers
2855
2856
// Helper class for updating pointers in HeapObjects.
2857
class UpdatingVisitor: public ObjectVisitor {
2858
public:
2859
explicit UpdatingVisitor(Heap* heap) : heap_(heap) {}
2860
2861
void VisitPointer(Object** p) {
2862
UpdatePointer(p);
2863
}
2864
2865
void VisitPointers(Object** start, Object** end) {
2866
// Mark all HeapObject pointers in [start, end)
2867
for (Object** p = start; p < end; p++) UpdatePointer(p);
2868
}
2869
2870
void VisitCodeTarget(RelocInfo* rinfo) {
2871
ASSERT(RelocInfo::IsCodeTarget(rinfo->rmode()));
2872
Object* target = Code::GetCodeFromTargetAddress(rinfo->target_address());
2873
VisitPointer(&target);
2874
rinfo->set_target_address(
2875
reinterpret_cast<Code*>(target)->instruction_start());
2876
}
2877
2878
void VisitDebugTarget(RelocInfo* rinfo) {
2879
ASSERT((RelocInfo::IsJSReturn(rinfo->rmode()) &&
2880
rinfo->IsPatchedReturnSequence()) ||
2881
(RelocInfo::IsDebugBreakSlot(rinfo->rmode()) &&
2882
rinfo->IsPatchedDebugBreakSlotSequence()));
2883
Object* target = Code::GetCodeFromTargetAddress(rinfo->call_address());
2884
VisitPointer(&target);
2885
rinfo->set_call_address(
2886
reinterpret_cast<Code*>(target)->instruction_start());
2887
}
2888
2889
inline Heap* heap() const { return heap_; }
2890
2891
private:
2892
void UpdatePointer(Object** p) {
2893
if (!(*p)->IsHeapObject()) return;
2894
2895
HeapObject* obj = HeapObject::cast(*p);
2896
Address old_addr = obj->address();
2897
Address new_addr;
2898
ASSERT(!heap()->InFromSpace(obj));
2899
2900
if (heap()->new_space()->Contains(obj)) {
2901
Address forwarding_pointer_addr =
2902
heap()->new_space()->FromSpaceLow() +
2903
heap()->new_space()->ToSpaceOffsetForAddress(old_addr);
2904
new_addr = Memory::Address_at(forwarding_pointer_addr);
2905
2906
#ifdef DEBUG
2907
ASSERT(heap()->old_pointer_space()->Contains(new_addr) ||
2908
heap()->old_data_space()->Contains(new_addr) ||
2909
heap()->new_space()->FromSpaceContains(new_addr) ||
2910
heap()->lo_space()->Contains(HeapObject::FromAddress(new_addr)));
2911
2912
if (heap()->new_space()->FromSpaceContains(new_addr)) {
2913
ASSERT(heap()->new_space()->FromSpaceOffsetForAddress(new_addr) <=
2914
heap()->new_space()->ToSpaceOffsetForAddress(old_addr));
2915
}
2916
#endif
2917
2918
} else if (heap()->lo_space()->Contains(obj)) {
2919
// Don't move objects in the large object space.
2920
return;
2921
2922
} else {
2923
#ifdef DEBUG
2924
PagedSpaces spaces;
2925
PagedSpace* original_space = spaces.next();
2926
while (original_space != NULL) {
2927
if (original_space->Contains(obj)) break;
2928
original_space = spaces.next();
2929
}
2930
ASSERT(original_space != NULL);
2931
#endif
2932
new_addr = MarkCompactCollector::GetForwardingAddressInOldSpace(obj);
2933
ASSERT(original_space->Contains(new_addr));
2934
ASSERT(original_space->MCSpaceOffsetForAddress(new_addr) <=
2935
original_space->MCSpaceOffsetForAddress(old_addr));
2936
}
2937
2938
*p = HeapObject::FromAddress(new_addr);
2939
2940
#ifdef DEBUG
2941
if (FLAG_gc_verbose) {
2942
PrintF("update %p : %p -> %p\n",
2943
reinterpret_cast<Address>(p), old_addr, new_addr);
2944
}
2945
#endif
2946
}
2947
2948
Heap* heap_;
2949
};
2950
2951
2952
void MarkCompactCollector::UpdatePointers() {
2953
#ifdef DEBUG
2954
ASSERT(state_ == ENCODE_FORWARDING_ADDRESSES);
2955
state_ = UPDATE_POINTERS;
2956
#endif
2957
UpdatingVisitor updating_visitor(heap());
2958
heap()->isolate()->runtime_profiler()->UpdateSamplesAfterCompact(
2959
&updating_visitor);
2960
heap()->IterateRoots(&updating_visitor, VISIT_ONLY_STRONG);
2961
heap()->isolate()->global_handles()->IterateWeakRoots(&updating_visitor);
2962
2963
// Update the pointer to the head of the weak list of global contexts.
2964
updating_visitor.VisitPointer(&heap()->global_contexts_list_);
2965
2966
LiveObjectList::IterateElements(&updating_visitor);
2967
2968
int live_maps_size = IterateLiveObjects(
2969
heap()->map_space(), &MarkCompactCollector::UpdatePointersInOldObject);
2970
int live_pointer_olds_size = IterateLiveObjects(
2971
heap()->old_pointer_space(),
2972
&MarkCompactCollector::UpdatePointersInOldObject);
2973
int live_data_olds_size = IterateLiveObjects(
2974
heap()->old_data_space(),
2975
&MarkCompactCollector::UpdatePointersInOldObject);
2976
int live_codes_size = IterateLiveObjects(
2977
heap()->code_space(), &MarkCompactCollector::UpdatePointersInOldObject);
2978
int live_cells_size = IterateLiveObjects(
2979
heap()->cell_space(), &MarkCompactCollector::UpdatePointersInOldObject);
2980
int live_news_size = IterateLiveObjects(
2981
heap()->new_space(), &MarkCompactCollector::UpdatePointersInNewObject);
2982
2983
// Large objects do not move, the map word can be updated directly.
2984
LargeObjectIterator it(heap()->lo_space());
2985
for (HeapObject* obj = it.next(); obj != NULL; obj = it.next()) {
2986
UpdatePointersInNewObject(obj);
2987
}
2988
2989
USE(live_maps_size);
2990
USE(live_pointer_olds_size);
2991
USE(live_data_olds_size);
2992
USE(live_codes_size);
2993
USE(live_cells_size);
2994
USE(live_news_size);
2995
ASSERT(live_maps_size == live_map_objects_size_);
2996
ASSERT(live_data_olds_size == live_old_data_objects_size_);
2997
ASSERT(live_pointer_olds_size == live_old_pointer_objects_size_);
2998
ASSERT(live_codes_size == live_code_objects_size_);
2999
ASSERT(live_cells_size == live_cell_objects_size_);
3000
ASSERT(live_news_size == live_young_objects_size_);
3001
}
3002
3003
3004
int MarkCompactCollector::UpdatePointersInNewObject(HeapObject* obj) {
3005
// Keep old map pointers
3006
Map* old_map = obj->map();
3007
ASSERT(old_map->IsHeapObject());
3008
3009
Address forwarded = GetForwardingAddressInOldSpace(old_map);
3010
3011
ASSERT(heap()->map_space()->Contains(old_map));
3012
ASSERT(heap()->map_space()->Contains(forwarded));
3013
#ifdef DEBUG
3014
if (FLAG_gc_verbose) {
3015
PrintF("update %p : %p -> %p\n", obj->address(), old_map->address(),
3016
forwarded);
3017
}
3018
#endif
3019
// Update the map pointer.
3020
obj->set_map(reinterpret_cast<Map*>(HeapObject::FromAddress(forwarded)));
3021
3022
// We have to compute the object size relying on the old map because
3023
// map objects are not relocated yet.
3024
int obj_size = obj->SizeFromMap(old_map);
3025
3026
// Update pointers in the object body.
3027
UpdatingVisitor updating_visitor(heap());
3028
obj->IterateBody(old_map->instance_type(), obj_size, &updating_visitor);
3029
return obj_size;
3030
}
3031
3032
3033
int MarkCompactCollector::UpdatePointersInOldObject(HeapObject* obj) {
3034
// Decode the map pointer.
3035
MapWord encoding = obj->map_word();
3036
Address map_addr = encoding.DecodeMapAddress(heap()->map_space());
3037
ASSERT(heap()->map_space()->Contains(HeapObject::FromAddress(map_addr)));
3038
3039
// At this point, the first word of map_addr is also encoded, cannot
3040
// cast it to Map* using Map::cast.
3041
Map* map = reinterpret_cast<Map*>(HeapObject::FromAddress(map_addr));
3042
int obj_size = obj->SizeFromMap(map);
3043
InstanceType type = map->instance_type();
3044
3045
// Update map pointer.
3046
Address new_map_addr = GetForwardingAddressInOldSpace(map);
3047
int offset = encoding.DecodeOffset();
3048
obj->set_map_word(MapWord::EncodeAddress(new_map_addr, offset));
3049
3050
#ifdef DEBUG
3051
if (FLAG_gc_verbose) {
3052
PrintF("update %p : %p -> %p\n", obj->address(),
3053
map_addr, new_map_addr);
3054
}
3055
#endif
3056
3057
// Update pointers in the object body.
3058
UpdatingVisitor updating_visitor(heap());
3059
obj->IterateBody(type, obj_size, &updating_visitor);
3060
return obj_size;
3061
}
3062
3063
3064
Address MarkCompactCollector::GetForwardingAddressInOldSpace(HeapObject* obj) {
3065
// Object should either in old or map space.
3066
MapWord encoding = obj->map_word();
3067
3068
// Offset to the first live object's forwarding address.
3069
int offset = encoding.DecodeOffset();
3070
Address obj_addr = obj->address();
3071
3072
// Find the first live object's forwarding address.
3073
Page* p = Page::FromAddress(obj_addr);
3074
Address first_forwarded = p->mc_first_forwarded;
3075
3076
// Page start address of forwarded address.
3077
Page* forwarded_page = Page::FromAddress(first_forwarded);
3078
int forwarded_offset = forwarded_page->Offset(first_forwarded);
3079
3080
// Find end of allocation in the page of first_forwarded.
3081
int mc_top_offset = forwarded_page->AllocationWatermarkOffset();
3082
3083
// Check if current object's forward pointer is in the same page
3084
// as the first live object's forwarding pointer
3085
if (forwarded_offset + offset < mc_top_offset) {
3086
// In the same page.
3087
return first_forwarded + offset;
3088
}
3089
3090
// Must be in the next page, NOTE: this may cross chunks.
3091
Page* next_page = forwarded_page->next_page();
3092
ASSERT(next_page->is_valid());
3093
3094
offset -= (mc_top_offset - forwarded_offset);
3095
offset += Page::kObjectStartOffset;
3096
3097
ASSERT_PAGE_OFFSET(offset);
3098
ASSERT(next_page->OffsetToAddress(offset) < next_page->AllocationTop());
3099
3100
return next_page->OffsetToAddress(offset);
3101
}
3102
3103
3104
// -------------------------------------------------------------------------
3105
// Phase 4: Relocate objects
3106
3107
void MarkCompactCollector::RelocateObjects() {
3108
#ifdef DEBUG
3109
ASSERT(state_ == UPDATE_POINTERS);
3110
state_ = RELOCATE_OBJECTS;
3111
#endif
3112
// Relocates objects, always relocate map objects first. Relocating
3113
// objects in other space relies on map objects to get object size.
3114
int live_maps_size = IterateLiveObjects(
3115
heap()->map_space(), &MarkCompactCollector::RelocateMapObject);
3116
int live_pointer_olds_size = IterateLiveObjects(
3117
heap()->old_pointer_space(),
3118
&MarkCompactCollector::RelocateOldPointerObject);
3119
int live_data_olds_size = IterateLiveObjects(
3120
heap()->old_data_space(), &MarkCompactCollector::RelocateOldDataObject);
3121
int live_codes_size = IterateLiveObjects(
3122
heap()->code_space(), &MarkCompactCollector::RelocateCodeObject);
3123
int live_cells_size = IterateLiveObjects(
3124
heap()->cell_space(), &MarkCompactCollector::RelocateCellObject);
3125
int live_news_size = IterateLiveObjects(
3126
heap()->new_space(), &MarkCompactCollector::RelocateNewObject);
3127
3128
USE(live_maps_size);
3129
USE(live_pointer_olds_size);
3130
USE(live_data_olds_size);
3131
USE(live_codes_size);
3132
USE(live_cells_size);
3133
USE(live_news_size);
3134
ASSERT(live_maps_size == live_map_objects_size_);
3135
ASSERT(live_data_olds_size == live_old_data_objects_size_);
3136
ASSERT(live_pointer_olds_size == live_old_pointer_objects_size_);
3137
ASSERT(live_codes_size == live_code_objects_size_);
3138
ASSERT(live_cells_size == live_cell_objects_size_);
3139
ASSERT(live_news_size == live_young_objects_size_);
3140
3141
// Flip from and to spaces
3142
heap()->new_space()->Flip();
3143
3144
heap()->new_space()->MCCommitRelocationInfo();
3145
3146
// Set age_mark to bottom in to space
3147
Address mark = heap()->new_space()->bottom();
3148
heap()->new_space()->set_age_mark(mark);
3149
3150
PagedSpaces spaces;
3151
for (PagedSpace* space = spaces.next(); space != NULL; space = spaces.next())
3152
space->MCCommitRelocationInfo();
3153
3154
heap()->CheckNewSpaceExpansionCriteria();
3155
heap()->IncrementYoungSurvivorsCounter(live_news_size);
3156
}
3157
3158
3159
int MarkCompactCollector::RelocateMapObject(HeapObject* obj) {
3160
// Recover map pointer.
3161
MapWord encoding = obj->map_word();
3162
Address map_addr = encoding.DecodeMapAddress(heap()->map_space());
3163
ASSERT(heap()->map_space()->Contains(HeapObject::FromAddress(map_addr)));
3164
3165
// Get forwarding address before resetting map pointer
3166
Address new_addr = GetForwardingAddressInOldSpace(obj);
3167
3168
// Reset map pointer. The meta map object may not be copied yet so
3169
// Map::cast does not yet work.
3170
obj->set_map(reinterpret_cast<Map*>(HeapObject::FromAddress(map_addr)));
3171
3172
Address old_addr = obj->address();
3173
3174
if (new_addr != old_addr) {
3175
// Move contents.
3176
heap()->MoveBlockToOldSpaceAndUpdateRegionMarks(new_addr,
3177
old_addr,
3178
Map::kSize);
3179
}
3180
3181
#ifdef DEBUG
3182
if (FLAG_gc_verbose) {
3183
PrintF("relocate %p -> %p\n", old_addr, new_addr);
3184
}
3185
#endif
3186
3187
return Map::kSize;
3188
}
3189
3190
3191
static inline int RestoreMap(HeapObject* obj,
3192
PagedSpace* space,
3193
Address new_addr,
3194
Address map_addr) {
3195
// This must be a non-map object, and the function relies on the
3196
// assumption that the Map space is compacted before the other paged
3197
// spaces (see RelocateObjects).
3198
3199
// Reset map pointer.
3200
obj->set_map(Map::cast(HeapObject::FromAddress(map_addr)));
3201
3202
int obj_size = obj->Size();
3203
ASSERT_OBJECT_SIZE(obj_size);
3204
3205
ASSERT(space->MCSpaceOffsetForAddress(new_addr) <=
3206
space->MCSpaceOffsetForAddress(obj->address()));
3207
3208
#ifdef DEBUG
3209
if (FLAG_gc_verbose) {
3210
PrintF("relocate %p -> %p\n", obj->address(), new_addr);
3211
}
3212
#endif
3213
3214
return obj_size;
3215
}
3216
3217
3218
int MarkCompactCollector::RelocateOldNonCodeObject(HeapObject* obj,
3219
PagedSpace* space) {
3220
// Recover map pointer.
3221
MapWord encoding = obj->map_word();
3222
Address map_addr = encoding.DecodeMapAddress(heap()->map_space());
3223
ASSERT(heap()->map_space()->Contains(map_addr));
3224
3225
// Get forwarding address before resetting map pointer.
3226
Address new_addr = GetForwardingAddressInOldSpace(obj);
3227
3228
// Reset the map pointer.
3229
int obj_size = RestoreMap(obj, space, new_addr, map_addr);
3230
3231
Address old_addr = obj->address();
3232
3233
if (new_addr != old_addr) {
3234
// Move contents.
3235
if (space == heap()->old_data_space()) {
3236
heap()->MoveBlock(new_addr, old_addr, obj_size);
3237
} else {
3238
heap()->MoveBlockToOldSpaceAndUpdateRegionMarks(new_addr,
3239
old_addr,
3240
obj_size);
3241
}
3242
}
3243
3244
ASSERT(!HeapObject::FromAddress(new_addr)->IsCode());
3245
3246
HeapObject* copied_to = HeapObject::FromAddress(new_addr);
3247
if (copied_to->IsSharedFunctionInfo()) {
3248
PROFILE(heap()->isolate(),
3249
SharedFunctionInfoMoveEvent(old_addr, new_addr));
3250
}
3251
HEAP_PROFILE(heap(), ObjectMoveEvent(old_addr, new_addr));
3252
3253
return obj_size;
3254
}
3255
3256
3257
int MarkCompactCollector::RelocateOldPointerObject(HeapObject* obj) {
3258
return RelocateOldNonCodeObject(obj, heap()->old_pointer_space());
3259
}
3260
3261
3262
int MarkCompactCollector::RelocateOldDataObject(HeapObject* obj) {
3263
return RelocateOldNonCodeObject(obj, heap()->old_data_space());
3264
}
3265
3266
3267
int MarkCompactCollector::RelocateCellObject(HeapObject* obj) {
3268
return RelocateOldNonCodeObject(obj, heap()->cell_space());
3269
}
3270
3271
3272
int MarkCompactCollector::RelocateCodeObject(HeapObject* obj) {
3273
// Recover map pointer.
3274
MapWord encoding = obj->map_word();
3275
Address map_addr = encoding.DecodeMapAddress(heap()->map_space());
3276
ASSERT(heap()->map_space()->Contains(HeapObject::FromAddress(map_addr)));
3277
3278
// Get forwarding address before resetting map pointer
3279
Address new_addr = GetForwardingAddressInOldSpace(obj);
3280
3281
// Reset the map pointer.
3282
int obj_size = RestoreMap(obj, heap()->code_space(), new_addr, map_addr);
3283
3284
Address old_addr = obj->address();
3285
3286
if (new_addr != old_addr) {
3287
// Move contents.
3288
heap()->MoveBlock(new_addr, old_addr, obj_size);
3289
}
3290
3291
HeapObject* copied_to = HeapObject::FromAddress(new_addr);
3292
if (copied_to->IsCode()) {
3293
// May also update inline cache target.
3294
Code::cast(copied_to)->Relocate(new_addr - old_addr);
3295
// Notify the logger that compiled code has moved.
3296
PROFILE(heap()->isolate(), CodeMoveEvent(old_addr, new_addr));
3297
}
3298
HEAP_PROFILE(heap(), ObjectMoveEvent(old_addr, new_addr));
3299
3300
return obj_size;
3301
}
3302
3303
3304
int MarkCompactCollector::RelocateNewObject(HeapObject* obj) {
3305
int obj_size = obj->Size();
3306
3307
// Get forwarding address
3308
Address old_addr = obj->address();
3309
int offset = heap()->new_space()->ToSpaceOffsetForAddress(old_addr);
3310
3311
Address new_addr =
3312
Memory::Address_at(heap()->new_space()->FromSpaceLow() + offset);
3313
3314
#ifdef DEBUG
3315
if (heap()->new_space()->FromSpaceContains(new_addr)) {
3316
ASSERT(heap()->new_space()->FromSpaceOffsetForAddress(new_addr) <=
3317
heap()->new_space()->ToSpaceOffsetForAddress(old_addr));
3318
} else {
3319
ASSERT(heap()->TargetSpace(obj) == heap()->old_pointer_space() ||
3320
heap()->TargetSpace(obj) == heap()->old_data_space());
3321
}
3322
#endif
3323
3324
// New and old addresses cannot overlap.
3325
if (heap()->InNewSpace(HeapObject::FromAddress(new_addr))) {
3326
heap()->CopyBlock(new_addr, old_addr, obj_size);
3327
} else {
3328
heap()->CopyBlockToOldSpaceAndUpdateRegionMarks(new_addr,
3329
old_addr,
3330
obj_size);
3331
}
3332
3333
#ifdef DEBUG
3334
if (FLAG_gc_verbose) {
3335
PrintF("relocate %p -> %p\n", old_addr, new_addr);
3336
}
3337
#endif
3338
3339
HeapObject* copied_to = HeapObject::FromAddress(new_addr);
3340
if (copied_to->IsSharedFunctionInfo()) {
3341
PROFILE(heap()->isolate(),
3342
SharedFunctionInfoMoveEvent(old_addr, new_addr));
3343
}
3344
HEAP_PROFILE(heap(), ObjectMoveEvent(old_addr, new_addr));
3345
3346
return obj_size;
3715
SweepSpace(heap()->old_pointer_space(), how_to_sweep);
3716
SweepSpace(heap()->old_data_space(), how_to_sweep);
3717
3718
RemoveDeadInvalidatedCode();
3719
SweepSpace(heap()->code_space(), PRECISE);
3720
3721
SweepSpace(heap()->cell_space(), PRECISE);
3722
3723
EvacuateNewSpaceAndCandidates();
3724
3725
// ClearNonLiveTransitions depends on precise sweeping of map space to
3726
// detect whether unmarked map became dead in this collection or in one
3727
// of the previous ones.
3728
SweepSpace(heap()->map_space(), PRECISE);
3729
3730
// Deallocate unmarked objects and clear marked bits for marked objects.
3731
heap_->lo_space()->FreeUnmarkedObjects();
3347
3732
}
3348
3733
3349
3734
3359
3744
}
3360
3745
3361
3746
3747
// TODO(1466) ReportDeleteIfNeeded is not called currently.
3748
// Our profiling tools do not expect intersections between
3749
// code objects. We should either reenable it or change our tools.
3362
3750
void MarkCompactCollector::ReportDeleteIfNeeded(HeapObject* obj,
3363
3751
Isolate* isolate) {
3364
3752
#ifdef ENABLE_GDB_JIT_INTERFACE
3372
3760
}
3373
3761
3374
3762
3375
int MarkCompactCollector::SizeOfMarkedObject(HeapObject* obj) {
3376
MapWord map_word = obj->map_word();
3377
map_word.ClearMark();
3378
return obj->SizeFromMap(map_word.ToMap());
3379
}
3380
3381
3382
3763
void MarkCompactCollector::Initialize() {
3383
StaticPointersToNewGenUpdatingVisitor::Initialize();
3384
3764
StaticMarkingVisitor::Initialize();
3385
3765
}
3386
3766
3387
3767
3768
bool SlotsBuffer::IsTypedSlot(ObjectSlot slot) {
3769
return reinterpret_cast<uintptr_t>(slot) < NUMBER_OF_SLOT_TYPES;
3770
}
3771
3772
3773
bool SlotsBuffer::AddTo(SlotsBufferAllocator* allocator,
3774
SlotsBuffer** buffer_address,
3775
SlotType type,
3776
Address addr,
3777
AdditionMode mode) {
3778
SlotsBuffer* buffer = *buffer_address;
3779
if (buffer == NULL || !buffer->HasSpaceForTypedSlot()) {
3780
if (mode == FAIL_ON_OVERFLOW && ChainLengthThresholdReached(buffer)) {
3781
allocator->DeallocateChain(buffer_address);
3782
return false;
3783
}
3784
buffer = allocator->AllocateBuffer(buffer);
3785
*buffer_address = buffer;
3786
}
3787
ASSERT(buffer->HasSpaceForTypedSlot());
3788
buffer->Add(reinterpret_cast<ObjectSlot>(type));
3789
buffer->Add(reinterpret_cast<ObjectSlot>(addr));
3790
return true;
3791
}
3792
3793
3794
static inline SlotsBuffer::SlotType SlotTypeForRMode(RelocInfo::Mode rmode) {
3795
if (RelocInfo::IsCodeTarget(rmode)) {
3796
return SlotsBuffer::CODE_TARGET_SLOT;
3797
} else if (RelocInfo::IsEmbeddedObject(rmode)) {
3798
return SlotsBuffer::EMBEDDED_OBJECT_SLOT;
3799
} else if (RelocInfo::IsDebugBreakSlot(rmode)) {
3800
return SlotsBuffer::DEBUG_TARGET_SLOT;
3801
} else if (RelocInfo::IsJSReturn(rmode)) {
3802
return SlotsBuffer::JS_RETURN_SLOT;
3803
}
3804
UNREACHABLE();
3805
return SlotsBuffer::NUMBER_OF_SLOT_TYPES;
3806
}
3807
3808
3809
void MarkCompactCollector::RecordRelocSlot(RelocInfo* rinfo, Object* target) {
3810
Page* target_page = Page::FromAddress(reinterpret_cast<Address>(target));
3811
if (target_page->IsEvacuationCandidate() &&
3812
(rinfo->host() == NULL ||
3813
!ShouldSkipEvacuationSlotRecording(rinfo->host()))) {
3814
if (!SlotsBuffer::AddTo(&slots_buffer_allocator_,
3815
target_page->slots_buffer_address(),
3816
SlotTypeForRMode(rinfo->rmode()),
3817
rinfo->pc(),
3818
SlotsBuffer::FAIL_ON_OVERFLOW)) {
3819
EvictEvacuationCandidate(target_page);
3820
}
3821
}
3822
}
3823
3824
3825
void MarkCompactCollector::RecordCodeEntrySlot(Address slot, Code* target) {
3826
Page* target_page = Page::FromAddress(reinterpret_cast<Address>(target));
3827
if (target_page->IsEvacuationCandidate() &&
3828
!ShouldSkipEvacuationSlotRecording(reinterpret_cast<Object**>(slot))) {
3829
if (!SlotsBuffer::AddTo(&slots_buffer_allocator_,
3830
target_page->slots_buffer_address(),
3831
SlotsBuffer::CODE_ENTRY_SLOT,
3832
slot,
3833
SlotsBuffer::FAIL_ON_OVERFLOW)) {
3834
EvictEvacuationCandidate(target_page);
3835
}
3836
}
3837
}
3838
3839
3840
static inline SlotsBuffer::SlotType DecodeSlotType(
3841
SlotsBuffer::ObjectSlot slot) {
3842
return static_cast<SlotsBuffer::SlotType>(reinterpret_cast<intptr_t>(slot));
3843
}
3844
3845
3846
void SlotsBuffer::UpdateSlots(Heap* heap) {
3847
PointersUpdatingVisitor v(heap);
3848
3849
for (int slot_idx = 0; slot_idx < idx_; ++slot_idx) {
3850
ObjectSlot slot = slots_[slot_idx];
3851
if (!IsTypedSlot(slot)) {
3852
PointersUpdatingVisitor::UpdateSlot(heap, slot);
3853
} else {
3854
++slot_idx;
3855
ASSERT(slot_idx < idx_);
3856
UpdateSlot(&v,
3857
DecodeSlotType(slot),
3858
reinterpret_cast<Address>(slots_[slot_idx]));
3859
}
3860
}
3861
}
3862
3863
3864
void SlotsBuffer::UpdateSlotsWithFilter(Heap* heap) {
3865
PointersUpdatingVisitor v(heap);
3866
3867
for (int slot_idx = 0; slot_idx < idx_; ++slot_idx) {
3868
ObjectSlot slot = slots_[slot_idx];
3869
if (!IsTypedSlot(slot)) {
3870
if (!IsOnInvalidatedCodeObject(reinterpret_cast<Address>(slot))) {
3871
PointersUpdatingVisitor::UpdateSlot(heap, slot);
3872
}
3873
} else {
3874
++slot_idx;
3875
ASSERT(slot_idx < idx_);
3876
Address pc = reinterpret_cast<Address>(slots_[slot_idx]);
3877
if (!IsOnInvalidatedCodeObject(pc)) {
3878
UpdateSlot(&v,
3879
DecodeSlotType(slot),
3880
reinterpret_cast<Address>(slots_[slot_idx]));
3881
}
3882
}
3883
}
3884
}
3885
3886
3887
SlotsBuffer* SlotsBufferAllocator::AllocateBuffer(SlotsBuffer* next_buffer) {
3888
return new SlotsBuffer(next_buffer);
3889
}
3890
3891
3892
void SlotsBufferAllocator::DeallocateBuffer(SlotsBuffer* buffer) {
3893
delete buffer;
3894
}
3895
3896
3897
void SlotsBufferAllocator::DeallocateChain(SlotsBuffer** buffer_address) {
3898
SlotsBuffer* buffer = *buffer_address;
3899
while (buffer != NULL) {
3900
SlotsBuffer* next_buffer = buffer->next();
3901
DeallocateBuffer(buffer);
3902
buffer = next_buffer;
3903
}
3904
*buffer_address = NULL;
3905
}
3906
3907
3388
3908
} } // namespace v8::internal
Loggerhead is a web-based interface for Breezy
Version: 2.0.1