~oif-team/ubuntu/natty/qt4-x11/xi2.1

if (n_elements > 1 && element_size && (totalBytes / element_size) != n_elements || (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= totalBytes))

248

return 0;

249

250

totalBytes += sizeof(AllocAlignmentInteger);

251

void* result = malloc(totalBytes);

252

if (!result)

253

return 0;

254

255

memset(result, 0, totalBytes);

256

*static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;

257

result = static_cast<AllocAlignmentInteger*>(result) + 1;

258

return result;

259

#else

201

260

return calloc(n_elements, element_size);

261

#endif

202

262

}

203

263

204

264

void* fastCalloc(size_t n_elements, size_t element_size)

205

265

{

206

266

ASSERT(!isForbidden());

267

268

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

269

TryMallocReturnValue returnValue = tryFastCalloc(n_elements, element_size);

270

void* result;

271

returnValue.getValue(result);

272

#else

207

273

void* result = calloc(n_elements, element_size);

274

#endif

275

208

276

if (!result)

209

277

CRASH();

210

278

return result;

213

281

void fastFree(void* p)

214

282

{

215

283

ASSERT(!isForbidden());

284

285

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

286

if (!p)

287

return;

288

289

AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p);

290

if (*header != Internal::AllocTypeMalloc)

291

Internal::fastMallocMatchFailed(p);

292

free(header);

293

#else

216

294

free(p);

295

#endif

217

296

}

218

297

219

void* tryFastRealloc(void* p, size_t n)

298

TryMallocReturnValue tryFastRealloc(void* p, size_t n)

220

299

{

221

300

ASSERT(!isForbidden());

301

302

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

303

if (p) {

304

if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= n) // If overflow would occur...

305

return 0;

306

AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p);

307

if (*header != Internal::AllocTypeMalloc)

308

Internal::fastMallocMatchFailed(p);

309

void* result = realloc(header, n + sizeof(AllocAlignmentInteger));

310

if (!result)

311

return 0;

312

313

// This should not be needed because the value is already there:

314

// *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;

315

result = static_cast<AllocAlignmentInteger*>(result) + 1;

316

return result;

317

} else {

318

return fastMalloc(n);

319

}

320

#else

222

321

return realloc(p, n);

322

#endif

223

323

}

224

324

225

325

void* fastRealloc(void* p, size_t n)

226

326

{

227

327

ASSERT(!isForbidden());

328

329

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

330

TryMallocReturnValue returnValue = tryFastRealloc(p, n);

331

void* result;

332

returnValue.getValue(result);

333

#else

228

334

void* result = realloc(p, n);

335

#endif

336

229

337

if (!result)

230

338

CRASH();

231

339

return result;

265

373

#include "TCSystemAlloc.h"

266

374

#include <algorithm>

267

375

#include <errno.h>

376

#include <limits>

268

377

#include <new>

269

378

#include <pthread.h>

270

379

#include <stdarg.h>

321

430

#define CHECK_CONDITION ASSERT

322

431

323

432

#if PLATFORM(DARWIN)

433

class Span;

434

class TCMalloc_Central_FreeListPadded;

324

435

class TCMalloc_PageHeap;

325

436

class TCMalloc_ThreadCache;

326

class TCMalloc_Central_FreeListPadded;

437

template <typename T> class PageHeapAllocator;

327

438

328

439

class FastMallocZone {

329

440

public:

339

450

static void statistics(malloc_zone_t*, malloc_statistics_t* stats) { memset(stats, 0, sizeof(malloc_statistics_t)); }

340

451

341

452

private:

342

FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*);

453

FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*, PageHeapAllocator<Span>*, PageHeapAllocator<TCMalloc_ThreadCache>*);

343

454

static size_t size(malloc_zone_t*, const void*);

344

455

static void* zoneMalloc(malloc_zone_t*, size_t);

345

456

static void* zoneCalloc(malloc_zone_t*, size_t numItems, size_t size);

352

463

TCMalloc_PageHeap* m_pageHeap;

353

464

TCMalloc_ThreadCache** m_threadHeaps;

354

465

TCMalloc_Central_FreeListPadded* m_centralCaches;

466

PageHeapAllocator<Span>* m_spanAllocator;

467

PageHeapAllocator<TCMalloc_ThreadCache>* m_pageHeapAllocator;

355

468

};

356

469

357

470

#endif

432

545

static const size_t kPageMapBigAllocationThreshold = 128 << 20;

433

546

434

547

// Minimum number of pages to fetch from system at a time. Must be

435

// significantly bigger than kBlockSize to amortize system-call

548

// significantly bigger than kPageSize to amortize system-call

436

549

// overhead, and also to reduce external fragementation. Also, we

437

550

// should keep this value big because various incarnations of Linux

438

551

// have small limits on the number of mmap() regions per

820

933

char* free_area_;

821

934

size_t free_avail_;

822

935

936

// Linked list of all regions allocated by this allocator

937

void* allocated_regions_;

938

823

939

// Free list of already carved objects

824

940

void* free_list_;

825

941

830

946

void Init() {

831

947

ASSERT(kAlignedSize <= kAllocIncrement);

832

948

inuse_ = 0;

949

allocated_regions_ = 0;

833

950

free_area_ = NULL;

834

951

free_avail_ = 0;

835

952

free_list_ = NULL;

844

961

} else {

845

962

if (free_avail_ < kAlignedSize) {

846

963

// Need more room

847

free_area_ = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));

848

if (free_area_ == NULL) CRASH();

849

free_avail_ = kAllocIncrement;

964

char* new_allocation = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));

965

if (!new_allocation)

966

CRASH();

967

968

*(void**)new_allocation = allocated_regions_;

969

allocated_regions_ = new_allocation;

970

free_area_ = new_allocation + kAlignedSize;

971

free_avail_ = kAllocIncrement - kAlignedSize;

850

972

}

851

973

result = free_area_;

852

974

free_area_ += kAlignedSize;

863

985

}

864

986

865

987

int inuse() const { return inuse_; }

988

989

#if defined(WTF_CHANGES) && PLATFORM(DARWIN)

990

template <class Recorder>

991

void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader& reader)

992

{

993

vm_address_t adminAllocation = reinterpret_cast<vm_address_t>(allocated_regions_);

994

while (adminAllocation) {

995

recorder.recordRegion(adminAllocation, kAllocIncrement);

996

adminAllocation = *reader(reinterpret_cast<vm_address_t*>(adminAllocation));

997

}

998

}

999

#endif

866

1000

};

867

1001

868

1002

// -------------------------------------------------------------------------

921

1055

#endif

922

1056

};

923

1057

924

#if TCMALLOC_TRACK_DECOMMITED_SPANS

925

1058

#define ASSERT_SPAN_COMMITTED(span) ASSERT(!span->decommitted)

926

#else

927

#define ASSERT_SPAN_COMMITTED(span)

928

#endif

929

1059

930

1060

#ifdef SPAN_HISTORY

931

1061

void Event(Span* span, char op, int v = 0) {

1037

1167

typedef PackedCache<BITS, uint64_t> CacheType;

1038

1168

};

1039

1169

1170

#if defined(WTF_CHANGES)

1171

#if PLATFORM(X86_64)

1172

// On all known X86-64 platforms, the upper 16 bits are always unused and therefore

1173

// can be excluded from the PageMap key.

1174

// See http://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details

1175

1176

static const size_t kBitsUnusedOn64Bit = 16;

1177

#else

1178

static const size_t kBitsUnusedOn64Bit = 0;

1179

#endif

1180

1181

// A three-level map for 64-bit machines

1182

template <> class MapSelector<64> {

1183

public:

1184

typedef TCMalloc_PageMap3<64 - kPageShift - kBitsUnusedOn64Bit> Type;

1185

typedef PackedCache<64, uint64_t> CacheType;

1186

};

1187

#endif

1188

1040

1189

// A two-level map for 32-bit machines

1041

1190

template <> class MapSelector<32> {

1042

1191

public:

1043

typedef TCMalloc_PageMap2<32-kPageShift> Type;

1044

typedef PackedCache<32-kPageShift, uint16_t> CacheType;

1192

typedef TCMalloc_PageMap2<32 - kPageShift> Type;

1193

typedef PackedCache<32 - kPageShift, uint16_t> CacheType;

1045

1194

};

1046

1195

1047

1196

// -------------------------------------------------------------------------

1052

1201

// contiguous runs of pages (called a "span").

1053

1202

// -------------------------------------------------------------------------

1054

1203

1204

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1205

// The central page heap collects spans of memory that have been deleted but are still committed until they are released

1206

// back to the system. We use a background thread to periodically scan the list of free spans and release some back to the

1207

// system. Every 5 seconds, the background thread wakes up and does the following:

1208

// - Check if we needed to commit memory in the last 5 seconds. If so, skip this scavenge because it's a sign that we are short

1209

// of free committed pages and so we should not release them back to the system yet.

1210

// - Otherwise, go through the list of free spans (from largest to smallest) and release up to a fraction of the free committed pages

1211

// back to the system.

1212

// - If the number of free committed pages reaches kMinimumFreeCommittedPageCount, we can stop the scavenging and block the

1213

// scavenging thread until the number of free committed pages goes above kMinimumFreeCommittedPageCount.

1214

1215

// Background thread wakes up every 5 seconds to scavenge as long as there is memory available to return to the system.

1216

static const int kScavengeTimerDelayInSeconds = 5;

1217

1218

// Number of free committed pages that we want to keep around.

1219

static const size_t kMinimumFreeCommittedPageCount = 512;

1220

1221

// During a scavenge, we'll release up to a fraction of the free committed pages.

1222

#if PLATFORM(WIN)

1223

// We are slightly less aggressive in releasing memory on Windows due to performance reasons.

1224

static const int kMaxScavengeAmountFactor = 3;

1225

#else

1226

static const int kMaxScavengeAmountFactor = 2;

1227

#endif

1228

#endif

1229

1055

1230

class TCMalloc_PageHeap {

1056

1231

public:

1057

1232

void init();

1151

1326

// Bytes allocated from system

1152

1327

uint64_t system_bytes_;

1153

1328

1329

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1330

// Number of pages kept in free lists that are still committed.

1331

Length free_committed_pages_;

1332

1333

// Number of pages that we committed in the last scavenge wait interval.

1334

Length pages_committed_since_last_scavenge_;

1335

#endif

1336

1154

1337

bool GrowHeap(Length n);

1155

1338

1156

1339

// REQUIRES span->length >= n

1173

1356

// span of exactly the specified length. Else, returns NULL.

1174

1357

Span* AllocLarge(Length n);

1175

1358

1359

#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1176

1360

// Incrementally release some memory to the system.

1177

1361

// IncrementalScavenge(n) is called whenever n pages are freed.

1178

1362

void IncrementalScavenge(Length n);

1363

#endif

1179

1364

1180

1365

// Number of pages to deallocate before doing more scavenging

1181

1366

int64_t scavenge_counter_;

1186

1371

#if defined(WTF_CHANGES) && PLATFORM(DARWIN)

1187

1372

friend class FastMallocZone;

1188

1373

#endif

1374

1375

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1376

static NO_RETURN void* runScavengerThread(void*);

1377

1378

NO_RETURN void scavengerThread();

1379

1380

void scavenge();

1381

1382

inline bool shouldContinueScavenging() const;

1383

1384

pthread_mutex_t m_scavengeMutex;

1385

1386

pthread_cond_t m_scavengeCondition;

1387

1388

// Keeps track of whether the background thread is actively scavenging memory every kScavengeTimerDelayInSeconds, or

1389

// it's blocked waiting for more pages to be deleted.

1390

bool m_scavengeThreadActive;

1391

#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1189

1392

};

1190

1393

1191

1394

void TCMalloc_PageHeap::init()

1194

1397

pagemap_cache_ = PageMapCache(0);

1195

1398

free_pages_ = 0;

1196

1399

system_bytes_ = 0;

1400

1401

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1402

free_committed_pages_ = 0;

1403

pages_committed_since_last_scavenge_ = 0;

1404

#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1405

1197

1406

scavenge_counter_ = 0;

1198

1407

// Start scavenging at kMaxPages list

1199

1408

scavenge_index_ = kMaxPages-1;

1204

1413

DLL_Init(&free_[i].normal);

1205

1414

DLL_Init(&free_[i].returned);

1206

1415

}

1207

}

1416

1417

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1418

pthread_mutex_init(&m_scavengeMutex, 0);

1419

pthread_cond_init(&m_scavengeCondition, 0);

1420

m_scavengeThreadActive = true;

1421

pthread_t thread;

1422

pthread_create(&thread, 0, runScavengerThread, this);

1423

#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1424

}

1425

1426

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1427

void* TCMalloc_PageHeap::runScavengerThread(void* context)

1428

{

1429

static_cast<TCMalloc_PageHeap*>(context)->scavengerThread();

1430

#if COMPILER(MSVC)

1431

// Without this, Visual Studio will complain that this method does not return a value.

1432

return 0;

1433

#endif

1434

}

1435

1436

void TCMalloc_PageHeap::scavenge()

1437

{

1438

// If we have to commit memory in the last 5 seconds, it means we don't have enough free committed pages

1439

// for the amount of allocations that we do. So hold off on releasing memory back to the system.

1440

if (pages_committed_since_last_scavenge_ > 0) {

1441

pages_committed_since_last_scavenge_ = 0;

1442

return;

1443

}

1444

Length pagesDecommitted = 0;

1445

for (int i = kMaxPages; i >= 0; i--) {

1446

SpanList* slist = (static_cast<size_t>(i) == kMaxPages) ? &large_ : &free_[i];

1447

if (!DLL_IsEmpty(&slist->normal)) {

1448

// Release the last span on the normal portion of this list

1449

Span* s = slist->normal.prev;

1450

// Only decommit up to a fraction of the free committed pages if pages_allocated_since_last_scavenge_ > 0.

1451

if ((pagesDecommitted + s->length) * kMaxScavengeAmountFactor > free_committed_pages_)

1452

continue;

1453

DLL_Remove(s);

1454

TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),

1455

static_cast<size_t>(s->length << kPageShift));

1456

if (!s->decommitted) {

1457

pagesDecommitted += s->length;

1458

s->decommitted = true;

1459

}

1460

DLL_Prepend(&slist->returned, s);

1461

// We can stop scavenging if the number of free committed pages left is less than or equal to the minimum number we want to keep around.

1462

if (free_committed_pages_ <= kMinimumFreeCommittedPageCount + pagesDecommitted)

1463

break;

1464

}

1465

}

1466

pages_committed_since_last_scavenge_ = 0;

1467

ASSERT(free_committed_pages_ >= pagesDecommitted);

1468

free_committed_pages_ -= pagesDecommitted;

1469

}

1470

1471

inline bool TCMalloc_PageHeap::shouldContinueScavenging() const

1472

{

1473

return free_committed_pages_ > kMinimumFreeCommittedPageCount;

1474

}

1475

1476

#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1208

1477

1209

1478

inline Span* TCMalloc_PageHeap::New(Length n) {

1210

1479

ASSERT(Check());

1228

1497

1229

1498

Span* result = ll->next;

1230

1499

Carve(result, n, released);

1231

#if TCMALLOC_TRACK_DECOMMITED_SPANS

1232

1500

if (result->decommitted) {

1233

1501

TCMalloc_SystemCommit(reinterpret_cast<void*>(result->start << kPageShift), static_cast<size_t>(n << kPageShift));

1234

1502

result->decommitted = false;

1235

}

1503

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1504

pages_committed_since_last_scavenge_ += n;

1236

1505

#endif

1506

}

1507

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1508

else {

1509

// The newly allocated memory is from a span that's in the normal span list (already committed). Update the

1510

// free committed pages count.

1511

ASSERT(free_committed_pages_ >= n);

1512

free_committed_pages_ -= n;

1513

}

1514

#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1237

1515

ASSERT(Check());

1238

1516

free_pages_ -= n;

1239

1517

return result;

1290

1568

1291

1569

if (best != NULL) {

1292

1570

Carve(best, n, from_released);

1293

#if TCMALLOC_TRACK_DECOMMITED_SPANS

1294

1571

if (best->decommitted) {

1295

1572

TCMalloc_SystemCommit(reinterpret_cast<void*>(best->start << kPageShift), static_cast<size_t>(n << kPageShift));

1296

1573

best->decommitted = false;

1297

}

1574

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1575

pages_committed_since_last_scavenge_ += n;

1298

1576

#endif

1577

}

1578

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1579

else {

1580

// The newly allocated memory is from a span that's in the normal span list (already committed). Update the

1581

// free committed pages count.

1582

ASSERT(free_committed_pages_ >= n);

1583

free_committed_pages_ -= n;

1584

}

1585

#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1299

1586

ASSERT(Check());

1300

1587

free_pages_ -= n;

1301

1588

return best;

1320

1607

return leftover;

1321

1608

}

1322

1609

1323

#if !TCMALLOC_TRACK_DECOMMITED_SPANS

1324

static ALWAYS_INLINE void propagateDecommittedState(Span*, Span*) { }

1325

#else

1326

1610

static ALWAYS_INLINE void propagateDecommittedState(Span* destination, Span* source)

1327

1611

{

1328

1612

destination->decommitted = source->decommitted;

1329

1613

}

1330

#endif

1331

1614

1332

1615

inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) {

1333

1616

ASSERT(n > 0);

1354

1637

}

1355

1638

}

1356

1639

1357

#if !TCMALLOC_TRACK_DECOMMITED_SPANS

1358

static ALWAYS_INLINE void mergeDecommittedStates(Span*, Span*) { }

1359

#else

1360

1640

static ALWAYS_INLINE void mergeDecommittedStates(Span* destination, Span* other)

1361

1641

{

1362

if (other->decommitted)

1642

if (destination->decommitted && !other->decommitted) {

1643

TCMalloc_SystemRelease(reinterpret_cast<void*>(other->start << kPageShift),

1644

static_cast<size_t>(other->length << kPageShift));

1645

} else if (other->decommitted && !destination->decommitted) {

1646

TCMalloc_SystemRelease(reinterpret_cast<void*>(destination->start << kPageShift),

1647

static_cast<size_t>(destination->length << kPageShift));

1363

1648

destination->decommitted = true;

1649

}

1364

1650

}

1365

#endif

1366

1651

1367

1652

inline void TCMalloc_PageHeap::Delete(Span* span) {

1368

1653

ASSERT(Check());

1379

1664

// necessary. We do not bother resetting the stale pagemap

1380

1665

// entries for the pieces we are merging together because we only

1381

1666

// care about the pagemap entries for the boundaries.

1382

1383

// Note that the spans we merge into "span" may come out of

1384

// a "returned" list. For simplicity, we move these into the

1385

// "normal" list of the appropriate size class.

1667

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1668

// Track the total size of the neighboring free spans that are committed.

1669

Length neighboringCommittedSpansLength = 0;

1670

#endif

1386

1671

const PageID p = span->start;

1387

1672

const Length n = span->length;

1388

1673

Span* prev = GetDescriptor(p-1);

1390

1675

// Merge preceding span into this span

1391

1676

ASSERT(prev->start + prev->length == p);

1392

1677

const Length len = prev->length;

1678

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1679

if (!prev->decommitted)

1680

neighboringCommittedSpansLength += len;

1681

#endif

1393

1682

mergeDecommittedStates(span, prev);

1394

1683

DLL_Remove(prev);

1395

1684

DeleteSpan(prev);

1403

1692

// Merge next span into this span

1404

1693

ASSERT(next->start == p+n);

1405

1694

const Length len = next->length;

1695

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1696

if (!next->decommitted)

1697

neighboringCommittedSpansLength += len;

1698

#endif

1406

1699

mergeDecommittedStates(span, next);

1407

1700

DLL_Remove(next);

1408

1701

DeleteSpan(next);

1413

1706

1414

1707

Event(span, 'D', span->length);

1415

1708

span->free = 1;

1416

if (span->length < kMaxPages) {

1417

DLL_Prepend(&free_[span->length].normal, span);

1709

if (span->decommitted) {

1710

if (span->length < kMaxPages)

1711

DLL_Prepend(&free_[span->length].returned, span);

1712

else

1713

DLL_Prepend(&large_.returned, span);

1418

1714

} else {

1419

DLL_Prepend(&large_.normal, span);

1715

if (span->length < kMaxPages)

1716

DLL_Prepend(&free_[span->length].normal, span);

1717

else

1718

DLL_Prepend(&large_.normal, span);

1420

1719

}

1421

1720

free_pages_ += n;

1422

1721

1722

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1723

if (span->decommitted) {

1724

// If the merged span is decommitted, that means we decommitted any neighboring spans that were

1725

// committed. Update the free committed pages count.

1726

free_committed_pages_ -= neighboringCommittedSpansLength;

1727

} else {

1728

// If the merged span remains committed, add the deleted span's size to the free committed pages count.

1729

free_committed_pages_ += n;

1730

}

1731

1732

// Make sure the scavenge thread becomes active if we have enough freed pages to release some back to the system.

1733

if (!m_scavengeThreadActive && shouldContinueScavenging())

1734

pthread_cond_signal(&m_scavengeCondition);

1735

#else

1423

1736

IncrementalScavenge(n);

1737

#endif

1738

1424

1739

ASSERT(Check());

1425

1740

}

1426

1741

1742

#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1427

1743

void TCMalloc_PageHeap::IncrementalScavenge(Length n) {

1428

1744

// Fast path; not yet time to release memory

1429

1745

scavenge_counter_ -= n;

1444

1760

DLL_Remove(s);

1445

1761

TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),

1446

1762

static_cast<size_t>(s->length << kPageShift));

1447

#if TCMALLOC_TRACK_DECOMMITED_SPANS

1448

1763

s->decommitted = true;

1449

#endif

1450

1764

DLL_Prepend(&slist->returned, s);

1451

1765

1452

1766

scavenge_counter_ = std::max<size_t>(64UL, std::min<size_t>(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay)));

1463

1777

// Nothing to scavenge, delay for a while

1464

1778

scavenge_counter_ = kDefaultReleaseDelay;

1465

1779

}

1780

#endif

1466

1781

1467

1782

void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) {

1468

1783

// Associate span object with all interior pages as well

1574

1889

}

1575

1890

ask = actual_size >> kPageShift;

1576

1891

1892

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

1893

pages_committed_since_last_scavenge_ += ask;

1894

#endif

1895

1577

1896

uint64_t old_system_bytes = system_bytes_;

1578

1897

system_bytes_ += (ask << kPageShift);

1579

1898

const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;

1954

2273

1955

2274

#define pageheap getPageHeap()

1956

2275

2276

#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY

2277

#if PLATFORM(WIN)

2278

static void sleep(unsigned seconds)

2279

{

2280

::Sleep(seconds * 1000);

2281

}

2282

#endif

2283

2284

void TCMalloc_PageHeap::scavengerThread()

2285

{

2286

#if HAVE(PTHREAD_SETNAME_NP)

2287

pthread_setname_np("JavaScriptCore: FastMalloc scavenger");

2288

#endif

2289

2290

while (1) {

2291

if (!shouldContinueScavenging()) {

2292

pthread_mutex_lock(&m_scavengeMutex);

2293

m_scavengeThreadActive = false;

2294

// Block until there are enough freed pages to release back to the system.

2295

pthread_cond_wait(&m_scavengeCondition, &m_scavengeMutex);

2296

m_scavengeThreadActive = true;

2297

pthread_mutex_unlock(&m_scavengeMutex);

2298

}

2299

sleep(kScavengeTimerDelayInSeconds);

2300

{

2301

SpinLockHolder h(&pageheap_lock);

2302

pageheap->scavenge();

2303

}

2304

}

2305

}

2306

#endif

2307

1957

2308

// If TLS is available, we also store a copy

1958

2309

// of the per-thread object in a __thread variable

1959

2310

// since __thread variables are faster to read

2041

2392

// The following check is expensive, so it is disabled by default

2042

2393

if (false) {

2043

2394

// Check that object does not occur in list

2044

int got = 0;

2395

unsigned got = 0;

2045

2396

for (void* p = span->objects; p != NULL; p = *((void**) p)) {

2046

2397

ASSERT(p != object);

2047

2398

got++;

3235

3586

return malloc<true>(size);

3236

3587

}

3237

3588

3238

void* tryFastMalloc(size_t size)

3589

TryMallocReturnValue tryFastMalloc(size_t size)

3239

3590

{

3240

3591

return malloc<false>(size);

3241

3592

}

3244

3595

ALWAYS_INLINE

3245

3596

#endif

3246

3597

void* malloc(size_t size) {

3247

void* result = do_malloc(size);

3598

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3599

if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= size) // If overflow would occur...

3600

return 0;

3601

size += sizeof(AllocAlignmentInteger);

3602

void* result = do_malloc(size);

3603

if (!result)

3604

return 0;

3605

3606

*static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;

3607

result = static_cast<AllocAlignmentInteger*>(result) + 1;

3608

#else

3609

void* result = do_malloc(size);

3610

#endif

3611

3248

3612

#ifndef WTF_CHANGES

3249

3613

MallocHook::InvokeNewHook(result, size);

3250

3614

#endif

3258

3622

#ifndef WTF_CHANGES

3259

3623

MallocHook::InvokeDeleteHook(ptr);

3260

3624

#endif

3261

do_free(ptr);

3625

3626

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3627

if (!ptr)

3628

return;

3629

3630

AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(ptr);

3631

if (*header != Internal::AllocTypeMalloc)

3632

Internal::fastMallocMatchFailed(ptr);

3633

do_free(header);

3634

#else

3635

do_free(ptr);

3636

#endif

3262

3637

}

3263

3638

3264

3639

#ifndef WTF_CHANGES

3272

3647

return calloc<true>(n, elem_size);

3273

3648

}

3274

3649

3275

void* tryFastCalloc(size_t n, size_t elem_size)

3650

TryMallocReturnValue tryFastCalloc(size_t n, size_t elem_size)

3276

3651

{

3277

3652

return calloc<false>(n, elem_size);

3278

3653

}

3281

3656

ALWAYS_INLINE

3282

3657

#endif

3283

3658

void* calloc(size_t n, size_t elem_size) {

3284

const size_t totalBytes = n * elem_size;

3659

size_t totalBytes = n * elem_size;

3285

3660

3286

3661

// Protect against overflow

3287

3662

if (n > 1 && elem_size && (totalBytes / elem_size) != n)

3288

3663

return 0;

3289

3290

void* result = do_malloc(totalBytes);

3291

if (result != NULL) {

3664

3665

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3666

if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= totalBytes) // If overflow would occur...

3667

return 0;

3668

3669

totalBytes += sizeof(AllocAlignmentInteger);

3670

void* result = do_malloc(totalBytes);

3671

if (!result)

3672

return 0;

3673

3292

3674

memset(result, 0, totalBytes);

3293

}

3675

*static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;

3676

result = static_cast<AllocAlignmentInteger*>(result) + 1;

3677

#else

3678

void* result = do_malloc(totalBytes);

3679

if (result != NULL) {

3680

memset(result, 0, totalBytes);

3681

}

3682

#endif

3683

3294

3684

#ifndef WTF_CHANGES

3295

3685

MallocHook::InvokeNewHook(result, totalBytes);

3296

3686

#endif

3297

3687

return result;

3298

3688

}

3299

3689

3690

// Since cfree isn't used anywhere, we don't compile it in.

3691

#ifndef WTF_CHANGES

3300

3692

#ifndef WTF_CHANGES

3301

3693

extern "C"

3302

3694

#endif

3306

3698

#endif

3307

3699

do_free(ptr);

3308

3700

}

3701

#endif

3309

3702

3310

3703

#ifndef WTF_CHANGES

3311

3704

extern "C"

3318

3711

return realloc<true>(old_ptr, new_size);

3319

3712

}

3320

3713

3321

void* tryFastRealloc(void* old_ptr, size_t new_size)

3714

TryMallocReturnValue tryFastRealloc(void* old_ptr, size_t new_size)

3322

3715

{

3323

3716

return realloc<false>(old_ptr, new_size);

3324

3717

}

3328

3721

#endif

3329

3722

void* realloc(void* old_ptr, size_t new_size) {

3330

3723

if (old_ptr == NULL) {

3724

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3725

void* result = malloc(new_size);

3726

#else

3331

3727

void* result = do_malloc(new_size);

3332

3728

#ifndef WTF_CHANGES

3333

3729

MallocHook::InvokeNewHook(result, new_size);

3334

3730

#endif

3731

#endif

3335

3732

return result;

3336

3733

}

3337

3734

if (new_size == 0) {

3342

3739

return NULL;

3343

3740

}

3344

3741

3742

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3743

if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= new_size) // If overflow would occur...

3744

return 0;

3745

new_size += sizeof(AllocAlignmentInteger);

3746

AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(old_ptr);

3747

if (*header != Internal::AllocTypeMalloc)

3748

Internal::fastMallocMatchFailed(old_ptr);

3749

old_ptr = header;

3750

#endif

3751

3345

3752

// Get the size of the old entry

3346

3753

const PageID p = reinterpret_cast<uintptr_t>(old_ptr) >> kPageShift;

3347

3754

size_t cl = pageheap->GetSizeClassIfCached(p);

3378

3785

// that we already know the sizeclass of old_ptr. The benefit

3379

3786

// would be small, so don't bother.

3380

3787

do_free(old_ptr);

3788

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3789

new_ptr = static_cast<AllocAlignmentInteger*>(new_ptr) + 1;

3790

#endif

3381

3791

return new_ptr;

3382

3792

} else {

3793

#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)

3794

old_ptr = pByte + sizeof(AllocAlignmentInteger); // Set old_ptr back to the user pointer.

3795

#endif

3383

3796

return old_ptr;

3384

3797

}

3385

3798

}

3608

4021

3609

4022

void visit(void* ptr) { m_freeObjects.add(ptr); }

3610

4023

bool isFreeObject(void* ptr) const { return m_freeObjects.contains(ptr); }

4024

bool isFreeObject(vm_address_t ptr) const { return isFreeObject(reinterpret_cast<void*>(ptr)); }

3611

4025

size_t freeObjectCount() const { return m_freeObjects.size(); }

3612

4026

3613

4027

void findFreeObjects(TCMalloc_ThreadCache* threadCache)

3658

4072

vm_range_recorder_t* m_recorder;

3659

4073

const RemoteMemoryReader& m_reader;

3660

4074

const FreeObjectFinder& m_freeObjectFinder;

3661

mutable HashSet<void*> m_seenPointers;

4075

4076

HashSet<void*> m_seenPointers;

4077

Vector<Span*> m_coalescedSpans;

3662

4078

3663

4079

public:

3664

4080

PageMapMemoryUsageRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader, const FreeObjectFinder& freeObjectFinder)

3670

4086

, m_freeObjectFinder(freeObjectFinder)

3671

4087

{ }

3672

4088

3673

int visit(void* ptr) const

3674

{

3675

if (!ptr)

3676

return 1;

3677

3678

Span* span = m_reader(reinterpret_cast<Span*>(ptr));

3679

if (m_seenPointers.contains(ptr))

3680

return span->length;

3681

m_seenPointers.add(ptr);

3682

3683

// Mark the memory used for the Span itself as an administrative region

3684

vm_range_t ptrRange = { reinterpret_cast<vm_address_t>(ptr), sizeof(Span) };

3685

if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE))

3686

(*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, &ptrRange, 1);

3687

3688

ptrRange.address = span->start << kPageShift;

3689

ptrRange.size = span->length * kPageSize;

3690

3691

// Mark the memory region the span represents as candidates for containing pointers

3692

if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE))

4089

~PageMapMemoryUsageRecorder()

4090

{

4091

ASSERT(!m_coalescedSpans.size());

4092

}

4093

4094

void recordPendingRegions()

4095

{

4096

Span* lastSpan = m_coalescedSpans[m_coalescedSpans.size() - 1];

4097

vm_range_t ptrRange = { m_coalescedSpans[0]->start << kPageShift, 0 };

4098

ptrRange.size = (lastSpan->start << kPageShift) - ptrRange.address + (lastSpan->length * kPageSize);

4099

4100

// Mark the memory region the spans represent as a candidate for containing pointers

4101

if (m_typeMask & MALLOC_PTR_REGION_RANGE_TYPE)

3693

4102

(*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1);

3694

4103

3695

if (!span->free && (m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) {

3696

// If it's an allocated large object span, mark it as in use

3697

if (span->sizeclass == 0 && !m_freeObjectFinder.isFreeObject(reinterpret_cast<void*>(ptrRange.address)))

3698

(*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &ptrRange, 1);

3699

else if (span->sizeclass) {

3700

const size_t byteSize = ByteSizeForClass(span->sizeclass);

3701

unsigned totalObjects = (span->length << kPageShift) / byteSize;

3702

ASSERT(span->refcount <= totalObjects);

3703

char* ptr = reinterpret_cast<char*>(span->start << kPageShift);

4104

if (!(m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) {

4105

m_coalescedSpans.clear();

4106

return;

4107

}

4108

4109

Vector<vm_range_t, 1024> allocatedPointers;

4110

for (size_t i = 0; i < m_coalescedSpans.size(); ++i) {

4111

Span *theSpan = m_coalescedSpans[i];

4112

if (theSpan->free)

4113

continue;

4114

4115

vm_address_t spanStartAddress = theSpan->start << kPageShift;

4116

vm_size_t spanSizeInBytes = theSpan->length * kPageSize;

4117

4118

if (!theSpan->sizeclass) {

4119

// If it's an allocated large object span, mark it as in use

4120

if (!m_freeObjectFinder.isFreeObject(spanStartAddress))

4121

allocatedPointers.append((vm_range_t){spanStartAddress, spanSizeInBytes});

4122

} else {

4123

const size_t objectSize = ByteSizeForClass(theSpan->sizeclass);

3704

4124

3705

4125

// Mark each allocated small object within the span as in use

3706

for (unsigned i = 0; i < totalObjects; i++) {

3707

char* thisObject = ptr + (i * byteSize);

3708

if (m_freeObjectFinder.isFreeObject(thisObject))

3709

continue;

3710

3711

vm_range_t objectRange = { reinterpret_cast<vm_address_t>(thisObject), byteSize };

3712

(*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &objectRange, 1);

4126

const vm_address_t endOfSpan = spanStartAddress + spanSizeInBytes;

4127

for (vm_address_t object = spanStartAddress; object + objectSize <= endOfSpan; object += objectSize) {

4128

if (!m_freeObjectFinder.isFreeObject(object))

4129

allocatedPointers.append((vm_range_t){object, objectSize});

3713

4130

}

3714

4131

}

3715

4132

}

3716

4133

4134

(*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, allocatedPointers.data(), allocatedPointers.size());

4135

4136

m_coalescedSpans.clear();

4137

}

4138

4139

int visit(void* ptr)

4140

{

4141

if (!ptr)

4142

return 1;

4143

4144

Span* span = m_reader(reinterpret_cast<Span*>(ptr));

4145

if (!span->start)

4146

return 1;

4147

4148

if (m_seenPointers.contains(ptr))

4149

return span->length;

4150

m_seenPointers.add(ptr);

4151

4152

if (!m_coalescedSpans.size()) {

4153

m_coalescedSpans.append(span);

4154

return span->length;

4155

}

4156

4157

Span* previousSpan = m_coalescedSpans[m_coalescedSpans.size() - 1];

4158

vm_address_t previousSpanStartAddress = previousSpan->start << kPageShift;

4159

vm_size_t previousSpanSizeInBytes = previousSpan->length * kPageSize;

4160

4161

// If the new span is adjacent to the previous span, do nothing for now.

4162

vm_address_t spanStartAddress = span->start << kPageShift;

4163

if (spanStartAddress == previousSpanStartAddress + previousSpanSizeInBytes) {

4164

m_coalescedSpans.append(span);

4165

return span->length;

4166

}

4167

4168

// New span is not adjacent to previous span, so record the spans coalesced so far.

4169

recordPendingRegions();

4170

m_coalescedSpans.append(span);

4171

3717

4172

return span->length;

3718

4173

}

3719

4174

};

3720

4175

4176

class AdminRegionRecorder {

4177

task_t m_task;

4178

void* m_context;

4179

unsigned m_typeMask;

4180

vm_range_recorder_t* m_recorder;

4181

const RemoteMemoryReader& m_reader;

4182

4183

Vector<vm_range_t, 1024> m_pendingRegions;

4184

4185

public:

4186

AdminRegionRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader)

4187

: m_task(task)

4188

, m_context(context)

4189

, m_typeMask(typeMask)

4190

, m_recorder(recorder)

4191

, m_reader(reader)

4192

{ }

4193

4194

void recordRegion(vm_address_t ptr, size_t size)

4195

{

4196

if (m_typeMask & MALLOC_ADMIN_REGION_RANGE_TYPE)

4197

m_pendingRegions.append((vm_range_t){ ptr, size });

4198

}

4199

4200

void visit(void *ptr, size_t size)

4201

{

4202

recordRegion(reinterpret_cast<vm_address_t>(ptr), size);

4203

}

4204

4205

void recordPendingRegions()

4206

{

4207

if (m_pendingRegions.size()) {

4208

(*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, m_pendingRegions.data(), m_pendingRegions.size());

4209

m_pendingRegions.clear();

4210

}

4211

}

4212

4213

~AdminRegionRecorder()

4214

{

4215

ASSERT(!m_pendingRegions.size());

4216

}

4217

};

4218

3721

4219

kern_return_t FastMallocZone::enumerate(task_t task, void* context, unsigned typeMask, vm_address_t zoneAddress, memory_reader_t reader, vm_range_recorder_t recorder)

3722

4220

{

3723

4221

RemoteMemoryReader memoryReader(task, reader);

3737

4235

3738

4236

TCMalloc_PageHeap::PageMap* pageMap = &pageHeap->pagemap_;

3739

4237

PageMapFreeObjectFinder pageMapFinder(memoryReader, finder);

3740

pageMap->visit(pageMapFinder, memoryReader);

4238

pageMap->visitValues(pageMapFinder, memoryReader);

3741

4239

3742

4240

PageMapMemoryUsageRecorder usageRecorder(task, context, typeMask, recorder, memoryReader, finder);

3743

pageMap->visit(usageRecorder, memoryReader);

4241

pageMap->visitValues(usageRecorder, memoryReader);

4242

usageRecorder.recordPendingRegions();

4243

4244

AdminRegionRecorder adminRegionRecorder(task, context, typeMask, recorder, memoryReader);

4245

pageMap->visitAllocations(adminRegionRecorder, memoryReader);

4246

4247

PageHeapAllocator<Span>* spanAllocator = memoryReader(mzone->m_spanAllocator);

4248

PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator = memoryReader(mzone->m_pageHeapAllocator);

4249

4250

spanAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader);

4251

pageHeapAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader);

4252

4253

adminRegionRecorder.recordPendingRegions();

3744

4254

3745

4255

return 0;

3746

4256

}

3781

4291

3782

4292

extern "C" {

3783

4293

malloc_introspection_t jscore_fastmalloc_introspection = { &FastMallocZone::enumerate, &FastMallocZone::goodSize, &FastMallocZone::check, &FastMallocZone::print,

3784

&FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics };

4294

&FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics

4295

4296

#if !defined(BUILDING_ON_TIGER) && !defined(BUILDING_ON_LEOPARD) && !PLATFORM(IPHONE)

4297

, 0 // zone_locked will not be called on the zone unless it advertises itself as version five or higher.

4298

#endif

4299

4300

};

3785

4301

}

3786

4302

3787

FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches)

4303

FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches, PageHeapAllocator<Span>* spanAllocator, PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator)

3788

4304

: m_pageHeap(pageHeap)

3789

4305

, m_threadHeaps(threadHeaps)

3790

4306

, m_centralCaches(centralCaches)

4307

, m_spanAllocator(spanAllocator)

4308

, m_pageHeapAllocator(pageHeapAllocator)

3791

4309

{

3792

4310

memset(&m_zone, 0, sizeof(m_zone));

4311

m_zone.version = 4;

3793

4312

m_zone.zone_name = "JavaScriptCore FastMalloc";

3794

4313

m_zone.size = &FastMallocZone::size;

3795

4314

m_zone.malloc = &FastMallocZone::zoneMalloc;

3805

4324

3806

4325

void FastMallocZone::init()

3807

4326

{

3808

static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache));

4327

static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator);

3809

4328

}

3810

4329

3811

4330

#endif

Older »