~ubuntu-branches/ubuntu/raring/qtwebkit-source/raring-proposed

ScratchBuffer* scratchBuffer = m_jit.globalData()->scratchBufferForSize(sizeof(EncodedJSValue) * std::max(haveUInt32s ? 2u : 0u, numberOfPoisonedVirtualRegisters + (numberOfDisplacedVirtualRegisters <= GPRInfo::numberOfRegisters ? 0 : numberOfDisplacedVirtualRegisters)));

267

EncodedJSValue* scratchDataBuffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;

268

269

// From here on, the code assumes that it is profitable to maximize the distance

270

// between when something is computed and when it is stored.

271

272

// 5) Perform all reboxing of integers.

273

274

if (haveUnboxedInt32s || haveUInt32s) {

275

for (size_t index = 0; index < operands.size(); ++index) {

276

const ValueRecovery& recovery = operands[index];

277

switch (recovery.technique()) {

278

case UnboxedInt32InGPR:

279

if (recovery.gpr() != alreadyBoxed)

280

m_jit.or64(GPRInfo::tagTypeNumberRegister, recovery.gpr());

281

break;

282

283

case AlreadyInJSStackAsUnboxedInt32:

284

m_jit.store32(AssemblyHelpers::TrustedImm32(static_cast<uint32_t>(TagTypeNumber >> 32)), AssemblyHelpers::tagFor(static_cast<VirtualRegister>(operands.operandForIndex(index))));

285

break;

286

287

case UInt32InGPR: {

288

// This occurs when the speculative JIT left an unsigned 32-bit integer

289

// in a GPR. If it's positive, we can just box the int. Otherwise we

290

// need to turn it into a boxed double.

291

292

// We don't try to be clever with register allocation here; we assume

293

// that the program is using FPRs and we don't try to figure out which

294

// ones it is using. Instead just temporarily save fpRegT0 and then

295

// restore it. This makes sense because this path is not cheap to begin

296

// with, and should happen very rarely.

297

298

GPRReg addressGPR = GPRInfo::regT0;

299

if (addressGPR == recovery.gpr())

300

addressGPR = GPRInfo::regT1;

301

302

m_jit.store64(addressGPR, scratchDataBuffer);

303

m_jit.move(AssemblyHelpers::TrustedImmPtr(scratchDataBuffer + 1), addressGPR);

304

m_jit.storeDouble(FPRInfo::fpRegT0, addressGPR);

305

306

AssemblyHelpers::Jump positive = m_jit.branch32(AssemblyHelpers::GreaterThanOrEqual, recovery.gpr(), AssemblyHelpers::TrustedImm32(0));

307

308

m_jit.convertInt32ToDouble(recovery.gpr(), FPRInfo::fpRegT0);

309

m_jit.addDouble(AssemblyHelpers::AbsoluteAddress(&AssemblyHelpers::twoToThe32), FPRInfo::fpRegT0);

310

m_jit.boxDouble(FPRInfo::fpRegT0, recovery.gpr());

311

312

AssemblyHelpers::Jump done = m_jit.jump();

313

314

positive.link(&m_jit);

315

316

m_jit.or64(GPRInfo::tagTypeNumberRegister, recovery.gpr());

317

318

done.link(&m_jit);

319

320

m_jit.loadDouble(addressGPR, FPRInfo::fpRegT0);

321

m_jit.load64(scratchDataBuffer, addressGPR);

322

break;

323

}

324

325

default:

326

break;

327

}

328

}

329

}

330

331

// 6) Dump all non-poisoned GPRs. For poisoned GPRs, save them into the scratch storage.

332

// Note that GPRs do not have a fast change (like haveFPRs) because we expect that

333

// most OSR failure points will have at least one GPR that needs to be dumped.

334

335

initializePoisoned(operands.numberOfLocals());

336

unsigned currentPoisonIndex = 0;

337

338

for (size_t index = 0; index < operands.size(); ++index) {

339

const ValueRecovery& recovery = operands[index];

340

int operand = operands.operandForIndex(index);

341

switch (recovery.technique()) {

342

case InGPR:

343

case UnboxedInt32InGPR:

344

case UInt32InGPR:

345

if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {

346

m_jit.store64(recovery.gpr(), scratchDataBuffer + currentPoisonIndex);

347

m_poisonScratchIndices[operands.variableForIndex(index)] = currentPoisonIndex;

348

currentPoisonIndex++;

349

} else

350

m_jit.store64(recovery.gpr(), AssemblyHelpers::addressFor((VirtualRegister)operand));

351

break;

352

default:

353

break;

354

}

355

}

356

357

// At this point all GPRs are available for scratch use.

358

359

if (haveFPRs) {

360

// 7) Box all doubles (relies on there being more GPRs than FPRs)

361

362

for (size_t index = 0; index < operands.size(); ++index) {

363

const ValueRecovery& recovery = operands[index];

364

if (recovery.technique() != InFPR)

365

continue;

366

FPRReg fpr = recovery.fpr();

367

GPRReg gpr = GPRInfo::toRegister(FPRInfo::toIndex(fpr));

368

m_jit.boxDouble(fpr, gpr);

369

}

370

371

// 8) Dump all doubles into the stack, or to the scratch storage if

372

// the destination virtual register is poisoned.

373

374

for (size_t index = 0; index < operands.size(); ++index) {

375

const ValueRecovery& recovery = operands[index];

376

if (recovery.technique() != InFPR)

377

continue;

378

GPRReg gpr = GPRInfo::toRegister(FPRInfo::toIndex(recovery.fpr()));

379

if (operands.isVariable(index) && poisonedVirtualRegisters[operands.variableForIndex(index)]) {

380

m_jit.store64(gpr, scratchDataBuffer + currentPoisonIndex);

381

m_poisonScratchIndices[operands.variableForIndex(index)] = currentPoisonIndex;

382

currentPoisonIndex++;

383

} else

384

m_jit.store64(gpr, AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));

385

}

386

}

387

388

// At this point all GPRs and FPRs are available for scratch use.

389

390

// 9) Box all unboxed doubles in the stack.

391

if (haveUnboxedDoubles) {

392

for (size_t index = 0; index < operands.size(); ++index) {

393

const ValueRecovery& recovery = operands[index];

394

if (recovery.technique() != AlreadyInJSStackAsUnboxedDouble)

395

continue;

396

m_jit.loadDouble(AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)), FPRInfo::fpRegT0);

397

m_jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);

398

m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));

399

}

400

}

401

402

ASSERT(currentPoisonIndex == numberOfPoisonedVirtualRegisters);

403

404

// 10) Reshuffle displaced virtual registers. Optimize for the case that

405

// the number of displaced virtual registers is not more than the number

406

// of available physical registers.

407

408

if (numberOfDisplacedVirtualRegisters) {

409

if (numberOfDisplacedVirtualRegisters <= GPRInfo::numberOfRegisters) {

410

// So far this appears to be the case that triggers all the time, but

411

// that is far from guaranteed.

412

413

unsigned displacementIndex = 0;

414

for (size_t index = 0; index < operands.size(); ++index) {

415

const ValueRecovery& recovery = operands[index];

416

switch (recovery.technique()) {

417

case DisplacedInJSStack:

418

m_jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));

419

break;

420

421

case Int32DisplacedInJSStack: {

422

GPRReg gpr = GPRInfo::toRegister(displacementIndex++);

423

m_jit.load32(AssemblyHelpers::addressFor(recovery.virtualRegister()), gpr);

424

m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);

425

break;

426

}

427

428

case DoubleDisplacedInJSStack: {

429

GPRReg gpr = GPRInfo::toRegister(displacementIndex++);

430

m_jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), gpr);

431

m_jit.sub64(GPRInfo::tagTypeNumberRegister, gpr);

432

break;

433

}

434

435

default:

436

break;

437

}

438

}

439

440

displacementIndex = 0;

441

for (size_t index = 0; index < operands.size(); ++index) {

442

const ValueRecovery& recovery = operands[index];

443

switch (recovery.technique()) {

444

case DisplacedInJSStack:

445

case Int32DisplacedInJSStack:

446

case DoubleDisplacedInJSStack:

447

m_jit.store64(GPRInfo::toRegister(displacementIndex++), AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));

448

break;

449

450

default:

451

break;

452

}

453

}

454

} else {

455

// FIXME: This should use the shuffling algorithm that we use

456

// for speculative->non-speculative jumps, if we ever discover that

457

// some hot code with lots of live values that get displaced and

458

// spilled really enjoys frequently failing speculation.

459

460

// For now this code is engineered to be correct but probably not

461

// super. In particular, it correctly handles cases where for example

462

// the displacements are a permutation of the destination values, like

463

464

// 1 -> 2

465

// 2 -> 1

466

467

// It accomplishes this by simply lifting all of the virtual registers

468

// from their old (DFG JIT) locations and dropping them in a scratch

469

// location in memory, and then transferring from that scratch location

470

// to their new (old JIT) locations.

471

472

unsigned scratchIndex = numberOfPoisonedVirtualRegisters;

473

for (size_t index = 0; index < operands.size(); ++index) {

474

const ValueRecovery& recovery = operands[index];

475

476

switch (recovery.technique()) {

477

case DisplacedInJSStack:

478

m_jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);

479

m_jit.store64(GPRInfo::regT0, scratchDataBuffer + scratchIndex++);

480

break;

481

482

case Int32DisplacedInJSStack: {

483

m_jit.load32(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);

484

m_jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);

485

m_jit.store64(GPRInfo::regT0, scratchDataBuffer + scratchIndex++);

486

break;

487

}

488

489

case DoubleDisplacedInJSStack: {

490

m_jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);

491

m_jit.sub64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);

492

m_jit.store64(GPRInfo::regT0, scratchDataBuffer + scratchIndex++);

493

break;

494

}

495

496

default:

497

break;

498

}

499

}

500

501

scratchIndex = numberOfPoisonedVirtualRegisters;

502

for (size_t index = 0; index < operands.size(); ++index) {

503

const ValueRecovery& recovery = operands[index];

504

switch (recovery.technique()) {

505

case DisplacedInJSStack:

506

case Int32DisplacedInJSStack:

507

case DoubleDisplacedInJSStack:

508

m_jit.load64(scratchDataBuffer + scratchIndex++, GPRInfo::regT0);

509

m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));

510

break;

511

512

default:

513

break;

514

}

515

}

516

517

ASSERT(scratchIndex == numberOfPoisonedVirtualRegisters + numberOfDisplacedVirtualRegisters);

518

}

519

}

520

521

// 11) Dump all poisoned virtual registers.

522

523

if (numberOfPoisonedVirtualRegisters) {

524

for (int virtualRegister = 0; virtualRegister < (int)operands.numberOfLocals(); ++virtualRegister) {

525

if (!poisonedVirtualRegisters[virtualRegister])

526

continue;

527

528

const ValueRecovery& recovery = operands.local(virtualRegister);

529

switch (recovery.technique()) {

530

case InGPR:

531

case UnboxedInt32InGPR:

532

case UInt32InGPR:

533

case InFPR:

534

m_jit.load64(scratchDataBuffer + poisonIndex(virtualRegister), GPRInfo::regT0);

535

m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)virtualRegister));

536

break;

537

538

default:

539

break;

540

}

541

}

542

}

543

544

// 12) Dump all constants. Optimize for Undefined, since that's a constant we see

545

// often.

546

547

if (haveConstants) {

548

if (haveUndefined)

549

m_jit.move(AssemblyHelpers::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);

550

551

for (size_t index = 0; index < operands.size(); ++index) {

552

const ValueRecovery& recovery = operands[index];

553

if (recovery.technique() != Constant)

554

continue;

555

if (recovery.constant().isUndefined())

556

m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));

557

else

558

m_jit.store64(AssemblyHelpers::TrustedImm64(JSValue::encode(recovery.constant())), AssemblyHelpers::addressFor((VirtualRegister)operands.operandForIndex(index)));

559

}

560

}

561

562

// 13) Adjust the old JIT's execute counter. Since we are exiting OSR, we know

563

// that all new calls into this code will go to the new JIT, so the execute

564

// counter only affects call frames that performed OSR exit and call frames

565

// that were still executing the old JIT at the time of another call frame's

566

// OSR exit. We want to ensure that the following is true:

567

568

// (a) Code the performs an OSR exit gets a chance to reenter optimized

569

// code eventually, since optimized code is faster. But we don't

570

// want to do such reentery too aggressively (see (c) below).

571

572

// (b) If there is code on the call stack that is still running the old

573

// JIT's code and has never OSR'd, then it should get a chance to

574

// perform OSR entry despite the fact that we've exited.

575

576

// (c) Code the performs an OSR exit should not immediately retry OSR

577

// entry, since both forms of OSR are expensive. OSR entry is

578

// particularly expensive.

579

580

// (d) Frequent OSR failures, even those that do not result in the code

581

// running in a hot loop, result in recompilation getting triggered.

582

583

// To ensure (c), we'd like to set the execute counter to

584

// counterValueForOptimizeAfterWarmUp(). This seems like it would endanger

585

// (a) and (b), since then every OSR exit would delay the opportunity for

586

// every call frame to perform OSR entry. Essentially, if OSR exit happens

587

// frequently and the function has few loops, then the counter will never

588

// become non-negative and OSR entry will never be triggered. OSR entry

589

// will only happen if a loop gets hot in the old JIT, which does a pretty

590

// good job of ensuring (a) and (b). But that doesn't take care of (d),

591

// since each speculation failure would reset the execute counter.

592

// So we check here if the number of speculation failures is significantly

593

// larger than the number of successes (we want 90% success rate), and if

594

// there have been a large enough number of failures. If so, we set the

595

// counter to 0; otherwise we set the counter to

596

// counterValueForOptimizeAfterWarmUp().

597

598

handleExitCounts(exit);

599

600

// 14) Reify inlined call frames.

601

602

ASSERT(m_jit.baselineCodeBlock()->getJITType() == JITCode::BaselineJIT);

603

m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(m_jit.baselineCodeBlock()), AssemblyHelpers::addressFor((VirtualRegister)JSStack::CodeBlock));

604

605

for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {

606

InlineCallFrame* inlineCallFrame = codeOrigin.inlineCallFrame;

607

CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(codeOrigin);

608

CodeBlock* baselineCodeBlockForCaller = m_jit.baselineCodeBlockFor(inlineCallFrame->caller);

609

Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlockForCaller);

610

unsigned returnBytecodeIndex = inlineCallFrame->caller.bytecodeIndex + OPCODE_LENGTH(op_call);

611

BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned, BytecodeAndMachineOffset::getBytecodeIndex>(decodedCodeMap.begin(), decodedCodeMap.size(), returnBytecodeIndex);

612

613

ASSERT(mapping);

614

ASSERT(mapping->m_bytecodeIndex == returnBytecodeIndex);

615

616

void* jumpTarget = baselineCodeBlockForCaller->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);

617

618

GPRReg callerFrameGPR;

619

if (inlineCallFrame->caller.inlineCallFrame) {

620

m_jit.addPtr(AssemblyHelpers::TrustedImm32(inlineCallFrame->caller.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT3);

621

callerFrameGPR = GPRInfo::regT3;

622

} else

623

callerFrameGPR = GPRInfo::callFrameRegister;

624

625

m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(baselineCodeBlock), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CodeBlock)));

626

m_jit.store64(AssemblyHelpers::TrustedImm64(JSValue::encode(JSValue(inlineCallFrame->callee->scope()))), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ScopeChain)));

627

m_jit.store64(callerFrameGPR, AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CallerFrame)));

628

m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(jumpTarget), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ReturnPC)));

629

m_jit.store32(AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ArgumentCount)));

630

m_jit.store64(AssemblyHelpers::TrustedImm64(JSValue::encode(JSValue(inlineCallFrame->callee.get()))), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::Callee)));

631

}

632

633

// 15) Create arguments if necessary and place them into the appropriate aliased

634

// registers.

635

636

if (haveArguments) {

637

HashSet<InlineCallFrame*, DefaultHash<InlineCallFrame*>::Hash,

638

NullableHashTraits<InlineCallFrame*> > didCreateArgumentsObject;

639

640

for (size_t index = 0; index < operands.size(); ++index) {

641

const ValueRecovery& recovery = operands[index];

642

if (recovery.technique() != ArgumentsThatWereNotCreated)

643

continue;

644

int operand = operands.operandForIndex(index);

645

// Find the right inline call frame.

646

InlineCallFrame* inlineCallFrame = 0;

647

for (InlineCallFrame* current = exit.m_codeOrigin.inlineCallFrame;

648

current;

649

current = current->caller.inlineCallFrame) {

650

if (current->stackOffset <= operand) {

651

inlineCallFrame = current;

652

break;

653

}

654

}

655

656

if (!m_jit.baselineCodeBlockFor(inlineCallFrame)->usesArguments())

657

continue;

658

int argumentsRegister = m_jit.argumentsRegisterFor(inlineCallFrame);

659

if (didCreateArgumentsObject.add(inlineCallFrame).isNewEntry) {

660

// We know this call frame optimized out an arguments object that

661

// the baseline JIT would have created. Do that creation now.

662

if (inlineCallFrame) {

663

m_jit.addPtr(AssemblyHelpers::TrustedImm32(inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT0);

664

m_jit.setupArguments(GPRInfo::regT0);

665

} else

666

m_jit.setupArgumentsExecState();

667

m_jit.move(

668

AssemblyHelpers::TrustedImmPtr(

669

bitwise_cast<void*>(operationCreateArguments)),

670

GPRInfo::nonArgGPR0);

671

m_jit.call(GPRInfo::nonArgGPR0);

672

m_jit.store64(GPRInfo::returnValueGPR, AssemblyHelpers::addressFor(argumentsRegister));

673

m_jit.store64(

674

GPRInfo::returnValueGPR,

675

AssemblyHelpers::addressFor(unmodifiedArgumentsRegister(argumentsRegister)));

676

m_jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0); // no-op move on almost all platforms.

677

}

678

679

m_jit.load64(AssemblyHelpers::addressFor(argumentsRegister), GPRInfo::regT0);

680

m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));

681

}

682

}

683

684

// 16) Load the result of the last bytecode operation into regT0.

685

686

if (exit.m_lastSetOperand != std::numeric_limits<int>::max())

687

m_jit.load64(AssemblyHelpers::addressFor((VirtualRegister)exit.m_lastSetOperand), GPRInfo::cachedResultRegister);

688

689

// 17) Adjust the call frame pointer.

690

691

if (exit.m_codeOrigin.inlineCallFrame)

692

m_jit.addPtr(AssemblyHelpers::TrustedImm32(exit.m_codeOrigin.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister);

693

694

// 18) Jump into the corresponding baseline JIT code.

695

696

CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(exit.m_codeOrigin);

697

Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlock);

698

699

BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned, BytecodeAndMachineOffset::getBytecodeIndex>(decodedCodeMap.begin(), decodedCodeMap.size(), exit.m_codeOrigin.bytecodeIndex);

700

701

ASSERT(mapping);

702

ASSERT(mapping->m_bytecodeIndex == exit.m_codeOrigin.bytecodeIndex);

703

704

void* jumpTarget = baselineCodeBlock->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);

705

706

ASSERT(GPRInfo::regT1 != GPRInfo::cachedResultRegister);

707

708

m_jit.move(AssemblyHelpers::TrustedImmPtr(jumpTarget), GPRInfo::regT1);

709

710

m_jit.jump(GPRInfo::regT1);

711

712

#if DFG_ENABLE(DEBUG_VERBOSE)

713

dataLogF("-> %p\n", jumpTarget);

714

#endif

715

}

716

717

} } // namespace JSC::DFG

718

719

#endif // ENABLE(DFG_JIT) && USE(JSVALUE64)

Older »