2
* Copyright (C) 2019 Apple Inc. All rights reserved.
4
* Redistribution and use in source and binary forms, with or without
5
* modification, are permitted provided that the following conditions
7
* 1. Redistributions of source code must retain the above copyright
8
* notice, this list of conditions and the following disclaimer.
9
* 2. Redistributions in binary form must reproduce the above copyright
10
* notice, this list of conditions and the following disclaimer in the
11
* documentation and/or other materials provided with the distribution.
13
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
#include "WasmSlowPaths.h"
29
#if ENABLE(WEBASSEMBLY)
31
#include "BytecodeStructs.h"
32
#include "ExceptionFuzz.h"
33
#include "FrameTracers.h"
34
#include "LLIntData.h"
35
#include "WasmBBQPlan.h"
36
#include "WasmCallee.h"
37
#include "WasmCompilationMode.h"
38
#include "WasmContextInlines.h"
39
#include "WasmFunctionCodeBlock.h"
40
#include "WasmInstance.h"
41
#include "WasmModuleInformation.h"
42
#include "WasmOMGForOSREntryPlan.h"
43
#include "WasmOMGPlan.h"
44
#include "WasmOperations.h"
45
#include "WasmSignatureInlines.h"
46
#include "WasmThunks.h"
47
#include "WasmWorklist.h"
49
namespace JSC { namespace LLInt {
51
#define WASM_RETURN_TWO(first, second) do { \
52
return encodeResult(first, second); \
55
#define WASM_END_IMPL() WASM_RETURN_TWO(pc, 0)
57
#define WASM_THROW(exceptionType) do { \
58
callFrame->setArgumentCountIncludingThis(static_cast<int>(exceptionType)); \
59
WASM_RETURN_TWO(LLInt::wasmExceptionInstructions(), 0); \
62
#define WASM_END() do { \
66
#define WASM_RETURN(value) do { \
67
callFrame->uncheckedR(instruction.m_dst) = static_cast<EncodedJSValue>(value); \
71
#define WASM_CALL_RETURN(targetInstance, callTarget, callTargetTag) do { \
72
WASM_RETURN_TWO(retagCodePtr(callTarget, callTargetTag, SlowPathPtrTag), targetInstance); \
75
#define CODE_BLOCK() \
76
bitwise_cast<Wasm::FunctionCodeBlock*>(callFrame->codeBlock())
78
#define READ(virtualRegister) \
79
(virtualRegister.isConstant() \
80
? JSValue::decode(CODE_BLOCK()->getConstant(virtualRegister)) \
81
: callFrame->r(virtualRegister))
83
inline bool jitCompileAndSetHeuristics(Wasm::LLIntCallee* callee, Wasm::FunctionCodeBlock* codeBlock, Wasm::Instance* instance)
85
Wasm::LLIntTierUpCounter& tierUpCounter = codeBlock->tierUpCounter();
86
if (!tierUpCounter.checkIfOptimizationThresholdReached()) {
87
dataLogLnIf(Options::verboseOSR(), " JIT threshold should be lifted.");
91
if (callee->replacement()) {
92
dataLogLnIf(Options::verboseOSR(), " Code was already compiled.");
93
tierUpCounter.optimizeSoon();
99
auto locker = holdLock(tierUpCounter.m_lock);
100
switch (tierUpCounter.m_compilationStatus) {
101
case Wasm::LLIntTierUpCounter::CompilationStatus::NotCompiled:
103
tierUpCounter.m_compilationStatus = Wasm::LLIntTierUpCounter::CompilationStatus::Compiling;
105
case Wasm::LLIntTierUpCounter::CompilationStatus::Compiling:
106
tierUpCounter.optimizeAfterWarmUp();
108
case Wasm::LLIntTierUpCounter::CompilationStatus::Compiled:
114
uint32_t functionIndex = codeBlock->functionIndex();
115
RefPtr<Wasm::Plan> plan;
116
if (Options::wasmLLIntTiersUpToBBQ())
117
plan = adoptRef(*new Wasm::BBQPlan(instance->context(), makeRef(const_cast<Wasm::ModuleInformation&>(instance->module().moduleInformation())), functionIndex, instance->codeBlock(), Wasm::Plan::dontFinalize()));
119
plan = adoptRef(*new Wasm::OMGPlan(instance->context(), Ref<Wasm::Module>(instance->module()), functionIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize()));
121
Wasm::ensureWorklist().enqueue(makeRef(*plan));
122
if (UNLIKELY(!Options::useConcurrentJIT()))
123
plan->waitForCompletion();
125
tierUpCounter.optimizeAfterWarmUp();
128
return !!callee->replacement();
131
WASM_SLOW_PATH_DECL(prologue_osr)
135
Wasm::LLIntCallee* callee = static_cast<Wasm::LLIntCallee*>(callFrame->callee().asWasmCallee());
136
Wasm::FunctionCodeBlock* codeBlock = CODE_BLOCK();
138
dataLogLnIf(Options::verboseOSR(), *callee, ": Entered epilogue_osr with tierUpCounter = ", codeBlock->tierUpCounter());
140
if (!jitCompileAndSetHeuristics(callee, codeBlock, instance))
141
WASM_RETURN_TWO(0, 0);
143
WASM_RETURN_TWO(callee->replacement()->entrypoint().executableAddress(), 0);
146
WASM_SLOW_PATH_DECL(loop_osr)
148
if (!Options::useWebAssemblyOSR()) {
149
slow_path_wasm_prologue_osr(callFrame, pc, instance);
150
WASM_RETURN_TWO(0, 0);
153
Wasm::LLIntCallee* callee = static_cast<Wasm::LLIntCallee*>(callFrame->callee().asWasmCallee());
154
Wasm::FunctionCodeBlock* codeBlock = CODE_BLOCK();
155
Wasm::LLIntTierUpCounter& tierUpCounter = codeBlock->tierUpCounter();
157
dataLogLnIf(Options::verboseOSR(), *callee, ": Entered loop_osr with tierUpCounter = ", codeBlock->tierUpCounter());
159
unsigned loopOSREntryBytecodeOffset = codeBlock->bytecodeOffset(pc);
160
const auto& osrEntryData = tierUpCounter.osrEntryDataForLoop(loopOSREntryBytecodeOffset);
162
if (!tierUpCounter.checkIfOptimizationThresholdReached()) {
163
dataLogLnIf(Options::verboseOSR(), " JIT threshold should be lifted.");
164
WASM_RETURN_TWO(0, 0);
167
const auto doOSREntry = [&] {
168
Wasm::OMGForOSREntryCallee* osrEntryCallee = callee->osrEntryCallee();
169
if (osrEntryCallee->loopIndex() != osrEntryData.loopIndex)
170
WASM_RETURN_TWO(0, 0);
172
size_t osrEntryScratchBufferSize = osrEntryCallee->osrEntryScratchBufferSize();
173
RELEASE_ASSERT(osrEntryScratchBufferSize == osrEntryData.values.size());
174
uint64_t* buffer = instance->context()->scratchBufferForSize(osrEntryScratchBufferSize);
176
WASM_RETURN_TWO(0, 0);
179
for (VirtualRegister reg : osrEntryData.values)
180
buffer[index++] = READ(reg).encodedJSValue();
182
WASM_RETURN_TWO(buffer, osrEntryCallee->entrypoint().executableAddress());
185
if (callee->osrEntryCallee())
188
bool compile = false;
190
auto locker = holdLock(tierUpCounter.m_lock);
191
switch (tierUpCounter.m_loopCompilationStatus) {
192
case Wasm::LLIntTierUpCounter::CompilationStatus::NotCompiled:
194
tierUpCounter.m_loopCompilationStatus = Wasm::LLIntTierUpCounter::CompilationStatus::Compiling;
196
case Wasm::LLIntTierUpCounter::CompilationStatus::Compiling:
197
tierUpCounter.optimizeAfterWarmUp();
199
case Wasm::LLIntTierUpCounter::CompilationStatus::Compiled:
205
Ref<Wasm::Plan> plan = adoptRef(*static_cast<Wasm::Plan*>(new Wasm::OMGForOSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::Callee>(*callee), codeBlock->functionIndex(), osrEntryData.loopIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize())));
206
Wasm::ensureWorklist().enqueue(plan.copyRef());
207
if (UNLIKELY(!Options::useConcurrentJIT()))
208
plan->waitForCompletion();
210
tierUpCounter.optimizeAfterWarmUp();
213
if (callee->osrEntryCallee())
216
WASM_RETURN_TWO(0, 0);
219
WASM_SLOW_PATH_DECL(epilogue_osr)
221
Wasm::LLIntCallee* callee = static_cast<Wasm::LLIntCallee*>(callFrame->callee().asWasmCallee());
222
Wasm::FunctionCodeBlock* codeBlock = CODE_BLOCK();
224
dataLogLnIf(Options::verboseOSR(), *callee, ": Entered epilogue_osr with tierUpCounter = ", codeBlock->tierUpCounter());
226
jitCompileAndSetHeuristics(callee, codeBlock, instance);
231
WASM_SLOW_PATH_DECL(trace)
233
UNUSED_PARAM(instance);
235
if (!Options::traceLLIntExecution())
238
WasmOpcodeID opcodeID = pc->opcodeID<WasmOpcodeTraits>();
239
dataLogF("<%p> %p / %p: executing bc#%zu, %s, pc = %p\n",
241
callFrame->codeBlock(),
243
static_cast<intptr_t>(CODE_BLOCK()->bytecodeOffset(pc)),
244
pc->name<WasmOpcodeTraits>(),
246
if (opcodeID == wasm_enter) {
247
dataLogF("Frame will eventually return to %p\n", callFrame->returnPC().value());
248
*removeCodePtrTag<volatile char*>(callFrame->returnPC().value());
250
if (opcodeID == wasm_ret) {
251
dataLogF("Will be returning to %p\n", callFrame->returnPC().value());
252
dataLogF("The new cfr will be %p\n", callFrame->callerFrame());
257
WASM_SLOW_PATH_DECL(out_of_line_jump_target)
259
UNUSED_PARAM(instance);
261
pc = CODE_BLOCK()->outOfLineJumpTarget(pc);
265
WASM_SLOW_PATH_DECL(ref_func)
267
auto instruction = pc->as<WasmRefFunc, WasmOpcodeTraits>();
268
WASM_RETURN(Wasm::operationWasmRefFunc(instance, instruction.m_functionIndex));
271
WASM_SLOW_PATH_DECL(table_get)
273
auto instruction = pc->as<WasmTableGet, WasmOpcodeTraits>();
274
int32_t index = READ(instruction.m_index).unboxedInt32();
275
EncodedJSValue result = Wasm::operationGetWasmTableElement(instance, instruction.m_tableIndex, index);
277
WASM_THROW(Wasm::ExceptionType::OutOfBoundsTableAccess);
281
WASM_SLOW_PATH_DECL(table_set)
283
auto instruction = pc->as<WasmTableSet, WasmOpcodeTraits>();
284
int32_t index = READ(instruction.m_index).unboxedInt32();
285
EncodedJSValue value = READ(instruction.m_value).encodedJSValue();
286
if (!Wasm::operationSetWasmTableElement(instance, instruction.m_tableIndex, index, value))
287
WASM_THROW(Wasm::ExceptionType::OutOfBoundsTableAccess);
291
WASM_SLOW_PATH_DECL(table_size)
293
auto instruction = pc->as<WasmTableSize, WasmOpcodeTraits>();
294
WASM_RETURN(Wasm::operationGetWasmTableSize(instance, instruction.m_tableIndex));
297
WASM_SLOW_PATH_DECL(table_fill)
299
auto instruction = pc->as<WasmTableFill, WasmOpcodeTraits>();
300
int32_t offset = READ(instruction.m_offset).unboxedInt32();
301
EncodedJSValue fill = READ(instruction.m_fill).encodedJSValue();
302
int32_t size = READ(instruction.m_size).unboxedInt32();
303
if (!Wasm::operationWasmTableFill(instance, instruction.m_tableIndex, offset, fill, size))
304
WASM_THROW(Wasm::ExceptionType::OutOfBoundsTableAccess);
308
WASM_SLOW_PATH_DECL(table_grow)
310
auto instruction = pc->as<WasmTableGrow, WasmOpcodeTraits>();
311
EncodedJSValue fill = READ(instruction.m_fill).encodedJSValue();
312
int32_t size = READ(instruction.m_size).unboxedInt32();
313
WASM_RETURN(Wasm::operationWasmTableGrow(instance, instruction.m_tableIndex, fill, size));
316
WASM_SLOW_PATH_DECL(grow_memory)
318
auto instruction = pc->as<WasmGrowMemory, WasmOpcodeTraits>();
319
int32_t delta = READ(instruction.m_delta).unboxedInt32();
320
WASM_RETURN(Wasm::operationGrowMemory(callFrame, instance, delta));
323
inline SlowPathReturnType doWasmCall(Wasm::Instance* instance, unsigned functionIndex)
325
uint32_t importFunctionCount = instance->module().moduleInformation().importFunctionCount();
327
MacroAssemblerCodePtr<WasmEntryPtrTag> codePtr;
329
if (functionIndex < importFunctionCount) {
330
Wasm::Instance::ImportFunctionInfo* functionInfo = instance->importFunctionInfo(functionIndex);
331
if (functionInfo->targetInstance) {
332
// target is a wasm function from a different instance
333
codePtr = instance->codeBlock()->wasmToWasmExitStub(functionIndex);
336
codePtr = functionInfo->wasmToEmbedderStub;
339
// Target is a wasm function within the same instance
340
codePtr = *instance->codeBlock()->entrypointLoadLocationFromFunctionIndexSpace(functionIndex);
343
WASM_CALL_RETURN(instance, codePtr.executableAddress(), WasmEntryPtrTag);
346
WASM_SLOW_PATH_DECL(call)
348
UNUSED_PARAM(callFrame);
350
auto instruction = pc->as<WasmCall, WasmOpcodeTraits>();
351
return doWasmCall(instance, instruction.m_functionIndex);
354
WASM_SLOW_PATH_DECL(call_no_tls)
356
UNUSED_PARAM(callFrame);
358
auto instruction = pc->as<WasmCallNoTls, WasmOpcodeTraits>();
359
return doWasmCall(instance, instruction.m_functionIndex);
362
inline SlowPathReturnType doWasmCallIndirect(CallFrame* callFrame, Wasm::Instance* instance, unsigned functionIndex, unsigned tableIndex, unsigned signatureIndex)
364
Wasm::FuncRefTable* table = instance->table(tableIndex)->asFuncrefTable();
366
if (functionIndex >= table->length())
367
WASM_THROW(Wasm::ExceptionType::OutOfBoundsCallIndirect);
369
Wasm::Instance* targetInstance = table->instance(functionIndex);
370
const Wasm::WasmToWasmImportableFunction& function = table->function(functionIndex);
372
if (function.signatureIndex == Wasm::Signature::invalidIndex)
373
WASM_THROW(Wasm::ExceptionType::NullTableEntry);
375
const Wasm::Signature& callSignature = CODE_BLOCK()->signature(signatureIndex);
376
if (function.signatureIndex != Wasm::SignatureInformation::get(callSignature))
377
WASM_THROW(Wasm::ExceptionType::BadSignature);
379
if (targetInstance != instance)
380
targetInstance->setCachedStackLimit(instance->cachedStackLimit());
382
WASM_CALL_RETURN(targetInstance, function.entrypointLoadLocation->executableAddress(), WasmEntryPtrTag);
385
WASM_SLOW_PATH_DECL(call_indirect)
387
auto instruction = pc->as<WasmCallIndirect, WasmOpcodeTraits>();
388
unsigned functionIndex = READ(instruction.m_functionIndex).unboxedInt32();
389
return doWasmCallIndirect(callFrame, instance, functionIndex, instruction.m_tableIndex, instruction.m_signatureIndex);
392
WASM_SLOW_PATH_DECL(call_indirect_no_tls)
394
auto instruction = pc->as<WasmCallIndirectNoTls, WasmOpcodeTraits>();
395
unsigned functionIndex = READ(instruction.m_functionIndex).unboxedInt32();
396
return doWasmCallIndirect(callFrame, instance, functionIndex, instruction.m_tableIndex, instruction.m_signatureIndex);
399
WASM_SLOW_PATH_DECL(set_global_ref)
401
auto instruction = pc->as<WasmSetGlobalRef, WasmOpcodeTraits>();
402
instance->setGlobal(instruction.m_globalIndex, READ(instruction.m_value).jsValue());
406
WASM_SLOW_PATH_DECL(set_global_ref_portable_binding)
408
auto instruction = pc->as<WasmSetGlobalRefPortableBinding, WasmOpcodeTraits>();
409
instance->setGlobal(instruction.m_globalIndex, READ(instruction.m_value).jsValue());
413
extern "C" SlowPathReturnType slow_path_wasm_throw_exception(CallFrame* callFrame, const Instruction* pc, Wasm::Instance* instance, Wasm::ExceptionType exceptionType)
416
WASM_RETURN_TWO(operationWasmToJSException(callFrame, exceptionType, instance), 0);
419
extern "C" SlowPathReturnType slow_path_wasm_popcount(const Instruction* pc, uint32_t x)
421
void* result = bitwise_cast<void*>(static_cast<uint64_t>(__builtin_popcount(x)));
422
WASM_RETURN_TWO(pc, result);
425
extern "C" SlowPathReturnType slow_path_wasm_popcountll(const Instruction* pc, uint64_t x)
427
void* result = bitwise_cast<void*>(static_cast<uint64_t>(__builtin_popcountll(x)));
428
WASM_RETURN_TWO(pc, result);
431
} } // namespace JSC::LLInt
433
#endif // ENABLE(WEBASSEMBLY)
added
removed
Source/JavaScriptCore/wasm/WasmSlowPaths.cpp
