From 079f16abdef47b756cbf6fe2d8ccc0633c56e9e0 Mon Sep 17 00:00:00 2001 From: Denis Palmeiro Date: Thu, 6 Apr 2023 12:05:36 +0000 Subject: [PATCH] Bug 1815538: part 6 - Emit an entry trampoline for each script that enters the C++ interpreter r=iain Differential Revision: https://phabricator.services.mozilla.com/D172635 --- js/src/jit/ABIFunctionList-inl.h | 1 + js/src/jit/InterpreterEntryTrampoline.cpp | 58 ++++++++++++++++++++++- js/src/jit/Jit.cpp | 8 ++++ js/src/jit/Jit.h | 2 + js/src/jit/JitRuntime.h | 7 +++ js/src/vm/Interpreter.cpp | 54 ++++++++++++++++++--- js/src/vm/Interpreter.h | 1 + 7 files changed, 123 insertions(+), 8 deletions(-) diff --git a/js/src/jit/ABIFunctionList-inl.h b/js/src/jit/ABIFunctionList-inl.h index 20e6e627aee7..1c30419fab33 100644 --- a/js/src/jit/ABIFunctionList-inl.h +++ b/js/src/jit/ABIFunctionList-inl.h @@ -102,6 +102,7 @@ namespace jit { _(js_free) \ _(js::hypot3) \ _(js::hypot4) \ + _(js::Interpret) \ _(js::Int32ToStringPure) \ _(js::irregexp::CaseInsensitiveCompareNonUnicode) \ _(js::irregexp::CaseInsensitiveCompareUnicode) \ diff --git a/js/src/jit/InterpreterEntryTrampoline.cpp b/js/src/jit/InterpreterEntryTrampoline.cpp index e463b9fd71cf..eb38d7cc4f48 100644 --- a/js/src/jit/InterpreterEntryTrampoline.cpp +++ b/js/src/jit/InterpreterEntryTrampoline.cpp @@ -7,6 +7,7 @@ #include "jit/InterpreterEntryTrampoline.h" #include "jit/JitRuntime.h" #include "jit/Linker.h" +#include "vm/Interpreter.h" #include "gc/Marking-inl.h" #include "jit/MacroAssembler-inl.h" @@ -143,6 +144,54 @@ void JitRuntime::generateBaselineInterpreterEntryTrampoline( masm.ret(); } +void JitRuntime::generateInterpreterEntryTrampoline(MacroAssembler& masm) { + AutoCreatedBy acb(masm, "JitRuntime::generateInterpreterEntryTrampoline"); + + // If BLI is disabled, we don't need an offset. + if (IsBaselineInterpreterEnabled()) { + uint32_t offset = startTrampolineCode(masm); + if (!vmInterpreterEntryOffset_) { + vmInterpreterEntryOffset_ = offset; + } + } + + masm.push(FramePointer); + masm.moveStackPtrTo(FramePointer); + + AllocatableRegisterSet regs(RegisterSet::Volatile()); + LiveRegisterSet save(regs.asLiveSet()); + masm.PushRegsInMask(save); + +#ifdef JS_CODEGEN_X86 + Register temp0 = regs.takeAnyGeneral(); + Register temp1 = regs.takeAnyGeneral(); + Address cxAddr(FramePointer, 2 * sizeof(void*)); + Address stateAddr(FramePointer, 3 * sizeof(void*)); + masm.loadPtr(cxAddr, temp0); + masm.loadPtr(stateAddr, temp1); +#else + Register temp0 = IntArgReg0; + Register temp1 = IntArgReg1; +#endif + + Register scratch = regs.takeAnyGeneral(); + + using Fn = bool (*)(JSContext * cx, js::RunState & state); + masm.setupUnalignedABICall(scratch); + masm.passABIArg(temp0); // cx + masm.passABIArg(temp1); // state + masm.callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + LiveRegisterSet ignore; + ignore.add(ReturnReg); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.moveToStackPtr(FramePointer); + masm.pop(FramePointer); + masm.ret(); +} + JitCode* JitRuntime::generateEntryTrampolineForScript(JSContext* cx, JSScript* script) { if (JitSpewEnabled(JitSpew_Codegen)) { @@ -162,8 +211,13 @@ JitCode* JitRuntime::generateEntryTrampolineForScript(JSContext* cx, StackMacroAssembler masm(cx, temp); PerfSpewerRangeRecorder rangeRecorder(masm); - generateBaselineInterpreterEntryTrampoline(masm); - rangeRecorder.recordOffset("BaselineInterpreter", cx, script); + if (IsBaselineInterpreterEnabled()) { + generateBaselineInterpreterEntryTrampoline(masm); + rangeRecorder.recordOffset("BaselineInterpreter", cx, script); + } + + generateInterpreterEntryTrampoline(masm); + rangeRecorder.recordOffset("Interpreter", cx, script); Linker linker(masm); JitCode* code = linker.newCode(cx, CodeKind::Other); diff --git a/js/src/jit/Jit.cpp b/js/src/jit/Jit.cpp index 554de34475c7..4566fc013e0a 100644 --- a/js/src/jit/Jit.cpp +++ b/js/src/jit/Jit.cpp @@ -128,6 +128,14 @@ static EnterJitStatus JS_HAZ_JSNATIVE_CALLER EnterJit(JSContext* cx, return EnterJitStatus::Ok; } +// Call the per-script interpreter entry trampoline. +bool js::jit::EnterInterpreterEntryTrampoline(uint8_t* code, JSContext* cx, + RunState* state) { + using EnterTrampolineCodePtr = bool (*)(JSContext * cx, RunState*); + auto funcPtr = JS_DATA_TO_FUNC_PTR(EnterTrampolineCodePtr, code); + return CALL_GENERATED_2(funcPtr, cx, state); +} + EnterJitStatus js::jit::MaybeEnterJit(JSContext* cx, RunState& state) { if (!IsBaselineInterpreterEnabled()) { // All JITs are disabled. diff --git a/js/src/jit/Jit.h b/js/src/jit/Jit.h index bd2b9b726a99..d9632513b69c 100644 --- a/js/src/jit/Jit.h +++ b/js/src/jit/Jit.h @@ -31,6 +31,8 @@ enum class EnterJitStatus { NotEntered, }; +extern bool EnterInterpreterEntryTrampoline(uint8_t* code, JSContext* cx, + RunState* state); extern EnterJitStatus MaybeEnterJit(JSContext* cx, RunState& state); } // namespace jit diff --git a/js/src/jit/JitRuntime.h b/js/src/jit/JitRuntime.h index 1d41ea046ab7..39c7019e2f2b 100644 --- a/js/src/jit/JitRuntime.h +++ b/js/src/jit/JitRuntime.h @@ -179,6 +179,10 @@ class JitRuntime { // Code for trampolines and VMFunction wrappers. WriteOnceData trampolineCode_{nullptr}; + // Thunk that calls into the C++ interpreter from the interpreter + // entry trampoline that is generated with --emit-interpreter-entry + WriteOnceData vmInterpreterEntryOffset_{0}; + // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_. using VMWrapperOffsets = Vector; VMWrapperOffsets functionWrapperOffsets_; @@ -265,6 +269,7 @@ class JitRuntime { } void generateBaselineInterpreterEntryTrampoline(MacroAssembler& masm); + void generateInterpreterEntryTrampoline(MacroAssembler& masm); public: JitCode* generateEntryTrampolineForScript(JSContext* cx, JSScript* script); @@ -329,6 +334,8 @@ class JitRuntime { return trampolineCode(argumentsRectifierOffset_); } + uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_; } + TrampolinePtr getArgumentsRectifierReturnAddr() const { return trampolineCode(argumentsRectifierReturnOffset_); } diff --git a/js/src/vm/Interpreter.cpp b/js/src/vm/Interpreter.cpp index d1ebd85cfcc3..5e402fd5978e 100644 --- a/js/src/vm/Interpreter.cpp +++ b/js/src/vm/Interpreter.cpp @@ -354,9 +354,6 @@ static bool AddRecordSpreadOperation(JSContext* cx, HandleValue recHandle, } #endif -static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx, - RunState& state); - InterpreterFrame* InvokeState::pushInterpreterFrame(JSContext* cx) { return cx->interpreterStack().pushInvokeFrame(cx, args_, construct_); } @@ -373,6 +370,36 @@ InterpreterFrame* RunState::pushInterpreterFrame(JSContext* cx) { return asExecute()->pushInterpreterFrame(cx); } +static MOZ_ALWAYS_INLINE bool MaybeEnterInterpreterTrampoline(JSContext* cx, + RunState& state) { +#ifdef NIGHTLY_BUILD + if (jit::JitOptions.emitInterpreterEntryTrampoline && + cx->runtime()->hasJitRuntime()) { + js::jit::JitRuntime* jitRuntime = cx->runtime()->jitRuntime(); + JSScript* script = state.script(); + + uint8_t* codeRaw = nullptr; + auto p = jitRuntime->getInterpreterEntryMap()->lookup(script); + if (p) { + codeRaw = p->value().raw(); + } else if (js::jit::JitCode* code = + jitRuntime->generateEntryTrampolineForScript(cx, script)) { + js::jit::EntryTrampoline entry(cx, code); + if (!jitRuntime->getInterpreterEntryMap()->put(script, entry)) { + return false; + } + codeRaw = code->raw(); + } + + MOZ_ASSERT(codeRaw, "Should have a valid trampoline here."); + // The C++ entry thunk is located at the vmInterpreterEntryOffset offset. + codeRaw += jitRuntime->vmInterpreterEntryOffset(); + return js::jit::EnterInterpreterEntryTrampoline(codeRaw, cx, &state); + } +#endif + return Interpret(cx, state); +} + // MSVC with PGO inlines a lot of functions in RunScript, resulting in large // stack frames and stack overflow issues, see bug 1167883. Turn off PGO to // avoid this. @@ -428,7 +455,7 @@ bool js::RunScript(JSContext* cx, RunState& state) { break; } - bool ok = Interpret(cx, state); + bool ok = MaybeEnterInterpreterTrampoline(cx, state); return ok; } @@ -1954,8 +1981,8 @@ void js::ReportInNotObjectError(JSContext* cx, HandleValue lref, InformalValueTypeName(rref)); } -static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx, - RunState& state) { +bool MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER js::Interpret(JSContext* cx, + RunState& state) { /* * Define macros for an interpreter loop. Opcode dispatch is done by * indirect goto (aka a threaded interpreter), which is technically @@ -3416,6 +3443,21 @@ static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx, case jit::EnterJitStatus::NotEntered: break; } + +#ifdef NIGHTLY_BUILD + // If entry trampolines are enabled, call back into + // MaybeEnterInterpreterTrampoline so we can generate an + // entry trampoline for the new frame. + if (jit::JitOptions.emitInterpreterEntryTrampoline) { + if (MaybeEnterInterpreterTrampoline(cx, state)) { + interpReturnOK = true; + CHECK_BRANCH(); + REGS.sp = args.spAfterCall(); + goto jit_return; + } + goto error; + } +#endif } funScript = fun->nonLazyScript(); diff --git a/js/src/vm/Interpreter.h b/js/src/vm/Interpreter.h index 550875115bcb..2635653c08f5 100644 --- a/js/src/vm/Interpreter.h +++ b/js/src/vm/Interpreter.h @@ -298,6 +298,7 @@ inline void RunState::setReturnValue(const Value& v) { } extern bool RunScript(JSContext* cx, RunState& state); +extern bool Interpret(JSContext* cx, RunState& state); extern JSType TypeOfObject(JSObject* obj);