Bug 1815538: part 6 - Emit an entry trampoline for each script that enters the C++ interpreter r=iain

Differential Revision: https://phabricator.services.mozilla.com/D172635

js/src/jit/ABIFunctionList-inl.h

@@ -102,6 +102,7 @@ namespace jit {
   _(js_free)                                                          \
   _(js::hypot3)                                                       \
   _(js::hypot4)                                                       \
+  _(js::Interpret)                                                    \
   _(js::Int32ToStringPure)                                            \
   _(js::irregexp::CaseInsensitiveCompareNonUnicode)                   \
   _(js::irregexp::CaseInsensitiveCompareUnicode)                      \

js/src/jit/InterpreterEntryTrampoline.cpp

@@ -7,6 +7,7 @@
 #include "jit/InterpreterEntryTrampoline.h"
 #include "jit/JitRuntime.h"
 #include "jit/Linker.h"
+#include "vm/Interpreter.h"
 
 #include "gc/Marking-inl.h"
 #include "jit/MacroAssembler-inl.h"
@@ -143,6 +144,54 @@ void JitRuntime::generateBaselineInterpreterEntryTrampoline(
   masm.ret();
 }
 
+void JitRuntime::generateInterpreterEntryTrampoline(MacroAssembler& masm) {
+  AutoCreatedBy acb(masm, "JitRuntime::generateInterpreterEntryTrampoline");
+
+  // If BLI is disabled, we don't need an offset.
+  if (IsBaselineInterpreterEnabled()) {
+    uint32_t offset = startTrampolineCode(masm);
+    if (!vmInterpreterEntryOffset_) {
+      vmInterpreterEntryOffset_ = offset;
+    }
+  }
+
+  masm.push(FramePointer);
+  masm.moveStackPtrTo(FramePointer);
+
+  AllocatableRegisterSet regs(RegisterSet::Volatile());
+  LiveRegisterSet save(regs.asLiveSet());
+  masm.PushRegsInMask(save);
+
+#ifdef JS_CODEGEN_X86
+  Register temp0 = regs.takeAnyGeneral();
+  Register temp1 = regs.takeAnyGeneral();
+  Address cxAddr(FramePointer, 2 * sizeof(void*));
+  Address stateAddr(FramePointer, 3 * sizeof(void*));
+  masm.loadPtr(cxAddr, temp0);
+  masm.loadPtr(stateAddr, temp1);
+#else
+  Register temp0 = IntArgReg0;
+  Register temp1 = IntArgReg1;
+#endif
+
+  Register scratch = regs.takeAnyGeneral();
+
+  using Fn = bool (*)(JSContext * cx, js::RunState & state);
+  masm.setupUnalignedABICall(scratch);
+  masm.passABIArg(temp0);  // cx
+  masm.passABIArg(temp1);  // state
+  masm.callWithABI<Fn, Interpret>(
+      MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+  LiveRegisterSet ignore;
+  ignore.add(ReturnReg);
+  masm.PopRegsInMaskIgnore(save, ignore);
+
+  masm.moveToStackPtr(FramePointer);
+  masm.pop(FramePointer);
+  masm.ret();
+}
+
 JitCode* JitRuntime::generateEntryTrampolineForScript(JSContext* cx,
                                                       JSScript* script) {
   if (JitSpewEnabled(JitSpew_Codegen)) {
@@ -162,8 +211,13 @@ JitCode* JitRuntime::generateEntryTrampolineForScript(JSContext* cx,
   StackMacroAssembler masm(cx, temp);
   PerfSpewerRangeRecorder rangeRecorder(masm);
 
-  generateBaselineInterpreterEntryTrampoline(masm);
-  rangeRecorder.recordOffset("BaselineInterpreter", cx, script);
+  if (IsBaselineInterpreterEnabled()) {
+    generateBaselineInterpreterEntryTrampoline(masm);
+    rangeRecorder.recordOffset("BaselineInterpreter", cx, script);
+  }
+
+  generateInterpreterEntryTrampoline(masm);
+  rangeRecorder.recordOffset("Interpreter", cx, script);
 
   Linker linker(masm);
   JitCode* code = linker.newCode(cx, CodeKind::Other);

js/src/jit/Jit.cpp

@@ -128,6 +128,14 @@ static EnterJitStatus JS_HAZ_JSNATIVE_CALLER EnterJit(JSContext* cx,
   return EnterJitStatus::Ok;
 }
 
+// Call the per-script interpreter entry trampoline.
+bool js::jit::EnterInterpreterEntryTrampoline(uint8_t* code, JSContext* cx,
+                                              RunState* state) {
+  using EnterTrampolineCodePtr = bool (*)(JSContext * cx, RunState*);
+  auto funcPtr = JS_DATA_TO_FUNC_PTR(EnterTrampolineCodePtr, code);
+  return CALL_GENERATED_2(funcPtr, cx, state);
+}
+
 EnterJitStatus js::jit::MaybeEnterJit(JSContext* cx, RunState& state) {
   if (!IsBaselineInterpreterEnabled()) {
     // All JITs are disabled.

js/src/jit/Jit.h

@@ -31,6 +31,8 @@ enum class EnterJitStatus {
   NotEntered,
 };
 
+extern bool EnterInterpreterEntryTrampoline(uint8_t* code, JSContext* cx,
+                                            RunState* state);
 extern EnterJitStatus MaybeEnterJit(JSContext* cx, RunState& state);
 
 }  // namespace jit

js/src/jit/JitRuntime.h

@@ -179,6 +179,10 @@ class JitRuntime {
   // Code for trampolines and VMFunction wrappers.
   WriteOnceData<JitCode*> trampolineCode_{nullptr};
 
+  // Thunk that calls into the C++ interpreter from the interpreter
+  // entry trampoline that is generated with --emit-interpreter-entry
+  WriteOnceData<uint32_t> vmInterpreterEntryOffset_{0};
+
   // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_.
   using VMWrapperOffsets = Vector<uint32_t, 0, SystemAllocPolicy>;
   VMWrapperOffsets functionWrapperOffsets_;
@@ -265,6 +269,7 @@ class JitRuntime {
   }
 
   void generateBaselineInterpreterEntryTrampoline(MacroAssembler& masm);
+  void generateInterpreterEntryTrampoline(MacroAssembler& masm);
 
  public:
   JitCode* generateEntryTrampolineForScript(JSContext* cx, JSScript* script);
@@ -329,6 +334,8 @@ class JitRuntime {
     return trampolineCode(argumentsRectifierOffset_);
   }
 
+  uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_; }
+
   TrampolinePtr getArgumentsRectifierReturnAddr() const {
     return trampolineCode(argumentsRectifierReturnOffset_);
   }

js/src/vm/Interpreter.cpp

@@ -354,9 +354,6 @@ static bool AddRecordSpreadOperation(JSContext* cx, HandleValue recHandle,
 }
 #endif
 
-static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx,
-                                                              RunState& state);
-
 InterpreterFrame* InvokeState::pushInterpreterFrame(JSContext* cx) {
   return cx->interpreterStack().pushInvokeFrame(cx, args_, construct_);
 }
@@ -373,6 +370,36 @@ InterpreterFrame* RunState::pushInterpreterFrame(JSContext* cx) {
   return asExecute()->pushInterpreterFrame(cx);
 }
 
+static MOZ_ALWAYS_INLINE bool MaybeEnterInterpreterTrampoline(JSContext* cx,
+                                                              RunState& state) {
+#ifdef NIGHTLY_BUILD
+  if (jit::JitOptions.emitInterpreterEntryTrampoline &&
+      cx->runtime()->hasJitRuntime()) {
+    js::jit::JitRuntime* jitRuntime = cx->runtime()->jitRuntime();
+    JSScript* script = state.script();
+
+    uint8_t* codeRaw = nullptr;
+    auto p = jitRuntime->getInterpreterEntryMap()->lookup(script);
+    if (p) {
+      codeRaw = p->value().raw();
+    } else if (js::jit::JitCode* code =
+                   jitRuntime->generateEntryTrampolineForScript(cx, script)) {
+      js::jit::EntryTrampoline entry(cx, code);
+      if (!jitRuntime->getInterpreterEntryMap()->put(script, entry)) {
+        return false;
+      }
+      codeRaw = code->raw();
+    }
+
+    MOZ_ASSERT(codeRaw, "Should have a valid trampoline here.");
+    // The C++ entry thunk is located at the vmInterpreterEntryOffset offset.
+    codeRaw += jitRuntime->vmInterpreterEntryOffset();
+    return js::jit::EnterInterpreterEntryTrampoline(codeRaw, cx, &state);
+  }
+#endif
+  return Interpret(cx, state);
+}
+
 // MSVC with PGO inlines a lot of functions in RunScript, resulting in large
 // stack frames and stack overflow issues, see bug 1167883. Turn off PGO to
 // avoid this.
@@ -428,7 +455,7 @@ bool js::RunScript(JSContext* cx, RunState& state) {
       break;
   }
 
-  bool ok = Interpret(cx, state);
+  bool ok = MaybeEnterInterpreterTrampoline(cx, state);
 
   return ok;
 }
@@ -1954,8 +1981,8 @@ void js::ReportInNotObjectError(JSContext* cx, HandleValue lref,
                             InformalValueTypeName(rref));
 }
 
-static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx,
-                                                              RunState& state) {
+bool MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER js::Interpret(JSContext* cx,
+                                                           RunState& state) {
 /*
  * Define macros for an interpreter loop. Opcode dispatch is done by
  * indirect goto (aka a threaded interpreter), which is technically
@@ -3416,6 +3443,21 @@ static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx,
             case jit::EnterJitStatus::NotEntered:
               break;
           }
+
+#ifdef NIGHTLY_BUILD
+          // If entry trampolines are enabled, call back into
+          // MaybeEnterInterpreterTrampoline so we can generate an
+          // entry trampoline for the new frame.
+          if (jit::JitOptions.emitInterpreterEntryTrampoline) {
+            if (MaybeEnterInterpreterTrampoline(cx, state)) {
+              interpReturnOK = true;
+              CHECK_BRANCH();
+              REGS.sp = args.spAfterCall();
+              goto jit_return;
+            }
+            goto error;
+          }
+#endif
         }
 
         funScript = fun->nonLazyScript();

js/src/vm/Interpreter.h

@@ -298,6 +298,7 @@ inline void RunState::setReturnValue(const Value& v) {
 }
 
 extern bool RunScript(JSContext* cx, RunState& state);
+extern bool Interpret(JSContext* cx, RunState& state);
 
 extern JSType TypeOfObject(JSObject* obj);