Bug 1914963 - Combine wasmMarkSlowCall and call(reg). r=jseward, a=dsmith

Differential Revision: https://phabricator.services.mozilla.com/D220051

js/src/jit/MacroAssembler.cpp

@@ -5395,7 +5395,7 @@ static ReturnCallTrampolineData MakeReturnCallTrampoline(MacroAssembler& masm) {
         wasm::FrameWithInstances::sizeOfInstanceFieldsAndShadowStack(),
         WasmStackAlignment));
 
-    masm.wasmMarkSlowCall();
+    masm.wasmMarkCallAsSlow();
   }
 
   masm.loadPtr(
@@ -5743,9 +5743,10 @@ CodeOffset MacroAssembler::wasmCallImport(const wasm::CallSiteDesc& desc,
            Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
   loadWasmPinnedRegsFromInstance();
 
-  CodeOffset res = call(desc, ABINonArgReg0);
 #ifdef ENABLE_WASM_TAIL_CALLS
-  wasmMarkSlowCall();
+  CodeOffset res = wasmMarkedSlowCall(desc, ABINonArgReg0);
+#else
+  CodeOffset res = call(desc, ABINonArgReg0);
 #endif
   return res;
 }
@@ -6009,9 +6010,10 @@ void MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc,
   loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, code)),
           calleeScratch);
 
-  *slowCallOffset = call(desc, calleeScratch);
 #ifdef ENABLE_WASM_TAIL_CALLS
-  wasmMarkSlowCall();
+  *slowCallOffset = wasmMarkedSlowCall(desc, calleeScratch);
+#else
+  *slowCallOffset = call(desc, calleeScratch);
 #endif
 
   // Restore registers and realm and join up with the fast path.
@@ -6202,9 +6204,10 @@ void MacroAssembler::wasmCallRef(const wasm::CallSiteDesc& desc,
       FunctionExtended::WASM_FUNC_UNCHECKED_ENTRY_SLOT);
   loadPtr(Address(calleeFnObj, uncheckedEntrySlotOffset), calleeScratch);
 
-  *slowCallOffset = call(desc, calleeScratch);
 #ifdef ENABLE_WASM_TAIL_CALLS
-  wasmMarkSlowCall();
+  *slowCallOffset = wasmMarkedSlowCall(desc, calleeScratch);
+#else
+  *slowCallOffset = call(desc, calleeScratch);
 #endif
 
   // Restore registers and realm and back to this caller's.

js/src/jit/MacroAssembler.h

@@ -3900,8 +3900,14 @@ class MacroAssembler : public MacroAssemblerSpecific {
                              Register temp2)
       DEFINED_ON(x86, x64, arm, arm64, loong64, mips64, riscv64);
 
-  void wasmMarkSlowCall()
+  // Places slow class marker for tail calls.
+  void wasmMarkCallAsSlow()
       DEFINED_ON(x86, x64, arm, arm64, loong64, mips64, riscv64);
+
+  // Combines slow class marker with actual assembler call.
+  CodeOffset wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                const Register reg)
+      DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64);
 #endif
 
   // WasmTableCallIndexReg must contain the index of the indirect call.  This is

js/src/jit/arm/MacroAssembler-arm.cpp

@@ -6090,7 +6090,7 @@ void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() { ma_and(lr, lr, lr); }
+void MacroAssembler::wasmMarkCallAsSlow() { ma_and(lr, lr, lr); }
 
 const int32_t SlowCallMarker = 0xe00ee00e;
 
@@ -6103,6 +6103,14 @@ void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
   ma_cmp(temp2, temp1);
   j(Assembler::NotEqual, notSlow);
 }
+
+CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                              const Register reg) {
+  AutoForbidPoolsAndNops afp(this, 2);
+  CodeOffset offset = call(desc, reg);
+  wasmMarkCallAsSlow();
+  return offset;
+}
 #endif  // ENABLE_WASM_TAIL_CALLS
 
 //}}} check_macroassembler_style

js/src/jit/arm64/MacroAssembler-arm64.cpp

@@ -3406,7 +3406,7 @@ void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() { Mov(x28, x28); }
+void MacroAssembler::wasmMarkCallAsSlow() { Mov(x28, x28); }
 
 const int32_t SlowCallMarker = 0xaa1c03fc;
 
@@ -3417,6 +3417,14 @@ void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
   Cmp(W(temp2), Operand(SlowCallMarker));
   B(Assembler::NotEqual, notSlow);
 }
+
+CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                              const Register reg) {
+  AutoForbidPoolsAndNops afp(this, !GetStackPointer64().Is(vixl::sp) ? 3 : 2);
+  CodeOffset offset = call(desc, reg);
+  wasmMarkCallAsSlow();
+  return offset;
+}
 #endif  // ENABLE_WASM_TAIL_CALLS
 
 //}}} check_macroassembler_style

js/src/jit/loong64/MacroAssembler-loong64.cpp

@@ -5525,7 +5525,7 @@ void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() { mov(ra, ra); }
+void MacroAssembler::wasmMarkCallAsSlow() { mov(ra, ra); }
 
 const int32_t SlowCallMarker = 0x03800021;  // ori ra, ra, 0
 
@@ -5535,6 +5535,13 @@ void MacroAssembler::wasmCheckSlowCallsite(Register ra_, Label* notSlow,
   load32(Address(ra_, 0), temp2);
   branch32(Assembler::NotEqual, temp2, Imm32(SlowCallMarker), notSlow);
 }
+
+CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                              const Register reg) {
+  CodeOffset offset = call(desc, reg);
+  wasmMarkCallAsSlow();
+  return offset;
+}
 #endif  // ENABLE_WASM_TAIL_CALLS
 
 //}}} check_macroassembler_style

js/src/jit/mips64/MacroAssembler-mips64.cpp

@@ -2856,7 +2856,7 @@ void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest,
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() { mov(ra, ra); }
+void MacroAssembler::wasmMarkCallAsSlow() { mov(ra, ra); }
 
 const int32_t SlowCallMarker = 0x37ff0000;  // ori ra, ra, 0
 
@@ -2866,6 +2866,13 @@ void MacroAssembler::wasmCheckSlowCallsite(Register ra_, Label* notSlow,
   load32(Address(ra_, 0), temp2);
   branch32(Assembler::NotEqual, temp2, Imm32(SlowCallMarker), notSlow);
 }
+
+CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                              const Register reg) {
+  CodeOffset offset = call(desc, reg);
+  wasmMarkCallAsSlow();
+  return offset;
+}
 #endif  // ENABLE_WASM_TAIL_CALLS
 
 //}}} check_macroassembler_style

js/src/jit/riscv64/MacroAssembler-riscv64.cpp

@@ -4330,7 +4330,7 @@ void MacroAssembler::widenInt32(Register r) {
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() { mv(ra, ra); }
+void MacroAssembler::wasmMarkCallAsSlow() { mv(ra, ra); }
 
 const int32_t SlowCallMarker = 0x8093;  // addi ra, ra, 0
 
@@ -4340,6 +4340,14 @@ void MacroAssembler::wasmCheckSlowCallsite(Register ra_, Label* notSlow,
   load32(Address(ra_, 0), temp2);
   branch32(Assembler::NotEqual, temp2, Imm32(SlowCallMarker), notSlow);
 }
+
+CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                              const Register reg) {
+  BlockTrampolinePoolScope block_trampoline_pool(this, 2);
+  CodeOffset offset = call(desc, reg);
+  wasmMarkCallAsSlow();
+  return offset;
+}
 #endif  // ENABLE_WASM_TAIL_CALLS
 //}}} check_macroassembler_style
 

js/src/jit/x64/MacroAssembler-x64.cpp

@@ -1654,7 +1654,7 @@ void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() {
+void MacroAssembler::wasmMarkCallAsSlow() {
   static_assert(InstanceReg == r14);
   orPtr(Imm32(0), r14);
 }

js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp

@@ -2151,4 +2151,13 @@ void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
   addPtr(indexTemp32, pointer);
 }
 
+#ifdef ENABLE_WASM_TAIL_CALLS
+CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
+                                              const Register reg) {
+  CodeOffset offset = call(desc, reg);
+  wasmMarkCallAsSlow();
+  return offset;
+}
+#endif  // ENABLE_WASM_TAIL_CALLS
+
 //}}} check_macroassembler_style

js/src/jit/x86/MacroAssembler-x86.cpp

@@ -1877,7 +1877,7 @@ void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
 }
 
 #ifdef ENABLE_WASM_TAIL_CALLS
-void MacroAssembler::wasmMarkSlowCall() {
+void MacroAssembler::wasmMarkCallAsSlow() {
   static_assert(esi == InstanceReg);
   or32(esi, esi);
 }

js/src/jsapi-tests/testWasmReturnCalls.cpp

@@ -17,7 +17,7 @@ using namespace js::jit;
 
 #if defined(ENABLE_WASM_TAIL_CALLS) && !defined(JS_CODEGEN_NONE)
 
-// Check if wasmMarkSlowCall produces the byte sequence that can
+// Check if wasmMarkCallAsSlow produces the byte sequence that can
 // wasmCheckSlowCallsite detect.
 BEGIN_TEST(testWasmCheckSlowCallMarkerHit) {
   js::LifoAlloc lifo(4096);
@@ -30,7 +30,7 @@ BEGIN_TEST(testWasmCheckSlowCallMarkerHit) {
 
   Label check, fail, end;
   masm.call(&check);
-  masm.wasmMarkSlowCall();
+  masm.wasmMarkCallAsSlow();
   masm.jump(&end);
 
   masm.bind(&check);