Bug 1904243 - Check if complex types with local refs are properly validated. r=rhunt, a=dmeehan

Differential Revision: https://phabricator.services.mozilla.com/D214750

js/src/jit-test/tests/wasm/gc/regress-1904243.js

@@ -0,0 +1,18 @@
+// Check if complex types with local refs are properly validated.
+
+var ins = wasmEvalText(`(module
+  (rec
+    (type $t0 (struct))
+    (type $t1 (sub (func (result (ref null $t1)))))
+  )
+)`);
+
+var ins = wasmEvalText(`(module
+  (rec
+    (type $t0 (struct))
+    (type $t1 (sub (func (result (ref null $t0)))))
+  )
+  (func (type $t1) (result (ref null $t0))
+    ref.null $t0
+  )
+)`);

js/src/wasm/WasmTypeDef.h

@@ -1301,7 +1301,10 @@ inline uintptr_t TypeDef::forMatch(const TypeDef* typeDef,
 
   // Return a tagged index for local type references
   if (typeDef && &typeDef->recGroup() == recGroup) {
-    return uintptr_t(recGroup->indexOf(typeDef)) | 0x1;
+    // recGroup->indexOf result is expected to be not greater than MaxTypes,
+    // and shall fit in 32-bit pointer without loss.
+    static_assert(MaxTypes <= 0x7FFFFFFF);
+    return (uintptr_t(recGroup->indexOf(typeDef)) << 1) | 0x1;
   }
 
   // Return an untagged pointer for non-local type references