Bug 1737733 - Update rlbox wasm configuration to use direct function calls r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D128566

config/recurse.mk

@@ -217,6 +217,7 @@ endif
 ifdef MOZ_USING_WASM_SANDBOXING
 security/rlbox/target-objects: config/external/wasm2c_sandbox_compiler/host
 security/rlbox/target: security/rlbox/target-objects
+dom/media/ogg/target-objects extensions/spellcheck/hunspell/glue/target-objects gfx/thebes/target-objects: security/rlbox/target-objects
 endif
 
 # Most things are built during compile (target/host), but some things happen during export

dom/media/ogg/OggDemuxer.cpp

@@ -89,8 +89,7 @@ OggDemuxer::nsAutoOggSyncState::~nsAutoOggSyncState() {
 rlbox_sandbox_ogg* OggDemuxer::CreateSandbox() {
   rlbox_sandbox_ogg* sandbox = new rlbox_sandbox_ogg();
 #ifdef MOZ_WASM_SANDBOXING_OGG
-  bool success = sandbox->create_sandbox(
+  bool success = sandbox->create_sandbox(false /* infallible */);
-      mozilla::ipc::GetSandboxedRLBoxPath().get(), false /* infallible */);
 #else
   bool success = sandbox->create_sandbox();
 #endif

dom/media/ogg/OggRLBox.h

@@ -12,9 +12,12 @@
 #include "mozilla/rlbox/rlbox_config.h"
 
 #ifdef MOZ_WASM_SANDBOXING_OGG
+// Include the generated header file so that we are able to resolve the symbols
+// in the wasm binary
+#  include "rlbox.wasm.h"
+#  define RLBOX_USE_STATIC_CALLS() rlbox_wasm2c_sandbox_lookup_symbol
 #  include "mozilla/rlbox/rlbox_wasm2c_sandbox.hpp"
 #else
-// Extra configuration for no-op sandbox
 #  define RLBOX_USE_STATIC_CALLS() rlbox_noop_sandbox_lookup_symbol
 #  include "mozilla/rlbox/rlbox_noop_sandbox.hpp"
 #endif

dom/media/ogg/moz.build

@@ -24,6 +24,8 @@ UNIFIED_SOURCES += [
     "OpusParser.cpp",
 ]
 
+LOCAL_INCLUDES += ["!/security/rlbox"]
+
 FINAL_LIBRARY = "xul"
 
 # Add libFuzzer configuration directives

extensions/spellcheck/hunspell/glue/RLBoxHunspell.cpp

@@ -41,11 +41,7 @@ RLBoxHunspell::RLBoxHunspell(const nsAutoCString& affpath,
                              const nsAutoCString& dpath)
     : mHandle(nullptr) {
   MOZ_DIAGNOSTIC_ASSERT(NS_IsMainThread());
-#ifdef MOZ_WASM_SANDBOXING_HUNSPELL
-  mSandbox.create_sandbox(mozilla::ipc::GetSandboxedRLBoxPath().get());
-#else
   mSandbox.create_sandbox();
-#endif
 
   // Add the aff and dict files to allow list
   if (!affpath.IsEmpty()) {

extensions/spellcheck/hunspell/glue/RLBoxHunspell.h

@@ -12,9 +12,12 @@
 #include "mozilla/rlbox/rlbox_config.h"
 
 #ifdef MOZ_WASM_SANDBOXING_HUNSPELL
+// Include the generated header file so that we are able to resolve the symbols
+// in the wasm binary
+#  include "rlbox.wasm.h"
+#  define RLBOX_USE_STATIC_CALLS() rlbox_wasm2c_sandbox_lookup_symbol
 #  include "mozilla/rlbox/rlbox_wasm2c_sandbox.hpp"
 #else
-// Extra configuration for no-op sandbox
 #  define RLBOX_USE_STATIC_CALLS() rlbox_noop_sandbox_lookup_symbol
 #  include "mozilla/rlbox/rlbox_noop_sandbox.hpp"
 #endif

extensions/spellcheck/hunspell/glue/moz.build

@@ -18,6 +18,7 @@ DEFINES["HUNSPELL_STATIC"] = True
 FINAL_LIBRARY = "xul"
 
 LOCAL_INCLUDES += [
+    "!/security/rlbox",
     "../src",
     "/dom/base",
     "/extensions/spellcheck/src",

gfx/thebes/ThebesRLBox.h

@@ -12,9 +12,12 @@
 #include "mozilla/rlbox/rlbox_config.h"
 
 #ifdef MOZ_WASM_SANDBOXING_GRAPHITE
+// Include the generated header file so that we are able to resolve the symbols
+// in the wasm binary
+#  include "rlbox.wasm.h"
+#  define RLBOX_USE_STATIC_CALLS() rlbox_wasm2c_sandbox_lookup_symbol
 #  include "mozilla/rlbox/rlbox_wasm2c_sandbox.hpp"
 #else
-// Extra configuration for no-op sandbox
 #  define RLBOX_USE_STATIC_CALLS() rlbox_noop_sandbox_lookup_symbol
 #  include "mozilla/rlbox/rlbox_noop_sandbox.hpp"
 #endif

gfx/thebes/gfxFontEntry.cpp

@@ -631,11 +631,7 @@ struct gfxFontEntry::GrSandboxData {
       grGetGlyphAdvanceCallback;
 
   GrSandboxData() {
-#ifdef MOZ_WASM_SANDBOXING_GRAPHITE
-    sandbox.create_sandbox(mozilla::ipc::GetSandboxedRLBoxPath().get());
-#else
     sandbox.create_sandbox();
-#endif
     grGetTableCallback = sandbox.register_callback(GrGetTable);
     grReleaseTableCallback = sandbox.register_callback(GrReleaseTable);
     grGetGlyphAdvanceCallback =

gfx/thebes/moz.build

@@ -257,6 +257,7 @@ include("/ipc/chromium/chromium-config.mozbuild")
 FINAL_LIBRARY = "xul"
 
 LOCAL_INCLUDES += [
+    "!/security/rlbox",
     "/dom/base",
     "/dom/media/platforms/apple",
     "/dom/xml",

security/rlbox/moz.build

@@ -20,9 +20,10 @@ SOURCES += [
     "/third_party/wasm2c/wasm2c/wasm-rt-wasi.c",
 ]
 
-FINAL_LIBRARY = "xul"
+# Configuration that removes the wasm2c functions from shared library exports
+DEFINES["WASM_DONT_EXPORT_FUNCS"] = True
 
-NoVisibilityFlags()
+FINAL_LIBRARY = "xul"
 
 # Allow compiler warnings both in wasm compiles (because they're third-party
 # libraries), and C compiles (because they're code generated by wasm2c).