Bug 1509549 - Add a ProfilerGetSymbols module which can dump symbols with the help of a dynamically-loaded WebAssembly module. r=kmag

The module can dump ELF binaries, Mach-O binaries, and pdb files. So it works
for all our supported platforms.

The module is currently hosted on https://zealous-rosalind-a98ce8.netlify.com/ ,
which is a netlify server that serves files from the following repo:
https://github.com/mstange/profiler-assets

To make all of this look a bit more official, I'm planning on doing two things:
 - Move the github repo under the devtools-html organization
 - Get a firefox.com subdomain such as profiler-assets.firefox.com for hosting

Depends on D13004

Differential Revision: https://phabricator.services.mozilla.com/D13005

--HG--
extra : moz-landing-system : lando

.eslintignore

@@ -65,6 +65,8 @@ browser/components/sessionstore/test/unit/data/sessionstore_invalid.js
 # for documentation purposes (policies.json) but to be accessed by the
 # code as a .jsm (schema.jsm)
 browser/components/enterprisepolicies/schemas/schema.jsm
+# Ignore generated code from wasm-bindgen
+browser/components/extensions/profiler_get_symbols.js
 # generated & special files in cld2
 browser/components/translation/cld2/**
 # Screenshots is imported as a system add-on and has

browser/components/extensions/ProfilerGetSymbols-worker.js

@@ -0,0 +1,128 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+/* eslint-env mozilla/chrome-worker */
+
+"use strict";
+
+importScripts("resource://gre/modules/osfile.jsm",
+              "resource://app/modules/profiler_get_symbols.js");
+
+// This worker uses the wasm module that was generated from https://github.com/mstange/profiler-get-symbols.
+// See ProfilerGetSymbols.jsm for more information.
+//
+// The worker instantiates the module, reads the binary into wasm memory, runs
+// the wasm code, and returns the symbol table or an error. Then it shuts down
+// itself.
+
+// Read an open OS.File instance into the Uint8Array dataBuf.
+function readFileInto(file, dataBuf) {
+  // Ideally we'd be able to call file.readTo(dataBuf) here, but readTo no
+  // longer exists.
+  // So instead, we copy the file over into wasm memory in 4MB chunks. This
+  // will take 425 invocations for a a 1.7GB file (such as libxul.so for a
+  // Firefox for Android build) and not take up too much memory per call.
+  const dataBufLen = dataBuf.byteLength;
+  const chunkSize = 4 * 1024 * 1024;
+  let pos = 0;
+  while (pos < dataBufLen) {
+    const chunkData = file.read({bytes: chunkSize});
+    const chunkBytes = chunkData.byteLength;
+    if (chunkBytes === 0) {
+      break;
+    }
+
+    dataBuf.set(chunkData, pos);
+    pos += chunkBytes;
+  }
+}
+
+onmessage = async e => {
+  try {
+    const {binaryPath, debugPath, breakpadId, module} = e.data;
+
+    if (!(module instanceof WebAssembly.Module)) {
+      throw new Error("invalid WebAssembly module");
+    }
+
+    // Instantiate the WASM module.
+    await wasm_bindgen(module);
+
+    const {CompactSymbolTable, wasm} = wasm_bindgen;
+
+    const binaryFile = OS.File.open(binaryPath, {read: true});
+    const binaryDataBufLen = binaryFile.stat().size;
+
+    // Read the binary file into WASM memory.
+    const binaryDataBufPtr = wasm.__wbindgen_malloc(binaryDataBufLen);
+    const binaryDataBuf = new Uint8Array(wasm.memory.buffer, binaryDataBufPtr, binaryDataBufLen);
+    readFileInto(binaryFile, binaryDataBuf);
+    binaryFile.close();
+
+    // Do the same for the debug file, if it is supplied and different from the
+    // binary file. This is only the case on Windows.
+    let debugDataBufLen = binaryDataBufLen;
+    let debugDataBufPtr = binaryDataBufPtr;
+    if (debugPath && debugPath !== binaryPath) {
+      const debugFile = OS.File.open(debugPath, {read: true});
+      debugDataBufLen = debugFile.stat().size;
+      debugDataBufPtr = wasm.__wbindgen_malloc(debugDataBufLen);
+      const debugDataBuf = new Uint8Array(wasm.memory.buffer, debugDataBufPtr, debugDataBufLen);
+      readFileInto(debugFile, debugDataBuf);
+      debugFile.close();
+    }
+
+    // Call get_compact_symbol_table. We're calling the raw wasm function
+    // instead of the binding function that wasm-bindgen generated for us,
+    // because the generated function doesn't let us pass binaryDataBufPtr
+    // or debugDataBufPtr and would force another copy.
+    //
+    // The rust definition of get_compact_symbol_table is:
+    //
+    // #[wasm_bindgen]
+    // pub fn get_compact_symbol_table(binary_data: &[u8], debug_data: &[u8], breakpad_id: &str, dest: &mut CompactSymbolTable) -> bool
+    //
+    // It gets exposed as a wasm function with the following signature:
+    //
+    // pub fn get_compact_symbol_table(binaryDataBufPtr: u32, binaryDataBufLen: u32, debugDataBufPtr: u32, debugDataBufLen: u32, breakpadIdPtr: u32, breakpadIdLen: u32, destPtr: u32) -> u32
+    //
+    // We're relying on implementation details of wasm-bindgen here. The above
+    // is true as of wasm-bindgen 0.2.32.
+
+    // Convert the breakpadId string into bytes in wasm memory.
+    const breakpadIdBuf = new TextEncoder("utf-8").encode(breakpadId);
+    const breakpadIdLen = breakpadIdBuf.length;
+    const breakpadIdPtr = wasm.__wbindgen_malloc(breakpadIdLen);
+    new Uint8Array(wasm.memory.buffer).set(breakpadIdBuf, breakpadIdPtr);
+
+    const output = new CompactSymbolTable();
+    let succeeded;
+    try {
+      succeeded =
+        wasm.get_compact_symbol_table(binaryDataBufPtr, binaryDataBufLen,
+                                      debugDataBufPtr, debugDataBufLen,
+                                      breakpadIdPtr, breakpadIdLen,
+                                      output.ptr) !== 0;
+    } catch (e) {
+      succeeded = false;
+    }
+
+    wasm.__wbindgen_free(breakpadIdPtr, breakpadIdLen);
+    wasm.__wbindgen_free(binaryDataBufPtr, binaryDataBufLen);
+    if (debugDataBufPtr != binaryDataBufPtr) {
+      wasm.__wbindgen_free(debugDataBufPtr, debugDataBufLen);
+    }
+
+    if (!succeeded) {
+      output.free();
+      throw new Error("get_compact_symbol_table returned false");
+    }
+
+    const result = [output.take_addr(), output.take_index(), output.take_buffer()];
+    output.free();
+
+    postMessage({result}, result.map(r => r.buffer));
+  } catch (error) {
+    postMessage({error: error.toString()});
+  }
+  close();
+};

browser/components/extensions/ProfilerGetSymbols.jsm

@@ -0,0 +1,95 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+"use strict";
+
+const EXPORTED_SYMBOLS = ["ProfilerGetSymbols"];
+
+ChromeUtils.defineModuleGetter(this, "setTimeout",
+                               "resource://gre/modules/Timer.jsm");
+ChromeUtils.defineModuleGetter(this, "clearTimeout",
+                               "resource://gre/modules/Timer.jsm");
+
+Cu.importGlobalProperties(["fetch"]);
+
+// This module obtains symbol tables for binaries.
+// It does so with the help of a WASM module which gets pulled in from the
+// internet on demand. We're doing this purely for the purposes of saving on
+// code size. The contents of the WASM module are expected to be static, they
+// are checked against the hash specified below.
+// The WASM code is run on a ChromeWorker thread. It takes the raw byte
+// contents of the to-be-dumped binary (and of an additional optional pdb file
+// on Windows) as its input, and returns a set of typed arrays which make up
+// the symbol table.
+
+// Don't let the strange looking URLs and strings below scare you.
+// The hash check ensures that the contents of the wasm module are what we
+// expect them to be.
+// The source code is at https://github.com/mstange/profiler-get-symbols/ .
+
+// Generated from https://github.com/mstange/profiler-get-symbols/commit/0a0aadc68d6196823a5f102feacb2f04424cd681
+const WASM_MODULE_URL =
+  "https://zealous-rosalind-a98ce8.netlify.com/wasm/4af7553b4848038c5a1e33a15ec107094bd0572e16fe2a367235bcb50a630b148ac4fe1d165859d7a9bb4ca4e2572c0e.wasm";
+const WASM_MODULE_INTEGRITY =
+  "sha384-SvdVO0hIA4xaHjOhXsEHCUvQVy4W/io2cjW8tQpjCxSKxP4dFlhZ16m7TKTiVywO";
+
+const EXPIRY_TIME_IN_MS = 5 * 60 * 1000; // 5 minutes
+
+let gCachedWASMModulePromise = null;
+let gCachedWASMModuleExpiryTimer = 0;
+
+function clearCachedWASMModule() {
+  gCachedWASMModulePromise = null;
+  gCachedWASMModuleExpiryTimer = 0;
+}
+
+function getWASMProfilerGetSymbolsModule() {
+  if (!gCachedWASMModulePromise) {
+    gCachedWASMModulePromise = (async function() {
+      const request = new Request(WASM_MODULE_URL, {
+        integrity: WASM_MODULE_INTEGRITY,
+        credentials: "omit",
+      });
+      return WebAssembly.compileStreaming(fetch(request));
+    })();
+  }
+
+  // Reset expiry timer.
+  clearTimeout(gCachedWASMModuleExpiryTimer);
+  gCachedWASMModuleExpiryTimer = setTimeout(clearCachedWASMModule, EXPIRY_TIME_IN_MS);
+
+  return gCachedWASMModulePromise;
+}
+
+this.ProfilerGetSymbols = {
+  /**
+   * Obtain symbols for the binary at the specified location.
+   *
+   * @param {string} binaryPath The absolute path to the binary on the local
+   *   file system.
+   * @param {string} debugPath The absolute path to the binary's pdb file on the
+   *   local file system if on Windows, otherwise the same as binaryPath.
+   * @param {string} breakpadId The breakpadId for the binary whose symbols
+   *   should be obtained. This is used for two purposes: 1) to locate the
+   *   correct single-arch binary in "FatArch" files, and 2) to make sure the
+   *   binary at the given path is actually the one that we want. If no ID match
+   *   is found, this function throws (rejects the promise).
+   * @returns {Promise} The symbol table in SymbolTableAsTuple format, see the
+   *   documentation for nsIProfiler.getSymbolTable.
+   */
+  async getSymbolTable(binaryPath, debugPath, breakpadId) {
+    const module = await getWASMProfilerGetSymbolsModule();
+
+    return new Promise((resolve, reject) => {
+      const worker =
+        new ChromeWorker("resource://app/modules/ProfilerGetSymbols-worker.js");
+      worker.onmessage = (e) => {
+        if (e.data.error) {
+          reject(e.data.error);
+          return;
+        }
+        resolve(e.data.result);
+      };
+      worker.postMessage({binaryPath, debugPath, breakpadId, module});
+    });
+  },
+};

browser/components/extensions/moz.build

@@ -19,6 +19,9 @@ EXTRA_JS_MODULES += [
     'ParseBreakpadSymbols-worker.js',
     'ParseNMSymbols-worker.js',
     'ParseSymbols.jsm',
+    'profiler_get_symbols.js',
+    'ProfilerGetSymbols-worker.js',
+    'ProfilerGetSymbols.jsm',
 ]
 
 DIRS += ['schemas']

browser/components/extensions/profiler_get_symbols.js

@@ -0,0 +1,186 @@
+//
+// THIS FILE IS AUTOGENERATED by wasm-bindgen.
+//
+// Generated from:
+// https://github.com/mstange/profiler-get-symbols/commit/0a0aadc68d6196823a5f102feacb2f04424cd681
+// by following the instructions in that repository's Readme.md
+//
+
+(function() {
+    var wasm;
+    const __exports = {};
+
+
+    let cachegetUint32Memory = null;
+    function getUint32Memory() {
+        if (cachegetUint32Memory === null || cachegetUint32Memory.buffer !== wasm.memory.buffer) {
+            cachegetUint32Memory = new Uint32Array(wasm.memory.buffer);
+        }
+        return cachegetUint32Memory;
+    }
+
+    function getArrayU32FromWasm(ptr, len) {
+        return getUint32Memory().subarray(ptr / 4, ptr / 4 + len);
+    }
+
+    let cachedGlobalArgumentPtr = null;
+    function globalArgumentPtr() {
+        if (cachedGlobalArgumentPtr === null) {
+            cachedGlobalArgumentPtr = wasm.__wbindgen_global_argument_ptr();
+        }
+        return cachedGlobalArgumentPtr;
+    }
+
+    let cachegetUint8Memory = null;
+    function getUint8Memory() {
+        if (cachegetUint8Memory === null || cachegetUint8Memory.buffer !== wasm.memory.buffer) {
+            cachegetUint8Memory = new Uint8Array(wasm.memory.buffer);
+        }
+        return cachegetUint8Memory;
+    }
+
+    function getArrayU8FromWasm(ptr, len) {
+        return getUint8Memory().subarray(ptr / 1, ptr / 1 + len);
+    }
+
+    function passArray8ToWasm(arg) {
+        const ptr = wasm.__wbindgen_malloc(arg.length * 1);
+        getUint8Memory().set(arg, ptr / 1);
+        return [ptr, arg.length];
+    }
+
+    let cachedTextEncoder = new TextEncoder('utf-8');
+
+    function passStringToWasm(arg) {
+
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = wasm.__wbindgen_malloc(buf.length);
+        getUint8Memory().set(buf, ptr);
+        return [ptr, buf.length];
+    }
+    /**
+    * @param {Uint8Array} arg0
+    * @param {Uint8Array} arg1
+    * @param {string} arg2
+    * @param {CompactSymbolTable} arg3
+    * @returns {boolean}
+    */
+    __exports.get_compact_symbol_table = function(arg0, arg1, arg2, arg3) {
+        const [ptr0, len0] = passArray8ToWasm(arg0);
+        const [ptr1, len1] = passArray8ToWasm(arg1);
+        const [ptr2, len2] = passStringToWasm(arg2);
+        try {
+            return (wasm.get_compact_symbol_table(ptr0, len0, ptr1, len1, ptr2, len2, arg3.ptr)) !== 0;
+
+        } finally {
+            wasm.__wbindgen_free(ptr0, len0 * 1);
+            wasm.__wbindgen_free(ptr1, len1 * 1);
+            wasm.__wbindgen_free(ptr2, len2 * 1);
+
+        }
+
+    };
+
+    function freeCompactSymbolTable(ptr) {
+
+        wasm.__wbg_compactsymboltable_free(ptr);
+    }
+    /**
+    */
+    class CompactSymbolTable {
+
+        free() {
+            const ptr = this.ptr;
+            this.ptr = 0;
+            freeCompactSymbolTable(ptr);
+        }
+
+        /**
+        * @returns {}
+        */
+        constructor() {
+            this.ptr = wasm.compactsymboltable_new();
+        }
+        /**
+        * @returns {Uint32Array}
+        */
+        take_addr() {
+            const retptr = globalArgumentPtr();
+            wasm.compactsymboltable_take_addr(retptr, this.ptr);
+            const mem = getUint32Memory();
+            const rustptr = mem[retptr / 4];
+            const rustlen = mem[retptr / 4 + 1];
+
+            const realRet = getArrayU32FromWasm(rustptr, rustlen).slice();
+            wasm.__wbindgen_free(rustptr, rustlen * 4);
+            return realRet;
+
+        }
+        /**
+        * @returns {Uint32Array}
+        */
+        take_index() {
+            const retptr = globalArgumentPtr();
+            wasm.compactsymboltable_take_index(retptr, this.ptr);
+            const mem = getUint32Memory();
+            const rustptr = mem[retptr / 4];
+            const rustlen = mem[retptr / 4 + 1];
+
+            const realRet = getArrayU32FromWasm(rustptr, rustlen).slice();
+            wasm.__wbindgen_free(rustptr, rustlen * 4);
+            return realRet;
+
+        }
+        /**
+        * @returns {Uint8Array}
+        */
+        take_buffer() {
+            const retptr = globalArgumentPtr();
+            wasm.compactsymboltable_take_buffer(retptr, this.ptr);
+            const mem = getUint32Memory();
+            const rustptr = mem[retptr / 4];
+            const rustlen = mem[retptr / 4 + 1];
+
+            const realRet = getArrayU8FromWasm(rustptr, rustlen).slice();
+            wasm.__wbindgen_free(rustptr, rustlen * 1);
+            return realRet;
+
+        }
+    }
+    __exports.CompactSymbolTable = CompactSymbolTable;
+
+    let cachedTextDecoder = new TextDecoder('utf-8');
+
+    function getStringFromWasm(ptr, len) {
+        return cachedTextDecoder.decode(getUint8Memory().subarray(ptr, ptr + len));
+    }
+
+    __exports.__wbindgen_throw = function(ptr, len) {
+        throw new Error(getStringFromWasm(ptr, len));
+    };
+
+    function init(path_or_module) {
+        let instantiation;
+        const imports = { './profiler_get_symbols': __exports };
+        if (path_or_module instanceof WebAssembly.Module) {
+            instantiation = WebAssembly.instantiate(path_or_module, imports)
+            .then(instance => {
+            return { instance, module: path_or_module }
+        });
+    } else {
+        const data = fetch(path_or_module);
+        if (typeof WebAssembly.instantiateStreaming === 'function') {
+            instantiation = WebAssembly.instantiateStreaming(data, imports);
+        } else {
+            instantiation = data
+            .then(response => response.arrayBuffer())
+            .then(buffer => WebAssembly.instantiate(buffer, imports));
+        }
+    }
+    return instantiation.then(({instance}) => {
+        wasm = init.wasm = instance.exports;
+        return;
+    });
+};
+self.wasm_bindgen = Object.assign(init, __exports);
+})();

tools/lint/eslint/modules.json

@@ -170,6 +170,7 @@
   "prefs.js": ["PrefsEngine", "PrefRec"],
   "prefs.jsm": ["Preference"],
   "pretty-fast.js": ["prettyFast"],
+  "profiler_get_symbols.js": ["wasm_bindgen"],
   "PromiseWorker.jsm": ["BasePromiseWorker"],
   "PushCrypto.jsm": ["PushCrypto", "concatArray"],
   "quit.js": ["goQuitApplication"],