From 6dbd19aac822f7c12fddc5bc9636b7d43e77e92f Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Thu, 7 Feb 2019 19:36:43 +0000 Subject: [PATCH] Bug 1509549 - Add a ProfilerGetSymbols module which can dump symbols with the help of a dynamically-loaded WebAssembly module. r=kmag The module can dump ELF binaries, Mach-O binaries, and pdb files. So it works for all our supported platforms. The module is currently hosted on https://zealous-rosalind-a98ce8.netlify.com/ , which is a netlify server that serves files from the following repo: https://github.com/mstange/profiler-assets To make all of this look a bit more official, I'm planning on doing two things: - Move the github repo under the devtools-html organization - Get a firefox.com subdomain such as profiler-assets.firefox.com for hosting Depends on D13004 Differential Revision: https://phabricator.services.mozilla.com/D13005 --HG-- extra : moz-landing-system : lando --- .eslintignore | 2 + .../extensions/ProfilerGetSymbols-worker.js | 128 ++++++++++++ .../extensions/ProfilerGetSymbols.jsm | 95 +++++++++ browser/components/extensions/moz.build | 3 + .../extensions/profiler_get_symbols.js | 186 ++++++++++++++++++ tools/lint/eslint/modules.json | 1 + 6 files changed, 415 insertions(+) create mode 100644 browser/components/extensions/ProfilerGetSymbols-worker.js create mode 100644 browser/components/extensions/ProfilerGetSymbols.jsm create mode 100644 browser/components/extensions/profiler_get_symbols.js diff --git a/.eslintignore b/.eslintignore index b6737bf89bf5..f5219d5be971 100644 --- a/.eslintignore +++ b/.eslintignore @@ -65,6 +65,8 @@ browser/components/sessionstore/test/unit/data/sessionstore_invalid.js # for documentation purposes (policies.json) but to be accessed by the # code as a .jsm (schema.jsm) browser/components/enterprisepolicies/schemas/schema.jsm +# Ignore generated code from wasm-bindgen +browser/components/extensions/profiler_get_symbols.js # generated & special files in cld2 browser/components/translation/cld2/** # Screenshots is imported as a system add-on and has diff --git a/browser/components/extensions/ProfilerGetSymbols-worker.js b/browser/components/extensions/ProfilerGetSymbols-worker.js new file mode 100644 index 000000000000..f1b90df7edf6 --- /dev/null +++ b/browser/components/extensions/ProfilerGetSymbols-worker.js @@ -0,0 +1,128 @@ +/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim: set sts=2 sw=2 et tw=80: */ +/* eslint-env mozilla/chrome-worker */ + +"use strict"; + +importScripts("resource://gre/modules/osfile.jsm", + "resource://app/modules/profiler_get_symbols.js"); + +// This worker uses the wasm module that was generated from https://github.com/mstange/profiler-get-symbols. +// See ProfilerGetSymbols.jsm for more information. +// +// The worker instantiates the module, reads the binary into wasm memory, runs +// the wasm code, and returns the symbol table or an error. Then it shuts down +// itself. + +// Read an open OS.File instance into the Uint8Array dataBuf. +function readFileInto(file, dataBuf) { + // Ideally we'd be able to call file.readTo(dataBuf) here, but readTo no + // longer exists. + // So instead, we copy the file over into wasm memory in 4MB chunks. This + // will take 425 invocations for a a 1.7GB file (such as libxul.so for a + // Firefox for Android build) and not take up too much memory per call. + const dataBufLen = dataBuf.byteLength; + const chunkSize = 4 * 1024 * 1024; + let pos = 0; + while (pos < dataBufLen) { + const chunkData = file.read({bytes: chunkSize}); + const chunkBytes = chunkData.byteLength; + if (chunkBytes === 0) { + break; + } + + dataBuf.set(chunkData, pos); + pos += chunkBytes; + } +} + +onmessage = async e => { + try { + const {binaryPath, debugPath, breakpadId, module} = e.data; + + if (!(module instanceof WebAssembly.Module)) { + throw new Error("invalid WebAssembly module"); + } + + // Instantiate the WASM module. + await wasm_bindgen(module); + + const {CompactSymbolTable, wasm} = wasm_bindgen; + + const binaryFile = OS.File.open(binaryPath, {read: true}); + const binaryDataBufLen = binaryFile.stat().size; + + // Read the binary file into WASM memory. + const binaryDataBufPtr = wasm.__wbindgen_malloc(binaryDataBufLen); + const binaryDataBuf = new Uint8Array(wasm.memory.buffer, binaryDataBufPtr, binaryDataBufLen); + readFileInto(binaryFile, binaryDataBuf); + binaryFile.close(); + + // Do the same for the debug file, if it is supplied and different from the + // binary file. This is only the case on Windows. + let debugDataBufLen = binaryDataBufLen; + let debugDataBufPtr = binaryDataBufPtr; + if (debugPath && debugPath !== binaryPath) { + const debugFile = OS.File.open(debugPath, {read: true}); + debugDataBufLen = debugFile.stat().size; + debugDataBufPtr = wasm.__wbindgen_malloc(debugDataBufLen); + const debugDataBuf = new Uint8Array(wasm.memory.buffer, debugDataBufPtr, debugDataBufLen); + readFileInto(debugFile, debugDataBuf); + debugFile.close(); + } + + // Call get_compact_symbol_table. We're calling the raw wasm function + // instead of the binding function that wasm-bindgen generated for us, + // because the generated function doesn't let us pass binaryDataBufPtr + // or debugDataBufPtr and would force another copy. + // + // The rust definition of get_compact_symbol_table is: + // + // #[wasm_bindgen] + // pub fn get_compact_symbol_table(binary_data: &[u8], debug_data: &[u8], breakpad_id: &str, dest: &mut CompactSymbolTable) -> bool + // + // It gets exposed as a wasm function with the following signature: + // + // pub fn get_compact_symbol_table(binaryDataBufPtr: u32, binaryDataBufLen: u32, debugDataBufPtr: u32, debugDataBufLen: u32, breakpadIdPtr: u32, breakpadIdLen: u32, destPtr: u32) -> u32 + // + // We're relying on implementation details of wasm-bindgen here. The above + // is true as of wasm-bindgen 0.2.32. + + // Convert the breakpadId string into bytes in wasm memory. + const breakpadIdBuf = new TextEncoder("utf-8").encode(breakpadId); + const breakpadIdLen = breakpadIdBuf.length; + const breakpadIdPtr = wasm.__wbindgen_malloc(breakpadIdLen); + new Uint8Array(wasm.memory.buffer).set(breakpadIdBuf, breakpadIdPtr); + + const output = new CompactSymbolTable(); + let succeeded; + try { + succeeded = + wasm.get_compact_symbol_table(binaryDataBufPtr, binaryDataBufLen, + debugDataBufPtr, debugDataBufLen, + breakpadIdPtr, breakpadIdLen, + output.ptr) !== 0; + } catch (e) { + succeeded = false; + } + + wasm.__wbindgen_free(breakpadIdPtr, breakpadIdLen); + wasm.__wbindgen_free(binaryDataBufPtr, binaryDataBufLen); + if (debugDataBufPtr != binaryDataBufPtr) { + wasm.__wbindgen_free(debugDataBufPtr, debugDataBufLen); + } + + if (!succeeded) { + output.free(); + throw new Error("get_compact_symbol_table returned false"); + } + + const result = [output.take_addr(), output.take_index(), output.take_buffer()]; + output.free(); + + postMessage({result}, result.map(r => r.buffer)); + } catch (error) { + postMessage({error: error.toString()}); + } + close(); +}; diff --git a/browser/components/extensions/ProfilerGetSymbols.jsm b/browser/components/extensions/ProfilerGetSymbols.jsm new file mode 100644 index 000000000000..f6d422806fc8 --- /dev/null +++ b/browser/components/extensions/ProfilerGetSymbols.jsm @@ -0,0 +1,95 @@ +/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim: set sts=2 sw=2 et tw=80: */ +"use strict"; + +const EXPORTED_SYMBOLS = ["ProfilerGetSymbols"]; + +ChromeUtils.defineModuleGetter(this, "setTimeout", + "resource://gre/modules/Timer.jsm"); +ChromeUtils.defineModuleGetter(this, "clearTimeout", + "resource://gre/modules/Timer.jsm"); + +Cu.importGlobalProperties(["fetch"]); + +// This module obtains symbol tables for binaries. +// It does so with the help of a WASM module which gets pulled in from the +// internet on demand. We're doing this purely for the purposes of saving on +// code size. The contents of the WASM module are expected to be static, they +// are checked against the hash specified below. +// The WASM code is run on a ChromeWorker thread. It takes the raw byte +// contents of the to-be-dumped binary (and of an additional optional pdb file +// on Windows) as its input, and returns a set of typed arrays which make up +// the symbol table. + +// Don't let the strange looking URLs and strings below scare you. +// The hash check ensures that the contents of the wasm module are what we +// expect them to be. +// The source code is at https://github.com/mstange/profiler-get-symbols/ . + +// Generated from https://github.com/mstange/profiler-get-symbols/commit/0a0aadc68d6196823a5f102feacb2f04424cd681 +const WASM_MODULE_URL = + "https://zealous-rosalind-a98ce8.netlify.com/wasm/4af7553b4848038c5a1e33a15ec107094bd0572e16fe2a367235bcb50a630b148ac4fe1d165859d7a9bb4ca4e2572c0e.wasm"; +const WASM_MODULE_INTEGRITY = + "sha384-SvdVO0hIA4xaHjOhXsEHCUvQVy4W/io2cjW8tQpjCxSKxP4dFlhZ16m7TKTiVywO"; + +const EXPIRY_TIME_IN_MS = 5 * 60 * 1000; // 5 minutes + +let gCachedWASMModulePromise = null; +let gCachedWASMModuleExpiryTimer = 0; + +function clearCachedWASMModule() { + gCachedWASMModulePromise = null; + gCachedWASMModuleExpiryTimer = 0; +} + +function getWASMProfilerGetSymbolsModule() { + if (!gCachedWASMModulePromise) { + gCachedWASMModulePromise = (async function() { + const request = new Request(WASM_MODULE_URL, { + integrity: WASM_MODULE_INTEGRITY, + credentials: "omit", + }); + return WebAssembly.compileStreaming(fetch(request)); + })(); + } + + // Reset expiry timer. + clearTimeout(gCachedWASMModuleExpiryTimer); + gCachedWASMModuleExpiryTimer = setTimeout(clearCachedWASMModule, EXPIRY_TIME_IN_MS); + + return gCachedWASMModulePromise; +} + +this.ProfilerGetSymbols = { + /** + * Obtain symbols for the binary at the specified location. + * + * @param {string} binaryPath The absolute path to the binary on the local + * file system. + * @param {string} debugPath The absolute path to the binary's pdb file on the + * local file system if on Windows, otherwise the same as binaryPath. + * @param {string} breakpadId The breakpadId for the binary whose symbols + * should be obtained. This is used for two purposes: 1) to locate the + * correct single-arch binary in "FatArch" files, and 2) to make sure the + * binary at the given path is actually the one that we want. If no ID match + * is found, this function throws (rejects the promise). + * @returns {Promise} The symbol table in SymbolTableAsTuple format, see the + * documentation for nsIProfiler.getSymbolTable. + */ + async getSymbolTable(binaryPath, debugPath, breakpadId) { + const module = await getWASMProfilerGetSymbolsModule(); + + return new Promise((resolve, reject) => { + const worker = + new ChromeWorker("resource://app/modules/ProfilerGetSymbols-worker.js"); + worker.onmessage = (e) => { + if (e.data.error) { + reject(e.data.error); + return; + } + resolve(e.data.result); + }; + worker.postMessage({binaryPath, debugPath, breakpadId, module}); + }); + }, +}; diff --git a/browser/components/extensions/moz.build b/browser/components/extensions/moz.build index 82eed846fe53..1dc443cf90c5 100644 --- a/browser/components/extensions/moz.build +++ b/browser/components/extensions/moz.build @@ -19,6 +19,9 @@ EXTRA_JS_MODULES += [ 'ParseBreakpadSymbols-worker.js', 'ParseNMSymbols-worker.js', 'ParseSymbols.jsm', + 'profiler_get_symbols.js', + 'ProfilerGetSymbols-worker.js', + 'ProfilerGetSymbols.jsm', ] DIRS += ['schemas'] diff --git a/browser/components/extensions/profiler_get_symbols.js b/browser/components/extensions/profiler_get_symbols.js new file mode 100644 index 000000000000..beba5741eb13 --- /dev/null +++ b/browser/components/extensions/profiler_get_symbols.js @@ -0,0 +1,186 @@ +// +// THIS FILE IS AUTOGENERATED by wasm-bindgen. +// +// Generated from: +// https://github.com/mstange/profiler-get-symbols/commit/0a0aadc68d6196823a5f102feacb2f04424cd681 +// by following the instructions in that repository's Readme.md +// + +(function() { + var wasm; + const __exports = {}; + + + let cachegetUint32Memory = null; + function getUint32Memory() { + if (cachegetUint32Memory === null || cachegetUint32Memory.buffer !== wasm.memory.buffer) { + cachegetUint32Memory = new Uint32Array(wasm.memory.buffer); + } + return cachegetUint32Memory; + } + + function getArrayU32FromWasm(ptr, len) { + return getUint32Memory().subarray(ptr / 4, ptr / 4 + len); + } + + let cachedGlobalArgumentPtr = null; + function globalArgumentPtr() { + if (cachedGlobalArgumentPtr === null) { + cachedGlobalArgumentPtr = wasm.__wbindgen_global_argument_ptr(); + } + return cachedGlobalArgumentPtr; + } + + let cachegetUint8Memory = null; + function getUint8Memory() { + if (cachegetUint8Memory === null || cachegetUint8Memory.buffer !== wasm.memory.buffer) { + cachegetUint8Memory = new Uint8Array(wasm.memory.buffer); + } + return cachegetUint8Memory; + } + + function getArrayU8FromWasm(ptr, len) { + return getUint8Memory().subarray(ptr / 1, ptr / 1 + len); + } + + function passArray8ToWasm(arg) { + const ptr = wasm.__wbindgen_malloc(arg.length * 1); + getUint8Memory().set(arg, ptr / 1); + return [ptr, arg.length]; + } + + let cachedTextEncoder = new TextEncoder('utf-8'); + + function passStringToWasm(arg) { + + const buf = cachedTextEncoder.encode(arg); + const ptr = wasm.__wbindgen_malloc(buf.length); + getUint8Memory().set(buf, ptr); + return [ptr, buf.length]; + } + /** + * @param {Uint8Array} arg0 + * @param {Uint8Array} arg1 + * @param {string} arg2 + * @param {CompactSymbolTable} arg3 + * @returns {boolean} + */ + __exports.get_compact_symbol_table = function(arg0, arg1, arg2, arg3) { + const [ptr0, len0] = passArray8ToWasm(arg0); + const [ptr1, len1] = passArray8ToWasm(arg1); + const [ptr2, len2] = passStringToWasm(arg2); + try { + return (wasm.get_compact_symbol_table(ptr0, len0, ptr1, len1, ptr2, len2, arg3.ptr)) !== 0; + + } finally { + wasm.__wbindgen_free(ptr0, len0 * 1); + wasm.__wbindgen_free(ptr1, len1 * 1); + wasm.__wbindgen_free(ptr2, len2 * 1); + + } + + }; + + function freeCompactSymbolTable(ptr) { + + wasm.__wbg_compactsymboltable_free(ptr); + } + /** + */ + class CompactSymbolTable { + + free() { + const ptr = this.ptr; + this.ptr = 0; + freeCompactSymbolTable(ptr); + } + + /** + * @returns {} + */ + constructor() { + this.ptr = wasm.compactsymboltable_new(); + } + /** + * @returns {Uint32Array} + */ + take_addr() { + const retptr = globalArgumentPtr(); + wasm.compactsymboltable_take_addr(retptr, this.ptr); + const mem = getUint32Memory(); + const rustptr = mem[retptr / 4]; + const rustlen = mem[retptr / 4 + 1]; + + const realRet = getArrayU32FromWasm(rustptr, rustlen).slice(); + wasm.__wbindgen_free(rustptr, rustlen * 4); + return realRet; + + } + /** + * @returns {Uint32Array} + */ + take_index() { + const retptr = globalArgumentPtr(); + wasm.compactsymboltable_take_index(retptr, this.ptr); + const mem = getUint32Memory(); + const rustptr = mem[retptr / 4]; + const rustlen = mem[retptr / 4 + 1]; + + const realRet = getArrayU32FromWasm(rustptr, rustlen).slice(); + wasm.__wbindgen_free(rustptr, rustlen * 4); + return realRet; + + } + /** + * @returns {Uint8Array} + */ + take_buffer() { + const retptr = globalArgumentPtr(); + wasm.compactsymboltable_take_buffer(retptr, this.ptr); + const mem = getUint32Memory(); + const rustptr = mem[retptr / 4]; + const rustlen = mem[retptr / 4 + 1]; + + const realRet = getArrayU8FromWasm(rustptr, rustlen).slice(); + wasm.__wbindgen_free(rustptr, rustlen * 1); + return realRet; + + } + } + __exports.CompactSymbolTable = CompactSymbolTable; + + let cachedTextDecoder = new TextDecoder('utf-8'); + + function getStringFromWasm(ptr, len) { + return cachedTextDecoder.decode(getUint8Memory().subarray(ptr, ptr + len)); + } + + __exports.__wbindgen_throw = function(ptr, len) { + throw new Error(getStringFromWasm(ptr, len)); + }; + + function init(path_or_module) { + let instantiation; + const imports = { './profiler_get_symbols': __exports }; + if (path_or_module instanceof WebAssembly.Module) { + instantiation = WebAssembly.instantiate(path_or_module, imports) + .then(instance => { + return { instance, module: path_or_module } + }); + } else { + const data = fetch(path_or_module); + if (typeof WebAssembly.instantiateStreaming === 'function') { + instantiation = WebAssembly.instantiateStreaming(data, imports); + } else { + instantiation = data + .then(response => response.arrayBuffer()) + .then(buffer => WebAssembly.instantiate(buffer, imports)); + } + } + return instantiation.then(({instance}) => { + wasm = init.wasm = instance.exports; + return; + }); +}; +self.wasm_bindgen = Object.assign(init, __exports); +})(); diff --git a/tools/lint/eslint/modules.json b/tools/lint/eslint/modules.json index 83a738487490..4b030f19dbe6 100644 --- a/tools/lint/eslint/modules.json +++ b/tools/lint/eslint/modules.json @@ -170,6 +170,7 @@ "prefs.js": ["PrefsEngine", "PrefRec"], "prefs.jsm": ["Preference"], "pretty-fast.js": ["prettyFast"], + "profiler_get_symbols.js": ["wasm_bindgen"], "PromiseWorker.jsm": ["BasePromiseWorker"], "PushCrypto.jsm": ["PushCrypto", "concatArray"], "quit.js": ["goQuitApplication"],