From 02841791400cf7cf5760c0cfaf31f5d772624253 Mon Sep 17 00:00:00 2001 From: Noemi Erli Date: Tue, 7 Nov 2023 05:24:56 +0200 Subject: [PATCH] Backed out 5 changesets (bug 1861516) for causing mass failures CLOSED TREE Backed out changeset 05f23c29b41c (bug 1861516) Backed out changeset 683094c8544b (bug 1861516) Backed out changeset 7dd9e944c8af (bug 1861516) Backed out changeset f3a921cb6c03 (bug 1861516) Backed out changeset a2088075a1e8 (bug 1861516) --- .prettierignore | 2 + .../performance/browser_startup_content.js | 1 + ...preferences_manage_downloaded_languages.js | 10 +- modules/libpref/init/all.js | 7 + .../actors/AboutTranslationsChild.sys.mjs | 54 ++ .../actors/TranslationsChild.sys.mjs | 23 + .../actors/TranslationsParent.sys.mjs | 240 +++++++- .../content/language-id-engine-worker.js | 327 +++++++++++ .../content/language-id-engine.sys.mjs | 224 ++++++++ .../translations/content/translations.mjs | 24 +- .../docs/resources/01_overview.md | 12 +- .../docs/resources/02_contributing.md | 293 +++++++++- .../components/translations/fasttext/LICENSE | 21 + .../translations/fasttext/fasttext.js | 536 ++++++++++++++++++ .../translations/fasttext/fasttext_wasm.js | 8 + .../components/translations/fasttext/moz.yaml | 44 ++ toolkit/components/translations/jar.mn | 4 + ...browser_about_translations_translations.js | 5 +- ...owser_translations_actor_detected_langs.js | 4 +- .../browser_translations_actor_empty_langs.js | 3 + .../browser/browser_translations_full_page.js | 4 + .../translations/tests/browser/shared-head.js | 84 ++- .../components/translations/translations.d.ts | 31 + toolkit/content/license.html | 2 + tools/rewriting/ThirdPartyPaths.txt | 2 + 25 files changed, 1937 insertions(+), 28 deletions(-) create mode 100644 toolkit/components/translations/content/language-id-engine-worker.js create mode 100644 toolkit/components/translations/content/language-id-engine.sys.mjs create mode 100644 toolkit/components/translations/fasttext/LICENSE create mode 100644 toolkit/components/translations/fasttext/fasttext.js create mode 100644 toolkit/components/translations/fasttext/fasttext_wasm.js create mode 100644 toolkit/components/translations/fasttext/moz.yaml diff --git a/.prettierignore b/.prettierignore index 07265bd95491..9edd508e57bf 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1424,6 +1424,8 @@ toolkit/components/passwordmgr/PasswordRulesParser.sys.mjs toolkit/components/protobuf/ toolkit/components/translation/cld2/ toolkit/components/translations/bergamot-translator +toolkit/components/translations/fasttext/fasttext.js +toolkit/components/translations/fasttext/fasttext_wasm.js toolkit/components/url-classifier/chromium/ toolkit/components/utils/mozjexl.js toolkit/components/viaduct/fetch_msg_types.pb.cc diff --git a/browser/base/content/test/performance/browser_startup_content.js b/browser/base/content/test/performance/browser_startup_content.js index 92db50ad0c25..024b9960a2b9 100644 --- a/browser/base/content/test/performance/browser_startup_content.js +++ b/browser/base/content/test/performance/browser_startup_content.js @@ -72,6 +72,7 @@ const intermittently_loaded_scripts = { // Translations code which may be preffed on. "resource://gre/actors/TranslationsChild.sys.mjs", "resource://gre/modules/translation/LanguageDetector.sys.mjs", + "chrome://global/content/translations/language-id-engine.sys.mjs", "resource://gre/modules/ConsoleAPIStorage.sys.mjs", // Logging related. // Session store. diff --git a/browser/components/translations/tests/browser/browser_translations_about_preferences_manage_downloaded_languages.js b/browser/components/translations/tests/browser/browser_translations_about_preferences_manage_downloaded_languages.js index 598cd1661e7e..2ea694be9c73 100644 --- a/browser/components/translations/tests/browser/browser_translations_about_preferences_manage_downloaded_languages.js +++ b/browser/components/translations/tests/browser/browser_translations_about_preferences_manage_downloaded_languages.js @@ -117,8 +117,13 @@ add_task(async function test_about_preferences_manage_languages() { "All models were downloaded." ); Assert.deepEqual( - await remoteClients.translationsWasm.resolvePendingDownloads(1), - ["bergamot-translator"], + await remoteClients.languageIdModels.resolvePendingDownloads(1), + ["lid.176.ftz"], + "Language ID model was downloaded." + ); + Assert.deepEqual( + await remoteClients.translationsWasm.resolvePendingDownloads(2), + ["bergamot-translator", "fasttext-wasm"], "Wasm was downloaded." ); @@ -154,6 +159,7 @@ add_task(async function test_about_preferences_manage_languages() { ); remoteClients.translationsWasm.assertNoNewDownloads(); + remoteClients.languageIdModels.assertNoNewDownloads(); await assertVisibility({ message: "Everything is downloaded again.", diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index 3ff4c372138c..6f8dac92c9cb 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -3634,6 +3634,13 @@ pref("browser.translations.simulateUnsupportedEngine", false); pref("browser.translations.chaos.errors", false); pref("browser.translations.chaos.timeoutMS", 0); +// A pref to manage the use of fastText for language detection in Translations. +// The feature was initially built using fastText, but we are now putting it +// behind a pref while we investigate some performance improvements. +// In the meantime, we will use CLD2, which is already available in tree. +// See https://bugzilla.mozilla.org/show_bug.cgi?id=1836974 +pref("browser.translations.languageIdentification.useFastText", false); + // When a user cancels this number of authentication dialogs coming from // a single web page in a row, all following authentication dialogs will // be blocked (automatically canceled) for that page. The counter resets diff --git a/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs b/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs index c501c1b0cd1b..55a278529df9 100644 --- a/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs +++ b/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs @@ -2,6 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + const lazy = {}; ChromeUtils.defineLazyGetter(lazy, "console", () => { @@ -16,7 +18,14 @@ ChromeUtils.defineESModuleGetters(lazy, { "resource://gre/modules/translation/LanguageDetector.sys.mjs", }); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "useFastTextPref", + "browser.translations.languageIdentification.useFastText" +); + /** + * @typedef {import("./TranslationsChild.sys.mjs").LanguageIdEngine} LanguageIdEngine * @typedef {import("./TranslationsChild.sys.mjs").TranslationsEngine} TranslationsEngine * @typedef {import("./TranslationsChild.sys.mjs").SupportedLanguages} SupportedLanguages */ @@ -26,6 +35,9 @@ ChromeUtils.defineESModuleGetters(lazy, { * are exposed to the un-privileged scope of the about:translations page. */ export class AboutTranslationsChild extends JSWindowActorChild { + /** @type {LanguageIdEngine | null} */ + languageIdEngine = null; + /** * The translations engine uses text translations by default in about:translations, * but it can be changed to translate HTML by setting this pref to true. This is @@ -143,6 +155,7 @@ export class AboutTranslationsChild extends JSWindowActorChild { "AT_getSupportedLanguages", "AT_isTranslationEngineSupported", "AT_isHtmlTranslation", + "AT_createLanguageIdEngine", "AT_createTranslationsPort", "AT_identifyLanguage", "AT_getScriptDirection", @@ -211,6 +224,32 @@ export class AboutTranslationsChild extends JSWindowActorChild { return this.#isHtmlTranslation; } + /** + * Creates the LanguageIdEngine which attempts to identify in which + * human language a string is written. + * + * Unlike TranslationsEngine, which handles only a single language pair + * and must be rebuilt to handle a new language pair, the LanguageIdEngine + * is a one-to-many engine that can recognize all of its supported languages. + * + * Subsequent calls to this function after the engine is initialized will do nothing + * instead of rebuilding the engine. + * + * @returns {Promise} + */ + AT_createLanguageIdEngine() { + if (this.languageIdEngine) { + return this.#convertToContentPromise(Promise.resolve()); + } + return this.#convertToContentPromise( + this.#getTranslationsChild() + .getOrCreateLanguageIdEngine() + .then(engine => { + this.languageIdEngine = engine; + }) + ); + } + /** * Requests a port to the TranslationsEngine process. An engine will be created on * the fly for translation requests through this port. This port is unique to its @@ -231,11 +270,26 @@ export class AboutTranslationsChild extends JSWindowActorChild { /** * Attempts to identify the human language in which the message is written. + * @see LanguageIdEngine#identifyLanguage for more detailed documentation. * * @param {string} message * @returns {Promise<{ langTag: string, confidence: number }>} */ AT_identifyLanguage(message) { + if (lazy.useFastTextPref) { + if (!this.languageIdEngine) { + const { Promise, Error } = this.contentWindow; + return Promise.reject( + new Error("The language identification was not created.") + ); + } + + return this.#convertToContentPromise( + this.languageIdEngine + .identifyLanguage(message) + .then(data => Cu.cloneInto(data, this.contentWindow)) + ); + } return this.#convertToContentPromise( lazy.LanguageDetector.detectLanguage(message).then(data => Cu.cloneInto( diff --git a/toolkit/components/translations/actors/TranslationsChild.sys.mjs b/toolkit/components/translations/actors/TranslationsChild.sys.mjs index 73981f176e93..dd59f16d1e44 100644 --- a/toolkit/components/translations/actors/TranslationsChild.sys.mjs +++ b/toolkit/components/translations/actors/TranslationsChild.sys.mjs @@ -6,6 +6,10 @@ const lazy = {}; ChromeUtils.defineESModuleGetters(lazy, { TranslationsDocument: "chrome://global/content/translations/translations-document.sys.mjs", + // The fastText languageIdEngine + LanguageIdEngine: + "chrome://global/content/translations/language-id-engine.sys.mjs", + // The CLD2 language detector LanguageDetector: "resource://gre/modules/translation/LanguageDetector.sys.mjs", }); @@ -74,6 +78,16 @@ export class TranslationsChild extends JSWindowActorChild { } try { + // Try to use the fastText engine if directed to do so. + if (data.useFastText) { + const engine = await this.getOrCreateLanguageIdEngine(); + if (!engine) { + return null; + } + return engine.identifyLanguageFromDocument(this.document); + } + + // Use the CLD2 language detector otherwise. return lazy.LanguageDetector.detectLanguageFromDocument( this.document ); @@ -90,4 +104,13 @@ export class TranslationsChild extends JSWindowActorChild { throw new Error("Unknown message.", name); } } + + getOrCreateLanguageIdEngine() { + return lazy.LanguageIdEngine.getOrCreate(() => { + if (!this.manager || !this.manager.isCurrentGlobal) { + throw new Error("The page was already hidden."); + } + return this.sendQuery("Translations:GetLanguageIdEnginePayload"); + }); + } } diff --git a/toolkit/components/translations/actors/TranslationsParent.sys.mjs b/toolkit/components/translations/actors/TranslationsParent.sys.mjs index a9d70f061618..ed8a2e54a154 100644 --- a/toolkit/components/translations/actors/TranslationsParent.sys.mjs +++ b/toolkit/components/translations/actors/TranslationsParent.sys.mjs @@ -126,6 +126,12 @@ XPCOMUtils.defineLazyPreferenceGetter( "browser.translations.simulateUnsupportedEngine" ); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "useFastTextPref", + "browser.translations.languageIdentification.useFastText" +); + // At this time the signatures of the files are not being checked when they are being // loaded from disk. This signature check involves hitting the network, and translations // are explicitly an offline-capable feature. See Bug 1827265 for re-enabling this @@ -135,11 +141,13 @@ const VERIFY_SIGNATURES_FROM_FS = false; /** * @typedef {import("../translations").TranslationModelRecord} TranslationModelRecord * @typedef {import("../translations").RemoteSettingsClient} RemoteSettingsClient + * @typedef {import("../translations").LanguageIdEngineMockedPayload} LanguageIdEngineMockedPayload * @typedef {import("../translations").LanguageTranslationModelFiles} LanguageTranslationModelFiles * @typedef {import("../translations").WasmRecord} WasmRecord * @typedef {import("../translations").LangTags} LangTags * @typedef {import("../translations").LanguagePair} LanguagePair * @typedef {import("../translations").SupportedLanguages} SupportedLanguages + * @typedef {import("../translations").LanguageIdModelRecord} LanguageIdModelRecord * @typedef {import("../translations").TranslationErrors} TranslationErrors */ @@ -225,6 +233,13 @@ export class TranslationsParent extends JSWindowActorParent { } } + /** + * The remote settings client that retrieves the language-identification model binary. + * + * @type {RemoteSettingsClient | null} + */ + static #languageIdModelsRemoteClient = null; + /** * A map of the TranslationModelRecord["id"] to the record of the model in Remote Settings. * Used to coordinate the downloads. @@ -261,6 +276,22 @@ export class TranslationsParent extends JSWindowActorParent { */ static #isTranslationsEngineMocked = false; + /** + * The language identification engine can be mocked for testing + * by pre-defining this value. + * + * @type {string | null} + */ + static #mockedLangTag = null; + + /** + * The language identification engine can be mocked for testing + * by pre-defining this value. + * + * @type {number | null} + */ + static #mockedLanguageIdConfidence = null; + /** * @type {null | Promise} */ @@ -759,6 +790,18 @@ export class TranslationsParent extends JSWindowActorParent { async receiveMessage({ name, data }) { switch (name) { + case "Translations:GetLanguageIdEnginePayload": { + const [modelBuffer, wasmBuffer] = await Promise.all([ + TranslationsParent.#getLanguageIdModelArrayBuffer(), + TranslationsParent.#getLanguageIdWasmArrayBuffer(), + ]); + return { + modelBuffer, + wasmBuffer, + mockedConfidence: TranslationsParent.#mockedLanguageIdConfidence, + mockedLangTag: TranslationsParent.#mockedLangTag, + }; + } case "Translations:ReportLangTags": { const { documentElementLang, href } = data; const detectedLanguages = await this.getDetectedLanguages( @@ -935,6 +978,152 @@ export class TranslationsParent extends JSWindowActorParent { return true; } + /** @type {Promise | null} */ + static #languageIdModelRecord = null; + + /** + * Retrieves the language-identification model binary from remote settings. + * + * @returns {Promise} + */ + static async #getLanguageIdModelArrayBuffer() { + lazy.console.log("Getting language-identification model array buffer."); + const now = Date.now(); + const client = TranslationsParent.#getLanguageIdModelRemoteClient(); + + if (!TranslationsParent.#languageIdModelRecord) { + // Place the records into a promise to prevent any races. + TranslationsParent.#languageIdModelRecord = (async () => { + /** @type {LanguageIdModelRecord[]} */ + let modelRecords = await TranslationsParent.getMaxVersionRecords( + client + ); + + if (modelRecords.length === 0) { + throw new Error( + "Unable to get language-identification model record from remote settings" + ); + } + + if (modelRecords.length > 1) { + TranslationsParent.reportError( + new Error( + "Expected the language-identification model collection to have only 1 record." + ), + modelRecords + ); + } + return modelRecords[0]; + })(); + } + + await chaosMode(1 / 3); + + try { + /** @type {{buffer: ArrayBuffer}} */ + const { buffer } = await client.attachments.download( + await TranslationsParent.#languageIdModelRecord + ); + + const duration = (Date.now() - now) / 1000; + lazy.console.log( + `Remote language-identification model loaded in ${duration} seconds.` + ); + + return buffer; + } catch (error) { + TranslationsParent.#languageIdModelRecord = null; + throw error; + } + } + + /** + * Initializes the RemoteSettingsClient for the language-identification model binary. + * + * @returns {RemoteSettingsClient} + */ + static #getLanguageIdModelRemoteClient() { + if (TranslationsParent.#languageIdModelsRemoteClient) { + return TranslationsParent.#languageIdModelsRemoteClient; + } + + /** @type {RemoteSettingsClient} */ + const client = lazy.RemoteSettings("translations-identification-models"); + + TranslationsParent.#languageIdModelsRemoteClient = client; + return client; + } + + /** @type {Promise | null} */ + static #languageIdWasmRecord = null; + + /** + * Retrieves the language-identification wasm binary from remote settings. + * + * @returns {Promise} + */ + static async #getLanguageIdWasmArrayBuffer() { + const start = Date.now(); + const client = TranslationsParent.#getTranslationsWasmRemoteClient(); + + // Load the wasm binary from remote settings, if it hasn't been already. + lazy.console.log(`Getting remote language-identification wasm binary.`); + if (!TranslationsParent.#languageIdWasmRecord) { + // Place the records into a promise to prevent any races. + TranslationsParent.#languageIdWasmRecord = (async () => { + /** @type {WasmRecord[]} */ + let wasmRecords = await TranslationsParent.getMaxVersionRecords( + client, + { + filters: { name: "fasttext-wasm" }, + } + ); + + if (wasmRecords.length === 0) { + // The remote settings client provides an empty list of records when there is + // an error. + throw new Error( + 'Unable to get "fasttext-wasm" language-identification wasm binary from Remote Settings.' + ); + } + + if (wasmRecords.length > 1) { + TranslationsParent.reportError( + new Error( + 'Expected the "fasttext-wasm" language-identification wasm collection to only have 1 record.' + ), + wasmRecords + ); + } + return wasmRecords[0]; + })(); + } + + try { + // Unlike the models, greedily download the wasm. It will pull it from a locale + // cache on disk if it's already been downloaded. Do not retain a copy, as + // this will be running in the parent process. It's not worth holding onto + // this much memory, so reload it every time it is needed. + + await chaosMode(1 / 3); + + /** @type {{buffer: ArrayBuffer}} */ + const { buffer } = await client.attachments.download( + await TranslationsParent.#languageIdWasmRecord + ); + + const duration = (Date.now() - start) / 1000; + lazy.console.log( + `Remote language-identification wasm binary loaded in ${duration} seconds.` + ); + + return buffer; + } catch (error) { + TranslationsParent.#languageIdWasmRecord = null; + throw error; + } + } + /** * Creates a lookup key that is unique to each fromLanguage-toLanguage pair. * @@ -1154,7 +1343,7 @@ export class TranslationsParent extends JSWindowActorParent { * This function should take a record as input and return a string that represents the lookup key for the record. * For most record types, the name (default) is sufficient, however if a collection contains records with * non-unique name values, it may be necessary to provide an alternative function here. - * @returns {Array} + * @returns {Array} */ static async getMaxVersionRecords( remoteSettingsClient, @@ -1521,6 +1710,12 @@ export class TranslationsParent extends JSWindowActorParent { queue.push({ download: () => TranslationsParent.#getBergamotWasmArrayBuffer(), }); + queue.push({ + download: () => TranslationsParent.#getLanguageIdModelArrayBuffer(), + }); + queue.push({ + download: () => TranslationsParent.#getLanguageIdWasmArrayBuffer(), + }); return downloadManager(queue); } @@ -1771,10 +1966,13 @@ export class TranslationsParent extends JSWindowActorParent { // Records. TranslationsParent.#bergamotWasmRecord = null; TranslationsParent.#translationModelRecords = null; + TranslationsParent.#languageIdModelRecord = null; + TranslationsParent.#languageIdWasmRecord = null; // Clients. TranslationsParent.#translationModelsRemoteClient = null; TranslationsParent.#translationsWasmRemoteClient = null; + TranslationsParent.#languageIdModelsRemoteClient = null; // Derived data. TranslationsParent.#preferredLanguages = null; @@ -1798,6 +1996,33 @@ export class TranslationsParent extends JSWindowActorParent { TranslationsParent.#isTranslationsEngineMocked = false; } + /** + * For testing purposes, allow the LanguageIdEngine to be mocked. If called + * with `null` in each argument, the mock is removed. + * + * @param {string} langTag - The BCP 47 language tag. + * @param {number} confidence - The confidence score of the detected language. + * @param {RemoteSettingsClient} client + */ + static mockLanguageIdentification(langTag, confidence, client) { + lazy.console.log("Mocking language identification.", { + langTag, + confidence, + }); + TranslationsParent.#mockedLangTag = langTag; + TranslationsParent.#mockedLanguageIdConfidence = confidence; + TranslationsParent.#languageIdModelsRemoteClient = client; + } + + /** + * Remove the mocks for the language identification, make sure and call clearCache after + * to remove the cached values. + */ + static unmockLanguageIdentification() { + lazy.console.log("Removing language identification mock."); + TranslationsParent.#mockedLangTag = null; + TranslationsParent.#mockedLanguageIdConfidence = null; + } /** * Report an error. Having this as a method allows tests to check that an error * was properly reported. @@ -1946,7 +2171,15 @@ export class TranslationsParent extends JSWindowActorParent { } async queryIdentifyLanguage() { - return this.sendQuery("Translations:IdentifyLanguage").catch(error => { + if ( + TranslationsParent.isInAutomation() && + !TranslationsParent.#mockedLangTag + ) { + return null; + } + return this.sendQuery("Translations:IdentifyLanguage", { + useFastText: lazy.useFastTextPref, + }).catch(error => { if (this.#isDestroyed) { // The actor was destroyed while this message was still being resolved. return null; @@ -2052,7 +2285,8 @@ export class TranslationsParent extends JSWindowActorParent { } } } else { - // If the document's markup had no specified langTag, attempt to identify the page's language. + // If the document's markup had no specified langTag, attempt + // to identify the page's language using the LanguageIdEngine. langTags.docLangTag = await this.queryIdentifyLanguage(); if (this.#isDestroyed) { return null; diff --git a/toolkit/components/translations/content/language-id-engine-worker.js b/toolkit/components/translations/content/language-id-engine-worker.js new file mode 100644 index 000000000000..9c0bd0039697 --- /dev/null +++ b/toolkit/components/translations/content/language-id-engine-worker.js @@ -0,0 +1,327 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env worker */ +"use strict"; + +// Throw Promise rejection errors so that they are visible in the console. +self.addEventListener("unhandledrejection", event => { + throw event.reason; +}); + +/* global addOnPostRun FastText loadFastText */ +importScripts( + "chrome://global/content/translations/fasttext.js", + "chrome://global/content/translations/fasttext_wasm.js" +); + +/** + * The number of languages that should be returned when the model analyzes text. + * + * A value of 1 means only the most-likely language will be returned. + * A value of 5 would mean that the top 5 most-likely languages will be returned. + */ +const LANGUAGE_COUNT = 1; + +/** + * The threshold of likelihood in range [0.0, 1.0] that must pass + * for a language to be returned from the model. + * + * A value of 0.0 would mean that a language is always returned with any confidence. + * A value of 0.5 would mean that a language is only returned if the model + * is 50% confident that the analyzed text could be that language. + */ +const CONFIDENCE_THRESHOLD = 0.0; + +// Respect the preference "browser.translations.logLevel". +let _isLoggingEnabled = true; +function log(...args) { + if (_isLoggingEnabled) { + console.log("Translations:", ...args); + } +} + +// Wait for the initialization request. +addEventListener("message", handleInitializationMessage); + +/** + * Initialize the engine, and get it ready to handle language identification requests. + * The "initialize" message must be received before any other message handling + * requests will be processed. + * + * @param {Object} event + * @param {Object} event.data + * @param {string} event.data.type - The message type, expects "initialize". + * @param {ArrayBuffer} event.data.wasmBuffer - The buffer containing the wasm binary. + * @param {ArrayBuffer} event.data.modelBuffer - The buffer containing the language-id model binary. + * @param {null | string} event.data.mockedLangTag - The mocked language tag value (only present when mocking). + * @param {null | number} event.data.mockedConfidence - The mocked confidence value (only present when mocking). + * @param {boolean} event.data.isLoggingEnabled + */ +async function handleInitializationMessage({ data }) { + if (data.type !== "initialize") { + throw new Error( + "The LanguageIdEngine worker received a message before it was initialized." + ); + } + + try { + const { isLoggingEnabled } = data; + if (isLoggingEnabled) { + // Respect the "browser.translations.logLevel" preference. + _isLoggingEnabled = true; + } + + /** @type {LanguageIdEngine | MockedLanguageIdEngine} */ + let languageIdEngine; + const { mockedLangTag, mockedConfidence } = data; + if (mockedLangTag !== null && mockedConfidence !== null) { + // Don't actually use the engine as it is mocked. + languageIdEngine = new MockedLanguageIdEngine( + mockedLangTag, + mockedConfidence + ); + } else { + languageIdEngine = await initializeLanguageIdEngine(data); + } + + handleMessages(languageIdEngine); + postMessage({ type: "initialization-success" }); + } catch (error) { + console.error(error); + postMessage({ type: "initialization-error", error: error?.message }); + } + + removeEventListener("message", handleInitializationMessage); +} + +/** + * Initializes the fastText wasm runtime and returns the fastText model. + * + * @param {ArrayBuffer} data.wasmBuffer - The buffer containing the wasm binary. + * @param {ArrayBuffer} data.modelBuffer - The buffer containing the language-id model binary. + * @returns {FastTextModel} + */ +function initializeFastTextModel(modelBuffer, wasmBuffer) { + return new Promise((resolve, reject) => { + const initialModule = { + onAbort() { + reject(new Error("Error loading the fastText Wasm Module")); + }, + onRuntimeInitialized() { + addOnPostRun(() => { + const ft = new FastText(initialModule); + const model = ft.loadModelBinary(modelBuffer); + resolve(model); + }); + }, + wasmBinary: wasmBuffer, + }; + loadFastText(initialModule); + }); +} + +/** + * Initialize the LanguageIdEngine from the data payload by loading + * the fastText wasm runtime and model and constructing the engine. + * + * @param {Object} data + * @property {ArrayBuffer} data.wasmBuffer - The buffer containing the wasm binary. + * @property {ArrayBuffer} data.modelBuffer - The buffer containing the language-id model binary. + */ +async function initializeLanguageIdEngine(data) { + const { modelBuffer, wasmBuffer } = data; + if (!modelBuffer) { + throw new Error('LanguageIdEngine initialization missing "modelBuffer"'); + } + if (!wasmBuffer) { + throw new Error('LanguageIdEngine initialization missing "wasmBuffer"'); + } + const model = await initializeFastTextModel(modelBuffer, wasmBuffer); + return new LanguageIdEngine(model); +} + +/** + * Sets up the message handling for the worker. + * + * @param {LanguageIdEngine | MockedLanguageIdEngine} languageIdEngine + */ +function handleMessages(languageIdEngine) { + /** + * Handle any message after the initialization message. + * + * @param {Object} data + * @property {string} data.type - The message type. + * @property {string} data.message - The message text to identify the language of. + * @property {number} data.messageId - The ID of the message. + */ + addEventListener("message", ({ data }) => { + try { + if (data.type === "initialize") { + throw new Error( + "The language-identification engine must not be re-initialized." + ); + } + switch (data.type) { + case "language-id-request": { + const { message, messageId } = data; + try { + const [confidence, langTag] = + languageIdEngine.identifyLanguage(message); + postMessage({ + type: "language-id-response", + langTag, + confidence, + messageId, + }); + } catch (error) { + console.error(error); + postMessage({ + type: "language-id-error", + messageId, + }); + } + break; + } + default: { + console.warn("Unknown message type:", data.type); + } + } + } catch (error) { + // Ensure the unexpected errors are surfaced in the console. + console.error(error); + } + }); +} + +/** + * The LanguageIdEngine wraps around a machine-learning model that can identify text + * as being written in a given human language. The engine is responsible for invoking + * model and returning the language tag in the format that is expected by firefox + * translations code. + */ +class LanguageIdEngine { + /** @type {FastTextModel} */ + #model; + + /** + * @param {FastTextModel} model + */ + constructor(model) { + this.#model = model; + } + + /** + * Formats the language tag returned by the language-identification model to match + * conform to the format used internally by Firefox. + * + * This function is currently configured to handle the fastText language-identification + * model. Updating the language-identification model or moving to something other than + * fastText in the future will likely require updating this function. + * + * @param {string} langTag + * @returns {string} The correctly formatted langTag + */ + #formatLangTag(langTag) { + // The fastText language model returns values of the format "__label__{langTag}". + // As such, this function strips the "__label__" prefix, leaving only the langTag. + let formattedTag = langTag.replace("__label__", ""); + + // fastText is capable of returning any of a predetermined set of 176 langTags: + // https://fasttext.cc/docs/en/language-identification.html + // + // These tags come from ISO639-3: + // https://iso639-3.sil.org/code_tables/deprecated_codes/data + // + // Each of these tags have been cross checked for compatibility with the IANA + // language subtag registry, which is used by BCP 47, and any edge cases are handled below. + // https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry + switch (formattedTag) { + // fastText may return "eml" which is a deprecated ISO639-3 language tag for the language + // Emiliano-Romagnolo. It was split into two separate tags "egl" and "rgn": + // https://iso639-3.sil.org/request/2008-040 + // + // "eml" was once requested to be added to the IANA registry, but it was denied: + // https://www.alvestrand.no/pipermail/ietf-languages/2009-December/009754.html + // + // This case should return either "egl" or "rgn", given that the "eml" tag was split. + // However, given that the fastText model does not distinguish between the two by using + // the deprecated tag, this function will default to "egl" because it is alphabetically first. + // + // At such a time that Firefox Translations may support either of these languages, we should consider + // a way to further distinguish between the two languages at that time. + case "eml": { + formattedTag = "egl"; + break; + } + // The fastText model returns "no" for Norwegian Bokmål. + // + // According to advice from https://r12a.github.io/app-subtags/ + // "no" is a macro language that encompasses the following more specific primary language subtags: "nb" "nn". + // It is recommended to use more specific language subtags as long as it does not break legacy usage of an application. + // As such, this function will return "nb" for Norwegian Bokmål instead of "no" as reported by fastText. + case "no": { + formattedTag = "nb"; + break; + } + } + return formattedTag; + } + + /** + * Identifies the human language in which the message is written and returns + * the BCP 47 language tag of the language it is determined to be along along + * with a rating of how confident the model is that the label is correct. + * + * @param {string} message + * @returns {Array} An array containing the confidence and language tag. + * The confidence is a number between 0 and 1, representing a percentage. + * The language tag is a BCP 47 language tag such as "en" for English. + * + * e.g. [0.87, "en"] + */ + identifyLanguage(message) { + const mostLikelyLanguageData = this.#model + .predict(message.trim(), LANGUAGE_COUNT, CONFIDENCE_THRESHOLD) + .get(0); + + // This should never fail as long as + // LANGUAGE_COUNT > 1 && CONFIDENCE_THRESHOLD === 0.0 + if (!mostLikelyLanguageData) { + throw new Error("Unable to identify a language"); + } + + const [confidence, langTag] = mostLikelyLanguageData; + return [confidence, this.#formatLangTag(langTag)]; + } +} + +/** + * For testing purposes, provide a fully mocked engine. This allows for easy integration + * testing of the UI, without having to rely on downloading remote models and remote + * wasm binaries. + */ +class MockedLanguageIdEngine { + /** @type {string} */ + #langTag; + /** @type {number} */ + #confidence; + + /** + * @param {string} langTag + * @param {number} confidence + */ + constructor(langTag, confidence) { + this.#langTag = langTag; + this.#confidence = confidence; + } + + /** + * Mocks identifying a language by returning the mocked engine's pre-determined + * language tag and confidence values. + */ + identifyLanguage(_message) { + return [this.#confidence, this.#langTag]; + } +} diff --git a/toolkit/components/translations/content/language-id-engine.sys.mjs b/toolkit/components/translations/content/language-id-engine.sys.mjs new file mode 100644 index 000000000000..83b027ebc573 --- /dev/null +++ b/toolkit/components/translations/content/language-id-engine.sys.mjs @@ -0,0 +1,224 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "logLevel", + "browser.translations.logLevel" +); + +ChromeUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +ChromeUtils.defineESModuleGetters(lazy, { + setTimeout: "resource://gre/modules/Timer.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +/** + * The threshold that the language-identification confidence + * value must be greater than in order to provide the detected language + * tag for translations. + * + * This value should ideally be one that does not allow false positives + * while also not being too restrictive. + * + * At this time, this value is not driven by statistical data or analysis. + */ +const DOC_LANGUAGE_DETECTION_THRESHOLD = 0.65; + +/** + * The length of the substring to pull from the document's text for language + * identification. + * + * This value should ideally be one that is large enough to yield a confident + * identification result without being too large or expensive to extract. + * + * At this time, this value is not driven by statistical data or analysis. + * + * For the moment, while we investigate which language identification library + * we would like to use, keep this logic in sync with LanguageDetector.sys.mjs + */ +const DOC_TEXT_TO_IDENTIFY_LENGTH = 1024; + +export class LanguageIdEngine { + /** @type {Worker} */ + #languageIdWorker; + // Multiple messages can be sent before a response is received. This ID is used to keep + // track of the messages. It is incremented on every use. + #messageId = 0; + + static #cachedEngine = null; + static #cachedEngineTimeoutId = null; + static #cachedEngineTimeoutMS = 30_000; + + /** + * Gets a cached engine, or creates a new one. Returns `null` when the engine + * payload fails to download. + * + * @param {() => Object} getPayload + * @returns {LanguageIdEngine | null} + */ + static getOrCreate(getPayload) { + if (!this.#cachedEngine) { + this.#cachedEngine = LanguageIdEngine.#create(getPayload); + } + return this.#cachedEngine; + } + + /** + * @param {() => Object} getPayload + * @returns {Promise} + */ + static async #create(getPayload) { + let payload; + try { + payload = await getPayload(); + } catch (error) { + // The payload may not be able to be downloaded. Report this as a normal + // console.log, as this is the default behavior in automation. + lazy.console.log( + "The language id payload was unable to be downloaded.", + error + ); + return null; + } + + const engine = new LanguageIdEngine(payload); + await engine.isReady; + LanguageIdEngine.#resetCacheTimeout(); + return engine; + } + + static #resetCacheTimeout() { + if (LanguageIdEngine.#cachedEngineTimeoutId) { + lazy.clearTimeout(LanguageIdEngine.#cachedEngineTimeoutId); + } + LanguageIdEngine.#cachedEngineTimeoutId = lazy.setTimeout( + LanguageIdEngine.#clearEngineCache, + LanguageIdEngine.#cachedEngineTimeoutMS + ); + } + + static #clearEngineCache() { + lazy.console.log("Clearing the engine cache"); + LanguageIdEngine.#cachedEngine = null; + LanguageIdEngine.#cachedEngineTimeoutId = null; + } + + /** + * Construct and initialize the language-id worker. + * + * @param {Object} data + * @param {string} data.type - The message type, expects "initialize". + * @param {ArrayBuffer} data.wasmBuffer - The buffer containing the wasm binary. + * @param {ArrayBuffer} data.modelBuffer - The buffer containing the language-id model binary. + * @param {null | string} data.mockedLangTag - The mocked language tag value (only present when mocking). + * @param {null | number} data.mockedConfidence - The mocked confidence value (only present when mocking). + * @param {boolean} data.isLoggingEnabled + */ + constructor(data) { + this.#languageIdWorker = new Worker( + "chrome://global/content/translations/language-id-engine-worker.js" + ); + + this.isReady = new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + if (data.type === "initialization-success") { + resolve(); + } else if (data.type === "initialization-error") { + reject(data.error); + } + this.#languageIdWorker.removeEventListener("message", onMessage); + }; + this.#languageIdWorker.addEventListener("message", onMessage); + }); + + const transferables = []; + // Make sure the ArrayBuffers are transferred, not cloned. + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects + transferables.push(data.wasmBuffer, data.modelBuffer); + + this.#languageIdWorker.postMessage( + { + type: "initialize", + isLoggingEnabled: lazy.logLevel === "All", + ...data, + }, + transferables + ); + } + + /** + * Attempts to identify the human language in which the message is written. + * Generally, the longer a message is, the higher the likelihood that the + * identified language will be correct. Shorter messages increase the chance + * of false identification. + * + * The returned confidence is a number between 0.0 and 1.0 of how confident + * the language identification model was that it identified the correct language. + * + * @param {string} message + * @returns {Promise<{ langTag: string, confidence: number }>} + */ + identifyLanguage(message) { + LanguageIdEngine.#resetCacheTimeout(); + const messageId = this.#messageId++; + return new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + if (data.messageId !== messageId) { + // Multiple translation requests can be sent before a response is received. + // Ensure that the response received here is the correct one. + return; + } + if (data.type === "language-id-response") { + let { langTag, confidence } = data; + resolve({ langTag, confidence }); + } + if (data.type === "language-id-error") { + reject(data.error); + } + this.#languageIdWorker.removeEventListener("message", onMessage); + }; + this.#languageIdWorker.addEventListener("message", onMessage); + this.#languageIdWorker.postMessage({ + type: "language-id-request", + message, + messageId, + }); + }); + } + + /** + * Attempts to determine the language in which the document's content is written. + * + * For the moment, while we investigate which language identification library + * we would like to use, keep this logic in sync with LanguageDetector.sys.mjs + * @returns {string | null} + */ + async identifyLanguageFromDocument(document) { + // Grab a selection of text. + let encoder = Cu.createDocumentEncoder("text/plain"); + encoder.init(document, "text/plain", encoder.SkipInvisibleContent); + let text = encoder + .encodeToStringWithMaxLength(DOC_TEXT_TO_IDENTIFY_LENGTH) + .replaceAll("\r", "") + .replaceAll("\n", " "); + + let { langTag, confidence } = await this.identifyLanguage(text); + + lazy.console.log( + `${langTag}(${confidence.toFixed(2)}) Detected Page Language` + ); + return confidence >= DOC_LANGUAGE_DETECTION_THRESHOLD ? langTag : null; + } +} diff --git a/toolkit/components/translations/content/translations.mjs b/toolkit/components/translations/content/translations.mjs index 0ec8b2d47550..12bce88eb0e6 100644 --- a/toolkit/components/translations/content/translations.mjs +++ b/toolkit/components/translations/content/translations.mjs @@ -8,7 +8,7 @@ /* global AT_getSupportedLanguages, AT_log, AT_getScriptDirection, AT_logError, AT_createTranslationsPort, AT_isHtmlTranslation, - AT_isTranslationEngineSupported, AT_identifyLanguage */ + AT_isTranslationEngineSupported, AT_createLanguageIdEngine, AT_identifyLanguage */ // Allow tests to override this value so that they can run faster. // This is the delay in milliseconds. @@ -79,6 +79,14 @@ class TranslationsState { */ this.isTranslationEngineSupported = isSupported; + /** + * Allow code to wait for the engine to be created. + * @type {Promise} + */ + this.languageIdEngineCreated = isSupported + ? AT_createLanguageIdEngine() + : Promise.resolve(); + /** * @type {SupportedLanguages} */ @@ -90,13 +98,12 @@ class TranslationsState { this.ui.setup(); // Set the UI as ready after all of the state promises have settled. - this.supportedLanguages - .then(() => { - this.ui.setAsReady(); - }) - .catch(error => { - AT_logError("Failed to load the supported languages", error); - }); + Promise.allSettled([ + this.languageIdEngineCreated, + this.supportedLanguages, + ]).then(() => { + this.ui.setAsReady(); + }); } /** @@ -108,6 +115,7 @@ class TranslationsState { * @param {string} message */ async identifyLanguage(message) { + await this.languageIdEngineCreated; const start = performance.now(); const { langTag, confidence } = await AT_identifyLanguage(message); const duration = performance.now() - start; diff --git a/toolkit/components/translations/docs/resources/01_overview.md b/toolkit/components/translations/docs/resources/01_overview.md index 632b3002dfa9..7f4333265bb4 100644 --- a/toolkit/components/translations/docs/resources/01_overview.md +++ b/toolkit/components/translations/docs/resources/01_overview.md @@ -80,16 +80,20 @@ architecture to identify content as being written in a detected language. ### Technology -Firefox Translations utilizes a [CLD2] language detector to identify in which language content is written. +Firefox Translations utilizes a [WASM] version of the [fastText] library to identify in which +language content is written. ### Models -No models are currently used for language identification, since [CLD2] exists in the Firefox source tree. +Unlike the language translations models in the [section](#language-translations) above, the [fastText] +model is a is a one-to-many model that is capable of detecting all of our supported languages +from the single model. --- ## Remote Settings -Remote Settings is not currently used for language identification, since [CLD2] exists in the Firefox source tree. +Firefox Translations utilizes [Remote Settings] to download [WASM] binaries, [Language Translation](#language-translation) +models and [Language Identification](#language-identification) models to use locally on your system. --- ## Using Firefox Translations @@ -135,7 +139,7 @@ It is, however, useful and fun, so it is documented here. [Bergamot]: https://browser.mt/ -[CLD2]: https://github.com/CLD2Owners/cld2 +[fastText]: https://fasttext.cc/ [Firefox Nightly]: https://www.mozilla.org/en-US/firefox/channel/desktop/ [Marian]: https://aclanthology.org/P18-4020/ [Remote Settings]: https://remote-settings.readthedocs.io/en/latest/ diff --git a/toolkit/components/translations/docs/resources/02_contributing.md b/toolkit/components/translations/docs/resources/02_contributing.md index b04509723bc6..d7c66bc50bed 100644 --- a/toolkit/components/translations/docs/resources/02_contributing.md +++ b/toolkit/components/translations/docs/resources/02_contributing.md @@ -13,11 +13,11 @@ to provide helpful information regarding contributing to Firefox Translations. - [Versioning](#versioning) - [Non-Breaking Changes](#non-breaking-changes) - [Breaking Changes](#breaking-changes) +- [Building fastText](#building-fasttext) - [Downloading The Models](#downloading-the-models) - [Building the WASM Binary](#building-the-wasm-binary) - [Dependencies](#dependencies) - [Modifying the EMCXXFLAGS](#modifying-the-emcxxflags) -- [Language Identification](#language-identification) - [Building Bergamot](#building-bergamot) --- @@ -127,11 +127,290 @@ Tying breaking changes to releases in this way frees up Firefox Translations to switching one third-party library for another in the compiled source code, while allowing older versions of Firefox to continue utilizing the old library and allowing newer versions of Firefox to utilize the new library. --- -## Language Identification +## Building fastText -Translations currently uses the [CLD2] language detector. +### Downloading the Models -We have previously experimented with using the [fastText] language detector, but we opted to use [CLD2] due to complications with [fastText] [WASM] runtime performance. The benefit of the [CLD2] language detector is that it already exists in the Firefox source tree. In the future, we would still like to explore moving to a more modern language detector such as [CLD3], or perhaps something else. +The fastText model that we use can be downloaded directly from the fastText website:
+> [https://fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html) + +Firefox Translations uses the compressed, **`lid.176.ftz`** model. + +### Building the WASM Binary + +To build the fastText [WASM] binary, we can follow the steps in the [Requirements] section of the fastText website. + +#### Dependencies + +**C++ Compiler**
+Any of the C++ compilers from [Getting Set Up To Work On The Firefox Codebase] will be sufficient for this. + +**emskd**
+Follow the [Download and Install] instructions for setting up the emscripten sdk. + +#### Modifying the EMCXXFLAGS + +At the time of writing, the a latest commit on the fastText repo ([3697152e0fd772d9185697fdbd4a1d340ca5571d]) +is not compatible by default with the latest version of [emscripten (3.1.35)]. + +A few changes need to be made to the Makefile in order to generate the fastText [WASM] for use in Firefox. + +**1) Disable DYNAMIC_EXECUTION**
+In the `Makefile` for the fastText repo, there is a variable called **`EMCXXFLAGS`**.
+We need to add the following flag to this variable: + +``` +-s "DYNAMIC_EXECUTION=0" +``` + +If this flag is not set to **`0`**, then emscripten will [generate functions] that use the [eval()] function. +[eval()] is not allowed in the context that fastText runs in FireFox due to security reasons. + +**2) Rename EXTRA_EXPORTED_RUNTIME_METHODS**
+In [emscripten (2.0.18)], **`EXTRA_EXPORTED_RUNTIME_METHODS`** was deprecated in favor of **`EXPORTED_RUNTIME_METHODS`**. +The fastText Makefile still has the old flag, so we need to update the name. + +**3) Use the -r Flag When Appropriate**
+In [emscripten (2.0.3)] the following change was made: + +> "The default output format is now executable JavaScript. Previously we would default to output objecting files unless, for example, the output name ended in **`.js`**. This is contrary to behavior of clang and gcc. Now emscripten will always produce and executable unless the **`-c`**, **`-r`** or **`-shared`** flags are given. This is true even when the name of the output file ends in **`.o`**. e.g, **`emcc foo.c -o foo.o`** will produce a JavaScript file called **`foo.o`**. This might surprise some users (although it matches the behavior of existing toolchains) so we now produce a warning in this case." + +The Makefile needs to be modified to use the **`-r`** flag when appropriate. These changes are modeled after comments on this [GitHub Issue]. + +**Cumulative Changes**
+Here is a diff of the full changes needed for the Makefile at the time of writing: + +```diff +diff --git a/Makefile b/Makefile +index e246f79..396ae0b 100644 +--- a/Makefile ++++ b/Makefile +@@ -73,7 +73,9 @@ clean: + + EMCXX = em++ +-EMCXXFLAGS = --bind --std=c++11 -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s "EXTRA_EXPORTED_RUNTIME_METHODS=['addOnPostRun', 'FS']" -s "DISABLE_EXCEPTION_CATCHING=0" -s "EXCEPTION_DEBUG=1" -s "FORCE_FILESYSTEM=1" -s "MODULARIZE=1" -s "EXPORT_ES6=1" -s 'EXPORT_NAME="FastTextModule"' -Isrc/ ++EMCXXFLAGS_BASE = --bind --std=c++11 -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s "EXPORTED_RUNTIME_METHODS=['addOnPostRun', 'FS']" -s "DISABLE_EXCEPTION_CATCHING=0" -s "EXCEPTION_DEBUG=0" -s "DYNAMIC_EXECUTION=0" -s "FORCE_FILESYSTEM=1" -s "MODULARIZE=1" -s "EXPORT_ES6=1" -s 'EXPORT_NAME="FastTextModule"' -Isrc/ ++EMCXXFLAGS = $(EMCXXFLAGS_BASE) -r ++EMCXXFLAGS_JS = $(EMCXXFLAGS_BASE) + EMOBJS = args.bc autotune.bc matrix.bc dictionary.bc loss.bc productquantizer.bc densematrix.bc quantmatrix.bc vector.bc model.bc utils.bc meter.bc fasttext.bc main.bc + + +@@ -120,6 +122,6 @@ fasttext.bc: src/fasttext.cc src/*.h + $(EMCXX) $(EMCXXFLAGS) src/fasttext.cc -o fasttext.bc + + webassembly/fasttext_wasm.js: $(EMOBJS) webassembly/fasttext_wasm.cc Makefile +- $(EMCXX) $(EMCXXFLAGS) $(EMOBJS) -o webassembly/fasttext_wasm.js ++ $(EMCXX) $(EMCXXFLAGS_JS) $(EMOBJS) -o webassembly/fasttext_wasm.js +``` + +After modifying the Makefile in the previous section, running **`make wasm`** in the fastText repo should run without warnings or errors and the following files will be generated in the **`webassembly`** directory: + +``` +webassembly +├── fasttext.js +├── fasttext_wasm.js +└── fasttext_wasm.wasm +``` + +#### Modifying fasttext_wasm.js + +There are a few changes we need to make to the **`fasttext_wasm.js`** file to make it compatible with use in Firefox. + +**1) Define a function, not a module**
+The generated code exports a module, but this needs to be modified into a function for use in [importScripts()] in a worker. + +At the top of the file we need to make the following changes: + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js +index 64c6184a85851..4802343da2a03 100644 +--- a/toolkit/components/translations/fasttext/fasttext_wasm.js ++++ b/toolkit/components/translations/fasttext/fasttext_wasm.js +@@ -1,9 +1,6 @@ + +-var FastTextModule = (() => { +- var _scriptDir = import.meta.url; +- +- return ( +-async function(FastTextModule = {}) { ++async function loadFastTextModule(FastTextModule = {}) { ++ const _scriptDir = null; + + // include: shell.js + // The Module object: Our interface to the outside world. We import +``` + +Here we are defining a function rather than a variable, and we are setting **`_scriptDir`** to null +because **`import.meta.url`** is only available for use within modules. + +Next we need to modify the bottom of the file to match these changes: + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js +index 64c6184a85851..0a6fca3f524e4 100644 +--- a/toolkit/components/translations/fasttext/fasttext_wasm.js ++++ b/toolkit/components/translations/fasttext/fasttext_wasm.js +@@ -8287,7 +8287,3 @@ run(); + + return FastTextModule.ready + } +- +-); +-})(); +-export default FastTextModule; +``` + +**2) Remove unneeded environment checks**
+Next we need to remove unneeded checks for different environments: + +```JavaScript +if (ENVIRONMENT_IS_NODE) { + // ... +} else +if (ENVIRONMENT_IS_SHELL) { + // ... +} else +if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + // ... +} else +{ + throw new Error('environment detection error'); +} +``` + +Since this code will only be run inside of a worker, we want to delete the blocks that deal with **`ENVIRONMENT_IS_NODE`** and **`ENVIRONMENT_IS_SHELL`**. In fact, this code will fail to be imported by [importScripts()] if we don't do this. + +**3) Remove the use of `import.meta.url`**
+Finally, there is a use of **`import.meta.url`** that we need to remove. + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js +index 64c6184a85851..746cbae2ec952 100644 +--- a/toolkit/components/translations/fasttext/fasttext_wasm.js ++++ b/toolkit/components/translations/fasttext/fasttext_wasm.js +@@ -746,7 +746,7 @@ if (Module['locateFile']) { + } + } else { + // Use bundler-friendly `new URL(..., import.meta.url)` pattern; works in browsers too. +- wasmBinaryFile = new URL('fasttext_wasm.wasm', import.meta.url).href; ++ wasmBinaryFile = null; + } + + function getBinary(file) { +``` + +As mentioned before, **`import.meta.url`** is not allowed outside of modules and cannot be used with [importScripts()] +in the worker code that we are creating. + +It is okay to set this to null here, because we will be providing the **`wasmBinaryFile`** via [Remote Settings]. + +**4) Minifying the file**
+The generated **`fasttext_wasm.js`** file is very large. To minimize the impact on the size of the code in the Firefox source tree, we want to minify the file using the [minify] tool. + +``` +Size Name +291k ├── fasttext_wasm.js (original) +109k └── fasttext_wasm.js (minified) +``` + +**5) Adding the license**
+Finally, we should add a copy of the current fastText MIT license to the top of the minified **`fasttext_wasm.js`** file. +You should be able to paste this from the generated **`fasttext.js`** file. + +#### Modifying fasttext.js + +```{note} +It is likely that the source file in tree already has these changes and is already sufficient, +even if **`fasttext_wasm.js`** has been recently updated. Try running it first as-is before replacing +and re-modifying. +``` + +Next we need to modify **`fasttext.js`** to utilize the changes that we made to **`fasttext_wasm.js`** and also to +not be a module so that we can import it using [importScripts()]. + +These changes do the following: + +1) Define a variable called **`fastTextModule`** for use in the worker scripts. +2) Utilize the **`loadFastTextModule()`** function that we defined in **`fasttext_wasm.js`** +3) Add a function **`loadModelBinary()`** that takes the wasm binary directly, which we will provide through [Remote Settings]. +4) Remove any module exports. + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext.js b/toolkit/components/translations/fasttext/fasttext.js +index 86600b9ac9e28..2c49b3faaeedc 100644 +--- a/toolkit/components/translations/fasttext/fasttext.js ++++ b/toolkit/components/translations/fasttext/fasttext.js +@@ -6,20 +6,30 @@ + * LICENSE file in the root directory of this source tree. + */ + +-import fastTextModularized from './fasttext_wasm.js'; +-const fastTextModule = fastTextModularized(); ++let fastTextModule; ++ ++const _initFastTextModule = async function (wasmModule) { ++ try { ++ fastTextModule = await loadFastTextModule(wasmModule); ++ } catch(e) { ++ console.error(e); ++ } ++ return true ++} + + let postRunFunc = null; + const addOnPostRun = function(func) { + postRunFunc = func; + }; + +-fastTextModule.addOnPostRun(() => { +- if (postRunFunc) { +- postRunFunc(); +- } +-}); + ++const loadFastText = (wasmModule) => { ++ _initFastTextModule(wasmModule).then((res) => { ++ if (postRunFunc) { ++ postRunFunc(); ++ } ++ }) ++} + const thisModule = this; + const trainFileInWasmFs = 'train.txt'; + const testFileInWasmFs = 'test.txt'; +@@ -41,7 +51,7 @@ const getFloat32ArrayFromHeap = (len) => { + const heapToFloat32 = (r) => new Float32Array(r.buffer, r.ptr, r.size); + + class FastText { +- constructor() { ++ constructor(fastTextModule) { + this.f = new fastTextModule.FastText(); + } + +@@ -77,6 +87,15 @@ class FastText { + }); + } + ++ loadModelBinary(buffer) { ++ const fastTextNative = this.f; ++ const byteArray = new Uint8Array(buffer); ++ const FS = fastTextModule.FS; ++ FS.writeFile(modelFileInWasmFs, byteArray); ++ fastTextNative.loadModel(modelFileInWasmFs); ++ return new FastTextModel(fastTextNative); ++ } ++ + _train(url, modelName, kwargs = {}, callback = null) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + const fastTextNative = this.f; +@@ -515,6 +534,3 @@ class FastTextModel { + }); + } + } +- +- +-export {FastText, addOnPostRun}; +``` --- ## Building Bergamot @@ -140,21 +419,20 @@ TODO +[3697152e0fd772d9185697fdbd4a1d340ca5571d]: https://github.com/facebookresearch/fastText/tree/3697152e0fd772d9185697fdbd4a1d340ca5571d [Bugzilla]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Cloud%20Services&component=Server%3A%20Remote%20Settings [Child]: https://searchfox.org/mozilla-central/search?q=TranslationsChild -[CLD2]: https://github.com/CLD2Owners/cld2 -[CLD3]: https://github.com/google/cld3 [Download and Install]: https://emscripten.org/docs/getting_started/downloads.html#download-and-install [emscripten (2.0.3)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#203-09102020 [emscripten (2.0.18)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#2018-04232021 [emscripten (3.1.35)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3135---040323 [Environments]: https://remote-settings.readthedocs.io/en/latest/getting-started.html#environments [eval()]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/eval -[fastText]: https://fasttext.cc/ [Filter Expressions]: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions [Firefox Release Schedule]: https://wiki.mozilla.org/Release_Management/Calendar [generate functions]: https://emscripten.org/docs/api_reference/emscripten.h.html?highlight=dynamic_execution#functions [Getting Set Up To Work On The Firefox Codebase]: https://firefox-source-docs.mozilla.org/setup/index.html +[GitHub Issue]: https://github.com/facebookresearch/fastText/pull/1227#issuecomment-1353830003 [importScripts()]: https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope/importScripts [JSWindowActors]: https://firefox-source-docs.mozilla.org/dom/ipc/jsactors.html#jswindowactor [minify]: https://github.com/tdewolff/minify @@ -162,6 +440,7 @@ TODO [Step 3]: https://remote-settings.readthedocs.io/en/latest/getting-started.html#create-a-new-official-type-of-remote-settings [remote-settings-devtools]: https://github.com/mozilla-extensions/remote-settings-devtools/releases [Remote Settings]: https://remote-settings.readthedocs.io/en/latest/ +[Requirements]: https://fasttext.cc/docs/en/webassembly-module.html#requirements [toolkit/components/translations]: https://searchfox.org/mozilla-central/search?q=toolkit%2Fcomponents%2Ftranslations [WASM]: https://webassembly.org/ [Workers]: https://searchfox.org/mozilla-central/search?q=%2Ftranslations.*worker&path=&case=false®exp=true diff --git a/toolkit/components/translations/fasttext/LICENSE b/toolkit/components/translations/fasttext/LICENSE new file mode 100644 index 000000000000..5a14f2864aa9 --- /dev/null +++ b/toolkit/components/translations/fasttext/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016-present, Facebook, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkit/components/translations/fasttext/fasttext.js b/toolkit/components/translations/fasttext/fasttext.js new file mode 100644 index 000000000000..a79dfeffa010 --- /dev/null +++ b/toolkit/components/translations/fasttext/fasttext.js @@ -0,0 +1,536 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +let fastTextModule; + +const _initFastTextModule = async function (wasmModule) { + try { + fastTextModule = await loadFastTextModule(wasmModule); + } catch(e) { + console.error(e); + } + return true +} + +let postRunFunc = null; +const addOnPostRun = function (func) { + postRunFunc = func; +}; + + +const loadFastText = (wasmModule) => { + _initFastTextModule(wasmModule).then((res) => { + if (postRunFunc) { + postRunFunc(); + } + }) +} +const thisModule = this; +const trainFileInWasmFs = 'train.txt'; +const testFileInWasmFs = 'test.txt'; +const modelFileInWasmFs = 'model.bin'; + +const getFloat32ArrayFromHeap = (len) => { + const dataBytes = len * Float32Array.BYTES_PER_ELEMENT; + const dataPtr = fastTextModule._malloc(dataBytes); + const dataHeap = new Uint8Array(fastTextModule.HEAPU8.buffer, + dataPtr, + dataBytes); + return { + 'ptr':dataHeap.byteOffset, + 'size':len, + 'buffer':dataHeap.buffer + }; +}; + +const heapToFloat32 = (r) => new Float32Array(r.buffer, r.ptr, r.size); + +class FastText { + constructor(fastTextModule) { + this.f = new fastTextModule.FastText(); + } + + /** + * loadModel + * + * Loads the model file from the specified url, and returns the + * corresponding `FastTextModel` object. + * + * @param {string} url + * the url of the model file. + * + * @return {Promise} promise object that resolves to a `FastTextModel` + * + */ + loadModel(url) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + + const fastTextNative = this.f; + return new Promise(function(resolve, reject) { + fetchFunc(url).then(response => { + return response.arrayBuffer(); + }).then(bytes => { + const byteArray = new Uint8Array(bytes); + const FS = fastTextModule.FS; + FS.writeFile(modelFileInWasmFs, byteArray); + }).then(() => { + fastTextNative.loadModel(modelFileInWasmFs); + resolve(new FastTextModel(fastTextNative)); + }).catch(error => { + reject(error); + }); + }); + } + + loadModelBinary(buffer) { + const fastTextNative = this.f; + const byteArray = new Uint8Array(buffer); + const FS = fastTextModule.FS; + FS.writeFile(modelFileInWasmFs, byteArray); + fastTextNative.loadModel(modelFileInWasmFs); + return new FastTextModel(fastTextNative); + } + + _train(url, modelName, kwargs = {}, callback = null) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + const fastTextNative = this.f; + + return new Promise(function(resolve, reject) { + fetchFunc(url).then(response => { + return response.arrayBuffer(); + }).then(bytes => { + const byteArray = new Uint8Array(bytes); + const FS = fastTextModule.FS; + FS.writeFile(trainFileInWasmFs, byteArray); + }).then(() => { + const argsList = ['lr', 'lrUpdateRate', 'dim', 'ws', 'epoch', + 'minCount', 'minCountLabel', 'neg', 'wordNgrams', 'loss', + 'model', 'bucket', 'minn', 'maxn', 't', 'label', 'verbose', + 'pretrainedVectors', 'saveOutput', 'seed', 'qout', 'retrain', + 'qnorm', 'cutoff', 'dsub', 'qnorm', 'autotuneValidationFile', + 'autotuneMetric', 'autotunePredictions', 'autotuneDuration', + 'autotuneModelSize']; + const args = new fastTextModule.Args(); + argsList.forEach(k => { + if (k in kwargs) { + args[k] = kwargs[k]; + } + }); + args.model = fastTextModule.ModelName[modelName]; + args.loss = ('loss' in kwargs) ? + fastTextModule.LossName[kwargs['loss']] : 'hs'; + args.thread = 1; + args.input = trainFileInWasmFs; + + fastTextNative.train(args, callback); + + resolve(new FastTextModel(fastTextNative)); + }).catch(error => { + reject(error); + }); + }); + } + + /** + * trainSupervised + * + * Downloads the input file from the specified url, trains a supervised + * model and returns a `FastTextModel` object. + * + * @param {string} url + * the url of the input file. + * The input file must must contain at least one label per line. For an + * example consult the example datasets which are part of the fastText + * repository such as the dataset pulled by classification-example.sh. + * + * @param {dict} kwargs + * train parameters. + * For example {'lr': 0.5, 'epoch': 5} + * + * @param {function} callback + * train callback function + * `callback` function is called regularly from the train loop: + * `callback(progress, loss, wordsPerSec, learningRate, eta)` + * + * @return {Promise} promise object that resolves to a `FastTextModel` + * + */ + trainSupervised(url, kwargs = {}, callback) { + const self = this; + return new Promise(function(resolve, reject) { + self._train(url, 'supervised', kwargs, callback).then(model => { + resolve(model); + }).catch(error => { + reject(error); + }); + }); + } + + /** + * trainUnsupervised + * + * Downloads the input file from the specified url, trains an unsupervised + * model and returns a `FastTextModel` object. + * + * @param {string} url + * the url of the input file. + * The input file must not contain any labels or use the specified label + * prefixunless it is ok for those words to be ignored. For an example + * consult the dataset pulled by the example script word-vector-example.sh + * which is part of the fastText repository. + * + * @param {string} modelName + * Model to be used for unsupervised learning. `cbow` or `skipgram`. + * + * @param {dict} kwargs + * train parameters. + * For example {'lr': 0.5, 'epoch': 5} + * + * @param {function} callback + * train callback function + * `callback` function is called regularly from the train loop: + * `callback(progress, loss, wordsPerSec, learningRate, eta)` + * + * @return {Promise} promise object that resolves to a `FastTextModel` + * + */ + trainUnsupervised(url, modelName, kwargs = {}, callback) { + const self = this; + return new Promise(function(resolve, reject) { + self._train(url, modelName, kwargs, callback).then(model => { + resolve(model); + }).catch(error => { + reject(error); + }); + }); + } + +} + + +class FastTextModel { + /** + * `FastTextModel` represents a trained model. + * + * @constructor + * + * @param {object} fastTextNative + * webassembly object that makes the bridge between js and C++ + */ + constructor(fastTextNative) { + this.f = fastTextNative; + } + + /** + * isQuant + * + * @return {bool} true if the model is quantized + * + */ + isQuant() { + return this.f.isQuant; + } + + /** + * getDimension + * + * @return {int} the dimension (size) of a lookup vector (hidden layer) + * + */ + getDimension() { + return this.f.args.dim; + } + + /** + * getWordVector + * + * @param {string} word + * + * @return {Float32Array} the vector representation of `word`. + * + */ + getWordVector(word) { + const b = getFloat32ArrayFromHeap(this.getDimension()); + this.f.getWordVector(b, word); + + return heapToFloat32(b); + } + + /** + * getSentenceVector + * + * @param {string} text + * + * @return {Float32Array} the vector representation of `text`. + * + */ + getSentenceVector(text) { + if (text.indexOf('\n') != -1) { + "sentence vector processes one line at a time (remove '\\n')"; + } + text += '\n'; + const b = getFloat32ArrayFromHeap(this.getDimension()); + this.f.getSentenceVector(b, text); + + return heapToFloat32(b); + } + + /** + * getNearestNeighbors + * + * returns the nearest `k` neighbors of `word`. + * + * @param {string} word + * @param {int} k + * + * @return {Array.>} + * words and their corresponding cosine similarities. + * + */ + getNearestNeighbors(word, k = 10) { + return this.f.getNN(word, k); + } + + /** + * getAnalogies + * + * returns the nearest `k` neighbors of the operation + * `wordA - wordB + wordC`. + * + * @param {string} wordA + * @param {string} wordB + * @param {string} wordC + * @param {int} k + * + * @return {Array.>} + * words and their corresponding cosine similarities + * + */ + getAnalogies(wordA, wordB, wordC, k) { + return this.f.getAnalogies(k, wordA, wordB, wordC); + } + + /** + * getWordId + * + * Given a word, get the word id within the dictionary. + * Returns -1 if word is not in the dictionary. + * + * @return {int} word id + * + */ + getWordId(word) { + return this.f.getWordId(word); + } + + /** + * getSubwordId + * + * Given a subword, return the index (within input matrix) it hashes to. + * + * @return {int} subword id + * + */ + getSubwordId(subword) { + return this.f.getSubwordId(subword); + } + + /** + * getSubwords + * + * returns the subwords and their indicies. + * + * @param {string} word + * + * @return {Pair., Array.>} + * words and their corresponding indicies + * + */ + getSubwords(word) { + return this.f.getSubwords(word); + } + + /** + * getInputVector + * + * Given an index, get the corresponding vector of the Input Matrix. + * + * @param {int} ind + * + * @return {Float32Array} the vector of the `ind`'th index + * + */ + getInputVector(ind) { + const b = getFloat32ArrayFromHeap(this.getDimension()); + this.f.getInputVector(b, ind); + + return heapToFloat32(b); + } + + /** + * predict + * + * Given a string, get a list of labels and a list of corresponding + * probabilities. k controls the number of returned labels. + * + * @param {string} text + * @param {int} k, the number of predictions to be returned + * @param {number} probability threshold + * + * @return {Array.>} + * labels and their probabilities + * + */ + predict(text, k = 1, threshold = 0.0) { + return this.f.predict(text, k, threshold); + } + + /** + * getInputMatrix + * + * Get a reference to the full input matrix of a Model. This only + * works if the model is not quantized. + * + * @return {DenseMatrix} + * densematrix with functions: `rows`, `cols`, `at(i,j)` + * + * example: + * let inputMatrix = model.getInputMatrix(); + * let value = inputMatrix.at(1, 2); + */ + getInputMatrix() { + if (this.isQuant()) { + throw new Error("Can't get quantized Matrix"); + } + return this.f.getInputMatrix(); + } + + /** + * getOutputMatrix + * + * Get a reference to the full input matrix of a Model. This only + * works if the model is not quantized. + * + * @return {DenseMatrix} + * densematrix with functions: `rows`, `cols`, `at(i,j)` + * + * example: + * let outputMatrix = model.getOutputMatrix(); + * let value = outputMatrix.at(1, 2); + */ + getOutputMatrix() { + if (this.isQuant()) { + throw new Error("Can't get quantized Matrix"); + } + return this.f.getOutputMatrix(); + } + + /** + * getWords + * + * Get the entire list of words of the dictionary including the frequency + * of the individual words. This does not include any subwords. For that + * please consult the function get_subwords. + * + * @return {Pair., Array.>} + * words and their corresponding frequencies + * + */ + getWords() { + return this.f.getWords(); + } + + /** + * getLabels + * + * Get the entire list of labels of the dictionary including the frequency + * of the individual labels. + * + * @return {Pair., Array.>} + * labels and their corresponding frequencies + * + */ + getLabels() { + return this.f.getLabels(); + } + + /** + * getLine + * + * Split a line of text into words and labels. Labels must start with + * the prefix used to create the model (__label__ by default). + * + * @param {string} text + * + * @return {Pair., Array.>} + * words and labels + * + */ + getLine(text) { + return this.f.getLine(text); + } + + /** + * saveModel + * + * Saves the model file in web assembly in-memory FS and returns a blob + * + * @return {Blob} blob data of the file saved in web assembly FS + * + */ + saveModel() { + this.f.saveModel(modelFileInWasmFs); + const content = fastTextModule.FS.readFile(modelFileInWasmFs, + { encoding: 'binary' }); + return new Blob( + [new Uint8Array(content, content.byteOffset, content.length)], + { type: ' application/octet-stream' } + ); + } + + /** + * test + * + * Downloads the test file from the specified url, evaluates the supervised + * model with it. + * + * @param {string} url + * @param {int} k, the number of predictions to be returned + * @param {number} probability threshold + * + * @return {Promise} promise object that resolves to a `Meter` object + * + * example: + * model.test("/absolute/url/to/test.txt", 1, 0.0).then((meter) => { + * console.log(meter.precision); + * console.log(meter.recall); + * console.log(meter.f1Score); + * console.log(meter.nexamples()); + * }); + * + */ + test(url, k, threshold) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + const fastTextNative = this.f; + + return new Promise(function(resolve, reject) { + fetchFunc(url).then(response => { + return response.arrayBuffer(); + }).then(bytes => { + const byteArray = new Uint8Array(bytes); + const FS = fastTextModule.FS; + FS.writeFile(testFileInWasmFs, byteArray); + }).then(() => { + const meter = fastTextNative.test(testFileInWasmFs, k, threshold); + resolve(meter); + }).catch(error => { + reject(error); + }); + }); + } +} diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js new file mode 100644 index 000000000000..84d05459a3e6 --- /dev/null +++ b/toolkit/components/translations/fasttext/fasttext_wasm.js @@ -0,0 +1,8 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +async function loadFastTextModule(aJ={}){var b_=null,b=typeof aJ!='undefined'?aJ:{},bV,aA,bT,e$,bR,e_,aO,R,N,eZ,A,aQ,an,aR,eV,Z,r,gz,gy,gx,gw,ae,eS,af,aD,eR,gv,q,v,C,ah,h,l,au,aF,aE,bz,by,gu,bv,aZ,eC,Q,O,ad,W,bi,K,k,s,aw,bG,J,am,y,cG,cF,o,H,G,j,cp,aS,a,t,ap,Y,P,ax,cg,ce,bZ,aI,bU,T,aU,bt,ak,ac,aa,bw,x,S,bq,dU,bb,a$,bm,bo,bp,av,az,eE,bH,gg,gf,ge,eM,bJ,eO,gd,eQ,bK,B,e,al,bM,eW,eX,aP,f,g,b$,fa,fb,fc,fd,fe,ff,fg,fh,fi,fj,fk,fl,fm,fn,fo,gc,fq,fr,ft,gb,gA,eN,eT,ay;if(b.ready=new Promise(function(a,b){bV=a,aA=b}),["_main","getExceptionMessage","___get_exception_message","_free","___getTypeName","__embind_initialize_bindings","_fflush","onRuntimeInitialized"].forEach(a=>{Object.getOwnPropertyDescriptor(b.ready,a)||Object.defineProperty(b.ready,a,{get:()=>p('You are getting '+a+' on the Promise object, instead of the instance. Use .then() to get called back with the instance, see the MODULARIZE docs in src/settings.js'),set:()=>p('You are setting '+a+' on the Promise object, instead of the instance. Use .then() to get called back with the instance, see the MODULARIZE docs in src/settings.js')})}),bT=Object.assign({},b),e$=[],bR='./this.program',e_=(b,a)=>{throw a},aO=typeof window=='object',R=typeof importScripts=='function',N=typeof process=='object'&&typeof process.versions=='object'&&typeof process.versions.node=='string',eZ=!aO&&!N&&!R,b.ENVIRONMENT)throw new Error('Module.ENVIRONMENT has been deprecated. To force the environment, use the ENVIRONMENT compile-time option (for example, -sENVIRONMENT=web or -sENVIRONMENT=node)');A='';function eY(a){return b.locateFile?b.locateFile(a,A):A+a}if(aO||R){if(R?A=self.location.href:typeof document!='undefined'&&document.currentScript&&(A=document.currentScript.src),b_&&(A=b_),A.indexOf('blob:')!==0?A=A.substr(0,A.replace(/[?#].*/,"").lastIndexOf('/')+1):A='',!(typeof window=='object'||typeof importScripts=='function'))throw new Error('not compiled for this environment (did you build to HTML and try to run it not on the web, or set ENVIRONMENT to something - like node - and run it someplace else - like on the web?)');aQ=b=>{var a=new XMLHttpRequest;return a.open('GET',b,!1),a.send(null),a.responseText},R&&(aR=b=>{var a=new XMLHttpRequest;return a.open('GET',b,!1),a.responseType='arraybuffer',a.send(null),new Uint8Array(a.response)}),an=(c,d,b)=>{var a=new XMLHttpRequest;a.open('GET',c,!0),a.responseType='arraybuffer',a.onload=()=>{if(a.status==200||a.status==0&&a.response){d(a.response);return}b()},a.onerror=b,a.send(null)},eV=a=>document.title=a}else throw new Error('environment detection error');Z=b.print||console.log.bind(console),r=b.printErr||console.warn.bind(console),Object.assign(b,bT),bT=null,ga(),b.arguments&&(e$=b.arguments),F('arguments','arguments_'),b.thisProgram&&(bR=b.thisProgram),F('thisProgram','thisProgram'),b.quit&&(e_=b.quit),F('quit','quit_'),c(typeof b.memoryInitializerPrefixURL=='undefined','Module.memoryInitializerPrefixURL option was removed, use Module.locateFile instead'),c(typeof b.pthreadMainPrefixURL=='undefined','Module.pthreadMainPrefixURL option was removed, use Module.locateFile instead'),c(typeof b.cdInitializerPrefixURL=='undefined','Module.cdInitializerPrefixURL option was removed, use Module.locateFile instead'),c(typeof b.filePackagePrefixURL=='undefined','Module.filePackagePrefixURL option was removed, use Module.locateFile instead'),c(typeof b.read=='undefined','Module.read option was removed (modify read_ in JS)'),c(typeof b.readAsync=='undefined','Module.readAsync option was removed (modify readAsync in JS)'),c(typeof b.readBinary=='undefined','Module.readBinary option was removed (modify readBinary in JS)'),c(typeof b.setWindowTitle=='undefined','Module.setWindowTitle option was removed (modify setWindowTitle in JS)'),c(typeof b.TOTAL_MEMORY=='undefined','Module.TOTAL_MEMORY has been renamed Module.INITIAL_MEMORY'),F('read','read_'),F('readAsync','readAsync'),F('readBinary','readBinary'),F('setWindowTitle','setWindowTitle'),gz='IDBFS is no longer included by default; build with -lidbfs.js',gy='PROXYFS is no longer included by default; build with -lproxyfs.js',gx='WORKERFS is no longer included by default; build with -lworkerfs.js',gw='NODEFS is no longer included by default; build with -lnodefs.js',c(!eZ,"shell environment detected but not enabled at build time. Add 'shell' to `-sENVIRONMENT` to enable."),b.wasmBinary&&(ae=b.wasmBinary),F('wasmBinary','wasmBinary'),eS=b.noExitRuntime||!0,F('noExitRuntime','noExitRuntime'),typeof WebAssembly!='object'&&p('no native wasm support detected'),aD=!1;function c(b,a){b||p('Assertion failed'+(a?': '+a:''))}function bI(){var a=af.buffer;b.HEAP8=q=new Int8Array(a),b.HEAP16=C=new Int16Array(a),b.HEAP32=h=new Int32Array(a),b.HEAPU8=v=new Uint8Array(a),b.HEAPU16=ah=new Uint16Array(a),b.HEAPU32=l=new Uint32Array(a),b.HEAPF32=au=new Float32Array(a),b.HEAPF64=aF=new Float64Array(a)}c(!b.STACK_SIZE,'STACK_SIZE can no longer be set at runtime. Use -sSTACK_SIZE at link time'),c(typeof Int32Array!='undefined'&&typeof Float64Array!='undefined'&&Int32Array.prototype.subarray!=void 0&&Int32Array.prototype.set!=void 0,'JS engine does not provide full typed array support'),c(!b.wasmMemory,'Use of `wasmMemory` detected. Use -sIMPORTED_MEMORY to define wasmMemory externally'),c(!b.INITIAL_MEMORY,'Detected runtime INITIAL_MEMORY setting. Use -sIMPORTED_MEMORY to define wasmMemory dynamically');function eI(){var a=aP();c((a&3)==0),a==0&&(a+=4),l[a>>2]=34821223,l[a+4>>2]=2310721022,l[0]=1668509029}function aV(){var a,b,c;if(aD)return;a=aP(),a==0&&(a+=4),b=l[a>>2],c=l[a+4>>2],(b!=34821223||c!=2310721022)&&p('Stack overflow! Stack cookie has been overwritten at '+$(a)+', expected hex dwords 0x89BACDFE and 0x2135467, but received '+$(c)+' '+$(b)),l[0]!==1668509029&&p('Runtime error: The application has corrupted its heap memory area (address zero)!')}(function(){var a=new Int16Array(1),b=new Int8Array(a.buffer);if(a[0]=25459,b[0]!==115||b[1]!==99)throw'Runtime error: expected the system to be little-endian! (Run with -sSUPPORT_BIG_ENDIAN to bypass)'})(),bz=[],by=[],gu=[],bv=[],aZ=!1,eC=0;function go(){return eS||eC>0}function eB(){if(b.preRun)for(typeof b.preRun=='function'&&(b.preRun=[b.preRun]);b.preRun.length;)et(b.preRun.shift());aK(bz)}function ew(){c(!aZ),aZ=!0,aV(),!b.noFSInit&&!a.init.initialized&&a.init(),a.ignorePermissions=!1,G.init(),aK(by)}function ev(){if(aV(),b.postRun)for(typeof b.postRun=='function'&&(b.postRun=[b.postRun]);b.postRun.length;)br(b.postRun.shift());aK(bv)}function et(a){bz.unshift(a)}function eh(a){by.unshift(a)}function gt(a){}function br(a){bv.unshift(a)}c(Math.imul,'This browser does not support Math.imul(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),c(Math.fround,'This browser does not support Math.fround(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),c(Math.clz32,'This browser does not support Math.clz32(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),c(Math.trunc,'This browser does not support Math.trunc(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),Q=0,O=null,ad=null,W={};function bh(a){for(var b=a;1;){if(!W[a])return a;a=b+Math.random()}}function aH(a){Q++,b.monitorRunDependencies&&b.monitorRunDependencies(Q),a?(c(!W[a]),W[a]=1,O===null&&typeof setInterval!='undefined'&&(O=setInterval(function(){var a,b;if(aD){clearInterval(O),O=null;return}a=!1;for(b in W)a||(a=!0,r('still waiting on run dependencies:')),r('dependency: '+b);a&&r('(end of list)')},1e4))):r('warning: run dependency added without ID')}function ai(a){if(Q--,b.monitorRunDependencies&&b.monitorRunDependencies(Q),a?(c(W[a]),delete W[a]):r('warning: run dependency removed without ID'),Q==0)if(O!==null&&(clearInterval(O),O=null),ad){var d=ad;ad=null,d()}}function p(a){b.onAbort&&b.onAbort(a),a='Aborted('+a+')',r(a),aD=!0,eR=1;var c=new WebAssembly.RuntimeError(a);throw aA(c),c}bi='data:application/octet-stream;base64,';function bd(a){return a.startsWith(bi)}function bc(a){return a.startsWith('file://')}function n(a,d){return function(){var f=a,e=d;return d||(e=b.asm),c(aZ,'native function `'+f+'` called before runtime initialization'),e[a]||c(e[a],'exported native function `'+f+'` not found'),e[a].apply(null,arguments)}}class d extends Error{}class gs extends d{}class ba extends d{constructor(a){super(a),this.excPtr=a;const b=bx(a);this.name=b[0],this.message=b[1]}}b.locateFile?(K='fasttext_wasm.wasm',bd(K)||(K=eY(K))):K=null;function bj(a){try{if(a==K&&ae)return new Uint8Array(ae);if(aR)return aR(a);throw"both async and sync fetching of the wasm failed"}catch(a){p(a)}}function dH(a){if(!ae&&(aO||R)){if(typeof fetch=='function'&&!bc(a))return fetch(a,{credentials:'same-origin'}).then(function(b){if(!b.ok)throw"failed to load wasm binary file at '"+a+"'";return b.arrayBuffer()}).catch(function(){return bj(a)});if(an)return new Promise(function(b,c){an(a,function(a){b(new Uint8Array(a))},c)})}return Promise.resolve().then(function(){return bj(a)})}function bF(a,b,c){return dH(a).then(function(a){return WebAssembly.instantiate(a,b)}).then(function(a){return a}).then(c,function(a){r('failed to asynchronously prepare wasm: '+a),bc(K)&&r('warning: Loading from a file URI ('+K+') is not supported in most browsers. See https://emscripten.org/docs/getting_started/FAQ.html#how-do-i-run-a-local-webserver-for-testing-why-does-my-program-stall-in-downloading-or-preparing'),p(a)})}function dz(d,a,b,c){return!d&&typeof WebAssembly.instantiateStreaming=='function'&&!bd(a)&&!bc(a)&&!N&&typeof fetch=='function'?fetch(a,{credentials:'same-origin'}).then(function(d){var e=WebAssembly.instantiateStreaming(d,b);return e.then(c,function(d){return r('wasm streaming compile failed: '+d),r('falling back to ArrayBuffer instantiation'),bF(a,b,c)})}):bF(a,b,c)}function dx(){var a={env:bH,wasi_snapshot_preview1:bH},e;function d(d,e){var a=d.exports;return b.asm=a,af=b.asm.memory,c(af,"memory not found in wasm exports"),bI(),aE=b.asm.__indirect_function_table,c(aE,"table not found in wasm exports"),eh(b.asm.__wasm_call_ctors),ai('wasm-instantiate'),a}aH('wasm-instantiate'),e=b;function f(a){c(b===e,'the Module object should not be replaced during async compilation - perhaps the order of HTML elements is wrong?'),e=null,d(a.instance)}if(b.instantiateWasm)try{return b.instantiateWasm(a,d)}catch(a){r('Module.instantiateWasm callback failed with error: '+a),aA(a)}return dz(ae,K,a,f).catch(aA),{}}function F(a,c){Object.getOwnPropertyDescriptor(b,a)||Object.defineProperty(b,a,{configurable:!0,get:function(){p('Module.'+a+' has been replaced with plain '+c+' (the initial value can be provided on Module, but after startup the value is only looked for on a local variable of that name)')}})}function dt(a){Object.getOwnPropertyDescriptor(b,a)&&p('`Module.'+a+'` was supplied but `'+a+'` not included in INCOMING_MODULE_JS_API')}function bN(a){return a==='FS_createPath'||a==='FS_createDataFile'||a==='FS_createPreloadedFile'||a==='FS_unlink'||a==='addRunDependency'||a==='FS_createLazyFile'||a==='FS_createDevice'||a==='removeRunDependency'}function dq(a,b){typeof globalThis!='undefined'&&Object.defineProperty(globalThis,a,{configurable:!0,get:function(){D('`'+a+'` is not longer defined by emscripten. '+b)}})}dq('buffer','Please use HEAP8.buffer or wasmMemory.buffer');function dp(a){typeof globalThis!='undefined'&&!Object.getOwnPropertyDescriptor(globalThis,a)&&Object.defineProperty(globalThis,a,{configurable:!0,get:function(){var b='`'+a+'` is a library symbol and not included by default; add it to your library.js __deps or to DEFAULT_LIBRARY_FUNCS_TO_INCLUDE on the command line',c=a;c.startsWith('_')||(c='$'+a),b+=" (e.g. -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE="+c+")",bN(a)&&(b+='. Alternatively, forcing filesystem support (-sFORCE_FILESYSTEM) can export this for you'),D(b)}}),bX(a)}function bX(a){Object.getOwnPropertyDescriptor(b,a)||Object.defineProperty(b,a,{configurable:!0,get:function(){var b="'"+a+"' was not exported. add it to EXPORTED_RUNTIME_METHODS (see the FAQ)";bN(a)&&(b+='. Alternatively, forcing filesystem support (-sFORCE_FILESYSTEM) can export this for you'),p(b)}})}function gr(a){console.error(a)}function gq(a){this.name='ExitStatus',this.message='Program terminated with exit('+a+')',this.status=a}function aK(a){while(a.length>0)a.shift()(b)}aw=[];function i(a){var b=aw[a];return b||(a>=aw.length&&(aw.length=a+1),aw[a]=b=aE.get(a)),c(aE.get(a)==b,"JavaScript-side Wasm function table mirror is out of date!"),b}function bS(a){if(a.release_ref()&&!a.get_rethrown()){var b=a.get_destructor();b&&i(b)(a.excPtr),eM(a.excPtr)}}function X(a){this.excPtr=a,this.ptr=a-24,this.set_type=function(a){l[this.ptr+4>>2]=a},this.get_type=function(){return l[this.ptr+4>>2]},this.set_destructor=function(a){l[this.ptr+8>>2]=a},this.get_destructor=function(){return l[this.ptr+8>>2]},this.set_refcount=function(a){h[this.ptr>>2]=a},this.set_caught=function(a){a=a?1:0,q[this.ptr+12>>0]=a},this.get_caught=function(){return q[this.ptr+12>>0]!=0},this.set_rethrown=function(a){a=a?1:0,q[this.ptr+13>>0]=a},this.get_rethrown=function(){return q[this.ptr+13>>0]!=0},this.init=function(a,b){this.set_adjusted_ptr(0),this.set_type(a),this.set_destructor(b),this.set_refcount(0),this.set_caught(!1),this.set_rethrown(!1)},this.add_ref=function(){var a=h[this.ptr>>2];h[this.ptr>>2]=a+1},this.release_ref=function(){var a=h[this.ptr>>2];return h[this.ptr>>2]=a-1,c(a>0),a===1},this.set_adjusted_ptr=function(a){l[this.ptr+16>>2]=a},this.get_adjusted_ptr=function(){return l[this.ptr+16>>2]},this.get_exception_ptr=function(){var b=fd(this.get_type()),a;return b?l[this.excPtr>>2]:(a=this.get_adjusted_ptr(),a!==0)?a:this.excPtr}}function bP(a){if(!a)return;bS(new X(a))}function gp(a){bP(a)}function df(a){var b=f(),c=a();return g(b),c}bG=typeof TextDecoder!='undefined'?new TextDecoder('utf8'):void 0;function U(c,b,i){for(var j=b+i,d=b,e,a,f,g,h;c[d]&&!(d>=j);)++d;if(d-b>16&&c.buffer&&bG)return bG.decode(c.subarray(b,d));for(e='';b>10,56320|h&1023))}return e}function L(a,b){return c(typeof a=='number'),a?U(v,a,b):''}function ca(a){return df(function(){var c=b$(4),d=b$(4),e,b,g,f;return fb(a,c,d),e=l[c>>2],b=l[d>>2],g=L(e),B(e),b&&(f=L(b),B(b)),[g,f]})}function bx(a){return ca(a)}b.getExceptionMessage=bx;function gn(a,b='i8'){switch(b.endsWith('*')&&(b='*'),b){case'i1':return q[a>>0];case'i8':return q[a>>0];case'i16':return C[a>>1];case'i32':return h[a>>2];case'i64':return h[a>>2];case'float':return au[a>>2];case'double':return aF[a>>3];case'*':return l[a>>2];default:p('invalid type for getValue: '+b)}}function aY(a){a.add_ref()}function bs(a){if(!a)return;aY(new X(a))}function gm(a){bs(a)}function $(a){return c(typeof a=='number'),'0x'+a.toString(16).padStart(8,'0')}function gl(a,b,c='i8'){switch(c.endsWith('*')&&(c='*'),c){case'i1':q[a>>0]=b;break;case'i8':q[a>>0]=b;break;case'i16':C[a>>1]=b;break;case'i32':h[a>>2]=b;break;case'i64':s=[b>>>0,(k=b,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[a>>2]=s[0],h[a+4>>2]=s[1];break;case'float':au[a>>2]=b;break;case'double':aF[a>>3]=b;break;case'*':l[a>>2]=b;break;default:p('invalid type for setValue: '+c)}}function D(a){D.shown||(D.shown={}),D.shown[a]||(D.shown[a]=1,N&&(a='warning: '+a),r(a))}function cR(c,a,d,b){p('Assertion failed: '+L(c)+', at: '+[a?L(a):'unknown filename',d,b?L(b):'unknown function'])}J=[],am=0;function cP(b){var a=new X(b);return a.get_caught()||(a.set_caught(!0),am--),a.set_rethrown(!1),J.push(a),aY(a),a.get_exception_ptr()}function cO(){if(!J.length)return 0;var a=J[J.length-1];return aY(a),a.excPtr}y=0;function cN(){e(0),c(J.length>0);var a=J.pop();bS(a),y=0}function cH(a){throw y||(y=new ba(a)),y}function bk(){var a=y&&y.excPtr,d,b,e,c,f;if(!a)return al(0),0;if(d=new X(a),d.set_adjusted_ptr(a),b=d.get_type(),!b)return al(0),a;for(e=0;e>2]=a,a}o={isAbs:a=>a.charAt(0)==='/',splitPath:a=>{var b=/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/;return b.exec(a).slice(1)},normalizeArray:(a,e)=>{for(var c=0,b=a.length-1,d;b>=0;b--)d=a[b],d==='.'?a.splice(b,1):d==='..'?(a.splice(b,1),c++):c&&(a.splice(b,1),c--);if(e)for(;c;c--)a.unshift('..');return a},normalize:a=>{var b=o.isAbs(a),c=a.substr(-1)==='/';return a=o.normalizeArray(a.split('/').filter(a=>!!a),!b).join('/'),!a&&!b&&(a='.'),a&&c&&(a+='/'),(b?'/':'')+a},dirname:d=>{var b=o.splitPath(d),c=b[0],a=b[1];return!c&&!a?'.':(a&&(a=a.substr(0,a.length-1)),c+a)},basename:a=>{if(a==='/')return'/';a=o.normalize(a),a=a.replace(/\/$/,"");var b=a.lastIndexOf('/');return b===-1?a:a.substr(b+1)},join:function(){var a=Array.prototype.slice.call(arguments);return o.normalize(a.join('/'))},join2:(a,b)=>o.normalize(a+'/'+b)};function cu(){var a,b,c;if(typeof crypto=='object'&&typeof crypto.getRandomValues=='function')return a=>crypto.getRandomValues(a);if(N)try{return a=require('crypto'),b=a.randomFillSync,b?b=>a.randomFillSync(b):(c=a.randomBytes,a=>(a.set(c(a.byteLength)),a))}catch(a){}p("no cryptographic support found for randomDevice. consider polyfilling it if you want to use something insecure like Math.random(), e.g. put this in a --pre-js: var crypto = { getRandomValues: function(array) { for (var i = 0; i < array.length; i++) array[i] = (Math.random()*256)|0 } };")}function bu(a){return(bu=cu())(a)}H={resolve:function(){for(var b='',c=!1,d=arguments.length-1,e;d>=-1&&!c;d--){if(e=d>=0?arguments[d]:a.cwd(),typeof e!='string')throw new TypeError('Arguments to path.resolve must be strings');if(!e)return'';b=e+'/'+b,c=o.isAbs(e)}return b=o.normalizeArray(b.split('/').filter(a=>!!a),!c).join('/'),(c?'/':'')+b||'.'},relative:(g,h)=>{var d,e,f,c,a,b;g=H.resolve(g).substr(1),h=H.resolve(h).substr(1);function i(b){for(var a=0,c;a=0;c--)if(b[c]!=='')break;return a>c?[]:b.slice(a,c-a+1)}d=i(g.split('/')),e=i(h.split('/')),f=Math.min(d.length,e.length),c=f;for(a=0;a=55296&&c<=57343?(a+=4,++b):a+=3;return a}function aW(f,c,b,g){var h,e,d,a,i;if(!(g>0))return 0;h=b,e=b+g-1;for(d=0;d=55296&&a<=57343&&(i=f.charCodeAt(++d),a=65536+((a&1023)<<10)|i&1023),a<=127){if(b>=e)break;c[b++]=a}else if(a<=2047){if(b+1>=e)break;c[b++]=192|a>>6,c[b++]=128|a&63}else if(a<=65535){if(b+2>=e)break;c[b++]=224|a>>12,c[b++]=128|a>>6&63,c[b++]=128|a&63}else{if(b+3>=e)break;a>1114111&&D('Invalid Unicode code point '+$(a)+' encountered when serializing a JS string to a UTF-8 string in wasm memory! (Valid unicode code points should be in range 0-0x10FFFF).'),c[b++]=240|a>>18,c[b++]=128|a>>12&63,c[b++]=128|a>>6&63,c[b++]=128|a&63}return c[b]=0,b-h}function aB(b,d,c){var e=c>0?c:aX(b)+1,a=new Array(e),f=aW(b,a,0,a.length);return d&&(a.length=f),a}G={ttys:[],init:function(){},shutdown:function(){},register:function(b,c){G.ttys[b]={input:[],output:[],ops:c},a.registerDevice(b,G.stream_ops)},stream_ops:{open:function(b){var c=G.ttys[b.node.rdev];if(!c)throw new a.ErrnoError(43);b.tty=c,b.seekable=!1},close:function(a){a.tty.ops.fsync(a.tty)},fsync:function(a){a.tty.ops.fsync(a.tty)},read:function(c,h,f,g,i){var d,e,b;if(!c.tty||!c.tty.ops.get_char)throw new a.ErrnoError(60);d=0;for(e=0;e0?a=e.slice(0,b).toString('utf-8'):a=null}else typeof window!='undefined'&&typeof window.prompt=='function'?(a=window.prompt('Input: '),a!==null&&(a+='\n')):typeof readline=='function'&&(a=readline(),a!==null&&(a+='\n'));if(!a)return null;c.input=aB(a,!0)}return c.input.shift()},put_char:function(b,a){a===null||a===10?(Z(U(b.output,0)),b.output=[]):a!=0&&b.output.push(a)},fsync:function(a){a.output&&a.output.length>0&&(Z(U(a.output,0)),a.output=[])}},default_tty1_ops:{put_char:function(b,a){a===null||a===10?(r(U(b.output,0)),b.output=[]):a!=0&&b.output.push(a)},fsync:function(a){a.output&&a.output.length>0&&(r(U(a.output,0)),a.output=[])}}};function gk(a,b){return v.fill(0,a,a+b),a}function gj(b,a){return c(a,"alignment argument is required"),Math.ceil(b/a)*a}function bC(a){p('internal error: mmapAlloc called but `emscripten_builtin_memalign` native symbol not exported')}j={ops_table:null,mount:function(a){return j.createNode(null,'/',16384|511,0)},createNode:function(c,e,d,f){if(a.isBlkdev(d)||a.isFIFO(d))throw new a.ErrnoError(63);j.ops_table||(j.ops_table={dir:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr,lookup:j.node_ops.lookup,mknod:j.node_ops.mknod,rename:j.node_ops.rename,unlink:j.node_ops.unlink,rmdir:j.node_ops.rmdir,readdir:j.node_ops.readdir,symlink:j.node_ops.symlink},stream:{llseek:j.stream_ops.llseek}},file:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr},stream:{llseek:j.stream_ops.llseek,read:j.stream_ops.read,write:j.stream_ops.write,allocate:j.stream_ops.allocate,mmap:j.stream_ops.mmap,msync:j.stream_ops.msync}},link:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr,readlink:j.node_ops.readlink},stream:{}},chrdev:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr},stream:a.chrdev_stream_ops}});var b=a.createNode(c,e,d,f);return a.isDir(b.mode)?(b.node_ops=j.ops_table.dir.node,b.stream_ops=j.ops_table.dir.stream,b.contents={}):a.isFile(b.mode)?(b.node_ops=j.ops_table.file.node,b.stream_ops=j.ops_table.file.stream,b.usedBytes=0,b.contents=null):a.isLink(b.mode)?(b.node_ops=j.ops_table.link.node,b.stream_ops=j.ops_table.link.stream):a.isChrdev(b.mode)&&(b.node_ops=j.ops_table.chrdev.node,b.stream_ops=j.ops_table.chrdev.stream),b.timestamp=Date.now(),c&&(c.contents[e]=b,c.timestamp=b.timestamp),b},getFileDataAsTypedArray:function(a){return a.contents?a.contents.subarray?a.contents.subarray(0,a.usedBytes):new Uint8Array(a.contents):new Uint8Array(0)},expandFileStorage:function(a,b){var c=a.contents?a.contents.length:0,d,e;if(c>=b)return;d=1024*1024,b=Math.max(b,c*(c>>0),c!=0&&(b=Math.max(b,256)),e=a.contents,a.contents=new Uint8Array(b),a.usedBytes>0&&a.contents.set(e.subarray(0,a.usedBytes),0)},resizeFileStorage:function(a,b){if(a.usedBytes==b)return;if(b==0)a.contents=null,a.usedBytes=0;else{var c=a.contents;a.contents=new Uint8Array(b),c&&a.contents.set(c.subarray(0,Math.min(b,a.usedBytes))),a.usedBytes=b}},node_ops:{getattr:function(c){var b={};return b.dev=a.isChrdev(c.mode)?c.id:1,b.ino=c.id,b.mode=c.mode,b.nlink=1,b.uid=0,b.gid=0,b.rdev=c.rdev,a.isDir(c.mode)?b.size=4096:a.isFile(c.mode)?b.size=c.usedBytes:a.isLink(c.mode)?b.size=c.link.length:b.size=0,b.atime=new Date(c.timestamp),b.mtime=new Date(c.timestamp),b.ctime=new Date(c.timestamp),b.blksize=4096,b.blocks=Math.ceil(b.size/b.blksize),b},setattr:function(b,a){a.mode!==void 0&&(b.mode=a.mode),a.timestamp!==void 0&&(b.timestamp=a.timestamp),a.size!==void 0&&j.resizeFileStorage(b,a.size)},lookup:function(b,c){throw a.genericErrors[44]},mknod:function(a,b,c,d){return j.createNode(a,b,c,d)},rename:function(b,c,e){var d,f;if(a.isDir(b.mode)){try{d=a.lookupNode(c,e)}catch(a){}if(d)for(f in d.contents)throw new a.ErrnoError(55)}delete b.parent.contents[b.name],b.parent.timestamp=Date.now(),b.name=e,c.contents[e]=b,c.timestamp=b.parent.timestamp,b.parent=c},unlink:function(a,b){delete a.contents[b],a.timestamp=Date.now()},rmdir:function(b,c){var d=a.lookupNode(b,c),e;for(e in d.contents)throw new a.ErrnoError(55);delete b.contents[c],b.timestamp=Date.now()},readdir:function(a){var b=['.','..'],c;for(c in a.contents){if(!a.contents.hasOwnProperty(c))continue;b.push(c)}return b},symlink:function(b,c,d){var a=j.createNode(b,c,511|40960,0);return a.link=d,a},readlink:function(b){if(!a.isLink(b.mode))throw new a.ErrnoError(28);return b.link}},stream_ops:{read:function(f,h,g,i,b){var e=f.node.contents,a,d;if(b>=f.node.usedBytes)return 0;if(a=Math.min(f.node.usedBytes-b,i),c(a>=0),a>8&&e.subarray)h.set(e.subarray(b,b+a),g);else for(d=0;d0||c+f{c(d,'Loading data file "'+a+'" failed (no arrayBuffer).'),e(new Uint8Array(d)),b&&ai(b)},b=>{if(d)d();else throw'Loading data file "'+a+'" failed.'}),b&&aH(b)}cp={0:"Success",1:"Arg list too long",2:"Permission denied",3:"Address already in use",4:"Address not available",5:"Address family not supported by protocol family",6:"No more processes",7:"Socket already connected",8:"Bad file number",9:"Trying to read unreadable message",10:"Mount device busy",11:"Operation canceled",12:"No children",13:"Connection aborted",14:"Connection refused",15:"Connection reset by peer",16:"File locking deadlock error",17:"Destination address required",18:"Math arg out of domain of func",19:"Quota exceeded",20:"File exists",21:"Bad address",22:"File too large",23:"Host is unreachable",24:"Identifier removed",25:"Illegal byte sequence",26:"Connection already in progress",27:"Interrupted system call",28:"Invalid argument",29:"I/O error",30:"Socket is already connected",31:"Is a directory",32:"Too many symbolic links",33:"Too many open files",34:"Too many links",35:"Message too long",36:"Multihop attempted",37:"File or path name too long",38:"Network interface is not configured",39:"Connection reset by network",40:"Network is unreachable",41:"Too many open files in system",42:"No buffer space available",43:"No such device",44:"No such file or directory",45:"Exec format error",46:"No record locks available",47:"The link has been severed",48:"Not enough core",49:"No message of desired type",50:"Protocol not available",51:"No space left on device",52:"Function not implemented",53:"Socket is not connected",54:"Not a directory",55:"Directory not empty",56:"State not recoverable",57:"Socket operation on non-socket",59:"Not a typewriter",60:"No such device or address",61:"Value too large for defined data type",62:"Previous owner died",63:"Not super-user",64:"Broken pipe",65:"Protocol error",66:"Unknown protocol",67:"Protocol wrong type for socket",68:"Math result not representable",69:"Read only file system",70:"Illegal seek",71:"No such process",72:"Stale file handle",73:"Connection timed out",74:"Text file busy",75:"Cross-device link",100:"Device not a stream",101:"Bad font file fmt",102:"Invalid slot",103:"Invalid request code",104:"No anode",105:"Block device required",106:"Channel number out of range",107:"Level 3 halted",108:"Level 3 reset",109:"Link number out of range",110:"Protocol driver not attached",111:"No CSI structure available",112:"Level 2 halted",113:"Invalid exchange",114:"Invalid request descriptor",115:"Exchange full",116:"No data (for no delay io)",117:"Timer expired",118:"Out of streams resources",119:"Machine is not on the network",120:"Package not installed",121:"The object is remote",122:"Advertise error",123:"Srmount error",124:"Communication error on send",125:"Cross mount point (not really error)",126:"Given log. name not unique",127:"f.d. invalid for this operation",128:"Remote address changed",129:"Can access a needed shared lib",130:"Accessing a corrupted shared lib",131:".lib section in a.out corrupted",132:"Attempting to link in too many libs",133:"Attempting to exec a shared library",135:"Streams pipe error",136:"Too many users",137:"Socket type not supported",138:"Not supported",139:"Protocol family not supported",140:"Can't send after socket shutdown",141:"Too many references",142:"Host is down",148:"No medium (in tape drive)",156:"Level 2 not synchronized"},aS={};function co(a){return D('warning: build with -sDEMANGLE_SUPPORT to link in libcxxabi demangling'),a}function ck(a){var b=/\b_Z[\w\d_]+/g;return a.replace(b,function(a){var b=co(a);return a===b?a:b+' ['+a+']'})}a={root:null,mounts:[],devices:{},streams:[],nextInode:1,nameTable:null,currentPath:"/",initialized:!1,ignorePermissions:!0,ErrnoError:null,genericErrors:{},filesystems:null,syncFSRequests:0,lookupPath:(h,d={})=>{var i,f,b,c,e,g,j,k,l;if(h=H.resolve(h),!h)return{path:'',node:null};if(i={follow_mount:!0,recurse_count:0},d=Object.assign(i,d),d.recurse_count>8)throw new a.ErrnoError(32);f=h.split('/').filter(a=>!!a),b=a.root,c='/';for(e=0;e40)throw new a.ErrnoError(32)}return{path:c,node:b}},getPath:b=>{for(var c,d;!0;){if(a.isRoot(b))return d=b.mount.mountpoint,!c?d:d[d.length-1]!=='/'?d+'/'+c:d+c;c=c?b.name+'/'+c:b.name,b=b.parent}},hashName:(e,d)=>{for(var b=0,c=0;c>>0)%a.nameTable.length},hashAddNode:b=>{var c=a.hashName(b.parent.id,b.name);b.name_next=a.nameTable[c],a.nameTable[c]=b},hashRemoveNode:b=>{var d=a.hashName(b.parent.id,b.name),c;if(a.nameTable[d]===b)a.nameTable[d]=b.name_next;else for(c=a.nameTable[d];c;){if(c.name_next===b){c.name_next=b.name_next;break}c=c.name_next}},lookupNode:(c,d)=>{var e=a.mayLookup(c),f,b,g;if(e)throw new a.ErrnoError(e,c);f=a.hashName(c.id,d);for(b=a.nameTable[f];b;b=b.name_next)if(g=b.name,b.parent.id===c.id&&g===d)return b;return a.lookup(c,d)},createNode:(b,e,f,g)=>{c(typeof b=='object');var d=new a.FSNode(b,e,f,g);return a.hashAddNode(d),d},destroyNode:b=>{a.hashRemoveNode(b)},isRoot:a=>a===a.parent,isMountpoint:a=>!!a.mounted,isFile:a=>(a&61440)===32768,isDir:a=>(a&61440)===16384,isLink:a=>(a&61440)===40960,isChrdev:a=>(a&61440)===8192,isBlkdev:a=>(a&61440)===24576,isFIFO:a=>(a&61440)===4096,isSocket:a=>(a&49152)===49152,flagModes:{r:0,"r+":2,w:577,"w+":578,a:1089,"a+":1090},modeStringToFlags:b=>{var c=a.flagModes[b];if(typeof c=='undefined')throw new Error('Unknown file open mode: '+b);return c},flagsToPermissionString:a=>{var b=['r','w','rw'][a&3];return a&512&&(b+='w'),b},nodePermissions:(b,c)=>a.ignorePermissions?0:c.includes('r')&&!(b.mode&292)?2:c.includes('w')&&!(b.mode&146)?2:c.includes('x')&&!(b.mode&73)?2:0,mayLookup:b=>{var c=a.nodePermissions(b,'x');return c?c:b.node_ops.lookup?0:2},mayCreate:(b,c)=>{try{var d=a.lookupNode(b,c);return 20}catch(a){}return a.nodePermissions(b,'wx')},mayDelete:(c,e,f)=>{var b,d;try{b=a.lookupNode(c,e)}catch(a){return a.errno}if(d=a.nodePermissions(c,'wx'),d)return d;if(f){{if(!a.isDir(b.mode))return 54;if(a.isRoot(b)||a.getPath(b)===a.cwd())return 10}}else if(a.isDir(b.mode))return 31;return 0},mayOpen:(b,c)=>{if(!b)return 44;if(a.isLink(b.mode))return 32;if(a.isDir(b.mode))if(a.flagsToPermissionString(c)!=='r'||c&512)return 31;return a.nodePermissions(b,a.flagsToPermissionString(c))},MAX_OPEN_FDS:4096,nextfd:(c=0,d=a.MAX_OPEN_FDS)=>{for(var b=c;b<=d;b++)if(!a.streams[b])return b;throw new a.ErrnoError(33)},getStream:b=>a.streams[b],createStream:(b,d,e)=>{a.FSStream||(a.FSStream=function(){this.shared={}},a.FSStream.prototype={},Object.defineProperties(a.FSStream.prototype,{object:{get:function(){return this.node},set:function(a){this.node=a}},isRead:{get:function(){return(this.flags&2097155)!==1}},isWrite:{get:function(){return(this.flags&2097155)!==0}},isAppend:{get:function(){return this.flags&1024}},flags:{get:function(){return this.shared.flags},set:function(a){this.shared.flags=a}},position:{get:function(){return this.shared.position},set:function(a){this.shared.position=a}}})),b=Object.assign(new a.FSStream,b);var c=a.nextfd(d,e);return b.fd=c,a.streams[c]=b,b},closeStream:b=>{a.streams[b]=null},chrdev_stream_ops:{open:b=>{var c=a.getDevice(b.node.rdev);b.stream_ops=c.stream_ops,b.stream_ops.open&&b.stream_ops.open(b)},llseek:()=>{throw new a.ErrnoError(70)}},major:a=>a>>8,minor:a=>a&255,makedev:(a,b)=>a<<8|b,registerDevice:(b,c)=>{a.devices[b]={stream_ops:c}},getDevice:b=>a.devices[b],getMounts:d=>{for(var b=[],a=[d],c;a.length;)c=a.pop(),b.push(c),a.push.apply(a,c.mounts);return b},syncfs:(b,f)=>{var e,g;typeof b=='function'&&(f=b,b=!1),a.syncFSRequests++,a.syncFSRequests>1&&r('warning: '+a.syncFSRequests+' FS.syncfs operations in flight at once, probably just doing extra work'),e=a.getMounts(a.root.mount),g=0;function h(b){return c(a.syncFSRequests>0),a.syncFSRequests--,f(b)}function d(a){if(a){if(!d.errored)return d.errored=!0,h(a);return}++g>=e.length&&h(null)}e.forEach(a=>{if(!a.type.syncfs)return d(null);a.type.syncfs(a,b,d)})},mount:(f,j,d)=>{var g,i,b,h,c,e;if(typeof f=='string')throw f;if(g=d==='/',i=!d,g&&a.root)throw new a.ErrnoError(10);if(!g&&!i){if(h=a.lookupPath(d,{follow_mount:!1}),d=h.path,b=h.node,a.isMountpoint(b))throw new a.ErrnoError(10);if(!a.isDir(b.mode))throw new a.ErrnoError(54)}return c={type:f,opts:j,mountpoint:d,mounts:[]},e=f.mount(c),e.mount=c,c.root=e,g?a.root=e:b&&(b.mounted=c,b.mount&&b.mount.mounts.push(c)),e},unmount:g=>{var d=a.lookupPath(g,{follow_mount:!1}),b,e,h,f;if(!a.isMountpoint(d.node))throw new a.ErrnoError(28);b=d.node,e=b.mounted,h=a.getMounts(e),Object.keys(a.nameTable).forEach(c=>{for(var b=a.nameTable[c],d;b;)d=b.name_next,h.includes(b.mount)&&a.destroyNode(b),b=d}),b.mounted=null,f=b.mount.mounts.indexOf(e),c(f!==-1),b.mount.mounts.splice(f,1)},lookup:(a,b)=>a.node_ops.lookup(a,b),mknod:(d,f,g)=>{var h=a.lookupPath(d,{parent:!0}),c=h.node,b=o.basename(d),e;if(!b||b==='.'||b==='..')throw new a.ErrnoError(28);if(e=a.mayCreate(c,b),e)throw new a.ErrnoError(e);if(!c.node_ops.mknod)throw new a.ErrnoError(63);return c.node_ops.mknod(c,b,f,g)},create:(c,b)=>(b=b!==void 0?b:438,b&=4095,b|=32768,a.mknod(c,b,0)),mkdir:(c,b)=>(b=b!==void 0?b:511,b&=511|512,b|=16384,a.mknod(c,b,0)),mkdirTree:(e,f)=>{for(var c=e.split('/'),d='',b=0;b(typeof c=='undefined'&&(c=b,b=438),b|=8192,a.mknod(d,b,c)),symlink:(e,f)=>{var g,b,c,d;if(!H.resolve(e))throw new a.ErrnoError(44);if(g=a.lookupPath(f,{parent:!0}),b=g.node,!b)throw new a.ErrnoError(44);if(c=o.basename(f),d=a.mayCreate(b,c),d)throw new a.ErrnoError(d);if(!b.node_ops.symlink)throw new a.ErrnoError(63);return b.node_ops.symlink(b,c,e)},rename:(h,i)=>{var p=o.dirname(h),n=o.dirname(i),l=o.basename(h),j=o.basename(i),g=a.lookupPath(h,{parent:!0}),b=g.node,c,e,k,f,m,d;if(g=a.lookupPath(i,{parent:!0}),c=g.node,!b||!c)throw new a.ErrnoError(44);if(b.mount!==c.mount)throw new a.ErrnoError(75);if(e=a.lookupNode(b,l),k=H.relative(h,n),k.charAt(0)!=='.')throw new a.ErrnoError(28);if(k=H.relative(i,p),k.charAt(0)!=='.')throw new a.ErrnoError(55);try{f=a.lookupNode(c,j)}catch(a){}if(e===f)return;if(m=a.isDir(e.mode),d=a.mayDelete(b,l,m),d)throw new a.ErrnoError(d);if(d=f?a.mayDelete(c,j,m):a.mayCreate(c,j),d)throw new a.ErrnoError(d);if(!b.node_ops.rename)throw new a.ErrnoError(63);if(a.isMountpoint(e)||f&&a.isMountpoint(f))throw new a.ErrnoError(10);if(c!==b)if(d=a.nodePermissions(b,'w'),d)throw new a.ErrnoError(d);a.hashRemoveNode(e);try{b.node_ops.rename(e,c,j)}catch(a){throw a}finally{a.hashAddNode(e)}},rmdir:d=>{var g=a.lookupPath(d,{parent:!0}),b=g.node,c=o.basename(d),e=a.lookupNode(b,c),f=a.mayDelete(b,c,!0);if(f)throw new a.ErrnoError(f);if(!b.node_ops.rmdir)throw new a.ErrnoError(63);if(a.isMountpoint(e))throw new a.ErrnoError(10);b.node_ops.rmdir(b,c),a.destroyNode(e)},readdir:c=>{var d=a.lookupPath(c,{follow:!0}),b=d.node;if(!b.node_ops.readdir)throw new a.ErrnoError(54);return b.node_ops.readdir(b)},unlink:d=>{var g=a.lookupPath(d,{parent:!0}),b=g.node,c,e,f;if(!b)throw new a.ErrnoError(44);if(c=o.basename(d),e=a.lookupNode(b,c),f=a.mayDelete(b,c,!1),f)throw new a.ErrnoError(f);if(!b.node_ops.unlink)throw new a.ErrnoError(63);if(a.isMountpoint(e))throw new a.ErrnoError(10);b.node_ops.unlink(b,c),a.destroyNode(e)},readlink:c=>{var d=a.lookupPath(c),b=d.node;if(!b)throw new a.ErrnoError(44);if(!b.node_ops.readlink)throw new a.ErrnoError(28);return H.resolve(a.getPath(b.parent),b.node_ops.readlink(b))},stat:(c,d)=>{var e=a.lookupPath(c,{follow:!d}),b=e.node;if(!b)throw new a.ErrnoError(44);if(!b.node_ops.getattr)throw new a.ErrnoError(63);return b.node_ops.getattr(b)},lstat:b=>a.stat(b,!0),chmod:(c,d,e)=>{var b,f;if(typeof c=='string'?(f=a.lookupPath(c,{follow:!e}),b=f.node):b=c,!b.node_ops.setattr)throw new a.ErrnoError(63);b.node_ops.setattr(b,{mode:d&4095|b.mode&~4095,timestamp:Date.now()})},lchmod:(b,c)=>{a.chmod(b,c,!0)},fchmod:(c,d)=>{var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);a.chmod(b.node,d)},chown:(c,f,g,d)=>{var b,e;if(typeof c=='string'?(e=a.lookupPath(c,{follow:!d}),b=e.node):b=c,!b.node_ops.setattr)throw new a.ErrnoError(63);b.node_ops.setattr(b,{timestamp:Date.now()})},lchown:(b,c,d)=>{a.chown(b,c,d,!0)},fchown:(c,d,e)=>{var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);a.chown(b.node,d,e)},truncate:(c,e)=>{var b,f,d;if(e<0)throw new a.ErrnoError(28);if(typeof c=='string'?(f=a.lookupPath(c,{follow:!0}),b=f.node):b=c,!b.node_ops.setattr)throw new a.ErrnoError(63);if(a.isDir(b.mode))throw new a.ErrnoError(31);if(!a.isFile(b.mode))throw new a.ErrnoError(28);if(d=a.nodePermissions(b,'w'),d)throw new a.ErrnoError(d);b.node_ops.setattr(b,{size:e,timestamp:Date.now()})},ftruncate:(c,d)=>{var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);if((b.flags&2097155)===0)throw new a.ErrnoError(28);a.truncate(b.node,d)},utime:(c,d,e)=>{var f=a.lookupPath(c,{follow:!0}),b=f.node;b.node_ops.setattr(b,{timestamp:Math.max(d,e)})},open:(e,c,f)=>{var d,j,h,i,g;if(e==="")throw new a.ErrnoError(44);if(c=typeof c=='string'?a.modeStringToFlags(c):c,f=typeof f=='undefined'?438:f,c&64?f=f&4095|32768:f=0,typeof e=='object')d=e;else{e=o.normalize(e);try{j=a.lookupPath(e,{follow:!(c&131072)}),d=j.node}catch(a){}}if(h=!1,c&64)if(d){if(c&128)throw new a.ErrnoError(20)}else d=a.mknod(e,f,0),h=!0;if(!d)throw new a.ErrnoError(44);if(a.isChrdev(d.mode)&&(c&=~512),c&65536&&!a.isDir(d.mode))throw new a.ErrnoError(54);if(!h)if(i=a.mayOpen(d,c),i)throw new a.ErrnoError(i);return c&512&&!h&&a.truncate(d,0),c&=~(128|512|131072),g=a.createStream({node:d,path:a.getPath(d),flags:c,seekable:!0,position:0,stream_ops:d.stream_ops,ungotten:[],error:!1}),g.stream_ops.open&&g.stream_ops.open(g),b.logReadFiles&&!(c&1)&&(a.readFiles||(a.readFiles={}),e in a.readFiles||(a.readFiles[e]=1)),g},close:b=>{if(a.isClosed(b))throw new a.ErrnoError(8);b.getdents&&(b.getdents=null);try{b.stream_ops.close&&b.stream_ops.close(b)}catch(a){throw a}finally{a.closeStream(b.fd)}b.fd=null},isClosed:a=>a.fd===null,llseek:(b,d,c)=>{if(a.isClosed(b))throw new a.ErrnoError(8);if(!b.seekable||!b.stream_ops.llseek)throw new a.ErrnoError(70);if(c!=0&&c!=1&&c!=2)throw new a.ErrnoError(28);return b.position=b.stream_ops.llseek(b,d,c),b.ungotten=[],b.position},read:(b,g,h,f,c)=>{var d,e;if(f<0||c<0)throw new a.ErrnoError(28);if(a.isClosed(b))throw new a.ErrnoError(8);if((b.flags&2097155)===1)throw new a.ErrnoError(8);if(a.isDir(b.node.mode))throw new a.ErrnoError(31);if(!b.stream_ops.read)throw new a.ErrnoError(28);if(d=typeof c!='undefined',d){if(!b.seekable)throw new a.ErrnoError(70)}else c=b.position;return e=b.stream_ops.read(b,g,h,f,c),d||(b.position+=e),e},write:(b,i,g,f,c,h)=>{var e,d;if(f<0||c<0)throw new a.ErrnoError(28);if(a.isClosed(b))throw new a.ErrnoError(8);if((b.flags&2097155)===0)throw new a.ErrnoError(8);if(a.isDir(b.node.mode))throw new a.ErrnoError(31);if(!b.stream_ops.write)throw new a.ErrnoError(28);if(b.seekable&&b.flags&1024&&a.llseek(b,0,2),e=typeof c!='undefined',e){if(!b.seekable)throw new a.ErrnoError(70)}else c=b.position;return d=b.stream_ops.write(b,i,g,f,c,h),e||(b.position+=d),d},allocate:(b,c,d)=>{if(a.isClosed(b))throw new a.ErrnoError(8);if(c<0||d<=0)throw new a.ErrnoError(28);if((b.flags&2097155)===0)throw new a.ErrnoError(8);if(!a.isFile(b.node.mode)&&!a.isDir(b.node.mode))throw new a.ErrnoError(43);if(!b.stream_ops.allocate)throw new a.ErrnoError(138);b.stream_ops.allocate(b,c,d)},mmap:(b,e,f,c,d)=>{if((c&2)!==0&&(d&2)===0&&(b.flags&2097155)!==2)throw new a.ErrnoError(2);if((b.flags&2097155)===1)throw new a.ErrnoError(2);if(!b.stream_ops.mmap)throw new a.ErrnoError(43);return b.stream_ops.mmap(b,e,f,c,d)},msync:(a,b,c,d,e)=>a.stream_ops.msync?a.stream_ops.msync(a,b,c,d,e):0,munmap:a=>0,ioctl:(b,c,d)=>{if(!b.stream_ops.ioctl)throw new a.ErrnoError(59);return b.stream_ops.ioctl(b,c,d)},readFile:(h,b={})=>{var d,e,g,f,c;if(b.flags=b.flags||0,b.encoding=b.encoding||'binary',b.encoding!=='utf8'&&b.encoding!=='binary')throw new Error('Invalid encoding type "'+b.encoding+'"');return e=a.open(h,b.flags),g=a.stat(h),f=g.size,c=new Uint8Array(f),a.read(e,c,0,f,0),b.encoding==='utf8'?d=U(c,0):b.encoding==='binary'&&(d=c),a.close(e),d},writeFile:(g,b,c={})=>{var d,e,f;if(c.flags=c.flags||577,d=a.open(g,c.flags,c.mode),typeof b=='string')e=new Uint8Array(aX(b)+1),f=aW(b,e,0,e.length),a.write(d,e,0,f,void 0,c.canOwn);else if(ArrayBuffer.isView(b))a.write(d,b,0,b.byteLength,void 0,c.canOwn);else throw new Error('Unsupported data type');a.close(d)},cwd:()=>a.currentPath,chdir:d=>{var b=a.lookupPath(d,{follow:!0}),c;if(b.node===null)throw new a.ErrnoError(44);if(!a.isDir(b.node.mode))throw new a.ErrnoError(54);if(c=a.nodePermissions(b.node,'x'),c)throw new a.ErrnoError(c);a.currentPath=b.path},createDefaultDirectories:()=>{a.mkdir('/tmp'),a.mkdir('/home'),a.mkdir('/home/web_user')},createDefaultDevices:()=>{var c,b,d;a.mkdir('/dev'),a.registerDevice(a.makedev(1,3),{read:()=>0,write:(b,c,d,a,e)=>a}),a.mkdev('/dev/null',a.makedev(1,3)),G.register(a.makedev(5,0),G.default_tty_ops),G.register(a.makedev(6,0),G.default_tty1_ops),a.mkdev('/dev/tty',a.makedev(5,0)),a.mkdev('/dev/tty1',a.makedev(6,0)),c=new Uint8Array(1024),b=0,d=()=>(b===0&&(b=bu(c).byteLength),c[--b]),a.createDevice('/dev','random',d),a.createDevice('/dev','urandom',d),a.mkdir('/dev/shm'),a.mkdir('/dev/shm/tmp')},createSpecialDirectories:()=>{a.mkdir('/proc');var b=a.mkdir('/proc/self');a.mkdir('/proc/self/fd'),a.mount({mount:()=>{var c=a.createNode(b,'fd',16384|511,73);return c.node_ops={lookup:(f,d)=>{var e=+d,c=a.getStream(e),b;if(!c)throw new a.ErrnoError(8);return b={parent:null,mount:{mountpoint:'fake'},node_ops:{readlink:()=>c.path}},b.parent=b,b}},c}},{},'/proc/self/fd')},createStandardStreams:()=>{var d,e,f;b.stdin?a.createDevice('/dev','stdin',b.stdin):a.symlink('/dev/tty','/dev/stdin'),b.stdout?a.createDevice('/dev','stdout',null,b.stdout):a.symlink('/dev/tty','/dev/stdout'),b.stderr?a.createDevice('/dev','stderr',null,b.stderr):a.symlink('/dev/tty1','/dev/stderr'),d=a.open('/dev/stdin',0),e=a.open('/dev/stdout',1),f=a.open('/dev/stderr',1),c(d.fd===0,'invalid handle for stdin ('+d.fd+')'),c(e.fd===1,'invalid handle for stdout ('+e.fd+')'),c(f.fd===2,'invalid handle for stderr ('+f.fd+')')},ensureErrnoError:()=>{if(a.ErrnoError)return;a.ErrnoError=function(a,b){this.name='ErrnoError',this.node=b,this.setErrno=function(a){this.errno=a;for(var b in aS)if(aS[b]===a){this.code=b;break}},this.setErrno(a),this.message=cp[a],this.stack&&(Object.defineProperty(this,"stack",{value:(new Error).stack,writable:!0}),this.stack=ck(this.stack))},a.ErrnoError.prototype=new Error,a.ErrnoError.prototype.constructor=a.ErrnoError,[44].forEach(b=>{a.genericErrors[b]=new a.ErrnoError(b),a.genericErrors[b].stack=''})},staticInit:()=>{a.ensureErrnoError(),a.nameTable=new Array(4096),a.mount(j,{},'/'),a.createDefaultDirectories(),a.createDefaultDevices(),a.createSpecialDirectories(),a.filesystems={MEMFS:j}},init:(d,e,f)=>{c(!a.init.initialized,'FS.init was previously called. If you want to initialize later with custom parameters, remove any earlier calls (note that one is automatically added to the generated code)'),a.init.initialized=!0,a.ensureErrnoError(),b.stdin=d||b.stdin,b.stdout=e||b.stdout,b.stderr=f||b.stderr,a.createStandardStreams()},quit:()=>{var b,c;a.init.initialized=!1,bK(0);for(b=0;b{var a=0;return b&&(a|=292|73),c&&(a|=146),a},findObject:(c,d)=>{var b=a.analyzePath(c,d);return b.exists?b.object:null},analyzePath:(d,e)=>{var c,b;try{c=a.lookupPath(d,{follow:!e}),d=c.path}catch(a){}b={isRoot:!1,exists:!1,error:0,name:null,path:null,object:null,parentExists:!1,parentPath:null,parentObject:null};try{c=a.lookupPath(d,{parent:!0}),b.parentExists=!0,b.parentPath=c.path,b.parentObject=c.node,b.name=o.basename(d),c=a.lookupPath(d,{follow:!e}),b.exists=!0,b.path=c.path,b.object=c.node,b.name=c.node.name,b.isRoot=c.path==='/'}catch(a){b.error=a.errno}return b},createPath:(b,f,g,h)=>{var d,e,c;for(b=typeof b=='string'?b:a.getPath(b),d=f.split('/').reverse();d.length;){if(e=d.pop(),!e)continue;c=o.join2(b,e);try{a.mkdir(c)}catch(a){}b=c}return c},createFile:(b,c,h,d,e)=>{var f=o.join2(typeof b=='string'?b:a.getPath(b),c),g=a.getMode(d,e);return a.create(f,g)},createDataFile:(c,f,b,m,l,k)=>{var i=f,g,e,h,d,n,j;if(c&&(c=typeof c=='string'?c:a.getPath(c),i=f?o.join2(c,f):c),g=a.getMode(m,l),e=a.create(i,g),b){if(typeof b=='string'){h=new Array(b.length);for(d=0,n=b.length;d{var g=o.join2(typeof c=='string'?c:a.getPath(c),f),h=a.getMode(!!d,!!b),e;return a.createDevice.major||(a.createDevice.major=64),e=a.makedev(a.createDevice.major++,0),a.registerDevice(e,{open:a=>{a.seekable=!1},close:a=>{b&&b.buffer&&b.buffer.length&&b(10)},read:(h,i,f,g,j)=>{for(var c=0,e=0,b;e{for(var c=0;c{if(b.isDevice||b.isFolder||b.link||b.contents)return!0;if(typeof XMLHttpRequest!='undefined')throw new Error("Lazy loading should have been performed (contents set) in createLazyFile, but it was not. Lazy loading only works in web workers. Use --embed-file or --preload-file in emcc on the main thread.");if(aQ)try{b.contents=aB(aQ(b.url),!0),b.usedBytes=b.contents.length}catch(b){throw new a.ErrnoError(29)}else throw new Error('Cannot load without read() or XMLHttpRequest.')},createLazyFile:(k,l,f,m,n)=>{var h,d,b,e,i;function g(){this.lengthKnown=!1,this.chunks=[]}if(g.prototype.get=function(a){var b,c;return a>this.length-1||a<0?void 0:(b=a%this.chunkSize,c=a/this.chunkSize|0,this.getter(c)[b])},g.prototype.setDataGetter=function(a){this.getter=a},g.prototype.cacheLength=function(){var b=new XMLHttpRequest,a,e,g,h,c,i,d;if(b.open('HEAD',f,!1),b.send(null),!(b.status>=200&&b.status<300||b.status===304))throw new Error("Couldn't load "+f+". Status: "+b.status);a=Number(b.getResponseHeader("Content-length")),g=(e=b.getResponseHeader("Accept-Ranges"))&&e==="bytes",h=(e=b.getResponseHeader("Content-Encoding"))&&e==="gzip",c=1024*1024,g||(c=a),i=(e,d)=>{if(e>d)throw new Error("invalid range ("+e+", "+d+") or no bytes requested!");if(d>a-1)throw new Error("only "+a+" bytes available! programmer error!");var b=new XMLHttpRequest;if(b.open('GET',f,!1),a!==c&&b.setRequestHeader("Range","bytes="+e+"-"+d),b.responseType='arraybuffer',b.overrideMimeType&&b.overrideMimeType('text/plain; charset=x-user-defined'),b.send(null),!(b.status>=200&&b.status<300||b.status===304))throw new Error("Couldn't load "+f+". Status: "+b.status);return b.response!==void 0?new Uint8Array(b.response||[]):aB(b.responseText||'',!0)},d=this,d.setDataGetter(b=>{var f=b*c,e=(b+1)*c-1;if(e=Math.min(e,a-1),typeof d.chunks[b]=='undefined'&&(d.chunks[b]=i(f,e)),typeof d.chunks[b]=='undefined')throw new Error('doXHR failed!');return d.chunks[b]}),(h||!a)&&(c=a=1,a=this.getter(0).length,c=a,Z("LazyFiles on gzip forces download of the whole file when length is accessed")),this._length=a,this._chunkSize=c,this.lengthKnown=!0},typeof XMLHttpRequest!='undefined'){if(!R)throw'Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc';h=new g,Object.defineProperties(h,{length:{get:function(){return this.lengthKnown||this.cacheLength(),this._length}},chunkSize:{get:function(){return this.lengthKnown||this.cacheLength(),this._chunkSize}}}),d={isDevice:!1,contents:h}}else d={isDevice:!1,url:f};b=a.createFile(k,l,d,m,n),d.contents?b.contents=d.contents:d.url&&(b.contents=null,b.url=d.url),Object.defineProperties(b,{usedBytes:{get:function(){return this.contents.length}}}),e={},i=Object.keys(b.stream_ops),i.forEach(c=>{var d=b.stream_ops[c];e[c]=function(){return a.forceLoadFile(b),d.apply(null,arguments)}});function j(i,g,f,h,e){var b=i.node.contents,d,a;if(e>=b.length)return 0;if(d=Math.min(b.length-e,h),c(d>=0),b.slice)for(a=0;a(a.forceLoadFile(b),j(c,d,e,f,g)),e.mmap=(e,d,f,g,h)=>{a.forceLoadFile(b);var c=bC(d);if(!c)throw new a.ErrnoError(48);return j(e,q,c,d,f),{ptr:c,allocated:!0}},b.stream_ops=e,b},createPreloadedFile:(f,c,d,m,n,g,b,k,l,i)=>{var j=c?H.resolve(o.join2(f,c)):f,e=bh('cp '+j);function h(d){function h(b){i&&i(),k||a.createDataFile(f,c,b,m,n,l),g&&g(),ai(e)}if(Browser.handledByPreloadPlugin(d,j,h,()=>{b&&b(),ai(e)}))return;h(d)}aH(e),typeof d=='string'?cq(d,a=>h(a),b):h(d)},absolutePath:()=>{p('FS.absolutePath has been removed; use PATH_FS.resolve instead')},createFolder:()=>{p('FS.createFolder has been removed; use FS.mkdir instead')},createLink:()=>{p('FS.createLink has been removed; use FS.symlink instead')},joinPath:()=>{p('FS.joinPath has been removed; use PATH.join instead')},mmapAlloc:()=>{p('FS.mmapAlloc has been replaced by the top level function mmapAlloc')},standardizePath:()=>{p('FS.standardizePath has been removed; use PATH.normalize instead')}},t={DEFAULT_POLLMASK:5,calculateAt:function(d,c,f){var b,e;if(o.isAbs(c))return c;if(d===-100?b=a.cwd():(e=t.getStreamFromFD(d),b=e.path),c.length==0){if(!f)throw new a.ErrnoError(44);return b}return o.join2(b,c)},doStat:function(i,g,b){var c,e,f,d;try{c=i(g)}catch(b){if(b&&b.node&&o.normalize(g)!==o.normalize(a.getPath(b.node)))return-54;throw b}return h[b>>2]=c.dev,h[b+8>>2]=c.ino,h[b+12>>2]=c.mode,l[b+16>>2]=c.nlink,h[b+20>>2]=c.uid,h[b+24>>2]=c.gid,h[b+28>>2]=c.rdev,(s=[c.size>>>0,(k=c.size,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+40>>2]=s[0],h[b+44>>2]=s[1]),h[b+48>>2]=4096,h[b+52>>2]=c.blocks,e=c.atime.getTime(),f=c.mtime.getTime(),d=c.ctime.getTime(),(s=[Math.floor(e/1e3)>>>0,(k=Math.floor(e/1e3),+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+56>>2]=s[0],h[b+60>>2]=s[1]),l[b+64>>2]=e%1e3*1e3,(s=[Math.floor(f/1e3)>>>0,(k=Math.floor(f/1e3),+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+72>>2]=s[0],h[b+76>>2]=s[1]),l[b+80>>2]=f%1e3*1e3,(s=[Math.floor(d/1e3)>>>0,(k=Math.floor(d/1e3),+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+88>>2]=s[0],h[b+92>>2]=s[1]),l[b+96>>2]=d%1e3*1e3,(s=[c.ino>>>0,(k=c.ino,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+104>>2]=s[0],h[b+108>>2]=s[1]),0},doMsync:function(b,c,d,e,f){if(!a.isFile(c.node.mode))throw new a.ErrnoError(43);if(e&2)return 0;var g=v.slice(b,b+d);a.msync(c,g,f,d,e)},varargs:void 0,get:function(){c(t.varargs!=void 0),t.varargs+=4;var a=h[t.varargs-4>>2];return a},getStr:function(a){var b=L(a);return b},getStreamFromFD:function(c){var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);return b}};function cj(h,f,g){var c,b,d,e;t.varargs=g;try{switch(c=t.getStreamFromFD(h),f){case 0:return b=t.get(),b<0?-28:(d=a.createStream(c,b),d.fd);case 1:case 2:return 0;case 3:return c.flags;case 4:return b=t.get(),c.flags|=b,0;case 5:return b=t.get(),e=0,C[b+e>>1]=2,0;case 6:case 7:return 0;case 16:case 8:return-28;case 9:return cx(28),-1;default:return-28}}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return-b.errno}}function ci(e,d,f){var b,c;t.varargs=f;try{switch(b=t.getStreamFromFD(e),d){case 21509:case 21505:return b.tty?0:-59;case 21510:case 21511:case 21512:case 21506:case 21507:case 21508:return b.tty?0:-59;case 21519:return b.tty?(c=t.get(),h[c>>2]=0,0):-59;case 21520:return b.tty?-28:-59;case 21531:return c=t.get(),a.ioctl(b,d,c);case 21523:return b.tty?0:-59;case 21524:return b.tty?0:-59;default:return-28}}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return-b.errno}}function ch(d,b,e,c){t.varargs=c;try{b=t.getStr(b),b=t.calculateAt(d,b);var f=c?t.get():0;return a.open(b,e,f).fd}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return-b.errno}}ap={};function aq(a){for(var b,c;a.length;)b=a.pop(),c=a.pop(),c(b)}function _(a){return this.fromWireType(h[a>>2])}Y={},P={},ax={},cg=48,ce=57;function bW(a){if(void 0===a)return'_unknown';a=a.replace(/[^a-zA-Z0-9_]/g,'$');var b=a.charCodeAt(0);return b>=cg&&b<=ce?'_'+a:a}function aM(a,b){return a=bW(a),{[a]:function(){return b.apply(this,arguments)}}[a]}function aL(c,b){var a=aM(b,function(a){this.name=b,this.message=a;var c=new Error(a).stack;c!==void 0&&(this.stack=this.toString()+'\n'+c.replace(/^Error(:[^\n]*)?\n/,''))});return a.prototype=Object.create(c.prototype),a.prototype.constructor=a,a.prototype.toString=function(){return this.message===void 0?this.name:this.name+': '+this.message},a}bZ=void 0;function aG(a){throw new bZ(a)}function E(b,d,g){var a,c,e;b.forEach(function(a){ax[a]=d});function f(d){var c=g(d),a;c.length!==b.length&&aG('Mismatched type converter count');for(a=0;a{P.hasOwnProperty(b)?a[d]=P[b]:(c.push(b),Y.hasOwnProperty(b)||(Y[b]=[]),Y[b].push(()=>{a[d]=P[b],++e,e===c.length&&f(a)}))}),0===c.length&&f(a)}function cb(d){var c=ap[d],a,b,f,g,e;delete ap[d],a=c.elements,b=a.length,f=a.map(function(a){return a.getterReturnType}).concat(a.map(function(a){return a.setterArgumentType})),g=c.rawConstructor,e=c.rawDestructor,E([d],f,function(d){return a.forEach((a,c)=>{var e=d[c],f=a.getter,g=a.getterContext,h=d[c+b],i=a.setter,j=a.setterContext;a.read=a=>e.fromWireType(f(g,a)),a.write=(b,c)=>{var a=[];i(j,b,h.toWireType(a,c)),aq(a)}}),[{name:c.name,fromWireType:function(d){for(var f=new Array(b),c=0;ca()))}function cl(c,a,b,d,e){var f=aC(b);a=u(a),z(c,{name:a,fromWireType:function(a){return!!a},toWireType:function(b,a){return a?d:e},argPackAdvance:8,readValueFromPointer:function(d){var c;if(b===1)c=q;else if(b===2)c=C;else if(b===4)c=h;else throw new TypeError("Unknown boolean type size: "+a);return this.fromWireType(c[d>>f])},destructorFunction:null})}function cm(e){var a,c,b,d;if(!(this instanceof M))return!1;if(!(e instanceof M))return!1;for(a=this.$$.ptrType.registeredClass,c=this.$$.ptr,b=e.$$.ptrType.registeredClass,d=e.$$.ptr;a.baseClass;)c=a.upcast(c),a=a.baseClass;while(b.baseClass)d=b.upcast(d),b=b.baseClass;return a===b&&c===d}function cn(a){return{count:a.count,deleteScheduled:a.deleteScheduled,preservePointerOnDelete:a.preservePointerOnDelete,ptr:a.ptr,ptrType:a.ptrType,smartPtr:a.smartPtr,smartPtrType:a.smartPtrType}}function aN(a){function b(a){return a.$$.ptrType.registeredClass.name}m(b(a)+' instance already deleted')}aU=!1;function bE(a){}function cr(a){a.smartPtr?a.smartPtrType.rawDestructor(a.smartPtr):a.ptrType.registeredClass.rawDestructor(a.ptr)}function bB(a){a.count.value-=1;var b=0===a.count.value;b&&cr(a)}function bA(b,c,a){if(c===a)return b;if(void 0===a.baseClass)return null;var d=bA(b,c,a.baseClass);return d===null?null:a.downcast(d)}bt={};function cv(){return Object.keys(aa).length}function cw(){var a=[],b;for(b in aa)aa.hasOwnProperty(b)&&a.push(aa[b]);return a}ak=[];function a_(){while(ak.length){var a=ak.pop();a.$$.deleteScheduled=!1,a.delete()}}ac=void 0;function cA(a){ac=a,ak.length&&ac&&ac(a_)}function cB(){b.getInheritedInstanceCount=cv,b.getLiveInheritedInstances=cw,b.flushPendingDeletes=a_,b.setDelayFunction=cA}aa={};function cD(a,b){for(b===void 0&&m('ptr should not be undefined');a.baseClass;)b=a.upcast(b),a=a.baseClass;return b}function cE(b,a){return a=cD(b,a),aa[a]}function at(d,a){var b,c;return(!a.ptrType||!a.ptr)&&aG('makeClassHandle requires ptr and ptrType'),b=!!a.smartPtrType,c=!!a.smartPtr,b!==c&&aG('Both smartPtrType and smartPtr must be specified'),a.count={value:1},aj(Object.create(d,{$$:{value:a}}))}function bl(a){var d=this.getPointee(a),c,h,i,e,b,f;if(!d)return this.destructor(a),null;if(c=cE(this.registeredClass,d),void 0!==c)return 0===c.$$.count.value?(c.$$.ptr=d,c.$$.smartPtr=a,c.clone()):(h=c.clone(),this.destructor(a),h);function g(){return this.isSmartPointer?at(this.registeredClass.instancePrototype,{ptrType:this.pointeeType,ptr:d,smartPtrType:this,smartPtr:a}):at(this.registeredClass.instancePrototype,{ptrType:this,ptr:a})}return i=this.registeredClass.getActualType(d),e=bt[i],!e?g.call(this):(this.isConst?b=e.constPointerType:b=e.pointerType,f=bA(d,this.registeredClass,b.registeredClass),f===null)?g.call(this):this.isSmartPointer?at(b.registeredClass.instancePrototype,{ptrType:b,ptr:f,smartPtrType:this,smartPtr:a}):at(b.registeredClass.instancePrototype,{ptrType:b,ptr:f})}function aj(a){return'undefined'==typeof FinalizationRegistry?(aj=a=>a,a):(aU=new FinalizationRegistry(a=>{console.warn(a.leakWarning.stack.replace(/^Error: /,'')),bB(a.$$)}),aj=a=>{var b=a.$$,d=!!b.smartPtr,c,e;return d&&(c={$$:b},e=b.ptrType.registeredClass,c.leakWarning=new Error("Embind found a leaked C++ instance "+e.name+" <"+$(b.ptr)+">.\n"+"We'll free it automatically in this case, but this functionality is not reliable across various environments.\n"+"Make sure to invoke .delete() manually once you're done with the instance instead.\n"+"Originally allocated"),'captureStackTrace'in Error&&Error.captureStackTrace(c.leakWarning,bl),aU.register(a,c,a)),a},bE=a=>aU.unregister(a),aj(a))}function cI(){if(this.$$.ptr||aN(this),this.$$.preservePointerOnDelete)return this.$$.count.value+=1,this;var a=aj(Object.create(Object.getPrototypeOf(this),{$$:{value:cn(this.$$)}}));return a.$$.count.value+=1,a.$$.deleteScheduled=!1,a}function cJ(){this.$$.ptr||aN(this),this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete&&m('Object already scheduled for deletion'),bE(this),bB(this.$$),this.$$.preservePointerOnDelete||(this.$$.smartPtr=void 0,this.$$.ptr=void 0)}function cK(){return!this.$$.ptr}function cL(){return this.$$.ptr||aN(this),this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete&&m('Object already scheduled for deletion'),ak.push(this),ak.length===1&&ac&&ac(a_),this.$$.deleteScheduled=!0,this}function cM(){M.prototype.isAliasOf=cm,M.prototype.clone=cI,M.prototype.delete=cJ,M.prototype.isDeleted=cK,M.prototype.deleteLater=cL}function M(){}function bg(a,b,d){if(void 0===a[b].overloadTable){var c=a[b];a[b]=function(){return a[b].overloadTable.hasOwnProperty(arguments.length)||m("Function '"+d+"' called with an invalid number of arguments ("+arguments.length+") - expects one of ("+a[b].overloadTable+")!"),a[b].overloadTable[arguments.length].apply(this,arguments)},a[b].overloadTable=[],a[b].overloadTable[c.argCount]=c}}function bf(a,d,c){b.hasOwnProperty(a)?((void 0===c||void 0!==b[a].overloadTable&&void 0!==b[a].overloadTable[c])&&m("Cannot register public name '"+a+"' twice"),bg(b,a,a),b.hasOwnProperty(c)&&m("Cannot register multiple overloads of a function with the same number of arguments ("+c+")!"),b[a].overloadTable[c]=d):(b[a]=d,void 0!==c&&(b[a].numArguments=c))}function cQ(a,b,c,d,e,f,g,h){this.name=a,this.constructor=b,this.instancePrototype=c,this.rawDestructor=d,this.baseClass=e,this.getActualType=f,this.upcast=g,this.downcast=h,this.pureVirtualFunctions=[]}function ao(b,a,c){while(a!==c)a.upcast||m("Expected null or instance of "+c.name+", got an instance of "+a.name),b=a.upcast(b),a=a.baseClass;return b}function cS(d,a){var b,c;return a===null?(this.isReference&&m('null is not a valid '+this.name),0):(a.$$||m('Cannot pass "'+V(a)+'" as a '+this.name),a.$$.ptr||m('Cannot pass deleted object as a pointer of type '+this.name),b=a.$$.ptrType.registeredClass,c=ao(a.$$.ptr,b,this.registeredClass),c)}function cT(c,a){var b,d,e;if(a===null)return this.isReference&&m('null is not a valid '+this.name),this.isSmartPointer?(b=this.rawConstructor(),c!==null&&c.push(this.rawDestructor,b),b):0;if(a.$$||m('Cannot pass "'+V(a)+'" as a '+this.name),a.$$.ptr||m('Cannot pass deleted object as a pointer of type '+this.name),!this.isConst&&a.$$.ptrType.isConst&&m('Cannot convert argument of type '+(a.$$.smartPtrType?a.$$.smartPtrType.name:a.$$.ptrType.name)+' to parameter type '+this.name),d=a.$$.ptrType.registeredClass,b=ao(a.$$.ptr,d,this.registeredClass),this.isSmartPointer)switch(void 0===a.$$.smartPtr&&m('Passing raw pointer to smart pointer is illegal'),this.sharingPolicy){case 0:a.$$.smartPtrType===this?b=a.$$.smartPtr:m('Cannot convert argument of type '+(a.$$.smartPtrType?a.$$.smartPtrType.name:a.$$.ptrType.name)+' to parameter type '+this.name);break;case 1:b=a.$$.smartPtr;break;case 2:a.$$.smartPtrType===this?b=a.$$.smartPtr:(e=a.clone(),b=this.rawShare(b,S.toHandle(function(){e.delete()})),c!==null&&c.push(this.rawDestructor,b));break;default:m('Unsupporting sharing policy')}return b}function cU(d,a){var b,c;return a===null?(this.isReference&&m('null is not a valid '+this.name),0):(a.$$||m('Cannot pass "'+V(a)+'" as a '+this.name),a.$$.ptr||m('Cannot pass deleted object as a pointer of type '+this.name),a.$$.ptrType.isConst&&m('Cannot convert argument of type '+a.$$.ptrType.name+' to parameter type '+this.name),b=a.$$.ptrType.registeredClass,c=ao(a.$$.ptr,b,this.registeredClass),c)}function cV(a){return this.rawGetPointee&&(a=this.rawGetPointee(a)),a}function cW(a){this.rawDestructor&&this.rawDestructor(a)}function cX(a){a!==null&&a.delete()}function cY(){I.prototype.getPointee=cV,I.prototype.destructor=cW,I.prototype.argPackAdvance=8,I.prototype.readValueFromPointer=_,I.prototype.deleteObject=cX,I.prototype.fromWireType=bl}function I(d,a,e,b,c,f,g,h,i,j,k){this.name=d,this.registeredClass=a,this.isReference=e,this.isConst=b,this.isSmartPointer=c,this.pointeeType=f,this.sharingPolicy=g,this.rawGetPointee=h,this.rawConstructor=i,this.rawShare=j,this.rawDestructor=k,!c&&a.baseClass===void 0?b?(this.toWireType=cS,this.destructorFunction=null):(this.toWireType=cU,this.destructorFunction=null):this.toWireType=cT}function c_(a,d,c){b.hasOwnProperty(a)||aG('Replacing nonexistant public symbol'),void 0!==b[a].overloadTable&&void 0!==c?b[a].overloadTable[c]=d:(b[a]=d,b[a].argCount=c)}function c$(d,e,a){c('dynCall_'+d in b,"bad function pointer type - dynCall function not found for sig '"+d+"'"),a&&a.length?c(a.length===d.substring(1).replace(/j/g,'--').length):c(d.length==1);var f=b['dynCall_'+d];return a&&a.length?f.apply(null,[e].concat(a)):f.call(null,e)}function da(b,a,d){if(b.includes('j'))return c$(b,a,d);c(i(a),'missing table entry in dynCall: '+a);var e=i(a).apply(null,d);return e}function db(a,d){c(a.includes('j')||a.includes('p'),'getDynCaller should only be called with i64 sigs');var b=[];return function(){return b.length=0,Object.assign(b,arguments),da(a,d,b)}}function w(a,b){a=u(a);function d(){return a.includes('j')?db(a,b):i(b)}var c=d();return typeof c!="function"&&m("unknown function pointer with signature "+a+": "+b),c}bw=void 0;function bD(b){var a=eO(b),c=u(a);return B(a),c}function ag(d,e){var a=[],b={};function c(d){if(b[d])return;if(P[d])return;if(ax[d]){ax[d].forEach(c);return}a.push(d),b[d]=!0}throw e.forEach(c),new bw(d+': '+a.map(bD).join([', ']))}function dg(h,m,k,c,j,f,n,b,i,d,a,l,e){a=u(a),f=w(j,f),b&&(b=w(n,b)),d&&(d=w(i,d)),e=w(l,e);var g=bW(a);bf(g,function(){ag('Cannot construct '+a+' due to unbound types',[c])}),E([h,m,k],c?[c]:[],function(n){var k,l,j,m,i,q,o,p;return n=n[0],c?(k=n.registeredClass,l=k.instancePrototype):l=M.prototype,j=aM(g,function(){if(Object.getPrototypeOf(this)!==m)throw new T("Use 'new' to construct "+a);if(void 0===i.constructor_body)throw new T(a+" has no accessible constructor");var b=i.constructor_body[arguments.length];if(void 0===b)throw new T("Tried to invoke ctor of "+a+" with invalid number of parameters ("+arguments.length+") - expected ("+Object.keys(i.constructor_body).toString()+") parameters instead!");return b.apply(this,arguments)}),m=Object.create(l,{constructor:{value:j}}),j.prototype=m,i=new cQ(a,j,m,e,k,f,b,d),q=new I(a,i,!0,!1,!1),o=new I(a+'*',i,!1,!1,!1),p=new I(a+' const*',i,!1,!0,!1),bt[h]={pointerType:o,constPointerType:p},c_(g,j),[q,o,p]})}function bO(c,d){for(var b=[],a=0;a>2]);return b}function be(q,a,p,o,n,k){var i=a.length,h,j,f,l,e,g,b,d;i<2&&m("argTypes array size mismatch! Must at least get return value and 'this' types!"),c(!k,'Async bindings are only supported with JSPI.'),h=a[1]!==null&&p!==null,j=!1;for(f=1;f0),b=bO(a,f),d=w(g,d),i=[e],j=[],E([],[h],function(c){c=c[0];var f='constructor '+c.name;if(void 0===c.registeredClass.constructor_body&&(c.registeredClass.constructor_body=[]),void 0!==c.registeredClass.constructor_body[a-1])throw new T("Cannot register multiple constructors with identical number of parameters ("+(a-1)+") for class '"+c.name+"'! Overload resolution is currently only performed using the parameter count, not actual type info!");return c.registeredClass.constructor_body[a-1]=()=>{ag('Cannot construct '+c.name+' due to unbound types',b)},E([],b,function(b){return b.splice(1,0,null),c.registeredClass.constructor_body[a-1]=be(f,b,null,d,e),[]}),[]})}function dk(e,a,b,j,f,c,g,h,i){var d=bO(b,j);a=u(a),c=w(f,c),E([],[e],function(e){var k,f,j;e=e[0],k=e.name+'.'+a,a.startsWith("@@")&&(a=Symbol[a.substring(2)]),h&&e.registeredClass.pureVirtualFunctions.push(a);function l(){ag('Cannot call '+k+' due to unbound types',d)}return f=e.registeredClass.instancePrototype,j=f[a],void 0===j||void 0===j.overloadTable&&j.className!==e.name&&j.argCount===b-2?(l.argCount=b-2,l.className=e.name,f[a]=l):(bg(f,a,k),f[a].overloadTable[b-2]=l),E([],d,function(h){var d=be(k,h,e,c,g,i);return void 0===f[a].overloadTable?(d.argCount=b-2,f[a]=d):f[a].overloadTable[b-2]=d,[]}),[]})}function bY(a,c,b){return a instanceof Object||m(b+' with invalid "this": '+a),a instanceof c.registeredClass.constructor||m(b+' incompatible with "this" of type '+a.constructor.name),a.$$.ptr||m('cannot call emscripten binding method '+b+' on deleted object'),ao(a.$$.ptr,a.$$.ptrType.registeredClass,c.registeredClass)}function dm(h,b,c,f,e,g,d,i,a,j){b=u(b),e=w(f,e),E([],[h],function(f){var h,k;return f=f[0],h=f.name+'.'+b,k={get:function(){ag('Cannot access '+h+' due to unbound types',[c,d])},enumerable:!0,configurable:!0},a?k.set=()=>{ag('Cannot access '+h+' due to unbound types',[c,d])}:k.set=a=>{m(h+' is a read-only property')},Object.defineProperty(f.registeredClass.instancePrototype,b,k),E([],a?[c,d]:[c],function(c){var k=c[0],d={get:function(){var a=bY(this,f,h+' getter');return k.fromWireType(e(g,a))},enumerable:!0},l;return a&&(a=w(i,a),l=c[1],d.set=function(c){var d=bY(this,f,h+' setter'),b=[];a(j,d,l.toWireType(b,c)),aq(b)}),Object.defineProperty(f.registeredClass.instancePrototype,b,d),[]}),[]})}function dn(){this.allocated=[void 0],this.freelist=[],this.get=function(a){return c(this.allocated[a]!==void 0,'invalid handle: '+a),this.allocated[a]},this.allocate=function(b){let a=this.freelist.pop()||this.allocated.length;return this.allocated[a]=b,a},this.free=function(a){c(this.allocated[a]!==void 0),this.allocated[a]=void 0,this.freelist.push(a)}}x=new dn;function bQ(a){a>=x.reserved&&0===--x.get(a).refcount&&x.free(a)}function dr(){for(var b=0,a=x.reserved;a(a||m('Cannot use deleted val. handle = '+a),x.get(a).value),toHandle:a=>{switch(a){case void 0:return 1;case null:return 2;case!0:return 3;case!1:return 4;default:return x.allocate({refcount:1,value:a})}}};function du(b,a){a=u(a),z(b,{name:a,fromWireType:function(a){var b=S.toValue(a);return bQ(a),b},toWireType:function(b,a){return S.toHandle(a)},argPackAdvance:8,readValueFromPointer:_,destructorFunction:null})}function dv(b,c,a){switch(c){case 0:return function(b){var c=a?q:v;return this.fromWireType(c[b])};case 1:return function(b){var c=a?C:ah;return this.fromWireType(c[b>>1])};case 2:return function(b){var c=a?h:l;return this.fromWireType(c[b>>2])};default:throw new TypeError("Unknown integer type: "+b)}}function dw(c,a,d,e){var f=aC(d);a=u(a);function b(){}b.values={},z(c,{name:a,constructor:b,fromWireType:function(a){return this.constructor.values[a]},toWireType:function(b,a){return a.value},argPackAdvance:8,readValueFromPointer:dv(a,f,e),destructorFunction:null}),bf(a,b)}function aT(a,c){var b=P[a];return void 0===b&&m(c+" has unknown type "+bD(a)),b}function dy(f,a,c){var b=aT(f,'enum'),d,e;a=u(a),d=b.constructor,e=Object.create(b.constructor.prototype,{value:{value:c},constructor:{value:aM(b.name+'_'+a,function(){})}}),d.values[c]=e,d[a]=e}function V(a){if(a===null)return'null';var b=typeof a;return b==='object'||b==='array'||b==='function'?a.toString():''+a}function dA(a,b){switch(b){case 2:return function(a){return this.fromWireType(au[a>>2])};case 3:return function(a){return this.fromWireType(aF[a>>3])};default:throw new TypeError("Unknown float type: "+a)}}function dB(b,a,c){var d=aC(c);a=u(a),z(b,{name:a,fromWireType:function(a){return a},toWireType:function(b,a){if(typeof a!="number"&&typeof a!="boolean")throw new TypeError('Cannot convert "'+V(a)+'" to '+this.name);return a},argPackAdvance:8,readValueFromPointer:dA(a,d),destructorFunction:null})}function dC(b,c,a){switch(c){case 0:return a?function(a){return q[a]}:function(a){return v[a]};case 1:return a?function(a){return C[a>>1]}:function(a){return ah[a>>1]};case 2:return a?function(a){return h[a>>2]}:function(a){return l[a>>2]};default:throw new TypeError("Unknown integer type: "+b)}}function dD(k,a,h,b,c){var i,e,f,j,g,d;a=u(a),c===-1&&(c=4294967295),i=aC(h),e=a=>a,b===0&&(f=32-8*h,e=a=>a<>>f),j=a.includes('unsigned'),g=(d,e)=>{if(typeof d!="number"&&typeof d!="boolean")throw new TypeError('Cannot convert "'+V(d)+'" to '+e);if(dc)throw new TypeError('Passing a number "'+V(d)+'" from JS side to C/C++ side to an argument of type "'+a+'", which is outside the valid range ['+b+', '+c+']!')},j?d=function(b,a){return g(a,this.name),a>>>0}:d=function(b,a){return g(a,this.name),a},z(k,{name:a,fromWireType:e,toWireType:d,argPackAdvance:8,readValueFromPointer:dC(a,i,b!==0),destructorFunction:null})}function dE(c,d,a){var e=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array],f=e[d];function b(a){var b,c,d;return a=a>>2,b=l,c=b[a],d=b[a+1],new f(b.buffer,d,c)}a=u(a),z(c,{name:a,fromWireType:b,argPackAdvance:8,readValueFromPointer:b},{ignoreDuplicateRegistrations:!0})}function dF(b,d,a){return c(typeof a=='number','stringToUTF8(str, outPtr, maxBytesToWrite) is missing the third parameter that specifies the length of the output buffer!'),aW(b,v,d,a)}function dG(c,a){a=u(a);var b=a==="std::string";z(c,{name:a,fromWireType:function(e){var d=l[e>>2],f=e+4,c,g,a,h,k,i,j;if(b){{g=f;for(a=0;a<=d;++a)h=f+a,(a==d||v[h]==0)&&(k=h-g,i=L(g,k),c===void 0?c=i:(c+=String.fromCharCode(0),c+=i),g=h+1)}}else{j=new Array(d);for(a=0;a>2]=d,b&&e)dF(a,g,d+1);else if(e)for(c=0;c255&&(B(g),m('String has UTF-16 code units that do not fit in 8 bits')),v[g+c]=h;else for(c=0;c>1,h=a+i/2;!(a>=h)&&ah[a];)++a;if(d=a<<1,d-b>32&&bq)return bq.decode(v.subarray(b,d));g='';for(e=0;!(e>=i/2);++e){if(f=C[b+e*2>>1],f==0)break;g+=String.fromCharCode(f)}return g}function dJ(e,b,a){var f,g,d,h;if(c(b%2==0,'Pointer passed to stringToUTF16 must be aligned to two bytes!'),c(typeof a=='number','stringToUTF16(str, outPtr, maxBytesToWrite) is missing the third parameter that specifies the length of the output buffer!'),a===void 0&&(a=2147483647),a<2)return 0;a-=2,f=b,g=a>1]=h,b+=2;return C[b>>1]=0,b-f}function dK(a){return a.length*2}function dL(f,g){var b,d,a,e;for(c(f%4==0,'Pointer passed to UTF32ToString must be aligned to four bytes!'),b=0,d='';!(b>=g/4);){if(a=h[f+b*4>>2],a==0)break;++b,a>=65536?(e=a-65536,d+=String.fromCharCode(55296|e>>10,56320|e&1023)):d+=String.fromCharCode(a)}return d}function dM(g,a,d){var f,i,e,b,j;if(c(a%4==0,'Pointer passed to stringToUTF32 must be aligned to four bytes!'),c(typeof d=='number','stringToUTF32(str, outPtr, maxBytesToWrite) is missing the third parameter that specifies the length of the output buffer!'),d===void 0&&(d=2147483647),d<4)return 0;f=a,i=f+d-4;for(e=0;e=55296&&b<=57343&&(j=g.charCodeAt(++e),b=65536+((b&1023)<<10)|j&1023),h[a>>2]=b,a+=4,a+4>i)break;return h[a>>2]=0,a-f}function dN(b){for(var c=0,a=0,d;a=55296&&d<=57343&&++a,c+=4;return c}function dO(h,a,b){b=u(b);var e,f,g,d,c;a===2?(e=dI,f=dJ,d=dK,g=()=>ah,c=1):a===4&&(e=dL,f=dM,d=dN,g=()=>l,c=2),z(h,{name:b,fromWireType:function(d){for(var j=l[d>>2],n=g(),h=d+4,f=0,b,i,m,k;f<=j;++f)i=d+4+f*a,(f==j||n[i>>c]==0)&&(m=i-h,k=e(h,m),b===void 0?b=k:(b+=String.fromCharCode(0),b+=k),h=i+a);return B(d),b},toWireType:function(i,h){var g,e;return typeof h=='string'||m('Cannot pass non-string to C++ string type '+b),g=d(h),e=bJ(4+g+a),l[e>>2]=g>>c,f(h,e+4,g+a),i!==null&&i.push(B,e),e},argPackAdvance:8,readValueFromPointer:_,destructorFunction:function(a){B(a)}})}function dP(a,b,c,d,e,f){ap[a]={name:u(b),rawConstructor:w(c,d),rawDestructor:w(e,f),elements:[]}}function dQ(a,b,c,d,e,f,g,h,i){ap[a].elements.push({getterReturnType:b,getter:w(c,d),getterContext:e,setterArgumentType:f,setter:w(g,h),setterContext:i})}function dR(a,b,c,d,e,f){aI[a]={name:u(b),rawConstructor:w(c,d),rawDestructor:w(e,f),fields:[]}}function dS(a,b,c,d,e,f,g,h,i,j){aI[a].fields.push({fieldName:u(b),getterReturnType:c,getter:w(d,e),getterContext:f,setterArgumentType:g,setter:w(h,i),setterContext:j})}function dT(b,a){a=u(a),z(b,{isVoid:!0,name:a,argPackAdvance:0,fromWireType:function(){},toWireType:function(a,b){}})}dU=!0;function dV(){return dU}function dW(b,d){for(var c=new Array(b),a=0;a>2],"parameter "+a);return c}function dX(d,b,i,g){var h,c,a,e,f;d=S.toValue(d),h=dW(b,i),c=new Array(b);for(a=0;a4&&(x.get(a).refcount+=1)}function dZ(a,b){a=aT(a,'_emval_take_value');var c=a.readValueFromPointer(b);return S.toHandle(c)}function d_(){p('native code called abort()')}function d$(){return Date.now()}N?bb=()=>{var a=process.hrtime();return a[0]*1e3+a[1]/1e6}:bb=()=>performance.now();function eb(b,a,c){v.copyWithin(b,a,a+c)}function ec(){return 2147483648}function ed(a){var b=af.buffer;try{return af.grow(a-b.byteLength+65535>>>16),bI(),1}catch(c){r('emscripten_realloc_buffer: Attempted to grow heap from '+b.byteLength+' bytes to '+a+' bytes, but got error: '+c)}}function ee(a){var b=v.length,d,e,f,g,i;if(a=a>>>0,c(a>b),d=ec(),a>d)return r('Cannot enlarge memory, asked to go up to '+a+' bytes, but the limit is '+d+' bytes!'),!1;let h=(b,a)=>b+(a-b%a)%a;for(e=1;e<=4;e*=2)if(f=b*(1+.2/e),f=Math.min(f,a+100663296),g=Math.min(d,h(Math.max(a,f),65536)),i=ed(g),i)return!0;return r('Failed to grow the heap from '+b+' bytes to '+g+' bytes, not enough memory!'),!1}a$={};function eg(){return bR||'./this.program'}function ab(){var d,b,a,c;if(!ab.strings){d=(typeof navigator=='object'&&navigator.languages&&navigator.languages[0]||'C').replace('-','_')+'.UTF-8',b={USER:'web_user',LOGNAME:'web_user',PATH:'/',PWD:'/',HOME:'/home/web_user',LANG:d,_:eg()};for(a in a$)a$[a]===void 0?delete b[a]:b[a]=a$[a];c=[];for(a in b)c.push(a+'='+b[a]);ab.strings=c}return ab.strings}function ei(b,d){for(var a=0;a>0]=b.charCodeAt(a);q[d>>0]=0}function ej(b,c){var a=0;return ab().forEach(function(d,f){var e=c+a;l[b+f*4>>2]=e,ei(d,e),a+=d.length+1}),0}function ek(c,d){var a=ab(),b;return l[c>>2]=a.length,b=0,a.forEach(function(a){b+=a.length+1}),l[d>>2]=b,0}function el(b){try{var c=t.getStreamFromFD(b);return a.close(c),0}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function em(h,c,j,d){for(var e=0,f=0,i,g,b;f>2],g=l[c+4>>2],c+=8,b=a.read(h,q,i,g,d),b<0)return-1;if(e+=b,b>2]=c,0}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function eo(a,b){return c(a==a>>>0||a==(a|0)),c(b===(b|0)),b+2097152>>>0<4194305-!!a?(a>>>0)+b*4294967296:NaN}function ep(i,f,g,d,e){var c,b;try{return c=eo(f,g),isNaN(c)?61:(b=t.getStreamFromFD(i),a.llseek(b,c,d),(s=[b.position>>>0,(k=b.position,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[e>>2]=s[0],h[e+4>>2]=s[1]),b.getdents&&c===0&&d===0&&(b.getdents=null),0)}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function eq(g,b,j,d){for(var e=0,f=0,h,i,c;f>2],i=l[b+4>>2],b+=8,c=a.write(g,q,h,i,d),c<0)return-1;e+=c,typeof d!='undefined'&&(d+=c)}return e}function er(d,e,f,g){var b,c;try{return b=t.getStreamFromFD(d),c=eq(b,e,f),l[g>>2]=c,0}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function es(a){return a}function ar(a){return a%4===0&&(a%100!==0||a%400===0)}function eu(c,d){for(var a=0,b=0;b<=d;a+=c[b++]);return a}bm=[31,29,31,30,31,30,31,31,30,31,30,31],bo=[31,28,31,30,31,30,31,31,30,31,30,31];function ex(e,b){for(var a=new Date(e.getTime()),f,c,d;b>0;)if(f=ar(a.getFullYear()),c=a.getMonth(),d=(f?bm:bo)[c],b>d-a.getDate())b-=d-a.getDate()+1,a.setDate(1),c<11?a.setMonth(c+1):(a.setMonth(0),a.setFullYear(a.getFullYear()+1));else return a.setDate(a.getDate()+b),a;return a}function ey(a,b){c(a.length>=0,'writeArrayToMemory array must have a length (should be an array or typed array)'),q.set(a,b)}function ez(s,r,q,b){var j=h[b+40>>2],p={tm_sec:h[b>>2],tm_min:h[b+4>>2],tm_hour:h[b+8>>2],tm_mday:h[b+12>>2],tm_mon:h[b+16>>2],tm_year:h[b+20>>2],tm_wday:h[b+24>>2],tm_yday:h[b+28>>2],tm_isdst:h[b+32>>2],tm_gmtoff:h[b+36>>2],tm_zone:j?L(j):''},c=L(q),i={'%c':'%a %b %d %H:%M:%S %Y','%D':'%m/%d/%y','%F':'%Y-%m-%d','%h':'%b','%r':'%I:%M:%S %p','%R':'%H:%M','%T':'%H:%M:%S','%x':'%m/%d/%y','%X':'%H:%M:%S','%Ec':'%c','%EC':'%C','%Ex':'%m/%d/%y','%EX':'%H:%M:%S','%Ey':'%y','%EY':'%Y','%Od':'%d','%Oe':'%e','%OH':'%H','%OI':'%I','%Om':'%m','%OM':'%M','%OS':'%S','%Ou':'%u','%OU':'%U','%OV':'%V','%Ow':'%w','%OW':'%W','%Oy':'%y'},d,f,l,n,e;for(d in i)c=c.replace(new RegExp(d,'g'),i[d]);f=['Sunday','Monday','Tuesday','Wednesday','Thursday','Friday','Saturday'],l=['January','February','March','April','May','June','July','August','September','October','November','December'];function m(b,c,d){for(var a=typeof b=='number'?b.toString():b||'';a.length0?1:0}var a;return(a=d(b.getFullYear()-c.getFullYear()))===0&&(a=d(b.getMonth()-c.getMonth()))===0&&(a=d(b.getDate()-c.getDate())),a}function g(a){switch(a.getDay()){case 0:return new Date(a.getFullYear()-1,11,29);case 1:return a;case 2:return new Date(a.getFullYear(),0,3);case 3:return new Date(a.getFullYear(),0,2);case 4:return new Date(a.getFullYear(),0,1);case 5:return new Date(a.getFullYear()-1,11,31);case 6:return new Date(a.getFullYear()-1,11,30)}}function k(b){var a=ex(new Date(b.tm_year+1900,0,1),b.tm_yday),c=new Date(a.getFullYear(),0,4),d=new Date(a.getFullYear()+1,0,4),e=g(c),f=g(d);return o(e,a)<=0?o(f,a)<=0?a.getFullYear()+1:a.getFullYear():a.getFullYear()-1}n={'%a':function(a){return f[a.tm_wday].substring(0,3)},'%A':function(a){return f[a.tm_wday]},'%b':function(a){return l[a.tm_mon].substring(0,3)},'%B':function(a){return l[a.tm_mon]},'%C':function(b){var c=b.tm_year+1900;return a(c/100|0,2)},'%d':function(b){return a(b.tm_mday,2)},'%e':function(a){return m(a.tm_mday,2,' ')},'%g':function(a){return k(a).toString().substring(2)},'%G':function(a){return k(a)},'%H':function(b){return a(b.tm_hour,2)},'%I':function(c){var b=c.tm_hour;return b==0?b=12:b>12&&(b-=12),a(b,2)},'%j':function(b){return a(b.tm_mday+eu(ar(b.tm_year+1900)?bm:bo,b.tm_mon-1),3)},'%m':function(b){return a(b.tm_mon+1,2)},'%M':function(b){return a(b.tm_min,2)},'%n':function(){return'\n'},'%p':function(a){return a.tm_hour>=0&&a.tm_hour<12?'AM':'PM'},'%S':function(b){return a(b.tm_sec,2)},'%t':function(){return' '},'%u':function(a){return a.tm_wday||7},'%U':function(b){var c=b.tm_yday+7-b.tm_wday;return a(Math.floor(c/7),2)},'%V':function(b){var c=Math.floor((b.tm_yday+7-(b.tm_wday+6)%7)/7),d,e;return(b.tm_wday+371-b.tm_yday-2)%7<=2&&c++,c?c==53&&(e=(b.tm_wday+371-b.tm_yday)%7,e!=4&&(e!=3||!ar(b.tm_year))&&(c=1)):(c=52,d=(b.tm_wday+7-b.tm_yday-1)%7,(d==4||d==5&&ar(b.tm_year%400-1))&&c++),a(c,2)},'%w':function(a){return a.tm_wday},'%W':function(b){var c=b.tm_yday+7-(b.tm_wday+6)%7;return a(Math.floor(c/7),2)},'%y':function(a){return(a.tm_year+1900).toString().substring(2)},'%Y':function(a){return a.tm_year+1900},'%z':function(b){var a=b.tm_gmtoff,c=a>=0;return a=Math.abs(a)/60,a=a/60*100+a%60,(c?'+':'-')+String("0000"+a).slice(-4)},'%Z':function(a){return a.tm_zone},'%%':function(){return'%'}},c=c.replace(/%%/g,'\0\0');for(d in n)c.includes(d)&&(c=c.replace(new RegExp(d,'g'),n[d](p)));return c=c.replace(/\0\0/g,'%'),e=aB(c,!1),e.length>r?0:(ey(e,s),e.length-1)}function eA(a,b,c,d,e){return ez(a,b,c,d)}bp=function(b,c,d,e){b||(b=this),this.parent=b,this.mount=b.mount,this.mounted=null,this.id=a.nextInode++,this.name=c,this.mode=d,this.node_ops={},this.stream_ops={},this.rdev=e},av=292|73,az=146,Object.defineProperties(bp.prototype,{read:{get:function(){return(this.mode&av)===av},set:function(a){a?this.mode|=av:this.mode&=~av}},write:{get:function(){return(this.mode&az)===az},set:function(a){a?this.mode|=az:this.mode&=~az}},isFolder:{get:function(){return a.isDir(this.mode)}},isDevice:{get:function(){return a.isChrdev(this.mode)}}}),a.FSNode=bp,a.staticInit(),b.FS_createPath=a.createPath,b.FS_createDataFile=a.createDataFile,b.FS_createPreloadedFile=a.createPreloadedFile,b.FS_unlink=a.unlink,b.FS_createLazyFile=a.createLazyFile,b.FS_createDevice=a.createDevice,aS={EPERM:63,ENOENT:44,ESRCH:71,EINTR:27,EIO:29,ENXIO:60,E2BIG:1,ENOEXEC:45,EBADF:8,ECHILD:12,EAGAIN:6,EWOULDBLOCK:6,ENOMEM:48,EACCES:2,EFAULT:21,ENOTBLK:105,EBUSY:10,EEXIST:20,EXDEV:75,ENODEV:43,ENOTDIR:54,EISDIR:31,EINVAL:28,ENFILE:41,EMFILE:33,ENOTTY:59,ETXTBSY:74,EFBIG:22,ENOSPC:51,ESPIPE:70,EROFS:69,EMLINK:34,EPIPE:64,EDOM:18,ERANGE:68,ENOMSG:49,EIDRM:24,ECHRNG:106,EL2NSYNC:156,EL3HLT:107,EL3RST:108,ELNRNG:109,EUNATCH:110,ENOCSI:111,EL2HLT:112,EDEADLK:16,ENOLCK:46,EBADE:113,EBADR:114,EXFULL:115,ENOANO:104,EBADRQC:103,EBADSLT:102,EDEADLOCK:16,EBFONT:101,ENOSTR:100,ENODATA:116,ETIME:117,ENOSR:118,ENONET:119,ENOPKG:120,EREMOTE:121,ENOLINK:47,EADV:122,ESRMNT:123,ECOMM:124,EPROTO:65,EMULTIHOP:36,EDOTDOT:125,EBADMSG:9,ENOTUNIQ:126,EBADFD:127,EREMCHG:128,ELIBACC:129,ELIBBAD:130,ELIBSCN:131,ELIBMAX:132,ELIBEXEC:133,ENOSYS:52,ENOTEMPTY:55,ENAMETOOLONG:37,ELOOP:32,EOPNOTSUPP:138,EPFNOSUPPORT:139,ECONNRESET:15,ENOBUFS:42,EAFNOSUPPORT:5,EPROTOTYPE:67,ENOTSOCK:57,ENOPROTOOPT:50,ESHUTDOWN:140,ECONNREFUSED:14,EADDRINUSE:3,ECONNABORTED:13,ENETUNREACH:40,ENETDOWN:38,ETIMEDOUT:73,EHOSTDOWN:142,EHOSTUNREACH:23,EINPROGRESS:26,EALREADY:7,EDESTADDRREQ:17,EMSGSIZE:35,EPROTONOSUPPORT:66,ESOCKTNOSUPPORT:137,EADDRNOTAVAIL:4,ENETRESET:39,EISCONN:30,ENOTCONN:53,ETOOMANYREFS:141,EUSERS:136,EDQUOT:19,ESTALE:72,ENOTSUP:138,ENOMEDIUM:148,EILSEQ:25,EOVERFLOW:61,ECANCELED:11,ENOTRECOVERABLE:56,EOWNERDEAD:62,ESTRPIPE:135},bZ=b.InternalError=aL(Error,'InternalError'),cf(),T=b.BindingError=aL(Error,'BindingError'),cM(),cB(),cY(),bw=b.UnboundTypeError=aL(Error,'UnboundTypeError'),ds(),eE=typeof atob=='function'?atob:function(a){var d='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=',b='',c=0,k,h,i,g,e,f,j;a=a.replace(/[^A-Za-z0-9\+\/\=]/g,'');do i=d.indexOf(a.charAt(c++)),g=d.indexOf(a.charAt(c++)),e=d.indexOf(a.charAt(c++)),f=d.indexOf(a.charAt(c++)),j=i<<2|g>>4,k=(g&15)<<4|e>>2,h=(e&3)<<6|f,b=b+String.fromCharCode(j),e!==64&&(b=b+String.fromCharCode(k)),f!==64&&(b=b+String.fromCharCode(h));while(c0)return;if(eU(),eB(),Q>0)return;function a(){if(ay)return;if(ay=!0,b.calledRun=!0,aD)return;ew(),bV(b),b.onRuntimeInitialized&&b.onRuntimeInitialized(),c(!b._main,'compiled without a main, but one is present. if you added it from JS, use Module["onRuntimeInitialized"]'),ev()}b.setStatus?(b.setStatus('Running...'),setTimeout(function(){setTimeout(function(){b.setStatus('')},1),a()},1)):a(),aV()}function gh(){var c=Z,d=r,b=!1;Z=r=a=>{b=!0};try{bK(0),['stdout','stderr'].forEach(function(e){var d=a.analyzePath('/dev/'+e),f,g,c;if(!d)return;f=d.object,g=f.rdev,c=G.ttys[g],c&&c.output&&c.output.length&&(b=!0)})}catch(a){}Z=c,r=d,b&&D('stdio streams had content in them that was not flushed. you should set EXIT_RUNTIME to 1 (see the FAQ), or make sure to emit a newline when you printf etc.')}if(b.preInit)for(typeof b.preInit=='function'&&(b.preInit=[b.preInit]);b.preInit.length>0;)b.preInit.pop()();return bL(),aJ.ready} diff --git a/toolkit/components/translations/fasttext/moz.yaml b/toolkit/components/translations/fasttext/moz.yaml new file mode 100644 index 000000000000..053df8727c39 --- /dev/null +++ b/toolkit/components/translations/fasttext/moz.yaml @@ -0,0 +1,44 @@ +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: Firefox + component: Translation + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: fasttext + + description: The JavaScript emscripten worker to run fastText + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: https://github.com/facebookresearch/fastText + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: v0.9.2 + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: 3697152e0fd772d9185697fdbd4a1d340ca5571d + + # The package's license, where possible using the mnemonic from + # https://spdx.org/licenses/ + # Multiple licenses can be specified (as a YAML list) + # A "LICENSE" file must exist containing the full license text + license: MIT + + notes: > + This code was generated from the fastText repository on the following revision: + 3697152e0fd772d9185697fdbd4a1d340ca5571d + + https://github.com/facebookresearch/fastText + + There are detailed instructions in the Firefox Source Docs on how to build these + dependencies locally. + + https://firefox-source-docs.mozilla.org/toolkit/components/translations/resources/02_contributing.html#building-fasttext diff --git a/toolkit/components/translations/jar.mn b/toolkit/components/translations/jar.mn index 34f2773fcc69..65b80f2e9c26 100644 --- a/toolkit/components/translations/jar.mn +++ b/toolkit/components/translations/jar.mn @@ -4,6 +4,10 @@ toolkit.jar: content/global/translations/bergamot-translator.js (bergamot-translator/bergamot-translator.js) + content/global/translations/fasttext.js (fasttext/fasttext.js) + content/global/translations/fasttext_wasm.js (fasttext/fasttext_wasm.js) + content/global/translations/language-id-engine.sys.mjs (content/language-id-engine.sys.mjs) + content/global/translations/language-id-engine-worker.js (content/language-id-engine-worker.js) content/global/translations/simd-detect-worker.js (content/simd-detect-worker.js) content/global/translations/translations-document.sys.mjs (content/translations-document.sys.mjs) content/global/translations/translations-engine.html (content/translations-engine.html) diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_translations.js b/toolkit/components/translations/tests/browser/browser_about_translations_translations.js index 033adf5ad8fb..3443fe2b2b29 100644 --- a/toolkit/components/translations/tests/browser/browser_about_translations_translations.js +++ b/toolkit/components/translations/tests/browser/browser_about_translations_translations.js @@ -173,10 +173,13 @@ add_task(async function test_about_translations_html() { add_task(async function test_about_translations_language_identification() { await openAboutTranslations({ + detectedLangTag: "en", + detectedLanguageConfidence: "0.98", languagePairs: [ { fromLang: "en", toLang: "fr" }, { fromLang: "fr", toLang: "en" }, ], + prefs: [["browser.translations.languageIdentification.useFastText", true]], runInPage: async ({ selectors }) => { const { document, window } = content; Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. @@ -218,7 +221,7 @@ add_task(async function test_about_translations_language_identification() { is( translation, translationResult.innerText, - "The language identification correctly informs the translation." + "The language identification engine correctly informs the translation." ); } diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js b/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js index 65479a968e10..6930465cf775 100644 --- a/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js @@ -4,9 +4,11 @@ "use strict"; add_task(async function test_detected_language() { + const detectedLangTag = "en"; const { cleanup, tab } = await loadTestPage({ // This page will get its language changed by the test. page: ENGLISH_PAGE_URL, + detectedLangTag, autoDownloadFromRemoteSettings: true, languagePairs: [ // Spanish @@ -75,7 +77,7 @@ add_task(async function test_detected_language() { Assert.deepEqual( await getDetectedLanguagesFor("gibberish"), { - docLangTag: "en", + docLangTag: detectedLangTag, userLangTag: null, isDocLangTagSupported: true, }, diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js b/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js index 132bb9a1f24a..a930b00c546a 100644 --- a/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js @@ -8,9 +8,12 @@ * issues. */ add_task(async function test_detected_language() { + const detectedLangTag = "en"; + const { cleanup, tab } = await loadTestPage({ // This page will get its language changed by the test. page: ENGLISH_PAGE_URL, + detectedLangTag, autoDownloadFromRemoteSettings: true, // Empty out the accept languages. languagePairs: [ diff --git a/toolkit/components/translations/tests/browser/browser_translations_full_page.js b/toolkit/components/translations/tests/browser/browser_translations_full_page.js index 7ee15902f8a6..2fe7a4557e3b 100644 --- a/toolkit/components/translations/tests/browser/browser_translations_full_page.js +++ b/toolkit/components/translations/tests/browser/browser_translations_full_page.js @@ -98,6 +98,10 @@ add_task(async function test_about_translations_enabled() { add_task(async function test_language_identification_for_page_translation() { await autoTranslatePage({ page: NO_LANGUAGE_URL, + detectedLangTag: "es", + detectedLanguageConfidence: 0.95, + resolveLanguageIdDownloads: true, + prefs: [["browser.translations.languageIdentification.useFastText", true]], languagePairs: [ { fromLang: "es", toLang: "en" }, { fromLang: "en", toLang: "es" }, diff --git a/toolkit/components/translations/tests/browser/shared-head.js b/toolkit/components/translations/tests/browser/shared-head.js index 7d15f0b2ca95..2e9cc73033e1 100644 --- a/toolkit/components/translations/tests/browser/shared-head.js +++ b/toolkit/components/translations/tests/browser/shared-head.js @@ -60,6 +60,14 @@ const NEVER_TRANSLATE_LANGS_PREF = * @param {boolean} [options.disabled] * Disable the panel through a pref. * + * @param {number} detectedLanguageConfidence + * This is the value for the MockedLanguageIdEngine to give as a confidence score for + * the mocked detected language. + * + * @param {string} detectedLangTag + * This is the BCP 47 language tag for the MockedLanguageIdEngine to return as + * the mocked detected language. + * * @param {Array<{ fromLang: string, toLang: string }>} options.languagePairs * The translation languages pairs to mock for the test. * @@ -70,6 +78,8 @@ async function openAboutTranslations({ dataForContent, disabled, runInPage, + detectedLanguageConfidence, + detectedLangTag, languagePairs = LANGUAGE_PAIRS, prefs, }) { @@ -108,6 +118,8 @@ async function openAboutTranslations({ // TODO(Bug 1814168) - Do not test download behavior as this is not robustly // handled for about:translations yet. autoDownloadFromRemoteSettings: true, + detectedLangTag, + detectedLanguageConfidence, }); // Now load the about:translations page, since the actor could be mocked. @@ -117,7 +129,10 @@ async function openAboutTranslations({ ); await BrowserTestUtils.browserLoaded(tab.linkedBrowser); - await remoteClients.translationsWasm.resolvePendingDownloads(1); + // Resolve the files. + await remoteClients.languageIdModels.resolvePendingDownloads(1); + // The language id and translation engine each have a wasm file, so expect 2 downloads. + await remoteClients.translationsWasm.resolvePendingDownloads(2); await remoteClients.translationModels.resolvePendingDownloads( languagePairs.length * FILES_PER_LANGUAGE_PAIR ); @@ -389,6 +404,8 @@ async function closeTranslationsPanelIfOpen() { async function setupActorTest({ languagePairs, prefs, + detectedLanguageConfidence, + detectedLangTag, autoDownloadFromRemoteSettings = false, }) { await SpecialPowers.pushPrefEnv({ @@ -402,6 +419,8 @@ async function setupActorTest({ const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ languagePairs, + detectedLangTag, + detectedLanguageConfidence, autoDownloadFromRemoteSettings, }); @@ -429,6 +448,8 @@ async function setupActorTest({ async function createAndMockRemoteSettings({ languagePairs = LANGUAGE_PAIRS, + detectedLanguageConfidence = 0.5, + detectedLangTag = "en", autoDownloadFromRemoteSettings = false, }) { const remoteClients = { @@ -439,6 +460,9 @@ async function createAndMockRemoteSettings({ translationsWasm: await createTranslationsWasmRemoteClient( autoDownloadFromRemoteSettings ), + languageIdModels: await createLanguageIdModelsRemoteClient( + autoDownloadFromRemoteSettings + ), }; // The TranslationsParent will pull the language pair values from the JSON dump @@ -450,13 +474,23 @@ async function createAndMockRemoteSettings({ remoteClients.translationsWasm.client ); + TranslationsParent.mockLanguageIdentification( + detectedLangTag, + detectedLanguageConfidence, + remoteClients.languageIdModels.client + ); return { async removeMocks() { await remoteClients.translationModels.client.attachments.deleteAll(); + await remoteClients.translationsWasm.client.attachments.deleteAll(); + await remoteClients.languageIdModels.client.attachments.deleteAll(); + await remoteClients.translationModels.client.db.clear(); await remoteClients.translationsWasm.client.db.clear(); + await remoteClients.languageIdModels.client.db.clear(); TranslationsParent.unmockTranslationsEngine(); + TranslationsParent.unmockLanguageIdentification(); TranslationsParent.clearCache(); }, remoteClients, @@ -466,6 +500,8 @@ async function createAndMockRemoteSettings({ async function loadTestPage({ languagePairs, autoDownloadFromRemoteSettings = false, + detectedLanguageConfidence, + detectedLangTag, page, prefs, autoOffer, @@ -506,6 +542,8 @@ async function loadTestPage({ const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ languagePairs, + detectedLanguageConfidence, + detectedLangTag, autoDownloadFromRemoteSettings, }); @@ -544,6 +582,11 @@ async function loadTestPage({ ); }, + async resolveLanguageIdDownloads() { + await remoteClients.translationsWasm.resolvePendingDownloads(1); + await remoteClients.languageIdModels.resolvePendingDownloads(1); + }, + /** * @returns {Promise} */ @@ -813,7 +856,7 @@ async function createTranslationModelsRemoteClient( async function createTranslationsWasmRemoteClient( autoDownloadFromRemoteSettings ) { - const records = ["bergamot-translator"].map(name => ({ + const records = ["bergamot-translator", "fasttext-wasm"].map(name => ({ id: crypto.randomUUID(), name, version: "1.0", @@ -839,6 +882,43 @@ async function createTranslationsWasmRemoteClient( ); } +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {boolean} autoDownloadFromRemoteSettings + * @returns {RemoteSettingsClient} + */ +async function createLanguageIdModelsRemoteClient( + autoDownloadFromRemoteSettings +) { + const records = [ + { + id: crypto.randomUUID(), + name: "lid.176.ftz", + version: "1.0", + last_modified: Date.now(), + schema: Date.now(), + }, + ]; + + const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" + ); + const client = RemoteSettings( + "test-language-id-models" + _remoteSettingsMockId++ + ); + const mockedCollectionName = "test-language-id-models"; + const metadata = {}; + await client.db.clear(); + await client.db.importChanges(metadata, Date.now(), records); + + return createAttachmentMock( + client, + mockedCollectionName, + autoDownloadFromRemoteSettings + ); +} + async function selectAboutPreferencesElements() { const document = gBrowser.selectedBrowser.contentDocument; diff --git a/toolkit/components/translations/translations.d.ts b/toolkit/components/translations/translations.d.ts index ec1e899af42a..829f22a9142d 100644 --- a/toolkit/components/translations/translations.d.ts +++ b/toolkit/components/translations/translations.d.ts @@ -24,6 +24,25 @@ export interface Attachment { mimetype: string; } +/** + * The JSON that is synced from Remote Settings for the language-id models. + */ +export interface LanguageIdModelRecord { + // e.g. "0d4db293-a17c-4085-9bd8-e2e146c85000" + id: string; + // The full model name, e.g. "lid.176.ftz" + name: string; + // The semver number, used for handling future format changes. e.g. 1.0 + version: string; + // The file attachment for this record + attachment: Attachment; + // e.g. 1673455932527 + last_modified: string; + // A JEXL expression to determine whether this record should be pulled from Remote Settings + // See: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions + filter_expression: string; +} + /** * The JSON that is synced from Remote Settings for the translation models. */ @@ -242,6 +261,18 @@ interface TranslationsEnginePayload { isMocked: boolean, } +/** + * These are the files that are downloaded from Remote Settings that are necessary + * to start the language-identification engine. These may not be available if running + * in tests. + */ +interface LanguageIdEnginePayload { + wasmBuffer: ArrayBuffer, + modelBuffer: ArrayBuffer, + mockedConfidence: null | number, + mockedLangTag: null | string, +} + /** * Nodes that are being translated are given priority according to their visibility. */ diff --git a/toolkit/content/license.html b/toolkit/content/license.html index a6dc1069f420..9663397dd62e 100644 --- a/toolkit/content/license.html +++ b/toolkit/content/license.html @@ -3657,6 +3657,8 @@ SOFTWARE.
  • third_party/js/cfworker/json-schema.js
  • security/nss/lib/freebl/ecl/ecp_secp384r1.c and security/nss/lib/freebl/ecl/ecp_secp521r1.c
    • +
  • toolkit/components/translations/fasttext/fasttext.js and + toolkit/components/translations/fasttext/fasttext_wasm.js
  • security/nss/lib/freebl/ecl/curve25519_32.c, security/nss/lib/freebl/ecl/ecp_secp384r1.c and security/nss/lib/freebl/ecl/ecp_secp521r1.c
    • diff --git a/tools/rewriting/ThirdPartyPaths.txt b/tools/rewriting/ThirdPartyPaths.txt index 79eaa0b0f627..f00dc46fe831 100644 --- a/tools/rewriting/ThirdPartyPaths.txt +++ b/tools/rewriting/ThirdPartyPaths.txt @@ -180,6 +180,8 @@ toolkit/components/passwordmgr/PasswordRulesParser.sys.mjs toolkit/components/protobuf/ toolkit/components/translation/cld2/ toolkit/components/translations/bergamot-translator +toolkit/components/translations/fasttext/fasttext.js +toolkit/components/translations/fasttext/fasttext_wasm.js toolkit/components/url-classifier/chromium/ toolkit/components/utils/mozjexl.js toolkit/components/viaduct/fetch_msg_types.pb.cc