diff --git a/.cron.yml b/.cron.yml index 6747703e8abb..33e1af2c4f1d 100644 --- a/.cron.yml +++ b/.cron.yml @@ -335,3 +335,13 @@ jobs: by-project: mozilla-central: [{hour: 11, minute: 0}, {hour: 23, minute: 0}] default: [] + + - name: are-we-esmified-yet + job: + type: decision-task + treeherder-symbol: are-we-esmified-yet + target-tasks-method: are-we-esmified-yet + run-on-projects: + - mozilla-central + when: + - {hour: 0, minute: 00} diff --git a/taskcluster/ci/are-we-esmified-yet/kind.yml b/taskcluster/ci/are-we-esmified-yet/kind.yml new file mode 100644 index 000000000000..e0eb8ecff24e --- /dev/null +++ b/taskcluster/ci/are-we-esmified-yet/kind.yml @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: gecko_taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - fetch + +transforms: + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +jobs: + check: + description: Extracts statistics on remaining JSMs in the codebase + run-on-projects: [] + attributes: + cron: true + treeherder: + platform: lint/opt + kind: other + symbol: js(esm) + tier: 2 + worker-type: t-linux-xlarge-source + worker: + docker-image: {in-tree: "lint"} + max-run-time: 3600 + artifacts: + - type: file + name: public/are-we-esmified-yet.json + path: /builds/worker/artifacts/are-we-esmified-yet.json + run: + using: run-task + command: >- + ./taskcluster/scripts/misc/run-esmification-check.sh diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index d0007bb490cc..ae11b7da80de 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -726,3 +726,7 @@ fxrecord Visual metrics computation of desktop Firefox startup. The performance team monitors this task to watch for regressions in Firefox startup performance. + +are-we-esmified-yet +--------------------- +Collects data about the transition to ECMAScript Modules from JSMs. diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py index fd7adddd3997..d6b586996d0d 100644 --- a/taskcluster/gecko_taskgraph/target_tasks.py +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -1322,3 +1322,13 @@ def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): return task.kind in ["l10n-cross-channel"] return [l for l, t in full_task_graph.tasks.items() if filter(t)] + + +@_target_task("are-we-esmified-yet") +def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config): + """ + select the task to track the progress of the esmification project + """ + return [ + l for l, t in full_task_graph.tasks.items() if t.kind == "are-we-esmified-yet" + ] diff --git a/taskcluster/scripts/misc/are-we-esmified-yet.py b/taskcluster/scripts/misc/are-we-esmified-yet.py new file mode 100644 index 000000000000..5ec610d3de66 --- /dev/null +++ b/taskcluster/scripts/misc/are-we-esmified-yet.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import pathlib +import re +import subprocess +import sys + +TBPL_FAILURE = 2 + +excluded_files = [ + # Testcase for loader. + "js/xpconnect/tests/unit/environment_script.js" + "js/xpconnect/tests/chrome/file_expandosharing.jsm", + "js/xpconnect/tests/unit/bogus_element_type.jsm", + "js/xpconnect/tests/unit/bogus_exports_type.jsm", + "js/xpconnect/tests/unit/environment_checkscript.jsm", + "js/xpconnect/tests/unit/environment_loadscript.jsm", + "js/xpconnect/tests/unit/importer.jsm", + "js/xpconnect/tests/unit/not-esmified-not-exported.jsm", + "js/xpconnect/tests/unit/recursive_importA.jsm", + "js/xpconnect/tests/unit/recursive_importB.jsm", + "js/xpconnect/tests/unit/syntax_error.jsm", + "js/xpconnect/tests/unit/uninitialized_lexical.jsm", + "dom/url/tests/file_url.jsm", + "dom/url/tests/file_worker_url.jsm", + "dom/url/tests/test_bug883784.jsm", + "dom/workers/test/WorkerTest.jsm", + "dom/encoding/test/file_stringencoding.jsm", + "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.jsm", + # Testcase for build system. + "python/mozbuild/mozbuild/test/backend/data/build/bar.jsm", + "python/mozbuild/mozbuild/test/backend/data/build/baz.jsm", + "python/mozbuild/mozbuild/test/backend/data/build/foo.jsm", + "python/mozbuild/mozbuild/test/backend/data/build/qux.jsm", + # Testcase for test harness. + "testing/mochitest/tests/Harness_sanity/ImportTesting.jsm", + # EXPORTED_SYMBOLS inside testcase. + "tools/lint/eslint/eslint-plugin-mozilla/tests/mark-exported-symbols-as-used.js", +] + +if pathlib.Path(".hg").exists(): + mode = "hg" +elif pathlib.Path(".git").exists(): + mode = "git" +else: + print( + "Error: This script needs to be run inside mozilla-central checkout " + "of either mercurial or git.", + file=sys.stderr, + ) + sys.exit(TBPL_FAILURE) + + +def new_files_struct(): + return { + "jsm": [], + "esm": [], + "subdir": {}, + } + + +def put_file(files, kind, path): + """Put a path into files tree structure.""" + + if str(path) in excluded_files: + return + + name = path.name + + current_files = files + for part in path.parent.parts: + if part not in current_files["subdir"]: + current_files["subdir"][part] = new_files_struct() + current_files = current_files["subdir"][part] + + current_files[kind].append(name) + + +def run(cmd): + """Run command and return output as lines, excluding empty line.""" + lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode() + return filter(lambda x: x != "", lines.split("\n")) + + +def collect_jsm(files): + """Collect JSM files.""" + kind = "jsm" + + # jsm files + if mode == "hg": + cmd = ["hg", "files", "set:glob:**/*.jsm"] + else: + cmd = ["git", "ls-files", "*.jsm"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + # js files with EXPORTED_SYMBOLS + if mode == "hg": + cmd = ["hg", "files", "set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + else: + handled = {} + cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", "*.js"] + for line in run(cmd): + m = re.search("^([^:]+):", line) + if not m: + continue + path = m.group(1) + if path in handled: + continue + handled[path] = True + put_file(files, kind, pathlib.Path(path)) + + +def collect_esm(files): + """Collect system ESM files.""" + kind = "esm" + + # sys.mjs files + if mode == "hg": + cmd = ["hg", "files", "set:glob:**/*.sys.mjs"] + else: + cmd = ["git", "ls-files", "*.sys.mjs"] + for line in run(cmd): + put_file(files, kind, pathlib.Path(line)) + + +def to_stat(files): + """Convert files tree into status tree.""" + jsm = len(files["jsm"]) + esm = len(files["esm"]) + subdir = {} + + for key, sub_files in files["subdir"].items(): + sub_stat = to_stat(sub_files) + + subdir[key] = sub_stat + jsm += sub_stat["jsm"] + esm += sub_stat["esm"] + + stat = { + "jsm": jsm, + "esm": esm, + } + if len(subdir): + stat["subdir"] = subdir + + return stat + + +files = new_files_struct() +collect_jsm(files) +collect_esm(files) + +stat = to_stat(files) +print(json.dumps(stat, indent=2)) diff --git a/taskcluster/scripts/misc/run-esmification-check.sh b/taskcluster/scripts/misc/run-esmification-check.sh new file mode 100755 index 000000000000..3b090d3d6db2 --- /dev/null +++ b/taskcluster/scripts/misc/run-esmification-check.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +cd /builds/worker +mkdir -p artifacts +python3 "$GECKO_PATH/taskcluster/scripts/misc/are-we-esmified-yet.py" > artifacts/are-we-esmified-yet.json