Bug 1771587 - Implement are-we-esmified-yet check on ci; r=marco,jmaher

Differential Revision: https://phabricator.services.mozilla.com/D150369

.cron.yml

@@ -335,3 +335,13 @@ jobs:
           by-project:
               mozilla-central: [{hour: 11, minute: 0}, {hour: 23, minute: 0}]
               default: []
+
+    - name: are-we-esmified-yet
+      job:
+          type: decision-task
+          treeherder-symbol: are-we-esmified-yet
+          target-tasks-method: are-we-esmified-yet
+      run-on-projects:
+          - mozilla-central
+      when:
+          - {hour: 0, minute: 00}

taskcluster/ci/are-we-esmified-yet/kind.yml

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: gecko_taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - toolchain
+    - fetch
+
+transforms:
+    - gecko_taskgraph.transforms.job:transforms
+    - gecko_taskgraph.transforms.task:transforms
+
+jobs:
+    check:
+        description: Extracts statistics on remaining JSMs in the codebase
+        run-on-projects: []
+        attributes:
+            cron: true
+        treeherder:
+            platform: lint/opt
+            kind: other
+            symbol: js(esm)
+            tier: 2
+        worker-type: t-linux-xlarge-source
+        worker:
+            docker-image: {in-tree: "lint"}
+            max-run-time: 3600
+            artifacts:
+                - type: file
+                  name: public/are-we-esmified-yet.json
+                  path: /builds/worker/artifacts/are-we-esmified-yet.json
+        run:
+            using: run-task
+            command: >-
+                ./taskcluster/scripts/misc/run-esmification-check.sh

taskcluster/docs/kinds.rst

@@ -726,3 +726,7 @@ fxrecord
 
 Visual metrics computation of desktop Firefox startup. The performance team
 monitors this task to watch for regressions in Firefox startup performance.
+
+are-we-esmified-yet
+---------------------
+Collects data about the transition to ECMAScript Modules from JSMs.

taskcluster/gecko_taskgraph/target_tasks.py

@@ -1322,3 +1322,13 @@ def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config):
         return task.kind in ["l10n-cross-channel"]
 
     return [l for l, t in full_task_graph.tasks.items() if filter(t)]
+
+
+@_target_task("are-we-esmified-yet")
+def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config):
+    """
+    select the task to track the progress of the esmification project
+    """
+    return [
+        l for l, t in full_task_graph.tasks.items() if t.kind == "are-we-esmified-yet"
+    ]

taskcluster/scripts/misc/are-we-esmified-yet.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import pathlib
+import re
+import subprocess
+import sys
+
+TBPL_FAILURE = 2
+
+excluded_files = [
+    # Testcase for loader.
+    "js/xpconnect/tests/unit/environment_script.js"
+    "js/xpconnect/tests/chrome/file_expandosharing.jsm",
+    "js/xpconnect/tests/unit/bogus_element_type.jsm",
+    "js/xpconnect/tests/unit/bogus_exports_type.jsm",
+    "js/xpconnect/tests/unit/environment_checkscript.jsm",
+    "js/xpconnect/tests/unit/environment_loadscript.jsm",
+    "js/xpconnect/tests/unit/importer.jsm",
+    "js/xpconnect/tests/unit/not-esmified-not-exported.jsm",
+    "js/xpconnect/tests/unit/recursive_importA.jsm",
+    "js/xpconnect/tests/unit/recursive_importB.jsm",
+    "js/xpconnect/tests/unit/syntax_error.jsm",
+    "js/xpconnect/tests/unit/uninitialized_lexical.jsm",
+    "dom/url/tests/file_url.jsm",
+    "dom/url/tests/file_worker_url.jsm",
+    "dom/url/tests/test_bug883784.jsm",
+    "dom/workers/test/WorkerTest.jsm",
+    "dom/encoding/test/file_stringencoding.jsm",
+    "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.jsm",
+    # Testcase for build system.
+    "python/mozbuild/mozbuild/test/backend/data/build/bar.jsm",
+    "python/mozbuild/mozbuild/test/backend/data/build/baz.jsm",
+    "python/mozbuild/mozbuild/test/backend/data/build/foo.jsm",
+    "python/mozbuild/mozbuild/test/backend/data/build/qux.jsm",
+    # Testcase for test harness.
+    "testing/mochitest/tests/Harness_sanity/ImportTesting.jsm",
+    # EXPORTED_SYMBOLS inside testcase.
+    "tools/lint/eslint/eslint-plugin-mozilla/tests/mark-exported-symbols-as-used.js",
+]
+
+if pathlib.Path(".hg").exists():
+    mode = "hg"
+elif pathlib.Path(".git").exists():
+    mode = "git"
+else:
+    print(
+        "Error: This script needs to be run inside mozilla-central checkout "
+        "of either mercurial or git.",
+        file=sys.stderr,
+    )
+    sys.exit(TBPL_FAILURE)
+
+
+def new_files_struct():
+    return {
+        "jsm": [],
+        "esm": [],
+        "subdir": {},
+    }
+
+
+def put_file(files, kind, path):
+    """Put a path into files tree structure."""
+
+    if str(path) in excluded_files:
+        return
+
+    name = path.name
+
+    current_files = files
+    for part in path.parent.parts:
+        if part not in current_files["subdir"]:
+            current_files["subdir"][part] = new_files_struct()
+        current_files = current_files["subdir"][part]
+
+    current_files[kind].append(name)
+
+
+def run(cmd):
+    """Run command and return output as lines, excluding empty line."""
+    lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode()
+    return filter(lambda x: x != "", lines.split("\n"))
+
+
+def collect_jsm(files):
+    """Collect JSM files."""
+    kind = "jsm"
+
+    # jsm files
+    if mode == "hg":
+        cmd = ["hg", "files", "set:glob:**/*.jsm"]
+    else:
+        cmd = ["git", "ls-files", "*.jsm"]
+    for line in run(cmd):
+        put_file(files, kind, pathlib.Path(line))
+
+    # js files with EXPORTED_SYMBOLS
+    if mode == "hg":
+        cmd = ["hg", "files", "set:grep('EXPORTED_SYMBOLS = \[') and glob:**/*.js"]
+        for line in run(cmd):
+            put_file(files, kind, pathlib.Path(line))
+    else:
+        handled = {}
+        cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", "*.js"]
+        for line in run(cmd):
+            m = re.search("^([^:]+):", line)
+            if not m:
+                continue
+            path = m.group(1)
+            if path in handled:
+                continue
+            handled[path] = True
+            put_file(files, kind, pathlib.Path(path))
+
+
+def collect_esm(files):
+    """Collect system ESM files."""
+    kind = "esm"
+
+    # sys.mjs files
+    if mode == "hg":
+        cmd = ["hg", "files", "set:glob:**/*.sys.mjs"]
+    else:
+        cmd = ["git", "ls-files", "*.sys.mjs"]
+    for line in run(cmd):
+        put_file(files, kind, pathlib.Path(line))
+
+
+def to_stat(files):
+    """Convert files tree into status tree."""
+    jsm = len(files["jsm"])
+    esm = len(files["esm"])
+    subdir = {}
+
+    for key, sub_files in files["subdir"].items():
+        sub_stat = to_stat(sub_files)
+
+        subdir[key] = sub_stat
+        jsm += sub_stat["jsm"]
+        esm += sub_stat["esm"]
+
+    stat = {
+        "jsm": jsm,
+        "esm": esm,
+    }
+    if len(subdir):
+        stat["subdir"] = subdir
+
+    return stat
+
+
+files = new_files_struct()
+collect_jsm(files)
+collect_esm(files)
+
+stat = to_stat(files)
+print(json.dumps(stat, indent=2))

taskcluster/scripts/misc/run-esmification-check.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+cd /builds/worker
+mkdir -p artifacts
+python3 "$GECKO_PATH/taskcluster/scripts/misc/are-we-esmified-yet.py" > artifacts/are-we-esmified-yet.json