From 1473c23cbf6b545ec98efa5d4228f9b9b92f5bd2 Mon Sep 17 00:00:00 2001 From: Martin Robinson Date: Tue, 16 Apr 2024 09:31:17 +0000 Subject: [PATCH] Bug 1885597 - Remove GitHub synchronization task r=jrmuizel,jnicol,taskgraph-reviewers,releng-reviewers,ahal,bhearsum This synchronization is now handled in the downstream repositories. In the case of WebRender by the Servo project and in the case of qcms by a Gecko-managed GitHub Action. This change removes the github-sync task from mozilla-central. Differential Revision: https://phabricator.services.mozilla.com/D204787 --- build/sparse-profiles/github-sync | 7 - taskcluster/ci/config.yml | 1 - taskcluster/ci/docker-image/kind.yml | 3 - taskcluster/ci/github-sync/kind.yml | 53 -- taskcluster/docker/github-sync/Dockerfile | 14 - taskcluster/docker/github-sync/prepare.sh | 31 -- .../docker/github-sync/requirements.txt | 3 - taskcluster/docs/kinds.rst | 5 - .../gecko_taskgraph/transforms/github_sync.py | 23 - .../test/params/mb-promote-android.yml | 1 - .../params/mb-promote-devedition-partials.yml | 1 - .../test/params/mb-promote-devedition.yml | 1 - .../params/mb-promote-firefox-partials.yml | 1 - .../test/params/mb-promote-firefox.yml | 1 - taskcluster/test/params/mb-push-android.yml | 1 - .../test/params/mb-push-devedition.yml | 1 - .../test/params/mb-push-firefox-partials.yml | 1 - taskcluster/test/params/mb-push-firefox.yml | 1 - taskcluster/test/params/mb-ship-android.yml | 1 - .../test/params/mb-ship-devedition.yml | 1 - .../test/params/mb-ship-firefox-partials.yml | 1 - taskcluster/test/params/mb-ship-firefox.yml | 1 - .../test/params/mc-android-nightly.yml | 1 - .../test/params/mc-desktop-nightly.yml | 1 - taskcluster/test/params/mc-nightly-all.yml | 1 - .../test/params/me-promote-firefox.yml | 1 - taskcluster/test/params/me-push-firefox.yml | 1 - taskcluster/test/params/me-ship-firefox.yml | 1 - .../test/params/mr-promote-firefox-rc.yml | 1 - .../test/params/mr-promote-firefox.yml | 1 - taskcluster/test/params/mr-push-firefox.yml | 1 - .../test/params/mr-ship-firefox-rc.yml | 1 - taskcluster/test/params/mr-ship-firefox.yml | 1 - tools/github-sync/converter.py | 481 ------------------ tools/github-sync/read-json.py | 42 -- tools/github-sync/readme.md | 106 ---- tools/github-sync/sync-to-github.sh | 159 ------ tools/moz.build | 3 - 38 files changed, 955 deletions(-) delete mode 100644 build/sparse-profiles/github-sync delete mode 100644 taskcluster/ci/github-sync/kind.yml delete mode 100644 taskcluster/docker/github-sync/Dockerfile delete mode 100755 taskcluster/docker/github-sync/prepare.sh delete mode 100644 taskcluster/docker/github-sync/requirements.txt delete mode 100644 taskcluster/gecko_taskgraph/transforms/github_sync.py delete mode 100755 tools/github-sync/converter.py delete mode 100755 tools/github-sync/read-json.py delete mode 100644 tools/github-sync/readme.md delete mode 100755 tools/github-sync/sync-to-github.sh diff --git a/build/sparse-profiles/github-sync b/build/sparse-profiles/github-sync deleted file mode 100644 index 72f265b3a13c..000000000000 --- a/build/sparse-profiles/github-sync +++ /dev/null @@ -1,7 +0,0 @@ -%include build/sparse-profiles/mach -%include build/sparse-profiles/taskgraph - -[include] -path:gfx/wr/ -path:taskcluster/scripts/misc/ -path:tools/github-sync/ diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 7757a79700a3..577b116a2fa6 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -235,7 +235,6 @@ treeherder: 'test-info': 'Test manifest skip/fail information' 'condprof': 'Conditioned Profile Builder' 'doc': 'Documentation' - 'GhS': 'GitHub Synchronization' 'perftest': 'Performance tests' 'perftest-detect': 'Performance regression detection tooling' 'perftest-chrome': 'Performance tests using Chrone' diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index 12dc16cdd10e..22b18cf16f46 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -214,9 +214,6 @@ jobs: condprof: symbol: I(condprof) parent: ubuntu1804-test - github-sync: - symbol: I(github-sync) - parent: debian12-base sentry: symbol: I(sentry) parent: debian12-base diff --git a/taskcluster/ci/github-sync/kind.yml b/taskcluster/ci/github-sync/kind.yml deleted file mode 100644 index 7c0b8136c411..000000000000 --- a/taskcluster/ci/github-sync/kind.yml +++ /dev/null @@ -1,53 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. ---- -loader: gecko_taskgraph.loader.transform:loader - -kind-dependencies: - - toolchain - - build - -transforms: - - gecko_taskgraph.transforms.github_sync:transforms - - gecko_taskgraph.transforms.job:transforms - - gecko_taskgraph.transforms.task:transforms - -job-defaults: - attributes: - retrigger: true - run-on-projects: ['mozilla-central-only'] - worker-type: b-linux-gcp - treeherder: - platform: other/opt - tier: 2 - kind: other - worker: - max-run-time: 3600 - docker-image: {in-tree: github-sync} - run: - sparse-profile: github-sync - using: run-task - -jobs: - webrender: - description: Sync webrender to github mirror - secret: gecko/gfx-github-sync/token - run: - command: '$GECKO_PATH/tools/github-sync/sync-to-github.sh webrender gfx/wr servo/webrender @bors-servo' - when: - files-changed: - - 'gfx/wr/**' - treeherder: - symbol: GhS(wr) - - qcms: - description: Sync qcms to github mirror - secret: gecko/gfx-github-sync/token - run: - command: '$GECKO_PATH/tools/github-sync/sync-to-github.sh qcms gfx/qcms FirefoxGraphics/qcms @jrmuizel' - when: - files-changed: - - 'gfx/qcms/**' - treeherder: - symbol: GhS(qcms) diff --git a/taskcluster/docker/github-sync/Dockerfile b/taskcluster/docker/github-sync/Dockerfile deleted file mode 100644 index bff131ad7bde..000000000000 --- a/taskcluster/docker/github-sync/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM $DOCKER_IMAGE_PARENT -MAINTAINER Kartikaya Gupta - -VOLUME /builds/worker/checkouts -VOLUME /builds/worker/workspace -VOLUME /builds/worker/tooltool-cache -VOLUME /builds/worker/.ghsync - -ADD prepare.sh /setup/prepare-docker.sh -ADD requirements.txt /setup/requirements.txt -RUN /bin/bash /setup/prepare-docker.sh /setup/requirements.txt && rm -R /setup - -# Set a default command useful for debugging -CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/github-sync/prepare.sh b/taskcluster/docker/github-sync/prepare.sh deleted file mode 100755 index 6c0be271680e..000000000000 --- a/taskcluster/docker/github-sync/prepare.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -test "$(whoami)" == 'root' - -# Install stuff we need -apt-get -y update -apt-get install -y \ - cmake \ - curl \ - gcc \ - git \ - g++ \ - libffi-dev \ - libgit2-dev \ - libssl-dev \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools - -# Python packages -pip3 install --break-system-packages -r "$1" diff --git a/taskcluster/docker/github-sync/requirements.txt b/taskcluster/docker/github-sync/requirements.txt deleted file mode 100644 index 13fd42681dae..000000000000 --- a/taskcluster/docker/github-sync/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests == 2.21.0 -pygit2 == 1.11.1 -python-hglib == 2.6.1 diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index b54f95f0faf7..b05c9542bb07 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -703,11 +703,6 @@ webrender Tasks used to do testing of WebRender standalone (without gecko). The WebRender code lives in gfx/wr and has its own testing infrastructure. -github-sync ------------- -Tasks used to do synchronize parts of Gecko that have downstream GitHub -repositories. - instrumented-build ------------------ Tasks that generate builds with PGO instrumentation enabled. This is an diff --git a/taskcluster/gecko_taskgraph/transforms/github_sync.py b/taskcluster/gecko_taskgraph/transforms/github_sync.py deleted file mode 100644 index 6f48f794ce3d..000000000000 --- a/taskcluster/gecko_taskgraph/transforms/github_sync.py +++ /dev/null @@ -1,23 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from taskgraph.transforms.base import TransformSequence - -transforms = TransformSequence() - - -@transforms.add -def sync_github(config, tasks): - """Do transforms specific to github-sync tasks.""" - for task in tasks: - # Add the secret to the scopes, only in m-c. - # Doing this on any other tree will result in decision task failure - # because m-c is the only one allowed to have that scope. - secret = task["secret"] - if config.params["project"] == "mozilla-central": - task.setdefault("scopes", []) - task["scopes"].append("secrets:get:" + secret) - task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret - del task["secret"] - yield task diff --git a/taskcluster/test/params/mb-promote-android.yml b/taskcluster/test/params/mb-promote-android.yml index 90a8c1e60297..b1fed85cb7df 100644 --- a/taskcluster/test/params/mb-promote-android.yml +++ b/taskcluster/test/params/mb-promote-android.yml @@ -160,7 +160,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-promote-devedition-partials.yml b/taskcluster/test/params/mb-promote-devedition-partials.yml index 5517cf96ce4b..b70e622ad838 100644 --- a/taskcluster/test/params/mb-promote-devedition-partials.yml +++ b/taskcluster/test/params/mb-promote-devedition-partials.yml @@ -160,7 +160,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-promote-devedition.yml b/taskcluster/test/params/mb-promote-devedition.yml index 306c117f44cb..002cf55c7344 100644 --- a/taskcluster/test/params/mb-promote-devedition.yml +++ b/taskcluster/test/params/mb-promote-devedition.yml @@ -160,7 +160,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-promote-firefox-partials.yml b/taskcluster/test/params/mb-promote-firefox-partials.yml index 39bcac3f871d..2092b1e63842 100644 --- a/taskcluster/test/params/mb-promote-firefox-partials.yml +++ b/taskcluster/test/params/mb-promote-firefox-partials.yml @@ -160,7 +160,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-promote-firefox.yml b/taskcluster/test/params/mb-promote-firefox.yml index 1fe4f6930471..0c066c00f7ae 100644 --- a/taskcluster/test/params/mb-promote-firefox.yml +++ b/taskcluster/test/params/mb-promote-firefox.yml @@ -160,7 +160,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-push-android.yml b/taskcluster/test/params/mb-push-android.yml index 1acb1628ea9e..2f18c0f6f3ac 100644 --- a/taskcluster/test/params/mb-push-android.yml +++ b/taskcluster/test/params/mb-push-android.yml @@ -2405,7 +2405,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-push-devedition.yml b/taskcluster/test/params/mb-push-devedition.yml index 1952bac83334..bed653b3a480 100644 --- a/taskcluster/test/params/mb-push-devedition.yml +++ b/taskcluster/test/params/mb-push-devedition.yml @@ -2405,7 +2405,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-push-firefox-partials.yml b/taskcluster/test/params/mb-push-firefox-partials.yml index 5fcf72ec14c8..a73885592ceb 100644 --- a/taskcluster/test/params/mb-push-firefox-partials.yml +++ b/taskcluster/test/params/mb-push-firefox-partials.yml @@ -2405,7 +2405,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-push-firefox.yml b/taskcluster/test/params/mb-push-firefox.yml index 78f4199b13ab..1db380ec51d1 100644 --- a/taskcluster/test/params/mb-push-firefox.yml +++ b/taskcluster/test/params/mb-push-firefox.yml @@ -2405,7 +2405,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-ship-android.yml b/taskcluster/test/params/mb-ship-android.yml index e2d19707e27a..0dbdb029f2d8 100644 --- a/taskcluster/test/params/mb-ship-android.yml +++ b/taskcluster/test/params/mb-ship-android.yml @@ -147,7 +147,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-ship-devedition.yml b/taskcluster/test/params/mb-ship-devedition.yml index 08447d8139b4..35f851826870 100644 --- a/taskcluster/test/params/mb-ship-devedition.yml +++ b/taskcluster/test/params/mb-ship-devedition.yml @@ -147,7 +147,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-ship-firefox-partials.yml b/taskcluster/test/params/mb-ship-firefox-partials.yml index f1ad991f9935..835fb532d923 100644 --- a/taskcluster/test/params/mb-ship-firefox-partials.yml +++ b/taskcluster/test/params/mb-ship-firefox-partials.yml @@ -147,7 +147,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mb-ship-firefox.yml b/taskcluster/test/params/mb-ship-firefox.yml index 4a5eb2161a2b..42778db53948 100644 --- a/taskcluster/test/params/mb-ship-firefox.yml +++ b/taskcluster/test/params/mb-ship-firefox.yml @@ -147,7 +147,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: H2NDEiu8RJWciIqy0yu0IA diff --git a/taskcluster/test/params/mc-android-nightly.yml b/taskcluster/test/params/mc-android-nightly.yml index ea1b87a23694..72cd7247852f 100644 --- a/taskcluster/test/params/mc-android-nightly.yml +++ b/taskcluster/test/params/mc-android-nightly.yml @@ -206,7 +206,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: UGlRm6fZSJOQc1JcPC23lw diff --git a/taskcluster/test/params/mc-desktop-nightly.yml b/taskcluster/test/params/mc-desktop-nightly.yml index 37d7f5de7448..cadc2a8910c8 100644 --- a/taskcluster/test/params/mc-desktop-nightly.yml +++ b/taskcluster/test/params/mc-desktop-nightly.yml @@ -206,7 +206,6 @@ existing_tasks: docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: UGlRm6fZSJOQc1JcPC23lw diff --git a/taskcluster/test/params/mc-nightly-all.yml b/taskcluster/test/params/mc-nightly-all.yml index 7196e1292081..1e726aa539b8 100644 --- a/taskcluster/test/params/mc-nightly-all.yml +++ b/taskcluster/test/params/mc-nightly-all.yml @@ -345,7 +345,6 @@ existing_tasks: docker-image-firefox-snap: TaMBeARtSCKe485qQborMA docker-image-funsize-update-generator: RwrXtrceR_aEmcsEUxGiCg docker-image-gdb-test: ahyis7rkQTqhCapWXDqneg - docker-image-github-sync: ds8yNkB5ScKKqLW2Cv0OWQ docker-image-image_builder: Z3gDOem4TSWtoayyvtW_ng docker-image-index-task: dbQOfI1hTp6sh4sGNnW9sQ docker-image-lint: GntDF25JRUaCu5nuedyceA diff --git a/taskcluster/test/params/me-promote-firefox.yml b/taskcluster/test/params/me-promote-firefox.yml index fc713a788eed..6a4a9ceaee9b 100644 --- a/taskcluster/test/params/me-promote-firefox.yml +++ b/taskcluster/test/params/me-promote-firefox.yml @@ -97,7 +97,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg docker-image-gdb-test: WdZGFUBiTzyn5er0FJQUvw - docker-image-github-sync: SmOuKw2vSA2P0B6MxH6xXw docker-image-image_builder: MWRymupRSheUkuqIQwF8TA docker-image-index-task: VIneOK4VRXGrcEmrEpleSA docker-image-lint: Eeq8v2PJSbSivC61RBNaKg diff --git a/taskcluster/test/params/me-push-firefox.yml b/taskcluster/test/params/me-push-firefox.yml index 861a7676606d..57af7d97c55b 100644 --- a/taskcluster/test/params/me-push-firefox.yml +++ b/taskcluster/test/params/me-push-firefox.yml @@ -2219,7 +2219,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg docker-image-gdb-test: WdZGFUBiTzyn5er0FJQUvw - docker-image-github-sync: SmOuKw2vSA2P0B6MxH6xXw docker-image-image_builder: MWRymupRSheUkuqIQwF8TA docker-image-index-task: VIneOK4VRXGrcEmrEpleSA docker-image-lint: Eeq8v2PJSbSivC61RBNaKg diff --git a/taskcluster/test/params/me-ship-firefox.yml b/taskcluster/test/params/me-ship-firefox.yml index 062b3218f24f..f8bed8a843af 100644 --- a/taskcluster/test/params/me-ship-firefox.yml +++ b/taskcluster/test/params/me-ship-firefox.yml @@ -2219,7 +2219,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg docker-image-gdb-test: WdZGFUBiTzyn5er0FJQUvw - docker-image-github-sync: SmOuKw2vSA2P0B6MxH6xXw docker-image-image_builder: MWRymupRSheUkuqIQwF8TA docker-image-index-task: VIneOK4VRXGrcEmrEpleSA docker-image-lint: Eeq8v2PJSbSivC61RBNaKg diff --git a/taskcluster/test/params/mr-promote-firefox-rc.yml b/taskcluster/test/params/mr-promote-firefox-rc.yml index 0059b6b7aa11..c373c7947cc7 100644 --- a/taskcluster/test/params/mr-promote-firefox-rc.yml +++ b/taskcluster/test/params/mr-promote-firefox-rc.yml @@ -113,7 +113,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ diff --git a/taskcluster/test/params/mr-promote-firefox.yml b/taskcluster/test/params/mr-promote-firefox.yml index 0e132051ee03..356ca3536a2a 100644 --- a/taskcluster/test/params/mr-promote-firefox.yml +++ b/taskcluster/test/params/mr-promote-firefox.yml @@ -113,7 +113,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA - docker-image-github-sync: P_Ymo66MSamplL2DUkKafA docker-image-image_builder: MWRymupRSheUkuqIQwF8TA docker-image-index-task: VIneOK4VRXGrcEmrEpleSA docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ diff --git a/taskcluster/test/params/mr-push-firefox.yml b/taskcluster/test/params/mr-push-firefox.yml index de12bc2d85cd..fd4f5e5016ab 100644 --- a/taskcluster/test/params/mr-push-firefox.yml +++ b/taskcluster/test/params/mr-push-firefox.yml @@ -2358,7 +2358,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ diff --git a/taskcluster/test/params/mr-ship-firefox-rc.yml b/taskcluster/test/params/mr-ship-firefox-rc.yml index d2701afb4868..b7b7ca162e56 100644 --- a/taskcluster/test/params/mr-ship-firefox-rc.yml +++ b/taskcluster/test/params/mr-ship-firefox-rc.yml @@ -2358,7 +2358,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ diff --git a/taskcluster/test/params/mr-ship-firefox.yml b/taskcluster/test/params/mr-ship-firefox.yml index 552baee4b827..ecd616f800ea 100644 --- a/taskcluster/test/params/mr-ship-firefox.yml +++ b/taskcluster/test/params/mr-ship-firefox.yml @@ -2358,7 +2358,6 @@ existing_tasks: docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA - docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ docker-image-index-task: RwDns0jjRregWw0RNoOKeg docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ diff --git a/tools/github-sync/converter.py b/tools/github-sync/converter.py deleted file mode 100755 index 104229e29974..000000000000 --- a/tools/github-sync/converter.py +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env python3 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import re -import subprocess -import sys - -import hglib -import pygit2 - -DEBUG = False - - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def debugprint(*args, **kwargs): - if DEBUG: - eprint(*args, **kwargs) - - -class HgCommit: - def __init__(self, parent1, parent2): - self.parents = [] - if parent1 == NULL_PARENT_REV: - raise Exception( - "Encountered a hg changeset with no parents! We don't handle this...." - ) - self.parents.append(parent1) - if parent2 != NULL_PARENT_REV: - self.parents.append(parent2) - self.touches_sync_code = False - self.children = [] - - def add_child(self, rev): - self.children.append(rev) - - -class GitCommit: - def __init__(self, hg_rev, commit_obj): - self.hg_rev = hg_rev - self.commit_obj = commit_obj - - -def load_git_repository(): - commit_map = dict() - # First, scan the tags for "mozilla-xxx" that keep track of manually synchronized changes - sync_tags = filter( - lambda ref: ref.startswith("refs/tags/mozilla-"), - list(downstream_git_repo.references), - ) - for desc in sync_tags: - commit = downstream_git_repo.lookup_reference(desc).peel() - # cut out the revision hash from the output - hg_rev = desc[18:] - commit_map[hg_rev] = GitCommit(hg_rev, commit) - debugprint("Loaded pre-existing tag hg %s -> git %s" % (hg_rev, commit.oid)) - - # Next, scan the commits for a specific message format - re_commitmsg = re.compile( - r"^\[(ghsync|wrupdater)\] From https://hg.mozilla.org/mozilla-central/rev/([0-9a-fA-F]+)$", - re.MULTILINE, - ) - for commit in downstream_git_repo.walk(downstream_git_repo.head.target): - m = re_commitmsg.search(commit.message) - if not m: - continue - hg_rev = m.group(2) - commit_map[hg_rev] = GitCommit(hg_rev, commit) - debugprint("Loaded pre-existing commit hg %s -> git %s" % (hg_rev, commit.oid)) - return commit_map - - -def timeof(git_commit): - return git_commit.commit_obj.commit_time + git_commit.commit_obj.commit_time_offset - - -def find_newest_commit(commit_map): - newest_hg_rev = None - newest_commit_time = None - - for hg_rev, git_commit in commit_map.items(): - if newest_hg_rev is None or timeof(git_commit) > newest_commit_time: - newest_hg_rev = hg_rev - newest_commit_time = timeof(git_commit) - - return newest_hg_rev - - -def get_single_rev(revset): - output = subprocess.check_output( - ["hg", "log", "-r", revset, "--template", "{node}"] - ) - output = str(output, "ascii") - return output - - -def get_multiple_revs(revset, template): - output = subprocess.check_output( - ["hg", "log", "-r", revset, "--template", template + "\\n"] - ) - for line in output.splitlines(): - yield str(line, "ascii") - - -def get_base_hg_rev(commit_map): - base_hg_rev = find_newest_commit(commit_map) - eprint("Using %s as base hg revision" % base_hg_rev) - return base_hg_rev - - -def load_hg_commits(commits, query): - for cset in get_multiple_revs(query, "{node} {p1node} {p2node}"): - tokens = cset.split() - commits[tokens[0]] = HgCommit(tokens[1], tokens[2]) - return commits - - -def get_real_base_hg_rev(hg_data, commit_map): - # Some of the HG commits we want to port to github may have landed on codelines - # that branched off central prior to base_hg_rev. So when we create the git - # equivalents, they will have parents that are not the HEAD of the git repo, - # but instead will be descendants of older commits in the git repo. In order - # to do this correctly, we need to find the hg-equivalents of all of those - # possible git parents. So first we identify all the "tail" hg revisions in - # our hg_data set (think "tail" as in opposite of "head" which is the tipmost - # commit). The "tail" hg revisions are the ones for which we don't have their - # ancestors in hg_data. - tails = [] - for rev, cset in hg_data.items(): - for parent in cset.parents: - if parent not in hg_data: - tails.append(rev) - eprint("Found hg tail revisions %s" % tails) - # Then we find their common ancestor, which will be some ancestor of base_hg_rev - # from which those codelines. - if len(tails) == 0: - common_ancestor = get_single_rev(".") - else: - common_ancestor = get_single_rev("ancestor(" + ",".join(tails) + ")") - eprint("Found common ancestor of tail revisions: %s" % common_ancestor) - - # And then we find the newest git commit whose hg-equivalent is an ancestor of - # that common ancestor, to make sure we are starting from a known hg/git - # commit pair. - for git_commit in sorted(commit_map.values(), key=timeof, reverse=True): - new_base = get_single_rev( - "ancestor(" + common_ancestor + "," + git_commit.hg_rev + ")" - ) - if new_base == common_ancestor: - eprint( - "Pre-existing git commit %s from hg rev %s is descendant of common ancestor; %s" - % ( - git_commit.commit_obj.id, - git_commit.hg_rev, - "walking back further...", - ) - ) - continue - if new_base != git_commit.hg_rev: - eprint( - "Pre-existing git commit %s from hg rev %s is on sibling branch" - " of common ancestor; %s" - % ( - git_commit.commit_obj.id, - git_commit.hg_rev, - "walking back further...", - ) - ) - continue - eprint( - "Pre-existing git commit %s from hg rev %s is sufficiently old; stopping walk" - % (git_commit.commit_obj.id, git_commit.hg_rev) - ) - common_ancestor = new_base - break - - return common_ancestor - - -# Now we prune out all the uninteresting changesets from hg_commits. The -# uninteresting ones are ones that don't touch the target code, are not merges, -# and are not referenced by mozilla tags in the git repo. -# We do this by rewriting the parents to the "interesting" ancestor. -def prune_boring(rev): - while rev in hg_commits: - parent_pruned = False - for i in range(len(hg_commits[rev].parents)): - parent_rev = hg_commits[rev].parents[i] - if parent_rev not in hg_commits: - continue - if hg_commits[parent_rev].touches_sync_code: - continue - if len(hg_commits[parent_rev].parents) > 1: - continue - if parent_rev in hg_to_git_commit_map: - continue - - # If we get here, then `parent_rev` is a boring revision and we can - # prune it. Connect `rev` to its grandparent, and prune the parent - grandparent_rev = hg_commits[parent_rev].parents[0] - hg_commits[rev].parents[i] = grandparent_rev - # eprint("Pruned %s as boring parent of %s, using %s now" % - # (parent_rev, rev, grandparent_rev)) - parent_pruned = True - - if parent_pruned: - # If we pruned a parent, process `rev` again as we might want to - # prune more parents - continue - - # Collapse identical parents, because if the parents are identical - # we don't need to keep multiple copies of them. - hg_commits[rev].parents = list(dict.fromkeys(hg_commits[rev].parents)) - - # If we get here, all of `rev`s parents are interesting, so we can't - # prune them. Move up to the parent rev and start processing that, or - # if we have multiple parents then recurse on those nodes. - if len(hg_commits[rev].parents) == 1: - rev = hg_commits[rev].parents[0] - continue - - for parent_rev in hg_commits[rev].parents: - prune_boring(parent_rev) - return - - -class FakeCommit: - def __init__(self, oid): - self.oid = oid - - -def fake_commit(hg_rev, parent1, parent2): - if parent1 is None: - eprint("ERROR: Trying to build on None") - exit(1) - oid = "githash_%s" % hash(parent1) - eprint("Fake-built %s" % oid) - return FakeCommit(oid) - - -def build_tree(builder, treedata): - for name, value in treedata.items(): - if isinstance(value, dict): - subbuilder = downstream_git_repo.TreeBuilder() - build_tree(subbuilder, value) - builder.insert(name, subbuilder.write(), pygit2.GIT_FILEMODE_TREE) - else: - (filemode, contents) = value - blob_oid = downstream_git_repo.create_blob(contents) - builder.insert(name, blob_oid, filemode) - - -def author_to_signature(author): - pieces = author.strip().split("<") - if len(pieces) != 2 or pieces[1][-1] != ">": - # We could probably handle this better - return pygit2.Signature(author, "") - name = pieces[0].strip() - email = pieces[1][:-1].strip() - return pygit2.Signature(name, email) - - -def real_commit(hg_rev, parent1, parent2): - filetree = dict() - manifest = mozilla_hg_repo.manifest(rev=hg_rev) - for nodeid, permission, executable, symlink, filename in manifest: - if not filename.startswith(relative_path.encode("utf-8")): - continue - if symlink: - filemode = pygit2.GIT_FILEMODE_LINK - elif executable: - filemode = pygit2.GIT_FILEMODE_BLOB_EXECUTABLE - else: - filemode = pygit2.GIT_FILEMODE_BLOB - filecontent = mozilla_hg_repo.cat([filename], rev=hg_rev) - subtree = filetree - for component in filename.split(b"/")[2:-1]: - subtree = subtree.setdefault(component.decode("latin-1"), dict()) - filename = filename.split(b"/")[-1] - subtree[filename.decode("latin-1")] = (filemode, filecontent) - - builder = downstream_git_repo.TreeBuilder() - build_tree(builder, filetree) - tree_oid = builder.write() - - parent1_obj = downstream_git_repo.get(parent1) - if parent1_obj.tree_id == tree_oid: - eprint("Early-exit; tree matched that of parent git commit %s" % parent1) - return parent1_obj - - if parent2 is not None: - parent2_obj = downstream_git_repo.get(parent2) - if parent2_obj.tree_id == tree_oid: - eprint("Early-exit; tree matched that of parent git commit %s" % parent2) - return parent2_obj - - hg_rev_obj = mozilla_hg_repo.log(revrange=hg_rev, limit=1)[0] - commit_author = hg_rev_obj[4].decode("latin-1") - commit_message = hg_rev_obj[5].decode("latin-1") - commit_message += ( - "\n\n[ghsync] From https://hg.mozilla.org/mozilla-central/rev/%s" % hg_rev - + "\n" - ) - - parents = [parent1] - if parent2 is not None: - parents.append(parent2) - commit_oid = downstream_git_repo.create_commit( - None, - author_to_signature(commit_author), - author_to_signature(commit_author), - commit_message, - tree_oid, - parents, - ) - eprint("Built git commit %s" % commit_oid) - return downstream_git_repo.get(commit_oid) - - -def try_commit(hg_rev, parent1, parent2=None): - if False: - return fake_commit(hg_rev, parent1, parent2) - else: - return real_commit(hg_rev, parent1, parent2) - - -def build_git_commits(rev): - debugprint("build_git_commit(%s)..." % rev) - if rev in hg_to_git_commit_map: - debugprint(" maps to %s" % hg_to_git_commit_map[rev].commit_obj.oid) - return hg_to_git_commit_map[rev].commit_obj.oid - - if rev not in hg_commits: - debugprint(" not in hg_commits") - return None - - if len(hg_commits[rev].parents) == 1: - git_parent = build_git_commits(hg_commits[rev].parents[0]) - if not hg_commits[rev].touches_sync_code: - eprint( - "WARNING: Found rev %s that is non-merge and not related to the target" - % rev - ) - return git_parent - eprint("Building git equivalent for %s on top of %s" % (rev, git_parent)) - commit_obj = try_commit(rev, git_parent) - hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) - debugprint(" built %s as %s" % (rev, commit_obj.oid)) - return commit_obj.oid - - git_parent_1 = build_git_commits(hg_commits[rev].parents[0]) - git_parent_2 = build_git_commits(hg_commits[rev].parents[1]) - if git_parent_1 is None or git_parent_2 is None or git_parent_1 == git_parent_2: - git_parent = git_parent_1 if git_parent_2 is None else git_parent_2 - if not hg_commits[rev].touches_sync_code: - debugprint( - " %s is merge with no parents or doesn't touch WR, returning %s" - % (rev, git_parent) - ) - return git_parent - - eprint( - "WARNING: Found merge rev %s whose parents have identical target code" - ", but modifies the target" % rev - ) - eprint("Building git equivalent for %s on top of %s" % (rev, git_parent)) - commit_obj = try_commit(rev, git_parent) - hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) - debugprint(" built %s as %s" % (rev, commit_obj.oid)) - return commit_obj.oid - - # An actual merge - eprint( - "Building git equivalent for %s on top of %s, %s" - % (rev, git_parent_1, git_parent_2) - ) - commit_obj = try_commit(rev, git_parent_1, git_parent_2) - hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) - debugprint(" built %s as %s" % (rev, commit_obj.oid)) - return commit_obj.oid - - -def pretty_print(rev, cset): - desc = " %s" % rev - desc += " parents: %s" % cset.parents - if rev in hg_to_git_commit_map: - desc += " git: %s" % hg_to_git_commit_map[rev].commit_obj.oid - if rev == hg_tip: - desc += " (tip)" - return desc - - -if len(sys.argv) < 3: - eprint("Usage: %s " % sys.argv[0]) - eprint("Current dir must be the mozilla hg repo") - exit(1) - -local_checkout_path = sys.argv[1] -relative_path = sys.argv[2] -mozilla_hg_path = os.getcwd() -NULL_PARENT_REV = "0000000000000000000000000000000000000000" - -downstream_git_repo = pygit2.Repository(pygit2.discover_repository(local_checkout_path)) -mozilla_hg_repo = hglib.open(mozilla_hg_path) -hg_to_git_commit_map = load_git_repository() -base_hg_rev = get_base_hg_rev(hg_to_git_commit_map) -if base_hg_rev is None: - eprint("Found no sync commits or 'mozilla-xxx' tags") - exit(1) - -hg_commits = load_hg_commits(dict(), "only(.," + base_hg_rev + ")") -eprint("Initial set has %s changesets" % len(hg_commits)) -base_hg_rev = get_real_base_hg_rev(hg_commits, hg_to_git_commit_map) -eprint("Using hg rev %s as common ancestor of all interesting changesets" % base_hg_rev) - -# Refresh hg_commits with our wider dataset -hg_tip = get_single_rev(".") -wider_range = "%s::%s" % (base_hg_rev, hg_tip) -hg_commits = load_hg_commits(hg_commits, wider_range) -eprint("Updated set has %s changesets" % len(hg_commits)) - -if DEBUG: - eprint("Graph of descendants of %s" % base_hg_rev) - output = subprocess.check_output( - [ - "hg", - "log", - "--graph", - "-r", - "descendants(" + base_hg_rev + ")", - "--template", - "{node} {desc|firstline}\\n", - ] - ) - for line in output.splitlines(): - eprint(line.decode("utf-8", "ignore")) - -# Also flag any changes that touch the project -query = "(" + wider_range + ') & file("glob:' + relative_path + '/**")' -for cset in get_multiple_revs(query, "{node}"): - debugprint("Changeset %s modifies %s" % (cset, relative_path)) - hg_commits[cset].touches_sync_code = True -eprint( - "Identified %s changesets that touch the target code" - % sum([1 if v.touches_sync_code else 0 for (k, v) in hg_commits.items()]) -) - -prune_boring(hg_tip) - -# hg_tip itself might be boring -if not hg_commits[hg_tip].touches_sync_code and len(hg_commits[hg_tip].parents) == 1: - new_tip = hg_commits[hg_tip].parents[0] - eprint("Pruned tip %s as boring, using %s now" % (hg_tip, new_tip)) - hg_tip = new_tip - -eprint("--- Interesting changesets ---") -for rev, cset in hg_commits.items(): - if cset.touches_sync_code or len(cset.parents) > 1 or rev in hg_to_git_commit_map: - eprint(pretty_print(rev, cset)) -if DEBUG: - eprint("--- Other changesets (not really interesting) ---") - for rev, cset in hg_commits.items(): - if not ( - cset.touches_sync_code - or len(cset.parents) > 1 - or rev in hg_to_git_commit_map - ): - eprint(pretty_print(rev, cset)) - -git_tip = build_git_commits(hg_tip) -if git_tip is None: - eprint("No new changesets generated, exiting.") -else: - downstream_git_repo.create_reference("refs/heads/github-sync", git_tip, force=True) - eprint("Updated github-sync branch to %s, done!" % git_tip) diff --git a/tools/github-sync/read-json.py b/tools/github-sync/read-json.py deleted file mode 100755 index 87264d7df4df..000000000000 --- a/tools/github-sync/read-json.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import json -import sys - -j = json.load(sys.stdin) -components = sys.argv[1].split("/") - - -def next_match(json_fragment, components): - if len(components) == 0: - yield json_fragment - else: - component = components[0] - if type(json_fragment) == list: - if component == "*": - for item in json_fragment: - yield from next_match(item, components[1:]) - else: - component = int(component) - if component >= len(j): - sys.exit(1) - yield from next_match(json_fragment[component], components[1:]) - elif type(json_fragment) == dict: - if component == "*": - for key in sorted(json_fragment.keys()): - yield from next_match(json_fragment[key], components[1:]) - elif component not in json_fragment: - sys.exit(1) - else: - yield from next_match(json_fragment[component], components[1:]) - - -for match in list(next_match(j, components)): - if type(match) == dict: - print(" ".join(match.keys())) - else: - print(match) diff --git a/tools/github-sync/readme.md b/tools/github-sync/readme.md deleted file mode 100644 index d691071336b4..000000000000 --- a/tools/github-sync/readme.md +++ /dev/null @@ -1,106 +0,0 @@ -# Github synchronization scripts - -This tool aims to help synchronizing changes from mozilla-central to Github on pushes. -This is useful for Gecko sub-projects that have Github mirrors, like `gfx/wr` linking to `https://github.com/servo/webrender`. -Originally, the tools were developed in `https://github.com/staktrace/wrupdater`, -then got moved under `gfx/wr/ci-scripts/wrupdater`, -and finally migrated here while also abstracting away from WebRender specifically. - -The main entry point is the `sync-to-github.sh` script that is called with the following arguments: - 1. name of the project, matching the repository under `https://github.com/moz-gfx` user (e.g. `webrender`) - 2. relative folder in mozilla-central, which is the upstream for the changes (e.g. `gfx/wr`) - 3. downstream repository specified as "organization/project-name" (e.g. `servo/webrender`) - 4. name to call for auto-approving the pull request (e.g. `bors` or `@bors-servo`) - -It creates a staging directory at `~/.ghsync` if one doesn't already exist, -and clones the the downstream repo into it. -The script also requires the `GECKO_PATH` environment variable -to point to a mercurial clone of `mozilla-central`, and access to the -taskcluster secrets service to get a Github API token. - -The `sync-to-github.sh` script does some setup steps but the bulk of the actual work -is done by the `converter.py` script. This script scans the mercurial -repository for new changes to the relative folder in m-c, -and adds commits to the git repository corresponding to those changes. -There are some details in the implementation that make it more robust -than simply exporting patches and attempting to reapply them; -in particular it builds a commit tree structure that mirrors what is found in -the `mozilla-central` repository with respect to branches and merges. -So if conflicting changes land on autoland and inbound, and then get -merged, the git repository commits will have the same structure with -a fork/merge in the commit history. This was discovered to be -necessary after a previous version ran into multiple cases where -the simple patch approach didn't really work. - -One of the actions the `converter.py` takes is to find the last sync point -between Github and mozilla-central. This is done based on the following markers: - - commit message containing the string "[ghsync] From https://hg.mozilla.org/mozilla-central/rev/xxx" - - commit message containing the string "[wrupdater] From https://hg.mozilla.org/mozilla-central/rev/xxx" - - commit with tag "mozilla-xxx" -(where xxx is always a mozilla-central hg revision identifier). - -Once the converter is done converting, the `sync-to-github.sh` script -finishes the process by pushing the new commits to the `github-sync` branch -of the `https://github.com/moz-gfx/` repository, -and generating a pull request against the downstream repository. It also -leaves a comment on the PR that triggers testing and automatic merge of the PR. -If there is already a pull request (perhaps from a previous run) the -pre-existing PR is force-updated instead. This allows for graceful -handling of scenarios where the PR failed to get merged (e.g. due to -CI failures on the Github side). - -The script is intended to by run by taskcluster for any changes that -touch the relative folder that land on `mozilla-central`. This may mean -that multiple instances of this script run concurrently, or even out -of order (i.e. the task for an older m-c push runs after the task for -a newer m-c push). The script was written with these possibilities in -mind and should be able to eventually recover from any such scenario -automatically (although it may take additional changes to mozilla-central -for such recovery to occur). That being said, the number of pathological -scenarios here is quite large and they were not really tested. - -## Ownership and access - -When this tool is run in Firefox CI, it needs to have push permissions to -the `moz-gfx` github user's account. It gets this permission via a secret token -stored in the Firefox CI taskcluster secrets service. If you need to update -the token, you need to find somebody who is a member of the -[webrender-ci access group](https://people.mozilla.org/a/webrender-ci/). The -Google Drive associated with that access group has additional documentation -on the `moz-gfx` github user and the secret token. - -## Debugging - -To debug the converter.py script, you need to have a hg checkout of -mozilla-central, let's assume it's at $MOZILLA. First create a virtualenv -with the right dependencies installed: - -``` -mkdir -p $HOME/.ghsync -virtualenv --python=python3 $HOME/.ghsync/venv -source $HOME/.ghsync/venv/bin/activate -pip3 install -r $MOZILLA/taskcluster/docker/github-sync/requirements.txt -``` - -Also create a checkout of the downstream github repo and set up a `github-sync` -branch to the point where you want port commits to. For example, for WebRender -you'd do: - -``` -cd $HOME/.ghsync -git clone https://github.com/servo/webrender -cd webrender -git checkout -b github-sync master -``` - -(You can set the github-sync branch to a past revision if you want to replicate -a failure that already got committed). - -Then run the converter from your hg checkout: - -``` -cd $MOZILLA -tools/github-sync/converter.py $HOME/.ghsync/webrender gfx/wr -``` - -You can set the DEBUG variable in the script to True to get more output. diff --git a/tools/github-sync/sync-to-github.sh b/tools/github-sync/sync-to-github.sh deleted file mode 100755 index d67764974896..000000000000 --- a/tools/github-sync/sync-to-github.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -# Do NOT set -x here, since that will expose a secret API token! -set -o errexit -set -o nounset -set -o pipefail - -if [[ "$(uname)" != "Linux" ]]; then - echo "Error: this script must be run on Linux due to readlink semantics" - exit 1 -fi - -# GECKO_PATH should definitely be set -if [[ -z "${GECKO_PATH}" ]]; then - echo "Error: GECKO_PATH must point to a hg clone of mozilla-central" - exit 1 -fi - -# Internal variables, don't fiddle with these -MYSELF=$(readlink -f ${0}) -MYDIR=$(dirname "${MYSELF}") -WORKDIR="${HOME}/.ghsync" -TMPDIR="${WORKDIR}/tmp" - -NAME="$1" -RELATIVE_PATH="$2" -DOWNSTREAM_REPO="$3" -BORS="$4" -BRANCH="github-sync" - -mkdir -p "${TMPDIR}" - -# Bring the project clone to a known good up-to-date state -if [[ ! -d "${WORKDIR}/${NAME}" ]]; then - echo "Setting up ${NAME} repo..." - git clone "https://github.com/${DOWNSTREAM_REPO}" "${WORKDIR}/${NAME}" - pushd "${WORKDIR}/${NAME}" - git remote add moz-gfx https://github.com/moz-gfx/${NAME} - popd -else - echo "Updating ${NAME} repo..." - pushd "${WORKDIR}/${NAME}" - git checkout master - git pull - popd -fi - -if [[ -n "${GITHUB_SECRET:-}" ]]; then - echo "Obtaining github API token..." - # Be careful, GITHUB_TOKEN is secret, so don't log it (or any variables - # built using it). - GITHUB_TOKEN=$( - curl -sSfL "$TASKCLUSTER_PROXY_URL/secrets/v1/secret/${GITHUB_SECRET}" | - ${MYDIR}/read-json.py "secret/token" - ) - AUTH="moz-gfx:${GITHUB_TOKEN}" - CURL_AUTH="Authorization: bearer ${GITHUB_TOKEN}" -fi - -echo "Pushing base ${BRANCH} branch..." -pushd "${WORKDIR}/${NAME}" -git fetch moz-gfx -git checkout -B ${BRANCH} moz-gfx/${BRANCH} || git checkout -B ${BRANCH} master - -if [[ -n "${GITHUB_SECRET:-}" ]]; then - # git may emit error messages that contain the URL, so let's sanitize them - # or we might leak the auth token to the task log. - git push "https://${AUTH}@github.com/moz-gfx/${NAME}" \ - "${BRANCH}:${BRANCH}" 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g" - # Re-fetch to update the remote moz-gfx/$BRANCH branch in the local repo; - # normally the push does this but we use a fully-qualified URL for - # pushing so it doesn't happen. - git fetch moz-gfx -fi -popd - -# Run the converter -echo "Running converter..." -pushd "${GECKO_PATH}" -"${MYDIR}/converter.py" "${WORKDIR}/${NAME}" "${RELATIVE_PATH}" -popd - -# Check to see if we have changes that need pushing -echo "Checking for new changes..." -pushd "${WORKDIR}/${NAME}" -PATCHCOUNT=$(git log --oneline moz-gfx/${BRANCH}..${BRANCH}| wc -l) -if [[ ${PATCHCOUNT} -eq 0 ]]; then - echo "No new patches found, aborting..." - exit 0 -fi - -# Log the new changes, just for logging purposes -echo "Here are the new changes:" -git log --graph --stat moz-gfx/${BRANCH}..${BRANCH} - -# Collect PR numbers of PRs opened on Github and merged to m-c -set +e -FIXES=$( - git log master..${BRANCH} | - grep "\[import_pr\] From https://github.com/${DOWNSTREAM_REPO}/pull" | - sed -e "s%.*pull/% Fixes #%" | - uniq | - tr '\n' ',' -) -echo "${FIXES}" -set -e - -if [[ -z "${GITHUB_SECRET:-}" ]]; then - echo "Running in try push, exiting now" - exit 0 -fi - -echo "Pushing new changes to moz-gfx..." -# git may emit error messages that contain the URL, so let's sanitize them -# or we might leak the auth token to the task log. -git push "https://${AUTH}@github.com/moz-gfx/${NAME}" +${BRANCH}:${BRANCH} \ - 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g" - -CURL_HEADER="Accept: application/vnd.github.v3+json" -CURL=(curl -sSfL -H "${CURL_HEADER}" -H "${CURL_AUTH}") -# URL extracted here mostly to make servo-tidy happy with line lengths -API_URL="https://api.github.com/repos/${DOWNSTREAM_REPO}" - -# Check if there's an existing PR open -echo "Listing pre-existing pull requests..." -"${CURL[@]}" "${API_URL}/pulls?head=moz-gfx:${BRANCH}" | - tee "${TMPDIR}/pr.get" -set +e -COMMENT_URL=$(cat "${TMPDIR}/pr.get" | ${MYDIR}/read-json.py "0/comments_url") -HAS_COMMENT_URL="${?}" -set -e - -if [[ ${HAS_COMMENT_URL} -ne 0 ]]; then - echo "Pull request not found, creating..." - # The PR doesn't exist yet, so let's create it - ( echo -n '{ "title": "Sync changes from mozilla-central '"${RELATIVE_PATH}"'"' - echo -n ', "body": "'"${FIXES}"'"' - echo -n ', "head": "moz-gfx:'"${BRANCH}"'"' - echo -n ', "base": "master" }' - ) > "${TMPDIR}/pr.create" - "${CURL[@]}" -d "@${TMPDIR}/pr.create" "${API_URL}/pulls" | - tee "${TMPDIR}/pr.response" - COMMENT_URL=$( - cat "${TMPDIR}/pr.response" | - ${MYDIR}/read-json.py "comments_url" - ) -fi - -# At this point COMMENTS_URL should be set, so leave a comment to tell bors -# to merge the PR. -echo "Posting r+ comment to ${COMMENT_URL}..." -echo '{ "body": "'"$BORS"' r=auto" }' > "${TMPDIR}/bors_rplus" -"${CURL[@]}" -d "@${TMPDIR}/bors_rplus" "${COMMENT_URL}" - -echo "All done!" diff --git a/tools/moz.build b/tools/moz.build index bca499db618f..041c99880086 100644 --- a/tools/moz.build +++ b/tools/moz.build @@ -13,9 +13,6 @@ with Files("code-coverage/**"): with Files("compare-locales/mach_commands.py"): BUG_COMPONENT = ("Localization Infrastructure and Tools", "compare-locales") -with Files("github-sync/**"): - BUG_COMPONENT = ("Core", "Graphics") - with Files("lint/**"): BUG_COMPONENT = ("Developer Infrastructure", "Lint and Formatting")