forked from mirrors/gecko-dev
Bug 1885597 - Remove GitHub synchronization task r=jrmuizel,jnicol,taskgraph-reviewers,releng-reviewers,ahal,bhearsum
This synchronization is now handled in the downstream repositories. In the case of WebRender by the Servo project and in the case of qcms by a Gecko-managed GitHub Action. This change removes the github-sync task from mozilla-central. Differential Revision: https://phabricator.services.mozilla.com/D204787
This commit is contained in:
Martin Robinson
parent
712dfef63e
commit
1473c23cbf
38 changed files with 0 additions and 955 deletions
|
|
@ -1,7 +0,0 @@
|
|||
%include build/sparse-profiles/mach
|
||||
%include build/sparse-profiles/taskgraph
|
||||
|
||||
[include]
|
||||
path:gfx/wr/
|
||||
path:taskcluster/scripts/misc/
|
||||
path:tools/github-sync/
|
||||
|
|
@ -235,7 +235,6 @@ treeherder:
|
|||
'test-info': 'Test manifest skip/fail information'
|
||||
'condprof': 'Conditioned Profile Builder'
|
||||
'doc': 'Documentation'
|
||||
'GhS': 'GitHub Synchronization'
|
||||
'perftest': 'Performance tests'
|
||||
'perftest-detect': 'Performance regression detection tooling'
|
||||
'perftest-chrome': 'Performance tests using Chrone'
|
||||
|
|
|
|||
|
|
@ -214,9 +214,6 @@ jobs:
|
|||
condprof:
|
||||
symbol: I(condprof)
|
||||
parent: ubuntu1804-test
|
||||
github-sync:
|
||||
symbol: I(github-sync)
|
||||
parent: debian12-base
|
||||
sentry:
|
||||
symbol: I(sentry)
|
||||
parent: debian12-base
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: gecko_taskgraph.loader.transform:loader
|
||||
|
||||
kind-dependencies:
|
||||
- toolchain
|
||||
- build
|
||||
|
||||
transforms:
|
||||
- gecko_taskgraph.transforms.github_sync:transforms
|
||||
- gecko_taskgraph.transforms.job:transforms
|
||||
- gecko_taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
attributes:
|
||||
retrigger: true
|
||||
run-on-projects: ['mozilla-central-only']
|
||||
worker-type: b-linux-gcp
|
||||
treeherder:
|
||||
platform: other/opt
|
||||
tier: 2
|
||||
kind: other
|
||||
worker:
|
||||
max-run-time: 3600
|
||||
docker-image: {in-tree: github-sync}
|
||||
run:
|
||||
sparse-profile: github-sync
|
||||
using: run-task
|
||||
|
||||
jobs:
|
||||
webrender:
|
||||
description: Sync webrender to github mirror
|
||||
secret: gecko/gfx-github-sync/token
|
||||
run:
|
||||
command: '$GECKO_PATH/tools/github-sync/sync-to-github.sh webrender gfx/wr servo/webrender @bors-servo'
|
||||
when:
|
||||
files-changed:
|
||||
- 'gfx/wr/**'
|
||||
treeherder:
|
||||
symbol: GhS(wr)
|
||||
|
||||
qcms:
|
||||
description: Sync qcms to github mirror
|
||||
secret: gecko/gfx-github-sync/token
|
||||
run:
|
||||
command: '$GECKO_PATH/tools/github-sync/sync-to-github.sh qcms gfx/qcms FirefoxGraphics/qcms @jrmuizel'
|
||||
when:
|
||||
files-changed:
|
||||
- 'gfx/qcms/**'
|
||||
treeherder:
|
||||
symbol: GhS(qcms)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
FROM $DOCKER_IMAGE_PARENT
|
||||
MAINTAINER Kartikaya Gupta <kgupta@mozilla.com>
|
||||
|
||||
VOLUME /builds/worker/checkouts
|
||||
VOLUME /builds/worker/workspace
|
||||
VOLUME /builds/worker/tooltool-cache
|
||||
VOLUME /builds/worker/.ghsync
|
||||
|
||||
ADD prepare.sh /setup/prepare-docker.sh
|
||||
ADD requirements.txt /setup/requirements.txt
|
||||
RUN /bin/bash /setup/prepare-docker.sh /setup/requirements.txt && rm -R /setup
|
||||
|
||||
# Set a default command useful for debugging
|
||||
CMD ["/bin/bash", "--login"]
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
set -o xtrace
|
||||
|
||||
test "$(whoami)" == 'root'
|
||||
|
||||
# Install stuff we need
|
||||
apt-get -y update
|
||||
apt-get install -y \
|
||||
cmake \
|
||||
curl \
|
||||
gcc \
|
||||
git \
|
||||
g++ \
|
||||
libffi-dev \
|
||||
libgit2-dev \
|
||||
libssl-dev \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-setuptools
|
||||
|
||||
# Python packages
|
||||
pip3 install --break-system-packages -r "$1"
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
requests == 2.21.0
|
||||
pygit2 == 1.11.1
|
||||
python-hglib == 2.6.1
|
||||
|
|
@ -703,11 +703,6 @@ webrender
|
|||
Tasks used to do testing of WebRender standalone (without gecko). The
|
||||
WebRender code lives in gfx/wr and has its own testing infrastructure.
|
||||
|
||||
github-sync
|
||||
------------
|
||||
Tasks used to do synchronize parts of Gecko that have downstream GitHub
|
||||
repositories.
|
||||
|
||||
instrumented-build
|
||||
------------------
|
||||
Tasks that generate builds with PGO instrumentation enabled. This is an
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def sync_github(config, tasks):
|
||||
"""Do transforms specific to github-sync tasks."""
|
||||
for task in tasks:
|
||||
# Add the secret to the scopes, only in m-c.
|
||||
# Doing this on any other tree will result in decision task failure
|
||||
# because m-c is the only one allowed to have that scope.
|
||||
secret = task["secret"]
|
||||
if config.params["project"] == "mozilla-central":
|
||||
task.setdefault("scopes", [])
|
||||
task["scopes"].append("secrets:get:" + secret)
|
||||
task["worker"].setdefault("env", {})["GITHUB_SECRET"] = secret
|
||||
del task["secret"]
|
||||
yield task
|
||||
|
|
@ -160,7 +160,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -160,7 +160,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -160,7 +160,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -160,7 +160,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -160,7 +160,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -2405,7 +2405,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -2405,7 +2405,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -2405,7 +2405,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -2405,7 +2405,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -147,7 +147,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -147,7 +147,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -147,7 +147,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -147,7 +147,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: H2NDEiu8RJWciIqy0yu0IA
|
||||
|
|
|
|||
|
|
@ -206,7 +206,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: UGlRm6fZSJOQc1JcPC23lw
|
||||
|
|
|
|||
|
|
@ -206,7 +206,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: elwDDkImTPaIJXsdkPOgHA
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: Cfu6AIh4REm_lP4w2WJ34A
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: UGlRm6fZSJOQc1JcPC23lw
|
||||
|
|
|
|||
|
|
@ -345,7 +345,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: TaMBeARtSCKe485qQborMA
|
||||
docker-image-funsize-update-generator: RwrXtrceR_aEmcsEUxGiCg
|
||||
docker-image-gdb-test: ahyis7rkQTqhCapWXDqneg
|
||||
docker-image-github-sync: ds8yNkB5ScKKqLW2Cv0OWQ
|
||||
docker-image-image_builder: Z3gDOem4TSWtoayyvtW_ng
|
||||
docker-image-index-task: dbQOfI1hTp6sh4sGNnW9sQ
|
||||
docker-image-lint: GntDF25JRUaCu5nuedyceA
|
||||
|
|
|
|||
|
|
@ -97,7 +97,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg
|
||||
docker-image-gdb-test: WdZGFUBiTzyn5er0FJQUvw
|
||||
docker-image-github-sync: SmOuKw2vSA2P0B6MxH6xXw
|
||||
docker-image-image_builder: MWRymupRSheUkuqIQwF8TA
|
||||
docker-image-index-task: VIneOK4VRXGrcEmrEpleSA
|
||||
docker-image-lint: Eeq8v2PJSbSivC61RBNaKg
|
||||
|
|
|
|||
|
|
@ -2219,7 +2219,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg
|
||||
docker-image-gdb-test: WdZGFUBiTzyn5er0FJQUvw
|
||||
docker-image-github-sync: SmOuKw2vSA2P0B6MxH6xXw
|
||||
docker-image-image_builder: MWRymupRSheUkuqIQwF8TA
|
||||
docker-image-index-task: VIneOK4VRXGrcEmrEpleSA
|
||||
docker-image-lint: Eeq8v2PJSbSivC61RBNaKg
|
||||
|
|
|
|||
|
|
@ -2219,7 +2219,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg
|
||||
docker-image-gdb-test: WdZGFUBiTzyn5er0FJQUvw
|
||||
docker-image-github-sync: SmOuKw2vSA2P0B6MxH6xXw
|
||||
docker-image-image_builder: MWRymupRSheUkuqIQwF8TA
|
||||
docker-image-index-task: VIneOK4VRXGrcEmrEpleSA
|
||||
docker-image-lint: Eeq8v2PJSbSivC61RBNaKg
|
||||
|
|
|
|||
|
|
@ -113,7 +113,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ
|
||||
|
|
|
|||
|
|
@ -113,7 +113,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: H14MbupWQoyjX2KXVFC7Cg
|
||||
docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA
|
||||
docker-image-github-sync: P_Ymo66MSamplL2DUkKafA
|
||||
docker-image-image_builder: MWRymupRSheUkuqIQwF8TA
|
||||
docker-image-index-task: VIneOK4VRXGrcEmrEpleSA
|
||||
docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ
|
||||
|
|
|
|||
|
|
@ -2358,7 +2358,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ
|
||||
|
|
|
|||
|
|
@ -2358,7 +2358,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ
|
||||
|
|
|
|||
|
|
@ -2358,7 +2358,6 @@ existing_tasks:
|
|||
docker-image-firefox-snap: YmvNSMSfQWW3YG6x_Fcdbg
|
||||
docker-image-funsize-update-generator: YpyGQfXQQbGvpCIFyfWZ7A
|
||||
docker-image-gdb-test: SqgEZnNpSoqvNxaY-P2OCA
|
||||
docker-image-github-sync: GwZWXkYyQtGNRZnzVYuzCg
|
||||
docker-image-image_builder: OCRS9sC9SSeoQ96W91n8sQ
|
||||
docker-image-index-task: RwDns0jjRregWw0RNoOKeg
|
||||
docker-image-lint: PvzC5kHhQiGDcIo5GhW6TQ
|
||||
|
|
|
|||
|
|
@ -1,481 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import hglib
|
||||
import pygit2
|
||||
|
||||
DEBUG = False
|
||||
|
||||
|
||||
def eprint(*args, **kwargs):
|
||||
print(*args, file=sys.stderr, **kwargs)
|
||||
|
||||
|
||||
def debugprint(*args, **kwargs):
|
||||
if DEBUG:
|
||||
eprint(*args, **kwargs)
|
||||
|
||||
|
||||
class HgCommit:
|
||||
def __init__(self, parent1, parent2):
|
||||
self.parents = []
|
||||
if parent1 == NULL_PARENT_REV:
|
||||
raise Exception(
|
||||
"Encountered a hg changeset with no parents! We don't handle this...."
|
||||
)
|
||||
self.parents.append(parent1)
|
||||
if parent2 != NULL_PARENT_REV:
|
||||
self.parents.append(parent2)
|
||||
self.touches_sync_code = False
|
||||
self.children = []
|
||||
|
||||
def add_child(self, rev):
|
||||
self.children.append(rev)
|
||||
|
||||
|
||||
class GitCommit:
|
||||
def __init__(self, hg_rev, commit_obj):
|
||||
self.hg_rev = hg_rev
|
||||
self.commit_obj = commit_obj
|
||||
|
||||
|
||||
def load_git_repository():
|
||||
commit_map = dict()
|
||||
# First, scan the tags for "mozilla-xxx" that keep track of manually synchronized changes
|
||||
sync_tags = filter(
|
||||
lambda ref: ref.startswith("refs/tags/mozilla-"),
|
||||
list(downstream_git_repo.references),
|
||||
)
|
||||
for desc in sync_tags:
|
||||
commit = downstream_git_repo.lookup_reference(desc).peel()
|
||||
# cut out the revision hash from the output
|
||||
hg_rev = desc[18:]
|
||||
commit_map[hg_rev] = GitCommit(hg_rev, commit)
|
||||
debugprint("Loaded pre-existing tag hg %s -> git %s" % (hg_rev, commit.oid))
|
||||
|
||||
# Next, scan the commits for a specific message format
|
||||
re_commitmsg = re.compile(
|
||||
r"^\[(ghsync|wrupdater)\] From https://hg.mozilla.org/mozilla-central/rev/([0-9a-fA-F]+)$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
for commit in downstream_git_repo.walk(downstream_git_repo.head.target):
|
||||
m = re_commitmsg.search(commit.message)
|
||||
if not m:
|
||||
continue
|
||||
hg_rev = m.group(2)
|
||||
commit_map[hg_rev] = GitCommit(hg_rev, commit)
|
||||
debugprint("Loaded pre-existing commit hg %s -> git %s" % (hg_rev, commit.oid))
|
||||
return commit_map
|
||||
|
||||
|
||||
def timeof(git_commit):
|
||||
return git_commit.commit_obj.commit_time + git_commit.commit_obj.commit_time_offset
|
||||
|
||||
|
||||
def find_newest_commit(commit_map):
|
||||
newest_hg_rev = None
|
||||
newest_commit_time = None
|
||||
|
||||
for hg_rev, git_commit in commit_map.items():
|
||||
if newest_hg_rev is None or timeof(git_commit) > newest_commit_time:
|
||||
newest_hg_rev = hg_rev
|
||||
newest_commit_time = timeof(git_commit)
|
||||
|
||||
return newest_hg_rev
|
||||
|
||||
|
||||
def get_single_rev(revset):
|
||||
output = subprocess.check_output(
|
||||
["hg", "log", "-r", revset, "--template", "{node}"]
|
||||
)
|
||||
output = str(output, "ascii")
|
||||
return output
|
||||
|
||||
|
||||
def get_multiple_revs(revset, template):
|
||||
output = subprocess.check_output(
|
||||
["hg", "log", "-r", revset, "--template", template + "\\n"]
|
||||
)
|
||||
for line in output.splitlines():
|
||||
yield str(line, "ascii")
|
||||
|
||||
|
||||
def get_base_hg_rev(commit_map):
|
||||
base_hg_rev = find_newest_commit(commit_map)
|
||||
eprint("Using %s as base hg revision" % base_hg_rev)
|
||||
return base_hg_rev
|
||||
|
||||
|
||||
def load_hg_commits(commits, query):
|
||||
for cset in get_multiple_revs(query, "{node} {p1node} {p2node}"):
|
||||
tokens = cset.split()
|
||||
commits[tokens[0]] = HgCommit(tokens[1], tokens[2])
|
||||
return commits
|
||||
|
||||
|
||||
def get_real_base_hg_rev(hg_data, commit_map):
|
||||
# Some of the HG commits we want to port to github may have landed on codelines
|
||||
# that branched off central prior to base_hg_rev. So when we create the git
|
||||
# equivalents, they will have parents that are not the HEAD of the git repo,
|
||||
# but instead will be descendants of older commits in the git repo. In order
|
||||
# to do this correctly, we need to find the hg-equivalents of all of those
|
||||
# possible git parents. So first we identify all the "tail" hg revisions in
|
||||
# our hg_data set (think "tail" as in opposite of "head" which is the tipmost
|
||||
# commit). The "tail" hg revisions are the ones for which we don't have their
|
||||
# ancestors in hg_data.
|
||||
tails = []
|
||||
for rev, cset in hg_data.items():
|
||||
for parent in cset.parents:
|
||||
if parent not in hg_data:
|
||||
tails.append(rev)
|
||||
eprint("Found hg tail revisions %s" % tails)
|
||||
# Then we find their common ancestor, which will be some ancestor of base_hg_rev
|
||||
# from which those codelines.
|
||||
if len(tails) == 0:
|
||||
common_ancestor = get_single_rev(".")
|
||||
else:
|
||||
common_ancestor = get_single_rev("ancestor(" + ",".join(tails) + ")")
|
||||
eprint("Found common ancestor of tail revisions: %s" % common_ancestor)
|
||||
|
||||
# And then we find the newest git commit whose hg-equivalent is an ancestor of
|
||||
# that common ancestor, to make sure we are starting from a known hg/git
|
||||
# commit pair.
|
||||
for git_commit in sorted(commit_map.values(), key=timeof, reverse=True):
|
||||
new_base = get_single_rev(
|
||||
"ancestor(" + common_ancestor + "," + git_commit.hg_rev + ")"
|
||||
)
|
||||
if new_base == common_ancestor:
|
||||
eprint(
|
||||
"Pre-existing git commit %s from hg rev %s is descendant of common ancestor; %s"
|
||||
% (
|
||||
git_commit.commit_obj.id,
|
||||
git_commit.hg_rev,
|
||||
"walking back further...",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if new_base != git_commit.hg_rev:
|
||||
eprint(
|
||||
"Pre-existing git commit %s from hg rev %s is on sibling branch"
|
||||
" of common ancestor; %s"
|
||||
% (
|
||||
git_commit.commit_obj.id,
|
||||
git_commit.hg_rev,
|
||||
"walking back further...",
|
||||
)
|
||||
)
|
||||
continue
|
||||
eprint(
|
||||
"Pre-existing git commit %s from hg rev %s is sufficiently old; stopping walk"
|
||||
% (git_commit.commit_obj.id, git_commit.hg_rev)
|
||||
)
|
||||
common_ancestor = new_base
|
||||
break
|
||||
|
||||
return common_ancestor
|
||||
|
||||
|
||||
# Now we prune out all the uninteresting changesets from hg_commits. The
|
||||
# uninteresting ones are ones that don't touch the target code, are not merges,
|
||||
# and are not referenced by mozilla tags in the git repo.
|
||||
# We do this by rewriting the parents to the "interesting" ancestor.
|
||||
def prune_boring(rev):
|
||||
while rev in hg_commits:
|
||||
parent_pruned = False
|
||||
for i in range(len(hg_commits[rev].parents)):
|
||||
parent_rev = hg_commits[rev].parents[i]
|
||||
if parent_rev not in hg_commits:
|
||||
continue
|
||||
if hg_commits[parent_rev].touches_sync_code:
|
||||
continue
|
||||
if len(hg_commits[parent_rev].parents) > 1:
|
||||
continue
|
||||
if parent_rev in hg_to_git_commit_map:
|
||||
continue
|
||||
|
||||
# If we get here, then `parent_rev` is a boring revision and we can
|
||||
# prune it. Connect `rev` to its grandparent, and prune the parent
|
||||
grandparent_rev = hg_commits[parent_rev].parents[0]
|
||||
hg_commits[rev].parents[i] = grandparent_rev
|
||||
# eprint("Pruned %s as boring parent of %s, using %s now" %
|
||||
# (parent_rev, rev, grandparent_rev))
|
||||
parent_pruned = True
|
||||
|
||||
if parent_pruned:
|
||||
# If we pruned a parent, process `rev` again as we might want to
|
||||
# prune more parents
|
||||
continue
|
||||
|
||||
# Collapse identical parents, because if the parents are identical
|
||||
# we don't need to keep multiple copies of them.
|
||||
hg_commits[rev].parents = list(dict.fromkeys(hg_commits[rev].parents))
|
||||
|
||||
# If we get here, all of `rev`s parents are interesting, so we can't
|
||||
# prune them. Move up to the parent rev and start processing that, or
|
||||
# if we have multiple parents then recurse on those nodes.
|
||||
if len(hg_commits[rev].parents) == 1:
|
||||
rev = hg_commits[rev].parents[0]
|
||||
continue
|
||||
|
||||
for parent_rev in hg_commits[rev].parents:
|
||||
prune_boring(parent_rev)
|
||||
return
|
||||
|
||||
|
||||
class FakeCommit:
|
||||
def __init__(self, oid):
|
||||
self.oid = oid
|
||||
|
||||
|
||||
def fake_commit(hg_rev, parent1, parent2):
|
||||
if parent1 is None:
|
||||
eprint("ERROR: Trying to build on None")
|
||||
exit(1)
|
||||
oid = "githash_%s" % hash(parent1)
|
||||
eprint("Fake-built %s" % oid)
|
||||
return FakeCommit(oid)
|
||||
|
||||
|
||||
def build_tree(builder, treedata):
|
||||
for name, value in treedata.items():
|
||||
if isinstance(value, dict):
|
||||
subbuilder = downstream_git_repo.TreeBuilder()
|
||||
build_tree(subbuilder, value)
|
||||
builder.insert(name, subbuilder.write(), pygit2.GIT_FILEMODE_TREE)
|
||||
else:
|
||||
(filemode, contents) = value
|
||||
blob_oid = downstream_git_repo.create_blob(contents)
|
||||
builder.insert(name, blob_oid, filemode)
|
||||
|
||||
|
||||
def author_to_signature(author):
|
||||
pieces = author.strip().split("<")
|
||||
if len(pieces) != 2 or pieces[1][-1] != ">":
|
||||
# We could probably handle this better
|
||||
return pygit2.Signature(author, "")
|
||||
name = pieces[0].strip()
|
||||
email = pieces[1][:-1].strip()
|
||||
return pygit2.Signature(name, email)
|
||||
|
||||
|
||||
def real_commit(hg_rev, parent1, parent2):
|
||||
filetree = dict()
|
||||
manifest = mozilla_hg_repo.manifest(rev=hg_rev)
|
||||
for nodeid, permission, executable, symlink, filename in manifest:
|
||||
if not filename.startswith(relative_path.encode("utf-8")):
|
||||
continue
|
||||
if symlink:
|
||||
filemode = pygit2.GIT_FILEMODE_LINK
|
||||
elif executable:
|
||||
filemode = pygit2.GIT_FILEMODE_BLOB_EXECUTABLE
|
||||
else:
|
||||
filemode = pygit2.GIT_FILEMODE_BLOB
|
||||
filecontent = mozilla_hg_repo.cat([filename], rev=hg_rev)
|
||||
subtree = filetree
|
||||
for component in filename.split(b"/")[2:-1]:
|
||||
subtree = subtree.setdefault(component.decode("latin-1"), dict())
|
||||
filename = filename.split(b"/")[-1]
|
||||
subtree[filename.decode("latin-1")] = (filemode, filecontent)
|
||||
|
||||
builder = downstream_git_repo.TreeBuilder()
|
||||
build_tree(builder, filetree)
|
||||
tree_oid = builder.write()
|
||||
|
||||
parent1_obj = downstream_git_repo.get(parent1)
|
||||
if parent1_obj.tree_id == tree_oid:
|
||||
eprint("Early-exit; tree matched that of parent git commit %s" % parent1)
|
||||
return parent1_obj
|
||||
|
||||
if parent2 is not None:
|
||||
parent2_obj = downstream_git_repo.get(parent2)
|
||||
if parent2_obj.tree_id == tree_oid:
|
||||
eprint("Early-exit; tree matched that of parent git commit %s" % parent2)
|
||||
return parent2_obj
|
||||
|
||||
hg_rev_obj = mozilla_hg_repo.log(revrange=hg_rev, limit=1)[0]
|
||||
commit_author = hg_rev_obj[4].decode("latin-1")
|
||||
commit_message = hg_rev_obj[5].decode("latin-1")
|
||||
commit_message += (
|
||||
"\n\n[ghsync] From https://hg.mozilla.org/mozilla-central/rev/%s" % hg_rev
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
parents = [parent1]
|
||||
if parent2 is not None:
|
||||
parents.append(parent2)
|
||||
commit_oid = downstream_git_repo.create_commit(
|
||||
None,
|
||||
author_to_signature(commit_author),
|
||||
author_to_signature(commit_author),
|
||||
commit_message,
|
||||
tree_oid,
|
||||
parents,
|
||||
)
|
||||
eprint("Built git commit %s" % commit_oid)
|
||||
return downstream_git_repo.get(commit_oid)
|
||||
|
||||
|
||||
def try_commit(hg_rev, parent1, parent2=None):
|
||||
if False:
|
||||
return fake_commit(hg_rev, parent1, parent2)
|
||||
else:
|
||||
return real_commit(hg_rev, parent1, parent2)
|
||||
|
||||
|
||||
def build_git_commits(rev):
|
||||
debugprint("build_git_commit(%s)..." % rev)
|
||||
if rev in hg_to_git_commit_map:
|
||||
debugprint(" maps to %s" % hg_to_git_commit_map[rev].commit_obj.oid)
|
||||
return hg_to_git_commit_map[rev].commit_obj.oid
|
||||
|
||||
if rev not in hg_commits:
|
||||
debugprint(" not in hg_commits")
|
||||
return None
|
||||
|
||||
if len(hg_commits[rev].parents) == 1:
|
||||
git_parent = build_git_commits(hg_commits[rev].parents[0])
|
||||
if not hg_commits[rev].touches_sync_code:
|
||||
eprint(
|
||||
"WARNING: Found rev %s that is non-merge and not related to the target"
|
||||
% rev
|
||||
)
|
||||
return git_parent
|
||||
eprint("Building git equivalent for %s on top of %s" % (rev, git_parent))
|
||||
commit_obj = try_commit(rev, git_parent)
|
||||
hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj)
|
||||
debugprint(" built %s as %s" % (rev, commit_obj.oid))
|
||||
return commit_obj.oid
|
||||
|
||||
git_parent_1 = build_git_commits(hg_commits[rev].parents[0])
|
||||
git_parent_2 = build_git_commits(hg_commits[rev].parents[1])
|
||||
if git_parent_1 is None or git_parent_2 is None or git_parent_1 == git_parent_2:
|
||||
git_parent = git_parent_1 if git_parent_2 is None else git_parent_2
|
||||
if not hg_commits[rev].touches_sync_code:
|
||||
debugprint(
|
||||
" %s is merge with no parents or doesn't touch WR, returning %s"
|
||||
% (rev, git_parent)
|
||||
)
|
||||
return git_parent
|
||||
|
||||
eprint(
|
||||
"WARNING: Found merge rev %s whose parents have identical target code"
|
||||
", but modifies the target" % rev
|
||||
)
|
||||
eprint("Building git equivalent for %s on top of %s" % (rev, git_parent))
|
||||
commit_obj = try_commit(rev, git_parent)
|
||||
hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj)
|
||||
debugprint(" built %s as %s" % (rev, commit_obj.oid))
|
||||
return commit_obj.oid
|
||||
|
||||
# An actual merge
|
||||
eprint(
|
||||
"Building git equivalent for %s on top of %s, %s"
|
||||
% (rev, git_parent_1, git_parent_2)
|
||||
)
|
||||
commit_obj = try_commit(rev, git_parent_1, git_parent_2)
|
||||
hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj)
|
||||
debugprint(" built %s as %s" % (rev, commit_obj.oid))
|
||||
return commit_obj.oid
|
||||
|
||||
|
||||
def pretty_print(rev, cset):
|
||||
desc = " %s" % rev
|
||||
desc += " parents: %s" % cset.parents
|
||||
if rev in hg_to_git_commit_map:
|
||||
desc += " git: %s" % hg_to_git_commit_map[rev].commit_obj.oid
|
||||
if rev == hg_tip:
|
||||
desc += " (tip)"
|
||||
return desc
|
||||
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
eprint("Usage: %s <local-checkout-path> <repo-relative-path>" % sys.argv[0])
|
||||
eprint("Current dir must be the mozilla hg repo")
|
||||
exit(1)
|
||||
|
||||
local_checkout_path = sys.argv[1]
|
||||
relative_path = sys.argv[2]
|
||||
mozilla_hg_path = os.getcwd()
|
||||
NULL_PARENT_REV = "0000000000000000000000000000000000000000"
|
||||
|
||||
downstream_git_repo = pygit2.Repository(pygit2.discover_repository(local_checkout_path))
|
||||
mozilla_hg_repo = hglib.open(mozilla_hg_path)
|
||||
hg_to_git_commit_map = load_git_repository()
|
||||
base_hg_rev = get_base_hg_rev(hg_to_git_commit_map)
|
||||
if base_hg_rev is None:
|
||||
eprint("Found no sync commits or 'mozilla-xxx' tags")
|
||||
exit(1)
|
||||
|
||||
hg_commits = load_hg_commits(dict(), "only(.," + base_hg_rev + ")")
|
||||
eprint("Initial set has %s changesets" % len(hg_commits))
|
||||
base_hg_rev = get_real_base_hg_rev(hg_commits, hg_to_git_commit_map)
|
||||
eprint("Using hg rev %s as common ancestor of all interesting changesets" % base_hg_rev)
|
||||
|
||||
# Refresh hg_commits with our wider dataset
|
||||
hg_tip = get_single_rev(".")
|
||||
wider_range = "%s::%s" % (base_hg_rev, hg_tip)
|
||||
hg_commits = load_hg_commits(hg_commits, wider_range)
|
||||
eprint("Updated set has %s changesets" % len(hg_commits))
|
||||
|
||||
if DEBUG:
|
||||
eprint("Graph of descendants of %s" % base_hg_rev)
|
||||
output = subprocess.check_output(
|
||||
[
|
||||
"hg",
|
||||
"log",
|
||||
"--graph",
|
||||
"-r",
|
||||
"descendants(" + base_hg_rev + ")",
|
||||
"--template",
|
||||
"{node} {desc|firstline}\\n",
|
||||
]
|
||||
)
|
||||
for line in output.splitlines():
|
||||
eprint(line.decode("utf-8", "ignore"))
|
||||
|
||||
# Also flag any changes that touch the project
|
||||
query = "(" + wider_range + ') & file("glob:' + relative_path + '/**")'
|
||||
for cset in get_multiple_revs(query, "{node}"):
|
||||
debugprint("Changeset %s modifies %s" % (cset, relative_path))
|
||||
hg_commits[cset].touches_sync_code = True
|
||||
eprint(
|
||||
"Identified %s changesets that touch the target code"
|
||||
% sum([1 if v.touches_sync_code else 0 for (k, v) in hg_commits.items()])
|
||||
)
|
||||
|
||||
prune_boring(hg_tip)
|
||||
|
||||
# hg_tip itself might be boring
|
||||
if not hg_commits[hg_tip].touches_sync_code and len(hg_commits[hg_tip].parents) == 1:
|
||||
new_tip = hg_commits[hg_tip].parents[0]
|
||||
eprint("Pruned tip %s as boring, using %s now" % (hg_tip, new_tip))
|
||||
hg_tip = new_tip
|
||||
|
||||
eprint("--- Interesting changesets ---")
|
||||
for rev, cset in hg_commits.items():
|
||||
if cset.touches_sync_code or len(cset.parents) > 1 or rev in hg_to_git_commit_map:
|
||||
eprint(pretty_print(rev, cset))
|
||||
if DEBUG:
|
||||
eprint("--- Other changesets (not really interesting) ---")
|
||||
for rev, cset in hg_commits.items():
|
||||
if not (
|
||||
cset.touches_sync_code
|
||||
or len(cset.parents) > 1
|
||||
or rev in hg_to_git_commit_map
|
||||
):
|
||||
eprint(pretty_print(rev, cset))
|
||||
|
||||
git_tip = build_git_commits(hg_tip)
|
||||
if git_tip is None:
|
||||
eprint("No new changesets generated, exiting.")
|
||||
else:
|
||||
downstream_git_repo.create_reference("refs/heads/github-sync", git_tip, force=True)
|
||||
eprint("Updated github-sync branch to %s, done!" % git_tip)
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
j = json.load(sys.stdin)
|
||||
components = sys.argv[1].split("/")
|
||||
|
||||
|
||||
def next_match(json_fragment, components):
|
||||
if len(components) == 0:
|
||||
yield json_fragment
|
||||
else:
|
||||
component = components[0]
|
||||
if type(json_fragment) == list:
|
||||
if component == "*":
|
||||
for item in json_fragment:
|
||||
yield from next_match(item, components[1:])
|
||||
else:
|
||||
component = int(component)
|
||||
if component >= len(j):
|
||||
sys.exit(1)
|
||||
yield from next_match(json_fragment[component], components[1:])
|
||||
elif type(json_fragment) == dict:
|
||||
if component == "*":
|
||||
for key in sorted(json_fragment.keys()):
|
||||
yield from next_match(json_fragment[key], components[1:])
|
||||
elif component not in json_fragment:
|
||||
sys.exit(1)
|
||||
else:
|
||||
yield from next_match(json_fragment[component], components[1:])
|
||||
|
||||
|
||||
for match in list(next_match(j, components)):
|
||||
if type(match) == dict:
|
||||
print(" ".join(match.keys()))
|
||||
else:
|
||||
print(match)
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
# Github synchronization scripts
|
||||
|
||||
This tool aims to help synchronizing changes from mozilla-central to Github on pushes.
|
||||
This is useful for Gecko sub-projects that have Github mirrors, like `gfx/wr` linking to `https://github.com/servo/webrender`.
|
||||
Originally, the tools were developed in `https://github.com/staktrace/wrupdater`,
|
||||
then got moved under `gfx/wr/ci-scripts/wrupdater`,
|
||||
and finally migrated here while also abstracting away from WebRender specifically.
|
||||
|
||||
The main entry point is the `sync-to-github.sh` script that is called with the following arguments:
|
||||
1. name of the project, matching the repository under `https://github.com/moz-gfx` user (e.g. `webrender`)
|
||||
2. relative folder in mozilla-central, which is the upstream for the changes (e.g. `gfx/wr`)
|
||||
3. downstream repository specified as "organization/project-name" (e.g. `servo/webrender`)
|
||||
4. name to call for auto-approving the pull request (e.g. `bors` or `@bors-servo`)
|
||||
|
||||
It creates a staging directory at `~/.ghsync` if one doesn't already exist,
|
||||
and clones the the downstream repo into it.
|
||||
The script also requires the `GECKO_PATH` environment variable
|
||||
to point to a mercurial clone of `mozilla-central`, and access to the
|
||||
taskcluster secrets service to get a Github API token.
|
||||
|
||||
The `sync-to-github.sh` script does some setup steps but the bulk of the actual work
|
||||
is done by the `converter.py` script. This script scans the mercurial
|
||||
repository for new changes to the relative folder in m-c,
|
||||
and adds commits to the git repository corresponding to those changes.
|
||||
There are some details in the implementation that make it more robust
|
||||
than simply exporting patches and attempting to reapply them;
|
||||
in particular it builds a commit tree structure that mirrors what is found in
|
||||
the `mozilla-central` repository with respect to branches and merges.
|
||||
So if conflicting changes land on autoland and inbound, and then get
|
||||
merged, the git repository commits will have the same structure with
|
||||
a fork/merge in the commit history. This was discovered to be
|
||||
necessary after a previous version ran into multiple cases where
|
||||
the simple patch approach didn't really work.
|
||||
|
||||
One of the actions the `converter.py` takes is to find the last sync point
|
||||
between Github and mozilla-central. This is done based on the following markers:
|
||||
- commit message containing the string "[ghsync] From https://hg.mozilla.org/mozilla-central/rev/xxx"
|
||||
- commit message containing the string "[wrupdater] From https://hg.mozilla.org/mozilla-central/rev/xxx"
|
||||
- commit with tag "mozilla-xxx"
|
||||
(where xxx is always a mozilla-central hg revision identifier).
|
||||
|
||||
Once the converter is done converting, the `sync-to-github.sh` script
|
||||
finishes the process by pushing the new commits to the `github-sync` branch
|
||||
of the `https://github.com/moz-gfx/<project-name>` repository,
|
||||
and generating a pull request against the downstream repository. It also
|
||||
leaves a comment on the PR that triggers testing and automatic merge of the PR.
|
||||
If there is already a pull request (perhaps from a previous run) the
|
||||
pre-existing PR is force-updated instead. This allows for graceful
|
||||
handling of scenarios where the PR failed to get merged (e.g. due to
|
||||
CI failures on the Github side).
|
||||
|
||||
The script is intended to by run by taskcluster for any changes that
|
||||
touch the relative folder that land on `mozilla-central`. This may mean
|
||||
that multiple instances of this script run concurrently, or even out
|
||||
of order (i.e. the task for an older m-c push runs after the task for
|
||||
a newer m-c push). The script was written with these possibilities in
|
||||
mind and should be able to eventually recover from any such scenario
|
||||
automatically (although it may take additional changes to mozilla-central
|
||||
for such recovery to occur). That being said, the number of pathological
|
||||
scenarios here is quite large and they were not really tested.
|
||||
|
||||
## Ownership and access
|
||||
|
||||
When this tool is run in Firefox CI, it needs to have push permissions to
|
||||
the `moz-gfx` github user's account. It gets this permission via a secret token
|
||||
stored in the Firefox CI taskcluster secrets service. If you need to update
|
||||
the token, you need to find somebody who is a member of the
|
||||
[webrender-ci access group](https://people.mozilla.org/a/webrender-ci/). The
|
||||
Google Drive associated with that access group has additional documentation
|
||||
on the `moz-gfx` github user and the secret token.
|
||||
|
||||
## Debugging
|
||||
|
||||
To debug the converter.py script, you need to have a hg checkout of
|
||||
mozilla-central, let's assume it's at $MOZILLA. First create a virtualenv
|
||||
with the right dependencies installed:
|
||||
|
||||
```
|
||||
mkdir -p $HOME/.ghsync
|
||||
virtualenv --python=python3 $HOME/.ghsync/venv
|
||||
source $HOME/.ghsync/venv/bin/activate
|
||||
pip3 install -r $MOZILLA/taskcluster/docker/github-sync/requirements.txt
|
||||
```
|
||||
|
||||
Also create a checkout of the downstream github repo and set up a `github-sync`
|
||||
branch to the point where you want port commits to. For example, for WebRender
|
||||
you'd do:
|
||||
|
||||
```
|
||||
cd $HOME/.ghsync
|
||||
git clone https://github.com/servo/webrender
|
||||
cd webrender
|
||||
git checkout -b github-sync master
|
||||
```
|
||||
|
||||
(You can set the github-sync branch to a past revision if you want to replicate
|
||||
a failure that already got committed).
|
||||
|
||||
Then run the converter from your hg checkout:
|
||||
|
||||
```
|
||||
cd $MOZILLA
|
||||
tools/github-sync/converter.py $HOME/.ghsync/webrender gfx/wr
|
||||
```
|
||||
|
||||
You can set the DEBUG variable in the script to True to get more output.
|
||||
|
|
@ -1,159 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
# Do NOT set -x here, since that will expose a secret API token!
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
if [[ "$(uname)" != "Linux" ]]; then
|
||||
echo "Error: this script must be run on Linux due to readlink semantics"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# GECKO_PATH should definitely be set
|
||||
if [[ -z "${GECKO_PATH}" ]]; then
|
||||
echo "Error: GECKO_PATH must point to a hg clone of mozilla-central"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Internal variables, don't fiddle with these
|
||||
MYSELF=$(readlink -f ${0})
|
||||
MYDIR=$(dirname "${MYSELF}")
|
||||
WORKDIR="${HOME}/.ghsync"
|
||||
TMPDIR="${WORKDIR}/tmp"
|
||||
|
||||
NAME="$1"
|
||||
RELATIVE_PATH="$2"
|
||||
DOWNSTREAM_REPO="$3"
|
||||
BORS="$4"
|
||||
BRANCH="github-sync"
|
||||
|
||||
mkdir -p "${TMPDIR}"
|
||||
|
||||
# Bring the project clone to a known good up-to-date state
|
||||
if [[ ! -d "${WORKDIR}/${NAME}" ]]; then
|
||||
echo "Setting up ${NAME} repo..."
|
||||
git clone "https://github.com/${DOWNSTREAM_REPO}" "${WORKDIR}/${NAME}"
|
||||
pushd "${WORKDIR}/${NAME}"
|
||||
git remote add moz-gfx https://github.com/moz-gfx/${NAME}
|
||||
popd
|
||||
else
|
||||
echo "Updating ${NAME} repo..."
|
||||
pushd "${WORKDIR}/${NAME}"
|
||||
git checkout master
|
||||
git pull
|
||||
popd
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_SECRET:-}" ]]; then
|
||||
echo "Obtaining github API token..."
|
||||
# Be careful, GITHUB_TOKEN is secret, so don't log it (or any variables
|
||||
# built using it).
|
||||
GITHUB_TOKEN=$(
|
||||
curl -sSfL "$TASKCLUSTER_PROXY_URL/secrets/v1/secret/${GITHUB_SECRET}" |
|
||||
${MYDIR}/read-json.py "secret/token"
|
||||
)
|
||||
AUTH="moz-gfx:${GITHUB_TOKEN}"
|
||||
CURL_AUTH="Authorization: bearer ${GITHUB_TOKEN}"
|
||||
fi
|
||||
|
||||
echo "Pushing base ${BRANCH} branch..."
|
||||
pushd "${WORKDIR}/${NAME}"
|
||||
git fetch moz-gfx
|
||||
git checkout -B ${BRANCH} moz-gfx/${BRANCH} || git checkout -B ${BRANCH} master
|
||||
|
||||
if [[ -n "${GITHUB_SECRET:-}" ]]; then
|
||||
# git may emit error messages that contain the URL, so let's sanitize them
|
||||
# or we might leak the auth token to the task log.
|
||||
git push "https://${AUTH}@github.com/moz-gfx/${NAME}" \
|
||||
"${BRANCH}:${BRANCH}" 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g"
|
||||
# Re-fetch to update the remote moz-gfx/$BRANCH branch in the local repo;
|
||||
# normally the push does this but we use a fully-qualified URL for
|
||||
# pushing so it doesn't happen.
|
||||
git fetch moz-gfx
|
||||
fi
|
||||
popd
|
||||
|
||||
# Run the converter
|
||||
echo "Running converter..."
|
||||
pushd "${GECKO_PATH}"
|
||||
"${MYDIR}/converter.py" "${WORKDIR}/${NAME}" "${RELATIVE_PATH}"
|
||||
popd
|
||||
|
||||
# Check to see if we have changes that need pushing
|
||||
echo "Checking for new changes..."
|
||||
pushd "${WORKDIR}/${NAME}"
|
||||
PATCHCOUNT=$(git log --oneline moz-gfx/${BRANCH}..${BRANCH}| wc -l)
|
||||
if [[ ${PATCHCOUNT} -eq 0 ]]; then
|
||||
echo "No new patches found, aborting..."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Log the new changes, just for logging purposes
|
||||
echo "Here are the new changes:"
|
||||
git log --graph --stat moz-gfx/${BRANCH}..${BRANCH}
|
||||
|
||||
# Collect PR numbers of PRs opened on Github and merged to m-c
|
||||
set +e
|
||||
FIXES=$(
|
||||
git log master..${BRANCH} |
|
||||
grep "\[import_pr\] From https://github.com/${DOWNSTREAM_REPO}/pull" |
|
||||
sed -e "s%.*pull/% Fixes #%" |
|
||||
uniq |
|
||||
tr '\n' ','
|
||||
)
|
||||
echo "${FIXES}"
|
||||
set -e
|
||||
|
||||
if [[ -z "${GITHUB_SECRET:-}" ]]; then
|
||||
echo "Running in try push, exiting now"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Pushing new changes to moz-gfx..."
|
||||
# git may emit error messages that contain the URL, so let's sanitize them
|
||||
# or we might leak the auth token to the task log.
|
||||
git push "https://${AUTH}@github.com/moz-gfx/${NAME}" +${BRANCH}:${BRANCH} \
|
||||
2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g"
|
||||
|
||||
CURL_HEADER="Accept: application/vnd.github.v3+json"
|
||||
CURL=(curl -sSfL -H "${CURL_HEADER}" -H "${CURL_AUTH}")
|
||||
# URL extracted here mostly to make servo-tidy happy with line lengths
|
||||
API_URL="https://api.github.com/repos/${DOWNSTREAM_REPO}"
|
||||
|
||||
# Check if there's an existing PR open
|
||||
echo "Listing pre-existing pull requests..."
|
||||
"${CURL[@]}" "${API_URL}/pulls?head=moz-gfx:${BRANCH}" |
|
||||
tee "${TMPDIR}/pr.get"
|
||||
set +e
|
||||
COMMENT_URL=$(cat "${TMPDIR}/pr.get" | ${MYDIR}/read-json.py "0/comments_url")
|
||||
HAS_COMMENT_URL="${?}"
|
||||
set -e
|
||||
|
||||
if [[ ${HAS_COMMENT_URL} -ne 0 ]]; then
|
||||
echo "Pull request not found, creating..."
|
||||
# The PR doesn't exist yet, so let's create it
|
||||
( echo -n '{ "title": "Sync changes from mozilla-central '"${RELATIVE_PATH}"'"'
|
||||
echo -n ', "body": "'"${FIXES}"'"'
|
||||
echo -n ', "head": "moz-gfx:'"${BRANCH}"'"'
|
||||
echo -n ', "base": "master" }'
|
||||
) > "${TMPDIR}/pr.create"
|
||||
"${CURL[@]}" -d "@${TMPDIR}/pr.create" "${API_URL}/pulls" |
|
||||
tee "${TMPDIR}/pr.response"
|
||||
COMMENT_URL=$(
|
||||
cat "${TMPDIR}/pr.response" |
|
||||
${MYDIR}/read-json.py "comments_url"
|
||||
)
|
||||
fi
|
||||
|
||||
# At this point COMMENTS_URL should be set, so leave a comment to tell bors
|
||||
# to merge the PR.
|
||||
echo "Posting r+ comment to ${COMMENT_URL}..."
|
||||
echo '{ "body": "'"$BORS"' r=auto" }' > "${TMPDIR}/bors_rplus"
|
||||
"${CURL[@]}" -d "@${TMPDIR}/bors_rplus" "${COMMENT_URL}"
|
||||
|
||||
echo "All done!"
|
||||
|
|
@ -13,9 +13,6 @@ with Files("code-coverage/**"):
|
|||
with Files("compare-locales/mach_commands.py"):
|
||||
BUG_COMPONENT = ("Localization Infrastructure and Tools", "compare-locales")
|
||||
|
||||
with Files("github-sync/**"):
|
||||
BUG_COMPONENT = ("Core", "Graphics")
|
||||
|
||||
with Files("lint/**"):
|
||||
BUG_COMPONENT = ("Developer Infrastructure", "Lint and Formatting")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue