Bug 1887210 - Repackage the nightly linux64-aarch64 builds as .deb packages and upload them to the APT repository r=releng-reviewers,taskgraph-reviewers,bhearsum

Depends on D205402

Differential Revision: https://phabricator.services.mozilla.com/D205501

python/mozbuild/mozbuild/repackaging/deb.py

@@ -40,18 +40,33 @@ class HgServerError(Exception):
         super().__init__(msg)
 
 
+# Maps our CI/release pipeline's architecture names (e.g., "x86_64")
+# into architectures ("amd64") compatible with Debian's dpkg-buildpackage tool.
+# This is the target architecture we are building the .deb package for.
 _DEB_ARCH = {
     "all": "all",
     "x86": "i386",
     "x86_64": "amd64",
+    "aarch64": "arm64",
+}
+
+# Defines the sysroot (build host's) architecture for each target architecture in the pipeline.
+# It defines the architecture dpkg-buildpackage runs on.
+_DEB_SYSROOT_ARCH = {
+    "all": "amd64",
+    "x86": "i386",
+    "x86_64": "amd64",
+    "aarch64": "amd64",
+}
+
+# Assigns the Debian distribution version for the sysroot based on the target architecture.
+# It defines the Debian distribution dpkg-buildpackage runs on.
+_DEB_SYSROOT_DIST = {
+    "all": "jessie",
+    "x86": "jessie",
+    "x86_64": "jessie",
+    "aarch64": "buster",
 }
-# At the moment the Firefox build baseline is jessie.
-# The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile
-# bootstraps the /srv/jessie-i386 and /srv/jessie-amd64 chroot environments we use to
-# create the `.deb` repackages. By running the repackage using chroot we generate shared
-# library dependencies that match the Firefox build baseline
-# defined in taskcluster/scripts/misc/build-sysroot.sh
-_DEB_DIST = "jessie"
 
 
 def repackage_deb(
@@ -729,8 +744,15 @@ def _is_chroot_available(arch):
 
 
 def _get_chroot_path(arch):
-    deb_arch = "amd64" if arch == "all" else _DEB_ARCH[arch]
+    # At the moment the Firefox build baseline for i386 and amd64 is jessie and the baseline for arm64 is buster.
-    return f"/srv/{_DEB_DIST}-{deb_arch}"
+    # These baselines are defined in taskcluster/scripts/misc/build-sysroot.sh
+    # The debian-repackage image defined in taskcluster/docker/debian-repackage/Dockerfile
+    # bootstraps /srv/jessie-i386, /srv/jessie-amd64, and /srv/buster-amd64 roots.
+    # We use these roots to run the repackage step and generate shared
+    # library dependencies that match the Firefox build baseline.
+    deb_sysroot_dist = _DEB_SYSROOT_DIST[arch]
+    deb_sysroot_arch = _DEB_SYSROOT_ARCH[arch]
+    return f"/srv/{deb_sysroot_dist}-{deb_sysroot_arch}"
 
 
 _MANIFEST_FILE_NAME = "manifest.json"

taskcluster/ci/beetmover-apt/kind.yml

@@ -16,6 +16,7 @@ kind-dependencies:
 only-for-build-platforms:
     - linux-shippable/opt
     - linux64-shippable/opt
+    - linux64-aarch64-shippable/opt
     - linux-devedition/opt
     - linux64-devedition/opt
 

taskcluster/ci/repackage-deb/kind.yml

@@ -21,6 +21,7 @@ only-for-build-platforms:
     - linux-devedition/opt
     - linux64-shippable/opt
     - linux64-devedition/opt
+    - linux64-aarch64-shippable/opt
 
 only-for-attributes:
     - nightly

taskcluster/docker/debian-repackage/Dockerfile

@@ -3,22 +3,16 @@ MAINTAINER Mozilla Releng <release@mozilla.com>
 
 VOLUME /builds/worker/workspace
 
-# At the moment the Firefox build baseline is jessie.
+# At the moment the Firefox build sysroot for i386 and amd64 use jessie and arm64 uses buster.
-# Things in the build/CI system that run in the task's container
+# To generate Firefox Debian packages we bootstrap jessie and buster sysroots.
-# (mach, run-task, etc.) do not support jessie.
+# We use these sysroots to generate shared library dependencies for
-# There's also no jessie `base` images or packages generated in the CI.
+# i386 (jessie), amd64 (jessie), and arm64 (buster) packages.
-# To generate Firefox Debian packages compatible with the baseline
+# To keep the build and repackage enviroments consistent the Debian distributions used by the sysroots
-# we bootstrap basic i386 and amd64 jessie systems on bullseye.
+# created here should match the distributions used by the sysroots created in taskcluster/scripts/misc/build-sysroot.sh
-# We use these to generate shared library dependencies in jessie using chroot.
-# python/mozbuild/mozbuild/repackaging/deb.py checks for jessie systems
-# bootstrapped under /srv/jessie-i386 and /srv/jessie-amd64
-# If they aren't there, deb.py runs the repackage in the working directory without using chroot.
-# To keep the build and repackage enviroments consistent the Debian baseline used
-# here (jessie) should be synchronized with the baseline used in
-# taskcluster/scripts/misc/build-sysroot.sh
 ARG TASKCLUSTER_ROOT_URL
 ARG DOCKER_IMAGE_PACKAGES
 RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
+    dpkg --add-architecture arm64 && \
     apt-get update && \
     apt-get install debootstrap && set -xe && \
     for arch in i386 amd64; do \
@@ -30,6 +24,20 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG
         --verbose \
         jessie \
         /srv/jessie-$arch \
-        https://archive.debian.org/debian; \
+        https://archive.debian.org/debian || (echo "Debootstrap failed, outputting log:" && cat /srv/jessie-$arch/debootstrap/debootstrap.log && exit 1); \
     done && \
+    # bootstrap a buster/amd64 root to generate an arm64 .deb package.
+    debootstrap \
+    --arch=amd64 \
+    --variant=buildd \
+    --include=debhelper,binutils-multiarch,binutils-aarch64-linux-gnu \
+    --keyring=/usr/share/keyrings/debian-archive-buster-stable.gpg \
+    --verbose \
+    buster \
+    /srv/buster-amd64 \
+    http://deb.debian.org/debian || (echo "Debootstrap failed, outputting log:" && cat /srv/buster-amd64/debootstrap/debootstrap.log && exit 1) && \
+    chroot /srv/buster-amd64 /bin/bash -c "\
+        dpkg --add-architecture arm64 && \
+        apt-get update && \
+        apt-get install -y libasound2:arm64 libgtk-3-0:arm64 libx11-xcb1:arm64 libxtst6:arm64" && \
     /usr/local/sbin/clean_packages.sh $DOCKER_IMAGE_PACKAGES

taskcluster/gecko_taskgraph/util/platforms.py

@@ -27,7 +27,8 @@ _executable_extension = {
 
 _architectures = {
     r"linux\b.*": "x86",
-    r"linux64\b.*": "x86_64",
+    r"linux64\b(?!-aarch64).*": "x86_64",
+    r"linux64-aarch64\b.*": "aarch64",
     r"macosx64\b.*": "macos-x86_64-aarch64",
     r"win32\b.*": "x86",
     r"win64\b(?!-aarch64).*": "x86_64",