Bug 1966470 - Part 2: Use mach artifact install --unfiltered-project-package in single-locale repacks. r=firefox-build-system-reviewers,glandium

There's a lot here.  The main thing is that rather than invoking
`INNER_UNMAKE_PACKAGE`, we use the new `mach artifact install
--unfiltered-project-package` flag to do that work.  In automation,
this gets configured using `MOZ_ARTIFACT_TASK` and is deterministic;
locally, this is far easier to work with.

This replacement allows us to clean up a bunch of Makefile goo, some
of which is done here.  More clean up is surely possible; the
`mozharness` config files are a good place to look next.

The `MOZ_PKG_MAC_*` settings previously referred to the unpacked DMG
resources.  There's no need for that complication; we can always take
them from the branding resources.  (The relevant `mozconfig`
entries *look* to always have Nightly branding, but merge automation
ensures that the branding is correctly set for Beta and Release.)

Differential Revision: https://phabricator.services.mozilla.com/D249448

browser/installer/Makefile.in

@@ -96,13 +96,6 @@ ifdef MOZ_NOTIFICATION_SERVER
 DEFINES += -DMOZ_NOTIFICATION_SERVER=1
 endif
 
-ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
-MOZ_PKG_MAC_DSSTORE=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/dsstore
-MOZ_PKG_MAC_BACKGROUND=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/background.png
-MOZ_PKG_MAC_ICON=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/disk.icns
-MOZ_PKG_MAC_EXTRA=--symlink '/Applications:/ '
-endif
-
 include $(topsrcdir)/toolkit/mozapps/installer/packager.mk
 
 ifeq (Darwin,$(OS_TARGET))

browser/locales/Makefile.in

@@ -16,8 +16,6 @@ SUBMAKEFILES += \
 
 PWD := $(CURDIR)
 
-# These are defaulted to be compatible with the files the wget-en-US target
-# pulls. You may override them if you provide your own files.
 ZIP_IN ?= $(ABS_DIST)/$(PACKAGE)
 
 ifdef MOZ_DEV_EDITION
@@ -25,17 +23,6 @@ MOZ_LANGPACK_EID=langpack-$(AB_CD)@devedition.mozilla.org
 else
 MOZ_LANGPACK_EID=langpack-$(AB_CD)@firefox.mozilla.org
 endif
-# For Nightly, we know where to get the builds from to do local repacks
-ifdef NIGHTLY_BUILD
-export EN_US_BINARY_URL ?= https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central
-endif
-
-ifneq (,$(filter cocoa,$(MOZ_WIDGET_TOOLKIT)))
-MOZ_PKG_MAC_DSSTORE=$(ABS_DIST)/branding/dsstore
-MOZ_PKG_MAC_BACKGROUND=$(ABS_DIST)/branding/background.png
-MOZ_PKG_MAC_ICON=$(ABS_DIST)/branding/disk.icns
-MOZ_PKG_MAC_EXTRA=--symlink '/Applications:/ '
-endif
 
 # Required for l10n.mk - defines a list of app sub dirs that should
 # be included in langpack xpis.

config/makefiles/makeutils.mk

@@ -116,6 +116,3 @@ endif
 ifdef USE_AUTOTARGETS_MK # mkdir_deps
   include $(topORerr)/config/makefiles/autotargets.mk
 endif
-
-## copy(src, dst): recursive copy
-copy_dir = (cd $(1)/. && $(TAR) $(TAR_CREATE_FLAGS) - .) | (cd $(2)/. && tar -xf -)

taskcluster/kinds/l10n/kind.yml

@@ -89,18 +89,12 @@ tasks:
                     win64: windows2012-64/opt
         env:
             by-build-platform:
-                linux.*:    # linux64 and 32 get same treatment here
+                linux.*:
-                    EN_US_PACKAGE_NAME: target.tar.xz
+                    MOZ_ARTIFACT_TASK: {task-reference: '<build-signing>'}
-                    EN_US_BINARY_URL:
-                        artifact-reference: <build-signing/{artifact_prefix}>
                 macosx64.*:
-                    EN_US_PACKAGE_NAME: target.dmg
+                    MOZ_ARTIFACT_TASK: {task-reference: '<repackage>'}
-                    EN_US_BINARY_URL:
-                        artifact-reference: <repackage/{artifact_prefix}>
                 win.*:
-                    EN_US_PACKAGE_NAME: target.zip
+                    MOZ_ARTIFACT_TASK: {task-reference: '<build-signing>'}
-                    EN_US_BINARY_URL:
-                        artifact-reference: <build-signing/{artifact_prefix}>
         mozharness:
             config:
                 by-build-platform:

taskcluster/kinds/shippable-l10n/kind.yml

@@ -112,18 +112,12 @@ tasks:
                     win64-aarch64-devedition: windows2012-aarch64-devedition/opt
         env:
             by-build-platform:
-                linux.*:    # linux64 and 32 get same treatment here
+                linux.*:
-                    EN_US_PACKAGE_NAME: target.tar.xz
+                    MOZ_ARTIFACT_TASK: {task-reference: '<build-signing>'}
-                    EN_US_BINARY_URL:
-                        artifact-reference: <build-signing/{artifact_prefix}>
                 macosx64.*:
-                    EN_US_PACKAGE_NAME: target.dmg
+                    MOZ_ARTIFACT_TASK: {task-reference: '<repackage>'}
-                    EN_US_BINARY_URL:
-                        artifact-reference: <repackage/{artifact_prefix}>
                 win.*:
-                    EN_US_PACKAGE_NAME: target.zip
+                    MOZ_ARTIFACT_TASK: {task-reference: '<build-signing>'}
-                    EN_US_BINARY_URL:
-                        artifact-reference: <build-signing/{artifact_prefix}>
         mozharness:
             config:
                 by-build-platform:

testing/mozharness/configs/single_locale/tc_linux32.py

@@ -8,7 +8,6 @@ config = {
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "%(abs_obj_dir)s",
-        "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"],
         "DIST": "%(abs_obj_dir)s",
         "L10NBASEDIR": "../../l10n",
         "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"),

testing/mozharness/configs/single_locale/tc_linux_common.py

@@ -8,7 +8,6 @@ config = {
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "%(abs_obj_dir)s",
-        "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"],
         "DIST": "%(abs_obj_dir)s",
         "L10NBASEDIR": "../../l10n",
         "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"),

testing/mozharness/configs/single_locale/tc_macosx64.py

@@ -8,7 +8,6 @@ config = {
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "%(abs_obj_dir)s",
-        "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"],
         "DIST": "%(abs_obj_dir)s",
         "L10NBASEDIR": "../../l10n",
         "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"),

testing/mozharness/configs/single_locale/tc_win32.py

@@ -8,10 +8,8 @@ config = {
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "%(abs_obj_dir)s",
-        "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"],
         "DIST": "%(abs_obj_dir)s",
         "L10NBASEDIR": "../../l10n",
         "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"),
-        "EN_US_PACKAGE_NAME": "target.zip",
     },
 }

testing/mozharness/configs/single_locale/tc_win64.py

@@ -8,10 +8,8 @@ config = {
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "%(abs_obj_dir)s",
-        "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"],
         "DIST": "%(abs_obj_dir)s",
         "L10NBASEDIR": "../../l10n",
         "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"),
-        "EN_US_PACKAGE_NAME": "target.zip",
     },
 }

testing/mozharness/scripts/desktop_l10n.py

@@ -58,17 +58,6 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
                 " revision separated by colon, en-GB:default.",
             },
         ],
-        [
-            [
-                "--en-us-installer-url",
-            ],
-            {
-                "action": "store",
-                "dest": "en_us_installer_url",
-                "type": "string",
-                "help": "Specify the url of the en-us binary",
-            },
-        ],
     ]
 
     def __init__(self, require_config_file=True):
@@ -201,11 +190,9 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
         self._copy_mozconfig()
         self._mach_configure()
         self._make_export()
-        self.make_wget_en_US()
-        self.make_unpack_en_US()
 
     def _make_export(self):
-        """this step creates nsinstall, needed my make_wget_en_US() on Windows hosts
+        """this step creates nsinstall, needed on Windows hosts
         and creates buildid.h, used by NSIS installer for Windows UBR telemetry
         """
         dirs = self.query_abs_dirs()
@@ -280,21 +267,6 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
             ignore_errors=ignore_errors,
         )
 
-    def make_unpack_en_US(self):
-        """wrapper for make unpack"""
-        config = self.config
-        dirs = self.query_abs_dirs()
-        env = self.query_bootstrap_env()
-        cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"])
-        return self._make(target=["unpack"], cwd=cwd, env=env)
-
-    def make_wget_en_US(self):
-        """wrapper for make wget-en-US"""
-        env = self.query_bootstrap_env()
-        dirs = self.query_abs_dirs()
-        cwd = dirs["abs_locales_dir"]
-        return self._make(target=["wget-en-US"], cwd=cwd, env=env)
-
     def make_upload(self, locale):
         """wrapper for make upload command"""
         env = self.query_l10n_env()

toolkit/locales/l10n.mk

@@ -37,11 +37,6 @@ LPROJ_ROOT := $(subst -,_,$(AB_CD))
 endif
 endif
 
-# These are defaulted to be compatible with the files the wget-en-US target
-# pulls. You may override them if you provide your own files.
-ZIP_IN ?= $(ABS_DIST)/$(PACKAGE)
-WIN32_INSTALLER_IN ?= $(ABS_DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe
-
 # Allows overriding the final destination of the repackaged file
 ZIP_OUT ?= $(ABS_DIST)/$(PACKAGE)
 
@@ -76,37 +71,17 @@ STAGEDIST = $(ABS_DIST)/l10n-stage/$(MOZ_PKG_DIR)/$(_APPNAME)/Contents/Resources
 else
 STAGEDIST = $(ABS_DIST)/l10n-stage/$(MOZ_PKG_DIR)
 endif
-UNPACKED_INSTALLER = $(ABS_DIST)/unpacked-installer
 
 include $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.mk
 
-$(UNPACKED_INSTALLER): AB_CD:=en-US
+unpack:
-$(UNPACKED_INSTALLER): UNPACKAGE=$(call ESCAPE_WILDCARD,$(ZIP_IN))
+	$(RM) -r -f '$(ABS_DIST)/l10n-stage'
-$(UNPACKED_INSTALLER): $(call ESCAPE_WILDCARD,$(ZIP_IN))
+	$(PYTHON3) $(topsrcdir)/mach --log-no-times artifact install --unfiltered-project-package --distdir '$(ABS_DIST)/l10n-stage/$(MOZ_PKG_DIR)' --verbose
-# only mac needs to remove the parent of STAGEDIST...
-ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
-	$(RM) -r -v $(UNPACKED_INSTALLER)
-else
-# ... and windows doesn't like removing STAGEDIST itself, remove all children
-	find $(UNPACKED_INSTALLER) -maxdepth 1 -print0 | xargs -0 $(RM) -r
-endif
-	$(NSINSTALL) -D $(UNPACKED_INSTALLER)
-	$(call INNER_UNMAKE_PACKAGE,$(UNPACKED_INSTALLER))
-
-unpack: $(UNPACKED_INSTALLER)
-ifeq ($(OS_ARCH), WINNT)
-	$(RM) -r -f $(ABS_DIST)/l10n-stage
-	$(NSINSTALL) -D $(ABS_DIST)/l10n-stage
-	$(call copy_dir, $(UNPACKED_INSTALLER), $(ABS_DIST)/l10n-stage)
-else
-	rsync -rav --delete $(UNPACKED_INSTALLER)/ $(ABS_DIST)/l10n-stage
-endif
 
 # The path to the object dir for the mozilla-central build system,
 # may be overridden if necessary.
 MOZDEPTH ?= $(DEPTH)
 
-repackage-zip: UNPACKAGE='$(ZIP_IN)'
 repackage-zip:
 	$(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/l10n-repack.py '$(STAGEDIST)' $(ABS_DIST)/xpi-stage/locale-$(AB_CD) \
 		$(MOZ_PKG_EXTRAL10N) \
@@ -136,7 +111,7 @@ endif
 	if test -f '$(DIST)/l10n-stage/$(PACKAGE).asc'; then mv -f '$(DIST)/l10n-stage/$(PACKAGE).asc' '$(ZIP_OUT).asc'; fi
 
 repackage-zip-%: unpack
-	@$(MAKE) repackage-zip AB_CD=$* ZIP_IN='$(ZIP_IN)'
+	@$(MAKE) repackage-zip AB_CD=$*
 
 # Dealing with app sub dirs: If DIST_SUBDIRS is defined it contains a
 # listing of app sub-dirs we should include in langpack xpis. If not,
@@ -178,24 +153,3 @@ package-langpack-%:
 	$(NSINSTALL) -D $(DIST)/$(PKG_LANGPACK_PATH)
 	$(call py_action,langpack_manifest $(AB_CD),--locales $(AB_CD) --app-version $(MOZ_APP_VERSION) --max-app-ver $(MOZ_APP_MAXVERSION) --app-name '$(MOZ_APP_DISPLAYNAME)' --l10n-basedir '$(L10NBASEDIR)' --metadata $(LANGPACK_METADATA) --langpack-eid '$(MOZ_LANGPACK_EID)' --input $(DIST)/xpi-stage/locale-$(AB_CD))
 	$(call py_action,zip $(PKG_LANGPACK_BASENAME).xpi,-C $(DIST)/xpi-stage/locale-$(AB_CD) -x **/*.manifest -x **/*.js -x **/*.ini $(LANGPACK_FILE) $(PKG_ZIP_DIRS) manifest.json)
-
-# This variable is to allow the wget-en-US target to know which ftp server to download from
-ifndef EN_US_BINARY_URL
-EN_US_BINARY_URL = $(error You must set EN_US_BINARY_URL)
-endif
-
-# Allow the overriding of PACKAGE format so we can get an EN_US build with a different
-# PACKAGE format than we are creating l10n packages with.
-EN_US_PACKAGE_NAME ?= $(PACKAGE)
-
-# This make target allows us to wget the latest en-US binary from a specified website
-# The make installers-% target needs the en-US binary in dist/
-# and for the windows repackages we need the .installer.exe in dist/sea
-wget-en-US:
-ifndef WGET
-	$(error Wget not installed)
-endif
-	$(NSINSTALL) -D $(ABS_DIST)/$(PKG_PATH)
-	(cd $(ABS_DIST)/$(PKG_PATH) && \
-        $(WGET) --no-cache -nv --no-iri -N -O $(PACKAGE) '$(EN_US_BINARY_URL)/$(EN_US_PACKAGE_NAME)')
-	@echo 'Downloaded $(EN_US_BINARY_URL)/$(EN_US_PACKAGE_NAME) to $(ABS_DIST)/$(PKG_PATH)/$(PACKAGE)'

toolkit/mozapps/installer/upload-files.mk

@@ -93,24 +93,20 @@ endif
 TAR_CREATE_FLAGS := --exclude=.mkdir.done $(TAR_CREATE_FLAGS)
 CREATE_FINAL_TAR = $(TAR) -c --owner=0 --group=0 --numeric-owner \
   --mode=go-w --exclude=.mkdir.done -f
-UNPACK_TAR       = tar -xf-
 
 ifeq ($(MOZ_PKG_FORMAT),TAR)
   PKG_SUFFIX	= .tar
   INNER_MAKE_PACKAGE 	= cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) > $(PACKAGE)
-  INNER_UNMAKE_PACKAGE	= cd $(1) && $(UNPACK_TAR) < $(UNPACKAGE)
 endif
 
 ifeq ($(MOZ_PKG_FORMAT),TGZ)
   PKG_SUFFIX	= .tar.gz
   INNER_MAKE_PACKAGE 	= cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) | gzip -vf9 > $(PACKAGE)
-  INNER_UNMAKE_PACKAGE	= cd $(1) && gunzip -c $(UNPACKAGE) | $(UNPACK_TAR)
 endif
 
 ifeq ($(MOZ_PKG_FORMAT),XZ)
   PKG_SUFFIX = .tar.xz
   INNER_MAKE_PACKAGE 	= cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) | xz --compress --stdout -9 --extreme > $(PACKAGE)
-  INNER_UNMAKE_PACKAGE	= cd $(1) && xz --decompress --stdout $(UNPACKAGE) | $(UNPACK_TAR)
 endif
 
 ifeq ($(MOZ_PKG_FORMAT),BZ2)
@@ -120,13 +116,11 @@ ifeq ($(MOZ_PKG_FORMAT),BZ2)
   else
     INNER_MAKE_PACKAGE 	= cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) | bzip2 -vf > $(PACKAGE)
   endif
-  INNER_UNMAKE_PACKAGE	= cd $(1) && bunzip2 -c $(UNPACKAGE) | $(UNPACK_TAR)
 endif
 
 ifeq ($(MOZ_PKG_FORMAT),ZIP)
   PKG_SUFFIX	= .zip
   INNER_MAKE_PACKAGE = $(call py_action,zip,'$(PACKAGE)' '$(MOZ_PKG_DIR)' -x '**/.mkdir.done',$(1))
-  INNER_UNMAKE_PACKAGE = $(call py_action,make_unzip,$(UNPACKAGE),$(1))
 endif
 
 #Create an RPM file
@@ -199,15 +193,11 @@ ifeq ($(MOZ_PKG_FORMAT),RPM)
   endif
 
   INNER_MAKE_PACKAGE = cd $(1) && $(RPM_CMD)
-  #Avoiding rpm repacks, going to try creating/uploading xpi in rpm files instead
-  INNER_UNMAKE_PACKAGE = $(error Try using rpm2cpio and cpio)
-
 endif #Create an RPM file
 
 
 ifeq ($(MOZ_PKG_FORMAT),APK)
 INNER_MAKE_PACKAGE = true
-INNER_UNMAKE_PACKAGE = true
 endif
 
 ifeq ($(MOZ_PKG_FORMAT),DMG)
@@ -215,6 +205,9 @@ ifeq ($(MOZ_PKG_FORMAT),DMG)
 
   _ABS_MOZSRCDIR = $(shell cd $(MOZILLA_DIR) && pwd)
   PKG_DMG_SOURCE = $(MOZ_PKG_DIR)
+  MOZ_PKG_MAC_DSSTORE=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/dsstore
+  MOZ_PKG_MAC_BACKGROUND=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/background.png
+  MOZ_PKG_MAC_ICON=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/disk.icns
   INNER_MAKE_PACKAGE = \
     $(call py_action,make_dmg, \
         $(if $(MOZ_PKG_MAC_DSSTORE),--dsstore '$(MOZ_PKG_MAC_DSSTORE)') \
@@ -223,13 +216,6 @@ ifeq ($(MOZ_PKG_FORMAT),DMG)
         --volume-name '$(MOZ_APP_DISPLAYNAME)' \
         '$(PKG_DMG_SOURCE)' '$(PACKAGE)', \
         $(1))
-  INNER_UNMAKE_PACKAGE = \
-    $(call py_action,unpack_dmg, \
-        $(if $(MOZ_PKG_MAC_DSSTORE),--dsstore '$(MOZ_PKG_MAC_DSSTORE)') \
-        $(if $(MOZ_PKG_MAC_BACKGROUND),--background '$(MOZ_PKG_MAC_BACKGROUND)') \
-        $(if $(MOZ_PKG_MAC_ICON),--icon '$(MOZ_PKG_MAC_ICON)') \
-        $(UNPACKAGE) $(MOZ_PKG_DIR), \
-        $(1))
 endif
 
 MAKE_PACKAGE = $(INNER_MAKE_PACKAGE)