From 88e52afb397c96e667134ae5bc95f00d3790d29a Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 21 May 2025 16:32:56 +0000 Subject: [PATCH] Bug 1966470 - Part 2: Use `mach artifact install --unfiltered-project-package` in single-locale repacks. r=firefox-build-system-reviewers,glandium There's a lot here. The main thing is that rather than invoking `INNER_UNMAKE_PACKAGE`, we use the new `mach artifact install --unfiltered-project-package` flag to do that work. In automation, this gets configured using `MOZ_ARTIFACT_TASK` and is deterministic; locally, this is far easier to work with. This replacement allows us to clean up a bunch of Makefile goo, some of which is done here. More clean up is surely possible; the `mozharness` config files are a good place to look next. The `MOZ_PKG_MAC_*` settings previously referred to the unpacked DMG resources. There's no need for that complication; we can always take them from the branding resources. (The relevant `mozconfig` entries *look* to always have Nightly branding, but merge automation ensures that the branding is correctly set for Beta and Release.) Differential Revision: https://phabricator.services.mozilla.com/D249448 --- browser/installer/Makefile.in | 7 --- browser/locales/Makefile.in | 13 ----- config/makefiles/makeutils.mk | 3 -- taskcluster/kinds/l10n/kind.yml | 14 ++--- taskcluster/kinds/shippable-l10n/kind.yml | 14 ++--- .../configs/single_locale/tc_linux32.py | 1 - .../configs/single_locale/tc_linux_common.py | 1 - .../configs/single_locale/tc_macosx64.py | 1 - .../configs/single_locale/tc_win32.py | 2 - .../configs/single_locale/tc_win64.py | 2 - testing/mozharness/scripts/desktop_l10n.py | 30 +---------- toolkit/locales/l10n.mk | 54 ++----------------- toolkit/mozapps/installer/upload-files.mk | 20 ++----- 13 files changed, 16 insertions(+), 146 deletions(-) diff --git a/browser/installer/Makefile.in b/browser/installer/Makefile.in index 2e55ed4b83dc..7cf7f19976d9 100644 --- a/browser/installer/Makefile.in +++ b/browser/installer/Makefile.in @@ -96,13 +96,6 @@ ifdef MOZ_NOTIFICATION_SERVER DEFINES += -DMOZ_NOTIFICATION_SERVER=1 endif -ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT)) -MOZ_PKG_MAC_DSSTORE=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/dsstore -MOZ_PKG_MAC_BACKGROUND=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/background.png -MOZ_PKG_MAC_ICON=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/disk.icns -MOZ_PKG_MAC_EXTRA=--symlink '/Applications:/ ' -endif - include $(topsrcdir)/toolkit/mozapps/installer/packager.mk ifeq (Darwin,$(OS_TARGET)) diff --git a/browser/locales/Makefile.in b/browser/locales/Makefile.in index d3b76c92e48a..2f6a2c9be2e9 100644 --- a/browser/locales/Makefile.in +++ b/browser/locales/Makefile.in @@ -16,8 +16,6 @@ SUBMAKEFILES += \ PWD := $(CURDIR) -# These are defaulted to be compatible with the files the wget-en-US target -# pulls. You may override them if you provide your own files. ZIP_IN ?= $(ABS_DIST)/$(PACKAGE) ifdef MOZ_DEV_EDITION @@ -25,17 +23,6 @@ MOZ_LANGPACK_EID=langpack-$(AB_CD)@devedition.mozilla.org else MOZ_LANGPACK_EID=langpack-$(AB_CD)@firefox.mozilla.org endif -# For Nightly, we know where to get the builds from to do local repacks -ifdef NIGHTLY_BUILD -export EN_US_BINARY_URL ?= https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central -endif - -ifneq (,$(filter cocoa,$(MOZ_WIDGET_TOOLKIT))) -MOZ_PKG_MAC_DSSTORE=$(ABS_DIST)/branding/dsstore -MOZ_PKG_MAC_BACKGROUND=$(ABS_DIST)/branding/background.png -MOZ_PKG_MAC_ICON=$(ABS_DIST)/branding/disk.icns -MOZ_PKG_MAC_EXTRA=--symlink '/Applications:/ ' -endif # Required for l10n.mk - defines a list of app sub dirs that should # be included in langpack xpis. diff --git a/config/makefiles/makeutils.mk b/config/makefiles/makeutils.mk index 95c7791a4b3f..5be39dec7fad 100644 --- a/config/makefiles/makeutils.mk +++ b/config/makefiles/makeutils.mk @@ -116,6 +116,3 @@ endif ifdef USE_AUTOTARGETS_MK # mkdir_deps include $(topORerr)/config/makefiles/autotargets.mk endif - -## copy(src, dst): recursive copy -copy_dir = (cd $(1)/. && $(TAR) $(TAR_CREATE_FLAGS) - .) | (cd $(2)/. && tar -xf -) diff --git a/taskcluster/kinds/l10n/kind.yml b/taskcluster/kinds/l10n/kind.yml index d5a8a4ead7e6..537fe85fda49 100644 --- a/taskcluster/kinds/l10n/kind.yml +++ b/taskcluster/kinds/l10n/kind.yml @@ -89,18 +89,12 @@ tasks: win64: windows2012-64/opt env: by-build-platform: - linux.*: # linux64 and 32 get same treatment here - EN_US_PACKAGE_NAME: target.tar.xz - EN_US_BINARY_URL: - artifact-reference: + linux.*: + MOZ_ARTIFACT_TASK: {task-reference: ''} macosx64.*: - EN_US_PACKAGE_NAME: target.dmg - EN_US_BINARY_URL: - artifact-reference: + MOZ_ARTIFACT_TASK: {task-reference: ''} win.*: - EN_US_PACKAGE_NAME: target.zip - EN_US_BINARY_URL: - artifact-reference: + MOZ_ARTIFACT_TASK: {task-reference: ''} mozharness: config: by-build-platform: diff --git a/taskcluster/kinds/shippable-l10n/kind.yml b/taskcluster/kinds/shippable-l10n/kind.yml index ac7615d33037..2525369fa22a 100644 --- a/taskcluster/kinds/shippable-l10n/kind.yml +++ b/taskcluster/kinds/shippable-l10n/kind.yml @@ -112,18 +112,12 @@ tasks: win64-aarch64-devedition: windows2012-aarch64-devedition/opt env: by-build-platform: - linux.*: # linux64 and 32 get same treatment here - EN_US_PACKAGE_NAME: target.tar.xz - EN_US_BINARY_URL: - artifact-reference: + linux.*: + MOZ_ARTIFACT_TASK: {task-reference: ''} macosx64.*: - EN_US_PACKAGE_NAME: target.dmg - EN_US_BINARY_URL: - artifact-reference: + MOZ_ARTIFACT_TASK: {task-reference: ''} win.*: - EN_US_PACKAGE_NAME: target.zip - EN_US_BINARY_URL: - artifact-reference: + MOZ_ARTIFACT_TASK: {task-reference: ''} mozharness: config: by-build-platform: diff --git a/testing/mozharness/configs/single_locale/tc_linux32.py b/testing/mozharness/configs/single_locale/tc_linux32.py index 4ae752847378..bb6ede48db52 100644 --- a/testing/mozharness/configs/single_locale/tc_linux32.py +++ b/testing/mozharness/configs/single_locale/tc_linux32.py @@ -8,7 +8,6 @@ config = { "bootstrap_env": { "NO_MERCURIAL_SETUP_CHECK": "1", "MOZ_OBJDIR": "%(abs_obj_dir)s", - "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"], "DIST": "%(abs_obj_dir)s", "L10NBASEDIR": "../../l10n", "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"), diff --git a/testing/mozharness/configs/single_locale/tc_linux_common.py b/testing/mozharness/configs/single_locale/tc_linux_common.py index 4ae752847378..bb6ede48db52 100644 --- a/testing/mozharness/configs/single_locale/tc_linux_common.py +++ b/testing/mozharness/configs/single_locale/tc_linux_common.py @@ -8,7 +8,6 @@ config = { "bootstrap_env": { "NO_MERCURIAL_SETUP_CHECK": "1", "MOZ_OBJDIR": "%(abs_obj_dir)s", - "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"], "DIST": "%(abs_obj_dir)s", "L10NBASEDIR": "../../l10n", "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"), diff --git a/testing/mozharness/configs/single_locale/tc_macosx64.py b/testing/mozharness/configs/single_locale/tc_macosx64.py index 4ae752847378..bb6ede48db52 100644 --- a/testing/mozharness/configs/single_locale/tc_macosx64.py +++ b/testing/mozharness/configs/single_locale/tc_macosx64.py @@ -8,7 +8,6 @@ config = { "bootstrap_env": { "NO_MERCURIAL_SETUP_CHECK": "1", "MOZ_OBJDIR": "%(abs_obj_dir)s", - "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"], "DIST": "%(abs_obj_dir)s", "L10NBASEDIR": "../../l10n", "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"), diff --git a/testing/mozharness/configs/single_locale/tc_win32.py b/testing/mozharness/configs/single_locale/tc_win32.py index 34bcde89bc0b..bb6ede48db52 100644 --- a/testing/mozharness/configs/single_locale/tc_win32.py +++ b/testing/mozharness/configs/single_locale/tc_win32.py @@ -8,10 +8,8 @@ config = { "bootstrap_env": { "NO_MERCURIAL_SETUP_CHECK": "1", "MOZ_OBJDIR": "%(abs_obj_dir)s", - "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"], "DIST": "%(abs_obj_dir)s", "L10NBASEDIR": "../../l10n", "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"), - "EN_US_PACKAGE_NAME": "target.zip", }, } diff --git a/testing/mozharness/configs/single_locale/tc_win64.py b/testing/mozharness/configs/single_locale/tc_win64.py index 34bcde89bc0b..bb6ede48db52 100644 --- a/testing/mozharness/configs/single_locale/tc_win64.py +++ b/testing/mozharness/configs/single_locale/tc_win64.py @@ -8,10 +8,8 @@ config = { "bootstrap_env": { "NO_MERCURIAL_SETUP_CHECK": "1", "MOZ_OBJDIR": "%(abs_obj_dir)s", - "EN_US_BINARY_URL": os.environ["EN_US_BINARY_URL"], "DIST": "%(abs_obj_dir)s", "L10NBASEDIR": "../../l10n", "TOOLTOOL_CACHE": os.environ.get("TOOLTOOL_CACHE"), - "EN_US_PACKAGE_NAME": "target.zip", }, } diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py index 13f4508828f6..70d56e978960 100755 --- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -58,17 +58,6 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): " revision separated by colon, en-GB:default.", }, ], - [ - [ - "--en-us-installer-url", - ], - { - "action": "store", - "dest": "en_us_installer_url", - "type": "string", - "help": "Specify the url of the en-us binary", - }, - ], ] def __init__(self, require_config_file=True): @@ -201,11 +190,9 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): self._copy_mozconfig() self._mach_configure() self._make_export() - self.make_wget_en_US() - self.make_unpack_en_US() def _make_export(self): - """this step creates nsinstall, needed my make_wget_en_US() on Windows hosts + """this step creates nsinstall, needed on Windows hosts and creates buildid.h, used by NSIS installer for Windows UBR telemetry """ dirs = self.query_abs_dirs() @@ -280,21 +267,6 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript): ignore_errors=ignore_errors, ) - def make_unpack_en_US(self): - """wrapper for make unpack""" - config = self.config - dirs = self.query_abs_dirs() - env = self.query_bootstrap_env() - cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"]) - return self._make(target=["unpack"], cwd=cwd, env=env) - - def make_wget_en_US(self): - """wrapper for make wget-en-US""" - env = self.query_bootstrap_env() - dirs = self.query_abs_dirs() - cwd = dirs["abs_locales_dir"] - return self._make(target=["wget-en-US"], cwd=cwd, env=env) - def make_upload(self, locale): """wrapper for make upload command""" env = self.query_l10n_env() diff --git a/toolkit/locales/l10n.mk b/toolkit/locales/l10n.mk index 66e91f6a9593..4e27d1815a0e 100644 --- a/toolkit/locales/l10n.mk +++ b/toolkit/locales/l10n.mk @@ -37,11 +37,6 @@ LPROJ_ROOT := $(subst -,_,$(AB_CD)) endif endif -# These are defaulted to be compatible with the files the wget-en-US target -# pulls. You may override them if you provide your own files. -ZIP_IN ?= $(ABS_DIST)/$(PACKAGE) -WIN32_INSTALLER_IN ?= $(ABS_DIST)/$(PKG_INST_PATH)$(PKG_INST_BASENAME).exe - # Allows overriding the final destination of the repackaged file ZIP_OUT ?= $(ABS_DIST)/$(PACKAGE) @@ -76,37 +71,17 @@ STAGEDIST = $(ABS_DIST)/l10n-stage/$(MOZ_PKG_DIR)/$(_APPNAME)/Contents/Resources else STAGEDIST = $(ABS_DIST)/l10n-stage/$(MOZ_PKG_DIR) endif -UNPACKED_INSTALLER = $(ABS_DIST)/unpacked-installer include $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.mk -$(UNPACKED_INSTALLER): AB_CD:=en-US -$(UNPACKED_INSTALLER): UNPACKAGE=$(call ESCAPE_WILDCARD,$(ZIP_IN)) -$(UNPACKED_INSTALLER): $(call ESCAPE_WILDCARD,$(ZIP_IN)) -# only mac needs to remove the parent of STAGEDIST... -ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT)) - $(RM) -r -v $(UNPACKED_INSTALLER) -else -# ... and windows doesn't like removing STAGEDIST itself, remove all children - find $(UNPACKED_INSTALLER) -maxdepth 1 -print0 | xargs -0 $(RM) -r -endif - $(NSINSTALL) -D $(UNPACKED_INSTALLER) - $(call INNER_UNMAKE_PACKAGE,$(UNPACKED_INSTALLER)) - -unpack: $(UNPACKED_INSTALLER) -ifeq ($(OS_ARCH), WINNT) - $(RM) -r -f $(ABS_DIST)/l10n-stage - $(NSINSTALL) -D $(ABS_DIST)/l10n-stage - $(call copy_dir, $(UNPACKED_INSTALLER), $(ABS_DIST)/l10n-stage) -else - rsync -rav --delete $(UNPACKED_INSTALLER)/ $(ABS_DIST)/l10n-stage -endif +unpack: + $(RM) -r -f '$(ABS_DIST)/l10n-stage' + $(PYTHON3) $(topsrcdir)/mach --log-no-times artifact install --unfiltered-project-package --distdir '$(ABS_DIST)/l10n-stage/$(MOZ_PKG_DIR)' --verbose # The path to the object dir for the mozilla-central build system, # may be overridden if necessary. MOZDEPTH ?= $(DEPTH) -repackage-zip: UNPACKAGE='$(ZIP_IN)' repackage-zip: $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/l10n-repack.py '$(STAGEDIST)' $(ABS_DIST)/xpi-stage/locale-$(AB_CD) \ $(MOZ_PKG_EXTRAL10N) \ @@ -136,7 +111,7 @@ endif if test -f '$(DIST)/l10n-stage/$(PACKAGE).asc'; then mv -f '$(DIST)/l10n-stage/$(PACKAGE).asc' '$(ZIP_OUT).asc'; fi repackage-zip-%: unpack - @$(MAKE) repackage-zip AB_CD=$* ZIP_IN='$(ZIP_IN)' + @$(MAKE) repackage-zip AB_CD=$* # Dealing with app sub dirs: If DIST_SUBDIRS is defined it contains a # listing of app sub-dirs we should include in langpack xpis. If not, @@ -178,24 +153,3 @@ package-langpack-%: $(NSINSTALL) -D $(DIST)/$(PKG_LANGPACK_PATH) $(call py_action,langpack_manifest $(AB_CD),--locales $(AB_CD) --app-version $(MOZ_APP_VERSION) --max-app-ver $(MOZ_APP_MAXVERSION) --app-name '$(MOZ_APP_DISPLAYNAME)' --l10n-basedir '$(L10NBASEDIR)' --metadata $(LANGPACK_METADATA) --langpack-eid '$(MOZ_LANGPACK_EID)' --input $(DIST)/xpi-stage/locale-$(AB_CD)) $(call py_action,zip $(PKG_LANGPACK_BASENAME).xpi,-C $(DIST)/xpi-stage/locale-$(AB_CD) -x **/*.manifest -x **/*.js -x **/*.ini $(LANGPACK_FILE) $(PKG_ZIP_DIRS) manifest.json) - -# This variable is to allow the wget-en-US target to know which ftp server to download from -ifndef EN_US_BINARY_URL -EN_US_BINARY_URL = $(error You must set EN_US_BINARY_URL) -endif - -# Allow the overriding of PACKAGE format so we can get an EN_US build with a different -# PACKAGE format than we are creating l10n packages with. -EN_US_PACKAGE_NAME ?= $(PACKAGE) - -# This make target allows us to wget the latest en-US binary from a specified website -# The make installers-% target needs the en-US binary in dist/ -# and for the windows repackages we need the .installer.exe in dist/sea -wget-en-US: -ifndef WGET - $(error Wget not installed) -endif - $(NSINSTALL) -D $(ABS_DIST)/$(PKG_PATH) - (cd $(ABS_DIST)/$(PKG_PATH) && \ - $(WGET) --no-cache -nv --no-iri -N -O $(PACKAGE) '$(EN_US_BINARY_URL)/$(EN_US_PACKAGE_NAME)') - @echo 'Downloaded $(EN_US_BINARY_URL)/$(EN_US_PACKAGE_NAME) to $(ABS_DIST)/$(PKG_PATH)/$(PACKAGE)' diff --git a/toolkit/mozapps/installer/upload-files.mk b/toolkit/mozapps/installer/upload-files.mk index 923bbffa7869..648b030b48f4 100644 --- a/toolkit/mozapps/installer/upload-files.mk +++ b/toolkit/mozapps/installer/upload-files.mk @@ -93,24 +93,20 @@ endif TAR_CREATE_FLAGS := --exclude=.mkdir.done $(TAR_CREATE_FLAGS) CREATE_FINAL_TAR = $(TAR) -c --owner=0 --group=0 --numeric-owner \ --mode=go-w --exclude=.mkdir.done -f -UNPACK_TAR = tar -xf- ifeq ($(MOZ_PKG_FORMAT),TAR) PKG_SUFFIX = .tar INNER_MAKE_PACKAGE = cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) > $(PACKAGE) - INNER_UNMAKE_PACKAGE = cd $(1) && $(UNPACK_TAR) < $(UNPACKAGE) endif ifeq ($(MOZ_PKG_FORMAT),TGZ) PKG_SUFFIX = .tar.gz INNER_MAKE_PACKAGE = cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) | gzip -vf9 > $(PACKAGE) - INNER_UNMAKE_PACKAGE = cd $(1) && gunzip -c $(UNPACKAGE) | $(UNPACK_TAR) endif ifeq ($(MOZ_PKG_FORMAT),XZ) PKG_SUFFIX = .tar.xz INNER_MAKE_PACKAGE = cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) | xz --compress --stdout -9 --extreme > $(PACKAGE) - INNER_UNMAKE_PACKAGE = cd $(1) && xz --decompress --stdout $(UNPACKAGE) | $(UNPACK_TAR) endif ifeq ($(MOZ_PKG_FORMAT),BZ2) @@ -120,13 +116,11 @@ ifeq ($(MOZ_PKG_FORMAT),BZ2) else INNER_MAKE_PACKAGE = cd $(1) && $(CREATE_FINAL_TAR) - $(MOZ_PKG_DIR) | bzip2 -vf > $(PACKAGE) endif - INNER_UNMAKE_PACKAGE = cd $(1) && bunzip2 -c $(UNPACKAGE) | $(UNPACK_TAR) endif ifeq ($(MOZ_PKG_FORMAT),ZIP) PKG_SUFFIX = .zip INNER_MAKE_PACKAGE = $(call py_action,zip,'$(PACKAGE)' '$(MOZ_PKG_DIR)' -x '**/.mkdir.done',$(1)) - INNER_UNMAKE_PACKAGE = $(call py_action,make_unzip,$(UNPACKAGE),$(1)) endif #Create an RPM file @@ -199,15 +193,11 @@ ifeq ($(MOZ_PKG_FORMAT),RPM) endif INNER_MAKE_PACKAGE = cd $(1) && $(RPM_CMD) - #Avoiding rpm repacks, going to try creating/uploading xpi in rpm files instead - INNER_UNMAKE_PACKAGE = $(error Try using rpm2cpio and cpio) - endif #Create an RPM file ifeq ($(MOZ_PKG_FORMAT),APK) INNER_MAKE_PACKAGE = true -INNER_UNMAKE_PACKAGE = true endif ifeq ($(MOZ_PKG_FORMAT),DMG) @@ -215,6 +205,9 @@ ifeq ($(MOZ_PKG_FORMAT),DMG) _ABS_MOZSRCDIR = $(shell cd $(MOZILLA_DIR) && pwd) PKG_DMG_SOURCE = $(MOZ_PKG_DIR) + MOZ_PKG_MAC_DSSTORE=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/dsstore + MOZ_PKG_MAC_BACKGROUND=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/background.png + MOZ_PKG_MAC_ICON=$(topsrcdir)/$(MOZ_BRANDING_DIRECTORY)/disk.icns INNER_MAKE_PACKAGE = \ $(call py_action,make_dmg, \ $(if $(MOZ_PKG_MAC_DSSTORE),--dsstore '$(MOZ_PKG_MAC_DSSTORE)') \ @@ -223,13 +216,6 @@ ifeq ($(MOZ_PKG_FORMAT),DMG) --volume-name '$(MOZ_APP_DISPLAYNAME)' \ '$(PKG_DMG_SOURCE)' '$(PACKAGE)', \ $(1)) - INNER_UNMAKE_PACKAGE = \ - $(call py_action,unpack_dmg, \ - $(if $(MOZ_PKG_MAC_DSSTORE),--dsstore '$(MOZ_PKG_MAC_DSSTORE)') \ - $(if $(MOZ_PKG_MAC_BACKGROUND),--background '$(MOZ_PKG_MAC_BACKGROUND)') \ - $(if $(MOZ_PKG_MAC_ICON),--icon '$(MOZ_PKG_MAC_ICON)') \ - $(UNPACKAGE) $(MOZ_PKG_DIR), \ - $(1)) endif MAKE_PACKAGE = $(INNER_MAKE_PACKAGE)