diff --git a/.gitignore b/.gitignore index 4ad0724dc14d..7fdb280335e4 100644 --- a/.gitignore +++ b/.gitignore @@ -158,9 +158,6 @@ tps_result.json # Ignore files created when running a reftest. lextab.py -# tup database -/.tup - # Ignore Visual Studio Code workspace files. .vscode/ !.vscode/extensions.json diff --git a/.hgignore b/.hgignore index 0f1da5c84198..5ef1290e1a35 100644 --- a/.hgignore +++ b/.hgignore @@ -180,9 +180,6 @@ _OPT\.OBJ/ # Ignore files created when running a reftest. ^lextab.py$ -# tup database -^\.tup - # Ignore sync tps logs and reports tps\.log tps_result\.json diff --git a/browser/config/mozconfigs/linux64/tup b/browser/config/mozconfigs/linux64/tup deleted file mode 100644 index 22ab99be7025..000000000000 --- a/browser/config/mozconfigs/linux64/tup +++ /dev/null @@ -1,16 +0,0 @@ -MOZ_AUTOMATION_BUILD_SYMBOLS=0 -MOZ_AUTOMATION_PACKAGE=0 -MOZ_AUTOMATION_UPLOAD=0 -MOZ_AUTOMATION_UPLOAD_SYMBOLS=0 - -TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir} -export TUP=${MOZ_FETCHES_DIR}/tup/tup - -. "$topsrcdir/browser/config/mozconfigs/linux64/common-opt" - -ac_add_options --enable-build-backends=Tup -unset ENABLE_CLANG_PLUGIN -# To enable the option to upload the tup database, uncomment the line below -# ac_add_options --upload-tup-db - -. "$topsrcdir/build/mozconfig.common.override" diff --git a/build/docs/index.rst b/build/docs/index.rst index c6d5ea283b59..07edbf873cdb 100644 --- a/build/docs/index.rst +++ b/build/docs/index.rst @@ -16,7 +16,6 @@ Important Concepts files-metadata Profile Guided Optimization slow - tup environment-variables build-targets python diff --git a/build/docs/slow.rst b/build/docs/slow.rst index e8bb124399ac..b8ea1d2430ba 100644 --- a/build/docs/slow.rst +++ b/build/docs/slow.rst @@ -133,16 +133,14 @@ no-op build is spent in make traversal. make is inefficient =================== -Compared to modern build backends like Tup or Ninja, make is slow and -inefficient. We can only make make so fast. At some point, we'll hit a +Compared to modern build backends like Tup or Ninja, `make` is slow and +inefficient. We can only make `make` so fast. At some point, we'll hit a performance plateau and will need to use a different tool to make builds faster. -Please note that clobber and incremental builds are different. A clobber -build with make will likely be as fast as a clobber build with e.g. Tup. -However, Tup should vastly outperform make when it comes to incremental -builds. Therefore, this issue is mostly seen when performing incremental -builds. For more information, see :ref:`tup`. +Please note that clobber and incremental builds are different. A clobber build +with `make` will likely be as fast as a clobber build with a modern build +system. C++ header dependency hell ========================== diff --git a/build/docs/tup.rst b/build/docs/tup.rst deleted file mode 100644 index d0fc94c7f096..000000000000 --- a/build/docs/tup.rst +++ /dev/null @@ -1,170 +0,0 @@ -.. _tup: - -=========== -Tup Backend -=========== - -The Tup build backend is an alternative to the default Make backend. The `Tup -build system `_ is designed for fast and correct -incremental builds. A top-level no-op build should be under 2 seconds, and -clobbers should rarely be required. It is currently only available for Linux -Desktop builds -- other platforms like Windows or OSX are planned for the -future. - -As part of the mozbuild architecture, the Tup backend shares a significant -portion of frontend (developer-facing) code in the build system. When using the -Tup backend, ``./mach build`` is still the entry point to run the build system, -and moz.build files are still used for the build description. Familiar parts of -the build system like configure and generating the build files (the -``Reticulating splines...`` step) are virtually identical in both backends. The -difference is that ``mach`` invokes Tup instead of Make under the hood to do -the actual work of determining what needs to be rebuilt. Tup is able to perform -this work more efficiently by loading only the parts of the DAG that are -required for an incremental build. Additionally, Tup instruments processes to -see what files are read from and written to in order to verify that -dependencies are correct. - -For more detailed information on the rationale behind Tup, see the `Build -System Rules and Algorithms -`_ paper. - -Installation -============ - -You'll need to install the Tup executable, as well as the nightly rust/cargo -toolchain (Note: Replace $topsrcdir with the path to your mozilla-central -source tree):: - - cd ~/.mozbuild && $topsrcdir/mach artifact toolchain --from-build linux64-tup - rustup install nightly - rustup default nightly - -Configuration -============= - -Your mozconfig needs to describe how to find the executable if it's not in your -PATH, and enable the Tup backend:: - - export TUP=~/.mozbuild/tup/tup - ac_add_options --enable-build-backends=Tup - -Configuring Parallel Jobs -------------------------- - -To override the default number of jobs run in parallel, set MOZ_PARALLEL_BUILD -in your mozconfig:: - - mk_add_options MOZ_PARALLEL_BUILD=8 - -What Works -========== - -You should expect a Linux desktop build to generate a working Firefox binary -from a ``./mach build``, and be able to run test suites against it (eg: -mochitests, xpcshell, gtest). Top-level incremental builds should be fast -enough to use them during a regular compile/edit/test cycle. If you wish to -stop compilation partway through the build to more quickly iterate on a -particular file, you can expect ``./mach build objdir/path/to/file.o`` to -correctly produce all inputs required to build file.o before compiling it. For -example, you don't have to run the build system in various subdirectories to -get generated headers built in the right order. - -Currently Unsupported / Future Work -=================================== - -There are a number of features that you may use in the Make backend that are -currently unsupported for the Tup backend. We plan to add support for these in -the future according to developer demand and build team availability. - -* sccache - This is currently under active development to support icecream-like - functionality, which likely impacts the same parts that would affect Tup's - dependency checking mechanisms. Note that icecream itself should work with - Tup. - -* Incremental Rust compilation - see `bug 1468527 `_ - -* Watchman integration - This will allow Tup to skip the initial ``Scanning - filesystem...`` step, saving 1-2 seconds of startup overhead. - -* More platform support (Windows, OSX, etc.) - -* Packaging in automation - This is still quite intertwined with Makefiles - -* Tests in automation - Requires packaging - -Multiple object directories and object directories outside of the source tree -============================================================================= - -Common workflows involving multiple object directories which may be outside of -the source directory are supported by Tup, however there are some things to -consider when using these configurations. - -* Using multiple object directories works as expected, however you may find - pulling and attempting to build will fail due to tup attempting to - parse a Tupfile in an object directory other than the active object - directory. As a workaround, activate the object directory containing the - failing file and run configure before proceeding. -* If the currently active object directory is located outside of the source - directory, the tup backend will prompt the user to run ``tup init --no-sync`` - in a common ancestor directory of the source directory and object directory. - If this ancestor contains too many files (it's the home directory, for - instance), this will slow down tup's initial file scan. Anecdotally multiple - object directories will only incur marginal scanning overhead, but care - should be exercised when choosing a directory layout. - -Tup builds in automation -======================== - -Tup builds run on integration branches as ``Btup`` in treeherder. There are -some aspects of the Tup builds that are currently implemented outside of the -make build system, and divergences may cause the ``Btup`` job to fail until -the build is completely integrated. There are two known situations this has -come up. - -* Changes to the xpidl/webidl/ipdl build - these are given special treatment - by the make backend that is re-created in the tup backend. An update to a - Makefile implementing one of these parts of the build may require a - corresponding change to ``python/mozbuild/mozbuild/backend/tup.py`` -* Build scripts generating new outputs - due to the implementation of Tup, - outputs from ``build.rs`` that run during the build must be known to Tup - before the build starts. The current outputs are enumerated in - ``python/mozbuild/mozbuild/backend/cargo_build_defs.py``. Modifying the set - of outputs from a build script will require a corresponding update to this - file. - - -Partial tree builds in tup -========================== - -Partial tree builds are possible in tup by invoking -``./mach build ``, however the behavior differs from the -make backend. A partial tree build will result in running the commands in -Tupfiles in the specified subtree, building ``.o`` files and other outputs, but -unlike make it will take changed dependencies into account and build everything -necessary to update those outputs as well. Also unlike make it will not -attempt to find downstream commands that depend on these files, i.e. -programs and libraries from other parts of the tree will not be linked. A -top level incremental build should be performed to run all commands that depend -on changes to the local tree. - -How to Contribute -================= - -At the moment we're looking for early adopters who are developing on the Linux -desktop to try out the Tup backend, and share your experiences with the build -team (see `Contact`_). - - * Are there particular issues or missing features that prevent you from using - the Tup backend at this time? - - * Do you find that top-level incremental builds are fast enough to use for - every build invocation? - - * Have you needed to perform a clobber build to fix an issue? - -Contact -======== - -If you have any issues, feel free to file a bug blocking `buildtup -`_, or contact mshal or -chmanchester in #build on IRC. diff --git a/build/moz.configure/tup.configure b/build/moz.configure/tup.configure deleted file mode 100644 index 8d702f8fcdfe..000000000000 --- a/build/moz.configure/tup.configure +++ /dev/null @@ -1,76 +0,0 @@ -# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# tup detection -# ============================================================== - -tup = check_prog('TUP', ['tup']) - -@depends(tup) -@checking('for tup version') -@imports('re') -def tup_version(tup): - tup_min_version = '0.7.8' - out = check_cmd_output(tup, '--version', - onerror=lambda: die('Failed to get tup version')) - m = re.search(r'tup v?([0-9]+\.[0-9]+\.[0-9]+)', out) - - if not m: - raise FatalCheckError('Unknown version of tup: %s' % out) - ver = Version(m.group(1)) - - if ver < tup_min_version: - raise FatalCheckError('To use the tup backend you must have tup version ' - '%s or greater in your path' % tup_min_version) - return ver - -@depends(tup) -@checking('for tup ldpreload dependency checker') -def tup_is_ldpreload(tup): - out = check_cmd_output(tup, 'server', - onerror=lambda: die('Failed to get tup dependency checker')) - if out.rstrip() != 'ldpreload': - raise FatalCheckError('To use the tup backend, please use a version ' - 'of tup compiled with the ldpreload dependency ' - 'checker. Either compile tup locally with ' - 'CONFIG_TUP_SERVER=ldpreload in your tup.config ' - 'file, or use the version from the toolchain ' - 'task via |./mach artifact toolchain ' - '--from-build linux64-tup|') - return True - -@depends(tup, using_sccache) -def tup_and_sccache(tup, using_sccache): - if tup and using_sccache: - die('Cannot use sccache with tup yet. Please disable sccache or use ' - 'the make backend until it is supported.') - -@depends(tup, cargo) -@imports('re') -def check_tup_cargo_channel(tup, cargo): - channel = None - out = check_cmd_output(cargo, '--version') - VERSION_FORMAT = r'cargo \d\.\d+\.\d+-(\S+)' - m = re.match(VERSION_FORMAT, out) - if m: - channel = m.group(1) - if tup and channel != 'nightly': - die('Nightly Cargo is required when building with Tup.') - -@depends(tup, target, build_project) -def tup_and_non_linux(tup, target, build_project): - if tup and (target.kernel != 'Linux' or build_project != 'browser'): - die('The tup backend can only be used to build the browser on Linux. ' - 'Use the make backend until your target is supported.') - -option('--upload-tup-db', help= 'Upload the tup database from an automated build.') - -@depends('--upload-tup-db') -def upload_tdb(value): - if value: - return True - -set_config('UPLOAD_TUP_DB', upload_tdb) diff --git a/moz.configure b/moz.configure index e1752d6f676a..b80a48b72890 100755 --- a/moz.configure +++ b/moz.configure @@ -448,21 +448,6 @@ def possible_makes(make, host): check_prog('GMAKE', possible_makes) -@depends(build_backends, build_project) -def tup_include(build_backends, build_project): - # We need to check the rustc version when building with tup, but - # rustc_info isn't available when configuring js (and build_backends isn't - # available from project-specific configure), so as a workaround we only - # include the file when we know we'll need it. This can be removed when - # we globally require a rustc recent enough to build with tup. - if build_project not in ('browser', 'mobile/android'): - return None - for backend in build_backends: - if 'Tup' in backend: - return 'build/moz.configure/tup.configure' - -include(tup_include) - # watchman detection # ============================================================== diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml index b00c18f70ca6..83feabf9e865 100644 --- a/taskcluster/ci/build/linux.yml +++ b/taskcluster/ci/build/linux.yml @@ -1241,47 +1241,6 @@ linux64-rusttests/debug: - linux64-lucetc - wasi-sysroot -linux64-tup/opt: - description: "Linux64 Tup" - index: - product: firefox - job-name: linux64-tup-opt - treeherder: - platform: linux64/opt - symbol: Btup - tier: 3 - worker-type: b-linux - worker: - max-run-time: 3600 - env: - PERFHERDER_EXTRA_OPTIONS: tup - RUSTC_BOOTSTRAP: '1' - run: - using: mozharness - actions: [get-secrets, build] - config: - - builds/releng_base_firefox.py - - builds/releng_base_linux_64_builds.py - script: "mozharness/scripts/fx_desktop_build.py" - secrets: true - custom-build-variant-cfg: tup - mozconfig-variant: tup - tooltool-downloads: public - need-xvfb: true - run-on-projects: [] - fetches: - toolchain: - - linux64-binutils - - linux64-clang - - linux64-rust - - linux64-cbindgen - - linux64-sccache - - linux64-tup - - linux64-nasm - - linux64-node - - linux64-lucetc - - wasi-sysroot - linux64-ccov/opt: description: "Linux64-CCov Opt" index: diff --git a/taskcluster/ci/fetch/toolchains.yml b/taskcluster/ci/fetch/toolchains.yml index 1c2fbd33616a..f7b6ea95b6fd 100644 --- a/taskcluster/ci/fetch/toolchains.yml +++ b/taskcluster/ci/fetch/toolchains.yml @@ -324,13 +324,6 @@ llvm-for-dsymutil: repo: https://github.com/llvm/llvm-project revision: 3b7811f6441be13c9f613f81ef93297d231b4f8e -tup: - description: tup source code - fetch: - type: git - repo: https://github.com/gittup/tup - revision: 4371a41ba4ece660bf060b598b9eee4c2eb347d8 - rust-size: description: rust-size source code fetch: diff --git a/taskcluster/ci/toolchain/misc.yml b/taskcluster/ci/toolchain/misc.yml index 05db59278941..b3a75c9d0359 100644 --- a/taskcluster/ci/toolchain/misc.yml +++ b/taskcluster/ci/toolchain/misc.yml @@ -100,24 +100,6 @@ linux64-mar-tools: - toolkit/mozapps/update/updater/bspatch/ - tools/update-packaging/ -linux64-tup: - description: "tup toolchain build" - treeherder: - symbol: TL(tup) - worker: - max-run-time: 3600 - run: - script: build-tup-linux.sh - toolchain-artifact: public/build/tup.tar.xz - run-on-projects: - - trunk - - try - fetches: - fetch: - - tup - toolchain: - - linux64-gcc-7 - linux64-upx: description: "UPX build" treeherder: diff --git a/taskcluster/scripts/misc/build-tup-linux.sh b/taskcluster/scripts/misc/build-tup-linux.sh deleted file mode 100755 index eac7962e4a40..000000000000 --- a/taskcluster/scripts/misc/build-tup-linux.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -e -v - -# This script is for building tup on Linux. - -COMPRESS_EXT=xz -export PATH=$MOZ_FETCHES_DIR/gcc/bin:$PATH - -cd $MOZ_FETCHES_DIR/tup - -patch -p1 <<'EOF' -diff --git a/src/tup/link.sh b/src/tup/link.sh -index ed9a01c6..4ecc6d7d 100755 ---- a/src/tup/link.sh -+++ b/src/tup/link.sh -@@ -4,7 +4,7 @@ - # linking, so that the version is updated whenever we change anything that - # affects the tup binary. This used to live in the Tupfile, but to support - # Windows local builds we need to make it an explicit shell script. --version=`git describe` -+version=unknown - CC=$1 - CFLAGS=$2 - LDFLAGS=$3 -EOF - -(echo 'CONFIG_TUP_SERVER=ldpreload'; echo 'CONFIG_TUP_USE_SYSTEM_PCRE=n') > tup.config -./bootstrap-ldpreload.sh -cd .. -tar caf tup.tar.xz tup/tup tup/tup-ldpreload.so tup/tup.1 -cp tup.tar.xz $UPLOAD_DIR diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tup.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tup.py deleted file mode 100644 index 3ea8f58f9648..000000000000 --- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tup.py +++ /dev/null @@ -1,24 +0,0 @@ -import os - -config = { - 'stage_platform': 'linux64-tup-opt', - 'env': { - 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'), - 'DISPLAY': ':2', - 'HG_SHARE_BASE_DIR': '/builds/hg-shared', - 'MOZ_OBJDIR': '%(abs_obj_dir)s', - 'MOZ_CRASHREPORTER_NO_REPORT': '1', - 'LC_ALL': 'C', - 'XPCOM_DEBUG_BREAK': 'stack-and-abort', - # 64 bit specific - 'PATH': '/usr/local/bin:/bin:\ -/usr/bin:/usr/local/sbin:/usr/sbin:/sbin', - 'LD_LIBRARY_PATH': '%(abs_obj_dir)s/dist/bin', - 'TINDERBOX_OUTPUT': '1', - - # sccache doesn't work yet with tup due to the way the server is - # spawned, and because the server does the file I/O - 'SCCACHE_DISABLE': '1', - }, - 'disable_package_metrics': True, # TODO: the package needs to be created for this to work -} diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 3f0a8c6c5ce4..9b0b96f32427 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -314,7 +314,6 @@ class BuildOptionParser(object): 'aarch64-debug': 'builds/releng_sub_%s_configs/%s_aarch64_debug.py', 'android-geckoview-docs': 'builds/releng_sub_%s_configs/%s_geckoview_docs.py', 'valgrind': 'builds/releng_sub_%s_configs/%s_valgrind.py', - 'tup': 'builds/releng_sub_%s_configs/%s_tup.py', } build_pool_cfg_file = 'builds/build_pool_specifics.py'