Bug 1580028 - Always merge PGO profile data in the run task; r=firefox-build-system-reviewers,chmanchester

If the run task generates bad profile data, the merge step in the
profile-use task will fail. However, retrying the profile-use task
doesn't fix the problem, and there isn't a straightforward way to retry
the run task in this situation. Instead we can add a clang toolchain to
all the run tasks, and perform the merge there.

This means the output from the run task will always be a successfully
merged file called 'merged.profdata', and we no longer need to perform
the merge as part of the profile-use build as a GENERATED_FILES step.

Depends on D45262

Differential Revision: https://phabricator.services.mozilla.com/D45263

--HG--
extra : moz-landing-system : lando

build/merge_profdata.py

@@ -1,18 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import glob
-import subprocess
-import sys
-import buildconfig
-
-
-def main(_, profile_dir):
-    profraw_files = glob.glob(profile_dir + '/*.profraw')
-    if not profraw_files:
-        print('Could not find any profraw files in ' + profile_dir)
-        sys.exit(1)
-
-    subprocess.check_call([buildconfig.substs['LLVM_PROFDATA'], 'merge',
-                           '-o', 'merged.profdata'] + profraw_files)

build/moz.configure/lto-pgo.configure

@@ -43,17 +43,17 @@ set_config('MOZ_PROFILE_USE',
            depends_if('--enable-profile-use')(lambda _: True))
 
 
-@depends('--with-pgo-profile-path', '--enable-profile-use', llvm_profdata)
+@depends('--with-pgo-profile-path', '--enable-profile-use', llvm_profdata, check_build_environment)
 @imports('os')
-def pgo_profile_path(path, pgo_use, profdata):
+def pgo_profile_path(path, pgo_use, profdata, build_env):
     if not path:
-        return
+        return os.path.join(build_env.topobjdir, 'merged.profdata')
     if path and not pgo_use:
         die('Pass --enable-profile-use to use --with-pgo-profile-path.')
     if path and not profdata:
         die('LLVM_PROFDATA must be set to process the pgo profile.')
-    if not os.path.isdir(path[0]):
+    if not os.path.isfile(path[0]):
-        die('Argument to --with-pgo-profile-path must be a directory.')
+        die('Argument to --with-pgo-profile-path must be a file.')
     if not os.path.isabs(path[0]):
         die('Argument to --with-pgo-profile-path must be an absolute path.')
     return path[0]
@@ -61,10 +61,10 @@ def pgo_profile_path(path, pgo_use, profdata):
 
 set_config('PGO_PROFILE_PATH', pgo_profile_path)
 
-@depends(c_compiler, check_build_environment, target)
+@depends(c_compiler, check_build_environment, target, pgo_profile_path)
 @imports('multiprocessing')
 @imports(_from='__builtin__', _import='min')
-def pgo_flags(compiler, build_env, target):
+def pgo_flags(compiler, build_env, target, profdata):
     topobjdir = build_env.topobjdir
     if topobjdir.endswith('/js/src'):
         topobjdir = topobjdir[:-7]
@@ -79,7 +79,6 @@ def pgo_flags(compiler, build_env, target):
         )
 
     if compiler.type in ('clang-cl', 'clang'):
-        profdata = os.path.join(topobjdir, 'merged.profdata')
         prefix = ''
         if compiler.type == 'clang-cl':
             prefix = '/clang:'

build/mozconfig.win-common

@@ -10,6 +10,6 @@ if [ -z "$USE_ARTIFACT" ]; then
         export MOZ_LTO=cross
         ac_add_options --enable-profile-use
         ac_add_options --with-pgo-jarlog="${WORKSPACE}/fetches/en-US.log"
-        ac_add_options --with-pgo-profile-path="${WORKSPACE}/fetches"
+        ac_add_options --with-pgo-profile-path="${WORKSPACE}/fetches/merged.profdata"
     fi
 fi

build/pgo/profileserver.py

@@ -7,6 +7,8 @@
 import json
 import os
 import sys
+import glob
+import subprocess
 
 from mozbuild.base import MozbuildObject
 from mozfile import TemporaryDirectory
@@ -139,3 +141,21 @@ if __name__ == '__main__':
                 with open(logfile) as f:
                     print(f.read())
             sys.exit(ret)
+
+        llvm_profdata = env.get('LLVM_PROFDATA')
+        if llvm_profdata:
+            profraw_files = glob.glob('*.profraw')
+            if not profraw_files:
+                print('Could not find profraw files in the current directory: %s' % os.getcwd())
+                sys.exit(1)
+            merge_cmd = [
+                llvm_profdata,
+                'merge',
+                '-o',
+                'merged.profdata',
+            ] + profraw_files
+            rc = subprocess.call(merge_cmd)
+            if rc != 0:
+                print('INFRA-ERROR: Failed to merge profile data. Corrupt profile?')
+                # exit with TBPL_RETRY
+                sys.exit(4)

build/unix/mozconfig.unix

@@ -30,7 +30,7 @@ else
             if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
                 ac_add_options --enable-profile-use=cross
                 ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
-                ac_add_options --with-pgo-profile-path=/builds/worker/fetches
+                ac_add_options --with-pgo-profile-path=/builds/worker/fetches/merged.profdata
             fi
         fi
         export LLVM_PROFDATA="$MOZ_FETCHES_DIR/clang/bin/llvm-profdata"

config/makefiles/rust.mk

@@ -136,7 +136,7 @@ target_rust_ltoable := force-cargo-library-build
 target_rust_nonltoable := force-cargo-test-run force-cargo-library-check $(foreach b,build check,force-cargo-program-$(b))
 
 ifdef MOZ_PGO_RUST
-rust_pgo_flags := $(if $(MOZ_PROFILE_GENERATE),-C profile-generate=$(topobjdir)) $(if $(MOZ_PROFILE_USE),-C profile-use=$(topobjdir)/merged.profdata)
+rust_pgo_flags := $(if $(MOZ_PROFILE_GENERATE),-C profile-generate=$(topobjdir)) $(if $(MOZ_PROFILE_USE),-C profile-use=$(PGO_PROFILE_PATH))
 endif
 
 $(target_rust_ltoable): RUSTFLAGS:=$(rustflags_override) $(RUSTFLAGS) $(if $(MOZ_LTO_RUST),-Clinker-plugin-lto) $(rust_pgo_flags)

mobile/android/config/mozconfigs/common

@@ -100,7 +100,7 @@ if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
     # This is disabled because jarlog re-ordering breaks apk publishing tasks,
     # see bug 1539933.
     # ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
-    ac_add_options --with-pgo-profile-path=/builds/worker/fetches
+    ac_add_options --with-pgo-profile-path=/builds/worker/fetches/merged.profdata
 fi
 
 # Package js shell.

moz.build

@@ -153,16 +153,6 @@ if not CONFIG['JS_STANDALONE'] or not CONFIG['MOZ_BUILD_APP']:
         'build',
     ]
 
-if CONFIG['PGO_PROFILE_PATH']:
-    profdata_gen = ('merged.profdata.stub', 'merged.profdata')
-    GENERATED_FILES += [
-        profdata_gen
-    ]
-    GENERATED_FILES[profdata_gen].script = 'build/merge_profdata.py'
-    GENERATED_FILES[profdata_gen].inputs = [
-        '/' + CONFIG['PGO_PROFILE_PATH'],
-    ]
-
 DIRS += [
     'mfbt',
 ]

taskcluster/ci/generate-profile/kind.yml

@@ -56,6 +56,9 @@ jobs:
             cwd: '{checkout}'
             command: >
                 ./taskcluster/scripts/misc/run-profileserver.sh
+        fetches:
+            toolchain:
+                - linux64-clang
 
     linux64-shippable/opt:
         description: "Linux64 Profile Generation"
@@ -90,6 +93,9 @@ jobs:
             cwd: '{checkout}'
             command: >
                 ./taskcluster/scripts/misc/run-profileserver.sh
+        fetches:
+            toolchain:
+                - linux64-clang
 
     android-api-16/pgo:
         description: "Android 4.0 api-16+ Profile Generation"
@@ -159,6 +165,9 @@ jobs:
             sparse-profile: profile-generate
             cwd: '{checkout}'
             command: ./taskcluster/scripts/misc/run-profileserver.sh
+        fetches:
+            toolchain:
+                - win64-clang-cl
 
     win64-shippable/opt:
         description: "Win64 Profile Generation"
@@ -189,3 +198,6 @@ jobs:
             sparse-profile: profile-generate
             cwd: '{checkout}'
             command: ./taskcluster/scripts/misc/run-profileserver.sh
+        fetches:
+            toolchain:
+                - win64-clang-cl

taskcluster/scripts/misc/run-profileserver.sh

@@ -14,6 +14,7 @@ export UPLOAD_PATH
 
 PGO_RUNDIR=obj-firefox/dist
 export JARLOG_FILE="en-US.log"
+export LLVM_PROFDATA=$MOZ_FETCHES_DIR/clang/bin/llvm-profdata
 
 set -v
 
@@ -38,10 +39,4 @@ mkdir -p $UPLOAD_PATH
 mv $MOZ_FETCHES_DIR/firefox $PGO_RUNDIR
 ./mach python build/pgo/profileserver.py --binary $PGO_RUNDIR/firefox/firefox
 
-# Fail the build if for some reason we didn't collect any profile data.
+tar -acvf $UPLOAD_PATH/profdata.tar.xz merged.profdata en-US.log
-if test -z "$(find . -maxdepth 1 -name '*.profraw' -print -quit)"; then
-    echo "ERROR: no profile data produced"
-    exit 1
-fi
-
-tar -acvf $UPLOAD_PATH/profdata.tar.xz *.profraw en-US.log

testing/mozharness/scripts/android_emulator_pgo.py

@@ -270,10 +270,6 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin,
             self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError',
                        EXIT_STATUS_DICT[TBPL_RETRY])
 
-        # We normally merge as part of a GENERATED_FILES step in the profile-use
-        # build, but Android runs sometimes result in a truncated profile. We do
-        # a merge here to make sure the data isn't corrupt so we can retry the
-        # 'run' task if necessary.
         profraw_files = glob.glob('/builds/worker/workspace/*.profraw')
         if not profraw_files:
             self.fatal('Could not find any profraw files in /builds/worker/workspace')
@@ -281,7 +277,7 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin,
             os.path.join(os.environ['MOZ_FETCHES_DIR'], 'clang/bin/llvm-profdata'),
             'merge',
             '-o',
-            '/builds/worker/workspace/merged.profraw',
+            '/builds/worker/workspace/merged.profdata',
         ] + profraw_files
         rc = subprocess.call(merge_cmd)
         if rc != 0:
@@ -294,7 +290,7 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin,
             '-acvf',
             '/builds/worker/artifacts/profdata.tar.xz',
             '-C', '/builds/worker/workspace',
-            'merged.profraw',
+            'merged.profdata',
             'en-US.log',
         ]
         subprocess.check_call(tar_cmd)