nagai/fune
3
0
Fork 0
forked from mirrors/gecko-dev
Code Pull requests Activity

Bug 1580028 - Always merge PGO profile data in the run task; r=firefox-build-system-reviewers,chmanchester

Browse source 
If the run task generates bad profile data, the merge step in the
profile-use task will fail. However, retrying the profile-use task
doesn't fix the problem, and there isn't a straightforward way to retry
the run task in this situation. Instead we can add a clang toolchain to
all the run tasks, and perform the merge there.

This means the output from the run task will always be a successfully
merged file called 'merged.profdata', and we no longer need to perform
the merge as part of the profile-use build as a GENERATED_FILES step.

Depends on D45262

Differential Revision: https://phabricator.services.mozilla.com/D45263

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Mike Shal 2019-09-10 21:56:15 +00:00
parent 93f80b9e53
commit 3b8df1c395
11 changed files with 47 additions and 53 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

18
build/merge_profdata.py
View file

@ -1,18 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import glob
import subprocess
import sys
import buildconfig
def main(_, profile_dir):
profraw_files = glob.glob(profile_dir + '/*.profraw')
if not profraw_files:
print('Could not find any profraw files in ' + profile_dir)
sys.exit(1)
subprocess.check_call([buildconfig.substs['LLVM_PROFDATA'], 'merge',
'-o', 'merged.profdata'] + profraw_files)

15
build/moz.configure/lto-pgo.configure
View file

@ -43,17 +43,17 @@ set_config('MOZ_PROFILE_USE',
depends_if('--enable-profile-use')(lambda _: True))
@depends('--with-pgo-profile-path', '--enable-profile-use', llvm_profdata)
@depends('--with-pgo-profile-path', '--enable-profile-use', llvm_profdata, check_build_environment)
@imports('os')
def pgo_profile_path(path, pgo_use, profdata):
def pgo_profile_path(path, pgo_use, profdata, build_env):
if not path:
return
return os.path.join(build_env.topobjdir, 'merged.profdata')
if path and not pgo_use:
die('Pass --enable-profile-use to use --with-pgo-profile-path.')
if path and not profdata:
die('LLVM_PROFDATA must be set to process the pgo profile.')
if not os.path.isdir(path[0]):
die('Argument to --with-pgo-profile-path must be a directory.')
if not os.path.isfile(path[0]):
die('Argument to --with-pgo-profile-path must be a file.')
if not os.path.isabs(path[0]):
die('Argument to --with-pgo-profile-path must be an absolute path.')
return path[0]
@ -61,10 +61,10 @@ def pgo_profile_path(path, pgo_use, profdata):
set_config('PGO_PROFILE_PATH', pgo_profile_path)
@depends(c_compiler, check_build_environment, target)
@depends(c_compiler, check_build_environment, target, pgo_profile_path)
@imports('multiprocessing')
@imports(_from='__builtin__', _import='min')
def pgo_flags(compiler, build_env, target):
def pgo_flags(compiler, build_env, target, profdata):
topobjdir = build_env.topobjdir
if topobjdir.endswith('/js/src'):
topobjdir = topobjdir[:-7]
@ -79,7 +79,6 @@ def pgo_flags(compiler, build_env, target):
)
if compiler.type in ('clang-cl', 'clang'):
profdata = os.path.join(topobjdir, 'merged.profdata')
prefix = ''
if compiler.type == 'clang-cl':
prefix = '/clang:'

2
build/mozconfig.win-common
View file

@ -10,6 +10,6 @@ if [ -z "$USE_ARTIFACT" ]; then
export MOZ_LTO=cross
ac_add_options --enable-profile-use
ac_add_options --with-pgo-jarlog="${WORKSPACE}/fetches/en-US.log"
ac_add_options --with-pgo-profile-path="${WORKSPACE}/fetches"
ac_add_options --with-pgo-profile-path="${WORKSPACE}/fetches/merged.profdata"
fi
fi

20
build/pgo/profileserver.py
View file

@ -7,6 +7,8 @@
import json
import os
import sys
import glob
import subprocess
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
@ -139,3 +141,21 @@ if __name__ == '__main__':
with open(logfile) as f:
print(f.read())
sys.exit(ret)
llvm_profdata = env.get('LLVM_PROFDATA')
if llvm_profdata:
profraw_files = glob.glob('*.profraw')
if not profraw_files:
print('Could not find profraw files in the current directory: %s' % os.getcwd())
sys.exit(1)
merge_cmd = [
llvm_profdata,
'merge',
'-o',
'merged.profdata',
] + profraw_files
rc = subprocess.call(merge_cmd)
if rc != 0:
print('INFRA-ERROR: Failed to merge profile data. Corrupt profile?')
# exit with TBPL_RETRY
sys.exit(4)

2
build/unix/mozconfig.unix
View file

@ -30,7 +30,7 @@ else
if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
ac_add_options --enable-profile-use=cross
ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
ac_add_options --with-pgo-profile-path=/builds/worker/fetches
ac_add_options --with-pgo-profile-path=/builds/worker/fetches/merged.profdata
fi
fi
export LLVM_PROFDATA="$MOZ_FETCHES_DIR/clang/bin/llvm-profdata"

2
config/makefiles/rust.mk
View file

@ -136,7 +136,7 @@ target_rust_ltoable := force-cargo-library-build
target_rust_nonltoable := force-cargo-test-run force-cargo-library-check $(foreach b,build check,force-cargo-program-$(b))
ifdef MOZ_PGO_RUST
rust_pgo_flags := $(if $(MOZ_PROFILE_GENERATE),-C profile-generate=$(topobjdir)) $(if $(MOZ_PROFILE_USE),-C profile-use=$(topobjdir)/merged.profdata)
rust_pgo_flags := $(if $(MOZ_PROFILE_GENERATE),-C profile-generate=$(topobjdir)) $(if $(MOZ_PROFILE_USE),-C profile-use=$(PGO_PROFILE_PATH))
endif
$(target_rust_ltoable): RUSTFLAGS:=$(rustflags_override) $(RUSTFLAGS) $(if $(MOZ_LTO_RUST),-Clinker-plugin-lto) $(rust_pgo_flags)

2
mobile/android/config/mozconfigs/common
View file

@ -100,7 +100,7 @@ if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
# This is disabled because jarlog re-ordering breaks apk publishing tasks,
# see bug 1539933.
# ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
ac_add_options --with-pgo-profile-path=/builds/worker/fetches
ac_add_options --with-pgo-profile-path=/builds/worker/fetches/merged.profdata
fi
# Package js shell.

10
moz.build
View file

@ -153,16 +153,6 @@ if not CONFIG['JS_STANDALONE'] or not CONFIG['MOZ_BUILD_APP']:
'build',
]
if CONFIG['PGO_PROFILE_PATH']:
profdata_gen = ('merged.profdata.stub', 'merged.profdata')
GENERATED_FILES += [
profdata_gen
]
GENERATED_FILES[profdata_gen].script = 'build/merge_profdata.py'
GENERATED_FILES[profdata_gen].inputs = [
'/' + CONFIG['PGO_PROFILE_PATH'],
]
DIRS += [
'mfbt',
]

12
taskcluster/ci/generate-profile/kind.yml
View file

@ -56,6 +56,9 @@ jobs:
cwd: '{checkout}'
command: >
./taskcluster/scripts/misc/run-profileserver.sh
fetches:
toolchain:
- linux64-clang
linux64-shippable/opt:
description: "Linux64 Profile Generation"
@ -90,6 +93,9 @@ jobs:
cwd: '{checkout}'
command: >
./taskcluster/scripts/misc/run-profileserver.sh
fetches:
toolchain:
- linux64-clang
android-api-16/pgo:
description: "Android 4.0 api-16+ Profile Generation"
@ -159,6 +165,9 @@ jobs:
sparse-profile: profile-generate
cwd: '{checkout}'
command: ./taskcluster/scripts/misc/run-profileserver.sh
fetches:
toolchain:
- win64-clang-cl
win64-shippable/opt:
description: "Win64 Profile Generation"
@ -189,3 +198,6 @@ jobs:
sparse-profile: profile-generate
cwd: '{checkout}'
command: ./taskcluster/scripts/misc/run-profileserver.sh
fetches:
toolchain:
- win64-clang-cl

9
taskcluster/scripts/misc/run-profileserver.sh
View file

@ -14,6 +14,7 @@ export UPLOAD_PATH
PGO_RUNDIR=obj-firefox/dist
export JARLOG_FILE="en-US.log"
export LLVM_PROFDATA=$MOZ_FETCHES_DIR/clang/bin/llvm-profdata
set -v
@ -38,10 +39,4 @@ mkdir -p $UPLOAD_PATH
mv $MOZ_FETCHES_DIR/firefox $PGO_RUNDIR
./mach python build/pgo/profileserver.py --binary $PGO_RUNDIR/firefox/firefox
# Fail the build if for some reason we didn't collect any profile data.
if test -z "$(find . -maxdepth 1 -name '*.profraw' -print -quit)"; then
echo "ERROR: no profile data produced"
exit 1
fi
tar -acvf $UPLOAD_PATH/profdata.tar.xz *.profraw en-US.log
tar -acvf $UPLOAD_PATH/profdata.tar.xz merged.profdata en-US.log

8
testing/mozharness/scripts/android_emulator_pgo.py
View file

@ -270,10 +270,6 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin,
self.fatal('INFRA-ERROR: Failed with an ADBTimeoutError',
EXIT_STATUS_DICT[TBPL_RETRY])
# We normally merge as part of a GENERATED_FILES step in the profile-use
# build, but Android runs sometimes result in a truncated profile. We do
# a merge here to make sure the data isn't corrupt so we can retry the
# 'run' task if necessary.
profraw_files = glob.glob('/builds/worker/workspace/*.profraw')
if not profraw_files:
self.fatal('Could not find any profraw files in /builds/worker/workspace')
@ -281,7 +277,7 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin,
os.path.join(os.environ['MOZ_FETCHES_DIR'], 'clang/bin/llvm-profdata'),
'merge',
'-o',
'/builds/worker/workspace/merged.profraw',
'/builds/worker/workspace/merged.profdata',
] + profraw_files
rc = subprocess.call(merge_cmd)
if rc != 0:
@ -294,7 +290,7 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin,
'-acvf',
'/builds/worker/artifacts/profdata.tar.xz',
'-C', '/builds/worker/workspace',
'merged.profraw',
'merged.profdata',
'en-US.log',
]
subprocess.check_call(tar_cmd)