From 5dbfd833bbb114afe758db4d4bdbc5b13bcc33ef Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Wed, 15 Aug 2018 13:52:47 +0000 Subject: [PATCH] Bug 1472992 - [jsshell] Run javascript shell benchmarks against Google V8, r=jmaher This runs the jsshell benchmarks against Google's V8 engine in addition to spidermonkey. Both shells will run in the same task to keep things simple and decluttered. Though we could split them out to separate tasks at a later date if needed. Differential Revision: https://phabricator.services.mozilla.com/D3356 --HG-- extra : moz-landing-system : lando --- taskcluster/ci/config.yml | 3 +- taskcluster/ci/fetch/benchmarks.yml | 11 ++-- taskcluster/ci/source-test/jsshell.yml | 40 ++++++++------ .../taskgraph/transforms/source_test.py | 52 +++++++++++++++++++ testing/jsshell/benchmark.py | 24 ++++++--- 5 files changed, 103 insertions(+), 27 deletions(-) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 8233d78926ee..f8fbef4ef0f1 100755 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -3,7 +3,8 @@ project-repo-param-prefix: '' treeherder: group-names: 'cram': 'Cram tests' - 'js-bench': 'JavaScript shell benchmarks' + 'js-bench-sm': 'JavaScript shell benchmarks with Spidermonkey' + 'js-bench-v8': 'JavaScript shell benchmarks with Google V8' 'mocha': 'Mocha unit tests' 'py2': 'Python 2 unit tests' 'py3': 'Python 3 unit tests' diff --git a/taskcluster/ci/fetch/benchmarks.yml b/taskcluster/ci/fetch/benchmarks.yml index 68d957d25ffd..b53ef22e329b 100644 --- a/taskcluster/ci/fetch/benchmarks.yml +++ b/taskcluster/ci/fetch/benchmarks.yml @@ -1,8 +1,15 @@ +d8: + description: V8 debug shell + fetch: + type: static-url + url: https://github.com/mozilla/perf-automation/releases/download/d8-6.7.17/d8-6.7.17.zip + sha256: 0aa1c4e630de78373185fc1c0fa34bc87826f63fd4cbb664668891d6f6a6b24e + size: 20578358 + unity-webgl: description: unity-webgl benchmark fetch: type: static-url - artifact-name: unity-webgl.zip url: https://github.com/mozilla/perf-automation/releases/download/unity-webgl-v1/unity-webgl-6beb3d3e22ab.zip sha256: f71ee3a3f5b9513f041e1dd01c032d51f2071e1ad130e8ac2cf0c553c468b9ea size: 27062962 @@ -11,7 +18,6 @@ assorted-dom: description: assorted-dom benchmark fetch: type: static-url - artifact-name: assorted-dom.zip url: https://github.com/mozilla/perf-automation/releases/download/assorted-dom-v1/assorted-dom-4befd28725c6.zip sha256: e4eafe4a8e70c7ae6d42d668d3b1640b9fd9b696c486ff35aab754c368f78c2c size: 402665 @@ -20,7 +26,6 @@ web-tooling-benchmark: description: Web Tooling Benchmark fetch: type: static-url - artifact-name: web-tooling-benchmark.zip url: https://github.com/mozilla/perf-automation/releases/download/V1/web-tooling-benchmark-b2ac25c897c9.zip sha256: 93b0b51df0cec3ca9bfa0bdf81d782306dcf18532e39b3ff3180409125daaff1 size: 5444135 diff --git a/taskcluster/ci/source-test/jsshell.yml b/taskcluster/ci/source-test/jsshell.yml index f27d9a9b3272..22be4b5f3638 100644 --- a/taskcluster/ci/source-test/jsshell.yml +++ b/taskcluster/ci/source-test/jsshell.yml @@ -8,47 +8,57 @@ job-defaults: by-platform: linux64.*: env: - SHELL: /bin/bash - JSSHELL: /home/cltbld/fetches/js + by-shell: + sm: + SHELL: /bin/bash + JSSHELL: /home/cltbld/fetches/js + v8: + SHELL: /bin/bash + JSSHELL: /home/cltbld/fetches/d8/d8 max-run-time: 1800 treeherder: kind: test tier: 2 run: - using: mach + using: run-task workdir: /home/cltbld + command: > + cd $GECKO_PATH && + ./mach jsshell-bench --perfherder={shell} --binary=$JSSHELL {test} run-on-projects: ['mozilla-central', 'try'] fetches: build: - target.jsshell.zip + fetch: + - d8 bench-ares6: description: Ares6 JavaScript shell benchmark suite + shell: ['sm', 'v8'] + test: ares6 treeherder: - symbol: js-bench(ares6) - run: - mach: jsshell-bench --binary $JSSHELL --perfherder ares6 + symbol: ares6 bench-sixspeed: description: Six-Speed JavaScript shell benchmark suite + shell: ['sm', 'v8'] + test: six-speed treeherder: - symbol: js-bench(6speed) - run: - mach: jsshell-bench --binary $JSSHELL --perfherder six-speed + symbol: 6speed bench-sunspider: description: SunSpider JavaScript shell benchmark suite + shell: ['sm'] + test: sunspider treeherder: - symbol: js-bench(sunspider) - run: - mach: jsshell-bench --binary $JSSHELL --perfherder sunspider + symbol: sunspider bench-web-tooling: description: Web Tooling shell benchmark suite + shell: ['sm', 'v8'] + test: web-tooling-benchmark treeherder: - symbol: js-bench(webtool) - run: - mach: jsshell-bench --binary $JSSHELL --perfherder web-tooling-benchmark + symbol: webtool fetches: fetch: - web-tooling-benchmark diff --git a/taskcluster/taskgraph/transforms/source_test.py b/taskcluster/taskgraph/transforms/source_test.py index 174da18c4771..dca110735a6c 100644 --- a/taskcluster/taskgraph/transforms/source_test.py +++ b/taskcluster/taskgraph/transforms/source_test.py @@ -128,6 +128,36 @@ def split_python(config, jobs): yield pyjob +@transforms.add +def split_jsshell(config, jobs): + all_shells = { + 'sm': "Spidermonkey", + 'v8': "Google V8" + } + + for job in jobs: + if not job['name'].startswith('jsshell'): + yield job + continue + + test = job.pop('test') + for shell in job.get('shell', all_shells.keys()): + assert shell in all_shells + + new_job = copy.deepcopy(job) + new_job['name'] = '{}-{}'.format(new_job['name'], shell) + new_job['description'] = '{} on {}'.format(new_job['description'], all_shells[shell]) + new_job['shell'] = shell + + group = 'js-bench-{}'.format(shell) + symbol = split_symbol(new_job['treeherder']['symbol'])[1] + new_job['treeherder']['symbol'] = join_symbol(group, symbol) + + run = new_job['run'] + run['command'] = run['command'].format(shell=shell, SHELL=shell.upper(), test=test) + yield new_job + + def add_build_dependency(config, job): """ Add build dependency to the job and installer_url to env. @@ -172,3 +202,25 @@ def handle_platform(config, jobs): del job['platform'] yield job + + +@transforms.add +def handle_shell(config, jobs): + """ + Handle the 'shell' property. + """ + fields = [ + 'run-on-projects', + 'worker.env', + ] + + for job in jobs: + if not job.get('shell'): + yield job + continue + + for field in fields: + resolve_keyed_by(job, field, item_name=job['name']) + + del job['shell'] + yield job diff --git a/testing/jsshell/benchmark.py b/testing/jsshell/benchmark.py index f50bd5901a34..91e4d5c03ac2 100644 --- a/testing/jsshell/benchmark.py +++ b/testing/jsshell/benchmark.py @@ -31,9 +31,10 @@ class Benchmark(object): should_alert = False units = 'score' - def __init__(self, shell, args=None): + def __init__(self, shell, args=None, shell_name=None): self.shell = shell self.args = args + self.shell_name = shell_name @abstractproperty def name(self): @@ -70,6 +71,10 @@ class Benchmark(object): def reset(self): """Resets state between runs.""" + name = self.name + if self.shell_name: + name = '{}-{}'.format(name, self.shell_name) + self.perfherder_data = { 'framework': { 'name': 'js-bench', @@ -77,7 +82,7 @@ class Benchmark(object): 'suites': [ { 'lowerIsBetter': self.lower_is_better, - 'name': self.name, + 'name': name, 'shouldAlert': self.should_alert, 'subtests': [], 'units': self.units, @@ -266,8 +271,11 @@ class WebToolingBenchmark(Benchmark): if score_name == 'mean': bench_mean = mean self.suite['value'] = bench_mean - + def _provision_benchmark_script(self): + if os.path.isdir(self.path): + return + # Some benchmarks may have been downloaded from a fetch task, make # sure they get copied over. fetches_dir = os.environ.get('MOZ_FETCHES_DIR') @@ -275,7 +283,7 @@ class WebToolingBenchmark(Benchmark): webtool_fetchdir = os.path.join(fetches_dir, 'web-tooling-benchmark') if os.path.isdir(webtool_fetchdir): shutil.copytree(webtool_fetchdir, self.path) - + def run(self): self._provision_benchmark_script() return super(WebToolingBenchmark, self).run() @@ -289,7 +297,7 @@ all_benchmarks = { } -def run(benchmark, binary=None, extra_args=None, perfherder=False): +def run(benchmark, binary=None, extra_args=None, perfherder=None): if not binary: try: binary = os.path.join(build.bindir, 'js' + build.substs['BIN_SUFFIX']) @@ -300,7 +308,7 @@ def run(benchmark, binary=None, extra_args=None, perfherder=False): print(JSSHELL_NOT_FOUND) return 1 - bench = all_benchmarks.get(benchmark)(binary, args=extra_args) + bench = all_benchmarks.get(benchmark)(binary, args=extra_args, shell_name=perfherder) res = bench.run() if perfherder: @@ -316,8 +324,8 @@ def get_parser(): help="Path to the JS shell binary to use.") parser.add_argument('--arg', dest='extra_args', action='append', default=None, help="Extra arguments to pass to the JS shell.") - parser.add_argument('--perfherder', action='store_true', default=False, - help="Log PERFHERDER_DATA to stdout.") + parser.add_argument('--perfherder', default=None, + help="Log PERFHERDER_DATA to stdout using the given suite name.") return parser