From 7948524fe0b3567d45e68bc6e404307487eee2c6 Mon Sep 17 00:00:00 2001 From: Greg Mierzwinski Date: Mon, 8 Apr 2024 11:47:05 +0000 Subject: [PATCH] Bug 1855674 - Modify pdfpaint test to run more PDFs. r=aglavic,perftest-reviewers This patch modifies the pdfpaint test to run more pdfs that are found in the Mozilla pdf.js repository. The pdfpaint test is also moved to it's own suite due to the number of PDFs now being tested. These PDFs are pulled in locally from a toolchain task called talos-pdfs. The *ignore files are modified since the pdfpaint folder now contains a symbolic link to the local PDFs that should not be commit in-tree. To handle running the large number of PDFs, chunking is added to the test with the chunk size being 100 PDFs. Each chunk runs each of the 100 PDFs 5 times. A CLI option is also added for local runs so that users can select a specific pdfpaint PDF to test. An additional issue with the subtest/pdf file name parsing is also fixed for this to work. Differential Revision: https://phabricator.services.mozilla.com/D205824 --- .gitignore | 1 + .hgignore | 1 + .../mozharness/mozilla/testing/talos.py | 17 ++ testing/perfdocs/generated/talos.rst | 182 +----------------- testing/talos/perfdocs/config.yml | 8 +- testing/talos/talos.json | 4 +- testing/talos/talos/cmdline.py | 20 ++ testing/talos/talos/config.py | 81 ++++++++ .../talos/talos/pageloader/chrome/report.js | 5 + testing/talos/talos/test.py | 5 +- 10 files changed, 148 insertions(+), 176 deletions(-) diff --git a/.gitignore b/.gitignore index 70c8f7992b1c..fd42ca46d572 100644 --- a/.gitignore +++ b/.gitignore @@ -262,6 +262,7 @@ testing/talos/talos/fis/tp5n testing/talos/talos/tests/tp5n.zip testing/talos/talos/tests/tp5n.tar.gz testing/talos/talos/tests/tp5n +testing/talos/talos/tests/pdfpaint/pdfs testing/talos/talos/tests/devtools/damp.manifest.develop testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop testing/talos/talos/webextensions/ diff --git a/.hgignore b/.hgignore index 230fd78646c5..cfc08827f052 100644 --- a/.hgignore +++ b/.hgignore @@ -258,6 +258,7 @@ _OPT\.OBJ/ ^testing/talos/talos/tests/tp5n\.zip ^testing/talos/talos/tests/tp5n\.tar\.gz ^testing/talos/talos/tests/tp5n +^testing/talos/talos/tests/pdfpaint/pdfs ^testing/talos/talos/tests/devtools/damp\.manifest\.develop ^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint\.manifest\.develop ^testing/talos/talos/webextensions/ diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index 3866c18a68c3..96215b6f791f 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -280,6 +280,18 @@ class Talos( "help": "Take a screenshot when the test fails.", }, ], + [ + ["--pdfPaintChunk"], + { + "type": "int", + "dest": "pdfpaint_chunk", + "default": None, + "help": ( + "Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). " + "Defaults to None to run all the pdfs at the same time." + ), + }, + ], ] + testing_config_options + copy.deepcopy(code_coverage_config_options) @@ -558,6 +570,8 @@ class Talos( kw_options["symbolsPath"] = self.symbols_path if self.config.get("project", None): kw_options["project"] = self.config["project"] + if self.config.get("pdfpaint_chunk", None): + kw_options["pdfPaintChunk"] = str(self.config["pdfpaint_chunk"]) kw_options.update(kw) # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg' @@ -850,6 +864,9 @@ class Talos( env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] if not self.run_local: env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk() + env["MOZ_FETCHES_DIR"] = os.environ.get("MOZ_FETCHES_DIR") + else: + env["MOZBUILD_PATH"] = self.config.get("mozbuild_path") env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] env["RUST_BACKTRACE"] = "full" if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): diff --git a/testing/perfdocs/generated/talos.rst b/testing/perfdocs/generated/talos.rst index 06351b4df0ef..1ef674251def 100644 --- a/testing/perfdocs/generated/talos.rst +++ b/testing/perfdocs/generated/talos.rst @@ -7992,11 +7992,18 @@ For the sample commands found below, note that the capitalization used is import * source: * type: `Page load`_ * reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better) - * data: load a PDF 20 times + * data: loads a PDF 5 times + * description: + Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each. + If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested + by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js + repository, and this test pulls those in for testing locally through a toolchain artifact + called talos-pdfs. * pdfpaint: True - * timeout: 600 + * timeout: 1800 * tpmanifest: ${talos}/tests/pdfpaint/pdfpaint.manifest - * tppagecycles: 20 + * tppagecycles: 1 + * tptimeout: 60000 * unit: ms * Command @@ -8004,175 +8011,6 @@ For the sample commands found below, note that the capitalization used is import ./mach talos-test -a pdfpaint - * **Test Task**: - - .. list-table:: **test-linux1804-64-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-linux1804-64-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ✅ - - ✅ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ✅ - - ✅ - - ❌ - - ❌ - - - .. list-table:: **test-macosx1015-64-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ✅ - - ✅ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ✅ - - ✅ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-32-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-32-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-64-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-64-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ✅ - - ✅ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ✅ - - ✅ - - ❌ - - ❌ - - .. dropdown:: perf_reftest :class-container: anchor-id-perf_reftest diff --git a/testing/talos/perfdocs/config.yml b/testing/talos/perfdocs/config.yml index bbc71a74aa84..31caf54add01 100644 --- a/testing/talos/perfdocs/config.yml +++ b/testing/talos/perfdocs/config.yml @@ -607,7 +607,13 @@ suites: - source: - type: `Page load`_ - reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better) - - data: load a PDF 20 times + - data: loads a PDF 5 times + - description: + Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each. + If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested + by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js + repository, and this test pulls those in for testing locally through a toolchain artifact + called talos-pdfs. perf_reftest: > - contact: :emilio and css/layout team - source: `perf-reftest `__ diff --git a/testing/talos/talos.json b/testing/talos/talos.json index 32904f87a334..1446abd82a89 100644 --- a/testing/talos/talos.json +++ b/testing/talos/talos.json @@ -19,7 +19,6 @@ "tabpaint", "cpstartup", "startup_about_home_paint", - "pdfpaint", "cross_origin_pageload", "startup_about_home_paint_cached" ] @@ -121,6 +120,9 @@ "realworld-webextensions": { "tests": ["startup_about_home_paint_realworld_webextensions"], "webextensions_zip": "webextensions.zip" + }, + "pdfpaint": { + "tests": ["pdfpaint"] } } } diff --git a/testing/talos/talos/cmdline.py b/testing/talos/talos/cmdline.py index dc7ecd9a09e0..697342e96961 100644 --- a/testing/talos/talos/cmdline.py +++ b/testing/talos/talos/cmdline.py @@ -223,6 +223,26 @@ def create_parser(mach_interface=False): dest="pdfpaint", help="Wait for the first page of a PDF to be rendered", ) + add_arg( + "--pdfPaintChunk", + type=int, + default=None, + dest="pdfpaint_chunk", + help=( + "Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). " + "Defaults to None to run all the pdfs at the same time." + ), + ) + add_arg( + "--pdfPaintName", + type=str, + default=None, + dest="pdfpaint_name", + help=( + "Name of a pdfpaint test to run (e.g. xfa_imm5257e.pdf). Chunking will be " + "ignored/disabled if this option is used." + ), + ) add_arg("--webServer", dest="webserver", help="DEPRECATED") if not mach_interface: add_arg( diff --git a/testing/talos/talos/config.py b/testing/talos/talos/config.py index 02f8814bff8f..31162d692c63 100644 --- a/testing/talos/talos/config.py +++ b/testing/talos/talos/config.py @@ -2,7 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import copy +import json import os +import pathlib import re import sys @@ -198,6 +200,77 @@ def get_global_overrides(config): return global_overrides +def setup_pdfpaint_test(config, test_instance): + # Get the root of the location of the PDFs artifact + if os.environ.get("MOZ_FETCHES_DIR", None): + pdfs_root = pathlib.Path(os.environ.get("MOZ_FETCHES_DIR"), "talos-pdfs") + else: + pdfs_root = pathlib.Path(os.environ.get("MOZBUILD_PATH"), "talos-pdfs") + if not pdfs_root.exists(): + raise Exception(f"Cannot find webserver root: {pdfs_root}") + + pdfpaint_manifest_path = pathlib.Path(pdfs_root, "pdfpaint.manifest") + test_manifest_path = pathlib.Path(pdfs_root, "test_manifest.json") + test_manifest = json.loads(test_manifest_path.read_text(encoding="utf8")) + + # If a pdfpaint test was specified, prevent any chunking + chunk_number = config.get("pdfpaint_chunk", None) + pdfpaint_test = None + if config.get("pdfpaint_name") is not None: + chunk_number = None + pdfpaint_test = config["pdfpaint_name"] + + # Gather all the pdf files that can be used in the test, and write + # all the pdfs to be tested to the manifest file + start_ind = 0 + end_ind = None + pdfs_per_chunk = 100 + if chunk_number is not None: + start_ind = pdfs_per_chunk * (chunk_number - 1) + end_ind = pdfs_per_chunk * chunk_number + + pdf_files = set() + for pdf_info in test_manifest: + if pdf_info.get("password", None) is not None: + # PDFs that require passwords cause timeouts + continue + + pdf_name = pathlib.Path(pdf_info["file"]).name + if ( + config.get("pdfpaint_name", None) is not None + and config["pdfpaint_name"] != pdf_name + ): + # If a user passed a name of a pdf, skip all the others + continue + + pdf_files.add(pdf_name) + + if start_ind > len(pdf_files): + raise ConfigurationError( + f"Chunk {chunk_number} contains no PDFs to test. " + f"For {len(pdf_files)} PDFs, the max chunk is " + f"{int((len(pdf_files)-1)/pdfs_per_chunk)+1}." + ) + + with pdfpaint_manifest_path.open("w") as f: + for pdf_file in sorted(list(pdf_files))[start_ind:end_ind]: + print(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}") + f.write(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}") + + # Make a symbolic link to the mozbuild pdf folder since the talos + # webserver depends on having the doc root as the talos folder for getInfo.html + symlink_dest = pathlib.Path(__file__).parent / "tests" / "pdfpaint" / "pdfs" + symlink_dest.unlink(missing_ok=True) + symlink_dest.symlink_to(pdfs_root, target_is_directory=True) + test_instance.tpmanifest = str(pdfpaint_manifest_path) + + # Increase the pagecycles for each pdf to 5 if we're running chunks, otherwise + # it can take a very long time to complete testing of all pdfs + if chunk_number is not None or pdfpaint_test is not None: + print("Setting pdfpaint tppagecycles to 5") + test_instance.tppagecycles = 5 + + def get_test_host(manifest_line): match = re.match(r"^http://localhost/page_load_test/tp5n/([^/]+)/", manifest_line) host = match.group(1) @@ -248,6 +321,9 @@ def get_test(config, global_overrides, counters, test_instance): if pdfPaint is not None: test_instance.pdfpaint = pdfPaint + if test_instance.pdfpaint: + setup_pdfpaint_test(config, test_instance) + # fix up url url = getattr(test_instance, "url", None) if url: @@ -359,6 +435,11 @@ def get_config(argv=None): elif not cli_opts.activeTests: raise ConfigurationError("--activeTests or --suite required!") + if cli_opts.pdfpaint_chunk is not None and cli_opts.pdfpaint_chunk < 1: + raise ConfigurationError( + "pdfpaint chunk must be a positive integer greater than or equal to 1" + ) + cli_opts = parse_args(argv=argv) setup_logging("talos", cli_opts, {"tbpl": sys.stdout}) config = copy.deepcopy(DEFAULTS) diff --git a/testing/talos/talos/pageloader/chrome/report.js b/testing/talos/talos/pageloader/chrome/report.js index 7f5d61922a22..d1eea8f3c9a6 100644 --- a/testing/talos/talos/pageloader/chrome/report.js +++ b/testing/talos/talos/pageloader/chrome/report.js @@ -4,6 +4,11 @@ // given an array of strings, finds the longest common prefix function findCommonPrefixLength(strs) { + if (strs.every(str => str.includes("/pdfs/"))) { + // In all cases for pdfpaint PDFs, return the full file name + return strs[0].lastIndexOf("/") + 1; + } + if (strs.length < 2) { // only one page in the manifest // i.e. http://localhost/tests/perf-reftest/bloom-basic.html diff --git a/testing/talos/talos/test.py b/testing/talos/talos/test.py index 6a7f44feb943..f64d5004ef6b 100644 --- a/testing/talos/talos/test.py +++ b/testing/talos/talos/test.py @@ -419,8 +419,9 @@ class pdfpaint(PageloaderTest): """ tpmanifest = "${talos}/tests/pdfpaint/pdfpaint.manifest" - tppagecycles = 20 - timeout = 600 + tppagecycles = 1 + timeout = 1800 + tptimeout = 60000 pdfpaint = True unit = "ms"