diff --git a/.gitignore b/.gitignore index 70c8f7992b1c..fd42ca46d572 100644 --- a/.gitignore +++ b/.gitignore @@ -262,6 +262,7 @@ testing/talos/talos/fis/tp5n testing/talos/talos/tests/tp5n.zip testing/talos/talos/tests/tp5n.tar.gz testing/talos/talos/tests/tp5n +testing/talos/talos/tests/pdfpaint/pdfs testing/talos/talos/tests/devtools/damp.manifest.develop testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop testing/talos/talos/webextensions/ diff --git a/.hgignore b/.hgignore index 230fd78646c5..cfc08827f052 100644 --- a/.hgignore +++ b/.hgignore @@ -258,6 +258,7 @@ _OPT\.OBJ/ ^testing/talos/talos/tests/tp5n\.zip ^testing/talos/talos/tests/tp5n\.tar\.gz ^testing/talos/talos/tests/tp5n +^testing/talos/talos/tests/pdfpaint/pdfs ^testing/talos/talos/tests/devtools/damp\.manifest\.develop ^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint\.manifest\.develop ^testing/talos/talos/webextensions/ diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index 3866c18a68c3..96215b6f791f 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -280,6 +280,18 @@ class Talos( "help": "Take a screenshot when the test fails.", }, ], + [ + ["--pdfPaintChunk"], + { + "type": "int", + "dest": "pdfpaint_chunk", + "default": None, + "help": ( + "Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). " + "Defaults to None to run all the pdfs at the same time." + ), + }, + ], ] + testing_config_options + copy.deepcopy(code_coverage_config_options) @@ -558,6 +570,8 @@ class Talos( kw_options["symbolsPath"] = self.symbols_path if self.config.get("project", None): kw_options["project"] = self.config["project"] + if self.config.get("pdfpaint_chunk", None): + kw_options["pdfPaintChunk"] = str(self.config["pdfpaint_chunk"]) kw_options.update(kw) # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg' @@ -850,6 +864,9 @@ class Talos( env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] if not self.run_local: env["MINIDUMP_STACKWALK"] = self.query_minidump_stackwalk() + env["MOZ_FETCHES_DIR"] = os.environ.get("MOZ_FETCHES_DIR") + else: + env["MOZBUILD_PATH"] = self.config.get("mozbuild_path") env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] env["RUST_BACKTRACE"] = "full" if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): diff --git a/testing/perfdocs/generated/talos.rst b/testing/perfdocs/generated/talos.rst index 06351b4df0ef..1ef674251def 100644 --- a/testing/perfdocs/generated/talos.rst +++ b/testing/perfdocs/generated/talos.rst @@ -7992,11 +7992,18 @@ For the sample commands found below, note that the capitalization used is import * source: * type: `Page load`_ * reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better) - * data: load a PDF 20 times + * data: loads a PDF 5 times + * description: + Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each. + If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested + by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js + repository, and this test pulls those in for testing locally through a toolchain artifact + called talos-pdfs. * pdfpaint: True - * timeout: 600 + * timeout: 1800 * tpmanifest: ${talos}/tests/pdfpaint/pdfpaint.manifest - * tppagecycles: 20 + * tppagecycles: 1 + * tptimeout: 60000 * unit: ms * Command @@ -8004,175 +8011,6 @@ For the sample commands found below, note that the capitalization used is import ./mach talos-test -a pdfpaint - * **Test Task**: - - .. list-table:: **test-linux1804-64-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-linux1804-64-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ✅ - - ✅ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ✅ - - ✅ - - ❌ - - ❌ - - - .. list-table:: **test-macosx1015-64-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ✅ - - ✅ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ✅ - - ✅ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-32-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-32-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-64-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ❌ - - ❌ - - ❌ - - ❌ - - - .. list-table:: **test-windows10-64-shippable-qr/opt** - :widths: 30 15 15 15 15 - :header-rows: 1 - - * - **Test Name** - - mozilla-central - - autoland - - mozilla-release - - mozilla-beta - * - **talos-other** - - ✅ - - ✅ - - ❌ - - ❌ - * - **talos-other-profiling** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-other-swr** - - ✅ - - ✅ - - ❌ - - ❌ - - .. dropdown:: perf_reftest :class-container: anchor-id-perf_reftest diff --git a/testing/talos/perfdocs/config.yml b/testing/talos/perfdocs/config.yml index bbc71a74aa84..31caf54add01 100644 --- a/testing/talos/perfdocs/config.yml +++ b/testing/talos/perfdocs/config.yml @@ -607,7 +607,13 @@ suites: - source: - type: `Page load`_ - reporting: time from *performance.timing.navigationStart* to *pagerendered* event in ms (lower is better) - - data: load a PDF 20 times + - data: loads a PDF 5 times + - description: + Runs through a set of chunks. Each chunk runs 100 PDFs with 5 iterations each. + If --pdfPaintChunk is not used when running the test locally, all PDFs will be tested + by default with only 1 cycle each. The PDFs that are run are found in the Mozilla pdf.js + repository, and this test pulls those in for testing locally through a toolchain artifact + called talos-pdfs. perf_reftest: > - contact: :emilio and css/layout team - source: `perf-reftest `__ diff --git a/testing/talos/talos.json b/testing/talos/talos.json index 32904f87a334..1446abd82a89 100644 --- a/testing/talos/talos.json +++ b/testing/talos/talos.json @@ -19,7 +19,6 @@ "tabpaint", "cpstartup", "startup_about_home_paint", - "pdfpaint", "cross_origin_pageload", "startup_about_home_paint_cached" ] @@ -121,6 +120,9 @@ "realworld-webextensions": { "tests": ["startup_about_home_paint_realworld_webextensions"], "webextensions_zip": "webextensions.zip" + }, + "pdfpaint": { + "tests": ["pdfpaint"] } } } diff --git a/testing/talos/talos/cmdline.py b/testing/talos/talos/cmdline.py index dc7ecd9a09e0..697342e96961 100644 --- a/testing/talos/talos/cmdline.py +++ b/testing/talos/talos/cmdline.py @@ -223,6 +223,26 @@ def create_parser(mach_interface=False): dest="pdfpaint", help="Wait for the first page of a PDF to be rendered", ) + add_arg( + "--pdfPaintChunk", + type=int, + default=None, + dest="pdfpaint_chunk", + help=( + "Chunk of the pdfpaint test to run (each chunk runs at most 100 pdfs). " + "Defaults to None to run all the pdfs at the same time." + ), + ) + add_arg( + "--pdfPaintName", + type=str, + default=None, + dest="pdfpaint_name", + help=( + "Name of a pdfpaint test to run (e.g. xfa_imm5257e.pdf). Chunking will be " + "ignored/disabled if this option is used." + ), + ) add_arg("--webServer", dest="webserver", help="DEPRECATED") if not mach_interface: add_arg( diff --git a/testing/talos/talos/config.py b/testing/talos/talos/config.py index 02f8814bff8f..31162d692c63 100644 --- a/testing/talos/talos/config.py +++ b/testing/talos/talos/config.py @@ -2,7 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import copy +import json import os +import pathlib import re import sys @@ -198,6 +200,77 @@ def get_global_overrides(config): return global_overrides +def setup_pdfpaint_test(config, test_instance): + # Get the root of the location of the PDFs artifact + if os.environ.get("MOZ_FETCHES_DIR", None): + pdfs_root = pathlib.Path(os.environ.get("MOZ_FETCHES_DIR"), "talos-pdfs") + else: + pdfs_root = pathlib.Path(os.environ.get("MOZBUILD_PATH"), "talos-pdfs") + if not pdfs_root.exists(): + raise Exception(f"Cannot find webserver root: {pdfs_root}") + + pdfpaint_manifest_path = pathlib.Path(pdfs_root, "pdfpaint.manifest") + test_manifest_path = pathlib.Path(pdfs_root, "test_manifest.json") + test_manifest = json.loads(test_manifest_path.read_text(encoding="utf8")) + + # If a pdfpaint test was specified, prevent any chunking + chunk_number = config.get("pdfpaint_chunk", None) + pdfpaint_test = None + if config.get("pdfpaint_name") is not None: + chunk_number = None + pdfpaint_test = config["pdfpaint_name"] + + # Gather all the pdf files that can be used in the test, and write + # all the pdfs to be tested to the manifest file + start_ind = 0 + end_ind = None + pdfs_per_chunk = 100 + if chunk_number is not None: + start_ind = pdfs_per_chunk * (chunk_number - 1) + end_ind = pdfs_per_chunk * chunk_number + + pdf_files = set() + for pdf_info in test_manifest: + if pdf_info.get("password", None) is not None: + # PDFs that require passwords cause timeouts + continue + + pdf_name = pathlib.Path(pdf_info["file"]).name + if ( + config.get("pdfpaint_name", None) is not None + and config["pdfpaint_name"] != pdf_name + ): + # If a user passed a name of a pdf, skip all the others + continue + + pdf_files.add(pdf_name) + + if start_ind > len(pdf_files): + raise ConfigurationError( + f"Chunk {chunk_number} contains no PDFs to test. " + f"For {len(pdf_files)} PDFs, the max chunk is " + f"{int((len(pdf_files)-1)/pdfs_per_chunk)+1}." + ) + + with pdfpaint_manifest_path.open("w") as f: + for pdf_file in sorted(list(pdf_files))[start_ind:end_ind]: + print(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}") + f.write(f"http://localhost/tests/pdfpaint/pdfs/{pdf_file}{os.linesep}") + + # Make a symbolic link to the mozbuild pdf folder since the talos + # webserver depends on having the doc root as the talos folder for getInfo.html + symlink_dest = pathlib.Path(__file__).parent / "tests" / "pdfpaint" / "pdfs" + symlink_dest.unlink(missing_ok=True) + symlink_dest.symlink_to(pdfs_root, target_is_directory=True) + test_instance.tpmanifest = str(pdfpaint_manifest_path) + + # Increase the pagecycles for each pdf to 5 if we're running chunks, otherwise + # it can take a very long time to complete testing of all pdfs + if chunk_number is not None or pdfpaint_test is not None: + print("Setting pdfpaint tppagecycles to 5") + test_instance.tppagecycles = 5 + + def get_test_host(manifest_line): match = re.match(r"^http://localhost/page_load_test/tp5n/([^/]+)/", manifest_line) host = match.group(1) @@ -248,6 +321,9 @@ def get_test(config, global_overrides, counters, test_instance): if pdfPaint is not None: test_instance.pdfpaint = pdfPaint + if test_instance.pdfpaint: + setup_pdfpaint_test(config, test_instance) + # fix up url url = getattr(test_instance, "url", None) if url: @@ -359,6 +435,11 @@ def get_config(argv=None): elif not cli_opts.activeTests: raise ConfigurationError("--activeTests or --suite required!") + if cli_opts.pdfpaint_chunk is not None and cli_opts.pdfpaint_chunk < 1: + raise ConfigurationError( + "pdfpaint chunk must be a positive integer greater than or equal to 1" + ) + cli_opts = parse_args(argv=argv) setup_logging("talos", cli_opts, {"tbpl": sys.stdout}) config = copy.deepcopy(DEFAULTS) diff --git a/testing/talos/talos/pageloader/chrome/report.js b/testing/talos/talos/pageloader/chrome/report.js index 7f5d61922a22..d1eea8f3c9a6 100644 --- a/testing/talos/talos/pageloader/chrome/report.js +++ b/testing/talos/talos/pageloader/chrome/report.js @@ -4,6 +4,11 @@ // given an array of strings, finds the longest common prefix function findCommonPrefixLength(strs) { + if (strs.every(str => str.includes("/pdfs/"))) { + // In all cases for pdfpaint PDFs, return the full file name + return strs[0].lastIndexOf("/") + 1; + } + if (strs.length < 2) { // only one page in the manifest // i.e. http://localhost/tests/perf-reftest/bloom-basic.html diff --git a/testing/talos/talos/test.py b/testing/talos/talos/test.py index 6a7f44feb943..f64d5004ef6b 100644 --- a/testing/talos/talos/test.py +++ b/testing/talos/talos/test.py @@ -419,8 +419,9 @@ class pdfpaint(PageloaderTest): """ tpmanifest = "${talos}/tests/pdfpaint/pdfpaint.manifest" - tppagecycles = 20 - timeout = 600 + tppagecycles = 1 + timeout = 1800 + tptimeout = 60000 pdfpaint = True unit = "ms"