diff --git a/.hgignore b/.hgignore index aa8ee10d9fd8..c398e60c0033 100644 --- a/.hgignore +++ b/.hgignore @@ -126,6 +126,8 @@ _OPT\.OBJ/ ^remote/test/puppeteer/test/build ^remote/test/puppeteer/test/output-firefox ^remote/test/puppeteer/test/output-chromium +^remote/test/puppeteer/testserver/lib/ +^remote/test/puppeteer/utils/mochaRunner/lib/ ^remote/test/puppeteer/website # git checkout of libstagefright diff --git a/remote/.gitignore b/remote/.gitignore index 33fc56a0b3cd..516011818958 100644 --- a/remote/.gitignore +++ b/remote/.gitignore @@ -15,4 +15,6 @@ test/puppeteer/src/generated test/puppeteer/test/build test/puppeteer/test/output-firefox test/puppeteer/test/output-chromium +test/puppeteer/testserver/lib/ +test/puppeteer/utils/mochaRunner/lib/ test/puppeteer/website diff --git a/remote/mach_commands.py b/remote/mach_commands.py index 40ac0a1378ad..b51f4d64c91a 100644 --- a/remote/mach_commands.py +++ b/remote/mach_commands.py @@ -17,7 +17,6 @@ import mozprofile from mach.decorators import Command, CommandArgument, SubCommand from mozbuild import nodeutil from mozbuild.base import BinaryNotFoundException, MozbuildObject -from six import iteritems EX_CONFIG = 78 EX_SOFTWARE = 70 @@ -261,8 +260,10 @@ class MochaOutputHandler(object): if not status and not test_start: return test_info = event[1] - test_name = test_info.get("fullTitle", "") + test_full_title = test_info.get("fullTitle", "") + test_name = test_full_title test_path = test_info.get("file", "") + test_file_name = os.path.basename(test_path).replace(".js", "") test_err = test_info.get("err") if status == "FAIL" and test_err: if "timeout" in test_err.lower(): @@ -276,7 +277,32 @@ class MochaOutputHandler(object): if test_start: self.logger.test_start(test_name) return - expected = self.expected.get(test_name, ["PASS"]) + expected_name = "[{}] {}".format(test_file_name, test_full_title) + expected_item = next( + ( + expectation + for expectation in list(self.expected) + if expectation["testIdPattern"] == expected_name + ), + None, + ) + if expected_item is None: + # if there is no expectation data for a specific test case, + # try to find data for a whole file. + expected_item_for_file = next( + ( + expectation + for expectation in list(self.expected) + if expectation["testIdPattern"] == f"[{test_file_name}]" + ), + None, + ) + if expected_item_for_file is None: + expected = ["PASS"] + else: + expected = expected_item_for_file["expectations"] + else: + expected = expected_item["expectations"] # mozlog doesn't really allow unexpected skip, # so if a test is disabled just expect that and note the unexpected skip # Also, mocha doesn't log test-start for skipped tests @@ -308,34 +334,7 @@ class MochaOutputHandler(object): known_intermittent=known_intermittent, ) - def new_expected(self): - new_expected = OrderedDict() - for test_name, status in iteritems(self.test_results): - if test_name not in self.expected: - new_status = [status] - else: - if status in self.expected[test_name]: - new_status = self.expected[test_name] - else: - new_status = [status] - new_expected[test_name] = new_status - return new_expected - - def after_end(self, subset=False): - if not subset: - missing = set(self.expected) - set(self.test_results) - extra = set(self.test_results) - set(self.expected) - if missing: - self.has_unexpected = True - for test_name in missing: - self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,)) - if self.expected and extra: - self.has_unexpected = True - for test_name in extra: - self.logger.error( - "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,) - ) - + def after_end(self): if self.unexpected_skips: self.has_unexpected = True for test_name in self.unexpected_skips: @@ -392,8 +391,6 @@ class PuppeteerRunner(MozbuildObject): before invoking npm. Overrides default preferences. `enable_webrender`: Boolean to indicate whether to enable WebRender compositor in Gecko. - `write_results`: - Path to write the results json file `subset` Indicates only a subset of tests are being run, so we should skip the check for missing results @@ -425,6 +422,7 @@ class PuppeteerRunner(MozbuildObject): "--timeout", "20000", "--no-parallel", + "--no-coverage", ] env["HEADLESS"] = str(params.get("headless", False)) @@ -454,15 +452,31 @@ class PuppeteerRunner(MozbuildObject): env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options) expected_path = os.path.join( - os.path.dirname(__file__), "test", "puppeteer-expected.json" + os.path.dirname(__file__), + "test", + "puppeteer", + "test", + "TestExpectations.json", ) - if product == "firefox" and os.path.exists(expected_path): + if os.path.exists(expected_path): with open(expected_path) as f: expected_data = json.load(f) else: - expected_data = {} + expected_data = [] + # Filter expectation data for the selected browser, + # headless or headful mode, and the operating system. + platform = os.uname().sysname.lower() if os.uname() else "win32" + expectations = filter( + lambda el: product in el["parameters"] + and ( + (env["HEADLESS"] == "False" and "headless" not in el["parameters"]) + or "headful" not in el["parameters"] + ) + and platform in el["platforms"], + expected_data, + ) - output_handler = MochaOutputHandler(logger, expected_data) + output_handler = MochaOutputHandler(logger, list(expectations)) proc = npm( *command, cwd=self.puppeteer_dir, @@ -476,7 +490,7 @@ class PuppeteerRunner(MozbuildObject): # failure, so use an output_timeout as a fallback wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False) - output_handler.after_end(params.get("subset", False)) + output_handler.after_end() # Non-zero return codes are non-fatal for now since we have some # issues with unresolved promises that shouldn't otherwise block @@ -484,12 +498,6 @@ class PuppeteerRunner(MozbuildObject): if proc.returncode != 0: logger.warning("npm exited with code %s" % proc.returncode) - if params["write_results"]: - with open(params["write_results"], "w") as f: - json.dump( - output_handler.new_expected(), f, indent=2, separators=(",", ": ") - ) - if output_handler.has_unexpected: exit(1, "Got unexpected results") @@ -547,18 +555,6 @@ def create_parser_puppeteer(): "debug level messages with -v, trace messages with -vv," "and to not truncate long trace messages with -vvv", ) - p.add_argument( - "--write-results", - action="store", - nargs="?", - default=None, - const=os.path.join( - os.path.dirname(__file__), "test", "puppeteer-expected.json" - ), - help="Path to write updated results to (defaults to the " - "expectations file if the argument is provided but " - "no path is passed)", - ) p.add_argument( "--subset", action="store_true", @@ -597,7 +593,6 @@ def puppeteer_test( verbosity=0, tests=None, product="firefox", - write_results=None, subset=False, **kwargs, ): @@ -657,7 +652,6 @@ def puppeteer_test( "extra_prefs": prefs, "product": product, "extra_launcher_options": options, - "write_results": write_results, "subset": subset, } puppeteer = command_context._spawn(PuppeteerRunner) diff --git a/remote/test/puppeteer-expected.json b/remote/test/puppeteer-expected.json deleted file mode 100644 index eb1a12a7fc83..000000000000 --- a/remote/test/puppeteer-expected.json +++ /dev/null @@ -1,2455 +0,0 @@ -{ - "Accessibility should work (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility should report uninteresting nodes (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility roledescription (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility orientation (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility autocomplete (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility multiselectable (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility keyshortcuts (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes should not report text nodes inside controls (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes rich text editable fields should have children (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes rich text editable fields with role should have children (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes non editable textbox with role and tabIndex and label should not have children (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes checkbox with and tabIndex and label should not have children (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes checkbox without label should not have children (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes plaintext contenteditable plain text field with role should not have children (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes root option should work a button (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes root option should work an input (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes root option should work a menu (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes root option should return null when the element is no longer in DOM (accessibility.spec.js)": [ - "SKIP" - ], - "Accessibility filtering children of leaf nodes root option should support the interestingOnly option (accessibility.spec.js)": [ - "SKIP" - ], - "Browser specs Browser.version should return whether we are in headless (browser.spec.js)": [ - "PASS" - ], - "Browser specs Browser.userAgent should include WebKit (browser.spec.js)": [ - "PASS" - ], - "Browser specs Browser.target should return browser target (browser.spec.js)": [ - "PASS" - ], - "Browser specs Browser.process should return child_process instance (browser.spec.js)": [ - "PASS" - ], - "Browser specs Browser.process should not return child_process for remote browser (browser.spec.js)": [ - "PASS" - ], - "Browser specs Browser.isConnected should set the browser connected state (browser.spec.js)": [ - "PASS" - ], - "BrowserContext should have default context (browsercontext.spec.js)": [ - "PASS" - ], - "BrowserContext should create new incognito context (browsercontext.spec.js)": [ - "PASS" - ], - "BrowserContext should close all belonging targets once closing context (browsercontext.spec.js)": [ - "PASS" - ], - "BrowserContext window.open should use parent tab context (browsercontext.spec.js)": [ - "FAIL" - ], - "BrowserContext should fire target events (browsercontext.spec.js)": [ - "FAIL" - ], - "BrowserContext should wait for a target (browsercontext.spec.js)": [ - "TIMEOUT" - ], - "BrowserContext should timeout waiting for a non-existent target (browsercontext.spec.js)": [ - "PASS" - ], - "BrowserContext should isolate localStorage and cookies (browsercontext.spec.js)": [ - "FAIL" - ], - "BrowserContext should work across sessions (browsercontext.spec.js)": [ - "FAIL" - ], - "Page.click should click the button (click.spec.js)": [ - "PASS" - ], - "Page.click should click svg (click.spec.js)": [ - "PASS" - ], - "Page.click should click the button if window.Node is removed (click.spec.js)": [ - "FAIL" - ], - "Page.click should click on a span with an inline element inside (click.spec.js)": [ - "PASS" - ], - "Page.click should not throw UnhandledPromiseRejection when page closes (click.spec.js)": [ - "PASS" - ], - "Page.click should click the button after navigation (click.spec.js)": [ - "PASS" - ], - "Page.click should click with disabled javascript (click.spec.js)": [ - "FAIL" - ], - "Page.click should click when one of inline box children is outside of viewport (click.spec.js)": [ - "PASS" - ], - "Page.click should select the text by triple clicking (click.spec.js)": [ - "PASS" - ], - "Page.click should click offscreen buttons (click.spec.js)": [ - "PASS" - ], - "Page.click should click wrapped links (click.spec.js)": [ - "PASS" - ], - "Page.click should click on checkbox input and toggle (click.spec.js)": [ - "PASS" - ], - "Page.click should click on checkbox label and toggle (click.spec.js)": [ - "FAIL", "PASS" - ], - "Page.click should fail to click a missing button (click.spec.js)": [ - "PASS" - ], - "Page.click should not hang with touch-enabled viewports (click.spec.js)": [ - "PASS" - ], - "Page.click should scroll and click the button (click.spec.js)": [ - "PASS" - ], - "Page.click should double click the button (click.spec.js)": [ - "PASS" - ], - "Page.click should click a partially obscured button (click.spec.js)": [ - "PASS" - ], - "Page.click should click a rotated button (click.spec.js)": [ - "PASS" - ], - "Page.click should fire contextmenu event on right click (click.spec.js)": [ - "PASS" - ], - "Page.click should fire aux event on middle click (click.spec.js)": [ - "PASS" - ], - "Page.click should fire back click (click.spec.js)": [ - "PASS" - ], - "Page.click should fire forward click (click.spec.js)": [ - "PASS" - ], - "Page.click should click links which cause navigation (click.spec.js)": [ - "PASS" - ], - "Page.click should click the button inside an iframe (click.spec.js)": [ - "PASS" - ], - "Page.click should click the button with fixed position inside an iframe (click.spec.js)": [ - "SKIP" - ], - "Page.click should click the button with deviceScaleFactor set (click.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should return no cookies in pristine browser context (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should get a cookie (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should properly report httpOnly cookie (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should properly report \"Strict\" sameSite cookie (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should properly report \"Lax\" sameSite cookie (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should get multiple cookies (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.cookies should get cookies from multiple urls (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should work (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should isolate cookies in browser contexts (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should set multiple cookies (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should have |expires| set to |-1| for session cookies (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should set cookie with reasonable defaults (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should set a cookie with a path (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should not set a cookie on a blank page (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should not set a cookie with blank page URL (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should not set a cookie on a data URL page (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should default to setting secure cookie for HTTPS websites (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should be able to set unsecure cookie for HTTP website (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.setCookie should set a cookie on a different domain (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should set cookies from a frame (cookies.spec.js)": [ - "FAIL" - ], - "Cookie specs Page.setCookie should set secure same-site cookies from a frame (cookies.spec.js)": [ - "PASS" - ], - "Cookie specs Page.deleteCookie should work (cookies.spec.js)": [ - "FAIL" - ], - "DefaultBrowserContext page.cookies() should work (defaultbrowsercontext.spec.js)": [ - "PASS" - ], - "DefaultBrowserContext page.setCookie() should work (defaultbrowsercontext.spec.js)": [ - "FAIL" - ], - "DefaultBrowserContext page.deleteCookie() should work (defaultbrowsercontext.spec.js)": [ - "FAIL" - ], - "Page.Events.Dialog should fire (dialog.spec.js)": [ - "PASS" - ], - "Page.Events.Dialog should allow accepting prompts (dialog.spec.js)": [ - "FAIL" - ], - "Page.Events.Dialog should dismiss the prompt (dialog.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.boundingBox should work (elementhandle.spec.js)": [ - "FAIL" - ], - "ElementHandle specs ElementHandle.boundingBox should handle nested frames (elementhandle.spec.js)": [ - "PASS", "FAIL" - ], - "ElementHandle specs ElementHandle.boundingBox should return null for invisible elements (elementhandle.spec.js)": [ - "FAIL" - ], - "ElementHandle specs ElementHandle.boundingBox should force a layout (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.boundingBox should work with SVG nodes (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.boxModel should work (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.boxModel should return null for invisible elements (elementhandle.spec.js)": [ - "FAIL" - ], - "ElementHandle specs ElementHandle.contentFrame should work (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should work (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should work for Shadow DOM v1 (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should not work for TextNodes (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should throw for detached nodes (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should throw for hidden nodes (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should throw for recursively hidden nodes (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.click should throw for
elements (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Element.waitForSelector should wait correctly with waitForSelector on an element (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Element.waitForXPath should wait correctly with waitForXPath on an element (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.hover should work (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.isIntersectingViewport should work (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.isIntersectingViewport should work with threshold (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs ElementHandle.isIntersectingViewport should work with threshold of 1 (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should register and unregister (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should throw with invalid query names (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should work for multiple elements (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should eval correctly (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should wait correctly with waitForSelector (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should wait correctly with waitForSelector on an element (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should wait correctly with waitFor (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should work when both queryOne and queryAll are registered (elementhandle.spec.js)": [ - "PASS" - ], - "ElementHandle specs Custom queries should eval when both queryOne and queryAll are registered (elementhandle.spec.js)": [ - "PASS" - ], - "Emulation Page.viewport should get the proper viewport size (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.viewport should support mobile emulation (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.viewport should support touch emulation (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.viewport should be detectable by Modernizr (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.viewport should detect touch when applying viewport with touches (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.viewport should support landscape emulation (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulate should work (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.emulate should support clicking (emulation.spec.js)": [ - "PASS", "FAIL" - ], - "Emulation Page.emulateMediaType should work (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulateMediaType should throw in case of bad argument (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.emulateMediaFeatures should work (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulateMediaFeatures should throw in case of bad argument (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.emulateTimezone should work (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulateTimezone should throw for invalid timezone IDs (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulateVisionDeficiency should work (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulateVisionDeficiency should throw for invalid vision deficiencies (emulation.spec.js)": [ - "PASS" - ], - "Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType (emulation.spec.js)": [ - "FAIL" - ], - "Emulation Page.emulateCPUThrottling should change the CPU throttling rate successfully (emulation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should work (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer BigInt (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer NaN (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer -0 (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer Infinity (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer -Infinity (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer arrays (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer arrays as arrays, not objects (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should modify global environment (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should evaluate in the page context (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return undefined for objects with symbols (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should work with function shorthands (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should work with unicode chars (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should throw when evaluation triggers reload (evaluation.spec.js)": [ - "TIMEOUT" - ], - "Evaluation specs Page.evaluate should await promise (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should work right after framenavigated (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should work from-inside an exposed function (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should reject promise with exception (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should support thrown strings as error messages (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should support thrown numbers as error messages (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return complex objects (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return BigInt (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return NaN (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return -0 (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return Infinity (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return -Infinity (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should accept \"null\" as one of multiple parameters (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should properly serialize null fields (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should return undefined for non-serializable objects (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should fail for circular object (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should be able to throw a tricky error (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should accept a string (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should accept a string with semi colons (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should accept a string with comments (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should accept element handle as an argument (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should throw if underlying element was disposed (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should throw if elementHandles are from other frames (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should simulate a user gesture (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should throw a nice error after a navigation (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluate should not throw an error when evaluation does a navigation (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should transfer 100Mb of data from page to node.js (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluate should throw error with detailed information on exception inside promise (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Page.evaluateOnNewDocument should evaluate before anything else on the page (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Page.evaluateOnNewDocument should work with CSP (evaluation.spec.js)": [ - "FAIL" - ], - "Evaluation specs Frame.evaluate should have different execution contexts (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Frame.evaluate should have correct execution contexts (evaluation.spec.js)": [ - "PASS" - ], - "Evaluation specs Frame.evaluate should execute after cross-site navigation (evaluation.spec.js)": [ - "PASS" - ], - "EventEmitter on on: adds an event listener that is fired when the event is emitted (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter on on sends the event data to the handler (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter on on: supports chaining (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter on addListener: adds an event listener that is fired when the event is emitted (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter on addListener sends the event data to the handler (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter on addListener: supports chaining (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter off off: removes the listener so it is no longer called (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter off off: supports chaining (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter off removeListener: removes the listener so it is no longer called (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter off removeListener: supports chaining (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter once only calls the listener once and then removes it (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter once supports chaining (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter emit calls all the listeners for an event (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter emit passes data through to the listener (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter emit returns true if the event has listeners (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter emit returns false if the event has listeners (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter listenerCount returns the number of listeners for the given event (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter removeAllListeners removes every listener from all events by default (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter removeAllListeners returns the emitter for chaining (EventEmitter.spec.js)": [ - "PASS" - ], - "EventEmitter removeAllListeners can filter to remove only listeners for a given event name (EventEmitter.spec.js)": [ - "PASS" - ], - "Fixtures dumpio option should work with pipe option (fixtures.spec.js)": [ - "SKIP" - ], - "Fixtures should dump browser process stderr (fixtures.spec.js)": [ - "PASS" - ], - "Fixtures should close the browser when the node process closes (fixtures.spec.js)": [ - "PASS" - ], - "Frame specs Frame.executionContext should work (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame.evaluateHandle should work (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame.evaluate should throw for detached frames (frame.spec.js)": [ - "TIMEOUT" - ], - "Frame specs Frame.evaluate allows readonly array to be an argument (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame.page should retrieve the page from a frame (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should handle nested frames (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should send events when frames are manipulated dynamically (frame.spec.js)": [ - "FAIL" - ], - "Frame specs Frame Management should send \"framenavigated\" when navigating on anchor URLs (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should persist mainFrame on cross-process navigation (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should not send attach/detach events for main frame (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should detach child frames on navigation (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should support framesets (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should report frame from-inside shadow DOM (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should report frame.name() (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should report frame.parent() (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should report different frame instance when frame re-attaches (frame.spec.js)": [ - "FAIL" - ], - "Frame specs Frame Management should support url fragment (frame.spec.js)": [ - "PASS" - ], - "Frame specs Frame Management should support lazy frames (frame.spec.js)": [ - "FAIL" - ], - "Frame specs Frame.client should return the client instance (frame.spec.js)": [ - "PASS" - ], - "Emulate idle state changing idle state emulation causes change of the IdleDetector state (idle_override.spec.js)": [ - "FAIL" - ], - "ignoreHTTPSErrors should work (ignorehttpserrors.spec.js)": [ - "PASS" - ], - "ignoreHTTPSErrors should work with request interception (ignorehttpserrors.spec.js)": [ - "FAIL" - ], - "ignoreHTTPSErrors should work with mixed content (ignorehttpserrors.spec.js)": [ - "PASS" - ], - "ignoreHTTPSErrors Response.securityDetails should work (ignorehttpserrors.spec.js)": [ - "FAIL" - ], - "ignoreHTTPSErrors Response.securityDetails should be |null| for non-secure requests (ignorehttpserrors.spec.js)": [ - "PASS" - ], - "ignoreHTTPSErrors Response.securityDetails Network redirects should report SecurityDetails (ignorehttpserrors.spec.js)": [ - "PASS", "FAIL" - ], - "InjectedUtil tests should work (injected.spec.js)": [ - "PASS" - ], - "input tests input should upload the file (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should work when file input is attached to DOM (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should work when file input is not attached to DOM (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should respect timeout (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should respect default timeout when there is no custom timeout (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should prioritize exact timeout over default timeout (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should work with no timeout (input.spec.js)": [ - "SKIP" - ], - "input tests Page.waitForFileChooser should return the same file chooser when there are many watchdogs simultaneously (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should accept single file (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should be able to read selected file (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should be able to reset selected files with empty file list (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should not accept multiple files for single-file input (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should succeed even for non-existent files (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should error on read of non-existent files (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.accept should fail when accepting file chooser twice (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.cancel should cancel dialog (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.cancel should fail when canceling file chooser twice (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.isMultiple should work for single file pick (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.isMultiple should work for \"multiple\" (input.spec.js)": [ - "SKIP" - ], - "input tests FileChooser.isMultiple should work for \"webkitdirectory\" (input.spec.js)": [ - "SKIP" - ], - "JSHandle Page.evaluateHandle should work (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle Page.evaluateHandle should return the RemoteObject (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle Page.evaluateHandle should accept object handle as an argument (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle Page.evaluateHandle should accept object handle to primitive types (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle Page.evaluateHandle should warn about recursive objects (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle Page.evaluateHandle should accept object handle to unserializable value (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle Page.evaluateHandle should use the same JS wrappers (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.getProperty should work (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.jsonValue should work (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.jsonValue works with jsonValues that are not objects (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.jsonValue works with jsonValues that are primitives (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.jsonValue should not work with dates (jshandle.spec.js)": [ - "FAIL" - ], - "JSHandle JSHandle.jsonValue should throw for circular objects (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.getProperties should work (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.getProperties should return even non-own properties (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.asElement should work (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.asElement should return null for non-elements (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.asElement should return ElementHandle for TextNodes (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.toString should work for primitives (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.toString should work for complicated objects (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.toString should work with different subtypes (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.clickablePoint should work (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.clickablePoint should work for iframes (jshandle.spec.js)": [ - "PASS" - ], - "JSHandle JSHandle.click should work (jshandle.spec.js)": [ - "FAIL" - ], - "Keyboard should type into a textarea (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should press the metaKey (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should move with the arrow keys (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should send a character with ElementHandle.press (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard ElementHandle.press should support |text| option (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should send a character with sendCharacter (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should report shiftKey (keyboard.spec.js)": [ - "PASS", "FAIL" - ], - "Keyboard should report multiple modifiers (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should send proper codes while typing (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should send proper codes while typing with shift (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should not type canceled events (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should specify repeat property (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should type all kinds of characters (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should specify location (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should throw on unknown keys (keyboard.spec.js)": [ - "PASS" - ], - "Keyboard should type emoji (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should type emoji into an iframe (keyboard.spec.js)": [ - "FAIL" - ], - "Keyboard should press the meta key (keyboard.spec.js)": [ - "PASS", "FAIL" - ], - "Launcher specs Puppeteer BrowserFetcher should download and extract chrome linux binary (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer BrowserFetcher should download and extract firefox linux binary (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Browser.disconnect should reject navigation when browser closes (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Browser.disconnect should reject waitForSelector when browser closes (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Browser.close should terminate network waiters (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should reject all promises when browser is closed (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should reject if executable path is invalid (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch userDataDir option (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.launch userDataDir option restores preferences (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch userDataDir argument (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch userDataDir argument with non-existent dir (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore state (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore cookies (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should return the default arguments (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should report the correct product (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should work with no default arguments (launcher.spec.js)": [ - "FAIL", "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should filter out ignored default arguments in Chrome (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.launch should filter out ignored default argument in Firefox (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should have default URL when launching browser (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should have custom URL when launching browser (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should pass the timeout parameter to browser.waitForTarget (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should work with timeout = 0 (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should set the default viewport (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should disable the default viewport (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should take fullPage screenshots when defaultViewport is null (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should set the debugging port (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should not allow setting debuggingPort and pipe (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.launch should launch Chrome properly with --no-startup-window and waitForInitialPage=false (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.launch should be able to launch Chrome (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox (launcher.spec.js)": [ - "FAIL" - ], - "Launcher specs Puppeteer Puppeteer.connect should be able to connect multiple times to the same browser (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should be able to close remote browser (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should be able to connect to a browser with no page targets (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should support ignoreHTTPSErrors option (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should support targetFilter option in puppeteer.launch (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should support targetFilter option (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should be able to reconnect to a disconnected browser (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.connect should be able to connect to the same page simultaneously (launcher.spec.js)": [ - "PASS", "TIMEOUT" - ], - "Launcher specs Puppeteer Puppeteer.connect should be able to reconnect (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.executablePath should work (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.executablePath returns executablePath for channel (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.executablePath when PUPPETEER_EXECUTABLE_PATH is set its value is returned (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable exists returns /usr/bin/chromium-browser (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable exists and PUPPETEER_EXECUTABLE_PATH is set its value is returned (launcher.spec.js)": [ - "PASS" - ], - "Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable does not exist does not return /usr/bin/chromium-browser (launcher.spec.js)": [ - "SKIP" - ], - "Launcher specs Browser target events should work (launcher.spec.js)": [ - "FAIL" - ], - "Launcher specs Browser.Events.disconnected should be emitted when: browser gets closed, disconnected or underlying websocket gets closed (launcher.spec.js)": [ - "PASS" - ], - "Mouse should click the document (mouse.spec.js)": [ - "PASS" - ], - "Mouse should resize the textarea (mouse.spec.js)": [ - "PASS" - ], - "Mouse should select the text with mouse (mouse.spec.js)": [ - "PASS" - ], - "Mouse should trigger hover state (mouse.spec.js)": [ - "PASS", "FAIL" - ], - "Mouse should trigger hover state with removed window.Node (mouse.spec.js)": [ - "FAIL" - ], - "Mouse should set modifier keys on click (mouse.spec.js)": [ - "PASS" - ], - "Mouse should send mouse wheel events (mouse.spec.js)": [ - "FAIL" - ], - "Mouse should tween mouse movement (mouse.spec.js)": [ - "PASS", "FAIL" - ], - "Mouse should work with mobile viewports and cross process navigations (mouse.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work with anchor navigation (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work with redirects (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should navigate to about:blank (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should return response when page changes its URL after load (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work with subframes return 204 (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should fail when server returns 204 (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Page.goto should navigate to empty page with domcontentloaded (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work when page calls history API in beforeunload (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should navigate to empty page with networkidle0 (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Page.goto should navigate to empty page with networkidle2 (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Page.goto should fail when navigating to bad url (navigation.spec.js)": [ - "FAIL" - ], - "navigation Page.goto should fail when navigating to bad SSL (navigation.spec.js)": [ - "FAIL" - ], - "navigation Page.goto should fail when navigating to bad SSL after redirects (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should fail when main resources failed to load (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should fail when exceeding maximum navigation timeout (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should fail when exceeding default maximum navigation timeout (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should fail when exceeding default maximum timeout (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should prioritize default navigation timeout over default timeout (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should disable timeout when its set to 0 (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work when navigating to valid url (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work when navigating to data url (navigation.spec.js)": [ - "FAIL" - ], - "navigation Page.goto should work when navigating to 404 (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should return last response in redirect chain (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should wait for network idle to succeed navigation (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Page.goto should not leak listeners during navigation (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should not leak listeners during bad navigation (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should not leak listeners during navigation of 11 pages (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should navigate to dataURL and fire dataURL requests (navigation.spec.js)": [ - "FAIL" - ], - "navigation Page.goto should navigate to URL with hash and fire requests without hash (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should work with self requesting page (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should fail when navigating and show the url at the error message (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goto should send referer (navigation.spec.js)": [ - "FAIL" - ], - "navigation Page.waitForNavigation should work (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.waitForNavigation should work with both domcontentloaded and load (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.waitForNavigation should work with clicking on anchor links (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.waitForNavigation should work with history.pushState() (navigation.spec.js)": [ - "FAIL", "TIMEOUT" - ], - "navigation Page.waitForNavigation should work with history.replaceState() (navigation.spec.js)": [ - "FAIL", "TIMEOUT" - ], - "navigation Page.waitForNavigation should work with DOM history.back()/history.forward() (navigation.spec.js)": [ - "FAIL", "TIMEOUT" - ], - "navigation Page.waitForNavigation should work when subframe issues window.stop() (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Page.goBack should work (navigation.spec.js)": [ - "PASS" - ], - "navigation Page.goBack should work with HistoryAPI (navigation.spec.js)": [ - "FAIL" - ], - "navigation Frame.goto should navigate subframes (navigation.spec.js)": [ - "FAIL" - ], - "navigation Frame.goto should reject when frame detaches (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Frame.goto should return matching responses (navigation.spec.js)": [ - "FAIL" - ], - "navigation Frame.waitForNavigation should work (navigation.spec.js)": [ - "FAIL" - ], - "navigation Frame.waitForNavigation should fail when frame detaches (navigation.spec.js)": [ - "TIMEOUT" - ], - "navigation Page.reload should work (navigation.spec.js)": [ - "PASS" - ], - "network Page.Events.Request should fire for navigation requests (network.spec.js)": [ - "PASS" - ], - "network Page.Events.Request should fire for iframes (network.spec.js)": [ - "PASS" - ], - "network Page.Events.Request should fire for fetches (network.spec.js)": [ - "PASS" - ], - "network Request.frame should work for main frame navigation request (network.spec.js)": [ - "PASS" - ], - "network Request.frame should work for subframe navigation request (network.spec.js)": [ - "PASS" - ], - "network Request.frame should work for fetch requests (network.spec.js)": [ - "PASS" - ], - "network Request.headers should define Chrome as user agent header (network.spec.js)": [ - "SKIP" - ], - "network Request.headers should define Firefox as user agent header (network.spec.js)": [ - "PASS" - ], - "network Response.headers should work (network.spec.js)": [ - "PASS" - ], - "network Request.initiator should return the initiator (network.spec.js)": [ - "FAIL" - ], - "network Response.fromCache should return |false| for non-cached content (network.spec.js)": [ - "PASS" - ], - "network Response.fromCache should work (network.spec.js)": [ - "FAIL" - ], - "network Response.fromServiceWorker should return |false| for non-service-worker content (network.spec.js)": [ - "PASS" - ], - "network Response.fromServiceWorker Response.fromServiceWorker (network.spec.js)": [ - "TIMEOUT" - ], - "network Request.postData should work (network.spec.js)": [ - "FAIL" - ], - "network Request.postData should be |undefined| when there is no post data (network.spec.js)": [ - "PASS" - ], - "network Response.text should work (network.spec.js)": [ - "TIMEOUT" - ], - "network Response.text should return uncompressed text (network.spec.js)": [ - "TIMEOUT" - ], - "network Response.text should throw when requesting body of redirected response (network.spec.js)": [ - "FAIL" - ], - "network Response.text should wait until response completes (network.spec.js)": [ - "TIMEOUT" - ], - "network Response.json should work (network.spec.js)": [ - "TIMEOUT" - ], - "network Response.buffer should work (network.spec.js)": [ - "FAIL", "TIMEOUT" - ], - "network Response.buffer should work with compression (network.spec.js)": [ - "FAIL", "TIMEOUT" - ], - "network Response.buffer should throw if the response does not have a body (network.spec.js)": [ - "TIMEOUT" - ], - "network Response.statusText should work (network.spec.js)": [ - "PASS" - ], - "network Response.statusText handles missing status text (network.spec.js)": [ - "PASS" - ], - "network Response.timing returns timing information (network.spec.js)": [ - "FAIL" - ], - "network Network Events Page.Events.Request (network.spec.js)": [ - "FAIL", "PASS" - ], - "network Network Events Page.Events.RequestServedFromCache (network.spec.js)": [ - "FAIL" - ], - "network Network Events Page.Events.Response (network.spec.js)": [ - "PASS", "FAIL" - ], - "network Network Events Page.Events.RequestFailed (network.spec.js)": [ - "FAIL" - ], - "network Network Events Page.Events.RequestFinished (network.spec.js)": [ - "FAIL" - ], - "network Network Events should fire events in proper order (network.spec.js)": [ - "FAIL" - ], - "network Network Events should support redirects (network.spec.js)": [ - "FAIL" - ], - "network Request.isNavigationRequest should work (network.spec.js)": [ - "FAIL" - ], - "network Request.isNavigationRequest should work with request interception (network.spec.js)": [ - "FAIL" - ], - "network Request.isNavigationRequest should work when navigating to image (network.spec.js)": [ - "SKIP" - ], - "network Page.setExtraHTTPHeaders should work (network.spec.js)": [ - "FAIL" - ], - "network Page.setExtraHTTPHeaders should throw for non-string header values (network.spec.js)": [ - "PASS" - ], - "network Page.authenticate should work (network.spec.js)": [ - "TIMEOUT" - ], - "network Page.authenticate should fail if wrong credentials (network.spec.js)": [ - "FAIL" - ], - "network Page.authenticate should allow disable authentication (network.spec.js)": [ - "FAIL" - ], - "network Page.authenticate should not disable caching (network.spec.js)": [ - "FAIL" - ], - "network raw network headers Same-origin set-cookie navigation (network.spec.js)": [ - "PASS" - ], - "network raw network headers Same-origin set-cookie subresource (network.spec.js)": [ - "PASS", - "FAIL" - ], - "network raw network headers Cross-origin set-cookie (network.spec.js)": [ - "PASS", - "FAIL" - ], - "Page Page.close should reject all promises when page is closed (page.spec.js)": [ - "PASS" - ], - "Page Page.close should not be visible in browser.pages (page.spec.js)": [ - "PASS" - ], - "Page Page.close should run beforeunload if asked for (page.spec.js)": [ - "FAIL" - ], - "Page Page.close should *not* run beforeunload by default (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.close should set the page close state (page.spec.js)": [ - "PASS" - ], - "Page Page.close should terminate network waiters (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.Events.Load should fire when expected (page.spec.js)": [ - "PASS" - ], - "Page removing and adding event handlers should correctly fire event handlers as they are added and then removed (page.spec.js)": [ - "PASS" - ], - "Page removing and adding event handlers should correctly added and removed request events (page.spec.js)": [ - "PASS" - ], - "Page Page.Events.error should throw when page crashes (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.Events.Popup should work (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Popup should work with noopener (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.Events.Popup should work with clicking target=_blank and without rel=opener (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.Events.Popup should work with clicking target=_blank and with rel=opener (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.Events.Popup should work with fake-clicking target=_blank and rel=noopener (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.Events.Popup should work with clicking target=_blank and rel=noopener (page.spec.js)": [ - "TIMEOUT" - ], - "Page BrowserContext.overridePermissions should be prompt by default (page.spec.js)": [ - "PASS" - ], - "Page BrowserContext.overridePermissions should deny permission when not listed (page.spec.js)": [ - "FAIL" - ], - "Page BrowserContext.overridePermissions should fail when bad permission is given (page.spec.js)": [ - "PASS" - ], - "Page BrowserContext.overridePermissions should grant permission when listed (page.spec.js)": [ - "FAIL" - ], - "Page BrowserContext.overridePermissions should reset permissions (page.spec.js)": [ - "FAIL" - ], - "Page BrowserContext.overridePermissions should trigger permission onchange (page.spec.js)": [ - "FAIL" - ], - "Page BrowserContext.overridePermissions should isolate permissions between browser contexts (page.spec.js)": [ - "FAIL" - ], - "Page BrowserContext.overridePermissions should grant persistent-storage (page.spec.js)": [ - "FAIL" - ], - "Page Page.setGeolocation should work (page.spec.js)": [ - "FAIL" - ], - "Page Page.setGeolocation should throw when invalid longitude (page.spec.js)": [ - "PASS" - ], - "Page Page.setOfflineMode should work (page.spec.js)": [ - "FAIL" - ], - "Page Page.setOfflineMode should emulate navigator.onLine (page.spec.js)": [ - "FAIL" - ], - "Page ExecutionContext.queryObjects should work (page.spec.js)": [ - "FAIL" - ], - "Page ExecutionContext.queryObjects should work for non-blank page (page.spec.js)": [ - "FAIL" - ], - "Page ExecutionContext.queryObjects should fail for disposed handles (page.spec.js)": [ - "PASS" - ], - "Page ExecutionContext.queryObjects should fail primitive values as prototypes (page.spec.js)": [ - "PASS" - ], - "Page Page.Events.Console should work (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Console should work for different console API calls (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Console should not fail for window object (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Console should trigger correct Log (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Console should have location when fetch fails (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Console should have location and stack trace for console API calls (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.Console should not throw when there are console messages in detached iframes (page.spec.js)": [ - "PASS" - ], - "Page Page.Events.DOMContentLoaded should fire when expected (page.spec.js)": [ - "PASS" - ], - "Page Page.metrics should get metrics from a page (page.spec.js)": [ - "FAIL" - ], - "Page Page.metrics metrics event fired on console.timeStamp (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.waitForRequest should work (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForRequest should work with predicate (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForRequest should respect timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForRequest should respect default timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForRequest should work with no timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForResponse should work (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForResponse should respect timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForResponse should respect default timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForResponse should work with predicate (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForResponse should work with async predicate (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForResponse should work with no timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForNetworkIdle should work (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForNetworkIdle should respect timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForNetworkIdle should respect idleTime (page.spec.js)": [ - "PASS" - ], - "Page Page.waitForNetworkIdle should work with no timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.exposeFunction should work (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should throw exception in page context (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should support throwing \"null\" (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should be callable from-inside evaluateOnNewDocument (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should survive navigation (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should await returned promise (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should work on frames (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should work on frames before navigation (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should not throw when frames detach (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should work with complex objects (page.spec.js)": [ - "FAIL" - ], - "Page Page.exposeFunction should fallback to default export when passed a module object (page.spec.js)": [ - "FAIL" - ], - "Page Page.Events.PageError should fire (page.spec.js)": [ - "PASS" - ], - "Page Page.setUserAgent should work (page.spec.js)": [ - "PASS" - ], - "Page Page.setUserAgent should work for subframes (page.spec.js)": [ - "PASS" - ], - "Page Page.setUserAgent should emulate device user-agent (page.spec.js)": [ - "PASS" - ], - "Page Page.setUserAgent should work with additional userAgentMetdata (page.spec.js)": [ - "FAIL" - ], - "Page Page.setContent should work (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work with doctype (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work with HTML 4 doctype (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should respect timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should respect default navigation timeout (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should await resources to load (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work fast enough (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work with tricky content (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work with accents (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work with emojis (page.spec.js)": [ - "PASS" - ], - "Page Page.setContent should work with newline (page.spec.js)": [ - "PASS" - ], - "Page Page.setBypassCSP should bypass CSP meta tag (page.spec.js)": [ - "FAIL" - ], - "Page Page.setBypassCSP should bypass CSP header (page.spec.js)": [ - "FAIL" - ], - "Page Page.setBypassCSP should bypass after cross-process navigation (page.spec.js)": [ - "FAIL" - ], - "Page Page.setBypassCSP should bypass CSP in iframes as well (page.spec.js)": [ - "FAIL" - ], - "Page Page.addScriptTag should throw an error if no options are provided (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should work with a url (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should work with a url and type=module (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should work with a path and type=module (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should work with a content and type=module (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should throw an error if loading from url fail (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should work with a path (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should include sourcemap when path is provided (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should work with content (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should add id when provided (page.spec.js)": [ - "PASS" - ], - "Page Page.addScriptTag should throw when added with content to the CSP page (page.spec.js)": [ - "SKIP" - ], - "Page Page.addScriptTag should throw when added with URL to the CSP page (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should throw an error if no options are provided (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should work with a url (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should throw an error if loading from url fail (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should work with a path (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should include sourcemap when path is provided (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should work with content (page.spec.js)": [ - "PASS" - ], - "Page Page.addStyleTag should throw when added with content to the CSP page (page.spec.js)": [ - "TIMEOUT" - ], - "Page Page.addStyleTag should throw when added with URL to the CSP page (page.spec.js)": [ - "PASS" - ], - "Page Page.url should work (page.spec.js)": [ - "PASS" - ], - "Page Page.setJavaScriptEnabled should work (page.spec.js)": [ - "FAIL" - ], - "Page Page.setCacheEnabled should enable or disable the cache based on the state passed (page.spec.js)": [ - "FAIL" - ], - "Page Page.setCacheEnabled should stay disabled when toggling request interception on/off (page.spec.js)": [ - "FAIL" - ], - "Page printing to PDF can print to PDF and save to file (page.spec.js)": [ - "PASS" - ], - "Page printing to PDF can print to PDF and stream the result (page.spec.js)": [ - "PASS" - ], - "Page printing to PDF should respect timeout (page.spec.js)": [ - "SKIP" - ], - "Page Page.title should return the page title (page.spec.js)": [ - "PASS" - ], - "Page Page.select should select single option (page.spec.js)": [ - "PASS" - ], - "Page Page.select should select only first option (page.spec.js)": [ - "PASS" - ], - "Page Page.select should not throw when select causes navigation (page.spec.js)": [ - "PASS" - ], - "Page Page.select should select multiple options (page.spec.js)": [ - "PASS" - ], - "Page Page.select should respect event bubbling (page.spec.js)": [ - "PASS" - ], - "Page Page.select should throw when element is not a '); + + const element = (await page.$( + 'text/a' + )) as ElementHandle; + expect( + await element?.evaluate(e => { + return e.value; + }) + ).toBe('a'); + }); + it('should not query radio', async () => { + const {page} = getTestState(); + + await page.setContent(''); + + expect(await page.$('text/a')).toBeNull(); + }); + it('should query text spanning multiple elements', async () => { + const {page} = getTestState(); + + await page.setContent('
a b
'); + + const element = await page.$('text/a b'); + expect( + await element?.evaluate(e => { + return e.textContent; + }) + ).toBe('a b'); + }); + it('should clear caches', async () => { + const {page} = getTestState(); + + await page.setContent( + '
text
text
' + ); + const div = (await page.$('#target1')) as ElementHandle; + const input = (await page.$( + '#target2' + )) as ElementHandle; + + await div.evaluate(div => { + div.textContent = 'text'; + }); + expect( + await page.$eval(`text/text`, e => { + return e.id; + }) + ).toBe('target1'); + await div.evaluate(div => { + div.textContent = 'foo'; + }); + expect( + await page.$eval(`text/text`, e => { + return e.id; + }) + ).toBe('target2'); + await input.evaluate(input => { + input.value = ''; + }); + await input.type('foo'); + expect( + await page.$eval(`text/text`, e => { + return e.id; + }) + ).toBe('target3'); + + await div.evaluate(div => { + div.textContent = 'text'; + }); + await input.evaluate(input => { + input.value = ''; + }); + await input.type('text'); + expect( + await page.$$eval(`text/text`, es => { + return es.length; + }) + ).toBe(3); + await div.evaluate(div => { + div.textContent = 'foo'; + }); + expect( + await page.$$eval(`text/text`, es => { + return es.length; + }) + ).toBe(2); + await input.evaluate(input => { + input.value = ''; + }); + await input.type('foo'); + expect( + await page.$$eval(`text/text`, es => { + return es.length; + }) + ).toBe(1); + }); + }); + describe('in ElementHandles', function () { + it('should query existing element', async () => { + const {page} = getTestState(); + + await page.setContent('
a
'); + + const elementHandle = (await page.$('div'))!; + expect(await elementHandle.$(`text/a`)).toBeTruthy(); + expect((await elementHandle.$$(`text/a`)).length).toBe(1); + }); + + it('should return null for non-existing element', async () => { + const {page} = getTestState(); + + await page.setContent('
'); + + const elementHandle = (await page.$('div'))!; + expect(await elementHandle.$(`text/a`)).toBeFalsy(); + expect((await elementHandle.$$(`text/a`)).length).toBe(0); + }); + }); + }); + describe('XPath selectors', function () { describe('in Page', function () { it('should query existing element', async () => { diff --git a/remote/test/puppeteer/test/src/screenshot.spec.ts b/remote/test/puppeteer/test/src/screenshot.spec.ts index 1d0b795a3baf..d6cf443e07f2 100644 --- a/remote/test/puppeteer/test/src/screenshot.spec.ts +++ b/remote/test/puppeteer/test/src/screenshot.spec.ts @@ -19,8 +19,6 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itHeadfulOnly, - itChromeOnly, } from './mocha-utils.js'; describe('Screenshots', function () { @@ -67,23 +65,20 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-rect-scale2.png'); }); - it( - 'should get screenshot bigger than the viewport', - async () => { - const {page, server} = getTestState(); - await page.setViewport({width: 50, height: 50}); - await page.goto(server.PREFIX + '/grid.html'); - const screenshot = await page.screenshot({ - clip: { - x: 25, - y: 25, - width: 100, - height: 100, - }, - }); - expect(screenshot).toBeGolden('screenshot-offscreen-clip.png'); - } - ); + it('should get screenshot bigger than the viewport', async () => { + const {page, server} = getTestState(); + await page.setViewport({width: 50, height: 50}); + await page.goto(server.PREFIX + '/grid.html'); + const screenshot = await page.screenshot({ + clip: { + x: 25, + y: 25, + width: 100, + height: 100, + }, + }); + expect(screenshot).toBeGolden('screenshot-offscreen-clip.png'); + }); it('should run in parallel', async () => { const {page, server} = getTestState(); @@ -205,7 +200,7 @@ describe('Screenshots', function () { 'screenshot-sanity.png' ); }); - itHeadfulOnly('should work in "fromSurface: false" mode', async () => { + it('should work in "fromSurface: false" mode', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -230,7 +225,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-bounding-box.png'); }); - itChromeOnly('should work with a null viewport', async () => { + it('should work with a null viewport', async () => { const {defaultBrowserOptions, puppeteer, server} = getTestState(); const browser = await puppeteer.launch({ @@ -270,14 +265,12 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-padding-border.png'); }); - it( - 'should capture full element when larger than viewport', - async () => { - const {page} = getTestState(); + it('should capture full element when larger than viewport', async () => { + const {page} = getTestState(); - await page.setViewport({width: 500, height: 500}); + await page.setViewport({width: 500, height: 500}); - await page.setContent(` + await page.setContent(` something above
`); - const elementHandle = (await page.$('div.to-screenshot'))!; - const screenshot = await elementHandle.screenshot(); - expect(screenshot).toBeGolden( - 'screenshot-element-larger-than-viewport.png' - ); + const elementHandle = (await page.$('div.to-screenshot'))!; + const screenshot = await elementHandle.screenshot(); + expect(screenshot).toBeGolden( + 'screenshot-element-larger-than-viewport.png' + ); - expect( - await page.evaluate(() => { - return { - w: window.innerWidth, - h: window.innerHeight, - }; - }) - ).toEqual({w: 500, h: 500}); - } - ); + expect( + await page.evaluate(() => { + return { + w: window.innerWidth, + h: window.innerHeight, + }; + }) + ).toEqual({w: 500, h: 500}); + }); it('should scroll element into view', async () => { const {page} = getTestState(); diff --git a/remote/test/puppeteer/test/src/target.spec.ts b/remote/test/puppeteer/test/src/target.spec.ts index 71501ffeeb7b..b92987c6ab28 100644 --- a/remote/test/puppeteer/test/src/target.spec.ts +++ b/remote/test/puppeteer/test/src/target.spec.ts @@ -20,11 +20,11 @@ import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {Target} from '../../lib/cjs/puppeteer/common/Target.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils from './utils.js'; + const {waitEvent} = utils; describe('Target', function () { @@ -79,10 +79,7 @@ describe('Target', function () { ).toBe('Hello world'); expect(await originalPage.$('body')).toBeTruthy(); }); - // This test should be skipped in mozilla-central (Firefox). - // It intermittently makes some tests fail and triggers errors in the test hooks. - // See https://bugzilla.mozilla.org/show_bug.cgi?id=1748255 - itFailsFirefox('should be able to use async waitForTarget', async () => { + it('should be able to use async waitForTarget', async () => { const {page, server, context} = getTestState(); const [otherPage] = await Promise.all([ @@ -104,88 +101,82 @@ describe('Target', function () { ); expect(page).not.toEqual(otherPage); }); - it( - 'should report when a new page is created and closed', - async () => { - const {page, server, context} = getTestState(); + it('should report when a new page is created and closed', async () => { + const {page, server, context} = getTestState(); - const [otherPage] = await Promise.all([ - context - .waitForTarget(target => { - return target.url() === server.CROSS_PROCESS_PREFIX + '/empty.html'; - }) - .then(target => { - return target.page(); - }), - page.evaluate((url: string) => { - return window.open(url); - }, server.CROSS_PROCESS_PREFIX + '/empty.html'), - ]); - expect(otherPage!.url()).toContain(server.CROSS_PROCESS_PREFIX); - expect( - await otherPage!.evaluate(() => { - return ['Hello', 'world'].join(' '); + const [otherPage] = await Promise.all([ + context + .waitForTarget(target => { + return target.url() === server.CROSS_PROCESS_PREFIX + '/empty.html'; }) - ).toBe('Hello world'); - expect(await otherPage!.$('body')).toBeTruthy(); - - let allPages = await context.pages(); - expect(allPages).toContain(page); - expect(allPages).toContain(otherPage); - - const closePagePromise = new Promise(fulfill => { - return context.once('targetdestroyed', target => { - return fulfill(target.page()); - }); - }); - await otherPage!.close(); - expect(await closePagePromise).toBe(otherPage); - - allPages = (await Promise.all( - context.targets().map(target => { + .then(target => { return target.page(); - }) - )) as Page[]; - expect(allPages).toContain(page); - expect(allPages).not.toContain(otherPage); - } - ); - it( - 'should report when a service worker is created and destroyed', - async () => { - const {page, server, context} = getTestState(); + }), + page.evaluate((url: string) => { + return window.open(url); + }, server.CROSS_PROCESS_PREFIX + '/empty.html'), + ]); + expect(otherPage!.url()).toContain(server.CROSS_PROCESS_PREFIX); + expect( + await otherPage!.evaluate(() => { + return ['Hello', 'world'].join(' '); + }) + ).toBe('Hello world'); + expect(await otherPage!.$('body')).toBeTruthy(); - await page.goto(server.EMPTY_PAGE); - const createdTarget = new Promise(fulfill => { - return context.once('targetcreated', target => { - return fulfill(target); - }); + let allPages = await context.pages(); + expect(allPages).toContain(page); + expect(allPages).toContain(otherPage); + + const closePagePromise = new Promise(fulfill => { + return context.once('targetdestroyed', target => { + return fulfill(target.page()); }); + }); + await otherPage!.close(); + expect(await closePagePromise).toBe(otherPage); - await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); + allPages = (await Promise.all( + context.targets().map(target => { + return target.page(); + }) + )) as Page[]; + expect(allPages).toContain(page); + expect(allPages).not.toContain(otherPage); + }); + it('should report when a service worker is created and destroyed', async () => { + const {page, server, context} = getTestState(); - expect((await createdTarget).type()).toBe('service_worker'); - expect((await createdTarget).url()).toBe( - server.PREFIX + '/serviceworkers/empty/sw.js' - ); - - const destroyedTarget = new Promise(fulfill => { - return context.once('targetdestroyed', target => { - return fulfill(target); - }); + await page.goto(server.EMPTY_PAGE); + const createdTarget = new Promise(fulfill => { + return context.once('targetcreated', target => { + return fulfill(target); }); - await page.evaluate(() => { - return ( - globalThis as unknown as { - registrationPromise: Promise<{unregister: () => void}>; - } - ).registrationPromise.then((registration: any) => { - return registration.unregister(); - }); + }); + + await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); + + expect((await createdTarget).type()).toBe('service_worker'); + expect((await createdTarget).url()).toBe( + server.PREFIX + '/serviceworkers/empty/sw.js' + ); + + const destroyedTarget = new Promise(fulfill => { + return context.once('targetdestroyed', target => { + return fulfill(target); }); - expect(await destroyedTarget).toBe(await createdTarget); - } - ); + }); + await page.evaluate(() => { + return ( + globalThis as unknown as { + registrationPromise: Promise<{unregister: () => void}>; + } + ).registrationPromise.then((registration: any) => { + return registration.unregister(); + }); + }); + expect(await destroyedTarget).toBe(await createdTarget); + }); it('should create a worker from a service worker', async () => { const {page, server, context} = getTestState(); @@ -271,36 +262,33 @@ describe('Target', function () { expect(targetChanged).toBe(false); context.removeListener('targetchanged', listener); }); - it( - 'should not crash while redirecting if original request was missed', - async () => { - const {page, server, context} = getTestState(); + it('should not crash while redirecting if original request was missed', async () => { + const {page, server, context} = getTestState(); - let serverResponse!: ServerResponse; - server.setRoute('/one-style.css', (_req, res) => { - return (serverResponse = res); - }); - // Open a new page. Use window.open to connect to the page later. - await Promise.all([ - page.evaluate((url: string) => { - return window.open(url); - }, server.PREFIX + '/one-style.html'), - server.waitForRequest('/one-style.css'), - ]); - // Connect to the opened page. - const target = await context.waitForTarget(target => { - return target.url().includes('one-style.html'); - }); - const newPage = (await target.page())!; - // Issue a redirect. - serverResponse.writeHead(302, {location: '/injectedstyle.css'}); - serverResponse.end(); - // Wait for the new page to load. - await waitEvent(newPage, 'load'); - // Cleanup. - await newPage.close(); - } - ); + let serverResponse!: ServerResponse; + server.setRoute('/one-style.css', (_req, res) => { + return (serverResponse = res); + }); + // Open a new page. Use window.open to connect to the page later. + await Promise.all([ + page.evaluate((url: string) => { + return window.open(url); + }, server.PREFIX + '/one-style.html'), + server.waitForRequest('/one-style.css'), + ]); + // Connect to the opened page. + const target = await context.waitForTarget(target => { + return target.url().includes('one-style.html'); + }); + const newPage = (await target.page())!; + // Issue a redirect. + serverResponse.writeHead(302, {location: '/injectedstyle.css'}); + serverResponse.end(); + // Wait for the new page to load. + await waitEvent(newPage, 'load'); + // Cleanup. + await newPage.close(); + }); it('should have an opener', async () => { const {page, server, context} = getTestState(); diff --git a/remote/test/puppeteer/test/src/tracing.spec.ts b/remote/test/puppeteer/test/src/tracing.spec.ts index 2a7a4675cbc2..b271a17243ae 100644 --- a/remote/test/puppeteer/test/src/tracing.spec.ts +++ b/remote/test/puppeteer/test/src/tracing.spec.ts @@ -17,11 +17,11 @@ import fs from 'fs'; import path from 'path'; import expect from 'expect'; -import {getTestState, describeChromeOnly} from './mocha-utils.js'; -import {Browser} from '../../lib/cjs/puppeteer/common/Browser.js'; +import {getTestState} from './mocha-utils.js'; +import {Browser} from '../../lib/cjs/puppeteer/api/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; -describeChromeOnly('Tracing', function () { +describe('Tracing', function () { let outputFile!: string; let browser!: Browser; let page!: Page; diff --git a/remote/test/puppeteer/test/src/waittask.spec.ts b/remote/test/puppeteer/test/src/waittask.spec.ts index 0535a49e4e00..6508f06a93e5 100644 --- a/remote/test/puppeteer/test/src/waittask.spec.ts +++ b/remote/test/puppeteer/test/src/waittask.spec.ts @@ -17,6 +17,7 @@ import expect from 'expect'; import {isErrorLike} from '../../lib/cjs/puppeteer/util/ErrorLike.js'; import { + createTimeout, getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, @@ -31,9 +32,9 @@ describe('waittask specs', function () { it('should accept a string', async () => { const {page} = getTestState(); - const watchdog = page.waitForFunction('window.__FOO === 1'); + const watchdog = page.waitForFunction('self.__FOO === 1'); await page.evaluate(() => { - return ((globalThis as any).__FOO = 1); + return ((self as unknown as {__FOO: number}).__FOO = 1); }); await watchdog; }); @@ -46,61 +47,25 @@ describe('waittask specs', function () { await page.waitForFunction(() => { if (!(globalThis as any).__RELOADED) { window.location.reload(); + return false; } return true; }); }); it('should poll on interval', async () => { const {page} = getTestState(); - - let success = false; const startTime = Date.now(); const polling = 100; - const watchdog = page - .waitForFunction( - () => { - return (globalThis as any).__FOO === 'hit'; - }, - { - polling, - } - ) - .then(() => { - return (success = true); - }); + const watchdog = page.waitForFunction( + () => { + return (globalThis as any).__FOO === 'hit'; + }, + {polling} + ); await page.evaluate(() => { - return ((globalThis as any).__FOO = 'hit'); - }); - expect(success).toBe(false); - await page.evaluate(() => { - return document.body.appendChild(document.createElement('div')); - }); - await watchdog; - expect(Date.now() - startTime).not.toBeLessThan(polling / 2); - }); - it('should poll on interval async', async () => { - const {page} = getTestState(); - let success = false; - const startTime = Date.now(); - const polling = 100; - const watchdog = page - .waitForFunction( - async () => { - return (globalThis as any).__FOO === 'hit'; - }, - { - polling, - } - ) - .then(() => { - return (success = true); - }); - await page.evaluate(async () => { - return ((globalThis as any).__FOO = 'hit'); - }); - expect(success).toBe(false); - await page.evaluate(async () => { - return document.body.appendChild(document.createElement('div')); + setTimeout(() => { + (globalThis as any).__FOO = 'hit'; + }, 50); }); await watchdog; expect(Date.now() - startTime).not.toBeLessThan(polling / 2); @@ -212,26 +177,6 @@ describe('waittask specs', function () { ]); expect(error).toBeUndefined(); }); - it('should throw on bad polling value', async () => { - const {page} = getTestState(); - - let error!: Error; - try { - await page.waitForFunction( - () => { - return !!document.body; - }, - { - polling: 'unknown', - } - ); - } catch (error_) { - if (isErrorLike(error_)) { - error = error_ as Error; - } - } - expect(error?.message).toContain('polling'); - }); it('should throw negative polling interval', async () => { const {page} = getTestState(); @@ -299,23 +244,34 @@ describe('waittask specs', function () { const {page, puppeteer} = getTestState(); let error!: Error; - await page.waitForFunction('false', {timeout: 10}).catch(error_ => { - return (error = error_); - }); + await page + .waitForFunction( + () => { + return false; + }, + {timeout: 10} + ) + .catch(error_ => { + return (error = error_); + }); expect(error).toBeInstanceOf(puppeteer.errors.TimeoutError); - expect(error?.message).toContain('waiting for function failed: timeout'); + expect(error?.message).toContain('Waiting failed: 10ms exceeded'); }); it('should respect default timeout', async () => { const {page, puppeteer} = getTestState(); page.setDefaultTimeout(1); let error!: Error; - await page.waitForFunction('false').catch(error_ => { - return (error = error_); - }); + await page + .waitForFunction(() => { + return false; + }) + .catch(error_ => { + return (error = error_); + }); expect(error).toBeInstanceOf(puppeteer.errors.TimeoutError); - expect(error?.message).toContain('waiting for function failed: timeout'); + expect(error?.message).toContain('Waiting failed: 1ms exceeded'); }); it('should disable timeout when its set to 0', async () => { const {page} = getTestState(); @@ -341,7 +297,9 @@ describe('waittask specs', function () { let fooFound = false; const waitForFunction = page - .waitForFunction('globalThis.__FOO === 1') + .waitForFunction(() => { + return (globalThis as unknown as {__FOO: number}).__FOO === 1; + }) .then(() => { return (fooFound = true); }); @@ -464,21 +422,18 @@ describe('waittask specs', function () { await watchdog; }); - it( - 'Page.waitForSelector is shortcut for main frame', - async () => { - const {page, server} = getTestState(); + it('Page.waitForSelector is shortcut for main frame', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - await attachFrame(page, 'frame1', server.EMPTY_PAGE); - const otherFrame = page.frames()[1]!; - const watchdog = page.waitForSelector('div'); - await otherFrame.evaluate(addElement, 'div'); - await page.evaluate(addElement, 'div'); - const eHandle = await watchdog; - expect(eHandle?.frame).toBe(page.mainFrame()); - } - ); + await page.goto(server.EMPTY_PAGE); + await attachFrame(page, 'frame1', server.EMPTY_PAGE); + const otherFrame = page.frames()[1]!; + const watchdog = page.waitForSelector('div'); + await otherFrame.evaluate(addElement, 'div'); + await page.evaluate(addElement, 'div'); + const eHandle = await watchdog; + expect(eHandle?.frame).toBe(page.mainFrame()); + }); it('should run in specified frame', async () => { const {page, server} = getTestState(); @@ -525,113 +480,186 @@ describe('waittask specs', function () { await waitForSelector; expect(boxFound).toBe(true); }); - it('should wait for visible', async () => { + it('should wait for element to be visible (display)', async () => { const {page} = getTestState(); - let divFound = false; - const waitForSelector = page - .waitForSelector('div', {visible: true}) - .then(() => { - return (divFound = true); - }); - await page.setContent( - `
1
` - ); - expect(divFound).toBe(false); - await page.evaluate(() => { - return document.querySelector('div')?.style.removeProperty('display'); + const promise = page.waitForSelector('div', {visible: true}); + await page.setContent('
text
'); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; }); - expect(divFound).toBe(false); - await page.evaluate(() => { - return document - .querySelector('div') - ?.style.removeProperty('visibility'); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.removeProperty('display'); }); - expect(await waitForSelector).toBe(true); - expect(divFound).toBe(true); + await expect(promise).resolves.toBeTruthy(); }); - it('should wait for visible recursively', async () => { + it('should wait for element to be visible (visibility)', async () => { const {page} = getTestState(); - let divVisible = false; - const waitForSelector = page - .waitForSelector('div#inner', {visible: true}) - .then(() => { - return (divVisible = true); - }); + const promise = page.waitForSelector('div', {visible: true}); + await page.setContent('
text
'); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('visibility', 'collapse'); + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.removeProperty('visibility'); + }); + await expect(promise).resolves.toBeTruthy(); + }); + it('should wait for element to be visible (bounding box)', async () => { + const {page} = getTestState(); + + const promise = page.waitForSelector('div', {visible: true}); + await page.setContent('
text
'); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('height', '0'); + e.style.removeProperty('width'); + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('position', 'absolute'); + e.style.setProperty('right', '100vw'); + e.style.removeProperty('height'); + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('left', '100vw'); + e.style.removeProperty('right'); + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('top', '100vh'); + e.style.removeProperty('left'); + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('bottom', '100vh'); + e.style.removeProperty('top'); + }); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + // Just peeking + e.style.setProperty('bottom', '99vh'); + }); + await expect(promise).resolves.toBeTruthy(); + }); + it('should wait for element to be visible recursively', async () => { + const {page} = getTestState(); + + const promise = page.waitForSelector('div#inner', { + visible: true, + }); await page.setContent( `
hi
` ); - expect(divVisible).toBe(false); - await page.evaluate(() => { - return document.querySelector('div')?.style.removeProperty('display'); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; }); - expect(divVisible).toBe(false); - await page.evaluate(() => { - return document - .querySelector('div') - ?.style.removeProperty('visibility'); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + return e.style.removeProperty('display'); }); - expect(await waitForSelector).toBe(true); - expect(divVisible).toBe(true); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + return e.style.removeProperty('visibility'); + }); + await expect(promise).resolves.toBeTruthy(); }); - it('hidden should wait for visibility: hidden', async () => { + it('should wait for element to be hidden (visibility)', async () => { const {page} = getTestState(); - let divHidden = false; - await page.setContent(`
`); - const waitForSelector = page - .waitForSelector('div', {hidden: true}) - .then(() => { - return (divHidden = true); - }); - await page.waitForSelector('div'); // do a round trip - expect(divHidden).toBe(false); - await page.evaluate(() => { - return document - .querySelector('div') - ?.style.setProperty('visibility', 'hidden'); + const promise = page.waitForSelector('div', {hidden: true}); + await page.setContent(`
text
`); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; }); - expect(await waitForSelector).toBe(true); - expect(divHidden).toBe(true); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + return e.style.setProperty('visibility', 'hidden'); + }); + await expect(promise).resolves.toBeTruthy(); }); - it('hidden should wait for display: none', async () => { + it('should wait for element to be hidden (display)', async () => { const {page} = getTestState(); - let divHidden = false; - await page.setContent(`
`); - const waitForSelector = page - .waitForSelector('div', {hidden: true}) - .then(() => { - return (divHidden = true); - }); - await page.waitForSelector('div'); // do a round trip - expect(divHidden).toBe(false); - await page.evaluate(() => { - return document - .querySelector('div') - ?.style.setProperty('display', 'none'); + const promise = page.waitForSelector('div', {hidden: true}); + await page.setContent(`
text
`); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; }); - expect(await waitForSelector).toBe(true); - expect(divHidden).toBe(true); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + return e.style.setProperty('display', 'none'); + }); + await expect(promise).resolves.toBeTruthy(); }); - it('hidden should wait for removal', async () => { + it('should wait for element to be hidden (bounding box)', async () => { const {page} = getTestState(); - await page.setContent(`
`); - let divRemoved = false; - const waitForSelector = page - .waitForSelector('div', {hidden: true}) - .then(() => { - return (divRemoved = true); - }); - await page.waitForSelector('div'); // do a round trip - expect(divRemoved).toBe(false); - await page.evaluate(() => { - return document.querySelector('div')?.remove(); + const promise = page.waitForSelector('div', {hidden: true}); + await page.setContent('
text
'); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; }); - expect(await waitForSelector).toBe(true); - expect(divRemoved).toBe(true); + await expect( + Promise.race([promise, createTimeout(40)]) + ).resolves.toBeFalsy(); + await element.evaluate(e => { + e.style.setProperty('height', '0'); + }); + await expect(promise).resolves.toBeTruthy(); + }); + it('should wait for element to be hidden (removal)', async () => { + const {page} = getTestState(); + + const promise = page.waitForSelector('div', {hidden: true}); + await page.setContent(`
text
`); + const element = await page.evaluateHandle(() => { + return document.getElementsByTagName('div')[0]!; + }); + await expect( + Promise.race([promise, createTimeout(40, true)]) + ).resolves.toBeTruthy(); + await element.evaluate(e => { + e.remove(); + }); + await expect(promise).resolves.toBeFalsy(); }); it('should return null if waiting to hide non-existing element', async () => { const {page} = getTestState(); @@ -650,13 +678,13 @@ describe('waittask specs', function () { }); expect(error).toBeInstanceOf(puppeteer.errors.TimeoutError); expect(error?.message).toContain( - 'waiting for selector `div` failed: timeout' + 'Waiting for selector `div` failed: Waiting failed: 10ms exceeded' ); }); it('should have an error message specifically for awaiting an element to be hidden', async () => { const {page} = getTestState(); - await page.setContent(`
`); + await page.setContent(`
text
`); let error!: Error; await page .waitForSelector('div', {hidden: true, timeout: 10}) @@ -665,7 +693,7 @@ describe('waittask specs', function () { }); expect(error).toBeTruthy(); expect(error?.message).toContain( - 'waiting for selector `div` to be hidden failed: timeout' + 'Waiting for selector `div` failed: Waiting failed: 10ms exceeded' ); }); @@ -701,9 +729,11 @@ describe('waittask specs', function () { await page.waitForSelector('.zombo', {timeout: 10}).catch(error_ => { return (error = error_); }); - expect(error?.stack).toContain('waiting for selector `.zombo` failed'); + expect(error?.stack).toContain( + 'Waiting for selector `.zombo` failed: Waiting failed: 10ms exceeded' + ); // The extension is ts here as Mocha maps back via sourcemaps. - expect(error?.stack).toContain('waittask.spec.ts'); + expect(error?.stack).toContain('WaitTask.ts'); }); }); @@ -733,9 +763,7 @@ describe('waittask specs', function () { return (error = error_); }); expect(error).toBeInstanceOf(puppeteer.errors.TimeoutError); - expect(error?.message).toContain( - 'waiting for selector `.//div` failed: timeout 10ms exceeded' - ); + expect(error?.message).toContain('Waiting failed: 10ms exceeded'); }); it('should run in specified frame', async () => { const {page, server} = getTestState(); @@ -772,7 +800,7 @@ describe('waittask specs', function () { const {page} = getTestState(); let divHidden = false; - await page.setContent(`
`); + await page.setContent(`
text
`); const waitForXPath = page .waitForXPath('//div', {hidden: true}) .then(() => { diff --git a/remote/test/puppeteer/test/src/worker.spec.ts b/remote/test/puppeteer/test/src/worker.spec.ts index c45febc3d4ce..c279f6efb99c 100644 --- a/remote/test/puppeteer/test/src/worker.spec.ts +++ b/remote/test/puppeteer/test/src/worker.spec.ts @@ -18,14 +18,13 @@ import expect from 'expect'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import {WebWorker} from '../../lib/cjs/puppeteer/common/WebWorker.js'; import { - describeFailsFirefox, getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import {waitEvent} from './utils.js'; -describeFailsFirefox('Workers', function () { +describe('Workers', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('Page.workers', async () => { diff --git a/remote/test/puppeteer/test/tsconfig.json b/remote/test/puppeteer/test/tsconfig.json index 962644f958df..6842bd8ed42f 100644 --- a/remote/test/puppeteer/test/tsconfig.json +++ b/remote/test/puppeteer/test/tsconfig.json @@ -9,7 +9,8 @@ }, "include": ["src"], "references": [ - {"path": "../tsconfig.lib.json"}, - {"path": "../utils/testserver/tsconfig.json"} + {"path": "../src/tsconfig.cjs.json"}, + {"path": "../utils/testserver/tsconfig.json"}, + {"path": "../utils/mochaRunner/tsconfig.json"} ] } diff --git a/remote/test/puppeteer/utils/generate-matrix.js b/remote/test/puppeteer/utils/generate-matrix.js new file mode 100644 index 000000000000..1cb65d34914d --- /dev/null +++ b/remote/test/puppeteer/utils/generate-matrix.js @@ -0,0 +1,43 @@ +const fs = require('fs'); + +const data = JSON.parse(fs.readFileSync('./test/TestSuites.json', 'utf-8')); + +/** + * @param {string} platform + * @returns {string} + */ +function mapPlatform(platform) { + switch (platform) { + case 'linux': + return 'ubuntu-latest'; + case 'win32': + return 'windows-latest'; + case 'darwin': + return 'macos-latest'; + default: + throw new Error('Unsupported platform'); + } +} + +const result = []; +for (const suite of data.testSuites) { + for (const platform of suite.platforms) { + if (platform === 'linux' && suite.id !== 'firefox-bidi') { + for (const node of [14, 16, 18]) { + result.push(`- name: ${suite.id} + machine: ${mapPlatform(platform)} + xvfb: true + node: ${node} + suite: ${suite.id}`); + } + } else { + result.push(`- name: ${suite.id} + machine: ${mapPlatform(platform)} + xvfb: ${platform === 'linux'} + node: 18 + suite: ${suite.id}`); + } + } +} + +console.log(result.join('\n')); diff --git a/remote/test/puppeteer/utils/generate_sources.ts b/remote/test/puppeteer/utils/generate_sources.ts index 1507d11e5e07..0e0ad1f67600 100644 --- a/remote/test/puppeteer/utils/generate_sources.ts +++ b/remote/test/puppeteer/utils/generate_sources.ts @@ -6,7 +6,7 @@ import {sync as glob} from 'glob'; import path from 'path'; import {job} from './internal/job.js'; -const INCLUDED_FOLDERS = ['common', 'node', 'generated', 'util']; +const INCLUDED_FOLDERS = ['common', 'node', 'generated', 'util', 'api']; (async () => { await job('', async ({outputs}) => { @@ -36,7 +36,7 @@ const INCLUDED_FOLDERS = ['common', 'node', 'generated', 'util']; outdir: tmp, format: 'cjs', platform: 'browser', - target: 'ES2019', + target: 'ES2022', }); const baseName = path.basename(input); const content = await readFile( diff --git a/remote/test/puppeteer/utils/mochaRunner/README.md b/remote/test/puppeteer/utils/mochaRunner/README.md new file mode 100644 index 000000000000..efbafbc3b5e1 --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/README.md @@ -0,0 +1,47 @@ +# Mocha Runner + +Mocha Runner is a test runner on top of mocha. It uses `/test/TestSuites.json` and `/test/TestExpectations.json` files to run mocha tests in multiple configurations and interpret results. + +## Running tests for Mocha Runner itself. + +``` +npm run build:dev && npx c8 node utils/mochaRunner/lib/test.js +``` + +## Running tests using Mocha Runner + +``` +npm run build:dev && npm run test +``` + +By default, the runner runs all test suites applicable to the current platform. +To pick a test suite, provide the `--test-suite` arguments. For example, + +``` +npm run build:dev && npm run test -- --test-suite chrome-headless +``` + +## TestSuites.json + +Define test suites via the `testSuites` attribute. `parameters` can be used in the `TestExpectations.json` to disable tests +based on parameters. The meaning for parameters is defined in `parameterDefinitons` which tell what env object corresponds +to the given parameter. + +## TestExpectations.json + +An expectation looks like this: + +``` + { + "testIdPattern": "[accessibility.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + } +``` + +`testIdPattern` defines a string that will be used to prefix-match tests. `platforms` defines the platforms the expectation is for (`or`-logic). +`parameters` defines the parameters that the test has to match (`and`-logic). `expectations` is the list of test results that are considered to be acceptable. + +Currently, expectations are updated manually. The test runner outputs the suggested changes to the expectation file if the test run does not match +expectations. diff --git a/remote/test/puppeteer/utils/mochaRunner/src/interface.ts b/remote/test/puppeteer/utils/mochaRunner/src/interface.ts new file mode 100644 index 000000000000..df48855bbca5 --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/src/interface.ts @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Mocha from 'mocha'; +import commonInterface from 'mocha/lib/interfaces/common'; +import {getTestId} from './utils.js'; + +type SuiteFunction = ((this: Mocha.Suite) => void) | undefined; +type ExclusiveSuiteFunction = (this: Mocha.Suite) => void; + +const skippedTests: Array<{testIdPattern: string; skip: true}> = process.env[ + 'PUPPETEER_SKIPPED_TEST_CONFIG' +] + ? JSON.parse(process.env['PUPPETEER_SKIPPED_TEST_CONFIG']) + : []; + +skippedTests.reverse(); + +function shouldSkipTest(test: Mocha.Test): boolean { + const testIdForFileName = getTestId(test.file!); + const testIdForTestName = getTestId(test.file!, test.fullTitle()); + // TODO: more efficient lookup. + const defintion = skippedTests.find(skippedTest => { + return ( + '' === skippedTest.testIdPattern || + testIdForFileName === skippedTest.testIdPattern || + testIdForTestName === skippedTest.testIdPattern + ); + }); + if (defintion && defintion.skip) { + return true; + } + return false; +} + +function customBDDInterface(suite: Mocha.Suite) { + const suites = [suite]; + + suite.on( + Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE, + function (context, file, mocha) { + const common = commonInterface(suites, context, mocha); + + context['before'] = common.before; + context['after'] = common.after; + context['beforeEach'] = common.beforeEach; + context['afterEach'] = common.afterEach; + if (mocha.options.delay) { + context['run'] = common.runWithSuite(suite); + } + function describe(title: string, fn: SuiteFunction) { + return common.suite.create({ + title: title, + file: file, + fn: fn, + }); + } + describe.only = function (title: string, fn: ExclusiveSuiteFunction) { + return common.suite.only({ + title: title, + file: file, + fn: fn, + }); + }; + + describe.skip = function (title: string, fn: SuiteFunction) { + return common.suite.skip({ + title: title, + file: file, + fn: fn, + }); + }; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + context['describe'] = describe; + + function it(title: string, fn: Mocha.TestFunction) { + const suite = suites[0]!; + const test = new Mocha.Test(title, suite.isPending() ? undefined : fn); + test.file = file; + test.parent = suite; + if (shouldSkipTest(test)) { + const test = new Mocha.Test(title); + test.file = file; + suite.addTest(test); + return test; + } else { + suite.addTest(test); + return test; + } + } + + it.only = function (title: string, fn: Mocha.TestFunction) { + return common.test.only(mocha, context['it'](title, fn)); + }; + + it.skip = function (title: string) { + return context['it'](title); + }; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + context.it = it; + } + ); +} + +customBDDInterface.description = 'Custom BDD'; + +module.exports = customBDDInterface; diff --git a/remote/test/puppeteer/utils/mochaRunner/src/main.ts b/remote/test/puppeteer/utils/mochaRunner/src/main.ts new file mode 100644 index 000000000000..9e2b0ab2d44a --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/src/main.ts @@ -0,0 +1,248 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TestExpectation, + MochaResults, + zTestSuiteFile, + zPlatform, + TestSuite, + TestSuiteFile, + Platform, +} from './types.js'; + +import path from 'path'; +import fs from 'fs'; +import os from 'os'; +import {spawn, SpawnOptions} from 'node:child_process'; +import { + extendProcessEnv, + filterByPlatform, + prettyPrintJSON, + readJSON, + filterByParameters, + getExpectationUpdates, +} from './utils.js'; + +function getApplicableTestSuites( + parsedSuitesFile: TestSuiteFile, + platform: Platform +): TestSuite[] { + const testSuiteArgIdx = process.argv.indexOf('--test-suite'); + let applicableSuites: TestSuite[] = []; + + if (testSuiteArgIdx === -1) { + applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform); + } else { + const testSuiteId = process.argv[testSuiteArgIdx + 1]; + const testSuite = parsedSuitesFile.testSuites.find(suite => { + return suite.id === testSuiteId; + }); + + if (!testSuite) { + console.error(`Test suite ${testSuiteId} is not defined`); + process.exit(1); + } + + if (!testSuite.platforms.includes(platform)) { + console.warn( + `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.` + ); + } + + applicableSuites = [testSuite]; + } + + return applicableSuites; +} + +async function main() { + const noCoverage = process.argv.indexOf('--no-coverage') !== -1; + + const platform = zPlatform.parse(os.platform()); + + const expectations = readJSON( + path.join(process.cwd(), 'test', 'TestExpectations.json') + ) as TestExpectation[]; + + const parsedSuitesFile = zTestSuiteFile.parse( + readJSON(path.join(process.cwd(), 'test', 'TestSuites.json')) + ); + + const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform); + + console.log('Planning to run the following test suites', applicableSuites); + + let fail = false; + const recommendations = []; + try { + for (const suite of applicableSuites) { + const parameters = suite.parameters; + + const applicableExpectations = filterByParameters( + filterByPlatform(expectations, platform), + parameters + ); + + const env = extendProcessEnv([ + ...parameters.map(param => { + return parsedSuitesFile.parameterDefinitons[param]; + }), + { + PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify( + applicableExpectations.map(ex => { + return { + testIdPattern: ex.testIdPattern, + skip: ex.expectations.includes('SKIP'), + }; + }) + ), + }, + ]); + + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-test-runner-') + ); + const tmpFilename = path.join(tmpDir, 'output.json'); + console.log('Running', JSON.stringify(parameters), tmpFilename); + const reporterArgumentIndex = process.argv.indexOf('--reporter'); + const args = [ + '-u', + path.join(__dirname, 'interface.js'), + '-R', + reporterArgumentIndex === -1 + ? path.join(__dirname, 'reporter.js') + : process.argv[reporterArgumentIndex + 1] || '', + '-O', + 'output=' + tmpFilename, + ]; + const retriesArgumentIndex = process.argv.indexOf('--retries'); + const timeoutArgumentIndex = process.argv.indexOf('--timeout'); + if (retriesArgumentIndex > -1) { + args.push('--retries', process.argv[retriesArgumentIndex + 1] || ''); + } + if (timeoutArgumentIndex > -1) { + args.push('--timeout', process.argv[timeoutArgumentIndex + 1] || ''); + } + if (process.argv.indexOf('--no-parallel')) { + args.push('--no-parallel'); + } + if (process.argv.indexOf('--fullTrace')) { + args.push('--fullTrace'); + } + + const spawnArgs: SpawnOptions = { + shell: true, + cwd: process.cwd(), + stdio: 'inherit', + env, + }; + const handle = noCoverage + ? spawn('npx', ['mocha', ...args], spawnArgs) + : spawn( + 'npx', + [ + 'c8', + '--check-coverage', + '--lines', + String(suite.expectedLineCoverage), + 'npx mocha', + ...args, + ], + spawnArgs + ); + await new Promise((resolve, reject) => { + handle.on('error', err => { + reject(err); + }); + handle.on('close', () => { + resolve(); + }); + }); + console.log('Finished', JSON.stringify(parameters)); + try { + const results = readJSON(tmpFilename) as MochaResults; + const recommendation = getExpectationUpdates( + results, + applicableExpectations, + { + platforms: [os.platform()], + parameters, + } + ); + if (recommendation.length > 0) { + fail = true; + recommendations.push(...recommendation); + } else { + console.log('Test run matches expecations'); + continue; + } + } catch (err) { + fail = true; + console.error(err); + } + } + } catch (err) { + fail = true; + console.error(err); + } finally { + const toAdd = recommendations.filter(item => { + return item.action === 'add'; + }); + if (toAdd.length) { + console.log( + 'Add the following to TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toAdd.map(item => { + return item.expectation; + }) + ); + } + const toRemove = recommendations.filter(item => { + return item.action === 'remove'; + }); + if (toRemove.length) { + console.log( + 'Remove the following from the TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toRemove.map(item => { + return item.expectation; + }) + ); + } + const toUpdate = recommendations.filter(item => { + return item.action === 'update'; + }); + if (toUpdate.length) { + console.log( + 'Update the following expectations in the TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toUpdate.map(item => { + return item.expectation; + }) + ); + } + process.exit(fail ? 1 : 0); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/remote/test/puppeteer/utils/mochaRunner/src/reporter.ts b/remote/test/puppeteer/utils/mochaRunner/src/reporter.ts new file mode 100644 index 000000000000..37ca58621532 --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/src/reporter.ts @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Mocha from 'mocha'; + +class SpecJSONReporter extends Mocha.reporters.Spec { + constructor(runner: Mocha.Runner, options?: Mocha.MochaOptions) { + super(runner, options); + Mocha.reporters.JSON.call(this, runner, options); + } +} + +module.exports = SpecJSONReporter; diff --git a/remote/test/puppeteer/utils/mochaRunner/src/test.ts b/remote/test/puppeteer/utils/mochaRunner/src/test.ts new file mode 100644 index 000000000000..ae8daed451e6 --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/src/test.ts @@ -0,0 +1,62 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert/strict'; +import test from 'node:test'; +import {filterByParameters, getTestResultForFailure} from './utils.js'; +import {TestExpectation} from './types.js'; +import {getFilename, extendProcessEnv} from './utils.js'; + +test('extendProcessEnv', () => { + const env = extendProcessEnv([{TEST: 'TEST'}, {TEST2: 'TEST2'}]); + assert.equal(env['TEST'], 'TEST'); + assert.equal(env['TEST2'], 'TEST2'); +}); + +test('getFilename', () => { + assert.equal(getFilename('/etc/test.ts'), 'test'); + assert.equal(getFilename('/etc/test.js'), 'test'); +}); + +test('getTestResultForFailure', () => { + assert.equal( + getTestResultForFailure({err: {code: 'ERR_MOCHA_TIMEOUT'}}), + 'TIMEOUT' + ); + assert.equal(getTestResultForFailure({err: {code: 'ERROR'}}), 'FAIL'); +}); + +test('filterByParameters', () => { + const expectations: TestExpectation[] = [ + { + testIdPattern: + '[oopif.spec] OOPIF "after all" hook for "should keep track of a frames OOP state"', + platforms: ['darwin'], + parameters: ['firefox', 'headless'], + expectations: ['FAIL'], + }, + ]; + assert.equal( + filterByParameters(expectations, ['firefox', 'headless']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['firefox']).length, 0); + assert.equal( + filterByParameters(expectations, ['firefox', 'headless', 'other']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['other']).length, 0); +}); diff --git a/remote/test/puppeteer/utils/mochaRunner/src/types.ts b/remote/test/puppeteer/utils/mochaRunner/src/types.ts new file mode 100644 index 000000000000..e8fac25afc36 --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/src/types.ts @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {z} from 'zod'; + +export const zPlatform = z.enum(['win32', 'linux', 'darwin']); + +export type Platform = z.infer; + +export const zTestSuite = z.object({ + id: z.string(), + platforms: z.array(zPlatform), + parameters: z.array(z.string()), + expectedLineCoverage: z.number(), +}); + +export type TestSuite = z.infer; + +export const zTestSuiteFile = z.object({ + testSuites: z.array(zTestSuite), + parameterDefinitons: z.record(z.any()), +}); + +export type TestSuiteFile = z.infer; + +export type TestResult = 'PASS' | 'FAIL' | 'TIMEOUT' | 'SKIP'; + +export type TestExpectation = { + testIdPattern: string; + platforms: NodeJS.Platform[]; + parameters: string[]; + expectations: TestResult[]; +}; + +export type MochaTestResult = { + fullTitle: string; + file: string; + err?: {code: string}; +}; + +export type MochaResults = { + stats: unknown; + pending: MochaTestResult[]; + passes: MochaTestResult[]; + failures: MochaTestResult[]; +}; diff --git a/remote/test/puppeteer/utils/mochaRunner/src/utils.ts b/remote/test/puppeteer/utils/mochaRunner/src/utils.ts new file mode 100644 index 000000000000..517d1f1553fe --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/src/utils.ts @@ -0,0 +1,161 @@ +/** + * Copyright 2022 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + MochaTestResult, + TestExpectation, + MochaResults, + TestResult, +} from './types.js'; +import path from 'path'; +import fs from 'fs'; + +export function extendProcessEnv(envs: object[]): NodeJS.ProcessEnv { + return envs.reduce( + (acc: object, item: object) => { + Object.assign(acc, item); + return acc; + }, + { + ...process.env, + } + ) as NodeJS.ProcessEnv; +} + +export function getFilename(file: string): string { + return path.basename(file).replace(path.extname(file), ''); +} + +export function readJSON(path: string): unknown { + return JSON.parse(fs.readFileSync(path, 'utf-8')); +} + +export function filterByPlatform( + items: T[], + platform: NodeJS.Platform +): T[] { + return items.filter(item => { + return item.platforms.includes(platform); + }); +} + +export function prettyPrintJSON(json: unknown): void { + console.log(JSON.stringify(json, null, 2)); +} + +export function filterByParameters( + expectations: TestExpectation[], + parameters: string[] +): TestExpectation[] { + const querySet = new Set(parameters); + return expectations.filter(ex => { + return ex.parameters.every(param => { + return querySet.has(param); + }); + }); +} + +/** + * The last expectation that matches an empty string as all tests pattern + * or the name of the file or the whole name of the test the filter wins. + */ +export function findEffectiveExpectationForTest( + expectations: TestExpectation[], + result: MochaTestResult +): TestExpectation | undefined { + return expectations + .filter(expectation => { + return ( + '' === expectation.testIdPattern || + getTestId(result.file) === expectation.testIdPattern || + getTestId(result.file, result.fullTitle) === expectation.testIdPattern + ); + }) + .pop(); +} + +type RecommendedExpecation = { + expectation: TestExpectation; + test: MochaTestResult; + action: 'remove' | 'add' | 'update'; +}; + +export function getExpectationUpdates( + results: MochaResults, + expecations: TestExpectation[], + context: { + platforms: NodeJS.Platform[]; + parameters: string[]; + } +): RecommendedExpecation[] { + const output: RecommendedExpecation[] = []; + + for (const pass of results.passes) { + const expectation = findEffectiveExpectationForTest(expecations, pass); + if (expectation && !expectation.expectations.includes('PASS')) { + output.push({ + expectation, + test: pass, + action: 'remove', + }); + } + } + + for (const failure of results.failures) { + const expectation = findEffectiveExpectationForTest(expecations, failure); + if (expectation) { + if ( + !expectation.expectations.includes(getTestResultForFailure(failure)) + ) { + output.push({ + expectation: { + ...expectation, + expectations: [ + ...expectation.expectations, + getTestResultForFailure(failure), + ], + }, + test: failure, + action: 'update', + }); + } + } else { + output.push({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + test: failure, + action: 'add', + }); + } + } + return output; +} + +export function getTestResultForFailure( + test: Pick +): TestResult { + return test.err?.code === 'ERR_MOCHA_TIMEOUT' ? 'TIMEOUT' : 'FAIL'; +} + +export function getTestId(file: string, fullTitle?: string): string { + return fullTitle + ? `[${getFilename(file)}] ${fullTitle}` + : `[${getFilename(file)}]`; +} diff --git a/remote/test/puppeteer/utils/mochaRunner/tsconfig.json b/remote/test/puppeteer/utils/mochaRunner/tsconfig.json new file mode 100644 index 000000000000..c2576c2564f7 --- /dev/null +++ b/remote/test/puppeteer/utils/mochaRunner/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "allowJs": true, + "composite": true, + "module": "CommonJS", + "outDir": "lib", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/taskcluster/ci/source-test/remote.yml b/taskcluster/ci/source-test/remote.yml index f292068d201d..bb6fe250ab55 100644 --- a/taskcluster/ci/source-test/remote.yml +++ b/taskcluster/ci/source-test/remote.yml @@ -23,9 +23,6 @@ job-defaults: docker-image: {in-tree: ubuntu1804-test} max-run-time: 3600 artifacts: - - type: file - name: public/remote/results.json - path: /builds/worker/results.json - type: file name: public/remote/pup_errorsummary.json path: /builds/worker/pup_errorsummary.json @@ -44,4 +41,4 @@ puppeteer: command: > cd $GECKO_PATH/ && $MOZ_FETCHES_DIR/firefox/firefox --screenshot http://example.org && - ./mach puppeteer-test --ci -vv --binary $MOZ_FETCHES_DIR/firefox/firefox --headless --write-results /builds/worker/results.json --log-tbpl - --log-errorsummary /builds/worker/pup_errorsummary.json --log-raw /builds/worker/pup_raw.log + ./mach puppeteer-test --ci -vv --binary $MOZ_FETCHES_DIR/firefox/firefox --headless --log-tbpl - --log-errorsummary /builds/worker/pup_errorsummary.json --log-raw /builds/worker/pup_raw.log