From 63c5bb98b47aded592964cba9263f016256239f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarek=20Ziad=C3=A9?= Date: Mon, 8 Apr 2019 13:04:20 +0000 Subject: [PATCH] Bug 1541189 - Fix intermittents on stream test - r=whimboo Tweak the Streaming test to fix intermittents. Differential Revision: https://phabricator.services.mozilla.com/D26288 --HG-- extra : moz-landing-system : lando --- .eslintignore | 1 + dom/media/test/marionette/test_youtube.py | 22 +++--- .../test/marionette/yttest/debug_info.js | 6 +- .../test/marionette/yttest/duration_test.js | 4 +- dom/media/test/marionette/yttest/playback.py | 15 ++++ dom/media/test/marionette/yttest/support.py | 25 ++++++- .../marionette/yttest/uR0N3DrybGQ.manifest | 10 +++ .../test/marionette/yttest/until_end_test.js | 4 +- .../yttest/video_playback_quality.js | 8 +-- dom/media/test/marionette/yttest/ytpage.py | 72 ++++++++++++++++++- 10 files changed, 136 insertions(+), 31 deletions(-) create mode 100644 dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest diff --git a/.eslintignore b/.eslintignore index 1c607b14d2bc..7777a3111f99 100644 --- a/.eslintignore +++ b/.eslintignore @@ -171,6 +171,7 @@ dom/grid/** dom/html/** dom/jsurl/** dom/media/test/** +!dom/media/test/marionette/yttest/*.js dom/media/tests/** dom/media/webaudio/** dom/media/webspeech/** diff --git a/dom/media/test/marionette/test_youtube.py b/dom/media/test/marionette/test_youtube.py index 34a17a0a68b4..89b87d1adb58 100644 --- a/dom/media/test/marionette/test_youtube.py +++ b/dom/media/test/marionette/test_youtube.py @@ -11,18 +11,12 @@ from yttest.support import VideoStreamTestCase class YoutubeTest(VideoStreamTestCase): # bug 1513511 - def test_stream_30_seconds(self): - # XXX use the VP9 video we will settle on. - with self.youtube_video("BZP1rYjoBgI") as page: + def test_stream_4K(self): + with self.youtube_video("uR0N3DrybGQ", duration=15) as page: res = page.run_test() - self.assertTrue(res is not None, "We did not get back the results") - self.assertLess(res["droppedVideoFrames"], res["totalVideoFrames"] * 0.04) - # extracting in/out from the debugInfo - video_state = res["debugInfo"][7] - video_in = int(video_state.split(" ")[10].split("=")[-1]) - video_out = int(video_state.split(" ")[11].split("=")[-1]) - # what's the ratio ? we want 99%+ - if video_out == video_in: - return - in_out_ratio = float(video_out) / float(video_in) * 100 - self.assertMore(in_out_ratio, 99.0) + self.assertVideoQuality(res) + + def test_stream_480p(self): + with self.youtube_video("BZP1rYjoBgI", duration=15) as page: + res = page.run_test() + self.assertVideoQuality(res) diff --git a/dom/media/test/marionette/yttest/debug_info.js b/dom/media/test/marionette/yttest/debug_info.js index 2009e3ff4028..2738d2b8ea55 100644 --- a/dom/media/test/marionette/yttest/debug_info.js +++ b/dom/media/test/marionette/yttest/debug_info.js @@ -1,4 +1,5 @@ video.mozRequestDebugInfo().then(debugInfo => { + // The parsing won't be necessary once we have bug 1542674 try { debugInfo = debugInfo.replace(/\t/g, '').split(/\n/g); var JSONDebugInfo = "{"; @@ -8,11 +9,10 @@ video.mozRequestDebugInfo().then(debugInfo => { } JSONDebugInfo = JSONDebugInfo.slice(0,JSONDebugInfo.length-1); JSONDebugInfo += "}"; - result["debugInfo"] = JSON.parse(JSONDebugInfo); + result["mozRequestDebugInfo"] = JSON.parse(JSONDebugInfo); } catch (err) { console.log(`Error '${err.toString()} in JSON.parse(${debugInfo})`); - result["debugInfo"] = debugInfo; + result["mozRequestDebugInfo"] = debugInfo; } - result["debugInfo"] = debugInfo; resolve(result); }); diff --git a/dom/media/test/marionette/yttest/duration_test.js b/dom/media/test/marionette/yttest/duration_test.js index 881847148aa3..3ebeb1055ed1 100644 --- a/dom/media/test/marionette/yttest/duration_test.js +++ b/dom/media/test/marionette/yttest/duration_test.js @@ -10,9 +10,11 @@ if (!video) { video.addEventListener("timeupdate", () => { if (video.currentTime >= %(duration)s) { - video.pause(); %(video_playback_quality)s %(debug_info)s + // Pausing after we get the debug info so + // we can also look at in/out data in buffers + video.pause(); } } ); diff --git a/dom/media/test/marionette/yttest/playback.py b/dom/media/test/marionette/yttest/playback.py index b96011f6001a..67c74d1ad519 100644 --- a/dom/media/test/marionette/yttest/playback.py +++ b/dom/media/test/marionette/yttest/playback.py @@ -12,6 +12,11 @@ import sys import datetime import time +try: + from urllib import unquote +except ImportError: + from urllib.parse import unquote + itags = { "5": { @@ -596,6 +601,13 @@ def OK(flow, code=204): def request(flow): + # in some cases, the YT client sends requests with a methode of the form: + # VAR=XX%3GET /xxx + # this will clean it up: + method = flow.request.method + method = unquote(method).split("=") + flow.request.method = method[-1] + # All requests made for stats purposes can be discarded and # a 204 sent back to the client. if flow.request.url.startswith("https://www.youtube.com/ptracking"): @@ -611,6 +623,9 @@ def request(flow): if "push.services.mozilla.com" in flow.request.url: OK(flow, code=200) return + if "tracking-protection.cdn.mozilla.net" in flow.request.url: + OK(flow, code=200) + return if "gen_204" in flow.request.url: OK(flow) return diff --git a/dom/media/test/marionette/yttest/support.py b/dom/media/test/marionette/yttest/support.py index 7c0e6fb8e021..c7efa0feaa8a 100644 --- a/dom/media/test/marionette/yttest/support.py +++ b/dom/media/test/marionette/yttest/support.py @@ -23,6 +23,7 @@ class VideoStreamTestCase(MarionetteTestCase): if "MOZ_UPLOAD_DIR" not in os.environ: os.environ["OBJ_PATH"] = "/tmp/" self.marionette.set_pref("media.autoplay.default", 1) + self.marionette.set_pref("privacy.trackingprotection.enabled", False) @contextmanager def using_proxy(self, video_id): @@ -56,8 +57,6 @@ class VideoStreamTestCase(MarionetteTestCase): playback_file = os.path.join(playback_dir, "%s.playback" % video_id) config["playback_tool_args"] = [ - "--set", - "stream_large_bodies=30", "--ssl-insecure", "--server-replay-nopop", "--set", @@ -82,9 +81,29 @@ class VideoStreamTestCase(MarionetteTestCase): def youtube_video(self, video_id, **options): proxy = options.get("proxy", True) if proxy: - with self.using_proxy(video_id): + with self.using_proxy(video_id) as proxy: + options["upload_dir"] = proxy.upload_dir with using_page(video_id, self.marionette, **options) as page: yield page else: with using_page(video_id, self.marionette, **options) as page: yield page + + def assertVideoQuality(self, res): + self.assertTrue(res is not None, "We did not get back the results") + debug_info = res["mozRequestDebugInfo"] + + # looking at mNumSamplesOutputTotal vs mNumSamplesSkippedTotal + decoded, skipped = debug_info["Video Frames Decoded"].split(" ", 1) + decoded = int(decoded) + skipped = int(skipped.split("=")[-1][:-1]) + self.assertLess(skipped, decoded * 0.04) + + # extracting in/out from the debugInfo + video_state = debug_info["Video State"] + video_in = int(video_state["in"]) + video_out = int(video_state["out"]) + # what's the ratio ? we want 99%+ + if video_out != video_in: + in_out_ratio = float(video_out) / float(video_in) * 100 + self.assertGreater(in_out_ratio, 99.0) diff --git a/dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest b/dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest new file mode 100644 index 000000000000..a9bba27cd4f9 --- /dev/null +++ b/dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 629013569, + "visibility": "public", + "digest": "213afa0e40411c26c86092a0803099a8c596b27cf789ed658ba0cf50dd8b404926dd784cd0236922aca22d3763edff666dd247c14bfe38359fb9d767f1869048", + "algorithm": "sha512", + "filename": "uR0N3DrybGQ.tar.gz", + "unpack": true + } +] diff --git a/dom/media/test/marionette/yttest/until_end_test.js b/dom/media/test/marionette/yttest/until_end_test.js index 61f56c94c797..70227a25d5f2 100644 --- a/dom/media/test/marionette/yttest/until_end_test.js +++ b/dom/media/test/marionette/yttest/until_end_test.js @@ -9,9 +9,11 @@ if (!video) { } video.addEventListener("ended", () => { - video.pause(); %(video_playback_quality)s %(debug_info)s + // Pausing after we get the debug info so + // we can also look at in/out data in buffers + video.pause(); }, {once: true} ); diff --git a/dom/media/test/marionette/yttest/video_playback_quality.js b/dom/media/test/marionette/yttest/video_playback_quality.js index fd02a776134e..52be76003a15 100644 --- a/dom/media/test/marionette/yttest/video_playback_quality.js +++ b/dom/media/test/marionette/yttest/video_playback_quality.js @@ -1,7 +1 @@ -var vpq = video.getVideoPlaybackQuality(); -var result = {"currentTime": video.currentTime}; -result["creationTime"] = vpq.creationTime; -result["corruptedVideoFrames"] = vpq.corruptedVideoFrames; -result["droppedVideoFrames"] = vpq.droppedVideoFrames; -result["totalVideoFrames"] = vpq.totalVideoFrames; -result["defaultPlaybackRate"] = video.playbackRate; +var result = {"getVideoPlaybackQuality": video.getVideoPlaybackQuality()}; diff --git a/dom/media/test/marionette/yttest/ytpage.py b/dom/media/test/marionette/yttest/ytpage.py index bb9282047fd2..d53383c68e89 100644 --- a/dom/media/test/marionette/yttest/ytpage.py +++ b/dom/media/test/marionette/yttest/ytpage.py @@ -6,6 +6,11 @@ Drives the browser during the playback test. """ import contextlib import os +import time +import json +import re + +from marionette_driver.by import By here = os.path.dirname(__file__) @@ -23,6 +28,19 @@ for script in JS_MACROS: with open(js) as f: JS_MACROS[script] = f.read() +SPLIT_FIELD = ( + "Audio State", + "Audio Track Buffer Details", + "AudioSink", + "MDSM", + "Video State", + "Video Track Buffer Details", + "Dumping Audio Track", + "Dumping Video Track", + "MediaDecoder", + "VideoSink", +) + class YoutubePage: def __init__(self, video_id, marionette, **options): @@ -53,14 +71,23 @@ class YoutubePage: def run_test(self): self.start_video() + # If we don't pause here for just a bit the media events + # are not intercepted. + time.sleep(5) + body = self.marionette.find_element(By.TAG_NAME, "html") + body.click() options = dict(JS_MACROS) options.update(self.options) if "duration" in options: script = DURATION_TEST % options else: script = UNTIL_END_TEST % options - self.marionette.set_pref("media.autoplay.default", 0) - return self.execute_async_script(script) + res = self.execute_async_script(script) + if res is None: + return res + res = self._parse_res(res) + self._dump_res(res) + return res def execute_async_script(self, script, context=None): if context is None: @@ -68,6 +95,47 @@ class YoutubePage: with self.marionette.using_context(context): return self.marionette.execute_async_script(script, sandbox="system") + def _parse_res(self, res): + debug_info = {} + # The parsing won't be necessary once we have bug 1542674 + for key, value in res["mozRequestDebugInfo"].items(): + key, value = key.strip(), value.strip() + if key.startswith(SPLIT_FIELD): + value_dict = {} + for field in re.findall(r"\S+\(.+\)\s|\S+", value): + field = field.strip() + if field == "": + continue + if field.startswith("VideoQueue"): + k = "VideoQueue" + v = field[len("VideoQueue(") : -2] # noqa: E203 + fields = {} + v = v.split(" ") + for h in v: + f, vv = h.split("=") + fields[f] = vv + v = fields + else: + if "=" in field: + k, v = field.split("=", 1) + else: + k, v = field.split(":", 1) + value_dict[k] = v + value = value_dict + debug_info[key] = value + res["mozRequestDebugInfo"] = debug_info + return res + + def _dump_res(self, res): + raw = json.dumps(res, indent=2, sort_keys=True) + print(raw) + if "upload_dir" in self.options: + fn = "%s-videoPlaybackQuality.json" % self.video_id + fn = os.path.join(self.options["upload_dir"], fn) + # dumping on disk + with open(fn, "w") as f: + f.write(raw) + def close(self): if self.started: self.marionette.delete_session()