diff --git a/.eslintignore b/.eslintignore index 779434f41bb2..4cd2a523254b 100644 --- a/.eslintignore +++ b/.eslintignore @@ -295,6 +295,10 @@ testing/modules/ajv-4.1.1.js testing/modules/sinon-7.2.7.js # octothorpe used for pref file comment causes parsing error testing/mozbase/mozprofile/tests/files/prefs_with_comments.js + +# Mozproxy third party +testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/** + testing/talos/talos/scripts/jszip.min.js testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js @@ -304,8 +308,6 @@ testing/talos/talos/tests/v8_7/** testing/talos/talos/tests/kraken/** # Runing Talos may extract data here, see bug 1435677. testing/talos/talos/tests/tp5n/** -# Raptor third party -testing/raptor/raptor/playback/scripts/catapult/** testing/web-platform/** testing/xpcshell/moz-http2/** diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py new file mode 100644 index 000000000000..e82b6582927d --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from .mitm import * diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py similarity index 91% rename from testing/mozbase/mozproxy/mozproxy/backends/mitm.py rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py index f6aa3a1655c1..39f5c5808dc8 100644 --- a/testing/mozbase/mozproxy/mozproxy/backends/mitm.py +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py @@ -195,7 +195,51 @@ class Mitmproxy(Playback): command = [mitmdump_path] if "playback_tool_args" in self.config: + LOG.info("Staring Proxy using provided command line!") command.extend(self.config["playback_tool_args"]) + elif "playback_files" in self.config: + script = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "scripts", + "alternate-server-replay-{}.py".format( + self.config["playback_version"])) + recording_paths = self.config["playback_files"] + # this part is platform-specific + if mozinfo.os == "win": + script = script.replace("\\", "\\\\\\") + recording_paths = [recording_path.replace("\\", "\\\\\\") + for recording_path in recording_paths] + + if self.config["playback_version"] == "2.0.2": + args = [ + "--replay-kill-extra", + "-v", + "--script", + '""{} {}""'.format(script, " ".join(recording_paths)), + ] + + if not self.config["playback_upstream_cert"]: + LOG.info("No upstream certificate sniffing") + args.insert(0, "--no-upstream-cert") + self.playback.config["playback_tool_args"] = args + elif self.config["playback_version"] == "4.0.4": + args = [ + "-v", + "--set", + "websocket=false", + "--set", + "server_replay_files={}".format(" ".join(recording_paths)), + "--scripts", + script, + ] + if not self.config["playback_upstream_cert"]: + LOG.info("No upstream certificate sniffing") + args = ["--set", "upstream_cert=false"] + args + command.extend(args) + else: + raise Exception("Mitmproxy version is unknown!") + + else: + raise Exception("Mitmproxy can't start playback! Playback settings missing.") LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command)) diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-linux64.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-linux64.manifest similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-linux64.manifest rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-linux64.manifest diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-osx.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-osx.manifest similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-osx.manifest rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-osx.manifest diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-win.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-win.manifest similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-win.manifest rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-win.manifest diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-linux64.manifest similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-linux64.manifest diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-osx.manifest similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-osx.manifest diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-win.manifest similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-win.manifest diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt similarity index 100% rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt diff --git a/testing/raptor/raptor/playback/alternate-server-replay-2.0.2.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-2.0.2.py similarity index 100% rename from testing/raptor/raptor/playback/alternate-server-replay-2.0.2.py rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-2.0.2.py diff --git a/testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py similarity index 99% rename from testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py index 5fed881bb193..1ff63b0a08f7 100644 --- a/testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py @@ -4,6 +4,7 @@ # Altered features: # * returns 404 rather than dropping the whole HTTP/2 connection on the floor # * remove the replay packages that don't have any content in their response package +from __future__ import absolute_import, print_function import os import json diff --git a/testing/raptor/raptor/playback/scripts/catapult/LICENSE b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE similarity index 100% rename from testing/raptor/raptor/playback/scripts/catapult/LICENSE rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE diff --git a/testing/raptor/raptor/playback/scripts/catapult/deterministic.js b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js similarity index 100% rename from testing/raptor/raptor/playback/scripts/catapult/deterministic.js rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js diff --git a/testing/raptor/raptor/playback/scripts/inject-deterministic.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py similarity index 99% rename from testing/raptor/raptor/playback/scripts/inject-deterministic.py rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py index b1567ced5937..1db81863e78d 100644 --- a/testing/raptor/raptor/playback/scripts/inject-deterministic.py +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function + import base64 import hashlib import re diff --git a/testing/mozbase/mozproxy/tests/test_proxy.py b/testing/mozbase/mozproxy/tests/test_proxy.py index 305c84937af9..e7ab2c730be5 100644 --- a/testing/mozbase/mozproxy/tests/test_proxy.py +++ b/testing/mozbase/mozproxy/tests/test_proxy.py @@ -31,13 +31,15 @@ class Process: @mock.patch("mozproxy.backends.mitm.tooltool_download", new=mock.DEFAULT) @mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy", lambda x: True) def test_mitm(*args): - bin_name = "mitmproxy-rel-bin-{platform}.manifest" - pageset_name = "mitmproxy-recordings-raptor-paypal.manifest" + bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest" + pageset_name = "mitm4-linux-firefox-amazon.manifest" config = { "playback_tool": "mitmproxy", "playback_binary_manifest": bin_name, "playback_pageset_manifest": pageset_name, + "playback_upstream_cert": 'false', + "playback_version": '4.0.4', "platform": mozinfo.os, "playback_recordings": os.path.join(here, "paypal.mp"), "run_local": True, @@ -49,7 +51,7 @@ def test_mitm(*args): with tempdir() as obj_path: config["obj_path"] = obj_path playback = get_playback(config) - + playback.config['playback_files'] = config['playback_recordings'] assert playback is not None try: playback.start() @@ -70,13 +72,15 @@ def test_playback_setup_failed(*args): return _s - bin_name = "mitmproxy-rel-bin-{platform}.manifest" - pageset_name = "mitmproxy-recordings-raptor-paypal.manifest" + bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest" + pageset_name = "mitm4-linux-firefox-amazon.manifest" config = { "playback_tool": "mitmproxy", "playback_binary_manifest": bin_name, "playback_pageset_manifest": pageset_name, + "playback_upstream_cert": 'false', + "playback_version": '4.0.4', "platform": mozinfo.os, "playback_recordings": os.path.join(here, "paypal.mp"), "run_local": True, @@ -93,6 +97,7 @@ def test_playback_setup_failed(*args): with mock.patch(prefix + "stop_mitmproxy_playback") as p: try: pb = get_playback(config) + pb.config['playback_files'] = config['playback_recordings'] pb.start() except SetupFailed: assert p.call_count == 1 diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 7c3630bb6d70..b3fab0141412 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -402,53 +402,12 @@ class Raptor(Perftest): if self.config['app'] in chrome_apps: self.profile.addons.remove(self.raptor_webext) - def get_proxy_command_for_mitm(self, test, version): - # Generate Mitmproxy playback args - script = os.path.join(here, "playback", "alternate-server-replay-{}.py".format(version)) - - recording_paths = self.get_recording_paths(test) - - # this part is platform-specific - if mozinfo.os == "win": - script = script.replace("\\", "\\\\\\") - recording_paths = [recording_path.replace("\\", "\\\\\\") - for recording_path in recording_paths] - - if version == "2.0.2": - args = [ - "--replay-kill-extra", - "-v", - "--script", - '""{} {}""'.format(script, " ".join(recording_paths)), - ] - - if not self.config["playback_upstream_cert"]: - LOG.info("No upstream certificate sniffing") - args.insert(0, "--no-upstream-cert") - self.playback.config["playback_tool_args"] = args - elif version == "4.0.4": - args = [ - "-v", - "--set", - "websocket=false", - "--set", - "server_replay_files={}".format(" ".join(recording_paths)), - "--scripts", - script, - ] - if not self.config["playback_upstream_cert"]: - LOG.info("No upstream certificate sniffing") - args = ["--set", "upstream_cert=false"] + args - self.playback.config["playback_tool_args"] = args - else: - raise Exception("Mitmproxy version is unknown!") - def start_playback(self, test): # creating the playback tool self.get_playback_config(test) self.playback = get_playback(self.config, self.device) - self.get_proxy_command_for_mitm(test, self.config['playback_version']) + self.playback.config['playback_files'] = self.get_recording_paths(test) # let's start it! self.playback.start() diff --git a/testing/raptor/test/test_playback.py b/testing/raptor/test/test_playback.py index 135a737c7f4e..dca55e000dc0 100644 --- a/testing/raptor/test/test_playback.py +++ b/testing/raptor/test/test_playback.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, unicode_literals import os +import time import mozinfo import mozunit @@ -27,17 +28,24 @@ def test_get_playback(get_binary): return config['obj_path'] = os.path.dirname(get_binary('firefox')) config['playback_tool'] = 'mitmproxy' - config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest' - config['playback_pageset_manifest'] = 'mitmproxy-recordings-raptor-tp6-1.manifest' + config['playback_version'] = '4.0.4' + config['playback_upstream_cert'] = 'false' + config['playback_binary_manifest'] = 'mitmproxy-rel-bin-4.0.4-{platform}.manifest' + config['playback_pageset_manifest'] = os.path.join( + os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "raptor", "playback", + 'mitm4-linux-firefox-amazon.manifest') config['playback_recordings'] = 'amazon.mp' config['binary'] = get_binary('firefox') config['run_local'] = run_local config['app'] = 'firefox' - config['host'] = 'example.com' + config['host'] = 'https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1' playback = get_playback(config) + playback.config['playback_files'] = [os.path.join(playback.mozproxy_dir, + config['playback_recordings'])] assert isinstance(playback, MitmproxyDesktop) playback.start() + time.sleep(1) playback.stop()