Bug 1809843 - Add DoH server and Http/3 server into mochitest, r=necko-reviewers,ahal,valentin,gbrown,jmaher

Differential Revision: https://phabricator.services.mozilla.com/D166644
This commit is contained in:
Kershaw Chang 2023-03-16 21:28:58 +00:00
parent cd31baf9a1
commit 79403e1620
29 changed files with 617 additions and 95 deletions

Binary file not shown.

View file

@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDKDCCAhCgAwIBAgIUZwyOUcu/hcoxGM+/mv6i/ZNoK8cwDQYJKoZIhvcNAQEL
BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa
GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9
PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3
HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg
Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7
EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK
lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C
AwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwSQYDVR0RBEIwQIIJbG9jYWxo
b3N0gg9mb28uZXhhbXBsZS5jb22CEGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhh
bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggEBAEPAH1dHcA0wyt6et0AjiRIBKla0
EaW/9hFQW8xphNsgZTw2tX6vUqjd2qz4kzKTSbW8AmB26VEfSDkwUxbFrbCP8Pri
FqVYwnWVLsBrr9PF0xg9SuRJnyHAwBv0xNXB3KRY2HtFcUaLn1C8alh2LAhX6eX+
+ozHvspMjMq7P1lsONGbtfmJaf6pNh7zzok4fOas+eoqhf64TUb4g4JNxfLyXh3P
DcLsP67kr6fZSlGjzhto9ILVhg9rflNmSkmrhVjCxNqDuLFyn5ljvhemw7X3/V8A
iCwqEP2/vOjnE1wCxF2Xdb6+3f/KJKqNwOmjraXZGupOUR6OA/+Z3V3RbGc=
-----END CERTIFICATE-----

View file

@ -0,0 +1 @@
default

View file

@ -0,0 +1,5 @@
issuer: HTTP2 Test CA
subject: HTTP2 Test Cert
validity:20170101-20270101
extension:extKeyUsage:serverAuth
extension:subjectAlternativeName:localhost,foo.example.com,alt1.example.com,alt2.example.com

Binary file not shown.

View file

@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDFjCCAf6gAwIBAgIUF75dZpigXpj3fOIppeG4xp8WwQowDQYJKoZIhvcNAQEL
BQAwHTEbMBkGA1UEAwwSIE1vY2hpdGVzdCBUZXN0IENBMCIYDzIwMjIwMTAxMDAw
MDAwWhgPMjAzMjAxMDEwMDAwMDBaMB8xHTAbBgNVBAMMFCBNb2NoaXRlc3QgVGVz
dCBDZXJ0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuohRqESOFtZB
/W62iAY2ED08E9nq5DVKtOz1aFdsJHvBxyWo4NgfvbGcBptuGobya+KvWnVramRx
CHqlWqdFh/cc1SScAn7NQ/weadA4ICmTqyDDSeTbuUzCa2wO7RWCD/F+rWkasdMC
OosqQe6ncOAPDY39ZgsrsCSSpH25iGF5kLFXkD3SO8XguEgfqDfTiEPvJxbYVbdm
Wqp+ApAvOnsQgAYkzBxsl62WYVu34pYSwHUxowyR3bTK9/ytHSXTCe+5Fw6naOGz
ey8ib2njtIqVYR3uJtYlnauRCE42yxwkBCy/Fosv5fGPmRcxuLP+SSP6clHEMdUD
rNoYCjXtjQIDAQABo0gwRjATBgNVHSUEDDAKBggrBgEFBQcDATAvBgNVHREEKDAm
ggptb2NoaS50ZXN0ggtleGFtcGxlLmNvbYILZXhhbXBsZS5vcmcwDQYJKoZIhvcN
AQELBQADggEBAGL9Tc9f41G7nFZKlneKAJCMK0k2BLOED4YRx9yyAXPDCPiywfkF
A/JLqSlnJH1roTxPf1BP3xIoe0K70IyBnwoRkZ3JWW5oKlDNkkIotLpGipLlOe3e
gmguxWCU4ANvUNgJ5YvEByCV5wJUZT25HiQCFvKvpnLDMmIcXegTULLI0nb3yJz6
NHqQX0SZYsQuw9+qSsnv/FypWX3VF+KAaP7tJVLpXDtJGrfboY9EbdnVrB36IHMQ
ODaPvnwE4cewHaEWDUE2/orI9Gev11c1hK5nEWLHTOKQ3ZYbV8872xtWC7wcYzIr
fJhMS05/mamP++g4rl5r4ZsOZ6nxIgOtISg=
-----END CERTIFICATE-----

View file

@ -0,0 +1 @@
default

View file

@ -0,0 +1,5 @@
issuer: Mochitest Test CA
subject: Mochitest Test Cert
validity:20220101-20320101
extension:extKeyUsage:serverAuth
extension:subjectAlternativeName:mochi.test,example.com,example.org

Binary file not shown.

View file

@ -371,3 +371,9 @@ http://expired.example.com:80 privileged
http://redirect-example.com:80 privileged
https://redirect-example.com:443 privileged,cert=bug1706126cert
https://www.redirect-example.com:443 privileged,cert=bug1706126cert
# DoH server
https://foo.example.com:4433 privileged,cert=http2-cert.pem
# Mochitest
https://mochi.test:443 privileged,cert=mochitest-cert.pem

View file

@ -1004,11 +1004,16 @@ impl ServersRunner {
}
pub fn init(&mut self) {
self.add_new_socket(0, ServerType::Http3);
self.add_new_socket(1, ServerType::Http3Fail);
self.add_new_socket(2, ServerType::Http3Ech);
self.add_new_socket(3, ServerType::Http3Proxy);
self.add_new_socket(5, ServerType::Http3NoResponse);
self.add_new_socket(0, ServerType::Http3, 0);
self.add_new_socket(1, ServerType::Http3Fail, 0);
self.add_new_socket(2, ServerType::Http3Ech, 0);
let proxy_port = match env::var("MOZ_HTTP3_PROXY_PORT") {
Ok(val) => val.parse::<u16>().unwrap(),
_ => 0,
};
self.add_new_socket(3, ServerType::Http3Proxy, proxy_port);
self.add_new_socket(5, ServerType::Http3NoResponse, 0);
println!(
"HTTP3 server listening on ports {}, {}, {}, {} and {}. EchConfig is @{}@",
@ -1024,8 +1029,8 @@ impl ServersRunner {
.unwrap();
}
fn add_new_socket(&mut self, count: usize, server_type: ServerType) -> u16 {
let addr = "127.0.0.1:0".parse().unwrap();
fn add_new_socket(&mut self, count: usize, server_type: ServerType, port: u16) -> u16 {
let addr = format!("127.0.0.1:{}", port).parse().unwrap();
let socket = match UdpSocket::bind(&addr) {
Err(err) => {

View file

@ -38,6 +38,7 @@ pth:testing/mozbase/mozprocess
pth:testing/mozbase/mozprofile
pth:testing/mozbase/mozproxy
pth:testing/mozbase/mozrunner
pth:testing/mozbase/mozserve
pth:testing/mozbase/mozsystemmonitor
pth:testing/mozbase/mozscreenshot
pth:testing/mozbase/moztest

View file

@ -18,6 +18,7 @@
../mozbase/mozprofile
../mozbase/mozproxy
../mozbase/mozrunner
../mozbase/mozserve
../mozbase/mozscreenshot
../mozbase/moztest
../mozbase/mozversion

View file

@ -18,6 +18,7 @@
--editable ../mozbase/mozprofile
--editable ../mozbase/mozproxy
--editable ../mozbase/mozrunner
--editable ../mozbase/mozserve
--editable ../mozbase/mozscreenshot
--editable ../mozbase/moztest
--editable ../mozbase/mozversion

View file

@ -0,0 +1,86 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
/* globals require, __dirname, global, Buffer, process */
const fs = require("fs");
const options = {
key: fs.readFileSync(__dirname + "/http2-cert.key.pem"),
cert: fs.readFileSync(__dirname + "/http2-cert.pem"),
};
const http2 = require("http2");
const http = require("http");
const url = require("url");
const dnsPacket = require("../../xpcshell/dns-packet");
const _ = require("lodash");
let serverPort = parseInt(process.argv[2].split("=")[1]);
let listeningPort = parseInt(process.argv[3].split("=")[1]);
let server = http2.createSecureServer(options, function handleRequest(
req,
res
) {
let u = "";
if (req.url != undefined) {
u = url.parse(req.url, true);
}
if (u.pathname === "/dns-query") {
let payload = Buffer.from("");
req.on("data", function receiveData(chunk) {
payload = Buffer.concat([payload, chunk]);
});
req.on("end", function finishedData() {
let packet = dnsPacket.decode(payload);
let answers = [];
// Return the HTTPS RR to let Firefox connect to the HTTP/3 server
if (packet.questions[0].type === "HTTPS") {
answers.push({
name: packet.questions[0].name,
type: "HTTPS",
ttl: 55,
class: "IN",
flush: false,
data: {
priority: 1,
name: packet.questions[0].name,
values: [
{ key: "alpn", value: ["h3"] },
{ key: "port", value: serverPort },
],
},
});
} else if (packet.questions[0].type === "A") {
answers.push({
name: packet.questions[0].name,
type: "A",
ttl: 55,
flush: false,
data: "127.0.0.1",
});
}
let buf = dnsPacket.encode({
type: "response",
id: packet.id,
flags: dnsPacket.RECURSION_DESIRED,
questions: packet.questions,
answers,
});
res.setHeader("Content-Type", "application/dns-message");
res.setHeader("Content-Length", buf.length);
res.writeHead(200);
res.write(buf);
res.end("");
});
}
});
server.listen(listeningPort);
console.log(`DoH server listening on ports ${server.address().port}`);

View file

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6iFGoRI4W1kH9
braIBjYQPTwT2erkNUq07PVoV2wke8HHJajg2B+9sZwGm24ahvJr4q9adWtqZHEI
eqVap0WH9xzVJJwCfs1D/B5p0DggKZOrIMNJ5Nu5TMJrbA7tFYIP8X6taRqx0wI6
iypB7qdw4A8Njf1mCyuwJJKkfbmIYXmQsVeQPdI7xeC4SB+oN9OIQ+8nFthVt2Za
qn4CkC86exCABiTMHGyXrZZhW7filhLAdTGjDJHdtMr3/K0dJdMJ77kXDqdo4bN7
LyJvaeO0ipVhHe4m1iWdq5EITjbLHCQELL8Wiy/l8Y+ZFzG4s/5JI/pyUcQx1QOs
2hgKNe2NAgMBAAECggEBAJ7LzjhhpFTsseD+j4XdQ8kvWCXOLpl4hNDhqUnaosWs
VZskBFDlrJ/gw+McDu+mUlpl8MIhlABO4atGPd6e6CKHzJPnRqkZKcXmrD2IdT9s
JbpZeec+XY+yOREaPNq4pLDN9fnKsF8SM6ODNcZLVWBSXn47kq18dQTPHcfLAFeI
r8vh6Pld90AqFRUw1YCDRoZOs3CqeZVqWHhiy1M3kTB/cNkcltItABppAJuSPGgz
iMnzbLm16+ZDAgQceNkIIGuHAJy4yrrK09vbJ5L7kRss9NtmA1hb6a4Mo7jmQXqg
SwbkcOoaO1gcoDpngckxW2KzDmAR8iRyWUbuxXxtlEECgYEA3W4dT//r9o2InE0R
TNqqnKpjpZN0KGyKXCmnF7umA3VkTVyqZ0xLi8cyY1hkYiDkVQ12CKwn1Vttt0+N
gSfvj6CQmLaRR94GVXNEfhg9Iv59iFrOtRPZWB3V4HwakPXOCHneExNx7O/JznLp
xD3BJ9I4GQ3oEXc8pdGTAfSMdCsCgYEA16dz2evDgKdn0v7Ak0rU6LVmckB3Gs3r
ta15b0eP7E1FmF77yVMpaCicjYkQL63yHzTi3UlA66jAnW0fFtzClyl3TEMnXpJR
3b5JCeH9O/Hkvt9Go5uLODMo70rjuVuS8gcK8myefFybWH/t3gXo59hspXiG+xZY
EKd7mEW8MScCgYEAlkcrQaYQwK3hryJmwWAONnE1W6QtS1oOtOnX6zWBQAul3RMs
2xpekyjHu8C7sBVeoZKXLt+X0SdR2Pz2rlcqMLHqMJqHEt1OMyQdse5FX8CT9byb
WS11bmYhR08ywHryL7J100B5KzK6JZC7smGu+5WiWO6lN2VTFb6cJNGRmS0CgYAo
tFCnp1qFZBOyvab3pj49lk+57PUOOCPvbMjo+ibuQT+LnRIFVA8Su+egx2got7pl
rYPMpND+KiIBFOGzXQPVqFv+Jwa9UPzmz83VcbRspiG47UfWBbvnZbCqSgZlrCU2
TaIBVAMuEgS4VZ0+NPtbF3yaVv+TUQpaSmKHwVHeLQKBgCgGe5NVgB0u9S36ltit
tYlnPPjuipxv9yruq+nva+WKT0q/BfeIlH3IUf2qNFQhR6caJGv7BU7naqNGq80m
ks/J5ExR5vBpxzXgc7oBn2pyFJYckbJoccrqv48GRBigJpDjmo1f8wZ7fNt/ULH1
NBinA5ZsT8d0v3QCr2xDJH9D
-----END PRIVATE KEY-----

View file

@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDKDCCAhCgAwIBAgIUZwyOUcu/hcoxGM+/mv6i/ZNoK8cwDQYJKoZIhvcNAQEL
BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa
GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9
PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3
HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg
Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7
EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK
lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C
AwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwSQYDVR0RBEIwQIIJbG9jYWxo
b3N0gg9mb28uZXhhbXBsZS5jb22CEGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhh
bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggEBAEPAH1dHcA0wyt6et0AjiRIBKla0
EaW/9hFQW8xphNsgZTw2tX6vUqjd2qz4kzKTSbW8AmB26VEfSDkwUxbFrbCP8Pri
FqVYwnWVLsBrr9PF0xg9SuRJnyHAwBv0xNXB3KRY2HtFcUaLn1C8alh2LAhX6eX+
+ozHvspMjMq7P1lsONGbtfmJaf6pNh7zzok4fOas+eoqhf64TUb4g4JNxfLyXh3P
DcLsP67kr6fZSlGjzhto9ILVhg9rflNmSkmrhVjCxNqDuLFyn5ljvhemw7X3/V8A
iCwqEP2/vOjnE1wCxF2Xdb6+3f/KJKqNwOmjraXZGupOUR6OA/+Z3V3RbGc=
-----END CERTIFICATE-----

View file

@ -504,6 +504,15 @@ class MochitestArguments(ArgumentContainer):
"suppress": True,
},
],
[
["--use-http3-server"],
{
"dest": "useHttp3Server",
"default": False,
"help": "Whether to use the Http3 server",
"action": "store_true",
},
],
[
["--setpref"],
{

View file

@ -94,6 +94,7 @@ TEST_HARNESS_FILES.testing.mochitest += [
"chrome-harness.js",
"chunkifyTests.js",
"document-builder.sjs",
"DoHServer/doh_server.js",
"favicon.ico",
"harness.xhtml",
"leaks.py",
@ -110,6 +111,12 @@ TEST_HARNESS_FILES.testing.mochitest += [
"start_desktop.js",
]
TEST_HARNESS_FILES.testing.mochitest.DoHServer += [
"DoHServer/doh_server.js",
"DoHServer/http2-cert.key.pem",
"DoHServer/http2-cert.pem",
]
TEST_HARNESS_FILES.testing.mochitest.embed += [
"embed/Xm5i5kbIXzc",
"embed/Xm5i5kbIXzc^headers^",

View file

@ -48,6 +48,8 @@ class JUnitTestRunner(MochitestDesktop):
def __init__(self, log, options):
self.log = log
self.verbose = False
self.http3Server = None
self.dohServer = None
if (
options.log_tbpl_level == "debug"
or options.log_mach_level == "debug"
@ -126,10 +128,12 @@ class JUnitTestRunner(MochitestDesktop):
self.options.webServer = self.options.remoteWebServer
self.options.webSocketPort = "9988"
self.options.httpdPath = None
self.options.http3ServerPath = None
self.options.keep_open = False
self.options.pidFile = ""
self.options.subsuite = None
self.options.xrePath = None
self.options.useHttp3Server = False
if build_obj and "MOZ_HOST_BIN" in os.environ:
self.options.xrePath = os.environ["MOZ_HOST_BIN"]
if not self.options.utilityPath:

View file

@ -54,6 +54,7 @@ from manifestparser.filters import (
)
from manifestparser.util import normsep
from mozgeckoprofiler import symbolicate_profile_json, view_gecko_profile
from mozserve import DoHServer, Http3Server
try:
from marionette_driver.addons import Addons
@ -91,6 +92,13 @@ import six
from six.moves.urllib.parse import quote_plus as encodeURIComponent
from six.moves.urllib_request import urlopen
try:
from mozbuild.base import MozbuildObject
build = MozbuildObject.from_environment(cwd=SCRIPT_DIR)
except ImportError:
build = None
here = os.path.abspath(os.path.dirname(__file__))
NO_TESTS_FOUND = """
@ -1369,6 +1377,56 @@ class MochitestDesktop(object):
break
return is_webrtc_tag_present and options.subsuite in ["media"]
def startHttp3Server(self, options):
"""
Start a Http3 test server.
"""
http3ServerPath = os.path.join(
options.utilityPath, "http3server" + mozinfo.info["bin_suffix"]
)
serverOptions = {}
serverOptions["http3ServerPath"] = http3ServerPath
serverOptions["profilePath"] = options.profilePath
serverOptions["isMochitest"] = True
serverOptions["isWin"] = mozinfo.isWin
serverOptions["proxyPort"] = options.http3ServerPort
env = test_environment(xrePath=options.xrePath, log=self.log)
self.http3Server = Http3Server(serverOptions, env, self.log)
self.http3Server.start()
port = self.http3Server.ports().get("MOZHTTP3_PORT_PROXY")
if int(port) != options.http3ServerPort:
self.http3Server = None
raise RuntimeError("Error: Unable to start Http/3 server")
def startDoHServer(self, options):
"""
Start a DoH test server.
"""
# We try to find the node executable in the path given to us by the user in
# the MOZ_NODE_PATH environment variable
nodeBin = os.getenv("MOZ_NODE_PATH", None)
self.log.info("Use MOZ_NODE_PATH at %s" % (nodeBin))
if not nodeBin and build:
nodeBin = build.substs.get("NODEJS")
self.log.info("Use build node at %s" % (nodeBin))
serverOptions = {}
serverOptions["serverPath"] = os.path.join(
SCRIPT_DIR, "DoHServer", "doh_server.js"
)
serverOptions["nodeBin"] = nodeBin
serverOptions["dstServerPort"] = options.http3ServerPort
serverOptions["isWin"] = mozinfo.isWin
serverOptions["port"] = options.dohServerPort
env = test_environment(xrePath=options.xrePath, log=self.log)
self.dohServer = DoHServer(serverOptions, env, self.log)
self.dohServer.start()
port = self.dohServer.port()
if port != options.dohServerPort:
raise RuntimeError("Error: Unable to start DoH server")
def startServers(self, options, debuggerInfo, public=None):
# start servers and set ports
# TODO: pass these values, don't set on `self`
@ -1403,6 +1461,13 @@ class MochitestDesktop(object):
if self.server is not None:
self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
self.log.info("use http3 server: %d" % options.useHttp3Server)
self.http3Server = None
self.dohServer = None
if options.useHttp3Server:
self.startHttp3Server(options)
self.startDoHServer(options)
def stopServers(self):
"""Servers are no longer needed, and perhaps more importantly, anything they
might spew to console might confuse things."""
@ -1434,6 +1499,16 @@ class MochitestDesktop(object):
self.log.info("Stopping websocket/process bridge")
except Exception:
self.log.critical("Exception stopping websocket/process bridge")
if self.http3Server is not None:
try:
self.http3Server.stop()
except Exception:
self.log.critical("Exception stopping http3 server")
if self.dohServer is not None:
try:
self.dohServer.stop()
except Exception:
self.log.critical("Exception stopping doh server")
if hasattr(self, "gstForV4l2loopbackProcess"):
try:
@ -2149,6 +2224,12 @@ toolbar#nav-bar {
os.unlink(pwfilePath)
return 0
def findFreePort(self, type):
with closing(socket.socket(socket.AF_INET, type)) as s:
s.bind(("127.0.0.1", 0))
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
return s.getsockname()[1]
def proxy(self, options):
# proxy
# use SSL port for legacy compatibility; see
@ -2156,13 +2237,21 @@ toolbar#nav-bar {
# - https://bugzilla.mozilla.org/show_bug.cgi?id=899221
# - https://github.com/mozilla/mozbase/commit/43f9510e3d58bfed32790c82a57edac5f928474d
# 'ws': str(self.webSocketPort)
return {
proxyOptions = {
"remote": options.webServer,
"http": options.httpPort,
"https": options.sslPort,
"ws": options.sslPort,
}
if options.useHttp3Server:
options.dohServerPort = self.findFreePort(socket.SOCK_STREAM)
options.http3ServerPort = self.findFreePort(socket.SOCK_DGRAM)
proxyOptions["dohServerPort"] = options.dohServerPort
self.log.info("use doh server at port: %d" % options.dohServerPort)
self.log.info("use http3 server at port: %d" % options.http3ServerPort)
return proxyOptions
def merge_base_profiles(self, options, category):
"""Merge extra profile data from testing/profiles."""

View file

@ -46,6 +46,7 @@ python_modules = [
"mozproxy",
"mozrunner",
"mozscreenshot",
"mozserve",
"mozsystemmonitor",
"moztest",
"mozversion",

View file

@ -234,6 +234,17 @@ class Permissions(object):
prefs = []
if proxy:
dohServerPort = proxy.get("dohServerPort")
if dohServerPort is not None:
# make sure we don't use proxy
user_prefs = [("network.proxy.type", 0)]
# Use TRR_ONLY mode
user_prefs.append(("network.trr.mode", 3))
trrUri = "https://foo.example.com:{}/dns-query".format(dohServerPort)
user_prefs.append(("network.trr.uri", trrUri))
user_prefs.append(("network.trr.bootstrapAddr", "127.0.0.1"))
user_prefs.append(("network.dns.force_use_https_rr", True))
else:
user_prefs = self.pac_prefs(proxy)
else:
user_prefs = []

View file

@ -31,7 +31,7 @@ def test_nw_prefs(perms):
assert len(user_prefs) == 0
assert len(prefs) == 0
prefs, user_prefs = perms.network_prefs(True)
prefs, user_prefs = perms.network_prefs({"http": 8888})
assert len(user_prefs) == 2
assert user_prefs[0] == ("network.proxy.type", 2)
assert user_prefs[1][0] == "network.proxy.autoconfig_url"
@ -50,6 +50,15 @@ def test_nw_prefs(perms):
)
assert all(c in user_prefs[1][1] for c in proxy_check)
prefs, user_prefs = perms.network_prefs({"dohServerPort": 443})
print(user_prefs)
assert len(user_prefs) == 5
assert user_prefs[0] == ("network.proxy.type", 0)
assert user_prefs[1] == ("network.trr.mode", 3)
assert user_prefs[2] == ("network.trr.uri", "https://foo.example.com:443/dns-query")
assert user_prefs[3] == ("network.trr.bootstrapAddr", "127.0.0.1")
assert user_prefs[4] == ("network.dns.force_use_https_rr", True)
if __name__ == "__main__":
mozunit.main()

View file

@ -0,0 +1,12 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
mozserve is a simple script that is used to launch test servers, and
is designed for use in mochitest and xpcshelltest.
"""
from .servers import DoHServer, Http3Server
__all__ = ["Http3Server", "DoHServer"]

View file

@ -0,0 +1,229 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import copy
import os
import re
import subprocess
import sys
import time
from argparse import Namespace
from contextlib import contextmanager
from subprocess import PIPE, Popen
@contextmanager
def popenCleanupHack(isWin):
"""
Hack to work around https://bugs.python.org/issue37380
The basic idea is that on old versions of Python on Windows,
we need to clear subprocess._cleanup before we call Popen(),
then restore it afterwards.
"""
savedCleanup = None
if isWin and sys.version_info[0] == 3 and sys.version_info < (3, 7, 5):
savedCleanup = subprocess._cleanup
subprocess._cleanup = lambda: None
try:
yield
finally:
if savedCleanup:
subprocess._cleanup = savedCleanup
class Http3Server(object):
"""
Class which encapsulates the Http3 server
"""
def __init__(self, options, env, logger):
if isinstance(options, Namespace):
options = vars(options)
self._log = logger
self._profileDir = options["profilePath"]
self._env = copy.deepcopy(env)
self._ports = {}
self._echConfig = ""
self._isMochitest = options["isMochitest"]
self._http3ServerPath = options["http3ServerPath"]
self._isWin = options["isWin"]
self._http3ServerProc = {}
self._proxyPort = -1
if options.get("proxyPort"):
self._proxyPort = options["proxyPort"]
def ports(self):
return self._ports
def echConfig(self):
return self._echConfig
def start(self):
if not os.path.exists(self._http3ServerPath):
raise Exception("Http3 server not found at %s" % self._http3ServerPath)
self._log.info("mozserve | Found Http3Server path: %s" % self._http3ServerPath)
dbPath = os.path.join(self._profileDir, "cert9.db")
if not os.path.exists(dbPath):
raise Exception("cert db not found at %s" % dbPath)
dbPath = self._profileDir
self._log.info("mozserve | cert db path: %s" % dbPath)
try:
if self._isMochitest:
self._env["MOZ_HTTP3_MOCHITEST"] = "1"
if self._proxyPort != -1:
self._env["MOZ_HTTP3_PROXY_PORT"] = str(self._proxyPort)
with popenCleanupHack(self._isWin):
process = Popen(
[self._http3ServerPath, dbPath],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
env=self._env,
cwd=os.getcwd(),
universal_newlines=True,
)
self._http3ServerProc["http3Server"] = process
# Check to make sure the server starts properly by waiting for it to
# tell us it's started
msg = process.stdout.readline()
self._log.info("mozserve | http3 server msg: %s" % msg)
if "server listening" in msg:
searchObj = re.search(
r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)."
" EchConfig is @([\x00-\x7F]+)@",
msg,
0,
)
if searchObj:
self._ports["MOZHTTP3_PORT"] = searchObj.group(1)
self._ports["MOZHTTP3_PORT_FAILED"] = searchObj.group(2)
self._ports["MOZHTTP3_PORT_ECH"] = searchObj.group(3)
self._ports["MOZHTTP3_PORT_PROXY"] = searchObj.group(4)
self._ports["MOZHTTP3_PORT_NO_RESPONSE"] = searchObj.group(5)
self._echConfig = searchObj.group(6)
except OSError as e:
# This occurs if the subprocess couldn't be started
self._log.error("Could not run the http3 server: %s" % (str(e)))
def stop(self):
"""
Shutdown our http3Server process, if it exists
"""
for name, proc in self._http3ServerProc.items():
self._log.info("%s server shutting down ..." % name)
if proc.poll() is not None:
self._log.info("Http3 server %s already dead %s" % (name, proc.poll()))
else:
proc.terminate()
retries = 0
while proc.poll() is None:
time.sleep(0.1)
retries += 1
if retries > 40:
self._log.info("Killing proc")
proc.kill()
break
def dumpOutput(fd, label):
firstTime = True
for msg in fd:
if firstTime:
firstTime = False
self._log.info("Process %s" % label)
self._log.info(msg)
dumpOutput(proc.stdout, "stdout")
dumpOutput(proc.stderr, "stderr")
self._http3ServerProc = {}
class DoHServer(object):
"""
Class which encapsulates the DoH server
"""
def __init__(self, options, env, logger):
if isinstance(options, Namespace):
options = vars(options)
self._log = logger
self._port = options["port"]
self._env = copy.deepcopy(env)
self._nodeBin = options["nodeBin"]
self._serverPath = options["serverPath"]
self._dstServerPort = options["dstServerPort"]
self._isWin = options["isWin"]
self._nodeProc = None
def port(self):
return self._port
def start(self):
if not os.path.exists(self._serverPath):
raise Exception("DoH server not found at %s" % self._serverPath)
self._log.info("mozserve | Found DoH server path: %s" % self._serverPath)
if not os.path.exists(self._nodeBin) or not os.path.isfile(self._nodeBin):
raise Exception("node not found at path %s" % (self._nodeBin))
self._log.info("Found node at %s" % (self._nodeBin))
try:
# We pipe stdin to node because the server will exit when its
# stdin reaches EOF
with popenCleanupHack(self._isWin):
process = Popen(
[
self._nodeBin,
self._serverPath,
"serverPort={}".format(self._dstServerPort),
"listeningPort={}".format(self._port),
],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
env=self._env,
cwd=os.getcwd(),
universal_newlines=True,
)
self._nodeProc = process
msg = process.stdout.readline()
self._log.info("runtests.py | DoH server msg: %s" % msg)
if "server listening" in msg:
searchObj = re.search(r"DoH server listening on ports ([0-9]+)", msg, 0)
if searchObj:
self._port = int(searchObj.group(1))
self._log.info("DoH server started at port: %d" % (self._port))
except OSError as e:
# This occurs if the subprocess couldn't be started
self._log.error("Could not run DoH server: %s" % (str(e)))
def stop(self):
"""
Shut down our node process, if it exists
"""
if self._nodeProc is not None:
if self._nodeProc.poll() is not None:
self._log.info("Node server already dead %s" % (self._nodeProc.poll()))
else:
self._nodeProc.terminate()
def dumpOutput(fd, label):
firstTime = True
for msg in fd:
if firstTime:
firstTime = False
self._log.info("Process %s" % label)
self._log.info(msg)
dumpOutput(self._nodeProc.stdout, "stdout")
dumpOutput(self._nodeProc.stderr, "stderr")
self._nodeProc = None

View file

@ -0,0 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup
PACKAGE_VERSION = "0.1"
setup(
name="mozserve",
version=PACKAGE_VERSION,
description="Python test server launcher intended for use with Mozilla testing",
long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html",
license="MPL",
packages=["mozserve"],
zip_safe=False,
)

View file

@ -29,6 +29,7 @@ SEARCH_PATHS = [
"mozbase/mozprofile",
"mozbase/mozrunner",
"mozbase/mozscreenshot",
"mozbase/mozserve",
"mozbase/mozsystemmonitor",
"mozbase/moztest",
"mozbase/mozversion",

View file

@ -29,6 +29,7 @@ from threading import Event, Thread, Timer, current_thread
import mozdebug
import six
from mozserve import Http3Server
try:
import psutil
@ -976,7 +977,7 @@ class XPCShellTests(object):
self.log = log
self.harness_timeout = HARNESS_TIMEOUT
self.nodeProc = {}
self.http3ServerProc = {}
self.http3Server = None
self.conditioned_profile_dir = None
def getTestManifest(self, manifest):
@ -1415,8 +1416,7 @@ class XPCShellTests(object):
binSuffix = ""
if sys.platform == "win32":
binSuffix = ".exe"
http3ServerPath = self.http3server
http3ServerPath = self.http3ServerPath
if not http3ServerPath:
http3ServerPath = os.path.join(
SCRIPT_DIR, "http3server", "http3server" + binSuffix
@ -1425,89 +1425,25 @@ class XPCShellTests(object):
http3ServerPath = os.path.join(
build.topobjdir, "dist", "bin", "http3server" + binSuffix
)
if not os.path.exists(http3ServerPath):
self.log.warning(
"Http3 server not found at "
+ http3ServerPath
+ ". Tests requiring http/3 will fail."
)
return
# OK, we found our server, let's try to get it running
self.log.info("Found %s" % (http3ServerPath))
try:
dbPath = os.path.join(SCRIPT_DIR, "http3server", "http3serverDB")
if build:
dbPath = os.path.join(
build.topsrcdir, "netwerk", "test", "http3serverDB"
)
self.log.info("Using %s" % (dbPath))
# We pipe stdin to the server because it will exit when its stdin
# reaches EOF
with popenCleanupHack():
process = Popen(
[http3ServerPath, dbPath],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
env=self.env,
cwd=os.getcwd(),
universal_newlines=True,
)
self.http3ServerProc["http3Server"] = process
# Check to make sure the server starts properly by waiting for it to
# tell us it's started
msg = process.stdout.readline()
if "server listening" in msg:
searchObj = re.search(
r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)."
" EchConfig is @([\x00-\x7F]+)@",
msg,
0,
)
if searchObj:
self.env["MOZHTTP3_PORT"] = searchObj.group(1)
self.env["MOZHTTP3_PORT_FAILED"] = searchObj.group(2)
self.env["MOZHTTP3_PORT_ECH"] = searchObj.group(3)
self.env["MOZHTTP3_PORT_PROXY"] = searchObj.group(4)
self.env["MOZHTTP3_PORT_NO_RESPONSE"] = searchObj.group(5)
self.env["MOZHTTP3_ECH"] = searchObj.group(6)
except OSError as e:
# This occurs if the subprocess couldn't be started
self.log.error("Could not run the http3 server: %s" % (str(e)))
dbPath = os.path.join(build.topsrcdir, "netwerk", "test", "http3serverDB")
options = {}
options["http3ServerPath"] = http3ServerPath
options["profilePath"] = dbPath
options["isMochitest"] = False
options["isWin"] = sys.platform == "win32"
self.http3Server = Http3Server(options, self.env, self.log)
self.http3Server.start()
for key, value in self.http3Server.ports().items():
self.env[key] = value
self.env["MOZHTTP3_ECH"] = self.http3Server.echConfig()
def shutdownHttp3Server(self):
"""
Shutdown our http3Server process, if it exists
"""
for name, proc in six.iteritems(self.http3ServerProc):
self.log.info("%s server shutting down ..." % name)
if proc.poll() is not None:
self.log.info("Http3 server %s already dead %s" % (name, proc.poll()))
else:
proc.terminate()
retries = 0
while proc.poll() is None:
time.sleep(0.1)
retries += 1
if retries > 40:
self.log.info("Killing proc")
proc.kill()
break
def dumpOutput(fd, label):
firstTime = True
for msg in fd:
if firstTime:
firstTime = False
self.log.info("Process %s" % label)
self.log.info(msg)
dumpOutput(proc.stdout, "stdout")
dumpOutput(proc.stderr, "stderr")
self.http3ServerProc = {}
if self.http3Server is None:
return
self.http3Server.stop()
self.http3Server = None
def buildXpcsRunArgs(self):
"""
@ -1747,7 +1683,7 @@ class XPCShellTests(object):
self.app_binary = options.get("app_binary")
self.xpcshell = options.get("xpcshell")
self.http3server = options.get("http3server")
self.http3ServerPath = options.get("http3server")
self.xrePath = options.get("xrePath")
self.utility_path = options.get("utility_path")
self.appPath = options.get("appPath")