Bug 1809843 - Add DoH server and Http/3 server into mochitest, r=necko-reviewers,ahal,valentin,gbrown,jmaher

Differential Revision: https://phabricator.services.mozilla.com/D166644

build/pgo/certs/http2-cert.ca

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDKDCCAhCgAwIBAgIUZwyOUcu/hcoxGM+/mv6i/ZNoK8cwDQYJKoZIhvcNAQEL
+BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa
+GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9
+PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3
+HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg
+Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7
+EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK
+lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C
+AwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwSQYDVR0RBEIwQIIJbG9jYWxo
+b3N0gg9mb28uZXhhbXBsZS5jb22CEGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhh
+bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggEBAEPAH1dHcA0wyt6et0AjiRIBKla0
+EaW/9hFQW8xphNsgZTw2tX6vUqjd2qz4kzKTSbW8AmB26VEfSDkwUxbFrbCP8Pri
+FqVYwnWVLsBrr9PF0xg9SuRJnyHAwBv0xNXB3KRY2HtFcUaLn1C8alh2LAhX6eX+
++ozHvspMjMq7P1lsONGbtfmJaf6pNh7zzok4fOas+eoqhf64TUb4g4JNxfLyXh3P
+DcLsP67kr6fZSlGjzhto9ILVhg9rflNmSkmrhVjCxNqDuLFyn5ljvhemw7X3/V8A
+iCwqEP2/vOjnE1wCxF2Xdb6+3f/KJKqNwOmjraXZGupOUR6OA/+Z3V3RbGc=
+-----END CERTIFICATE-----

build/pgo/certs/http2-cert.ca.keyspec

@@ -0,0 +1 @@
+default

build/pgo/certs/http2-cert.certspec

@@ -0,0 +1,5 @@
+issuer: HTTP2 Test CA
+subject: HTTP2 Test Cert
+validity:20170101-20270101
+extension:extKeyUsage:serverAuth
+extension:subjectAlternativeName:localhost,foo.example.com,alt1.example.com,alt2.example.com

build/pgo/certs/mochitest-cert.ca

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDFjCCAf6gAwIBAgIUF75dZpigXpj3fOIppeG4xp8WwQowDQYJKoZIhvcNAQEL
+BQAwHTEbMBkGA1UEAwwSIE1vY2hpdGVzdCBUZXN0IENBMCIYDzIwMjIwMTAxMDAw
+MDAwWhgPMjAzMjAxMDEwMDAwMDBaMB8xHTAbBgNVBAMMFCBNb2NoaXRlc3QgVGVz
+dCBDZXJ0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuohRqESOFtZB
+/W62iAY2ED08E9nq5DVKtOz1aFdsJHvBxyWo4NgfvbGcBptuGobya+KvWnVramRx
+CHqlWqdFh/cc1SScAn7NQ/weadA4ICmTqyDDSeTbuUzCa2wO7RWCD/F+rWkasdMC
+OosqQe6ncOAPDY39ZgsrsCSSpH25iGF5kLFXkD3SO8XguEgfqDfTiEPvJxbYVbdm
+Wqp+ApAvOnsQgAYkzBxsl62WYVu34pYSwHUxowyR3bTK9/ytHSXTCe+5Fw6naOGz
+ey8ib2njtIqVYR3uJtYlnauRCE42yxwkBCy/Fosv5fGPmRcxuLP+SSP6clHEMdUD
+rNoYCjXtjQIDAQABo0gwRjATBgNVHSUEDDAKBggrBgEFBQcDATAvBgNVHREEKDAm
+ggptb2NoaS50ZXN0ggtleGFtcGxlLmNvbYILZXhhbXBsZS5vcmcwDQYJKoZIhvcN
+AQELBQADggEBAGL9Tc9f41G7nFZKlneKAJCMK0k2BLOED4YRx9yyAXPDCPiywfkF
+A/JLqSlnJH1roTxPf1BP3xIoe0K70IyBnwoRkZ3JWW5oKlDNkkIotLpGipLlOe3e
+gmguxWCU4ANvUNgJ5YvEByCV5wJUZT25HiQCFvKvpnLDMmIcXegTULLI0nb3yJz6
+NHqQX0SZYsQuw9+qSsnv/FypWX3VF+KAaP7tJVLpXDtJGrfboY9EbdnVrB36IHMQ
+ODaPvnwE4cewHaEWDUE2/orI9Gev11c1hK5nEWLHTOKQ3ZYbV8872xtWC7wcYzIr
+fJhMS05/mamP++g4rl5r4ZsOZ6nxIgOtISg=
+-----END CERTIFICATE-----

build/pgo/certs/mochitest-cert.ca.keyspec

@@ -0,0 +1 @@
+default

build/pgo/certs/mochitest-cert.certspec

@@ -0,0 +1,5 @@
+issuer: Mochitest Test CA
+subject: Mochitest Test Cert
+validity:20220101-20320101
+extension:extKeyUsage:serverAuth
+extension:subjectAlternativeName:mochi.test,example.com,example.org

build/pgo/server-locations.txt

@@ -371,3 +371,9 @@ http://expired.example.com:80                   privileged
 http://redirect-example.com:80              privileged
 https://redirect-example.com:443            privileged,cert=bug1706126cert
 https://www.redirect-example.com:443        privileged,cert=bug1706126cert
+
+# DoH server
+https://foo.example.com:4433                privileged,cert=http2-cert.pem
+
+# Mochitest
+https://mochi.test:443                      privileged,cert=mochitest-cert.pem

netwerk/test/http3server/src/main.rs

@@ -1004,11 +1004,16 @@ impl ServersRunner {
     }
 
     pub fn init(&mut self) {
-        self.add_new_socket(0, ServerType::Http3);
+        self.add_new_socket(0, ServerType::Http3, 0);
-        self.add_new_socket(1, ServerType::Http3Fail);
+        self.add_new_socket(1, ServerType::Http3Fail, 0);
-        self.add_new_socket(2, ServerType::Http3Ech);
+        self.add_new_socket(2, ServerType::Http3Ech, 0);
-        self.add_new_socket(3, ServerType::Http3Proxy);
+
-        self.add_new_socket(5, ServerType::Http3NoResponse);
+        let proxy_port = match env::var("MOZ_HTTP3_PROXY_PORT") {
+            Ok(val) => val.parse::<u16>().unwrap(),
+            _ => 0,
+        };
+        self.add_new_socket(3, ServerType::Http3Proxy, proxy_port);
+        self.add_new_socket(5, ServerType::Http3NoResponse, 0);
 
         println!(
             "HTTP3 server listening on ports {}, {}, {}, {} and {}. EchConfig is @{}@",
@@ -1024,8 +1029,8 @@ impl ServersRunner {
             .unwrap();
     }
 
-    fn add_new_socket(&mut self, count: usize, server_type: ServerType) -> u16 {
+    fn add_new_socket(&mut self, count: usize, server_type: ServerType, port: u16) -> u16 {
-        let addr = "127.0.0.1:0".parse().unwrap();
+        let addr = format!("127.0.0.1:{}", port).parse().unwrap();
 
         let socket = match UdpSocket::bind(&addr) {
             Err(err) => {

python/sites/mach.txt

@@ -38,6 +38,7 @@ pth:testing/mozbase/mozprocess
 pth:testing/mozbase/mozprofile
 pth:testing/mozbase/mozproxy
 pth:testing/mozbase/mozrunner
+pth:testing/mozbase/mozserve
 pth:testing/mozbase/mozsystemmonitor
 pth:testing/mozbase/mozscreenshot
 pth:testing/mozbase/moztest

testing/config/mozbase_requirements.txt

@@ -18,6 +18,7 @@
 ../mozbase/mozprofile
 ../mozbase/mozproxy
 ../mozbase/mozrunner
+../mozbase/mozserve
 ../mozbase/mozscreenshot
 ../mozbase/moztest
 ../mozbase/mozversion

testing/config/mozbase_source_requirements.txt

@@ -18,6 +18,7 @@
 --editable ../mozbase/mozprofile
 --editable ../mozbase/mozproxy
 --editable ../mozbase/mozrunner
+--editable ../mozbase/mozserve
 --editable ../mozbase/mozscreenshot
 --editable ../mozbase/moztest
 --editable ../mozbase/mozversion

testing/mochitest/DoHServer/doh_server.js

@@ -0,0 +1,86 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* globals require, __dirname, global, Buffer, process */
+
+const fs = require("fs");
+const options = {
+  key: fs.readFileSync(__dirname + "/http2-cert.key.pem"),
+  cert: fs.readFileSync(__dirname + "/http2-cert.pem"),
+};
+const http2 = require("http2");
+const http = require("http");
+const url = require("url");
+const dnsPacket = require("../../xpcshell/dns-packet");
+const _ = require("lodash");
+
+let serverPort = parseInt(process.argv[2].split("=")[1]);
+let listeningPort = parseInt(process.argv[3].split("=")[1]);
+
+let server = http2.createSecureServer(options, function handleRequest(
+  req,
+  res
+) {
+  let u = "";
+  if (req.url != undefined) {
+    u = url.parse(req.url, true);
+  }
+
+  if (u.pathname === "/dns-query") {
+    let payload = Buffer.from("");
+    req.on("data", function receiveData(chunk) {
+      payload = Buffer.concat([payload, chunk]);
+    });
+    req.on("end", function finishedData() {
+      let packet = dnsPacket.decode(payload);
+      let answers = [];
+      // Return the HTTPS RR to let Firefox connect to the HTTP/3 server
+      if (packet.questions[0].type === "HTTPS") {
+        answers.push({
+          name: packet.questions[0].name,
+          type: "HTTPS",
+          ttl: 55,
+          class: "IN",
+          flush: false,
+          data: {
+            priority: 1,
+            name: packet.questions[0].name,
+            values: [
+              { key: "alpn", value: ["h3"] },
+              { key: "port", value: serverPort },
+            ],
+          },
+        });
+      } else if (packet.questions[0].type === "A") {
+        answers.push({
+          name: packet.questions[0].name,
+          type: "A",
+          ttl: 55,
+          flush: false,
+          data: "127.0.0.1",
+        });
+      }
+
+      let buf = dnsPacket.encode({
+        type: "response",
+        id: packet.id,
+        flags: dnsPacket.RECURSION_DESIRED,
+        questions: packet.questions,
+        answers,
+      });
+
+      res.setHeader("Content-Type", "application/dns-message");
+      res.setHeader("Content-Length", buf.length);
+      res.writeHead(200);
+      res.write(buf);
+      res.end("");
+    });
+  }
+});
+
+server.listen(listeningPort);
+
+console.log(`DoH server listening on ports ${server.address().port}`);

testing/mochitest/DoHServer/http2-cert.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6iFGoRI4W1kH9
+braIBjYQPTwT2erkNUq07PVoV2wke8HHJajg2B+9sZwGm24ahvJr4q9adWtqZHEI
+eqVap0WH9xzVJJwCfs1D/B5p0DggKZOrIMNJ5Nu5TMJrbA7tFYIP8X6taRqx0wI6
+iypB7qdw4A8Njf1mCyuwJJKkfbmIYXmQsVeQPdI7xeC4SB+oN9OIQ+8nFthVt2Za
+qn4CkC86exCABiTMHGyXrZZhW7filhLAdTGjDJHdtMr3/K0dJdMJ77kXDqdo4bN7
+LyJvaeO0ipVhHe4m1iWdq5EITjbLHCQELL8Wiy/l8Y+ZFzG4s/5JI/pyUcQx1QOs
+2hgKNe2NAgMBAAECggEBAJ7LzjhhpFTsseD+j4XdQ8kvWCXOLpl4hNDhqUnaosWs
+VZskBFDlrJ/gw+McDu+mUlpl8MIhlABO4atGPd6e6CKHzJPnRqkZKcXmrD2IdT9s
+JbpZeec+XY+yOREaPNq4pLDN9fnKsF8SM6ODNcZLVWBSXn47kq18dQTPHcfLAFeI
+r8vh6Pld90AqFRUw1YCDRoZOs3CqeZVqWHhiy1M3kTB/cNkcltItABppAJuSPGgz
+iMnzbLm16+ZDAgQceNkIIGuHAJy4yrrK09vbJ5L7kRss9NtmA1hb6a4Mo7jmQXqg
+SwbkcOoaO1gcoDpngckxW2KzDmAR8iRyWUbuxXxtlEECgYEA3W4dT//r9o2InE0R
+TNqqnKpjpZN0KGyKXCmnF7umA3VkTVyqZ0xLi8cyY1hkYiDkVQ12CKwn1Vttt0+N
+gSfvj6CQmLaRR94GVXNEfhg9Iv59iFrOtRPZWB3V4HwakPXOCHneExNx7O/JznLp
+xD3BJ9I4GQ3oEXc8pdGTAfSMdCsCgYEA16dz2evDgKdn0v7Ak0rU6LVmckB3Gs3r
+ta15b0eP7E1FmF77yVMpaCicjYkQL63yHzTi3UlA66jAnW0fFtzClyl3TEMnXpJR
+3b5JCeH9O/Hkvt9Go5uLODMo70rjuVuS8gcK8myefFybWH/t3gXo59hspXiG+xZY
+EKd7mEW8MScCgYEAlkcrQaYQwK3hryJmwWAONnE1W6QtS1oOtOnX6zWBQAul3RMs
+2xpekyjHu8C7sBVeoZKXLt+X0SdR2Pz2rlcqMLHqMJqHEt1OMyQdse5FX8CT9byb
+WS11bmYhR08ywHryL7J100B5KzK6JZC7smGu+5WiWO6lN2VTFb6cJNGRmS0CgYAo
+tFCnp1qFZBOyvab3pj49lk+57PUOOCPvbMjo+ibuQT+LnRIFVA8Su+egx2got7pl
+rYPMpND+KiIBFOGzXQPVqFv+Jwa9UPzmz83VcbRspiG47UfWBbvnZbCqSgZlrCU2
+TaIBVAMuEgS4VZ0+NPtbF3yaVv+TUQpaSmKHwVHeLQKBgCgGe5NVgB0u9S36ltit
+tYlnPPjuipxv9yruq+nva+WKT0q/BfeIlH3IUf2qNFQhR6caJGv7BU7naqNGq80m
+ks/J5ExR5vBpxzXgc7oBn2pyFJYckbJoccrqv48GRBigJpDjmo1f8wZ7fNt/ULH1
+NBinA5ZsT8d0v3QCr2xDJH9D
+-----END PRIVATE KEY-----

testing/mochitest/DoHServer/http2-cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDKDCCAhCgAwIBAgIUZwyOUcu/hcoxGM+/mv6i/ZNoK8cwDQYJKoZIhvcNAQEL
+BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa
+GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9
+PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3
+HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg
+Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7
+EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK
+lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C
+AwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwSQYDVR0RBEIwQIIJbG9jYWxo
+b3N0gg9mb28uZXhhbXBsZS5jb22CEGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhh
+bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggEBAEPAH1dHcA0wyt6et0AjiRIBKla0
+EaW/9hFQW8xphNsgZTw2tX6vUqjd2qz4kzKTSbW8AmB26VEfSDkwUxbFrbCP8Pri
+FqVYwnWVLsBrr9PF0xg9SuRJnyHAwBv0xNXB3KRY2HtFcUaLn1C8alh2LAhX6eX+
++ozHvspMjMq7P1lsONGbtfmJaf6pNh7zzok4fOas+eoqhf64TUb4g4JNxfLyXh3P
+DcLsP67kr6fZSlGjzhto9ILVhg9rflNmSkmrhVjCxNqDuLFyn5ljvhemw7X3/V8A
+iCwqEP2/vOjnE1wCxF2Xdb6+3f/KJKqNwOmjraXZGupOUR6OA/+Z3V3RbGc=
+-----END CERTIFICATE-----

testing/mochitest/mochitest_options.py

@@ -504,6 +504,15 @@ class MochitestArguments(ArgumentContainer):
                 "suppress": True,
             },
         ],
+        [
+            ["--use-http3-server"],
+            {
+                "dest": "useHttp3Server",
+                "default": False,
+                "help": "Whether to use the Http3 server",
+                "action": "store_true",
+            },
+        ],
         [
             ["--setpref"],
             {

testing/mochitest/moz.build

@@ -94,6 +94,7 @@ TEST_HARNESS_FILES.testing.mochitest += [
     "chrome-harness.js",
     "chunkifyTests.js",
     "document-builder.sjs",
+    "DoHServer/doh_server.js",
     "favicon.ico",
     "harness.xhtml",
     "leaks.py",
@@ -110,6 +111,12 @@ TEST_HARNESS_FILES.testing.mochitest += [
     "start_desktop.js",
 ]
 
+TEST_HARNESS_FILES.testing.mochitest.DoHServer += [
+    "DoHServer/doh_server.js",
+    "DoHServer/http2-cert.key.pem",
+    "DoHServer/http2-cert.pem",
+]
+
 TEST_HARNESS_FILES.testing.mochitest.embed += [
     "embed/Xm5i5kbIXzc",
     "embed/Xm5i5kbIXzc^headers^",

testing/mochitest/runjunit.py

@@ -48,6 +48,8 @@ class JUnitTestRunner(MochitestDesktop):
     def __init__(self, log, options):
         self.log = log
         self.verbose = False
+        self.http3Server = None
+        self.dohServer = None
         if (
             options.log_tbpl_level == "debug"
             or options.log_mach_level == "debug"
@@ -126,10 +128,12 @@ class JUnitTestRunner(MochitestDesktop):
         self.options.webServer = self.options.remoteWebServer
         self.options.webSocketPort = "9988"
         self.options.httpdPath = None
+        self.options.http3ServerPath = None
         self.options.keep_open = False
         self.options.pidFile = ""
         self.options.subsuite = None
         self.options.xrePath = None
+        self.options.useHttp3Server = False
         if build_obj and "MOZ_HOST_BIN" in os.environ:
             self.options.xrePath = os.environ["MOZ_HOST_BIN"]
             if not self.options.utilityPath:

testing/mochitest/runtests.py

@@ -54,6 +54,7 @@ from manifestparser.filters import (
 )
 from manifestparser.util import normsep
 from mozgeckoprofiler import symbolicate_profile_json, view_gecko_profile
+from mozserve import DoHServer, Http3Server
 
 try:
     from marionette_driver.addons import Addons
@@ -91,6 +92,13 @@ import six
 from six.moves.urllib.parse import quote_plus as encodeURIComponent
 from six.moves.urllib_request import urlopen
 
+try:
+    from mozbuild.base import MozbuildObject
+
+    build = MozbuildObject.from_environment(cwd=SCRIPT_DIR)
+except ImportError:
+    build = None
+
 here = os.path.abspath(os.path.dirname(__file__))
 
 NO_TESTS_FOUND = """
@@ -1369,6 +1377,56 @@ class MochitestDesktop(object):
                 break
         return is_webrtc_tag_present and options.subsuite in ["media"]
 
+    def startHttp3Server(self, options):
+        """
+        Start a Http3 test server.
+        """
+        http3ServerPath = os.path.join(
+            options.utilityPath, "http3server" + mozinfo.info["bin_suffix"]
+        )
+        serverOptions = {}
+        serverOptions["http3ServerPath"] = http3ServerPath
+        serverOptions["profilePath"] = options.profilePath
+        serverOptions["isMochitest"] = True
+        serverOptions["isWin"] = mozinfo.isWin
+        serverOptions["proxyPort"] = options.http3ServerPort
+        env = test_environment(xrePath=options.xrePath, log=self.log)
+        self.http3Server = Http3Server(serverOptions, env, self.log)
+        self.http3Server.start()
+
+        port = self.http3Server.ports().get("MOZHTTP3_PORT_PROXY")
+        if int(port) != options.http3ServerPort:
+            self.http3Server = None
+            raise RuntimeError("Error: Unable to start Http/3 server")
+
+    def startDoHServer(self, options):
+        """
+        Start a DoH test server.
+        """
+        # We try to find the node executable in the path given to us by the user in
+        # the MOZ_NODE_PATH environment variable
+        nodeBin = os.getenv("MOZ_NODE_PATH", None)
+        self.log.info("Use MOZ_NODE_PATH at %s" % (nodeBin))
+        if not nodeBin and build:
+            nodeBin = build.substs.get("NODEJS")
+            self.log.info("Use build node at %s" % (nodeBin))
+
+        serverOptions = {}
+        serverOptions["serverPath"] = os.path.join(
+            SCRIPT_DIR, "DoHServer", "doh_server.js"
+        )
+        serverOptions["nodeBin"] = nodeBin
+        serverOptions["dstServerPort"] = options.http3ServerPort
+        serverOptions["isWin"] = mozinfo.isWin
+        serverOptions["port"] = options.dohServerPort
+        env = test_environment(xrePath=options.xrePath, log=self.log)
+        self.dohServer = DoHServer(serverOptions, env, self.log)
+        self.dohServer.start()
+
+        port = self.dohServer.port()
+        if port != options.dohServerPort:
+            raise RuntimeError("Error: Unable to start DoH server")
+
     def startServers(self, options, debuggerInfo, public=None):
         # start servers and set ports
         # TODO: pass these values, don't set on `self`
@@ -1403,6 +1461,13 @@ class MochitestDesktop(object):
         if self.server is not None:
             self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
 
+        self.log.info("use http3 server: %d" % options.useHttp3Server)
+        self.http3Server = None
+        self.dohServer = None
+        if options.useHttp3Server:
+            self.startHttp3Server(options)
+            self.startDoHServer(options)
+
     def stopServers(self):
         """Servers are no longer needed, and perhaps more importantly, anything they
         might spew to console might confuse things."""
@@ -1434,6 +1499,16 @@ class MochitestDesktop(object):
                 self.log.info("Stopping websocket/process bridge")
             except Exception:
                 self.log.critical("Exception stopping websocket/process bridge")
+        if self.http3Server is not None:
+            try:
+                self.http3Server.stop()
+            except Exception:
+                self.log.critical("Exception stopping http3 server")
+        if self.dohServer is not None:
+            try:
+                self.dohServer.stop()
+            except Exception:
+                self.log.critical("Exception stopping doh server")
 
         if hasattr(self, "gstForV4l2loopbackProcess"):
             try:
@@ -2149,6 +2224,12 @@ toolbar#nav-bar {
         os.unlink(pwfilePath)
         return 0
 
+    def findFreePort(self, type):
+        with closing(socket.socket(socket.AF_INET, type)) as s:
+            s.bind(("127.0.0.1", 0))
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            return s.getsockname()[1]
+
     def proxy(self, options):
         # proxy
         # use SSL port for legacy compatibility; see
@@ -2156,13 +2237,21 @@ toolbar#nav-bar {
         # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221
         # - https://github.com/mozilla/mozbase/commit/43f9510e3d58bfed32790c82a57edac5f928474d
         #             'ws': str(self.webSocketPort)
-        return {
+        proxyOptions = {
             "remote": options.webServer,
             "http": options.httpPort,
             "https": options.sslPort,
             "ws": options.sslPort,
         }
 
+        if options.useHttp3Server:
+            options.dohServerPort = self.findFreePort(socket.SOCK_STREAM)
+            options.http3ServerPort = self.findFreePort(socket.SOCK_DGRAM)
+            proxyOptions["dohServerPort"] = options.dohServerPort
+            self.log.info("use doh server at port: %d" % options.dohServerPort)
+            self.log.info("use http3 server at port: %d" % options.http3ServerPort)
+        return proxyOptions
+
     def merge_base_profiles(self, options, category):
         """Merge extra profile data from testing/profiles."""
 

testing/mozbase/moz.build

@@ -46,6 +46,7 @@ python_modules = [
     "mozproxy",
     "mozrunner",
     "mozscreenshot",
+    "mozserve",
     "mozsystemmonitor",
     "moztest",
     "mozversion",

testing/mozbase/mozprofile/mozprofile/permissions.py

@@ -234,7 +234,18 @@ class Permissions(object):
         prefs = []
 
         if proxy:
-            user_prefs = self.pac_prefs(proxy)
+            dohServerPort = proxy.get("dohServerPort")
+            if dohServerPort is not None:
+                # make sure we don't use proxy
+                user_prefs = [("network.proxy.type", 0)]
+                # Use TRR_ONLY mode
+                user_prefs.append(("network.trr.mode", 3))
+                trrUri = "https://foo.example.com:{}/dns-query".format(dohServerPort)
+                user_prefs.append(("network.trr.uri", trrUri))
+                user_prefs.append(("network.trr.bootstrapAddr", "127.0.0.1"))
+                user_prefs.append(("network.dns.force_use_https_rr", True))
+            else:
+                user_prefs = self.pac_prefs(proxy)
         else:
             user_prefs = []
 

testing/mozbase/mozprofile/tests/test_permissions.py

@@ -31,7 +31,7 @@ def test_nw_prefs(perms):
     assert len(user_prefs) == 0
     assert len(prefs) == 0
 
-    prefs, user_prefs = perms.network_prefs(True)
+    prefs, user_prefs = perms.network_prefs({"http": 8888})
     assert len(user_prefs) == 2
     assert user_prefs[0] == ("network.proxy.type", 2)
     assert user_prefs[1][0] == "network.proxy.autoconfig_url"
@@ -50,6 +50,15 @@ def test_nw_prefs(perms):
     )
     assert all(c in user_prefs[1][1] for c in proxy_check)
 
+    prefs, user_prefs = perms.network_prefs({"dohServerPort": 443})
+    print(user_prefs)
+    assert len(user_prefs) == 5
+    assert user_prefs[0] == ("network.proxy.type", 0)
+    assert user_prefs[1] == ("network.trr.mode", 3)
+    assert user_prefs[2] == ("network.trr.uri", "https://foo.example.com:443/dns-query")
+    assert user_prefs[3] == ("network.trr.bootstrapAddr", "127.0.0.1")
+    assert user_prefs[4] == ("network.dns.force_use_https_rr", True)
+
 
 if __name__ == "__main__":
     mozunit.main()

testing/mozbase/mozserve/mozserve/__init__.py

@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+mozserve is a simple script that is used to launch test servers, and
+is designed for use in mochitest and xpcshelltest.
+"""
+
+from .servers import DoHServer, Http3Server
+
+__all__ = ["Http3Server", "DoHServer"]

testing/mozbase/mozserve/mozserve/servers.py

@@ -0,0 +1,229 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import copy
+import os
+import re
+import subprocess
+import sys
+import time
+from argparse import Namespace
+from contextlib import contextmanager
+from subprocess import PIPE, Popen
+
+
+@contextmanager
+def popenCleanupHack(isWin):
+    """
+    Hack to work around https://bugs.python.org/issue37380
+    The basic idea is that on old versions of Python on Windows,
+    we need to clear subprocess._cleanup before we call Popen(),
+    then restore it afterwards.
+    """
+    savedCleanup = None
+    if isWin and sys.version_info[0] == 3 and sys.version_info < (3, 7, 5):
+        savedCleanup = subprocess._cleanup
+        subprocess._cleanup = lambda: None
+    try:
+        yield
+    finally:
+        if savedCleanup:
+            subprocess._cleanup = savedCleanup
+
+
+class Http3Server(object):
+    """
+    Class which encapsulates the Http3 server
+    """
+
+    def __init__(self, options, env, logger):
+        if isinstance(options, Namespace):
+            options = vars(options)
+        self._log = logger
+        self._profileDir = options["profilePath"]
+        self._env = copy.deepcopy(env)
+        self._ports = {}
+        self._echConfig = ""
+        self._isMochitest = options["isMochitest"]
+        self._http3ServerPath = options["http3ServerPath"]
+        self._isWin = options["isWin"]
+        self._http3ServerProc = {}
+        self._proxyPort = -1
+        if options.get("proxyPort"):
+            self._proxyPort = options["proxyPort"]
+
+    def ports(self):
+        return self._ports
+
+    def echConfig(self):
+        return self._echConfig
+
+    def start(self):
+        if not os.path.exists(self._http3ServerPath):
+            raise Exception("Http3 server not found at %s" % self._http3ServerPath)
+
+        self._log.info("mozserve | Found Http3Server path: %s" % self._http3ServerPath)
+
+        dbPath = os.path.join(self._profileDir, "cert9.db")
+        if not os.path.exists(dbPath):
+            raise Exception("cert db not found at %s" % dbPath)
+
+        dbPath = self._profileDir
+        self._log.info("mozserve | cert db path: %s" % dbPath)
+
+        try:
+            if self._isMochitest:
+                self._env["MOZ_HTTP3_MOCHITEST"] = "1"
+            if self._proxyPort != -1:
+                self._env["MOZ_HTTP3_PROXY_PORT"] = str(self._proxyPort)
+            with popenCleanupHack(self._isWin):
+                process = Popen(
+                    [self._http3ServerPath, dbPath],
+                    stdin=PIPE,
+                    stdout=PIPE,
+                    stderr=PIPE,
+                    env=self._env,
+                    cwd=os.getcwd(),
+                    universal_newlines=True,
+                )
+            self._http3ServerProc["http3Server"] = process
+
+            # Check to make sure the server starts properly by waiting for it to
+            # tell us it's started
+            msg = process.stdout.readline()
+            self._log.info("mozserve | http3 server msg: %s" % msg)
+            if "server listening" in msg:
+                searchObj = re.search(
+                    r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)."
+                    " EchConfig is @([\x00-\x7F]+)@",
+                    msg,
+                    0,
+                )
+                if searchObj:
+                    self._ports["MOZHTTP3_PORT"] = searchObj.group(1)
+                    self._ports["MOZHTTP3_PORT_FAILED"] = searchObj.group(2)
+                    self._ports["MOZHTTP3_PORT_ECH"] = searchObj.group(3)
+                    self._ports["MOZHTTP3_PORT_PROXY"] = searchObj.group(4)
+                    self._ports["MOZHTTP3_PORT_NO_RESPONSE"] = searchObj.group(5)
+                    self._echConfig = searchObj.group(6)
+        except OSError as e:
+            # This occurs if the subprocess couldn't be started
+            self._log.error("Could not run the http3 server: %s" % (str(e)))
+
+    def stop(self):
+        """
+        Shutdown our http3Server process, if it exists
+        """
+        for name, proc in self._http3ServerProc.items():
+            self._log.info("%s server shutting down ..." % name)
+            if proc.poll() is not None:
+                self._log.info("Http3 server %s already dead %s" % (name, proc.poll()))
+            else:
+                proc.terminate()
+                retries = 0
+                while proc.poll() is None:
+                    time.sleep(0.1)
+                    retries += 1
+                    if retries > 40:
+                        self._log.info("Killing proc")
+                        proc.kill()
+                        break
+
+            def dumpOutput(fd, label):
+                firstTime = True
+                for msg in fd:
+                    if firstTime:
+                        firstTime = False
+                        self._log.info("Process %s" % label)
+                    self._log.info(msg)
+
+            dumpOutput(proc.stdout, "stdout")
+            dumpOutput(proc.stderr, "stderr")
+        self._http3ServerProc = {}
+
+
+class DoHServer(object):
+    """
+    Class which encapsulates the DoH server
+    """
+
+    def __init__(self, options, env, logger):
+        if isinstance(options, Namespace):
+            options = vars(options)
+        self._log = logger
+        self._port = options["port"]
+        self._env = copy.deepcopy(env)
+        self._nodeBin = options["nodeBin"]
+        self._serverPath = options["serverPath"]
+        self._dstServerPort = options["dstServerPort"]
+        self._isWin = options["isWin"]
+        self._nodeProc = None
+
+    def port(self):
+        return self._port
+
+    def start(self):
+        if not os.path.exists(self._serverPath):
+            raise Exception("DoH server not found at %s" % self._serverPath)
+
+        self._log.info("mozserve | Found DoH server path: %s" % self._serverPath)
+
+        if not os.path.exists(self._nodeBin) or not os.path.isfile(self._nodeBin):
+            raise Exception("node not found at path %s" % (self._nodeBin))
+
+        self._log.info("Found node at %s" % (self._nodeBin))
+
+        try:
+            # We pipe stdin to node because the server will exit when its
+            # stdin reaches EOF
+            with popenCleanupHack(self._isWin):
+                process = Popen(
+                    [
+                        self._nodeBin,
+                        self._serverPath,
+                        "serverPort={}".format(self._dstServerPort),
+                        "listeningPort={}".format(self._port),
+                    ],
+                    stdin=PIPE,
+                    stdout=PIPE,
+                    stderr=PIPE,
+                    env=self._env,
+                    cwd=os.getcwd(),
+                    universal_newlines=True,
+                )
+            self._nodeProc = process
+
+            msg = process.stdout.readline()
+            self._log.info("runtests.py | DoH server msg: %s" % msg)
+            if "server listening" in msg:
+                searchObj = re.search(r"DoH server listening on ports ([0-9]+)", msg, 0)
+                if searchObj:
+                    self._port = int(searchObj.group(1))
+                    self._log.info("DoH server started at port: %d" % (self._port))
+        except OSError as e:
+            # This occurs if the subprocess couldn't be started
+            self._log.error("Could not run DoH server: %s" % (str(e)))
+
+    def stop(self):
+        """
+        Shut down our node process, if it exists
+        """
+        if self._nodeProc is not None:
+            if self._nodeProc.poll() is not None:
+                self._log.info("Node server already dead %s" % (self._nodeProc.poll()))
+            else:
+                self._nodeProc.terminate()
+
+            def dumpOutput(fd, label):
+                firstTime = True
+                for msg in fd:
+                    if firstTime:
+                        firstTime = False
+                        self._log.info("Process %s" % label)
+                    self._log.info(msg)
+
+            dumpOutput(self._nodeProc.stdout, "stdout")
+            dumpOutput(self._nodeProc.stderr, "stderr")
+
+            self._nodeProc = None

testing/mozbase/mozserve/setup.py

@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+PACKAGE_VERSION = "0.1"
+
+setup(
+    name="mozserve",
+    version=PACKAGE_VERSION,
+    description="Python test server launcher intended for use with Mozilla testing",
+    long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html",
+    license="MPL",
+    packages=["mozserve"],
+    zip_safe=False,
+)

testing/tools/mach_test_package_initialize.py

@@ -29,6 +29,7 @@ SEARCH_PATHS = [
     "mozbase/mozprofile",
     "mozbase/mozrunner",
     "mozbase/mozscreenshot",
+    "mozbase/mozserve",
     "mozbase/mozsystemmonitor",
     "mozbase/moztest",
     "mozbase/mozversion",

testing/xpcshell/runxpcshelltests.py

@@ -29,6 +29,7 @@ from threading import Event, Thread, Timer, current_thread
 
 import mozdebug
 import six
+from mozserve import Http3Server
 
 try:
     import psutil
@@ -976,7 +977,7 @@ class XPCShellTests(object):
         self.log = log
         self.harness_timeout = HARNESS_TIMEOUT
         self.nodeProc = {}
-        self.http3ServerProc = {}
+        self.http3Server = None
         self.conditioned_profile_dir = None
 
     def getTestManifest(self, manifest):
@@ -1415,8 +1416,7 @@ class XPCShellTests(object):
         binSuffix = ""
         if sys.platform == "win32":
             binSuffix = ".exe"
-
+        http3ServerPath = self.http3ServerPath
-        http3ServerPath = self.http3server
         if not http3ServerPath:
             http3ServerPath = os.path.join(
                 SCRIPT_DIR, "http3server", "http3server" + binSuffix
@@ -1425,89 +1425,25 @@ class XPCShellTests(object):
                 http3ServerPath = os.path.join(
                     build.topobjdir, "dist", "bin", "http3server" + binSuffix
                 )
-
+        dbPath = os.path.join(SCRIPT_DIR, "http3server", "http3serverDB")
-        if not os.path.exists(http3ServerPath):
+        if build:
-            self.log.warning(
+            dbPath = os.path.join(build.topsrcdir, "netwerk", "test", "http3serverDB")
-                "Http3 server not found at "
+        options = {}
-                + http3ServerPath
+        options["http3ServerPath"] = http3ServerPath
-                + ". Tests requiring http/3 will fail."
+        options["profilePath"] = dbPath
-            )
+        options["isMochitest"] = False
-            return
+        options["isWin"] = sys.platform == "win32"
-
+        self.http3Server = Http3Server(options, self.env, self.log)
-        # OK, we found our server, let's try to get it running
+        self.http3Server.start()
-        self.log.info("Found %s" % (http3ServerPath))
+        for key, value in self.http3Server.ports().items():
-        try:
+            self.env[key] = value
-            dbPath = os.path.join(SCRIPT_DIR, "http3server", "http3serverDB")
+        self.env["MOZHTTP3_ECH"] = self.http3Server.echConfig()
-            if build:
-                dbPath = os.path.join(
-                    build.topsrcdir, "netwerk", "test", "http3serverDB"
-                )
-            self.log.info("Using %s" % (dbPath))
-            # We pipe stdin to the server because it will exit when its stdin
-            # reaches EOF
-            with popenCleanupHack():
-                process = Popen(
-                    [http3ServerPath, dbPath],
-                    stdin=PIPE,
-                    stdout=PIPE,
-                    stderr=PIPE,
-                    env=self.env,
-                    cwd=os.getcwd(),
-                    universal_newlines=True,
-                )
-            self.http3ServerProc["http3Server"] = process
-
-            # Check to make sure the server starts properly by waiting for it to
-            # tell us it's started
-            msg = process.stdout.readline()
-            if "server listening" in msg:
-                searchObj = re.search(
-                    r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)."
-                    " EchConfig is @([\x00-\x7F]+)@",
-                    msg,
-                    0,
-                )
-                if searchObj:
-                    self.env["MOZHTTP3_PORT"] = searchObj.group(1)
-                    self.env["MOZHTTP3_PORT_FAILED"] = searchObj.group(2)
-                    self.env["MOZHTTP3_PORT_ECH"] = searchObj.group(3)
-                    self.env["MOZHTTP3_PORT_PROXY"] = searchObj.group(4)
-                    self.env["MOZHTTP3_PORT_NO_RESPONSE"] = searchObj.group(5)
-                    self.env["MOZHTTP3_ECH"] = searchObj.group(6)
-        except OSError as e:
-            # This occurs if the subprocess couldn't be started
-            self.log.error("Could not run the http3 server: %s" % (str(e)))
 
     def shutdownHttp3Server(self):
-        """
+        if self.http3Server is None:
-        Shutdown our http3Server process, if it exists
+            return
-        """
+        self.http3Server.stop()
-        for name, proc in six.iteritems(self.http3ServerProc):
+        self.http3Server = None
-            self.log.info("%s server shutting down ..." % name)
-            if proc.poll() is not None:
-                self.log.info("Http3 server %s already dead %s" % (name, proc.poll()))
-            else:
-                proc.terminate()
-                retries = 0
-                while proc.poll() is None:
-                    time.sleep(0.1)
-                    retries += 1
-                    if retries > 40:
-                        self.log.info("Killing proc")
-                        proc.kill()
-                        break
-
-            def dumpOutput(fd, label):
-                firstTime = True
-                for msg in fd:
-                    if firstTime:
-                        firstTime = False
-                        self.log.info("Process %s" % label)
-                    self.log.info(msg)
-
-            dumpOutput(proc.stdout, "stdout")
-            dumpOutput(proc.stderr, "stderr")
-        self.http3ServerProc = {}
 
     def buildXpcsRunArgs(self):
         """
@@ -1747,7 +1683,7 @@ class XPCShellTests(object):
 
         self.app_binary = options.get("app_binary")
         self.xpcshell = options.get("xpcshell")
-        self.http3server = options.get("http3server")
+        self.http3ServerPath = options.get("http3server")
         self.xrePath = options.get("xrePath")
         self.utility_path = options.get("utility_path")
         self.appPath = options.get("appPath")