diff --git a/build/pgo/certs/cert9.db b/build/pgo/certs/cert9.db index 818c11415312..316cd8292384 100644 Binary files a/build/pgo/certs/cert9.db and b/build/pgo/certs/cert9.db differ diff --git a/build/pgo/certs/http2-cert.ca b/build/pgo/certs/http2-cert.ca new file mode 100644 index 000000000000..a6bd5d72e96e --- /dev/null +++ b/build/pgo/certs/http2-cert.ca @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKDCCAhCgAwIBAgIUZwyOUcu/hcoxGM+/mv6i/ZNoK8cwDQYJKoZIhvcNAQEL +BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa +GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9 +PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3 +HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg +Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7 +EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK +lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C +AwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwSQYDVR0RBEIwQIIJbG9jYWxo +b3N0gg9mb28uZXhhbXBsZS5jb22CEGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggEBAEPAH1dHcA0wyt6et0AjiRIBKla0 +EaW/9hFQW8xphNsgZTw2tX6vUqjd2qz4kzKTSbW8AmB26VEfSDkwUxbFrbCP8Pri +FqVYwnWVLsBrr9PF0xg9SuRJnyHAwBv0xNXB3KRY2HtFcUaLn1C8alh2LAhX6eX+ ++ozHvspMjMq7P1lsONGbtfmJaf6pNh7zzok4fOas+eoqhf64TUb4g4JNxfLyXh3P +DcLsP67kr6fZSlGjzhto9ILVhg9rflNmSkmrhVjCxNqDuLFyn5ljvhemw7X3/V8A +iCwqEP2/vOjnE1wCxF2Xdb6+3f/KJKqNwOmjraXZGupOUR6OA/+Z3V3RbGc= +-----END CERTIFICATE----- diff --git a/build/pgo/certs/http2-cert.ca.keyspec b/build/pgo/certs/http2-cert.ca.keyspec new file mode 100644 index 000000000000..4ad96d51599f --- /dev/null +++ b/build/pgo/certs/http2-cert.ca.keyspec @@ -0,0 +1 @@ +default diff --git a/build/pgo/certs/http2-cert.certspec b/build/pgo/certs/http2-cert.certspec new file mode 100644 index 000000000000..1277e6d444fc --- /dev/null +++ b/build/pgo/certs/http2-cert.certspec @@ -0,0 +1,5 @@ +issuer: HTTP2 Test CA +subject: HTTP2 Test Cert +validity:20170101-20270101 +extension:extKeyUsage:serverAuth +extension:subjectAlternativeName:localhost,foo.example.com,alt1.example.com,alt2.example.com diff --git a/build/pgo/certs/key4.db b/build/pgo/certs/key4.db index 12335f35d7e8..fe14fa2eecca 100644 Binary files a/build/pgo/certs/key4.db and b/build/pgo/certs/key4.db differ diff --git a/build/pgo/certs/mochitest-cert.ca b/build/pgo/certs/mochitest-cert.ca new file mode 100644 index 000000000000..6b409728c279 --- /dev/null +++ b/build/pgo/certs/mochitest-cert.ca @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDFjCCAf6gAwIBAgIUF75dZpigXpj3fOIppeG4xp8WwQowDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSIE1vY2hpdGVzdCBUZXN0IENBMCIYDzIwMjIwMTAxMDAw +MDAwWhgPMjAzMjAxMDEwMDAwMDBaMB8xHTAbBgNVBAMMFCBNb2NoaXRlc3QgVGVz +dCBDZXJ0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuohRqESOFtZB +/W62iAY2ED08E9nq5DVKtOz1aFdsJHvBxyWo4NgfvbGcBptuGobya+KvWnVramRx +CHqlWqdFh/cc1SScAn7NQ/weadA4ICmTqyDDSeTbuUzCa2wO7RWCD/F+rWkasdMC +OosqQe6ncOAPDY39ZgsrsCSSpH25iGF5kLFXkD3SO8XguEgfqDfTiEPvJxbYVbdm +Wqp+ApAvOnsQgAYkzBxsl62WYVu34pYSwHUxowyR3bTK9/ytHSXTCe+5Fw6naOGz +ey8ib2njtIqVYR3uJtYlnauRCE42yxwkBCy/Fosv5fGPmRcxuLP+SSP6clHEMdUD +rNoYCjXtjQIDAQABo0gwRjATBgNVHSUEDDAKBggrBgEFBQcDATAvBgNVHREEKDAm +ggptb2NoaS50ZXN0ggtleGFtcGxlLmNvbYILZXhhbXBsZS5vcmcwDQYJKoZIhvcN +AQELBQADggEBAGL9Tc9f41G7nFZKlneKAJCMK0k2BLOED4YRx9yyAXPDCPiywfkF +A/JLqSlnJH1roTxPf1BP3xIoe0K70IyBnwoRkZ3JWW5oKlDNkkIotLpGipLlOe3e +gmguxWCU4ANvUNgJ5YvEByCV5wJUZT25HiQCFvKvpnLDMmIcXegTULLI0nb3yJz6 +NHqQX0SZYsQuw9+qSsnv/FypWX3VF+KAaP7tJVLpXDtJGrfboY9EbdnVrB36IHMQ +ODaPvnwE4cewHaEWDUE2/orI9Gev11c1hK5nEWLHTOKQ3ZYbV8872xtWC7wcYzIr +fJhMS05/mamP++g4rl5r4ZsOZ6nxIgOtISg= +-----END CERTIFICATE----- diff --git a/build/pgo/certs/mochitest-cert.ca.keyspec b/build/pgo/certs/mochitest-cert.ca.keyspec new file mode 100644 index 000000000000..4ad96d51599f --- /dev/null +++ b/build/pgo/certs/mochitest-cert.ca.keyspec @@ -0,0 +1 @@ +default diff --git a/build/pgo/certs/mochitest-cert.certspec b/build/pgo/certs/mochitest-cert.certspec new file mode 100644 index 000000000000..cfac71f493d0 --- /dev/null +++ b/build/pgo/certs/mochitest-cert.certspec @@ -0,0 +1,5 @@ +issuer: Mochitest Test CA +subject: Mochitest Test Cert +validity:20220101-20320101 +extension:extKeyUsage:serverAuth +extension:subjectAlternativeName:mochi.test,example.com,example.org diff --git a/build/pgo/certs/mochitest.client b/build/pgo/certs/mochitest.client index 41870acc1d25..ce71cc6ff261 100644 Binary files a/build/pgo/certs/mochitest.client and b/build/pgo/certs/mochitest.client differ diff --git a/build/pgo/server-locations.txt b/build/pgo/server-locations.txt index 55a4ed9567e1..dc5db0d81d70 100644 --- a/build/pgo/server-locations.txt +++ b/build/pgo/server-locations.txt @@ -371,3 +371,9 @@ http://expired.example.com:80 privileged http://redirect-example.com:80 privileged https://redirect-example.com:443 privileged,cert=bug1706126cert https://www.redirect-example.com:443 privileged,cert=bug1706126cert + +# DoH server +https://foo.example.com:4433 privileged,cert=http2-cert.pem + +# Mochitest +https://mochi.test:443 privileged,cert=mochitest-cert.pem diff --git a/netwerk/test/http3server/src/main.rs b/netwerk/test/http3server/src/main.rs index 8015f4aa1fde..947c02719283 100644 --- a/netwerk/test/http3server/src/main.rs +++ b/netwerk/test/http3server/src/main.rs @@ -1004,11 +1004,16 @@ impl ServersRunner { } pub fn init(&mut self) { - self.add_new_socket(0, ServerType::Http3); - self.add_new_socket(1, ServerType::Http3Fail); - self.add_new_socket(2, ServerType::Http3Ech); - self.add_new_socket(3, ServerType::Http3Proxy); - self.add_new_socket(5, ServerType::Http3NoResponse); + self.add_new_socket(0, ServerType::Http3, 0); + self.add_new_socket(1, ServerType::Http3Fail, 0); + self.add_new_socket(2, ServerType::Http3Ech, 0); + + let proxy_port = match env::var("MOZ_HTTP3_PROXY_PORT") { + Ok(val) => val.parse::().unwrap(), + _ => 0, + }; + self.add_new_socket(3, ServerType::Http3Proxy, proxy_port); + self.add_new_socket(5, ServerType::Http3NoResponse, 0); println!( "HTTP3 server listening on ports {}, {}, {}, {} and {}. EchConfig is @{}@", @@ -1024,8 +1029,8 @@ impl ServersRunner { .unwrap(); } - fn add_new_socket(&mut self, count: usize, server_type: ServerType) -> u16 { - let addr = "127.0.0.1:0".parse().unwrap(); + fn add_new_socket(&mut self, count: usize, server_type: ServerType, port: u16) -> u16 { + let addr = format!("127.0.0.1:{}", port).parse().unwrap(); let socket = match UdpSocket::bind(&addr) { Err(err) => { diff --git a/python/sites/mach.txt b/python/sites/mach.txt index 44423da809e8..ce964d863248 100644 --- a/python/sites/mach.txt +++ b/python/sites/mach.txt @@ -38,6 +38,7 @@ pth:testing/mozbase/mozprocess pth:testing/mozbase/mozprofile pth:testing/mozbase/mozproxy pth:testing/mozbase/mozrunner +pth:testing/mozbase/mozserve pth:testing/mozbase/mozsystemmonitor pth:testing/mozbase/mozscreenshot pth:testing/mozbase/moztest diff --git a/testing/config/mozbase_requirements.txt b/testing/config/mozbase_requirements.txt index 21076e199e0a..fe277e2f6a38 100644 --- a/testing/config/mozbase_requirements.txt +++ b/testing/config/mozbase_requirements.txt @@ -18,6 +18,7 @@ ../mozbase/mozprofile ../mozbase/mozproxy ../mozbase/mozrunner +../mozbase/mozserve ../mozbase/mozscreenshot ../mozbase/moztest ../mozbase/mozversion diff --git a/testing/config/mozbase_source_requirements.txt b/testing/config/mozbase_source_requirements.txt index 880017e0456b..1182b28d956e 100644 --- a/testing/config/mozbase_source_requirements.txt +++ b/testing/config/mozbase_source_requirements.txt @@ -18,6 +18,7 @@ --editable ../mozbase/mozprofile --editable ../mozbase/mozproxy --editable ../mozbase/mozrunner +--editable ../mozbase/mozserve --editable ../mozbase/mozscreenshot --editable ../mozbase/moztest --editable ../mozbase/mozversion diff --git a/testing/mochitest/DoHServer/doh_server.js b/testing/mochitest/DoHServer/doh_server.js new file mode 100644 index 000000000000..f909be0b3261 --- /dev/null +++ b/testing/mochitest/DoHServer/doh_server.js @@ -0,0 +1,86 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/* globals require, __dirname, global, Buffer, process */ + +const fs = require("fs"); +const options = { + key: fs.readFileSync(__dirname + "/http2-cert.key.pem"), + cert: fs.readFileSync(__dirname + "/http2-cert.pem"), +}; +const http2 = require("http2"); +const http = require("http"); +const url = require("url"); +const dnsPacket = require("../../xpcshell/dns-packet"); +const _ = require("lodash"); + +let serverPort = parseInt(process.argv[2].split("=")[1]); +let listeningPort = parseInt(process.argv[3].split("=")[1]); + +let server = http2.createSecureServer(options, function handleRequest( + req, + res +) { + let u = ""; + if (req.url != undefined) { + u = url.parse(req.url, true); + } + + if (u.pathname === "/dns-query") { + let payload = Buffer.from(""); + req.on("data", function receiveData(chunk) { + payload = Buffer.concat([payload, chunk]); + }); + req.on("end", function finishedData() { + let packet = dnsPacket.decode(payload); + let answers = []; + // Return the HTTPS RR to let Firefox connect to the HTTP/3 server + if (packet.questions[0].type === "HTTPS") { + answers.push({ + name: packet.questions[0].name, + type: "HTTPS", + ttl: 55, + class: "IN", + flush: false, + data: { + priority: 1, + name: packet.questions[0].name, + values: [ + { key: "alpn", value: ["h3"] }, + { key: "port", value: serverPort }, + ], + }, + }); + } else if (packet.questions[0].type === "A") { + answers.push({ + name: packet.questions[0].name, + type: "A", + ttl: 55, + flush: false, + data: "127.0.0.1", + }); + } + + let buf = dnsPacket.encode({ + type: "response", + id: packet.id, + flags: dnsPacket.RECURSION_DESIRED, + questions: packet.questions, + answers, + }); + + res.setHeader("Content-Type", "application/dns-message"); + res.setHeader("Content-Length", buf.length); + res.writeHead(200); + res.write(buf); + res.end(""); + }); + } +}); + +server.listen(listeningPort); + +console.log(`DoH server listening on ports ${server.address().port}`); diff --git a/testing/mochitest/DoHServer/http2-cert.key.pem b/testing/mochitest/DoHServer/http2-cert.key.pem new file mode 100644 index 000000000000..09e044f5e087 --- /dev/null +++ b/testing/mochitest/DoHServer/http2-cert.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6iFGoRI4W1kH9 +braIBjYQPTwT2erkNUq07PVoV2wke8HHJajg2B+9sZwGm24ahvJr4q9adWtqZHEI +eqVap0WH9xzVJJwCfs1D/B5p0DggKZOrIMNJ5Nu5TMJrbA7tFYIP8X6taRqx0wI6 +iypB7qdw4A8Njf1mCyuwJJKkfbmIYXmQsVeQPdI7xeC4SB+oN9OIQ+8nFthVt2Za +qn4CkC86exCABiTMHGyXrZZhW7filhLAdTGjDJHdtMr3/K0dJdMJ77kXDqdo4bN7 +LyJvaeO0ipVhHe4m1iWdq5EITjbLHCQELL8Wiy/l8Y+ZFzG4s/5JI/pyUcQx1QOs +2hgKNe2NAgMBAAECggEBAJ7LzjhhpFTsseD+j4XdQ8kvWCXOLpl4hNDhqUnaosWs +VZskBFDlrJ/gw+McDu+mUlpl8MIhlABO4atGPd6e6CKHzJPnRqkZKcXmrD2IdT9s +JbpZeec+XY+yOREaPNq4pLDN9fnKsF8SM6ODNcZLVWBSXn47kq18dQTPHcfLAFeI +r8vh6Pld90AqFRUw1YCDRoZOs3CqeZVqWHhiy1M3kTB/cNkcltItABppAJuSPGgz +iMnzbLm16+ZDAgQceNkIIGuHAJy4yrrK09vbJ5L7kRss9NtmA1hb6a4Mo7jmQXqg +SwbkcOoaO1gcoDpngckxW2KzDmAR8iRyWUbuxXxtlEECgYEA3W4dT//r9o2InE0R +TNqqnKpjpZN0KGyKXCmnF7umA3VkTVyqZ0xLi8cyY1hkYiDkVQ12CKwn1Vttt0+N +gSfvj6CQmLaRR94GVXNEfhg9Iv59iFrOtRPZWB3V4HwakPXOCHneExNx7O/JznLp +xD3BJ9I4GQ3oEXc8pdGTAfSMdCsCgYEA16dz2evDgKdn0v7Ak0rU6LVmckB3Gs3r +ta15b0eP7E1FmF77yVMpaCicjYkQL63yHzTi3UlA66jAnW0fFtzClyl3TEMnXpJR +3b5JCeH9O/Hkvt9Go5uLODMo70rjuVuS8gcK8myefFybWH/t3gXo59hspXiG+xZY +EKd7mEW8MScCgYEAlkcrQaYQwK3hryJmwWAONnE1W6QtS1oOtOnX6zWBQAul3RMs +2xpekyjHu8C7sBVeoZKXLt+X0SdR2Pz2rlcqMLHqMJqHEt1OMyQdse5FX8CT9byb +WS11bmYhR08ywHryL7J100B5KzK6JZC7smGu+5WiWO6lN2VTFb6cJNGRmS0CgYAo +tFCnp1qFZBOyvab3pj49lk+57PUOOCPvbMjo+ibuQT+LnRIFVA8Su+egx2got7pl +rYPMpND+KiIBFOGzXQPVqFv+Jwa9UPzmz83VcbRspiG47UfWBbvnZbCqSgZlrCU2 +TaIBVAMuEgS4VZ0+NPtbF3yaVv+TUQpaSmKHwVHeLQKBgCgGe5NVgB0u9S36ltit +tYlnPPjuipxv9yruq+nva+WKT0q/BfeIlH3IUf2qNFQhR6caJGv7BU7naqNGq80m +ks/J5ExR5vBpxzXgc7oBn2pyFJYckbJoccrqv48GRBigJpDjmo1f8wZ7fNt/ULH1 +NBinA5ZsT8d0v3QCr2xDJH9D +-----END PRIVATE KEY----- diff --git a/testing/mochitest/DoHServer/http2-cert.pem b/testing/mochitest/DoHServer/http2-cert.pem new file mode 100644 index 000000000000..a6bd5d72e96e --- /dev/null +++ b/testing/mochitest/DoHServer/http2-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKDCCAhCgAwIBAgIUZwyOUcu/hcoxGM+/mv6i/ZNoK8cwDQYJKoZIhvcNAQEL +BQAwGTEXMBUGA1UEAwwOIEhUVFAyIFRlc3QgQ0EwIhgPMjAxNzAxMDEwMDAwMDBa +GA8yMDI3MDEwMTAwMDAwMFowGzEZMBcGA1UEAwwQIEhUVFAyIFRlc3QgQ2VydDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALqIUahEjhbWQf1utogGNhA9 +PBPZ6uQ1SrTs9WhXbCR7wcclqODYH72xnAabbhqG8mvir1p1a2pkcQh6pVqnRYf3 +HNUknAJ+zUP8HmnQOCApk6sgw0nk27lMwmtsDu0Vgg/xfq1pGrHTAjqLKkHup3Dg +Dw2N/WYLK7AkkqR9uYhheZCxV5A90jvF4LhIH6g304hD7ycW2FW3ZlqqfgKQLzp7 +EIAGJMwcbJetlmFbt+KWEsB1MaMMkd20yvf8rR0l0wnvuRcOp2jhs3svIm9p47SK +lWEd7ibWJZ2rkQhONsscJAQsvxaLL+Xxj5kXMbiz/kkj+nJRxDHVA6zaGAo17Y0C +AwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwSQYDVR0RBEIwQIIJbG9jYWxo +b3N0gg9mb28uZXhhbXBsZS5jb22CEGFsdDEuZXhhbXBsZS5jb22CEGFsdDIuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggEBAEPAH1dHcA0wyt6et0AjiRIBKla0 +EaW/9hFQW8xphNsgZTw2tX6vUqjd2qz4kzKTSbW8AmB26VEfSDkwUxbFrbCP8Pri +FqVYwnWVLsBrr9PF0xg9SuRJnyHAwBv0xNXB3KRY2HtFcUaLn1C8alh2LAhX6eX+ ++ozHvspMjMq7P1lsONGbtfmJaf6pNh7zzok4fOas+eoqhf64TUb4g4JNxfLyXh3P +DcLsP67kr6fZSlGjzhto9ILVhg9rflNmSkmrhVjCxNqDuLFyn5ljvhemw7X3/V8A +iCwqEP2/vOjnE1wCxF2Xdb6+3f/KJKqNwOmjraXZGupOUR6OA/+Z3V3RbGc= +-----END CERTIFICATE----- diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index 6128872a1f75..4c7ae70e3ee8 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -504,6 +504,15 @@ class MochitestArguments(ArgumentContainer): "suppress": True, }, ], + [ + ["--use-http3-server"], + { + "dest": "useHttp3Server", + "default": False, + "help": "Whether to use the Http3 server", + "action": "store_true", + }, + ], [ ["--setpref"], { diff --git a/testing/mochitest/moz.build b/testing/mochitest/moz.build index edae41386eb3..4c882a87b7ff 100644 --- a/testing/mochitest/moz.build +++ b/testing/mochitest/moz.build @@ -94,6 +94,7 @@ TEST_HARNESS_FILES.testing.mochitest += [ "chrome-harness.js", "chunkifyTests.js", "document-builder.sjs", + "DoHServer/doh_server.js", "favicon.ico", "harness.xhtml", "leaks.py", @@ -110,6 +111,12 @@ TEST_HARNESS_FILES.testing.mochitest += [ "start_desktop.js", ] +TEST_HARNESS_FILES.testing.mochitest.DoHServer += [ + "DoHServer/doh_server.js", + "DoHServer/http2-cert.key.pem", + "DoHServer/http2-cert.pem", +] + TEST_HARNESS_FILES.testing.mochitest.embed += [ "embed/Xm5i5kbIXzc", "embed/Xm5i5kbIXzc^headers^", diff --git a/testing/mochitest/runjunit.py b/testing/mochitest/runjunit.py index a0faffcc8467..c0f8851e4f15 100644 --- a/testing/mochitest/runjunit.py +++ b/testing/mochitest/runjunit.py @@ -48,6 +48,8 @@ class JUnitTestRunner(MochitestDesktop): def __init__(self, log, options): self.log = log self.verbose = False + self.http3Server = None + self.dohServer = None if ( options.log_tbpl_level == "debug" or options.log_mach_level == "debug" @@ -126,10 +128,12 @@ class JUnitTestRunner(MochitestDesktop): self.options.webServer = self.options.remoteWebServer self.options.webSocketPort = "9988" self.options.httpdPath = None + self.options.http3ServerPath = None self.options.keep_open = False self.options.pidFile = "" self.options.subsuite = None self.options.xrePath = None + self.options.useHttp3Server = False if build_obj and "MOZ_HOST_BIN" in os.environ: self.options.xrePath = os.environ["MOZ_HOST_BIN"] if not self.options.utilityPath: diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 90d9e98c2885..7efe3a3179e4 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -54,6 +54,7 @@ from manifestparser.filters import ( ) from manifestparser.util import normsep from mozgeckoprofiler import symbolicate_profile_json, view_gecko_profile +from mozserve import DoHServer, Http3Server try: from marionette_driver.addons import Addons @@ -91,6 +92,13 @@ import six from six.moves.urllib.parse import quote_plus as encodeURIComponent from six.moves.urllib_request import urlopen +try: + from mozbuild.base import MozbuildObject + + build = MozbuildObject.from_environment(cwd=SCRIPT_DIR) +except ImportError: + build = None + here = os.path.abspath(os.path.dirname(__file__)) NO_TESTS_FOUND = """ @@ -1369,6 +1377,56 @@ class MochitestDesktop(object): break return is_webrtc_tag_present and options.subsuite in ["media"] + def startHttp3Server(self, options): + """ + Start a Http3 test server. + """ + http3ServerPath = os.path.join( + options.utilityPath, "http3server" + mozinfo.info["bin_suffix"] + ) + serverOptions = {} + serverOptions["http3ServerPath"] = http3ServerPath + serverOptions["profilePath"] = options.profilePath + serverOptions["isMochitest"] = True + serverOptions["isWin"] = mozinfo.isWin + serverOptions["proxyPort"] = options.http3ServerPort + env = test_environment(xrePath=options.xrePath, log=self.log) + self.http3Server = Http3Server(serverOptions, env, self.log) + self.http3Server.start() + + port = self.http3Server.ports().get("MOZHTTP3_PORT_PROXY") + if int(port) != options.http3ServerPort: + self.http3Server = None + raise RuntimeError("Error: Unable to start Http/3 server") + + def startDoHServer(self, options): + """ + Start a DoH test server. + """ + # We try to find the node executable in the path given to us by the user in + # the MOZ_NODE_PATH environment variable + nodeBin = os.getenv("MOZ_NODE_PATH", None) + self.log.info("Use MOZ_NODE_PATH at %s" % (nodeBin)) + if not nodeBin and build: + nodeBin = build.substs.get("NODEJS") + self.log.info("Use build node at %s" % (nodeBin)) + + serverOptions = {} + serverOptions["serverPath"] = os.path.join( + SCRIPT_DIR, "DoHServer", "doh_server.js" + ) + serverOptions["nodeBin"] = nodeBin + serverOptions["dstServerPort"] = options.http3ServerPort + serverOptions["isWin"] = mozinfo.isWin + serverOptions["port"] = options.dohServerPort + env = test_environment(xrePath=options.xrePath, log=self.log) + self.dohServer = DoHServer(serverOptions, env, self.log) + self.dohServer.start() + + port = self.dohServer.port() + if port != options.dohServerPort: + raise RuntimeError("Error: Unable to start DoH server") + def startServers(self, options, debuggerInfo, public=None): # start servers and set ports # TODO: pass these values, don't set on `self` @@ -1403,6 +1461,13 @@ class MochitestDesktop(object): if self.server is not None: self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) + self.log.info("use http3 server: %d" % options.useHttp3Server) + self.http3Server = None + self.dohServer = None + if options.useHttp3Server: + self.startHttp3Server(options) + self.startDoHServer(options) + def stopServers(self): """Servers are no longer needed, and perhaps more importantly, anything they might spew to console might confuse things.""" @@ -1434,6 +1499,16 @@ class MochitestDesktop(object): self.log.info("Stopping websocket/process bridge") except Exception: self.log.critical("Exception stopping websocket/process bridge") + if self.http3Server is not None: + try: + self.http3Server.stop() + except Exception: + self.log.critical("Exception stopping http3 server") + if self.dohServer is not None: + try: + self.dohServer.stop() + except Exception: + self.log.critical("Exception stopping doh server") if hasattr(self, "gstForV4l2loopbackProcess"): try: @@ -2149,6 +2224,12 @@ toolbar#nav-bar { os.unlink(pwfilePath) return 0 + def findFreePort(self, type): + with closing(socket.socket(socket.AF_INET, type)) as s: + s.bind(("127.0.0.1", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + def proxy(self, options): # proxy # use SSL port for legacy compatibility; see @@ -2156,13 +2237,21 @@ toolbar#nav-bar { # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221 # - https://github.com/mozilla/mozbase/commit/43f9510e3d58bfed32790c82a57edac5f928474d # 'ws': str(self.webSocketPort) - return { + proxyOptions = { "remote": options.webServer, "http": options.httpPort, "https": options.sslPort, "ws": options.sslPort, } + if options.useHttp3Server: + options.dohServerPort = self.findFreePort(socket.SOCK_STREAM) + options.http3ServerPort = self.findFreePort(socket.SOCK_DGRAM) + proxyOptions["dohServerPort"] = options.dohServerPort + self.log.info("use doh server at port: %d" % options.dohServerPort) + self.log.info("use http3 server at port: %d" % options.http3ServerPort) + return proxyOptions + def merge_base_profiles(self, options, category): """Merge extra profile data from testing/profiles.""" diff --git a/testing/mozbase/moz.build b/testing/mozbase/moz.build index 3cb529731101..de3f4781db36 100644 --- a/testing/mozbase/moz.build +++ b/testing/mozbase/moz.build @@ -46,6 +46,7 @@ python_modules = [ "mozproxy", "mozrunner", "mozscreenshot", + "mozserve", "mozsystemmonitor", "moztest", "mozversion", diff --git a/testing/mozbase/mozprofile/mozprofile/permissions.py b/testing/mozbase/mozprofile/mozprofile/permissions.py index fdc13105ad0c..ffb4e5acdb51 100644 --- a/testing/mozbase/mozprofile/mozprofile/permissions.py +++ b/testing/mozbase/mozprofile/mozprofile/permissions.py @@ -234,7 +234,18 @@ class Permissions(object): prefs = [] if proxy: - user_prefs = self.pac_prefs(proxy) + dohServerPort = proxy.get("dohServerPort") + if dohServerPort is not None: + # make sure we don't use proxy + user_prefs = [("network.proxy.type", 0)] + # Use TRR_ONLY mode + user_prefs.append(("network.trr.mode", 3)) + trrUri = "https://foo.example.com:{}/dns-query".format(dohServerPort) + user_prefs.append(("network.trr.uri", trrUri)) + user_prefs.append(("network.trr.bootstrapAddr", "127.0.0.1")) + user_prefs.append(("network.dns.force_use_https_rr", True)) + else: + user_prefs = self.pac_prefs(proxy) else: user_prefs = [] diff --git a/testing/mozbase/mozprofile/tests/test_permissions.py b/testing/mozbase/mozprofile/tests/test_permissions.py index 65d00efdc228..3fe688366d7c 100755 --- a/testing/mozbase/mozprofile/tests/test_permissions.py +++ b/testing/mozbase/mozprofile/tests/test_permissions.py @@ -31,7 +31,7 @@ def test_nw_prefs(perms): assert len(user_prefs) == 0 assert len(prefs) == 0 - prefs, user_prefs = perms.network_prefs(True) + prefs, user_prefs = perms.network_prefs({"http": 8888}) assert len(user_prefs) == 2 assert user_prefs[0] == ("network.proxy.type", 2) assert user_prefs[1][0] == "network.proxy.autoconfig_url" @@ -50,6 +50,15 @@ def test_nw_prefs(perms): ) assert all(c in user_prefs[1][1] for c in proxy_check) + prefs, user_prefs = perms.network_prefs({"dohServerPort": 443}) + print(user_prefs) + assert len(user_prefs) == 5 + assert user_prefs[0] == ("network.proxy.type", 0) + assert user_prefs[1] == ("network.trr.mode", 3) + assert user_prefs[2] == ("network.trr.uri", "https://foo.example.com:443/dns-query") + assert user_prefs[3] == ("network.trr.bootstrapAddr", "127.0.0.1") + assert user_prefs[4] == ("network.dns.force_use_https_rr", True) + if __name__ == "__main__": mozunit.main() diff --git a/testing/mozbase/mozserve/mozserve/__init__.py b/testing/mozbase/mozserve/mozserve/__init__.py new file mode 100644 index 000000000000..2354857354d4 --- /dev/null +++ b/testing/mozbase/mozserve/mozserve/__init__.py @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +mozserve is a simple script that is used to launch test servers, and +is designed for use in mochitest and xpcshelltest. +""" + +from .servers import DoHServer, Http3Server + +__all__ = ["Http3Server", "DoHServer"] diff --git a/testing/mozbase/mozserve/mozserve/servers.py b/testing/mozbase/mozserve/mozserve/servers.py new file mode 100644 index 000000000000..da7075601f50 --- /dev/null +++ b/testing/mozbase/mozserve/mozserve/servers.py @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import copy +import os +import re +import subprocess +import sys +import time +from argparse import Namespace +from contextlib import contextmanager +from subprocess import PIPE, Popen + + +@contextmanager +def popenCleanupHack(isWin): + """ + Hack to work around https://bugs.python.org/issue37380 + The basic idea is that on old versions of Python on Windows, + we need to clear subprocess._cleanup before we call Popen(), + then restore it afterwards. + """ + savedCleanup = None + if isWin and sys.version_info[0] == 3 and sys.version_info < (3, 7, 5): + savedCleanup = subprocess._cleanup + subprocess._cleanup = lambda: None + try: + yield + finally: + if savedCleanup: + subprocess._cleanup = savedCleanup + + +class Http3Server(object): + """ + Class which encapsulates the Http3 server + """ + + def __init__(self, options, env, logger): + if isinstance(options, Namespace): + options = vars(options) + self._log = logger + self._profileDir = options["profilePath"] + self._env = copy.deepcopy(env) + self._ports = {} + self._echConfig = "" + self._isMochitest = options["isMochitest"] + self._http3ServerPath = options["http3ServerPath"] + self._isWin = options["isWin"] + self._http3ServerProc = {} + self._proxyPort = -1 + if options.get("proxyPort"): + self._proxyPort = options["proxyPort"] + + def ports(self): + return self._ports + + def echConfig(self): + return self._echConfig + + def start(self): + if not os.path.exists(self._http3ServerPath): + raise Exception("Http3 server not found at %s" % self._http3ServerPath) + + self._log.info("mozserve | Found Http3Server path: %s" % self._http3ServerPath) + + dbPath = os.path.join(self._profileDir, "cert9.db") + if not os.path.exists(dbPath): + raise Exception("cert db not found at %s" % dbPath) + + dbPath = self._profileDir + self._log.info("mozserve | cert db path: %s" % dbPath) + + try: + if self._isMochitest: + self._env["MOZ_HTTP3_MOCHITEST"] = "1" + if self._proxyPort != -1: + self._env["MOZ_HTTP3_PROXY_PORT"] = str(self._proxyPort) + with popenCleanupHack(self._isWin): + process = Popen( + [self._http3ServerPath, dbPath], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + env=self._env, + cwd=os.getcwd(), + universal_newlines=True, + ) + self._http3ServerProc["http3Server"] = process + + # Check to make sure the server starts properly by waiting for it to + # tell us it's started + msg = process.stdout.readline() + self._log.info("mozserve | http3 server msg: %s" % msg) + if "server listening" in msg: + searchObj = re.search( + r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)." + " EchConfig is @([\x00-\x7F]+)@", + msg, + 0, + ) + if searchObj: + self._ports["MOZHTTP3_PORT"] = searchObj.group(1) + self._ports["MOZHTTP3_PORT_FAILED"] = searchObj.group(2) + self._ports["MOZHTTP3_PORT_ECH"] = searchObj.group(3) + self._ports["MOZHTTP3_PORT_PROXY"] = searchObj.group(4) + self._ports["MOZHTTP3_PORT_NO_RESPONSE"] = searchObj.group(5) + self._echConfig = searchObj.group(6) + except OSError as e: + # This occurs if the subprocess couldn't be started + self._log.error("Could not run the http3 server: %s" % (str(e))) + + def stop(self): + """ + Shutdown our http3Server process, if it exists + """ + for name, proc in self._http3ServerProc.items(): + self._log.info("%s server shutting down ..." % name) + if proc.poll() is not None: + self._log.info("Http3 server %s already dead %s" % (name, proc.poll())) + else: + proc.terminate() + retries = 0 + while proc.poll() is None: + time.sleep(0.1) + retries += 1 + if retries > 40: + self._log.info("Killing proc") + proc.kill() + break + + def dumpOutput(fd, label): + firstTime = True + for msg in fd: + if firstTime: + firstTime = False + self._log.info("Process %s" % label) + self._log.info(msg) + + dumpOutput(proc.stdout, "stdout") + dumpOutput(proc.stderr, "stderr") + self._http3ServerProc = {} + + +class DoHServer(object): + """ + Class which encapsulates the DoH server + """ + + def __init__(self, options, env, logger): + if isinstance(options, Namespace): + options = vars(options) + self._log = logger + self._port = options["port"] + self._env = copy.deepcopy(env) + self._nodeBin = options["nodeBin"] + self._serverPath = options["serverPath"] + self._dstServerPort = options["dstServerPort"] + self._isWin = options["isWin"] + self._nodeProc = None + + def port(self): + return self._port + + def start(self): + if not os.path.exists(self._serverPath): + raise Exception("DoH server not found at %s" % self._serverPath) + + self._log.info("mozserve | Found DoH server path: %s" % self._serverPath) + + if not os.path.exists(self._nodeBin) or not os.path.isfile(self._nodeBin): + raise Exception("node not found at path %s" % (self._nodeBin)) + + self._log.info("Found node at %s" % (self._nodeBin)) + + try: + # We pipe stdin to node because the server will exit when its + # stdin reaches EOF + with popenCleanupHack(self._isWin): + process = Popen( + [ + self._nodeBin, + self._serverPath, + "serverPort={}".format(self._dstServerPort), + "listeningPort={}".format(self._port), + ], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + env=self._env, + cwd=os.getcwd(), + universal_newlines=True, + ) + self._nodeProc = process + + msg = process.stdout.readline() + self._log.info("runtests.py | DoH server msg: %s" % msg) + if "server listening" in msg: + searchObj = re.search(r"DoH server listening on ports ([0-9]+)", msg, 0) + if searchObj: + self._port = int(searchObj.group(1)) + self._log.info("DoH server started at port: %d" % (self._port)) + except OSError as e: + # This occurs if the subprocess couldn't be started + self._log.error("Could not run DoH server: %s" % (str(e))) + + def stop(self): + """ + Shut down our node process, if it exists + """ + if self._nodeProc is not None: + if self._nodeProc.poll() is not None: + self._log.info("Node server already dead %s" % (self._nodeProc.poll())) + else: + self._nodeProc.terminate() + + def dumpOutput(fd, label): + firstTime = True + for msg in fd: + if firstTime: + firstTime = False + self._log.info("Process %s" % label) + self._log.info(msg) + + dumpOutput(self._nodeProc.stdout, "stdout") + dumpOutput(self._nodeProc.stderr, "stderr") + + self._nodeProc = None diff --git a/testing/mozbase/mozserve/setup.py b/testing/mozbase/mozserve/setup.py new file mode 100644 index 000000000000..c00000bb0c63 --- /dev/null +++ b/testing/mozbase/mozserve/setup.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_VERSION = "0.1" + +setup( + name="mozserve", + version=PACKAGE_VERSION, + description="Python test server launcher intended for use with Mozilla testing", + long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html", + license="MPL", + packages=["mozserve"], + zip_safe=False, +) diff --git a/testing/tools/mach_test_package_initialize.py b/testing/tools/mach_test_package_initialize.py index 62af41a08cb9..e29d407af2b3 100644 --- a/testing/tools/mach_test_package_initialize.py +++ b/testing/tools/mach_test_package_initialize.py @@ -29,6 +29,7 @@ SEARCH_PATHS = [ "mozbase/mozprofile", "mozbase/mozrunner", "mozbase/mozscreenshot", + "mozbase/mozserve", "mozbase/mozsystemmonitor", "mozbase/moztest", "mozbase/mozversion", diff --git a/testing/xpcshell/runxpcshelltests.py b/testing/xpcshell/runxpcshelltests.py index c53652cf6777..5964ace8410a 100755 --- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -29,6 +29,7 @@ from threading import Event, Thread, Timer, current_thread import mozdebug import six +from mozserve import Http3Server try: import psutil @@ -976,7 +977,7 @@ class XPCShellTests(object): self.log = log self.harness_timeout = HARNESS_TIMEOUT self.nodeProc = {} - self.http3ServerProc = {} + self.http3Server = None self.conditioned_profile_dir = None def getTestManifest(self, manifest): @@ -1415,8 +1416,7 @@ class XPCShellTests(object): binSuffix = "" if sys.platform == "win32": binSuffix = ".exe" - - http3ServerPath = self.http3server + http3ServerPath = self.http3ServerPath if not http3ServerPath: http3ServerPath = os.path.join( SCRIPT_DIR, "http3server", "http3server" + binSuffix @@ -1425,89 +1425,25 @@ class XPCShellTests(object): http3ServerPath = os.path.join( build.topobjdir, "dist", "bin", "http3server" + binSuffix ) - - if not os.path.exists(http3ServerPath): - self.log.warning( - "Http3 server not found at " - + http3ServerPath - + ". Tests requiring http/3 will fail." - ) - return - - # OK, we found our server, let's try to get it running - self.log.info("Found %s" % (http3ServerPath)) - try: - dbPath = os.path.join(SCRIPT_DIR, "http3server", "http3serverDB") - if build: - dbPath = os.path.join( - build.topsrcdir, "netwerk", "test", "http3serverDB" - ) - self.log.info("Using %s" % (dbPath)) - # We pipe stdin to the server because it will exit when its stdin - # reaches EOF - with popenCleanupHack(): - process = Popen( - [http3ServerPath, dbPath], - stdin=PIPE, - stdout=PIPE, - stderr=PIPE, - env=self.env, - cwd=os.getcwd(), - universal_newlines=True, - ) - self.http3ServerProc["http3Server"] = process - - # Check to make sure the server starts properly by waiting for it to - # tell us it's started - msg = process.stdout.readline() - if "server listening" in msg: - searchObj = re.search( - r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)." - " EchConfig is @([\x00-\x7F]+)@", - msg, - 0, - ) - if searchObj: - self.env["MOZHTTP3_PORT"] = searchObj.group(1) - self.env["MOZHTTP3_PORT_FAILED"] = searchObj.group(2) - self.env["MOZHTTP3_PORT_ECH"] = searchObj.group(3) - self.env["MOZHTTP3_PORT_PROXY"] = searchObj.group(4) - self.env["MOZHTTP3_PORT_NO_RESPONSE"] = searchObj.group(5) - self.env["MOZHTTP3_ECH"] = searchObj.group(6) - except OSError as e: - # This occurs if the subprocess couldn't be started - self.log.error("Could not run the http3 server: %s" % (str(e))) + dbPath = os.path.join(SCRIPT_DIR, "http3server", "http3serverDB") + if build: + dbPath = os.path.join(build.topsrcdir, "netwerk", "test", "http3serverDB") + options = {} + options["http3ServerPath"] = http3ServerPath + options["profilePath"] = dbPath + options["isMochitest"] = False + options["isWin"] = sys.platform == "win32" + self.http3Server = Http3Server(options, self.env, self.log) + self.http3Server.start() + for key, value in self.http3Server.ports().items(): + self.env[key] = value + self.env["MOZHTTP3_ECH"] = self.http3Server.echConfig() def shutdownHttp3Server(self): - """ - Shutdown our http3Server process, if it exists - """ - for name, proc in six.iteritems(self.http3ServerProc): - self.log.info("%s server shutting down ..." % name) - if proc.poll() is not None: - self.log.info("Http3 server %s already dead %s" % (name, proc.poll())) - else: - proc.terminate() - retries = 0 - while proc.poll() is None: - time.sleep(0.1) - retries += 1 - if retries > 40: - self.log.info("Killing proc") - proc.kill() - break - - def dumpOutput(fd, label): - firstTime = True - for msg in fd: - if firstTime: - firstTime = False - self.log.info("Process %s" % label) - self.log.info(msg) - - dumpOutput(proc.stdout, "stdout") - dumpOutput(proc.stderr, "stderr") - self.http3ServerProc = {} + if self.http3Server is None: + return + self.http3Server.stop() + self.http3Server = None def buildXpcsRunArgs(self): """ @@ -1747,7 +1683,7 @@ class XPCShellTests(object): self.app_binary = options.get("app_binary") self.xpcshell = options.get("xpcshell") - self.http3server = options.get("http3server") + self.http3ServerPath = options.get("http3server") self.xrePath = options.get("xrePath") self.utility_path = options.get("utility_path") self.appPath = options.get("appPath")