Bug 1871963: Implement zstd content-encoding support r=necko-reviewers,valentin,devtools-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D205109

devtools/client/netmonitor/test/browser_net_copy_as_powershell.js

@@ -19,7 +19,7 @@ add_task(async function () {
 -Headers @{
 "Accept" = "*/*"
   "Accept-Language" = "en-US"
-  "Accept-Encoding" = "gzip, deflate, br"
+  "Accept-Encoding" = "gzip, deflate, br, zstd"
   "X-Custom-Header-1" = "Custom value"
   "X-Custom-Header-2" = "8.8.8.8"
   "X-Custom-Header-3" = "Mon, 3 Mar 2014 11:11:11 GMT"
@@ -42,7 +42,7 @@ Invoke-WebRequest -UseBasicParsing -Uri "https://example.com/browser/devtools/cl
 -Headers @{
 "Accept" = "*/*"
   "Accept-Language" = "en-US"
-  "Accept-Encoding" = "gzip, deflate, br"
+  "Accept-Encoding" = "gzip, deflate, br, zstd"
   "X-Custom-Header-1" = "Custom value"
   "X-Custom-Header-2" = "8.8.8.8"
   "X-Custom-Header-3" = "Mon, 3 Mar 2014 11:11:11 GMT"
@@ -66,7 +66,7 @@ Invoke-WebRequest -UseBasicParsing -Uri "https://example.com/browser/devtools/cl
 -Headers @{
 "Accept" = "*/*"
   "Accept-Language" = "en-US"
-  "Accept-Encoding" = "gzip, deflate, br"
+  "Accept-Encoding" = "gzip, deflate, br, zstd"
   "X-Custom-Header-1" = "Custom value"
   "X-Custom-Header-2" = "8.8.8.8"
   "X-Custom-Header-3" = "Mon, 3 Mar 2014 11:11:11 GMT"
@@ -95,7 +95,7 @@ Invoke-WebRequest -UseBasicParsing -Uri "https://example.com/browser/devtools/cl
 -Headers @{
 "Accept" = "*/*"
   "Accept-Language" = "en-US"
-  "Accept-Encoding" = "gzip, deflate, br"
+  "Accept-Encoding" = "gzip, deflate, br, zstd"
   "X-Custom-Header-1" = "Custom value"
   "X-Custom-Header-2" = "8.8.8.8"
   "X-Custom-Header-3" = "Mon, 3 Mar 2014 11:11:11 GMT"

modules/libpref/init/all.js

@@ -1203,7 +1203,7 @@ pref("network.http.redirection-limit", 20);
 // NOTE: support for "compress" has been disabled per bug 196406.
 // NOTE: separate values with comma+space (", "): see bug 576033
 pref("network.http.accept-encoding", "gzip, deflate");
-pref("network.http.accept-encoding.secure", "gzip, deflate, br");
+pref("network.http.accept-encoding.secure", "gzip, deflate, br, zstd");
 
 // Prompt for redirects resulting in unsafe HTTP requests
 pref("network.http.prompt-temp-redirect", false);

netwerk/mime/nsMimeTypes.h

@@ -31,6 +31,7 @@
 #define APPLICATION_GZIP2 "application/gzip"
 #define APPLICATION_GZIP3 "application/x-gunzip"
 #define APPLICATION_BROTLI "application/brotli"
+#define APPLICATION_ZSTD "application/zstd"
 #define APPLICATION_ZIP "application/zip"
 #define APPLICATION_HTTP_INDEX_FORMAT "application/http-index-format"
 #define APPLICATION_ECMASCRIPT "application/ecmascript"
@@ -245,6 +246,7 @@
 #define ENCODING_UUENCODE3 "uuencode"
 #define ENCODING_UUENCODE4 "uue"
 #define ENCODING_YENCODE "x-yencode"
+#define ENCODING_ZSTD "zstd"
 
 /* Some names of parameters that various MIME headers include.
  */

netwerk/protocol/http/HttpBaseChannel.cpp

@@ -1539,6 +1539,8 @@ HttpBaseChannel::DoApplyContentConversions(nsIStreamListener* aNextListener,
           mode = 2;
         } else if (from.EqualsLiteral("br")) {
           mode = 3;
+        } else if (from.EqualsLiteral("zstd")) {
+          mode = 4;
         }
         Telemetry::Accumulate(Telemetry::HTTP_CONTENT_ENCODING, mode);
       }
@@ -1643,6 +1645,14 @@ HttpBaseChannel::nsContentEncodings::GetNext(nsACString& aNextEncoding) {
     }
   }
 
+  if (!haveType) {
+    encoding.BeginReading(start);
+    if (CaseInsensitiveFindInReadable("zstd"_ns, start, end)) {
+      aNextEncoding.AssignLiteral(APPLICATION_ZSTD);
+      haveType = true;
+    }
+  }
+
   // Prepare to fetch the next encoding
   mCurEnd = mCurStart;
   mReady = false;

netwerk/protocol/http/nsHttpChannel.cpp

@@ -7690,6 +7690,7 @@ static nsLiteralCString ContentTypeToTelemetryLabel(nsHttpChannel* aChannel) {
       return "proxy"_ns;
     }
     if (contentType.EqualsLiteral(APPLICATION_BROTLI) ||
+        contentType.EqualsLiteral(APPLICATION_ZSTD) ||
         contentType.Find("zip") != kNotFound ||
         contentType.Find("compress") != kNotFound) {
       return "compressed"_ns;

netwerk/streamconv/converters/moz.build

@@ -29,3 +29,7 @@ LOCAL_INCLUDES += [
     "/modules/brotli/dec",
     "/netwerk/base",
 ]
+
+DIRS += [
+    "/third_party/zstd",
+]

netwerk/streamconv/converters/nsHTTPCompressConv.cpp

@@ -29,6 +29,8 @@
 #include "state.h"
 #include "brotli/decode.h"
 
+#include "zstd/zstd.h"
+
 namespace mozilla {
 namespace net {
 
@@ -53,6 +55,26 @@ class BrotliWrapper {
   uint64_t mSourceOffset{0};
 };
 
+class ZstdWrapper {
+ public:
+  ZstdWrapper() {
+    mDStream = ZSTD_createDStream();
+    ZSTD_DCtx_setParameter(mDStream, ZSTD_d_windowLogMax, 23 /*8*1024*1024*/);
+  }
+  ~ZstdWrapper() {
+    if (mDStream) {
+      ZSTD_freeDStream(mDStream);
+    }
+  }
+
+  UniquePtr<uint8_t[]> mOutBuffer;
+  nsresult mStatus = NS_OK;
+  nsIRequest* mRequest{nullptr};
+  nsISupports* mContext{nullptr};
+  uint64_t mSourceOffset{0};
+  ZSTD_DStream* mDStream{nullptr};
+};
+
 // nsISupports implementation
 NS_IMPL_ISUPPORTS(nsHTTPCompressConv, nsIStreamConverter, nsIStreamListener,
                   nsIRequestObserver, nsICompressConvStats,
@@ -112,6 +134,12 @@ nsHTTPCompressConv::AsyncConvertData(const char* aFromType, const char* aToType,
   } else if (!nsCRT::strncasecmp(aFromType, HTTP_BROTLI_TYPE,
                                  sizeof(HTTP_BROTLI_TYPE) - 1)) {
     mMode = HTTP_COMPRESS_BROTLI;
+  } else if (!nsCRT::strncasecmp(aFromType, HTTP_ZSTD_TYPE,
+                                 sizeof(HTTP_ZSTD_TYPE) - 1)) {
+    mMode = HTTP_COMPRESS_ZSTD;
+  } else if (!nsCRT::strncasecmp(aFromType, HTTP_ZST_TYPE,
+                                 sizeof(HTTP_ZST_TYPE) - 1)) {
+    mMode = HTTP_COMPRESS_ZSTD;
   }
   LOG(("nsHttpCompresssConv %p AsyncConvertData %s %s mode %d\n", this,
        aFromType, aToType, (CompressMode)mMode));
@@ -363,6 +391,71 @@ nsresult nsHTTPCompressConv::BrotliHandler(nsIInputStream* stream,
   return self->mBrotli->mStatus;
 }
 
+/* static */
+nsresult nsHTTPCompressConv::ZstdHandler(nsIInputStream* stream, void* closure,
+                                         const char* dataIn, uint32_t,
+                                         uint32_t aAvail, uint32_t* countRead) {
+  MOZ_ASSERT(stream);
+  nsHTTPCompressConv* self = static_cast<nsHTTPCompressConv*>(closure);
+  *countRead = 0;
+
+  const size_t kOutSize = ZSTD_DStreamOutSize();  // normally 128K
+  uint8_t* outPtr;
+  size_t avail = aAvail;
+
+  // Stop decompressing after an error
+  if (self->mZstd->mStatus != NS_OK) {
+    *countRead = aAvail;
+    return NS_OK;
+  }
+
+  if (!self->mZstd->mOutBuffer) {
+    self->mZstd->mOutBuffer = MakeUniqueFallible<uint8_t[]>(kOutSize);
+    if (!self->mZstd->mOutBuffer) {
+      self->mZstd->mStatus = NS_ERROR_OUT_OF_MEMORY;
+      return self->mZstd->mStatus;
+    }
+  }
+  ZSTD_inBuffer inBuffer = {.src = dataIn, .size = aAvail, .pos = 0};
+  uint32_t last_pos = 0;
+  while (inBuffer.pos < inBuffer.size) {
+    outPtr = self->mZstd->mOutBuffer.get();
+
+    LOG(("nsHttpCompresssConv %p zstdhandler decompress %zu\n", self, avail));
+    // Use ZSTD_(de)compressStream to (de)compress the input buffer into the
+    // output buffer, and fill aReadCount with the number of bytes consumed.
+    ZSTD_outBuffer outBuffer{.dst = outPtr, .size = kOutSize};
+    size_t result;
+    bool output_full;
+    do {
+      outBuffer.pos = 0;
+      result =
+          ZSTD_decompressStream(self->mZstd->mDStream, &outBuffer, &inBuffer);
+
+      // If we errored when writing, flag this and abort writing.
+      if (ZSTD_isError(result)) {
+        self->mZstd->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING;
+        return self->mZstd->mStatus;
+      }
+
+      nsresult rv = self->do_OnDataAvailable(
+          self->mZstd->mRequest, self->mZstd->mSourceOffset,
+          reinterpret_cast<const char*>(outPtr), outBuffer.pos);
+      if (NS_FAILED(rv)) {
+        self->mZstd->mStatus = rv;
+        return rv;
+      }
+      self->mZstd->mSourceOffset += inBuffer.pos - last_pos;
+      last_pos = inBuffer.pos;
+      output_full = outBuffer.pos == outBuffer.size;
+      // in the unlikely case that the output buffer was full, loop to
+      // drain it before processing more input
+    } while (output_full);
+  }
+  *countRead = inBuffer.pos;
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 nsHTTPCompressConv::OnDataAvailable(nsIRequest* request, nsIInputStream* iStr,
                                     uint64_t aSourceOffset, uint32_t aCount) {
@@ -596,6 +689,25 @@ nsHTTPCompressConv::OnDataAvailable(nsIRequest* request, nsIInputStream* iStr,
       }
     } break;
 
+    case HTTP_COMPRESS_ZSTD: {
+      if (!mZstd) {
+        mZstd = MakeUnique<ZstdWrapper>();
+      }
+
+      mZstd->mRequest = request;
+      mZstd->mContext = nullptr;
+      mZstd->mSourceOffset = aSourceOffset;
+
+      uint32_t countRead;
+      rv = iStr->ReadSegments(ZstdHandler, this, streamLen, &countRead);
+      if (NS_SUCCEEDED(rv)) {
+        rv = mZstd->mStatus;
+      }
+      if (NS_FAILED(rv)) {
+        return rv;
+      }
+    } break;
+
     default:
       nsCOMPtr<nsIStreamListener> listener;
       {

netwerk/streamconv/converters/nsHTTPCompressConv.h

@@ -34,11 +34,14 @@ class nsIStringInputStream;
 #  define HTTP_BROTLI_TYPE "br"
 #  define HTTP_IDENTITY_TYPE "identity"
 #  define HTTP_UNCOMPRESSED_TYPE "uncompressed"
+#  define HTTP_ZSTD_TYPE "zstd"
+#  define HTTP_ZST_TYPE "zst"
 
 namespace mozilla {
 namespace net {
 
 class BrotliWrapper;
+class ZstdWrapper;
 
 class nsHTTPCompressConv : public nsIStreamConverter,
                            public nsICompressConvStats {
@@ -60,7 +63,8 @@ class nsHTTPCompressConv : public nsIStreamConverter,
     HTTP_COMPRESS_DEFLATE,
     HTTP_COMPRESS_COMPRESS,
     HTTP_COMPRESS_BROTLI,
-    HTTP_COMPRESS_IDENTITY
+    HTTP_COMPRESS_IDENTITY,
+    HTTP_COMPRESS_ZSTD,
   };
 
  private:
@@ -77,6 +81,7 @@ class nsHTTPCompressConv : public nsIStreamConverter,
   uint32_t mInpBufferLen{0};
 
   UniquePtr<BrotliWrapper> mBrotli;
+  UniquePtr<ZstdWrapper> mZstd;
 
   nsCOMPtr<nsIStringInputStream> mStream;
 
@@ -84,6 +89,10 @@ class nsHTTPCompressConv : public nsIStreamConverter,
                                 const char* dataIn, uint32_t, uint32_t avail,
                                 uint32_t* countRead);
 
+  static nsresult ZstdHandler(nsIInputStream* stream, void* closure,
+                              const char* dataIn, uint32_t, uint32_t avail,
+                              uint32_t* countRead);
+
   nsresult do_OnDataAvailable(nsIRequest* request, uint64_t aSourceOffset,
                               const char* buffer, uint32_t aCount);
 

netwerk/test/unit/test_brotli_http.js

@@ -100,7 +100,7 @@ add_task(
     });
     equal(
       Services.prefs.getCharPref("network.http.accept-encoding.secure"),
-      "gzip, deflate, br"
+      "gzip, deflate, br, zstd"
     );
     let { req, buff } = await new Promise(resolve => {
       let chan = NetUtil.newChannel({

toolkit/components/telemetry/Histograms.json

@@ -4815,7 +4815,7 @@
     "expires_in_version": "never",
     "kind": "enumerated",
     "n_values": 6,
-    "description": "encoding removed: 0=unknown, 1=gzip, 2=deflate, 3=brotli"
+    "description": "encoding removed: 0=unknown, 1=gzip, 2=deflate, 3=brotli, 4=zstd"
   },
   "CACHE_LM_INCONSISTENT": {
     "record_in_processes": ["main", "content"],