Bug 1847659 - Use the new MimeType parser to parse content-types for response headers; r=kershaw,necko-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D195692

dom/base/MimeType.cpp

@@ -243,6 +243,88 @@ TMimeType<char_type>::Parse(const nsTSubstring<char_type>& aMimeType) {
   return mimeType;
 }
 
+template <typename char_type>
+/* static */ nsTArray<nsTDependentSubstring<char_type>>
+TMimeType<char_type>::SplitMimetype(const nsTSubstring<char_type>& aMimeType) {
+  nsTArray<nsTDependentSubstring<char_type>> mimeTypeParts;
+  bool inQuotes = false;
+  size_t start = 0;
+
+  for (size_t i = 0; i < aMimeType.Length(); i++) {
+    char_type c = aMimeType[i];
+
+    if (c == '\"' && (i == 0 || aMimeType[i - 1] != '\\')) {
+      inQuotes = !inQuotes;
+    } else if (c == ',' && !inQuotes) {
+      mimeTypeParts.AppendElement(Substring(aMimeType, start, i - start));
+      start = i + 1;
+    }
+  }
+  if (start < aMimeType.Length()) {
+    mimeTypeParts.AppendElement(Substring(aMimeType, start));
+  }
+  return mimeTypeParts;
+}
+
+template <typename char_type>
+/* static */ bool TMimeType<char_type>::Parse(
+    const nsTSubstring<char_type>& aMimeType,
+    nsTSubstring<char_type>& aOutEssence,
+    nsTSubstring<char_type>& aOutCharset) {
+  static char_type kCHARSET[] = {'c', 'h', 'a', 'r', 's', 'e', 't'};
+  static nsTDependentSubstring<char_type> kCharset(kCHARSET, 7);
+
+  mozilla::UniquePtr<TMimeType<char_type>> parsed;
+  nsTAutoString<char_type> prevContentType;
+  nsTAutoString<char_type> prevCharset;
+
+  prevContentType.Assign(aOutEssence);
+  prevCharset.Assign(aOutCharset);
+
+  nsTArray<nsTDependentSubstring<char_type>> mimeTypeParts =
+      SplitMimetype(aMimeType);
+
+  for (auto& mimeTypeString : mimeTypeParts) {
+    if (mimeTypeString.EqualsLiteral("error")) {
+      continue;
+    }
+
+    parsed = Parse(mimeTypeString);
+
+    if (!parsed) {
+      aOutEssence.Truncate();
+      aOutCharset.Truncate();
+      return false;
+    }
+
+    parsed->GetEssence(aOutEssence);
+
+    if (aOutEssence.EqualsLiteral("*/*")) {
+      aOutEssence.Assign(prevContentType);
+      continue;
+    }
+
+    bool eq = !prevContentType.IsEmpty() && aOutEssence.Equals(prevContentType);
+
+    if (!eq) {
+      prevContentType.Assign(aOutEssence);
+    }
+
+    bool typeHasCharset = false;
+    if (parsed->GetParameterValue(kCharset, aOutCharset, false, false)) {
+      typeHasCharset = true;
+    } else if (eq) {
+      aOutCharset.Assign(prevCharset);
+    }
+
+    if ((!eq && !prevCharset.IsEmpty()) || typeHasCharset) {
+      prevCharset.Assign(aOutCharset);
+    }
+  }
+
+  return true;
+}
+
 template <typename char_type>
 void TMimeType<char_type>::Serialize(nsTSubstring<char_type>& aOutput) const {
   aOutput.Assign(mType);
@@ -258,7 +340,7 @@ void TMimeType<char_type>::Serialize(nsTSubstring<char_type>& aOutput) const {
 }
 
 template <typename char_type>
-void TMimeType<char_type>::GetFullType(nsTSubstring<char_type>& aOutput) const {
+void TMimeType<char_type>::GetEssence(nsTSubstring<char_type>& aOutput) const {
   aOutput.Assign(mType);
   aOutput.AppendLiteral("/");
   aOutput.Append(mSubtype);
@@ -273,7 +355,7 @@ bool TMimeType<char_type>::HasParameter(
 template <typename char_type>
 bool TMimeType<char_type>::GetParameterValue(
     const nsTSubstring<char_type>& aName, nsTSubstring<char_type>& aOutput,
-    bool aAppend) const {
+    bool aAppend, bool aWithQuotes) const {
   if (!aAppend) {
     aOutput.Truncate();
   }
@@ -283,7 +365,7 @@ bool TMimeType<char_type>::GetParameterValue(
     return false;
   }
 
-  if (value.mRequiresQuoting || value.IsEmpty()) {
+  if (aWithQuotes && (value.mRequiresQuoting || value.IsEmpty())) {
     aOutput.AppendLiteral("\"");
     const char_type* vcur = value.BeginReading();
     const char_type* vend = value.EndReading();
@@ -320,22 +402,30 @@ template mozilla::UniquePtr<TMimeType<char16_t>> TMimeType<char16_t>::Parse(
     const nsTSubstring<char16_t>& aMimeType);
 template mozilla::UniquePtr<TMimeType<char>> TMimeType<char>::Parse(
     const nsTSubstring<char>& aMimeType);
+template bool TMimeType<char16_t>::Parse(
+    const nsTSubstring<char16_t>& aMimeType,
+    nsTSubstring<char16_t>& aOutEssence, nsTSubstring<char16_t>& aOutCharset);
+template bool TMimeType<char>::Parse(const nsTSubstring<char>& aMimeType,
+                                     nsTSubstring<char>& aOutEssence,
+                                     nsTSubstring<char>& aOutCharset);
+template nsTArray<nsTDependentSubstring<char>> TMimeType<char>::SplitMimetype(
+    const nsTSubstring<char>& aMimeType);
 template void TMimeType<char16_t>::Serialize(
     nsTSubstring<char16_t>& aOutput) const;
 template void TMimeType<char>::Serialize(nsTSubstring<char>& aOutput) const;
-template void TMimeType<char16_t>::GetFullType(
+template void TMimeType<char16_t>::GetEssence(
     nsTSubstring<char16_t>& aOutput) const;
-template void TMimeType<char>::GetFullType(nsTSubstring<char>& aOutput) const;
+template void TMimeType<char>::GetEssence(nsTSubstring<char>& aOutput) const;
 template bool TMimeType<char16_t>::HasParameter(
     const nsTSubstring<char16_t>& aName) const;
 template bool TMimeType<char>::HasParameter(
     const nsTSubstring<char>& aName) const;
 template bool TMimeType<char16_t>::GetParameterValue(
     const nsTSubstring<char16_t>& aName, nsTSubstring<char16_t>& aOutput,
-    bool aAppend) const;
+    bool aAppend, bool aWithQuotes) const;
 template bool TMimeType<char>::GetParameterValue(
-    const nsTSubstring<char>& aName, nsTSubstring<char>& aOutput,
-    bool aAppend) const;
+    const nsTSubstring<char>& aName, nsTSubstring<char>& aOutput, bool aAppend,
+    bool aWithQuotes) const;
 template void TMimeType<char16_t>::SetParameterValue(
     const nsTSubstring<char16_t>& aName, const nsTSubstring<char16_t>& aValue);
 template void TMimeType<char>::SetParameterValue(

dom/base/MimeType.h

@@ -33,6 +33,9 @@ class TMimeType final {
     ParameterValue() : mRequiresQuoting(false) {}
   };
 
+  static nsTArray<nsTDependentSubstring<char_type>> SplitMimetype(
+      const nsTSubstring<char_type>& aMimeType);
+
   bool mIsBase64{false};
   nsTString<char_type> mType;
   nsTString<char_type> mSubtype;
@@ -48,10 +51,18 @@ class TMimeType final {
   static mozilla::UniquePtr<TMimeType<char_type>> Parse(
       const nsTSubstring<char_type>& aStr);
 
+  // @param aMimeType - the mimetype string
+  // @param aOutEssence - will hold the value of the content-type
+  // @param aOutCharset - will hold the value of the charset
+  // @return true if the mimetype was parsed, false otherwise.
+  static bool Parse(const nsTSubstring<char_type>& aMimeType,
+                    nsTSubstring<char_type>& aOutEssence,
+                    nsTSubstring<char_type>& aOutCharset);
+
   void Serialize(nsTSubstring<char_type>& aStr) const;
 
   // Returns the `<mType>/<mSubtype>`
-  void GetFullType(nsTSubstring<char_type>& aStr) const;
+  void GetEssence(nsTSubstring<char_type>& aOutput) const;
 
   bool IsBase64() const { return mIsBase64; }
 
@@ -63,10 +74,11 @@ class TMimeType final {
   // @param aOutput - will hold the value of the parameter (quoted if necessary)
   // @param aAppend - if true, the method will append to the string;
   //                  otherwise the string is truncated before appending.
+  // @param aWithQuotes - if true, output can contain quoted string
   // @return true if the parameter name is found, false otherwise.
   bool GetParameterValue(const nsTSubstring<char_type>& aName,
-                         nsTSubstring<char_type>& aOutput,
-                         bool aAppend = false) const;
+                         nsTSubstring<char_type>& aOutput, bool aAppend = false,
+                         bool aWithQuotes = true) const;
 
   // @param aName - the name of the parameter
   // @param aValue - the value of the parameter

dom/base/test/gtest/TestMimeType.cpp

@@ -814,3 +814,265 @@ TEST(MimeType, LegacyCommentSyntax2)
   ASSERT_TRUE(out.EqualsLiteral("text/html;x=\"(\";charset=gbk"))
   << "Legacy comment syntax #2";
 }
+
+TEST(MimeTypeParsing, contentTypes1)
+{
+  const nsAutoCString val(",text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_FALSE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral(""));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes2)
+{
+  const nsAutoCString val("text/plain,");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes3)
+{
+  const nsAutoCString val("text/html,text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes4)
+{
+  const nsAutoCString val("text/plain;charset=gbk,text/html");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes5)
+{
+  const nsAutoCString val(
+      "text/plain;charset=gbk,text/html;charset=windows-1254");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral("windows-1254"));
+}
+
+TEST(MimeTypeParsing, contentTypes6)
+{
+  const nsAutoCString val("text/plain;charset=gbk,text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral("gbk"));
+}
+
+TEST(MimeTypeParsing, contentTypes7)
+{
+  const nsAutoCString val(
+      "text/plain;charset=gbk,text/plain;charset=windows-1252");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral("windows-1252"));
+}
+
+TEST(MimeTypeParsing, contentTypes8)
+{
+  const nsAutoCString val("text/html;charset=gbk,text/html;x=\",text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral("gbk"));
+}
+
+TEST(MimeTypeParsing, contentTypes9)
+{
+  const nsAutoCString val("text/plain;charset=gbk;x=foo,text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral("gbk"));
+}
+
+TEST(MimeTypeParsing, contentTypes10)
+{
+  const nsAutoCString val("text/html;charset=gbk,text/plain,text/html");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes11)
+{
+  const nsAutoCString val("text/plain,*/*");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes12)
+{
+  const nsAutoCString val("text/html,*/*");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes13)
+{
+  const nsAutoCString val("*/*,text/html");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes14)
+{
+  const nsAutoCString val("text/plain,*/*;charset=gbk");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes15)
+{
+  const nsAutoCString val("text/html,*/*;charset=gbk");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes16)
+{
+  const nsAutoCString val("text/html;x=\",text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes17)
+{
+  const nsAutoCString val("text/html;\",text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes18)
+{
+  const nsAutoCString val("text/html;\",\\\",text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}
+
+TEST(MimeTypeParsing, contentTypes19)
+{
+  const nsAutoCString val("text/html;\",\\\",text/plain,\";charset=GBK");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/html"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral("GBK"));
+}
+
+TEST(MimeTypeParsing, contentTypes20)
+{
+  const nsAutoCString val("text/html;\",\",text/plain");
+  nsCString contentType;
+  nsCString contentCharset;
+
+  bool parsed = CMimeType::Parse(val, contentType, contentCharset);
+
+  ASSERT_TRUE(parsed);
+  ASSERT_TRUE(contentType.EqualsLiteral("text/plain"));
+  ASSERT_TRUE(contentCharset.EqualsLiteral(""));
+}

modules/libpref/init/StaticPrefList.yaml

@@ -12799,6 +12799,12 @@
   value: true
   mirror: always
 
+  # If true, HTTP response content-type headers will be parsed using the standards-compliant MimeType parser
+- name: network.standard_content_type_parsing.response_headers
+  type: RelaxedAtomicBool
+  value: true
+  mirror: always
+
 # The maximum count that we allow socket prrocess to crash. If this count is
 # reached, we won't use networking over socket process.
 - name: network.max_socket_process_failed_count

netwerk/protocol/data/nsDataHandler.cpp

@@ -201,7 +201,7 @@ nsresult nsDataHandler::ParsePathWithoutRef(const nsACString& aPath,
   // This is against the current spec, but we're doing it because we have
   // historically seen webcompat issues relying on this (see bug 781693).
   if (mozilla::UniquePtr<CMimeType> parsed = CMimeType::Parse(mimeType)) {
-    parsed->GetFullType(aContentType);
+    parsed->GetEssence(aContentType);
     if (aContentCharset) {
       parsed->GetParameterValue(kCharset, *aContentCharset);
     }

netwerk/protocol/http/nsHttpResponseHead.cpp

@@ -7,6 +7,7 @@
 // HttpLog.h should generally be included first
 #include "HttpLog.h"
 
+#include "mozilla/dom/MimeType.h"
 #include "mozilla/StaticPrefs_network.h"
 #include "mozilla/TextUtils.h"
 #include "mozilla/Unused.h"
@@ -475,9 +476,14 @@ nsresult nsHttpResponseHead::ParseHeaderLine_locked(
     }
 
   } else if (hdr == nsHttp::Content_Type) {
-    LOG(("ParseContentType [type=%s]\n", val.get()));
-    bool dummy;
-    net_ParseContentType(val, mContentType, mContentCharset, &dummy);
+    if (StaticPrefs::network_standard_content_type_parsing_response_headers() &&
+        CMimeType::Parse(val, mContentType, mContentCharset)) {
+    } else {
+      bool dummy;
+      net_ParseContentType(val, mContentType, mContentCharset, &dummy);
+    }
+    LOG(("ParseContentType [input=%s, type=%s, charset=%s]\n", val.get(),
+         mContentType.get(), mContentCharset.get()));
   } else if (hdr == nsHttp::Cache_Control) {
     ParseCacheControl(val.get());
   } else if (hdr == nsHttp::Pragma) {

testing/web-platform/meta/fetch/content-type/response.window.js.ini

@@ -47,9 +47,6 @@
   [Request: combined response Content-Type: text/html;charset=gbk text/plain text/html]
     expected: FAIL
 
-  [<iframe>: separate response Content-Type: text/plain;charset=gbk text/html]
-    expected: FAIL
-
   [fetch(): combined response Content-Type: text/html */*;charset=gbk]
     expected: FAIL
 
@@ -77,9 +74,6 @@
   [Response: combined response Content-Type: text/html text/plain]
     expected: FAIL
 
-  [<iframe>: separate response Content-Type: text/html;charset=gbk text/plain text/html]
-    expected: FAIL
-
   [fetch(): separate response Content-Type: text/plain;charset=gbk text/plain;charset=windows-1252]
     expected: FAIL
 

testing/web-platform/meta/fetch/content-type/script.window.js.ini

@@ -1,18 +1,9 @@
 [script.window.html]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]
-  [separate text/javascript;";charset=windows-1252]
-    expected: FAIL
-
-  [separate x/x;charset=windows-1252 text/javascript]
-    expected: FAIL
-
   [separate text/javascript;" x/x]
     expected: FAIL
 
-  [separate text/javascript;charset=windows-1252 x/x text/javascript]
-    expected: FAIL
-
   [separate text/javascript;charset=windows-1252;" \\" x/x]
     expected: FAIL
 

testing/web-platform/meta/mimesniff/mime-types/charset-parameter.window.js.ini

@@ -1,39 +0,0 @@
-[charset-parameter.window.html]
-  expected:
-    if processor == "x86": [OK, TIMEOUT]
-  [text/html;charset=gbk(]
-    expected: FAIL
-
-  [text/html;charset=gbk;charset=windows-1255]
-    expected: FAIL
-
-  [text/html;";charset=gbk]
-    expected: FAIL
-
-  [text/html;charset=();charset=GBK]
-    expected: FAIL
-
-  [text/html;charset= "gbk"]
-    expected: FAIL
-
-  [text/html;charset=';charset=GBK]
-    expected: FAIL
-
-  [text/html;charset= ";charset=GBK]
-    expected: FAIL
-
-  [text/html;charset="";charset=GBK]
-    expected: FAIL
-
-  [text/html;charset=\x0cgbk]
-    expected:
-      FAIL
-
-  [text/html;charset="gbk"]
-    expected:
-      if (processor == "x86") and not debug: [PASS, TIMEOUT]
-
-  [text/html;;;;charset=gbk]
-    expected:
-      if (processor == "x86") and not debug: [PASS, TIMEOUT]
-