gecko-dev/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "chromium/safebrowsing.pb.h"
#include "nsEscape.h"
#include "nsString.h"
#include "nsIURI.h"
#include "nsIURIMutator.h"
#include "nsIURL.h"
#include "nsIXULRuntime.h"
#include "nsUrlClassifierUtils.h"
#include "nsTArray.h"
#include "nsReadableUtils.h"
#include "plbase64.h"
#include "nsPrintfCString.h"
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/Sprintf.h"
#include "mozilla/StaticPtr.h"
#include "mozilla/Mutex.h"
#include "nsIRedirectHistoryEntry.h"
#include "nsIHttpChannelInternal.h"
#include "mozIThirdPartyUtil.h"
#include "nsIDocShell.h"
#include "mozilla/TextUtils.h"
#include "mozilla/Preferences.h"
#include "mozilla/Services.h"
#include "mozilla/Telemetry.h"
#include "nsNetUtil.h"
#include "nsIHttpChannel.h"
#include "nsIObserverService.h"
#include "nsIPrefBranch.h"
#include "nsIPrefService.h"
#include "nsPIDOMWindow.h"
#include "nsServiceManagerUtils.h"
#include "nsThreadManager.h"
#include "nsTHashSet.h"
#include "Classifier.h"
#include "Entries.h"
#include "prprf.h"
#include "prtime.h"

#define DEFAULT_PROTOCOL_VERSION "2.2"

using namespace mozilla;
using namespace mozilla::safebrowsing;

static mozilla::StaticRefPtr<nsUrlClassifierUtils> gUrlClassifierUtils;

static char int_to_hex_digit(int32_t i) {
  NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit");
  return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A')));
}

static bool IsDecimal(const nsACString& num) {
  for (uint32_t i = 0; i < num.Length(); i++) {
    if (!mozilla::IsAsciiDigit(num[i])) {
      return false;
    }
  }

  return true;
}

static bool IsHex(const nsACString& num) {
  if (num.Length() < 3) {
    return false;
  }

  if (num[0] != '0' || !(num[1] == 'x' || num[1] == 'X')) {
    return false;
  }

  for (uint32_t i = 2; i < num.Length(); i++) {
    if (!mozilla::IsAsciiHexDigit(num[i])) {
      return false;
    }
  }

  return true;
}

static bool IsOctal(const nsACString& num) {
  if (num.Length() < 2) {
    return false;
  }

  if (num[0] != '0') {
    return false;
  }

  for (uint32_t i = 1; i < num.Length(); i++) {
    if (!mozilla::IsAsciiDigit(num[i]) || num[i] == '8' || num[i] == '9') {
      return false;
    }
  }

  return true;
}

/////////////////////////////////////////////////////////////////
// SafeBrowsing V4 related utits.

namespace mozilla {
namespace safebrowsing {

static PlatformType GetPlatformType() {
#if defined(ANDROID)
  return ANDROID_PLATFORM;
#elif defined(XP_MACOSX)
  return OSX_PLATFORM;
#elif defined(XP_LINUX)
  return LINUX_PLATFORM;
#elif defined(XP_WIN)
  return WINDOWS_PLATFORM;
#else
  // Default to Linux for other platforms (see bug 1362501).
  return LINUX_PLATFORM;
#endif
}

typedef FetchThreatListUpdatesRequest_ListUpdateRequest ListUpdateRequest;
typedef FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints Constraints;

static void InitListUpdateRequest(ThreatType aThreatType,
                                  const nsCString& aStateBase64,
                                  ListUpdateRequest* aListUpdateRequest) {
  aListUpdateRequest->set_threat_type(aThreatType);
  PlatformType platform = GetPlatformType();
#if defined(ANDROID)
  // Temporary hack to fix bug 1441345.
  if ((aThreatType == SOCIAL_ENGINEERING_PUBLIC) ||
      (aThreatType == SOCIAL_ENGINEERING)) {
    platform = LINUX_PLATFORM;
  }
#endif
  aListUpdateRequest->set_platform_type(platform);
  aListUpdateRequest->set_threat_entry_type(URL);

  Constraints* contraints = new Constraints();
  contraints->add_supported_compressions(RICE);
  aListUpdateRequest->set_allocated_constraints(contraints);

  // Only set non-empty state.
  if (!aStateBase64.IsEmpty()) {
    nsCString stateBinary;
    nsresult rv = Base64Decode(aStateBase64, stateBinary);
    if (NS_SUCCEEDED(rv)) {
      aListUpdateRequest->set_state(stateBinary.get(), stateBinary.Length());
    }
  }
}

static ClientInfo* CreateClientInfo() {
  ClientInfo* c = new ClientInfo();

  nsCOMPtr<nsIPrefBranch> prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID);

  nsAutoCString clientId;
  nsresult rv = prefBranch->GetCharPref("browser.safebrowsing.id", clientId);

  if (NS_FAILED(rv)) {
    clientId = "Firefox";  // Use "Firefox" as fallback.
  }

  c->set_client_id(clientId.get());

  return c;
}

static bool IsAllowedOnCurrentPlatform(uint32_t aThreatType) {
  PlatformType platform = GetPlatformType();

  switch (aThreatType) {
    case POTENTIALLY_HARMFUL_APPLICATION:
      // Bug 1388582 - Google server would respond 404 error if the request
      // contains PHA on non-mobile platform.
      return ANDROID_PLATFORM == platform;
    case MALICIOUS_BINARY:
    case CSD_DOWNLOAD_WHITELIST:
      // Bug 1392204 - 'goog-downloadwhite-proto' and 'goog-badbinurl-proto'
      // are not available on android.
      return ANDROID_PLATFORM != platform;
  }
  // We allow every threat type not listed in the switch cases.
  return true;
}

}  // end of namespace safebrowsing.
}  // end of namespace mozilla.

// static
already_AddRefed<nsUrlClassifierUtils>
nsUrlClassifierUtils::GetXPCOMSingleton() {
  if (gUrlClassifierUtils) {
    return do_AddRef(gUrlClassifierUtils);
  }

  RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils();
  if (NS_WARN_IF(NS_FAILED(utils->Init()))) {
    return nullptr;
  }

  // Note: This is cleared in the nsUrlClassifierUtils destructor.
  gUrlClassifierUtils = utils.get();
  ClearOnShutdown(&gUrlClassifierUtils);
  return utils.forget();
}

// static
nsUrlClassifierUtils* nsUrlClassifierUtils::GetInstance() {
  if (!gUrlClassifierUtils) {
    RefPtr<nsUrlClassifierUtils> utils = GetXPCOMSingleton();
  }

  return gUrlClassifierUtils;
}

nsUrlClassifierUtils::nsUrlClassifierUtils()
    : mProviderDictLock("nsUrlClassifierUtils.mProviderDictLock") {}

nsUrlClassifierUtils::~nsUrlClassifierUtils() {
  if (gUrlClassifierUtils) {
    MOZ_ASSERT(gUrlClassifierUtils == this);
    gUrlClassifierUtils = nullptr;
  }
}

nsresult nsUrlClassifierUtils::Init() {
  // nsIUrlClassifierUtils is a thread-safe service so it's
  // allowed to use on non-main threads. However, building
  // the provider dictionary must be on the main thread.
  // We forcefully load nsUrlClassifierUtils in
  // nsUrlClassifierDBService::Init() to ensure we must
  // now be on the main thread.
  nsresult rv = ReadProvidersFromPrefs(mProviderDict);
  NS_ENSURE_SUCCESS(rv, rv);

  // Add an observer for shutdown
  nsCOMPtr<nsIObserverService> observerService =
      mozilla::services::GetObserverService();
  if (!observerService) return NS_ERROR_FAILURE;

  observerService->AddObserver(this, "xpcom-shutdown-threads", false);
  mozilla::Preferences::AddStrongObserver(this, "browser.safebrowsing");

  return NS_OK;
}

NS_IMPL_ISUPPORTS(nsUrlClassifierUtils, nsIUrlClassifierUtils, nsIObserver)

/////////////////////////////////////////////////////////////////////////////
// nsIUrlClassifierUtils

NS_IMETHODIMP
nsUrlClassifierUtils::GetKeyForURI(nsIURI* uri, nsACString& _retval) {
  nsCOMPtr<nsIURI> innerURI = NS_GetInnermostURI(uri);
  if (!innerURI) innerURI = uri;

  nsAutoCString host;
  innerURI->GetAsciiHost(host);

  if (host.IsEmpty()) {
    return NS_ERROR_MALFORMED_URI;
  }

  nsresult rv = CanonicalizeHostname(host, _retval);
  NS_ENSURE_SUCCESS(rv, rv);

  nsAutoCString path;
  rv = innerURI->GetPathQueryRef(path);
  NS_ENSURE_SUCCESS(rv, rv);

  // Strip fragment and query because canonicalization only applies to path
  int32_t ref = path.FindChar('#');
  if (ref != kNotFound) {
    path.SetLength(ref);
  }

  int32_t query = path.FindChar('?');
  if (query != kNotFound) {
    path.SetLength(query);
  }

  nsAutoCString temp;
  rv = CanonicalizePath(path, temp);
  NS_ENSURE_SUCCESS(rv, rv);

  _retval.Append(temp);

  if (query != kNotFound) {
    nsAutoCString query;
    rv = innerURI->GetQuery(query);
    NS_ENSURE_SUCCESS(rv, rv);

    // We have to canonicalize the query too based on
    // https://developers.google.com/safe-browsing/v4/urls-hashing?hl=en#canonicalization
    rv = CanonicalizeQuery(query, temp);
    NS_ENSURE_SUCCESS(rv, rv);

    _retval.Append(temp);
  }

  return NS_OK;
}

// We use "goog-*-proto" as the list name for v4, where "proto" indicates
// it's updated (as well as hash completion) via protobuf.
//
// In the mozilla official build, we are allowed to use the
// private phishing list (goog-phish-proto). See Bug 1288840.
static const struct {
  const char* mListName;
  uint32_t mThreatType;
} THREAT_TYPE_CONV_TABLE[] = {
    {"goog-malware-proto", MALWARE_THREAT},                   // 1
    {"googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC},       // 2
    {"goog-unwanted-proto", UNWANTED_SOFTWARE},               // 3
    {"goog-harmful-proto", POTENTIALLY_HARMFUL_APPLICATION},  // 4
    {"goog-phish-proto", SOCIAL_ENGINEERING},                 // 5

    // For application reputation
    {"goog-badbinurl-proto", MALICIOUS_BINARY},            // 7
    {"goog-downloadwhite-proto", CSD_DOWNLOAD_WHITELIST},  // 9

    // For testing purpose.
    {"moztest-phish-proto", SOCIAL_ENGINEERING_PUBLIC},  // 2
    {"test-phish-proto", SOCIAL_ENGINEERING_PUBLIC},     // 2
    {"moztest-unwanted-proto", UNWANTED_SOFTWARE},       // 3
    {"test-unwanted-proto", UNWANTED_SOFTWARE},          // 3
};

NS_IMETHODIMP
nsUrlClassifierUtils::ConvertThreatTypeToListNames(uint32_t aThreatType,
                                                   nsACString& aListNames) {
  for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) {
    if (aThreatType == THREAT_TYPE_CONV_TABLE[i].mThreatType) {
      if (!aListNames.IsEmpty()) {
        aListNames.AppendLiteral(",");
      }
      aListNames += THREAT_TYPE_CONV_TABLE[i].mListName;
    }
  }

  return aListNames.IsEmpty() ? NS_ERROR_FAILURE : NS_OK;
}

NS_IMETHODIMP
nsUrlClassifierUtils::ConvertListNameToThreatType(const nsACString& aListName,
                                                  uint32_t* aThreatType) {
  for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) {
    if (aListName.EqualsASCII(THREAT_TYPE_CONV_TABLE[i].mListName)) {
      *aThreatType = THREAT_TYPE_CONV_TABLE[i].mThreatType;
      return NS_OK;
    }
  }

  return NS_ERROR_FAILURE;
}

NS_IMETHODIMP
nsUrlClassifierUtils::GetProvider(const nsACString& aTableName,
                                  nsACString& aProvider) {
  MutexAutoLock lock(mProviderDictLock);
  nsCString* provider = nullptr;

  if (IsTestTable(aTableName)) {
    aProvider = nsLiteralCString(TESTING_TABLE_PROVIDER_NAME);
  } else if (mProviderDict.Get(aTableName, &provider)) {
    aProvider = provider ? *provider : ""_ns;
  } else {
    aProvider.Truncate();
  }
  return NS_OK;
}

NS_IMETHODIMP
nsUrlClassifierUtils::GetTelemetryProvider(const nsACString& aTableName,
                                           nsACString& aProvider) {
  GetProvider(aTableName, aProvider);
  // Exceptionlist known providers to avoid reporting on private ones.
  // An empty provider is treated as "other"
  if (!"mozilla"_ns.Equals(aProvider) && !"google"_ns.Equals(aProvider) &&
      !"google4"_ns.Equals(aProvider) && !"baidu"_ns.Equals(aProvider) &&
      !"mozcn"_ns.Equals(aProvider) && !"yandex"_ns.Equals(aProvider) &&
      !nsLiteralCString(TESTING_TABLE_PROVIDER_NAME).Equals(aProvider)) {
    aProvider.AssignLiteral("other");
  }

  return NS_OK;
}

NS_IMETHODIMP
nsUrlClassifierUtils::GetProtocolVersion(const nsACString& aProvider,
                                         nsACString& aVersion) {
  nsCOMPtr<nsIPrefBranch> prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID);
  if (prefBranch) {
    nsPrintfCString prefName("browser.safebrowsing.provider.%s.pver",
                             nsCString(aProvider).get());
    nsAutoCString version;
    nsresult rv = prefBranch->GetCharPref(prefName.get(), version);

    aVersion = NS_SUCCEEDED(rv) ? version.get() : DEFAULT_PROTOCOL_VERSION;
  } else {
    aVersion = DEFAULT_PROTOCOL_VERSION;
  }

  return NS_OK;
}

NS_IMETHODIMP
nsUrlClassifierUtils::MakeUpdateRequestV4(
    const nsTArray<nsCString>& aListNames,
    const nsTArray<nsCString>& aStatesBase64, nsACString& aRequest) {
  using namespace mozilla::safebrowsing;

  if (aListNames.Length() != aStatesBase64.Length()) {
    return NS_ERROR_INVALID_ARG;
  }

  FetchThreatListUpdatesRequest r;
  r.set_allocated_client(CreateClientInfo());

  for (uint32_t i = 0; i < aListNames.Length(); i++) {
    uint32_t threatType;
    nsresult rv = ConvertListNameToThreatType(aListNames[i], &threatType);
    if (NS_FAILED(rv)) {
      continue;  // Unknown list name.
    }
    if (!IsAllowedOnCurrentPlatform(threatType)) {
      NS_WARNING(
          nsPrintfCString(
              "Threat type %d (%s) is unsupported on current platform: %d",
              threatType, aListNames[i].get(), GetPlatformType())
              .get());
      continue;  // Some threat types are not available on some platforms.
    }
    auto lur = r.mutable_list_update_requests()->Add();
    InitListUpdateRequest(static_cast<ThreatType>(threatType), aStatesBase64[i],
                          lur);
  }

  // Then serialize.
  std::string s;
  r.SerializeToString(&s);

  nsCString out;
  nsresult rv = Base64URLEncode(s.size(), (const uint8_t*)s.c_str(),
                                Base64URLEncodePaddingPolicy::Include, out);
  NS_ENSURE_SUCCESS(rv, rv);

  aRequest = out;

  return NS_OK;
}

NS_IMETHODIMP
nsUrlClassifierUtils::MakeFindFullHashRequestV4(
    const nsTArray<nsCString>& aListNames,
    const nsTArray<nsCString>& aListStatesBase64,
    const nsTArray<nsCString>& aPrefixesBase64, nsACString& aRequest) {
  if (aListNames.Length() != aListStatesBase64.Length()) {
    return NS_ERROR_INVALID_ARG;
  }

  FindFullHashesRequest r;
  r.set_allocated_client(CreateClientInfo());

  nsresult rv;

  //-------------------------------------------------------------------
  // Set up FindFullHashesRequest.threat_info.
  auto threatInfo = r.mutable_threat_info();

  PlatformType platform = GetPlatformType();

  // 1) Set threat types.
  for (uint32_t i = 0; i < aListNames.Length(); i++) {
    // Add threat types.
    uint32_t threatType;
    rv = ConvertListNameToThreatType(aListNames[i], &threatType);
    NS_ENSURE_SUCCESS(rv, rv);
    if (!IsAllowedOnCurrentPlatform(threatType)) {
      NS_WARNING(
          nsPrintfCString(
              "Threat type %d (%s) is unsupported on current platform: %d",
              threatType, aListNames[i].get(), GetPlatformType())
              .get());
      continue;
    }
    threatInfo->add_threat_types((ThreatType)threatType);

#if defined(ANDROID)
    // Temporary hack to fix bug 1441345.
    if (((ThreatType)threatType == SOCIAL_ENGINEERING_PUBLIC) ||
        ((ThreatType)threatType == SOCIAL_ENGINEERING)) {
      platform = LINUX_PLATFORM;
    }
#endif

    // Add client states for index 'i' only when the threat type is available
    // on current platform.
    nsCString stateBinary;
    rv = Base64Decode(aListStatesBase64[i], stateBinary);
    NS_ENSURE_SUCCESS(rv, rv);
    r.add_client_states(stateBinary.get(), stateBinary.Length());
  }

  // 2) Set platform type.
  threatInfo->add_platform_types(platform);

  // 3) Set threat entry type.
  threatInfo->add_threat_entry_types(URL);

  // 4) Set threat entries.
  for (const nsCString& prefix : aPrefixesBase64) {
    nsCString prefixBinary;
    rv = Base64Decode(prefix, prefixBinary);
    threatInfo->add_threat_entries()->set_hash(prefixBinary.get(),
                                               prefixBinary.Length());
  }
  //-------------------------------------------------------------------

  // Then serialize.
  std::string s;
  r.SerializeToString(&s);

  nsCString out;
  rv = Base64URLEncode(s.size(), (const uint8_t*)s.c_str(),
                       Base64URLEncodePaddingPolicy::Include, out);
  NS_ENSURE_SUCCESS(rv, rv);

  aRequest = out;

  return NS_OK;
}

// Remove ref, query, userpass, anypart which may contain sensitive data
static nsresult GetSpecWithoutSensitiveData(nsIURI* aUri, nsACString& aSpec) {
  if (NS_WARN_IF(!aUri)) {
    return NS_ERROR_INVALID_ARG;
  }

  nsresult rv;
  nsCOMPtr<nsIURL> url(do_QueryInterface(aUri));
  if (url) {
    nsCOMPtr<nsIURI> clone;
    rv = NS_MutateURI(url)
             .SetQuery(""_ns)
             .SetRef(""_ns)
             .SetUserPass(""_ns)
             .Finalize(clone);
    NS_ENSURE_SUCCESS(rv, rv);
    rv = clone->GetAsciiSpec(aSpec);
    NS_ENSURE_SUCCESS(rv, rv);
  }
  return NS_OK;
}

static nsresult AddThreatSourceFromChannel(ThreatHit& aHit,
                                           nsIChannel* aChannel,
                                           ThreatHit_ThreatSourceType aType) {
  if (NS_WARN_IF(!aChannel)) {
    return NS_ERROR_INVALID_ARG;
  }

  nsresult rv;

  auto matchingSource = aHit.add_resources();
  matchingSource->set_type(aType);

  nsCOMPtr<nsIURI> uri;
  rv = NS_GetFinalChannelURI(aChannel, getter_AddRefs(uri));
  NS_ENSURE_SUCCESS(rv, rv);

  nsCString spec;
  rv = GetSpecWithoutSensitiveData(uri, spec);
  NS_ENSURE_SUCCESS(rv, rv);
  matchingSource->set_url(spec.get());

  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aChannel);
  if (httpChannel) {
    nsCOMPtr<nsIReferrerInfo> referrerInfo = httpChannel->GetReferrerInfo();
    if (referrerInfo) {
      nsAutoCString referrerSpec;
      nsCOMPtr<nsIURI> referrer = referrerInfo->GetComputedReferrer();
      if (referrer) {
        rv = GetSpecWithoutSensitiveData(referrer, referrerSpec);
        NS_ENSURE_SUCCESS(rv, rv);
        matchingSource->set_referrer(referrerSpec.get());
      }
    }
  }

  nsCOMPtr<nsIHttpChannelInternal> httpChannelInternal =
      do_QueryInterface(aChannel);
  if (httpChannelInternal) {
    nsCString remoteIp;
    rv = httpChannelInternal->GetRemoteAddress(remoteIp);
    if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) {
      matchingSource->set_remote_ip(remoteIp.get());
    }
  }
  return NS_OK;
}
static nsresult AddThreatSourceFromRedirectEntry(
    ThreatHit& aHit, nsIRedirectHistoryEntry* aRedirectEntry,
    ThreatHit_ThreatSourceType aType) {
  if (NS_WARN_IF(!aRedirectEntry)) {
    return NS_ERROR_INVALID_ARG;
  }

  nsresult rv;

  nsCOMPtr<nsIPrincipal> principal;
  rv = aRedirectEntry->GetPrincipal(getter_AddRefs(principal));
  NS_ENSURE_SUCCESS(rv, rv);
  nsCString spec;
  rv = principal->GetExposableSpec(spec);
  NS_ENSURE_SUCCESS(rv, rv);
  auto source = aHit.add_resources();
  source->set_url(spec.get());
  source->set_type(aType);

  nsCOMPtr<nsIURI> referrer;
  rv = aRedirectEntry->GetReferrerURI(getter_AddRefs(referrer));
  if (NS_SUCCEEDED(rv) && referrer) {
    nsCString referrerSpec;
    rv = GetSpecWithoutSensitiveData(referrer, referrerSpec);
    NS_ENSURE_SUCCESS(rv, rv);
    source->set_referrer(referrerSpec.get());
  }

  nsCString remoteIp;
  rv = aRedirectEntry->GetRemoteAddress(remoteIp);
  if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) {
    source->set_remote_ip(remoteIp.get());
  }
  return NS_OK;
}

// Add top level tab url and redirect threatsources to threatHit message
static nsresult AddTabThreatSources(ThreatHit& aHit, nsIChannel* aChannel) {
  if (NS_WARN_IF(!aChannel)) {
    return NS_ERROR_INVALID_ARG;
  }

  nsresult rv;
  nsCOMPtr<mozIDOMWindowProxy> win;
  nsCOMPtr<mozIThirdPartyUtil> thirdPartyUtil =
      do_GetService(THIRDPARTYUTIL_CONTRACTID, &rv);
  NS_ENSURE_SUCCESS(rv, rv);

  rv = thirdPartyUtil->GetTopWindowForChannel(aChannel, nullptr,
                                              getter_AddRefs(win));
  NS_ENSURE_SUCCESS(rv, rv);

  auto* pwin = nsPIDOMWindowOuter::From(win);
  nsCOMPtr<nsIDocShell> docShell = pwin->GetDocShell();
  if (!docShell) {
    return NS_OK;
  }

  nsCOMPtr<nsIChannel> topChannel;
  docShell->GetCurrentDocumentChannel(getter_AddRefs(topChannel));
  if (!topChannel) {
    return NS_OK;
  }

  nsCOMPtr<nsIURI> uri;
  rv = aChannel->GetURI(getter_AddRefs(uri));
  NS_ENSURE_SUCCESS(rv, rv);

  nsCOMPtr<nsIURI> topUri;
  rv = topChannel->GetURI(getter_AddRefs(topUri));
  NS_ENSURE_SUCCESS(rv, rv);

  bool isTopUri = false;
  rv = topUri->Equals(uri, &isTopUri);
  if (NS_SUCCEEDED(rv) && !isTopUri) {
    nsCOMPtr<nsILoadInfo> loadInfo = aChannel->LoadInfo();
    if (loadInfo->RedirectChain().Length()) {
      AddThreatSourceFromRedirectEntry(aHit, loadInfo->RedirectChain()[0],
                                       ThreatHit_ThreatSourceType_TAB_RESOURCE);
    }
  }

  // Set top level tab_url threat source
  rv = AddThreatSourceFromChannel(aHit, topChannel,
                                  ThreatHit_ThreatSourceType_TAB_URL);
  Unused << NS_WARN_IF(NS_FAILED(rv));

  // Set tab_redirect threat sources if there's any
  nsCOMPtr<nsILoadInfo> topLoadInfo = topChannel->LoadInfo();
  nsIRedirectHistoryEntry* redirectEntry;
  size_t length = topLoadInfo->RedirectChain().Length();
  for (size_t i = 0; i < length; i++) {
    redirectEntry = topLoadInfo->RedirectChain()[i];
    AddThreatSourceFromRedirectEntry(aHit, redirectEntry,
                                     ThreatHit_ThreatSourceType_TAB_REDIRECT);
  }

  return NS_OK;
}

NS_IMETHODIMP
nsUrlClassifierUtils::MakeThreatHitReport(nsIChannel* aChannel,
                                          const nsACString& aListName,
                                          const nsACString& aHashBase64,
                                          nsACString& aRequest) {
  if (NS_WARN_IF(aListName.IsEmpty()) || NS_WARN_IF(aHashBase64.IsEmpty()) ||
      NS_WARN_IF(!aChannel)) {
    return NS_ERROR_INVALID_ARG;
  }

  ThreatHit hit;
  nsresult rv;

  uint32_t threatType;
  rv = ConvertListNameToThreatType(aListName, &threatType);
  NS_ENSURE_SUCCESS(rv, rv);
  hit.set_threat_type(static_cast<ThreatType>(threatType));

  hit.set_platform_type(GetPlatformType());

  nsCString hash;
  rv = Base64Decode(aHashBase64, hash);
  if (NS_FAILED(rv) || hash.Length() != COMPLETE_SIZE) {
    return NS_ERROR_FAILURE;
  }

  auto threatEntry = hit.mutable_entry();
  threatEntry->set_hash(hash.get(), hash.Length());

  // Set matching source
  rv = AddThreatSourceFromChannel(hit, aChannel,
                                  ThreatHit_ThreatSourceType_MATCHING_URL);
  Unused << NS_WARN_IF(NS_FAILED(rv));
  // Set tab url, tab resource url and redirect sources
  rv = AddTabThreatSources(hit, aChannel);
  Unused << NS_WARN_IF(NS_FAILED(rv));

  hit.set_allocated_client_info(CreateClientInfo());

  std::string s;
  hit.SerializeToString(&s);

  nsCString out;
  rv = Base64URLEncode(s.size(), reinterpret_cast<const uint8_t*>(s.c_str()),
                       Base64URLEncodePaddingPolicy::Include, out);
  NS_ENSURE_SUCCESS(rv, rv);

  aRequest = out;

  return NS_OK;
}

static uint32_t DurationToMs(const Duration& aDuration) {
  // Seconds precision is good enough. Ignore nanoseconds like Chrome does.
  return aDuration.seconds() * 1000;
}

NS_IMETHODIMP
nsUrlClassifierUtils::ParseFindFullHashResponseV4(
    const nsACString& aResponse,
    nsIUrlClassifierParseFindFullHashCallback* aCallback) {
  enum CompletionErrorType {
    SUCCESS = 0,
    PARSING_FAILURE = 1,
    UNKNOWN_THREAT_TYPE = 2,
  };

  FindFullHashesResponse r;
  if (!r.ParseFromArray(aResponse.BeginReading(), aResponse.Length())) {
    NS_WARNING("Invalid response");
    Telemetry::Accumulate(Telemetry::URLCLASSIFIER_COMPLETION_ERROR,
                          PARSING_FAILURE);
    return NS_ERROR_FAILURE;
  }

  bool hasUnknownThreatType = false;

  for (auto& m : r.matches()) {
    nsCString tableNames;
    nsresult rv = ConvertThreatTypeToListNames(m.threat_type(), tableNames);
    if (NS_FAILED(rv)) {
      hasUnknownThreatType = true;
      continue;  // Ignore un-convertable threat type.
    }
    auto& hash = m.threat().hash();
    auto cacheDurationSec = m.cache_duration().seconds();
    aCallback->OnCompleteHashFound(
        nsDependentCString(hash.c_str(), hash.length()), tableNames,
        cacheDurationSec);
  }

  auto minWaitDuration = DurationToMs(r.minimum_wait_duration());
  auto negCacheDurationSec = r.negative_cache_duration().seconds();

  aCallback->OnResponseParsed(minWaitDuration, negCacheDurationSec);

  Telemetry::Accumulate(Telemetry::URLCLASSIFIER_COMPLETION_ERROR,
                        hasUnknownThreatType ? UNKNOWN_THREAT_TYPE : SUCCESS);
  return NS_OK;
}

//////////////////////////////////////////////////////////
// nsIObserver

NS_IMETHODIMP
nsUrlClassifierUtils::Observe(nsISupports* aSubject, const char* aTopic,
                              const char16_t* aData) {
  if (0 == strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
    MutexAutoLock lock(mProviderDictLock);
    return ReadProvidersFromPrefs(mProviderDict);
  }

  if (0 == strcmp(aTopic, "xpcom-shutdown-threads")) {
    nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
    NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE);
    return prefs->RemoveObserver("browser.safebrowsing", this);
  }

  return NS_ERROR_UNEXPECTED;
}

/////////////////////////////////////////////////////////////////////////////
// non-interface methods

nsresult nsUrlClassifierUtils::ReadProvidersFromPrefs(ProviderDictType& aDict) {
  MOZ_ASSERT(NS_IsMainThread(),
             "ReadProvidersFromPrefs must be on main thread");

  nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
  NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE);
  nsCOMPtr<nsIPrefBranch> prefBranch;
  nsresult rv = prefs->GetBranch("browser.safebrowsing.provider.",
                                 getter_AddRefs(prefBranch));
  NS_ENSURE_SUCCESS(rv, rv);

  // We've got a pref branch for "browser.safebrowsing.provider.".
  // Enumerate all children prefs and parse providers.
  nsTArray<nsCString> childArray;
  rv = prefBranch->GetChildList("", childArray);
  NS_ENSURE_SUCCESS(rv, rv);

  // Collect providers from childArray.
  nsTHashSet<nsCString> providers;
  for (auto& child : childArray) {
    auto dotPos = child.FindChar('.');
    if (dotPos < 0) {
      continue;
    }

    nsDependentCSubstring provider = Substring(child, 0, dotPos);

    providers.Insert(provider);
  }

  // Now we have all providers. Check which one owns |aTableName|.
  // e.g. The owning lists of provider "google" is defined in
  // "browser.safebrowsing.provider.google.lists".
  for (const auto& provider : providers) {
    nsPrintfCString owninListsPref("%s.lists",
                                   nsPromiseFlatCString{provider}.get());

    nsAutoCString owningLists;
    nsresult rv = prefBranch->GetCharPref(owninListsPref.get(), owningLists);
    if (NS_FAILED(rv)) {
      continue;
    }

    // We've got the owning lists (represented as string) of |provider|.
    // Build the dictionary for the owning list and the current provider.
    nsTArray<nsCString> tables;
    Classifier::SplitTables(owningLists, tables);
    for (auto tableName : tables) {
      aDict.InsertOrUpdate(tableName, MakeUnique<nsCString>(provider));
    }
  }

  return NS_OK;
}

nsresult nsUrlClassifierUtils::CanonicalizeHostname(const nsACString& hostname,
                                                    nsACString& _retval) {
  nsAutoCString unescaped;
  if (!NS_UnescapeURL(PromiseFlatCString(hostname).get(),
                      PromiseFlatCString(hostname).Length(), 0, unescaped)) {
    unescaped.Assign(hostname);
  }

  nsAutoCString cleaned;
  CleanupHostname(unescaped, cleaned);

  nsAutoCString temp;
  ParseIPAddress(cleaned, temp);
  if (!temp.IsEmpty()) {
    cleaned.Assign(temp);
  }

  ToLowerCase(cleaned);
  SpecialEncode(cleaned, false, _retval);

  return NS_OK;
}

nsresult nsUrlClassifierUtils::CanonicalizePath(const nsACString& path,
                                                nsACString& _retval) {
  _retval.Truncate();

  nsAutoCString decodedPath(path);
  nsAutoCString temp;
  while (NS_UnescapeURL(decodedPath.get(), decodedPath.Length(), 0, temp)) {
    decodedPath.Assign(temp);
    temp.Truncate();
  }

  SpecialEncode(decodedPath, true, _retval);
  // XXX: lowercase the path?

  return NS_OK;
}

nsresult nsUrlClassifierUtils::CanonicalizeQuery(const nsACString& query,
                                                 nsACString& _retval) {
  _retval.Truncate();
  _retval.Append('?');

  // Unescape the query
  nsAutoCString unescaped;
  if (!NS_UnescapeURL(PromiseFlatCString(query).get(),
                      PromiseFlatCString(query).Length(), 0, unescaped)) {
    unescaped.Assign(query);
  }

  // slash folding does not apply to the query parameters, but we need to
  // percent-escape all characters that are <= ASCII 32, >= 127, "#", or "%"
  SpecialEncode(unescaped, false, _retval);

  return NS_OK;
}

void nsUrlClassifierUtils::CleanupHostname(const nsACString& hostname,
                                           nsACString& _retval) {
  _retval.Truncate();

  const char* curChar = hostname.BeginReading();
  const char* end = hostname.EndReading();
  char lastChar = '\0';
  while (curChar != end) {
    unsigned char c = static_cast<unsigned char>(*curChar);
    if (c == '.' && (lastChar == '\0' || lastChar == '.')) {
      // skip
    } else {
      _retval.Append(*curChar);
    }
    lastChar = c;
    ++curChar;
  }

  // cut off trailing dots
  while (_retval.Length() > 0 && _retval[_retval.Length() - 1] == '.') {
    _retval.SetLength(_retval.Length() - 1);
  }
}

void nsUrlClassifierUtils::ParseIPAddress(const nsACString& host,
                                          nsACString& _retval) {
  _retval.Truncate();
  nsACString::const_iterator iter, end;
  host.BeginReading(iter);
  host.EndReading(end);

  if (host.Length() <= 15) {
    // The Windows resolver allows a 4-part dotted decimal IP address to
    // have a space followed by any old rubbish, so long as the total length
    // of the string doesn't get above 15 characters. So, "10.192.95.89 xy"
    // is resolved to 10.192.95.89.
    // If the string length is greater than 15 characters, e.g.
    // "10.192.95.89 xy.wildcard.example.com", it will be resolved through
    // DNS.

    if (FindCharInReadable(' ', iter, end)) {
      end = iter;
    }
  }

  for (host.BeginReading(iter); iter != end; iter++) {
    if (!(mozilla::IsAsciiHexDigit(*iter) || *iter == 'x' || *iter == 'X' ||
          *iter == '.')) {
      // not an IP
      return;
    }
  }

  host.BeginReading(iter);
  nsTArray<nsCString> parts;
  ParseString(PromiseFlatCString(Substring(iter, end)), '.', parts);
  if (parts.Length() > 4) {
    return;
  }

  // If any potentially-octal numbers (start with 0 but not hex) have
  // non-octal digits, no part of the ip can be in octal
  // XXX: this came from the old javascript implementation, is it really
  // supposed to be like this?
  bool allowOctal = true;
  uint32_t i;

  for (i = 0; i < parts.Length(); i++) {
    const nsCString& part = parts[i];
    if (part[0] == '0') {
      for (uint32_t j = 1; j < part.Length(); j++) {
        if (part[j] == 'x') {
          break;
        }
        if (part[j] == '8' || part[j] == '9') {
          allowOctal = false;
          break;
        }
      }
    }
  }

  for (i = 0; i < parts.Length(); i++) {
    nsAutoCString canonical;

    if (i == parts.Length() - 1) {
      CanonicalNum(parts[i], 5 - parts.Length(), allowOctal, canonical);
    } else {
      CanonicalNum(parts[i], 1, allowOctal, canonical);
    }

    if (canonical.IsEmpty()) {
      _retval.Truncate();
      return;
    }

    if (_retval.IsEmpty()) {
      _retval.Assign(canonical);
    } else {
      _retval.Append('.');
      _retval.Append(canonical);
    }
  }
}

void nsUrlClassifierUtils::CanonicalNum(const nsACString& num, uint32_t bytes,
                                        bool allowOctal, nsACString& _retval) {
  _retval.Truncate();

  if (num.Length() < 1) {
    return;
  }

  uint32_t val;
  if (allowOctal && IsOctal(num)) {
    if (PR_sscanf(PromiseFlatCString(num).get(), "%o", &val) != 1) {
      return;
    }
  } else if (IsDecimal(num)) {
    if (PR_sscanf(PromiseFlatCString(num).get(), "%u", &val) != 1) {
      return;
    }
  } else if (IsHex(num)) {
    if (PR_sscanf(PromiseFlatCString(num).get(),
                  num[1] == 'X' ? "0X%x" : "0x%x", &val) != 1) {
      return;
    }
  } else {
    return;
  }

  while (bytes--) {
    char buf[20];
    SprintfLiteral(buf, "%u", val & 0xff);
    if (_retval.IsEmpty()) {
      _retval.Assign(buf);
    } else {
      _retval = nsDependentCString(buf) + "."_ns + _retval;
    }
    val >>= 8;
  }
}

// This function will encode all "special" characters in typical url
// encoding, that is %hh where h is a valid hex digit.  It will also fold
// any duplicated slashes.
bool nsUrlClassifierUtils::SpecialEncode(const nsACString& url,
                                         bool foldSlashes,
                                         nsACString& _retval) {
  bool changed = false;
  const char* curChar = url.BeginReading();
  const char* end = url.EndReading();

  unsigned char lastChar = '\0';
  while (curChar != end) {
    unsigned char c = static_cast<unsigned char>(*curChar);
    if (ShouldURLEscape(c)) {
      _retval.Append('%');
      _retval.Append(int_to_hex_digit(c / 16));
      _retval.Append(int_to_hex_digit(c % 16));

      changed = true;
    } else if (foldSlashes && (c == '/' && lastChar == '/')) {
      // skip
    } else {
      _retval.Append(*curChar);
    }
    lastChar = c;
    curChar++;
  }
  return changed;
}

bool nsUrlClassifierUtils::ShouldURLEscape(const unsigned char c) const {
  return c <= 32 || c == '%' || c == '#' || c >= 127;
}

// moztest- tables are built-in created in LookupCache, they contain hardcoded
// url entries in it. moztest tables don't support updates.
// static
bool nsUrlClassifierUtils::IsMozTestTable(const nsACString& aTableName) {
  return StringBeginsWith(aTableName, "moztest-"_ns);
}

// test- tables are used by testcases and can add custom test entries
// through update API.
// static
bool nsUrlClassifierUtils::IsTestTable(const nsACString& aTableName) {
  return IsMozTestTable(aTableName) || StringBeginsWith(aTableName, "test"_ns);
}

bool nsUrlClassifierUtils::IsInSafeMode() {
  static Maybe<bool> sIsInSafeMode;

  if (!sIsInSafeMode.isSome()) {
    nsCOMPtr<nsIXULRuntime> appInfo =
        do_GetService("@mozilla.org/xre/runtime;1");
    if (appInfo) {
      bool inSafeMode = false;
      appInfo->GetInSafeMode(&inSafeMode);
      sIsInSafeMode.emplace(inSafeMode);
    }
  }

  return sIsInSafeMode.value();
}