forked from mirrors/gecko-dev
81b83a1f76
After adopting the new thread model for safebrowsing, we will create a new lookup cache for update so we can still check lookup cache at the same time. Prefix set, completions will be generated when we open the new lookup cache but it won't include cache, so we will loss cache after that. This patch will copy cache data from old lookup cache to new lookup cache while update. MozReview-Commit-ID: L0WpiHOGIGm --HG-- extra : rebase_source : ebaf249535561b87a983a3f24dacb8fbab7c096e
1647 lines
48 KiB
C++
1647 lines
48 KiB
C++
//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#include "Classifier.h"
|
|
#include "LookupCacheV4.h"
|
|
#include "nsIPrefBranch.h"
|
|
#include "nsIPrefService.h"
|
|
#include "nsISimpleEnumerator.h"
|
|
#include "nsIRandomGenerator.h"
|
|
#include "nsIInputStream.h"
|
|
#include "nsISeekableStream.h"
|
|
#include "nsIFile.h"
|
|
#include "nsNetCID.h"
|
|
#include "nsPrintfCString.h"
|
|
#include "nsThreadUtils.h"
|
|
#include "mozilla/Telemetry.h"
|
|
#include "mozilla/IntegerPrintfMacros.h"
|
|
#include "mozilla/Logging.h"
|
|
#include "mozilla/SyncRunnable.h"
|
|
#include "mozilla/Base64.h"
|
|
#include "mozilla/Unused.h"
|
|
#include "mozilla/SizePrintfMacros.h"
|
|
#include "mozilla/UniquePtr.h"
|
|
#include "nsIUrlClassifierUtils.h"
|
|
#include "nsUrlClassifierDBService.h"
|
|
|
|
// MOZ_LOG=UrlClassifierDbService:5
|
|
extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
|
|
#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
|
|
#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
|
|
|
|
#define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing")
|
|
#define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
|
|
#define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup")
|
|
#define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating")
|
|
|
|
#define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata")
|
|
|
|
namespace mozilla {
|
|
namespace safebrowsing {
|
|
|
|
namespace {
|
|
|
|
// A scoped-clearer for nsTArray<TableUpdate*>.
|
|
// The owning elements will be deleted and the array itself
|
|
// will be cleared on exiting the scope.
|
|
class ScopedUpdatesClearer {
|
|
public:
|
|
explicit ScopedUpdatesClearer(nsTArray<TableUpdate*> *aUpdates)
|
|
: mUpdatesArrayRef(aUpdates)
|
|
{
|
|
for (auto update : *aUpdates) {
|
|
mUpdatesPointerHolder.AppendElement(update);
|
|
}
|
|
}
|
|
|
|
~ScopedUpdatesClearer()
|
|
{
|
|
mUpdatesArrayRef->Clear();
|
|
}
|
|
|
|
private:
|
|
nsTArray<TableUpdate*>* mUpdatesArrayRef;
|
|
nsTArray<UniquePtr<TableUpdate>> mUpdatesPointerHolder;
|
|
};
|
|
|
|
} // End of unnamed namespace.
|
|
|
|
void
|
|
Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
|
|
{
|
|
tables.Clear();
|
|
|
|
nsACString::const_iterator begin, iter, end;
|
|
str.BeginReading(begin);
|
|
str.EndReading(end);
|
|
while (begin != end) {
|
|
iter = begin;
|
|
FindCharInReadable(',', iter, end);
|
|
nsDependentCSubstring table = Substring(begin,iter);
|
|
if (!table.IsEmpty()) {
|
|
tables.AppendElement(Substring(begin, iter));
|
|
}
|
|
begin = iter;
|
|
if (begin != end) {
|
|
begin++;
|
|
}
|
|
}
|
|
}
|
|
|
|
nsresult
|
|
Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
|
|
const nsACString& aTableName,
|
|
const nsACString& aProvider,
|
|
nsIFile** aPrivateStoreDirectory)
|
|
{
|
|
NS_ENSURE_ARG_POINTER(aPrivateStoreDirectory);
|
|
|
|
if (!StringEndsWith(aTableName, NS_LITERAL_CSTRING("-proto"))) {
|
|
// Only V4 table names (ends with '-proto') would be stored
|
|
// to per-provider sub-directory.
|
|
nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
|
|
return NS_OK;
|
|
}
|
|
|
|
if (aProvider.IsEmpty()) {
|
|
// When failing to get provider, just store in the root folder.
|
|
nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
|
|
return NS_OK;
|
|
}
|
|
|
|
nsCOMPtr<nsIFile> providerDirectory;
|
|
|
|
// Clone first since we are gonna create a new directory.
|
|
nsresult rv = aRootStoreDirectory->Clone(getter_AddRefs(providerDirectory));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Append the provider name to the root store directory.
|
|
rv = providerDirectory->AppendNative(aProvider);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Ensure existence of the provider directory.
|
|
bool dirExists;
|
|
rv = providerDirectory->Exists(&dirExists);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (!dirExists) {
|
|
LOG(("Creating private directory for %s", nsCString(aTableName).get()));
|
|
rv = providerDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
providerDirectory.forget(aPrivateStoreDirectory);
|
|
return rv;
|
|
}
|
|
|
|
// Store directory exists. Check if it's a directory.
|
|
bool isDir;
|
|
rv = providerDirectory->IsDirectory(&isDir);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
if (!isDir) {
|
|
return NS_ERROR_FILE_DESTINATION_NOT_DIR;
|
|
}
|
|
|
|
providerDirectory.forget(aPrivateStoreDirectory);
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
Classifier::Classifier()
|
|
: mIsTableRequestResultOutdated(true)
|
|
, mUpdateInterrupted(true)
|
|
{
|
|
NS_NewNamedThread(NS_LITERAL_CSTRING("Classifier Update"),
|
|
getter_AddRefs(mUpdateThread));
|
|
}
|
|
|
|
Classifier::~Classifier()
|
|
{
|
|
Close();
|
|
}
|
|
|
|
nsresult
|
|
Classifier::SetupPathNames()
|
|
{
|
|
// Get the root directory where to store all the databases.
|
|
nsresult rv = mCacheDirectory->Clone(getter_AddRefs(mRootStoreDirectory));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
rv = mRootStoreDirectory->AppendNative(STORE_DIRECTORY);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Make sure LookupCaches (which are persistent and survive updates)
|
|
// are reading/writing in the right place. We will be moving their
|
|
// files "underneath" them during backup/restore.
|
|
for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
|
|
mLookupCaches[i]->UpdateRootDirHandle(mRootStoreDirectory);
|
|
}
|
|
|
|
// Directory where to move a backup before an update.
|
|
rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Directory where to be working on the update.
|
|
rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Directory where to move the backup so we can atomically
|
|
// delete (really move) it.
|
|
rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::CreateStoreDirectory()
|
|
{
|
|
// Ensure the safebrowsing directory exists.
|
|
bool storeExists;
|
|
nsresult rv = mRootStoreDirectory->Exists(&storeExists);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (!storeExists) {
|
|
rv = mRootStoreDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
} else {
|
|
bool storeIsDir;
|
|
rv = mRootStoreDirectory->IsDirectory(&storeIsDir);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
if (!storeIsDir)
|
|
return NS_ERROR_FILE_DESTINATION_NOT_DIR;
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::Open(nsIFile& aCacheDirectory)
|
|
{
|
|
// Remember the Local profile directory.
|
|
nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Create the handles to the update and backup directories.
|
|
rv = SetupPathNames();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Clean up any to-delete directories that haven't been deleted yet.
|
|
// This is still required for backward compatibility.
|
|
rv = CleanToDelete();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// If we met a crash during the previous update, "safebrowsing-updating"
|
|
// directory will exist and let's remove it.
|
|
rv = mUpdatingDirectory->Remove(true);
|
|
if (NS_SUCCEEDED(rv)) {
|
|
// If the "safebrowsing-updating" exists, it implies a crash occurred
|
|
// in the previous update.
|
|
LOG(("We may have hit a crash in the previous update."));
|
|
}
|
|
|
|
// Check whether we have an incomplete update and recover from the
|
|
// backup if so.
|
|
rv = RecoverBackups();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Make sure the main store directory exists.
|
|
rv = CreateStoreDirectory();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Build the list of know urlclassifier lists
|
|
// XXX: Disk IO potentially on the main thread during startup
|
|
RegenActiveTables();
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
void
|
|
Classifier::Close()
|
|
{
|
|
DropStores();
|
|
}
|
|
|
|
void
|
|
Classifier::Reset()
|
|
{
|
|
MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
|
|
"Reset() MUST NOT be called on update thread");
|
|
|
|
LOG(("Reset() is called so we interrupt the update."));
|
|
mUpdateInterrupted = true;
|
|
|
|
auto resetFunc = [=] {
|
|
DropStores();
|
|
|
|
mRootStoreDirectory->Remove(true);
|
|
mBackupDirectory->Remove(true);
|
|
mUpdatingDirectory->Remove(true);
|
|
mToDeleteDirectory->Remove(true);
|
|
|
|
CreateStoreDirectory();
|
|
|
|
RegenActiveTables();
|
|
};
|
|
|
|
if (!mUpdateThread) {
|
|
LOG(("Async update has been disabled. Just Reset() on worker thread."));
|
|
resetFunc();
|
|
return;
|
|
}
|
|
|
|
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(resetFunc);
|
|
SyncRunnable::DispatchToThread(mUpdateThread, r);
|
|
}
|
|
|
|
void
|
|
Classifier::ResetTables(ClearType aType, const nsTArray<nsCString>& aTables)
|
|
{
|
|
for (uint32_t i = 0; i < aTables.Length(); i++) {
|
|
LOG(("Resetting table: %s", aTables[i].get()));
|
|
LookupCache *cache = GetLookupCache(aTables[i]);
|
|
if (cache) {
|
|
// Remove any cached Completes for this table if clear type is Clear_Cache
|
|
if (aType == Clear_Cache) {
|
|
cache->ClearCache();
|
|
} else {
|
|
cache->ClearAll();
|
|
}
|
|
}
|
|
}
|
|
|
|
// Clear on-disk database if clear type is Clear_All
|
|
if (aType == Clear_All) {
|
|
DeleteTables(mRootStoreDirectory, aTables);
|
|
|
|
RegenActiveTables();
|
|
}
|
|
}
|
|
|
|
void
|
|
Classifier::DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables)
|
|
{
|
|
nsCOMPtr<nsISimpleEnumerator> entries;
|
|
nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
|
|
NS_ENSURE_SUCCESS_VOID(rv);
|
|
|
|
bool hasMore;
|
|
while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
|
|
nsCOMPtr<nsISupports> supports;
|
|
rv = entries->GetNext(getter_AddRefs(supports));
|
|
NS_ENSURE_SUCCESS_VOID(rv);
|
|
|
|
nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
|
|
NS_ENSURE_TRUE_VOID(file);
|
|
|
|
// If |file| is a directory, recurse to find its entries as well.
|
|
bool isDirectory;
|
|
if (NS_FAILED(file->IsDirectory(&isDirectory))) {
|
|
continue;
|
|
}
|
|
if (isDirectory) {
|
|
DeleteTables(file, aTables);
|
|
continue;
|
|
}
|
|
|
|
nsCString leafName;
|
|
rv = file->GetNativeLeafName(leafName);
|
|
NS_ENSURE_SUCCESS_VOID(rv);
|
|
|
|
leafName.Truncate(leafName.RFind("."));
|
|
if (aTables.Contains(leafName)) {
|
|
if (NS_FAILED(file->Remove(false))) {
|
|
NS_WARNING(nsPrintfCString("Fail to remove file %s from the disk",
|
|
leafName.get()).get());
|
|
}
|
|
}
|
|
}
|
|
NS_ENSURE_SUCCESS_VOID(rv);
|
|
}
|
|
|
|
void
|
|
Classifier::TableRequest(nsACString& aResult)
|
|
{
|
|
MOZ_ASSERT(!NS_IsMainThread(),
|
|
"TableRequest must be called on the classifier worker thread.");
|
|
|
|
// This function and all disk I/O are guaranteed to occur
|
|
// on the same thread so we don't need to add a lock around.
|
|
if (!mIsTableRequestResultOutdated) {
|
|
aResult = mTableRequestResult;
|
|
return;
|
|
}
|
|
|
|
// Generating v2 table info.
|
|
nsTArray<nsCString> tables;
|
|
ActiveTables(tables);
|
|
for (uint32_t i = 0; i < tables.Length(); i++) {
|
|
HashStore store(tables[i], GetProvider(tables[i]), mRootStoreDirectory);
|
|
|
|
nsresult rv = store.Open();
|
|
if (NS_FAILED(rv)) {
|
|
continue;
|
|
}
|
|
|
|
ChunkSet &adds = store.AddChunks();
|
|
ChunkSet &subs = store.SubChunks();
|
|
|
|
// Open HashStore will always succeed even that is not a v2 table.
|
|
// So skip tables without add and sub chunks.
|
|
if (adds.Length() == 0 && subs.Length() == 0) {
|
|
continue;
|
|
}
|
|
|
|
aResult.Append(store.TableName());
|
|
aResult.Append(';');
|
|
|
|
if (adds.Length() > 0) {
|
|
aResult.AppendLiteral("a:");
|
|
nsAutoCString addList;
|
|
adds.Serialize(addList);
|
|
aResult.Append(addList);
|
|
}
|
|
|
|
if (subs.Length() > 0) {
|
|
if (adds.Length() > 0)
|
|
aResult.Append(':');
|
|
aResult.AppendLiteral("s:");
|
|
nsAutoCString subList;
|
|
subs.Serialize(subList);
|
|
aResult.Append(subList);
|
|
}
|
|
|
|
aResult.Append('\n');
|
|
}
|
|
|
|
// Load meta data from *.metadata files in the root directory.
|
|
// Specifically for v4 tables.
|
|
nsCString metadata;
|
|
nsresult rv = LoadMetadata(mRootStoreDirectory, metadata);
|
|
if (NS_SUCCEEDED(rv)) {
|
|
aResult.Append(metadata);
|
|
}
|
|
|
|
// Update the TableRequest result in-memory cache.
|
|
mTableRequestResult = aResult;
|
|
mIsTableRequestResultOutdated = false;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::Check(const nsACString& aSpec,
|
|
const nsACString& aTables,
|
|
LookupResultArray& aResults)
|
|
{
|
|
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_CHECK_TIME> timer;
|
|
|
|
// Get the set of fragments based on the url. This is necessary because we
|
|
// only look up at most 5 URLs per aSpec, even if aSpec has more than 5
|
|
// components.
|
|
nsTArray<nsCString> fragments;
|
|
nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
nsTArray<nsCString> activeTables;
|
|
SplitTables(aTables, activeTables);
|
|
|
|
nsTArray<LookupCache*> cacheArray;
|
|
for (uint32_t i = 0; i < activeTables.Length(); i++) {
|
|
LOG(("Checking table %s", activeTables[i].get()));
|
|
LookupCache *cache = GetLookupCache(activeTables[i]);
|
|
if (cache) {
|
|
cacheArray.AppendElement(cache);
|
|
} else {
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
}
|
|
|
|
// Only record telemetry when both v2 and v4 have data.
|
|
bool isV2Empty = true, isV4Empty = true;
|
|
bool shouldDoTelemetry = false;
|
|
for (auto&& cache : cacheArray) {
|
|
bool& ref = LookupCache::Cast<LookupCacheV2>(cache) ? isV2Empty : isV4Empty;
|
|
ref = ref ? cache->IsEmpty() : false;
|
|
if (!isV2Empty && !isV4Empty) {
|
|
shouldDoTelemetry = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Now check each lookup fragment against the entries in the DB.
|
|
for (uint32_t i = 0; i < fragments.Length(); i++) {
|
|
Completion lookupHash;
|
|
lookupHash.FromPlaintext(fragments[i], mCryptoHash);
|
|
|
|
if (LOG_ENABLED()) {
|
|
nsAutoCString checking;
|
|
lookupHash.ToHexString(checking);
|
|
LOG(("Checking fragment %s, hash %s (%X)", fragments[i].get(),
|
|
checking.get(), lookupHash.ToUint32()));
|
|
}
|
|
|
|
for (uint32_t i = 0; i < cacheArray.Length(); i++) {
|
|
LookupCache *cache = cacheArray[i];
|
|
bool has, confirmed;
|
|
uint32_t matchLength;
|
|
|
|
rv = cache->Has(lookupHash, &has, &matchLength, &confirmed);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (has) {
|
|
LookupResult *result = aResults.AppendElement();
|
|
if (!result)
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
|
|
LOG(("Found a result in %s: %s",
|
|
cache->TableName().get(),
|
|
confirmed ? "confirmed." : "Not confirmed."));
|
|
|
|
result->hash.complete = lookupHash;
|
|
result->mConfirmed = confirmed;
|
|
result->mTableName.Assign(cache->TableName());
|
|
result->mPartialHashLength = confirmed ? COMPLETE_SIZE : matchLength;
|
|
result->mProtocolV2 = LookupCache::Cast<LookupCacheV2>(cache);
|
|
|
|
// There are two cases we are going to ignore the result for telemetry:
|
|
// 1. shouldDoTelemetry == false(when either v2 or v4 table is empty)
|
|
// 2. When match was found in the table which is not provided by google.
|
|
if (!shouldDoTelemetry ||
|
|
!StringBeginsWith(result->mTableName, NS_LITERAL_CSTRING("goog"))) {
|
|
continue;
|
|
}
|
|
|
|
result->mMatchResult = result->mProtocolV2 ?
|
|
MatchResult::eV2Prefix : MatchResult::eV4Prefix;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we cannot find the prefix in neither the v2 nor the v4 database, record the
|
|
// telemetry here because we won't reach nsUrlClassifierLookupCallback:::HandleResult.
|
|
if (shouldDoTelemetry && aResults.Length() == 0) {
|
|
Telemetry::Accumulate(Telemetry::URLCLASSIFIER_MATCH_RESULT,
|
|
static_cast<uint8_t>(MatchResult::eNoMatch));
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
static nsresult
|
|
SwapDirectoryContent(nsIFile* aDir1,
|
|
nsIFile* aDir2,
|
|
nsIFile* aParentDir,
|
|
nsIFile* aTempDir)
|
|
{
|
|
// Pre-condition: |aDir1| and |aDir2| are directory and their parent
|
|
// are both |aParentDir|.
|
|
//
|
|
// Post-condition: The locations where aDir1 and aDir2 point to will not
|
|
// change but their contents will be exchanged. If we failed
|
|
// to swap their content, everything will be rolled back.
|
|
|
|
nsAutoCString tempDirName;
|
|
aTempDir->GetNativeLeafName(tempDirName);
|
|
|
|
nsresult rv;
|
|
|
|
nsAutoCString dirName1, dirName2;
|
|
aDir1->GetNativeLeafName(dirName1);
|
|
aDir2->GetNativeLeafName(dirName2);
|
|
|
|
LOG(("Swapping directories %s and %s...", dirName1.get(),
|
|
dirName2.get()));
|
|
|
|
// 1. Rename "dirName1" to "temp"
|
|
rv = aDir1->RenameToNative(nullptr, tempDirName);
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Unable to rename %s to %s", dirName1.get(),
|
|
tempDirName.get()));
|
|
return rv; // Nothing to roll back.
|
|
}
|
|
|
|
// 1.1. Create a handle for temp directory. This is required since
|
|
// |nsIFile.rename| will not change the location where the
|
|
// object points to.
|
|
nsCOMPtr<nsIFile> tempDirectory;
|
|
rv = aParentDir->Clone(getter_AddRefs(tempDirectory));
|
|
rv = tempDirectory->AppendNative(tempDirName);
|
|
|
|
// 2. Rename "dirName2" to "dirName1".
|
|
rv = aDir2->RenameToNative(nullptr, dirName1);
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to rename %s to %s. Rename temp directory back to %s",
|
|
dirName2.get(), dirName1.get(), dirName1.get()));
|
|
nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
|
|
NS_ENSURE_SUCCESS(rbrv, rbrv);
|
|
return rv;
|
|
}
|
|
|
|
// 3. Rename "temp" to "dirName2".
|
|
rv = tempDirectory->RenameToNative(nullptr, dirName2);
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to rename temp directory to %s. ", dirName2.get()));
|
|
// We've done (1) renaming "dir1 to temp" and
|
|
// (2) renaming "dir2 to dir1"
|
|
// so the rollback is
|
|
// (1) renaming "dir1 to dir2" and
|
|
// (2) renaming "temp to dir1"
|
|
nsresult rbrv; // rollback result
|
|
rbrv = aDir1->RenameToNative(nullptr, dirName2);
|
|
NS_ENSURE_SUCCESS(rbrv, rbrv);
|
|
rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
|
|
NS_ENSURE_SUCCESS(rbrv, rbrv);
|
|
return rv;
|
|
}
|
|
|
|
return rv;
|
|
}
|
|
|
|
void
|
|
Classifier::RemoveUpdateIntermediaries()
|
|
{
|
|
// Remove old LookupCaches.
|
|
for (auto c: mNewLookupCaches) {
|
|
delete c;
|
|
}
|
|
mNewLookupCaches.Clear();
|
|
|
|
// Remove the "old" directory. (despite its looking-new name)
|
|
if (NS_FAILED(mUpdatingDirectory->Remove(true))) {
|
|
// If the directory is locked from removal for some reason,
|
|
// we will fail here and it doesn't matter until the next
|
|
// update. (the next udpate will fail due to the removable
|
|
// "safebrowsing-udpating" directory.)
|
|
LOG(("Failed to remove updating directory."));
|
|
}
|
|
}
|
|
|
|
void
|
|
Classifier::CopyFullHashCacheToNewLookupCache(LookupCache* aNewLookupCache)
|
|
{
|
|
MOZ_ASSERT(aNewLookupCache);
|
|
|
|
for (auto c: mLookupCaches) {
|
|
if (c->TableName() == aNewLookupCache->TableName()) {
|
|
aNewLookupCache->CopyFullHashCache(c);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
Classifier::MergeNewLookupCaches()
|
|
{
|
|
MOZ_ASSERT(NS_GetCurrentThread() != mUpdateThread,
|
|
"MergeNewLookupCaches cannot be called on update thread "
|
|
"since it mutates mLookupCaches which is only safe on "
|
|
"worker thread.");
|
|
|
|
for (auto& newCache: mNewLookupCaches) {
|
|
// For each element in mNewLookCaches, it will be swapped with
|
|
// - An old cache in mLookupCache with the same table name or
|
|
// - nullptr (mLookupCache will be expaned) otherwise.
|
|
size_t swapIndex = 0;
|
|
for (; swapIndex < mLookupCaches.Length(); swapIndex++) {
|
|
if (mLookupCaches[swapIndex]->TableName() == newCache->TableName()) {
|
|
break;
|
|
}
|
|
}
|
|
if (swapIndex == mLookupCaches.Length()) {
|
|
mLookupCaches.AppendElement(nullptr);
|
|
}
|
|
|
|
Swap(mLookupCaches[swapIndex], newCache);
|
|
mLookupCaches[swapIndex]->UpdateRootDirHandle(mRootStoreDirectory);
|
|
}
|
|
|
|
// At this point, mNewLookupCaches's length remains the same but
|
|
// will contain either old cache (override) or nullptr (append).
|
|
}
|
|
|
|
nsresult
|
|
Classifier::SwapInNewTablesAndCleanup()
|
|
{
|
|
nsresult rv;
|
|
|
|
// Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup"
|
|
// as the intermediary directory is we can get databases recovered if
|
|
// crash occurred in any step of the swap. (We will recover from
|
|
// "safebrowsing-backup" in OpenDb().)
|
|
rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables
|
|
mRootStoreDirectory, // contains old tables
|
|
mCacheDirectory, // common parent dir
|
|
mBackupDirectory); // intermediary dir for swap
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to swap in on-disk tables."));
|
|
RemoveUpdateIntermediaries();
|
|
return rv;
|
|
}
|
|
|
|
// Step 2. Merge mNewLookupCaches into mLookupCaches. The outdated
|
|
// LookupCaches will be stored in mNewLookupCaches and be cleaned
|
|
// up later.
|
|
MergeNewLookupCaches();
|
|
|
|
// Step 3. Re-generate active tables based on on-disk tables.
|
|
rv = RegenActiveTables();
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to re-generate active tables!"));
|
|
}
|
|
|
|
// Step 4. Clean up intermediaries for update.
|
|
RemoveUpdateIntermediaries();
|
|
|
|
// Step 5. Invalidate cached tableRequest request.
|
|
mIsTableRequestResultOutdated = true;
|
|
|
|
LOG(("Done swap in updated tables."));
|
|
|
|
return rv;
|
|
}
|
|
|
|
void Classifier::FlushAndDisableAsyncUpdate()
|
|
{
|
|
LOG(("Classifier::FlushAndDisableAsyncUpdate [%p, %p]", this, mUpdateThread.get()));
|
|
|
|
if (!mUpdateThread) {
|
|
LOG(("Async update has been disabled."));
|
|
return;
|
|
}
|
|
|
|
mUpdateThread->Shutdown();
|
|
mUpdateThread = nullptr;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::AsyncApplyUpdates(nsTArray<TableUpdate*>* aUpdates,
|
|
const AsyncUpdateCallback& aCallback)
|
|
{
|
|
LOG(("Classifier::AsyncApplyUpdates"));
|
|
|
|
if (!mUpdateThread) {
|
|
LOG(("Async update has already been disabled."));
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
// Caller thread | Update thread
|
|
// --------------------------------------------------------
|
|
// | ApplyUpdatesBackground
|
|
// (processing other task) | (bg-update done. ping back to caller thread)
|
|
// (processing other task) | idle...
|
|
// ApplyUpdatesForeground |
|
|
// callback |
|
|
|
|
mUpdateInterrupted = false;
|
|
nsresult rv = mRootStoreDirectory->Clone(getter_AddRefs(mRootStoreDirectoryForUpdate));
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to clone mRootStoreDirectory for update."));
|
|
return rv;
|
|
}
|
|
|
|
nsCOMPtr<nsIThread> callerThread = NS_GetCurrentThread();
|
|
MOZ_ASSERT(callerThread != mUpdateThread);
|
|
|
|
nsCOMPtr<nsIRunnable> bgRunnable = NS_NewRunnableFunction([=] {
|
|
MOZ_ASSERT(NS_GetCurrentThread() == mUpdateThread, "MUST be on update thread");
|
|
|
|
LOG(("Step 1. ApplyUpdatesBackground on update thread."));
|
|
nsCString failedTableName;
|
|
nsresult bgRv = ApplyUpdatesBackground(aUpdates, failedTableName);
|
|
|
|
nsCOMPtr<nsIRunnable> fgRunnable = NS_NewRunnableFunction([=] {
|
|
MOZ_ASSERT(NS_GetCurrentThread() == callerThread, "MUST be on caller thread");
|
|
|
|
LOG(("Step 2. ApplyUpdatesForeground on caller thread"));
|
|
nsresult rv = ApplyUpdatesForeground(bgRv, failedTableName);;
|
|
|
|
LOG(("Step 3. Updates applied! Fire callback."));
|
|
|
|
aCallback(rv);
|
|
});
|
|
callerThread->Dispatch(fgRunnable, NS_DISPATCH_NORMAL);
|
|
});
|
|
|
|
return mUpdateThread->Dispatch(bgRunnable, NS_DISPATCH_NORMAL);
|
|
}
|
|
|
|
nsresult
|
|
Classifier::ApplyUpdatesBackground(nsTArray<TableUpdate*>* aUpdates,
|
|
nsACString& aFailedTableName)
|
|
{
|
|
// |mUpdateInterrupted| is guaranteed to have been unset.
|
|
// If |mUpdateInterrupted| is set at any point, Reset() must have
|
|
// been called then we need to interrupt the update process.
|
|
// We only add checkpoints for non-trivial tasks.
|
|
|
|
if (!aUpdates || aUpdates->Length() == 0) {
|
|
return NS_OK;
|
|
}
|
|
|
|
nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
|
|
do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
|
|
|
|
nsCString provider;
|
|
// Assume all TableUpdate objects should have the same provider.
|
|
urlUtil->GetTelemetryProvider((*aUpdates)[0]->TableName(), provider);
|
|
|
|
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_KEYED_UPDATE_TIME>
|
|
keyedTimer(provider);
|
|
|
|
PRIntervalTime clockStart = 0;
|
|
if (LOG_ENABLED()) {
|
|
clockStart = PR_IntervalNow();
|
|
}
|
|
|
|
nsresult rv;
|
|
|
|
{
|
|
ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
|
|
|
|
{
|
|
// Check point 1: Copying file takes time so we check here.
|
|
if (mUpdateInterrupted) {
|
|
LOG(("Update is interrupted. Don't copy files."));
|
|
return NS_OK;
|
|
}
|
|
|
|
rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup.
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to copy in-use directory for update."));
|
|
return rv;
|
|
}
|
|
}
|
|
|
|
LOG(("Applying %" PRIuSIZE " table updates.", aUpdates->Length()));
|
|
|
|
for (uint32_t i = 0; i < aUpdates->Length(); i++) {
|
|
// Previous UpdateHashStore() may have consumed this update..
|
|
if ((*aUpdates)[i]) {
|
|
// Run all updates for one table
|
|
nsCString updateTable(aUpdates->ElementAt(i)->TableName());
|
|
|
|
// Check point 2: Processing downloaded data takes time.
|
|
if (mUpdateInterrupted) {
|
|
LOG(("Update is interrupted. Stop building new tables."));
|
|
return NS_OK;
|
|
}
|
|
|
|
// Will update the mirrored in-memory and on-disk databases.
|
|
if (TableUpdate::Cast<TableUpdateV2>((*aUpdates)[i])) {
|
|
rv = UpdateHashStore(aUpdates, updateTable);
|
|
} else {
|
|
rv = UpdateTableV4(aUpdates, updateTable);
|
|
}
|
|
|
|
if (NS_FAILED(rv)) {
|
|
aFailedTableName = updateTable;
|
|
RemoveUpdateIntermediaries();
|
|
return rv;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // End of scopedUpdatesClearer scope.
|
|
|
|
if (LOG_ENABLED()) {
|
|
PRIntervalTime clockEnd = PR_IntervalNow();
|
|
LOG(("update took %dms\n",
|
|
PR_IntervalToMilliseconds(clockEnd - clockStart)));
|
|
}
|
|
|
|
return rv;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::ApplyUpdatesForeground(nsresult aBackgroundRv,
|
|
const nsACString& aFailedTableName)
|
|
{
|
|
if (mUpdateInterrupted) {
|
|
LOG(("Update is interrupted! Just remove update intermediaries."));
|
|
RemoveUpdateIntermediaries();
|
|
return NS_OK;
|
|
}
|
|
if (NS_SUCCEEDED(aBackgroundRv)) {
|
|
return SwapInNewTablesAndCleanup();
|
|
}
|
|
if (NS_ERROR_OUT_OF_MEMORY != aBackgroundRv) {
|
|
ResetTables(Clear_All, nsTArray<nsCString> { nsCString(aFailedTableName) });
|
|
}
|
|
return aBackgroundRv;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::ApplyFullHashes(nsTArray<TableUpdate*>* aUpdates)
|
|
{
|
|
LOG(("Applying %" PRIuSIZE " table gethashes.", aUpdates->Length()));
|
|
|
|
ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
|
|
for (uint32_t i = 0; i < aUpdates->Length(); i++) {
|
|
TableUpdate *update = aUpdates->ElementAt(i);
|
|
|
|
nsresult rv = UpdateCache(update);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
aUpdates->ElementAt(i) = nullptr;
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
void
|
|
Classifier::GetCacheInfo(const nsACString& aTable,
|
|
nsIUrlClassifierCacheInfo** aCache)
|
|
{
|
|
LookupCache* lookupCache = GetLookupCache(aTable);
|
|
if (!lookupCache) {
|
|
return;
|
|
}
|
|
|
|
lookupCache->GetCacheInfo(aCache);
|
|
}
|
|
|
|
void
|
|
Classifier::DropStores()
|
|
{
|
|
for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
|
|
delete mLookupCaches[i];
|
|
}
|
|
mLookupCaches.Clear();
|
|
}
|
|
|
|
nsresult
|
|
Classifier::RegenActiveTables()
|
|
{
|
|
mActiveTablesCache.Clear();
|
|
|
|
nsTArray<nsCString> foundTables;
|
|
ScanStoreDir(mRootStoreDirectory, foundTables);
|
|
|
|
for (uint32_t i = 0; i < foundTables.Length(); i++) {
|
|
nsCString table(foundTables[i]);
|
|
|
|
LookupCache *lookupCache = GetLookupCache(table);
|
|
if (!lookupCache) {
|
|
continue;
|
|
}
|
|
|
|
if (!lookupCache->IsPrimed()) {
|
|
continue;
|
|
}
|
|
|
|
if (LookupCache::Cast<LookupCacheV4>(lookupCache)) {
|
|
LOG(("Active v4 table: %s", table.get()));
|
|
} else {
|
|
HashStore store(table, GetProvider(table), mRootStoreDirectory);
|
|
|
|
nsresult rv = store.Open();
|
|
if (NS_FAILED(rv)) {
|
|
continue;
|
|
}
|
|
|
|
const ChunkSet &adds = store.AddChunks();
|
|
const ChunkSet &subs = store.SubChunks();
|
|
|
|
if (adds.Length() == 0 && subs.Length() == 0) {
|
|
continue;
|
|
}
|
|
|
|
LOG(("Active v2 table: %s", store.TableName().get()));
|
|
}
|
|
|
|
mActiveTablesCache.AppendElement(table);
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::ScanStoreDir(nsIFile* aDirectory, nsTArray<nsCString>& aTables)
|
|
{
|
|
nsCOMPtr<nsISimpleEnumerator> entries;
|
|
nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
bool hasMore;
|
|
while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
|
|
nsCOMPtr<nsISupports> supports;
|
|
rv = entries->GetNext(getter_AddRefs(supports));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
|
|
|
|
// If |file| is a directory, recurse to find its entries as well.
|
|
bool isDirectory;
|
|
if (NS_FAILED(file->IsDirectory(&isDirectory))) {
|
|
continue;
|
|
}
|
|
if (isDirectory) {
|
|
ScanStoreDir(file, aTables);
|
|
continue;
|
|
}
|
|
|
|
nsCString leafName;
|
|
rv = file->GetNativeLeafName(leafName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Both v2 and v4 contain .pset file
|
|
nsCString suffix(NS_LITERAL_CSTRING(".pset"));
|
|
|
|
int32_t dot = leafName.RFind(suffix, 0);
|
|
if (dot != -1) {
|
|
leafName.Cut(dot, suffix.Length());
|
|
aTables.AppendElement(leafName);
|
|
}
|
|
}
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::ActiveTables(nsTArray<nsCString>& aTables)
|
|
{
|
|
aTables = mActiveTablesCache;
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::CleanToDelete()
|
|
{
|
|
bool exists;
|
|
nsresult rv = mToDeleteDirectory->Exists(&exists);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (exists) {
|
|
rv = mToDeleteDirectory->Remove(true);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
|
|
|
|
already_AddRefed<nsIFile>
|
|
Classifier::GetFailedUpdateDirectroy()
|
|
{
|
|
nsCString failedUpdatekDirName = STORE_DIRECTORY + nsCString("-failedupdate");
|
|
|
|
nsCOMPtr<nsIFile> failedUpdatekDirectory;
|
|
if (NS_FAILED(mCacheDirectory->Clone(getter_AddRefs(failedUpdatekDirectory))) ||
|
|
NS_FAILED(failedUpdatekDirectory->AppendNative(failedUpdatekDirName))) {
|
|
LOG(("Failed to init failedUpdatekDirectory."));
|
|
return nullptr;
|
|
}
|
|
|
|
return failedUpdatekDirectory.forget();
|
|
}
|
|
|
|
nsresult
|
|
Classifier::DumpRawTableUpdates(const nsACString& aRawUpdates)
|
|
{
|
|
LOG(("Dumping raw table updates..."));
|
|
|
|
DumpFailedUpdate();
|
|
|
|
nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy();
|
|
|
|
// Create tableupdate.bin and dump raw table update data.
|
|
nsCOMPtr<nsIFile> rawTableUpdatesFile;
|
|
nsCOMPtr<nsIOutputStream> outputStream;
|
|
if (NS_FAILED(failedUpdatekDirectory->Clone(getter_AddRefs(rawTableUpdatesFile))) ||
|
|
NS_FAILED(rawTableUpdatesFile->AppendNative(nsCString("tableupdates.bin"))) ||
|
|
NS_FAILED(NS_NewLocalFileOutputStream(getter_AddRefs(outputStream),
|
|
rawTableUpdatesFile,
|
|
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE))) {
|
|
LOG(("Failed to create file to dump raw table updates."));
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
// Write out the data.
|
|
uint32_t written;
|
|
nsresult rv = outputStream->Write(aRawUpdates.BeginReading(),
|
|
aRawUpdates.Length(), &written);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
NS_ENSURE_TRUE(written == aRawUpdates.Length(), NS_ERROR_FAILURE);
|
|
|
|
return rv;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::DumpFailedUpdate()
|
|
{
|
|
LOG(("Dumping failed update..."));
|
|
|
|
nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy();
|
|
|
|
// Remove the "failed update" directory no matter it exists or not.
|
|
// Failure is fine because the directory may not exist.
|
|
failedUpdatekDirectory->Remove(true);
|
|
|
|
nsCString failedUpdatekDirName;
|
|
nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Copy the in-use directory to a clean "failed update" directory.
|
|
nsCOMPtr<nsIFile> inUseDirectory;
|
|
if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) ||
|
|
NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) {
|
|
LOG(("Failed to move in-use to the \"failed update\" directory %s",
|
|
failedUpdatekDirName.get()));
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
return rv;
|
|
}
|
|
|
|
#endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
|
|
|
|
nsresult
|
|
Classifier::CopyInUseDirForUpdate()
|
|
{
|
|
LOG(("Copy in-use directory content for update."));
|
|
|
|
// We copy everything from in-use directory to a temporary directory
|
|
// for updating.
|
|
|
|
nsCString updatingDirName;
|
|
nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Remove the destination directory first (just in case) the do the copy.
|
|
mUpdatingDirectory->Remove(true);
|
|
if (!mRootStoreDirectoryForUpdate) {
|
|
LOG(("mRootStoreDirectoryForUpdate is null."));
|
|
return NS_ERROR_NULL_POINTER;
|
|
}
|
|
rv = mRootStoreDirectoryForUpdate->CopyToNative(nullptr, updatingDirName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::RecoverBackups()
|
|
{
|
|
bool backupExists;
|
|
nsresult rv = mBackupDirectory->Exists(&backupExists);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (backupExists) {
|
|
// Remove the safebrowsing dir if it exists
|
|
nsCString storeDirName;
|
|
rv = mRootStoreDirectory->GetNativeLeafName(storeDirName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
bool storeExists;
|
|
rv = mRootStoreDirectory->Exists(&storeExists);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (storeExists) {
|
|
rv = mRootStoreDirectory->Remove(true);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
}
|
|
|
|
// Move the backup to the store location
|
|
rv = mBackupDirectory->MoveToNative(nullptr, storeDirName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// mBackupDirectory now points to storeDir, fix up.
|
|
rv = SetupPathNames();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
bool
|
|
Classifier::CheckValidUpdate(nsTArray<TableUpdate*>* aUpdates,
|
|
const nsACString& aTable)
|
|
{
|
|
// take the quick exit if there is no valid update for us
|
|
// (common case)
|
|
uint32_t validupdates = 0;
|
|
|
|
for (uint32_t i = 0; i < aUpdates->Length(); i++) {
|
|
TableUpdate *update = aUpdates->ElementAt(i);
|
|
if (!update || !update->TableName().Equals(aTable))
|
|
continue;
|
|
if (update->Empty()) {
|
|
aUpdates->ElementAt(i) = nullptr;
|
|
continue;
|
|
}
|
|
validupdates++;
|
|
}
|
|
|
|
if (!validupdates) {
|
|
// This can happen if the update was only valid for one table.
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
nsCString
|
|
Classifier::GetProvider(const nsACString& aTableName)
|
|
{
|
|
nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
|
|
do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
|
|
|
|
nsCString provider;
|
|
nsresult rv = urlUtil->GetProvider(aTableName, provider);
|
|
|
|
return NS_SUCCEEDED(rv) ? provider : EmptyCString();
|
|
}
|
|
|
|
/*
|
|
* This will consume+delete updates from the passed nsTArray.
|
|
*/
|
|
nsresult
|
|
Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
|
|
const nsACString& aTable)
|
|
{
|
|
if (nsUrlClassifierDBService::ShutdownHasStarted()) {
|
|
return NS_ERROR_UC_UPDATE_SHUTDOWNING;
|
|
}
|
|
|
|
LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
|
|
|
|
HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory);
|
|
|
|
if (!CheckValidUpdate(aUpdates, store.TableName())) {
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult rv = store.Open();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
rv = store.BeginUpdate();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Read the part of the store that is (only) in the cache
|
|
LookupCacheV2* lookupCache =
|
|
LookupCache::Cast<LookupCacheV2>(GetLookupCacheForUpdate(store.TableName()));
|
|
if (!lookupCache) {
|
|
return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
|
|
}
|
|
|
|
// Clear cache when update
|
|
lookupCache->InvalidateExpiredCacheEntries();
|
|
|
|
FallibleTArray<uint32_t> AddPrefixHashes;
|
|
rv = lookupCache->GetPrefixes(AddPrefixHashes);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
rv = store.AugmentAdds(AddPrefixHashes);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
AddPrefixHashes.Clear();
|
|
|
|
uint32_t applied = 0;
|
|
|
|
for (uint32_t i = 0; i < aUpdates->Length(); i++) {
|
|
TableUpdate *update = aUpdates->ElementAt(i);
|
|
if (!update || !update->TableName().Equals(store.TableName()))
|
|
continue;
|
|
|
|
rv = store.ApplyUpdate(*update);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
applied++;
|
|
|
|
auto updateV2 = TableUpdate::Cast<TableUpdateV2>(update);
|
|
if (updateV2) {
|
|
LOG(("Applied update to table %s:", store.TableName().get()));
|
|
LOG((" %d add chunks", updateV2->AddChunks().Length()));
|
|
LOG((" %" PRIuSIZE " add prefixes", updateV2->AddPrefixes().Length()));
|
|
LOG((" %" PRIuSIZE " add completions", updateV2->AddCompletes().Length()));
|
|
LOG((" %d sub chunks", updateV2->SubChunks().Length()));
|
|
LOG((" %" PRIuSIZE " sub prefixes", updateV2->SubPrefixes().Length()));
|
|
LOG((" %" PRIuSIZE " sub completions", updateV2->SubCompletes().Length()));
|
|
LOG((" %d add expirations", updateV2->AddExpirations().Length()));
|
|
LOG((" %d sub expirations", updateV2->SubExpirations().Length()));
|
|
}
|
|
|
|
aUpdates->ElementAt(i) = nullptr;
|
|
}
|
|
|
|
LOG(("Applied %d update(s) to %s.", applied, store.TableName().get()));
|
|
|
|
rv = store.Rebuild();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
LOG(("Table %s now has:", store.TableName().get()));
|
|
LOG((" %d add chunks", store.AddChunks().Length()));
|
|
LOG((" %" PRIuSIZE " add prefixes", store.AddPrefixes().Length()));
|
|
LOG((" %" PRIuSIZE " add completions", store.AddCompletes().Length()));
|
|
LOG((" %d sub chunks", store.SubChunks().Length()));
|
|
LOG((" %" PRIuSIZE " sub prefixes", store.SubPrefixes().Length()));
|
|
LOG((" %" PRIuSIZE " sub completions", store.SubCompletes().Length()));
|
|
|
|
rv = store.WriteFile();
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// At this point the store is updated and written out to disk, but
|
|
// the data is still in memory. Build our quick-lookup table here.
|
|
rv = lookupCache->Build(store.AddPrefixes(), store.AddCompletes());
|
|
NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_BUILD_PREFIX_FAILURE);
|
|
|
|
#if defined(DEBUG)
|
|
lookupCache->DumpCompletions();
|
|
#endif
|
|
rv = lookupCache->WriteFile();
|
|
NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
|
|
|
|
LOG(("Successfully updated %s", store.TableName().get()));
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::UpdateTableV4(nsTArray<TableUpdate*>* aUpdates,
|
|
const nsACString& aTable)
|
|
{
|
|
MOZ_ASSERT(!NS_IsMainThread(),
|
|
"UpdateTableV4 must be called on the classifier worker thread.");
|
|
if (nsUrlClassifierDBService::ShutdownHasStarted()) {
|
|
return NS_ERROR_UC_UPDATE_SHUTDOWNING;
|
|
}
|
|
|
|
LOG(("Classifier::UpdateTableV4(%s)", PromiseFlatCString(aTable).get()));
|
|
|
|
if (!CheckValidUpdate(aUpdates, aTable)) {
|
|
return NS_OK;
|
|
}
|
|
|
|
LookupCacheV4* lookupCache =
|
|
LookupCache::Cast<LookupCacheV4>(GetLookupCacheForUpdate(aTable));
|
|
if (!lookupCache) {
|
|
return NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND;
|
|
}
|
|
|
|
// Remove cache entries whose negative cache time is expired when update.
|
|
// We don't check if positive cache time is expired here because we want to
|
|
// keep the eviction rule simple when doing an update.
|
|
lookupCache->InvalidateExpiredCacheEntries();
|
|
|
|
nsresult rv = NS_OK;
|
|
|
|
// If there are multiple updates for the same table, prefixes1 & prefixes2
|
|
// will act as input and output in turn to reduce memory copy overhead.
|
|
PrefixStringMap prefixes1, prefixes2;
|
|
PrefixStringMap* input = &prefixes1;
|
|
PrefixStringMap* output = &prefixes2;
|
|
|
|
TableUpdateV4* lastAppliedUpdate = nullptr;
|
|
for (uint32_t i = 0; i < aUpdates->Length(); i++) {
|
|
TableUpdate *update = aUpdates->ElementAt(i);
|
|
if (!update || !update->TableName().Equals(aTable)) {
|
|
continue;
|
|
}
|
|
|
|
auto updateV4 = TableUpdate::Cast<TableUpdateV4>(update);
|
|
NS_ENSURE_TRUE(updateV4, NS_ERROR_UC_UPDATE_TABLE_NOT_FOUND);
|
|
|
|
if (updateV4->IsFullUpdate()) {
|
|
input->Clear();
|
|
output->Clear();
|
|
rv = lookupCache->ApplyUpdate(updateV4, *input, *output);
|
|
if (NS_FAILED(rv)) {
|
|
return rv;
|
|
}
|
|
} else {
|
|
// If both prefix sets are empty, this means we are doing a partial update
|
|
// without a prior full/partial update in the loop. In this case we should
|
|
// get prefixes from the lookup cache first.
|
|
if (prefixes1.IsEmpty() && prefixes2.IsEmpty()) {
|
|
lookupCache->GetPrefixes(prefixes1);
|
|
} else {
|
|
MOZ_ASSERT(prefixes1.IsEmpty() ^ prefixes2.IsEmpty());
|
|
|
|
// When there are multiple partial updates, input should always point
|
|
// to the non-empty prefix set(filled by previous full/partial update).
|
|
// output should always point to the empty prefix set.
|
|
input = prefixes1.IsEmpty() ? &prefixes2 : &prefixes1;
|
|
output = prefixes1.IsEmpty() ? &prefixes1 : &prefixes2;
|
|
}
|
|
|
|
rv = lookupCache->ApplyUpdate(updateV4, *input, *output);
|
|
if (NS_FAILED(rv)) {
|
|
return rv;
|
|
}
|
|
|
|
input->Clear();
|
|
}
|
|
|
|
// Keep track of the last applied update.
|
|
lastAppliedUpdate = updateV4;
|
|
|
|
aUpdates->ElementAt(i) = nullptr;
|
|
}
|
|
|
|
rv = lookupCache->Build(*output);
|
|
NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_BUILD_PREFIX_FAILURE);
|
|
|
|
rv = lookupCache->WriteFile();
|
|
NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
|
|
|
|
if (lastAppliedUpdate) {
|
|
LOG(("Write meta data of the last applied update."));
|
|
rv = lookupCache->WriteMetadata(lastAppliedUpdate);
|
|
NS_ENSURE_SUCCESS(rv, NS_ERROR_UC_UPDATE_FAIL_TO_WRITE_DISK);
|
|
}
|
|
|
|
LOG(("Successfully updated %s\n", PromiseFlatCString(aTable).get()));
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::UpdateCache(TableUpdate* aUpdate)
|
|
{
|
|
if (!aUpdate) {
|
|
return NS_OK;
|
|
}
|
|
|
|
nsAutoCString table(aUpdate->TableName());
|
|
LOG(("Classifier::UpdateCache(%s)", table.get()));
|
|
|
|
LookupCache *lookupCache = GetLookupCache(table);
|
|
if (!lookupCache) {
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
auto lookupV2 = LookupCache::Cast<LookupCacheV2>(lookupCache);
|
|
if (lookupV2) {
|
|
auto updateV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate);
|
|
lookupV2->AddGethashResultToCache(updateV2->AddCompletes(),
|
|
updateV2->MissPrefixes());
|
|
} else {
|
|
auto lookupV4 = LookupCache::Cast<LookupCacheV4>(lookupCache);
|
|
if (!lookupV4) {
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
auto updateV4 = TableUpdate::Cast<TableUpdateV4>(aUpdate);
|
|
lookupV4->AddFullHashResponseToCache(updateV4->FullHashResponse());
|
|
}
|
|
|
|
#if defined(DEBUG)
|
|
lookupCache->DumpCache();
|
|
#endif
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
LookupCache *
|
|
Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate)
|
|
{
|
|
if (aForUpdate) {
|
|
MOZ_ASSERT(NS_GetCurrentThread() == mUpdateThread,
|
|
"GetLookupCache(aForUpdate==true) can only be called on update thread.");
|
|
}
|
|
|
|
nsTArray<LookupCache*>& lookupCaches = aForUpdate ? mNewLookupCaches
|
|
: mLookupCaches;
|
|
auto& rootStoreDirectory = aForUpdate ? mUpdatingDirectory
|
|
: mRootStoreDirectory;
|
|
|
|
for (auto c: lookupCaches) {
|
|
if (c->TableName().Equals(aTable)) {
|
|
return c;
|
|
}
|
|
}
|
|
|
|
// TODO : Bug 1302600, It would be better if we have a more general non-main
|
|
// thread method to convert table name to protocol version. Currently
|
|
// we can only know this by checking if the table name ends with '-proto'.
|
|
UniquePtr<LookupCache> cache;
|
|
nsCString provider = GetProvider(aTable);
|
|
if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
|
|
cache = MakeUnique<LookupCacheV4>(aTable, provider, rootStoreDirectory);
|
|
} else {
|
|
cache = MakeUnique<LookupCacheV2>(aTable, provider, rootStoreDirectory);
|
|
}
|
|
|
|
nsresult rv = cache->Init();
|
|
if (NS_FAILED(rv)) {
|
|
return nullptr;
|
|
}
|
|
rv = cache->Open();
|
|
if (NS_SUCCEEDED(rv)) {
|
|
if (aForUpdate) {
|
|
// Since update algorithm will invalidate expired cache entries, we need
|
|
// to copy fullhash cache in "old LookupCache" to "update Lookupcache" before
|
|
// applying an update.
|
|
CopyFullHashCacheToNewLookupCache(cache.get());
|
|
}
|
|
|
|
lookupCaches.AppendElement(cache.get());
|
|
return cache.release();
|
|
}
|
|
|
|
// At this point we failed to open LookupCache.
|
|
//
|
|
// GetLookupCache for update and for other usage will run on update thread
|
|
// and worker thread respectively (Bug 1339760). Removing stuff only in
|
|
// their own realms potentially increases the concurrency.
|
|
|
|
if (aForUpdate) {
|
|
// Remove intermediaries no matter if it's due to file corruption or not.
|
|
RemoveUpdateIntermediaries();
|
|
return nullptr;
|
|
}
|
|
|
|
// Non-update case.
|
|
if (rv == NS_ERROR_FILE_CORRUPTED) {
|
|
Reset(); // Not including the update intermediaries.
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::ReadNoiseEntries(const Prefix& aPrefix,
|
|
const nsACString& aTableName,
|
|
uint32_t aCount,
|
|
PrefixArray* aNoiseEntries)
|
|
{
|
|
FallibleTArray<uint32_t> prefixes;
|
|
nsresult rv;
|
|
|
|
LookupCache *cache = GetLookupCache(aTableName);
|
|
if (!cache) {
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
LookupCacheV2* cacheV2 = LookupCache::Cast<LookupCacheV2>(cache);
|
|
if (cacheV2) {
|
|
rv = cacheV2->GetPrefixes(prefixes);
|
|
} else {
|
|
rv = LookupCache::Cast<LookupCacheV4>(cache)->GetFixedLengthPrefixes(prefixes);
|
|
}
|
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
if (prefixes.Length() == 0) {
|
|
NS_WARNING("Could not find prefix in PrefixSet during noise lookup");
|
|
return NS_ERROR_FAILURE;
|
|
}
|
|
|
|
// We do not want to simply pick random prefixes, because this would allow
|
|
// averaging out the noise by analysing the traffic from Firefox users.
|
|
// Instead, we ensure the 'noise' is the same for the same prefix by seeding
|
|
// the random number generator with the prefix. We prefer not to use rand()
|
|
// which isn't thread safe, and the reseeding of which could trip up other
|
|
// parts othe code that expect actual random numbers.
|
|
// Here we use a simple LCG (Linear Congruential Generator) to generate
|
|
// random numbers. We seed the LCG with the prefix we are generating noise
|
|
// for.
|
|
// http://en.wikipedia.org/wiki/Linear_congruential_generator
|
|
|
|
uint32_t m = prefixes.Length();
|
|
uint32_t a = aCount % m;
|
|
uint32_t idx = aPrefix.ToUint32() % m;
|
|
|
|
for (size_t i = 0; i < aCount; i++) {
|
|
idx = (a * idx + a) % m;
|
|
|
|
Prefix newPrefix;
|
|
uint32_t hash = prefixes[idx];
|
|
// In the case V4 little endian, we did swapping endian when converting from char* to
|
|
// int, should revert endian to make sure we will send hex string correctly
|
|
// See https://bugzilla.mozilla.org/show_bug.cgi?id=1283007#c23
|
|
if (!cacheV2 && !bool(MOZ_BIG_ENDIAN)) {
|
|
hash = NativeEndian::swapFromBigEndian(prefixes[idx]);
|
|
}
|
|
|
|
newPrefix.FromUint32(hash);
|
|
if (newPrefix != aPrefix) {
|
|
aNoiseEntries->AppendElement(newPrefix);
|
|
}
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult
|
|
Classifier::LoadMetadata(nsIFile* aDirectory, nsACString& aResult)
|
|
{
|
|
nsCOMPtr<nsISimpleEnumerator> entries;
|
|
nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
NS_ENSURE_ARG_POINTER(entries);
|
|
|
|
bool hasMore;
|
|
while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
|
|
nsCOMPtr<nsISupports> supports;
|
|
rv = entries->GetNext(getter_AddRefs(supports));
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
|
|
|
|
// If |file| is a directory, recurse to find its entries as well.
|
|
bool isDirectory;
|
|
if (NS_FAILED(file->IsDirectory(&isDirectory))) {
|
|
continue;
|
|
}
|
|
if (isDirectory) {
|
|
LoadMetadata(file, aResult);
|
|
continue;
|
|
}
|
|
|
|
// Truncate file extension to get the table name.
|
|
nsCString tableName;
|
|
rv = file->GetNativeLeafName(tableName);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
int32_t dot = tableName.RFind(METADATA_SUFFIX, 0);
|
|
if (dot == -1) {
|
|
continue;
|
|
}
|
|
tableName.Cut(dot, METADATA_SUFFIX.Length());
|
|
|
|
LookupCacheV4* lookupCache =
|
|
LookupCache::Cast<LookupCacheV4>(GetLookupCache(tableName));
|
|
if (!lookupCache) {
|
|
continue;
|
|
}
|
|
|
|
nsCString state;
|
|
nsCString checksum;
|
|
rv = lookupCache->LoadMetadata(state, checksum);
|
|
if (NS_FAILED(rv)) {
|
|
LOG(("Failed to get metadata for table %s", tableName.get()));
|
|
continue;
|
|
}
|
|
|
|
// The state might include '\n' so that we have to encode.
|
|
nsAutoCString stateBase64;
|
|
rv = Base64Encode(state, stateBase64);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
nsAutoCString checksumBase64;
|
|
rv = Base64Encode(checksum, checksumBase64);
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
LOG(("Appending state '%s' and checksum '%s' for table %s",
|
|
stateBase64.get(), checksumBase64.get(), tableName.get()));
|
|
|
|
aResult.AppendPrintf("%s;%s:%s\n", tableName.get(),
|
|
stateBase64.get(),
|
|
checksumBase64.get());
|
|
}
|
|
|
|
return rv;
|
|
}
|
|
|
|
} // namespace safebrowsing
|
|
} // namespace mozilla
|