forked from mirrors/gecko-dev
20ab57f9ae
The goal of this patch is to reduce the number of memory reallocation during |MakePrefixSet|[1]. Here is the number of nsTArray memory reallocation occur during |MakePrefixSet| (test in my local platform): googpub-phish-proto: 58k times goog-malware-proto: 9k times goog-unwanted-proto: 25k times goog-badbinurl-proto: 6k times This patch improves the performance by: 1. For tables whose prefixes are less than 128*1024(malware, unwanted, badinurl). Store prefixes directly without dividing allocation into smaller chunks. Because the maximum size to store all the prefixes in a single array for these tables will be less than 512k, we can avoid Bug 1046038. This simplifies the internal prefixset data structure generation and total memory usage is also saved: goog-malware-proto : 437K -> 163k goog-unwanted-proto : 658k -> 446k goog-badbinurl-proto: 320k -> 110k The single largest allocated continuous memory size is: goog-malware-proto : 86k -> 163k goog-unwanted-proto : 86k -> 446k goog-badbinurl-proto: 77k -> 110k A further improvement can be done for this part is for tables with fewer prefixes, we can use an one-dimension delta array to reduce the size of a single continuous memory allocation. 2. For tables with more prefixes: According to experiment, when prefixes are more than 400k the delta arrays have very high chance that are full, in the case of phishing table, we can estimate the capacity accurately before applying delta algorithm. The shortcoming of this part is when prefixes are between 130k~400k, the capacity estimation is not accurate. [1] https://searchfox.org/mozilla-central/rev/b2015fdd464f598d645342614593d4ebda922d95/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp#99 Differential Revision: https://phabricator.services.mozilla.com/D30046 --HG-- extra : moz-landing-system : lando
91 lines
3.1 KiB
C++
91 lines
3.1 KiB
C++
//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
/* vim: set ts=2 et sw=2 tw=80: */
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#ifndef nsUrlClassifierPrefixSet_h_
|
|
#define nsUrlClassifierPrefixSet_h_
|
|
|
|
#include "nsISupportsUtils.h"
|
|
#include "nsID.h"
|
|
#include "nsIFile.h"
|
|
#include "nsIMemoryReporter.h"
|
|
#include "nsIMutableArray.h"
|
|
#include "nsIFileStreams.h"
|
|
#include "nsIUrlClassifierPrefixSet.h"
|
|
#include "nsTArray.h"
|
|
#include "nsToolkitCompsCID.h"
|
|
#include "mozilla/FileUtils.h"
|
|
#include "mozilla/MemoryReporting.h"
|
|
#include "mozilla/Mutex.h"
|
|
#include "mozilla/Poison.h"
|
|
|
|
namespace mozilla {
|
|
namespace safebrowsing {
|
|
|
|
class VariableLengthPrefixSet;
|
|
|
|
} // namespace safebrowsing
|
|
} // namespace mozilla
|
|
|
|
class nsUrlClassifierPrefixSet final : public nsIUrlClassifierPrefixSet,
|
|
public nsIMemoryReporter {
|
|
public:
|
|
nsUrlClassifierPrefixSet();
|
|
|
|
NS_IMETHOD Init(const nsACString& aName) override;
|
|
NS_IMETHOD SetPrefixes(const uint32_t* aArray, uint32_t aLength) override;
|
|
NS_IMETHOD GetPrefixes(uint32_t* aCount, uint32_t** aPrefixes) override;
|
|
NS_IMETHOD Contains(uint32_t aPrefix, bool* aFound) override;
|
|
NS_IMETHOD IsEmpty(bool* aEmpty) override;
|
|
|
|
nsresult GetPrefixesNative(FallibleTArray<uint32_t>& outArray);
|
|
nsresult WritePrefixes(nsCOMPtr<nsIOutputStream>& out) const;
|
|
nsresult LoadPrefixes(nsCOMPtr<nsIInputStream>& in);
|
|
uint32_t CalculatePreallocateSize() const;
|
|
|
|
size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
|
|
|
|
NS_DECL_THREADSAFE_ISUPPORTS
|
|
NS_DECL_NSIMEMORYREPORTER
|
|
|
|
friend class mozilla::safebrowsing::VariableLengthPrefixSet;
|
|
|
|
private:
|
|
virtual ~nsUrlClassifierPrefixSet();
|
|
|
|
static const uint32_t DELTAS_LIMIT = 120;
|
|
static const uint32_t MAX_INDEX_DIFF = (1 << 16);
|
|
static const uint32_t PREFIXSET_VERSION_MAGIC = 1;
|
|
|
|
void Clear();
|
|
nsresult MakePrefixSet(const uint32_t* aArray, uint32_t aLength);
|
|
uint32_t BinSearch(uint32_t start, uint32_t end, uint32_t target) const;
|
|
bool IsEmptyInternal() const;
|
|
|
|
// Lock to prevent races between the url-classifier thread (which does most
|
|
// of the operations) and the main thread (which does memory reporting).
|
|
// It should be held for all operations between Init() and destruction that
|
|
// touch this class's data members.
|
|
mutable mozilla::Mutex mLock;
|
|
// list of fully stored prefixes, that also form the
|
|
// start of a run of deltas in mIndexDeltas.
|
|
nsTArray<uint32_t> mIndexPrefixes;
|
|
// array containing arrays of deltas from indices.
|
|
// Index to the place that matches the closest lower
|
|
// prefix from mIndexPrefix. Then every "delta" corresponds
|
|
// to a prefix in the PrefixSet.
|
|
// This array could be empty when we decide to store all the prefixes
|
|
// in mIndexPrefixes.
|
|
nsTArray<nsTArray<uint16_t> > mIndexDeltas;
|
|
|
|
// how many prefixes we have.
|
|
uint32_t mTotalPrefixes;
|
|
|
|
nsCString mName;
|
|
nsCString mMemoryReportPath;
|
|
mozilla::CorruptionCanary mCanary;
|
|
};
|
|
|
|
#endif
|