Bug 1286798 - Part 25: Add checks for the group and global limit; r=asuth

This commit is contained in:
Jan Varga 2018-11-29 21:48:34 +01:00
parent 480f7ccea1
commit 7981be440b
10 changed files with 536 additions and 39 deletions

View file

@ -190,6 +190,7 @@ dom/grid/**
dom/html/**
dom/ipc/**
dom/jsurl/**
dom/localstorage/**
dom/manifest/**
dom/media/test/**
dom/media/tests/**

View file

@ -294,7 +294,7 @@ BodyMaybeUpdatePaddingSize(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir,
int64_t fileSize = 0;
RefPtr<QuotaObject> quotaObject =
quotaManager->GetQuotaObject(PERSISTENCE_TYPE_DEFAULT, aQuotaInfo.mGroup,
aQuotaInfo.mOrigin, bodyFile, &fileSize);
aQuotaInfo.mOrigin, bodyFile, -1, &fileSize);
MOZ_DIAGNOSTIC_ASSERT(quotaObject);
MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0);
// XXXtt: bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1422815

View file

@ -21,6 +21,7 @@
#include "mozilla/dom/PBackgroundLSSharedTypes.h"
#include "mozilla/dom/PBackgroundLSSimpleRequestParent.h"
#include "mozilla/dom/quota/QuotaManager.h"
#include "mozilla/dom/quota/QuotaObject.h"
#include "mozilla/dom/quota/UsageInfo.h"
#include "mozilla/ipc/BackgroundParent.h"
#include "mozilla/ipc/PBackgroundParent.h"
@ -862,6 +863,7 @@ class Datastore final
{
RefPtr<DirectoryLock> mDirectoryLock;
RefPtr<Connection> mConnection;
RefPtr<QuotaObject> mQuotaObject;
nsCOMPtr<nsITimer> mAutoCommitTimer;
nsCOMPtr<nsIRunnable> mCompleteCallback;
nsTHashtable<nsPtrHashKey<PrepareDatastoreOp>> mPrepareDatastoreOps;
@ -880,6 +882,7 @@ public:
int64_t aUsage,
already_AddRefed<DirectoryLock>&& aDirectoryLock,
already_AddRefed<Connection>&& aConnection,
already_AddRefed<QuotaObject>&& aQuotaObject,
nsDataHashtable<nsStringHashKey, nsString>& aValues);
const nsCString&
@ -1436,6 +1439,10 @@ class PrepareDatastoreOp
bool mRequestedDirectoryLock;
bool mInvalidated;
#ifdef DEBUG
int64_t mDEBUGUsage;
#endif
public:
PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
const LSRequestParams& aParams);
@ -1504,7 +1511,9 @@ private:
DatabaseNotAvailable();
nsresult
EnsureDirectoryEntry(nsIFile* aEntry, bool aDirectory);
EnsureDirectoryEntry(nsIFile* aEntry,
bool aDirectory,
bool* aAlreadyExisted = nullptr);
nsresult
VerifyDatabaseInformation(mozIStorageConnection* aConnection);
@ -1826,6 +1835,11 @@ StaticAutoPtr<ObserverHashtable> gObservers;
Atomic<uint32_t, Relaxed> gOriginLimitKB(kDefaultOriginLimitKB);
typedef nsDataHashtable<nsCStringHashKey, int64_t> UsageHashtable;
// Can only be touched on the Quota Manager I/O thread.
StaticAutoPtr<UsageHashtable> gUsages;
bool
IsOnConnectionThread()
{
@ -1840,6 +1854,19 @@ AssertIsOnConnectionThread()
gConnectionThread->AssertIsOnConnectionThread();
}
void
InitUsageForOrigin(const nsACString& aOrigin, int64_t aUsage)
{
AssertIsOnIOThread();
if (!gUsages) {
gUsages = new UsageHashtable();
}
MOZ_ASSERT(!gUsages->Contains(aOrigin));
gUsages->Put(aOrigin, aUsage);
}
} // namespace
/*******************************************************************************
@ -2623,9 +2650,11 @@ Datastore::Datastore(const nsACString& aOrigin,
int64_t aUsage,
already_AddRefed<DirectoryLock>&& aDirectoryLock,
already_AddRefed<Connection>&& aConnection,
already_AddRefed<QuotaObject>&& aQuotaObject,
nsDataHashtable<nsStringHashKey, nsString>& aValues)
: mDirectoryLock(std::move(aDirectoryLock))
, mConnection(std::move(aConnection))
, mQuotaObject(std::move(aQuotaObject))
, mOrigin(aOrigin)
, mPrivateBrowsingId(aPrivateBrowsingId)
, mUsage(aUsage)
@ -2654,6 +2683,7 @@ Datastore::Close()
if (IsPersistent()) {
MOZ_ASSERT(mConnection);
MOZ_ASSERT(mQuotaObject);
if (mConnection->InTransaction()) {
MOZ_ASSERT(mAutoCommitTimer);
@ -2673,6 +2703,7 @@ Datastore::Close()
mConnection->Close(callback);
} else {
MOZ_ASSERT(!mConnection);
MOZ_ASSERT(!mQuotaObject);
// There's no connection, so it's safe to release the directory lock and
// unregister itself from the hashtable.
@ -2987,8 +3018,12 @@ Datastore::ConnectionClosedCallback()
AssertIsOnBackgroundThread();
MOZ_ASSERT(mDirectoryLock);
MOZ_ASSERT(mConnection);
MOZ_ASSERT(mQuotaObject);
MOZ_ASSERT(mClosed);
// Release the quota object first.
mQuotaObject = nullptr;
// Now it's safe to release the directory lock and unregister itself from
// the hashtable.
@ -3021,13 +3056,42 @@ Datastore::UpdateUsage(int64_t aDelta)
{
AssertIsOnBackgroundThread();
// Check internal LocalStorage origin limit.
int64_t newUsage = mUsage + aDelta;
if (newUsage > gOriginLimitKB * 1024) {
return false;
}
// Check QuotaManager limits (group and global limit).
if (IsPersistent()) {
MOZ_ASSERT(mQuotaObject);
if (!mQuotaObject->MaybeUpdateSize(newUsage, /* aTruncate */ true)) {
return false;
}
}
// Quota checks passed, set new usage.
mUsage = newUsage;
if (IsPersistent()) {
RefPtr<Runnable> runnable = NS_NewRunnableFunction(
"Datastore::UpdateUsage",
[origin = mOrigin, newUsage] () {
MOZ_ASSERT(gUsages);
MOZ_ASSERT(gUsages->Contains(origin));
gUsages->Put(origin, newUsage);
});
QuotaManager* quotaManager = QuotaManager::Get();
MOZ_ASSERT(quotaManager);
MOZ_ALWAYS_SUCCEEDS(
quotaManager->IOThread()->Dispatch(runnable, NS_DISPATCH_NORMAL));
}
return true;
}
@ -3745,6 +3809,9 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
, mDatabaseNotAvailable(false)
, mRequestedDirectoryLock(false)
, mInvalidated(false)
#ifdef DEBUG
, mDEBUGUsage(0)
#endif
{
MOZ_ASSERT(aParams.type() ==
LSRequestParams::TLSRequestPrepareDatastoreParams);
@ -4076,7 +4143,10 @@ PrepareDatastoreOp::DatabaseWork()
return rv;
}
rv = EnsureDirectoryEntry(directoryEntry, /* aIsDirectory */ false);
bool alreadyExisted;
rv = EnsureDirectoryEntry(directoryEntry,
/* aIsDirectory */ false,
&alreadyExisted);
if (rv == NS_ERROR_NOT_AVAILABLE) {
return DatabaseNotAvailable();
}
@ -4084,6 +4154,13 @@ PrepareDatastoreOp::DatabaseWork()
return rv;
}
if (alreadyExisted) {
MOZ_ASSERT(gUsages);
MOZ_ASSERT(gUsages->Get(mOrigin, &mUsage));
} else {
InitUsageForOrigin(mOrigin, 0);
}
rv = directoryEntry->GetPath(mDatabaseFilePath);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
@ -4141,7 +4218,9 @@ PrepareDatastoreOp::DatabaseNotAvailable()
}
nsresult
PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry, bool aIsDirectory)
PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry,
bool aIsDirectory,
bool* aAlreadyExisted)
{
AssertIsOnIOThread();
MOZ_ASSERT(aEntry);
@ -4172,6 +4251,9 @@ PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry, bool aIsDirectory)
}
#endif
if (aAlreadyExisted) {
*aAlreadyExisted = exists;
}
return NS_OK;
}
@ -4316,11 +4398,30 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
}
if (!mDatastore) {
MOZ_ASSERT(mUsage == mDEBUGUsage);
RefPtr<QuotaObject> quotaObject;
if (mPrivateBrowsingId == 0) {
MOZ_ASSERT(!mDatabaseFilePath.IsEmpty());
QuotaManager* quotaManager = QuotaManager::Get();
MOZ_ASSERT(quotaManager);
quotaObject = quotaManager->GetQuotaObject(PERSISTENCE_TYPE_DEFAULT,
mGroup,
mOrigin,
mDatabaseFilePath,
mUsage);
MOZ_ASSERT(quotaObject);
}
mDatastore = new Datastore(mOrigin,
mPrivateBrowsingId,
mUsage,
mDirectoryLock.forget(),
mConnection.forget(),
quotaObject.forget(),
mValues);
mDatastore->NoteLivePrepareDatastoreOp(this);
@ -4541,7 +4642,9 @@ LoadDataOp::DoDatastoreWork()
}
mPrepareDatastoreOp->mValues.Put(key, value);
mPrepareDatastoreOp->mUsage += key.Length() + value.Length();
#ifdef DEBUG
mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
#endif
}
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
@ -4912,28 +5015,7 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
UsageInfo* aUsageInfo)
{
AssertIsOnIOThread();
if (!aUsageInfo) {
return NS_OK;
}
return GetUsageForOrigin(aPersistenceType,
aGroup,
aOrigin,
aCanceled,
aUsageInfo);
}
nsresult
QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
const nsACString& aGroup,
const nsACString& aOrigin,
const AtomicBool& aCanceled,
UsageInfo* aUsageInfo)
{
AssertIsOnIOThread();
MOZ_ASSERT(aPersistenceType == PERSISTENCE_TYPE_DEFAULT);
MOZ_ASSERT(aUsageInfo);
QuotaManager* quotaManager = QuotaManager::Get();
MOZ_ASSERT(quotaManager);
@ -4991,7 +5073,42 @@ QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
}
// TODO: Use a special file that contains logical size of the database.
// For now, don't add to origin usage.
// For now, get the usage from the database.
nsCOMPtr<mozIStorageConnection> connection;
rv = CreateStorageConnection(file, aOrigin, getter_AddRefs(connection));
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
nsCOMPtr<mozIStorageStatement> stmt;
rv = connection->CreateStatement(NS_LITERAL_CSTRING(
"SELECT sum(length(key) + length(value)) "
"FROM data"
), getter_AddRefs(stmt));
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
bool hasResult;
rv = stmt->ExecuteStep(&hasResult);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
if (NS_WARN_IF(!hasResult)) {
return NS_ERROR_FAILURE;
}
int64_t usage;
rv = stmt->GetInt64(0, &usage);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
InitUsageForOrigin(aOrigin, usage);
aUsageInfo->AppendToDatabaseUsage(uint64_t(usage));
}
// Report unknown files, don't fail, just warn.
@ -5051,17 +5168,51 @@ QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
return NS_OK;
}
nsresult
QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
const nsACString& aGroup,
const nsACString& aOrigin,
const AtomicBool& aCanceled,
UsageInfo* aUsageInfo)
{
AssertIsOnIOThread();
MOZ_ASSERT(aPersistenceType == PERSISTENCE_TYPE_DEFAULT);
MOZ_ASSERT(aUsageInfo);
// We can't open the database at this point, since it can be already used
// by the connection thread. Use the cached value instead.
if (gUsages) {
int64_t usage;
if (gUsages->Get(aOrigin, &usage)) {
aUsageInfo->AppendToDatabaseUsage(usage);
}
}
return NS_OK;
}
void
QuotaClient::OnOriginClearCompleted(PersistenceType aPersistenceType,
const nsACString& aOrigin)
{
AssertIsOnIOThread();
if (aPersistenceType != PERSISTENCE_TYPE_DEFAULT) {
return;
}
if (gUsages) {
gUsages->Remove(aOrigin);
}
}
void
QuotaClient::ReleaseIOThreadObjects()
{
AssertIsOnIOThread();
gUsages = nullptr;
}
void

View file

@ -4,6 +4,10 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
XPCSHELL_TESTS_MANIFESTS += [
'test/unit/xpcshell.ini'
]
XPIDL_SOURCES += [
'nsILocalStorageManager.idl',
]

View file

@ -0,0 +1,144 @@
/**
* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
const NS_ERROR_DOM_QUOTA_EXCEEDED_ERR = 22;
ChromeUtils.import("resource://gre/modules/Services.jsm");
function is(a, b, msg)
{
Assert.equal(a, b, msg);
}
function ok(cond, msg)
{
Assert.ok(!!cond, msg);
}
function run_test()
{
runTest();
};
if (!this.runTest) {
this.runTest = function()
{
do_get_profile();
enableTesting();
do_test_pending();
testGenerator.next();
}
}
function finishTest()
{
resetTesting();
executeSoon(function() {
do_test_finished();
})
}
function continueToNextStepSync()
{
testGenerator.next();
}
function enableTesting()
{
Services.prefs.setBoolPref("dom.storage.testing", true);
Services.prefs.setBoolPref("dom.quotaManager.testing", true);
}
function resetTesting()
{
Services.prefs.clearUserPref("dom.quotaManager.testing");
Services.prefs.clearUserPref("dom.storage.testing");
}
function setGlobalLimit(globalLimit)
{
Services.prefs.setIntPref("dom.quotaManager.temporaryStorage.fixedLimit",
globalLimit);
}
function resetGlobalLimit()
{
Services.prefs.clearUserPref("dom.quotaManager.temporaryStorage.fixedLimit");
}
function setOriginLimit(originLimit)
{
Services.prefs.setIntPref("dom.storage.default_quota", originLimit);
}
function resetOriginLimit()
{
Services.prefs.clearUserPref("dom.storage.default_quota");
}
function getOriginUsage(principal, callback)
{
let request = Services.qms.getUsageForPrincipal(principal, callback);
request.callback = callback;
return request;
}
function clear(callback)
{
let request = Services.qms.clear();
request.callback = callback;
return request;
}
function resetOrigin(principal, callback)
{
let request =
Services.qms.resetStoragesForPrincipal(principal, "default", "ls");
request.callback = callback;
return request;
}
function repeatChar(count, ch) {
if (count == 0) {
return "";
}
let result = ch;
let count2 = count / 2;
// Double the input until it is long enough.
while (result.length <= count2) {
result += result;
}
// Use substring to hit the precise length target without using extra memory.
return result + result.substring(0, count - result.length);
}
function getPrincipal(url)
{
let uri = Services.io.newURI(url);
return Services.scriptSecurityManager.createCodebasePrincipal(uri, {});
}
function getCurrentPrincipal()
{
return Cc["@mozilla.org/systemprincipal;1"].createInstance(Ci.nsIPrincipal);
}
function getLocalStorage(principal)
{
if (!principal) {
principal = getCurrentPrincipal();
}
return Services.domStorageManager.createStorage(null, principal, "");
}

View file

@ -0,0 +1,91 @@
/**
* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
var testGenerator = testSteps();
function* testSteps()
{
const globalLimitKB = 5 * 1024;
const data = {};
data.sizeKB = 1 * 1024;
data.key = "A";
data.value = repeatChar(data.sizeKB * 1024 - data.key.length, ".");
data.urlCount = globalLimitKB / data.sizeKB;
function getSpec(index) {
return "http://example" + index + ".com";
}
info("Setting pref");
Services.prefs.setBoolPref("dom.storage.next_gen", true);
info("Setting limits");
setGlobalLimit(globalLimitKB);
clear(continueToNextStepSync);
yield undefined;
info("Getting storages");
let storages = [];
for (let i = 0; i < data.urlCount; i++) {
let storage = getLocalStorage(getPrincipal(getSpec(i)));
storages.push(storage);
}
info("Filling up entire default storage");
for (let i = 0; i < data.urlCount; i++) {
storages[i].setItem(data.key, data.value);
}
info("Verifying no more data can be written");
for (let i = 0; i < data.urlCount; i++) {
try {
storages[i].setItem("B", "");
ok(false, "Should have thrown");
} catch(ex) {
ok(true, "Did throw");
ok(ex instanceof DOMException, "Threw DOMException");
is(ex.name, "QuotaExceededError", "Threw right DOMException");
is(ex.code, NS_ERROR_DOM_QUOTA_EXCEEDED_ERR, "Threw with right code");
}
}
info("Closing first origin");
storages[0].close();
let principal = getPrincipal("http://example0.com");
resetOrigin(principal, continueToNextStepSync);
yield undefined;
info("Getting usage for first origin");
let request = getOriginUsage(principal, continueToNextStepSync);
yield undefined;
is(request.result.usage, data.sizeKB * 1024, "Correct usage");
info("Verifying more data data can be written");
for (let i = 1; i < data.urlCount; i++) {
storages[i].setItem("B", "");
}
info("Getting usage for first origin");
request = getOriginUsage(principal, continueToNextStepSync);
yield undefined;
is(request.result.usage, 0, "Zero usage");
finishTest();
}

View file

@ -0,0 +1,77 @@
/**
* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
var testGenerator = testSteps();
function* testSteps()
{
const groupLimitKB = 10 * 1024;
const globalLimitKB = groupLimitKB * 5;
const originLimit = 10 * 1024;
const urls = [
"http://example.com",
"http://test1.example.com",
"https://test2.example.com",
"http://test3.example.com:8080"
];
const data = {};
data.sizeKB = 5 * 1024;
data.key = "A";
data.value = repeatChar(data.sizeKB * 1024 - data.key.length, ".");
data.urlCount = groupLimitKB / data.sizeKB;
info("Setting limits");
setGlobalLimit(globalLimitKB);
clear(continueToNextStepSync);
yield undefined;
setOriginLimit(originLimit);
info("Getting storages");
let storages = [];
for (let i = 0; i < urls.length; i++) {
let storage = getLocalStorage(getPrincipal(urls[i]));
storages.push(storage);
}
info("Filling up the whole group");
for (let i = 0; i < data.urlCount; i++) {
storages[i].setItem(data.key, data.value);
}
info("Verifying no more data can be written");
for (let i = 0; i < urls.length; i++) {
try {
storages[i].setItem("B", "");
ok(false, "Should have thrown");
} catch(ex) {
ok(true, "Did throw");
ok(ex instanceof DOMException, "Threw DOMException");
is(ex.name, "QuotaExceededError", "Threw right DOMException");
is(ex.code, NS_ERROR_DOM_QUOTA_EXCEEDED_ERR, "Threw with right code");
}
}
info("Clearing first origin");
storages[0].clear();
info("Verifying more data can be written");
for (let i = 0; i < urls.length; i++) {
storages[i].setItem("B", "");
}
finishTest();
}

View file

@ -0,0 +1,9 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
[DEFAULT]
head = head.js
[test_eviction.js]
[test_groupLimit.js]

View file

@ -3116,9 +3116,16 @@ QuotaObject::LockedMaybeUpdateSize(int64_t aSize, bool aTruncate)
// This will block the thread without holding the lock while waitting.
AutoTArray<RefPtr<DirectoryLockImpl>, 10> locks;
uint64_t sizeToBeFreed;
uint64_t sizeToBeFreed =
quotaManager->LockedCollectOriginsForEviction(delta, locks);
if (IsOnBackgroundThread()) {
MutexAutoUnlock autoUnlock(quotaManager->mQuotaMutex);
sizeToBeFreed = quotaManager->CollectOriginsForEviction(delta, locks);
} else {
sizeToBeFreed = quotaManager->LockedCollectOriginsForEviction(delta,
locks);
}
if (!sizeToBeFreed) {
uint64_t usage = quotaManager->mTemporaryStorageUsage;
@ -3876,6 +3883,7 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
const nsACString& aGroup,
const nsACString& aOrigin,
nsIFile* aFile,
int64_t aFileSize,
int64_t* aFileSizeOut /* = nullptr */)
{
NS_ASSERTION(!NS_IsMainThread(), "Wrong thread!");
@ -3894,16 +3902,20 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
int64_t fileSize;
bool exists;
rv = aFile->Exists(&exists);
NS_ENSURE_SUCCESS(rv, nullptr);
if (exists) {
rv = aFile->GetFileSize(&fileSize);
if (aFileSize == -1) {
bool exists;
rv = aFile->Exists(&exists);
NS_ENSURE_SUCCESS(rv, nullptr);
}
else {
fileSize = 0;
if (exists) {
rv = aFile->GetFileSize(&fileSize);
NS_ENSURE_SUCCESS(rv, nullptr);
}
else {
fileSize = 0;
}
} else {
fileSize = aFileSize;
}
// Re-escape our parameters above to make sure we get the right quota group.
@ -3969,6 +3981,7 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
const nsACString& aGroup,
const nsACString& aOrigin,
const nsAString& aPath,
int64_t aFileSize,
int64_t* aFileSizeOut /* = nullptr */)
{
NS_ASSERTION(!NS_IsMainThread(), "Wrong thread!");
@ -3981,7 +3994,12 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
nsresult rv = NS_NewLocalFile(aPath, false, getter_AddRefs(file));
NS_ENSURE_SUCCESS(rv, nullptr);
return GetQuotaObject(aPersistenceType, aGroup, aOrigin, file, aFileSizeOut);
return GetQuotaObject(aPersistenceType,
aGroup,
aOrigin,
file,
aFileSize,
aFileSizeOut);
}
Nullable<bool>

View file

@ -179,6 +179,7 @@ public:
const nsACString& aGroup,
const nsACString& aOrigin,
nsIFile* aFile,
int64_t aFileSize = -1,
int64_t* aFileSizeOut = nullptr);
already_AddRefed<QuotaObject>
@ -186,6 +187,7 @@ public:
const nsACString& aGroup,
const nsACString& aOrigin,
const nsAString& aPath,
int64_t aFileSize = -1,
int64_t* aFileSizeOut = nullptr);
Nullable<bool>