forked from mirrors/gecko-dev
1660 lines
52 KiB
JavaScript
1660 lines
52 KiB
JavaScript
"use strict";
|
|
|
|
const { BaseAction } = ChromeUtils.import(
|
|
"resource://normandy/actions/BaseAction.jsm"
|
|
);
|
|
const { BranchedAddonStudyAction } = ChromeUtils.import(
|
|
"resource://normandy/actions/BranchedAddonStudyAction.jsm"
|
|
);
|
|
const { Uptake } = ChromeUtils.import("resource://normandy/lib/Uptake.jsm");
|
|
|
|
const { NormandyTestUtils } = ChromeUtils.import(
|
|
"resource://testing-common/NormandyTestUtils.jsm"
|
|
);
|
|
const { branchedAddonStudyFactory } = NormandyTestUtils.factories;
|
|
|
|
function branchedAddonStudyRecipeFactory(overrides = {}) {
|
|
let args = {
|
|
slug: "fake-slug",
|
|
userFacingName: "Fake name",
|
|
userFacingDescription: "fake description",
|
|
isEnrollmentPaused: false,
|
|
branches: [
|
|
{
|
|
slug: "a",
|
|
ratio: 1,
|
|
extensionApiId: 1,
|
|
},
|
|
],
|
|
};
|
|
if (Object.hasOwnProperty.call(overrides, "arguments")) {
|
|
args = Object.assign(args, overrides.arguments);
|
|
delete overrides.arguments;
|
|
}
|
|
return recipeFactory(
|
|
Object.assign(
|
|
{
|
|
action: "branched-addon-study",
|
|
arguments: args,
|
|
},
|
|
overrides
|
|
)
|
|
);
|
|
}
|
|
|
|
function recipeFromStudy(study, overrides = {}) {
|
|
let args = {
|
|
slug: study.slug,
|
|
userFacingName: study.userFacingName,
|
|
isEnrollmentPaused: false,
|
|
branches: [
|
|
{
|
|
slug: "a",
|
|
ratio: 1,
|
|
extensionApiId: study.extensionApiId,
|
|
},
|
|
],
|
|
};
|
|
|
|
if (Object.hasOwnProperty.call(overrides, "arguments")) {
|
|
args = Object.assign(args, overrides.arguments);
|
|
delete overrides.arguments;
|
|
}
|
|
|
|
return branchedAddonStudyRecipeFactory(
|
|
Object.assign(
|
|
{
|
|
id: study.recipeId,
|
|
arguments: args,
|
|
},
|
|
overrides
|
|
)
|
|
);
|
|
}
|
|
|
|
// Test that enroll is not called if recipe is already enrolled and update does nothing
|
|
// if recipe is unchanged
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([branchedAddonStudyFactory()]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "1.0" }),
|
|
async function enrollTwiceFail({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
hash: study.extensionHash,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const enrollSpy = sinon.spy(action, "enroll");
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(!enrollSpy.called, "enroll should not be called");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([]);
|
|
}
|
|
);
|
|
|
|
// Test that if the add-on fails to install, the database is cleaned up and the
|
|
// error is correctly reported.
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
AddonStudies.withStudies(),
|
|
async function enrollDownloadFail({ mockNormandyApi, sendEventSpy }) {
|
|
const recipe = branchedAddonStudyRecipeFactory({
|
|
arguments: {
|
|
branches: [{ slug: "missing", ratio: 1, extensionApiId: 404 }],
|
|
},
|
|
});
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: extensionDetailsFactory({
|
|
id: recipe.arguments.branches[0].extensionApiId,
|
|
xpi: "https://example.com/404.xpi",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
const studies = await AddonStudies.getAll();
|
|
Assert.deepEqual(studies, [], "the study should not be in the database");
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"enrollFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
reason: "download-failure",
|
|
detail: "ERROR_NETWORK_FAILURE",
|
|
branch: "missing",
|
|
},
|
|
],
|
|
]);
|
|
}
|
|
);
|
|
|
|
// Ensure that the database is clean and error correctly reported if hash check fails
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
AddonStudies.withStudies(),
|
|
async function enrollHashCheckFails({ mockNormandyApi, sendEventSpy }) {
|
|
const recipe = branchedAddonStudyRecipeFactory();
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: extensionDetailsFactory({
|
|
id: recipe.arguments.branches[0].extensionApiId,
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
hash: "badhash",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
const studies = await AddonStudies.getAll();
|
|
Assert.deepEqual(studies, [], "the study should not be in the database");
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"enrollFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
reason: "download-failure",
|
|
detail: "ERROR_INCORRECT_HASH",
|
|
branch: "a",
|
|
},
|
|
],
|
|
]);
|
|
}
|
|
);
|
|
|
|
// Ensure that the database is clean and error correctly reported if there is a metadata mismatch
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
AddonStudies.withStudies(),
|
|
async function enrollFailsMetadataMismatch({
|
|
mockNormandyApi,
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = branchedAddonStudyRecipeFactory();
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: extensionDetailsFactory({
|
|
id: recipe.arguments.branches[0].extensionApiId,
|
|
version: "1.5",
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
hash: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].hash,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
const studies = await AddonStudies.getAll();
|
|
Assert.deepEqual(studies, [], "the study should not be in the database");
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"enrollFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
reason: "metadata-mismatch",
|
|
branch: "a",
|
|
},
|
|
],
|
|
]);
|
|
}
|
|
);
|
|
|
|
// Test that in the case of a study add-on conflicting with a non-study add-on, the study does not enroll
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ version: "0.1", id: FIXTURE_ADDON_ID }),
|
|
AddonStudies.withStudies(),
|
|
async function conflictingEnrollment({
|
|
mockNormandyApi,
|
|
sendEventSpy,
|
|
installedWebExtensionSafe: { addonId },
|
|
}) {
|
|
is(
|
|
addonId,
|
|
FIXTURE_ADDON_ID,
|
|
"Generated, installed add-on should have the same ID as the fixture"
|
|
);
|
|
const recipe = branchedAddonStudyRecipeFactory({
|
|
arguments: { slug: "conflicting" },
|
|
});
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: extensionDetailsFactory({
|
|
id: recipe.arguments.extensionApiId,
|
|
addonUrl: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
is(addon.version, "0.1", "The installed add-on should not be replaced");
|
|
|
|
Assert.deepEqual(
|
|
await AddonStudies.getAll(),
|
|
[],
|
|
"There should be no enrolled studies"
|
|
);
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"enrollFailed",
|
|
"addon_study",
|
|
recipe.arguments.slug,
|
|
{ reason: "conflicting-addon-id" },
|
|
],
|
|
]);
|
|
}
|
|
);
|
|
|
|
// Test a successful update
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: FIXTURE_ADDON_ID,
|
|
extensionHash: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].hash,
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "1.0",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "1.0" }),
|
|
async function successfulUpdate({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const addonUrl = FIXTURE_ADDON_DETAILS["normandydriver-a-2.0"].url;
|
|
const recipe = recipeFromStudy(study, {
|
|
arguments: {
|
|
branches: [
|
|
{ slug: "a", extensionApiId: study.extensionApiId, ratio: 1 },
|
|
],
|
|
},
|
|
});
|
|
const hash = FIXTURE_ADDON_DETAILS["normandydriver-a-2.0"].hash;
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: extensionDetailsFactory({
|
|
id: recipe.arguments.branches[0].extensionApiId,
|
|
extension_id: FIXTURE_ADDON_ID,
|
|
xpi: addonUrl,
|
|
hash,
|
|
version: "2.0",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const enrollSpy = sinon.spy(action, "enroll");
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(!enrollSpy.called, "enroll should not be called");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"update",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
addonId: FIXTURE_ADDON_ID,
|
|
addonVersion: "2.0",
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(
|
|
updatedStudy,
|
|
{
|
|
...study,
|
|
addonVersion: "2.0",
|
|
addonUrl,
|
|
extensionApiId: recipe.arguments.branches[0].extensionApiId,
|
|
extensionHash: hash,
|
|
},
|
|
"study data should be updated"
|
|
);
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "2.0", "add-on should be updated");
|
|
}
|
|
);
|
|
|
|
// Test update fails when addon ID does not match
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: "test@example.com",
|
|
extensionHash: "01d",
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "0.1",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "0.1" }),
|
|
async function updateFailsAddonIdMismatch({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: FIXTURE_ADDON_ID,
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"updateFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
reason: "addon-id-mismatch",
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(updatedStudy, study, "study data should be unchanged");
|
|
|
|
let addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "0.1", "add-on should be unchanged");
|
|
}
|
|
);
|
|
|
|
// Test update fails when original addon does not exist
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
extensionHash: "01d",
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "0.1",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: "test@example.com", version: "0.1" }),
|
|
async function updateFailsAddonDoesNotExist({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"updateFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
reason: "addon-does-not-exist",
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(updatedStudy, study, "study data should be unchanged");
|
|
|
|
let addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(!addon, "new add-on should not be installed");
|
|
|
|
addon = await AddonManager.getAddonByID("test@example.com");
|
|
ok(addon, "old add-on should still be installed");
|
|
}
|
|
);
|
|
|
|
// Test update fails when download fails
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: FIXTURE_ADDON_ID,
|
|
extensionHash: "01d",
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "0.1",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "0.1" }),
|
|
async function updateDownloadFailure({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
xpi: "https://example.com/404.xpi",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"updateFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
branch: "a",
|
|
reason: "download-failure",
|
|
detail: "ERROR_NETWORK_FAILURE",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(updatedStudy, study, "study data should be unchanged");
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "0.1", "add-on should be unchanged");
|
|
}
|
|
);
|
|
|
|
// Test update fails when hash check fails
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: FIXTURE_ADDON_ID,
|
|
extensionHash: "01d",
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "0.1",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "0.1" }),
|
|
async function updateFailsHashCheckFail({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
hash: "badhash",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"updateFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
branch: "a",
|
|
reason: "download-failure",
|
|
detail: "ERROR_INCORRECT_HASH",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(updatedStudy, study, "study data should be unchanged");
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "0.1", "add-on should be unchanged");
|
|
}
|
|
);
|
|
|
|
// Test update fails on downgrade when study version is greater than extension version
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: FIXTURE_ADDON_ID,
|
|
extensionHash: "01d",
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "2.0",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "2.0" }),
|
|
async function upgradeFailsNoDowngrades({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
version: "1.0",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"updateFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
reason: "no-downgrade",
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(updatedStudy, study, "study data should be unchanged");
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "2.0", "add-on should be unchanged");
|
|
}
|
|
);
|
|
|
|
// Test update fails when there is a version mismatch with metadata
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: FIXTURE_ADDON_ID,
|
|
extensionHash: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].hash,
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "1.0",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionFromURL(
|
|
FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url
|
|
),
|
|
async function upgradeFailsMetadataMismatchVersion({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const recipe = recipeFromStudy(study);
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-2.0"].url,
|
|
version: "3.0",
|
|
hash: FIXTURE_ADDON_DETAILS["normandydriver-a-2.0"].hash,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"updateFailed",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
branch: "a",
|
|
reason: "metadata-mismatch",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const updatedStudy = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(updatedStudy, study, "study data should be unchanged");
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "1.0", "add-on should be unchanged");
|
|
|
|
let addonSourceURI = addon.getResourceURI();
|
|
if (addonSourceURI instanceof Ci.nsIJARURI) {
|
|
addonSourceURI = addonSourceURI.JARFile;
|
|
}
|
|
const xpiFile = addonSourceURI.QueryInterface(Ci.nsIFileURL).file;
|
|
const installedHash = CryptoUtils.getFileHash(
|
|
xpiFile,
|
|
study.extensionHashAlgorithm
|
|
);
|
|
ok(installedHash === study.extensionHash, "add-on should be unchanged");
|
|
}
|
|
);
|
|
|
|
// Test that unenrolling fails if the study doesn't exist
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
AddonStudies.withStudies(),
|
|
async function unenrollNonexistent() {
|
|
const action = new BranchedAddonStudyAction();
|
|
await Assert.rejects(
|
|
action.unenroll(42),
|
|
/no study found/i,
|
|
"unenroll should fail when no study exists"
|
|
);
|
|
}
|
|
);
|
|
|
|
// Test that unenrolling an inactive study fails
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
AddonStudies.withStudies([branchedAddonStudyFactory({ active: false })]),
|
|
withSendEventSpy(),
|
|
async ({ addonStudies: [study], sendEventSpy }) => {
|
|
const action = new BranchedAddonStudyAction();
|
|
await Assert.rejects(
|
|
action.unenroll(study.recipeId),
|
|
/cannot stop study.*already inactive/i,
|
|
"unenroll should fail when the requested study is inactive"
|
|
);
|
|
}
|
|
);
|
|
|
|
// test a successful unenrollment
|
|
const testStopId = "testStop@example.com";
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
active: true,
|
|
addonId: testStopId,
|
|
studyEndDate: null,
|
|
}),
|
|
]),
|
|
withInstalledWebExtension({ id: testStopId }, { expectUninstall: true }),
|
|
withSendEventSpy(),
|
|
withStub(TelemetryEnvironment, "setExperimentInactive"),
|
|
async function unenrollTest({
|
|
addonStudies: [study],
|
|
installedWebExtension: { addonId },
|
|
sendEventSpy,
|
|
setExperimentInactiveStub,
|
|
}) {
|
|
let addon = await AddonManager.getAddonByID(addonId);
|
|
ok(addon, "the add-on should be installed before unenrolling");
|
|
|
|
const action = new BranchedAddonStudyAction();
|
|
await action.unenroll(study.recipeId, "test-reason");
|
|
|
|
const newStudy = AddonStudies.get(study.recipeId);
|
|
is(!newStudy, false, "stop should mark the study as inactive");
|
|
ok(newStudy.studyEndDate !== null, "the study should have an end date");
|
|
|
|
addon = await AddonManager.getAddonByID(addonId);
|
|
is(addon, null, "the add-on should be uninstalled after unenrolling");
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"unenroll",
|
|
"addon_study",
|
|
study.name,
|
|
{
|
|
addonId,
|
|
addonVersion: study.addonVersion,
|
|
reason: "test-reason",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
Assert.deepEqual(
|
|
setExperimentInactiveStub.args,
|
|
[[study.slug]],
|
|
"setExperimentInactive should be called"
|
|
);
|
|
}
|
|
);
|
|
|
|
// If the add-on for a study isn't installed, a warning should be logged, but the action is still successful
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
active: true,
|
|
addonId: "missingAddon@example.com",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
async function unenrollMissingAddonTest({
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const action = new BranchedAddonStudyAction();
|
|
|
|
await action.unenroll(study.recipeId);
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"unenroll",
|
|
"addon_study",
|
|
study.name,
|
|
{
|
|
addonId: study.addonId,
|
|
addonVersion: study.addonVersion,
|
|
reason: "unknown",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
SimpleTest.endMonitorConsole();
|
|
}
|
|
);
|
|
|
|
// Test that the action respects the study opt-out
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
withMockPreferences(),
|
|
AddonStudies.withStudies(),
|
|
async function testOptOut({
|
|
mockNormandyApi,
|
|
sendEventSpy,
|
|
mockPreferences,
|
|
}) {
|
|
mockPreferences.set("app.shield.optoutstudies.enabled", false);
|
|
const action = new BranchedAddonStudyAction();
|
|
const enrollSpy = sinon.spy(action, "enroll");
|
|
const recipe = branchedAddonStudyRecipeFactory();
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: extensionDetailsFactory({
|
|
id: recipe.arguments.branches[0].extensionApiId,
|
|
}),
|
|
};
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(
|
|
action.state,
|
|
BranchedAddonStudyAction.STATE_DISABLED,
|
|
"the action should be disabled"
|
|
);
|
|
await action.finalize();
|
|
is(
|
|
action.state,
|
|
BranchedAddonStudyAction.STATE_FINALIZED,
|
|
"the action should be finalized"
|
|
);
|
|
is(action.lastError, null, "lastError should be null");
|
|
Assert.deepEqual(enrollSpy.args, [], "enroll should not be called");
|
|
sendEventSpy.assertEvents([]);
|
|
}
|
|
);
|
|
|
|
// Test that the action does not enroll paused recipes
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
AddonStudies.withStudies(),
|
|
async function testEnrollmentPaused({ mockNormandyApi, sendEventSpy }) {
|
|
const action = new BranchedAddonStudyAction();
|
|
const enrollSpy = sinon.spy(action, "enroll");
|
|
const updateSpy = sinon.spy(action, "update");
|
|
const recipe = branchedAddonStudyRecipeFactory({
|
|
arguments: { isEnrollmentPaused: true },
|
|
});
|
|
const extensionDetails = extensionDetailsFactory({
|
|
id: recipe.arguments.extensionApiId,
|
|
});
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.extensionApiId]: extensionDetails,
|
|
};
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
const addon = await AddonManager.getAddonByID(
|
|
extensionDetails.extension_id
|
|
);
|
|
is(addon, null, "the add-on should not have been installed");
|
|
await action.finalize();
|
|
ok(!updateSpy.called, "update should not be called");
|
|
ok(enrollSpy.called, "enroll should be called");
|
|
sendEventSpy.assertEvents([]);
|
|
}
|
|
);
|
|
|
|
// Test that the action updates paused recipes
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
addonId: FIXTURE_ADDON_ID,
|
|
extensionHash: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].hash,
|
|
extensionHashAlgorithm: "sha256",
|
|
addonVersion: "1.0",
|
|
}),
|
|
]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "1.0" }),
|
|
async function testUpdateEnrollmentPaused({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
}) {
|
|
const addonUrl = FIXTURE_ADDON_DETAILS["normandydriver-a-2.0"].url;
|
|
const recipe = recipeFromStudy(study, {
|
|
arguments: { isEnrollmentPaused: true },
|
|
});
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
xpi: addonUrl,
|
|
hash: FIXTURE_ADDON_DETAILS["normandydriver-a-2.0"].hash,
|
|
version: "2.0",
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const enrollSpy = sinon.spy(action, "enroll");
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
ok(!enrollSpy.called, "enroll should not be called");
|
|
ok(updateSpy.called, "update should be called");
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"update",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
addonId: FIXTURE_ADDON_ID,
|
|
addonVersion: "2.0",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
const addon = await AddonManager.getAddonByID(FIXTURE_ADDON_ID);
|
|
ok(addon.version === "2.0", "add-on should be updated");
|
|
}
|
|
);
|
|
|
|
// Test that unenroll called if the study is no longer sent from the server
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
AddonStudies.withStudies([branchedAddonStudyFactory()]),
|
|
async function unenroll({ addonStudies: [study] }) {
|
|
const action = new BranchedAddonStudyAction();
|
|
const unenrollSpy = sinon.stub(action, "unenroll");
|
|
await action.finalize();
|
|
is(action.lastError, null, "lastError should be null");
|
|
Assert.deepEqual(
|
|
unenrollSpy.args,
|
|
[[study.recipeId, "recipe-not-seen"]],
|
|
"unenroll should be called"
|
|
);
|
|
}
|
|
);
|
|
|
|
// A test function that will be parameterized over the argument "branch" below.
|
|
// Mocks the branch selector, and then tests that the user correctly gets enrolled in that branch.
|
|
const successEnrollBranchedTest = decorate(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
withStub(TelemetryEnvironment, "setExperimentActive"),
|
|
AddonStudies.withStudies(),
|
|
async function({
|
|
branch,
|
|
mockNormandyApi,
|
|
sendEventSpy,
|
|
setExperimentActiveStub,
|
|
}) {
|
|
ok(branch == "a" || branch == "b", "Branch should be either a or b");
|
|
const initialAddonIds = (await AddonManager.getAllAddons()).map(
|
|
addon => addon.id
|
|
);
|
|
const addonId = `normandydriver-${branch}@example.com`;
|
|
const otherBranchAddonId = `normandydriver-${branch == "a" ? "b" : "a"}`;
|
|
is(
|
|
await AddonManager.getAddonByID(addonId),
|
|
null,
|
|
"The add-on should not be installed at the beginning of the test"
|
|
);
|
|
is(
|
|
await AddonManager.getAddonByID(otherBranchAddonId),
|
|
null,
|
|
"The other branch's add-on should not be installed at the beginning of the test"
|
|
);
|
|
|
|
const recipe = branchedAddonStudyRecipeFactory({
|
|
arguments: {
|
|
slug: "success",
|
|
branches: [
|
|
{ slug: "a", ratio: 1, extensionApiId: 1 },
|
|
{ slug: "b", ratio: 1, extensionApiId: 2 },
|
|
],
|
|
},
|
|
});
|
|
mockNormandyApi.extensionDetails = {
|
|
[recipe.arguments.branches[0].extensionApiId]: {
|
|
id: recipe.arguments.branches[0].extensionApiId,
|
|
name: "Normandy Fixture A",
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url,
|
|
extension_id: "normandydriver-a@example.com",
|
|
version: "1.0",
|
|
hash: FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].hash,
|
|
hash_algorithm: "sha256",
|
|
},
|
|
[recipe.arguments.branches[1].extensionApiId]: {
|
|
id: recipe.arguments.branches[1].extensionApiId,
|
|
name: "Normandy Fixture B",
|
|
xpi: FIXTURE_ADDON_DETAILS["normandydriver-b-1.0"].url,
|
|
extension_id: "normandydriver-b@example.com",
|
|
version: "1.0",
|
|
hash: FIXTURE_ADDON_DETAILS["normandydriver-b-1.0"].hash,
|
|
hash_algorithm: "sha256",
|
|
},
|
|
};
|
|
const extensionApiId =
|
|
recipe.arguments.branches[branch == "a" ? 0 : 1].extensionApiId;
|
|
const extensionDetails = mockNormandyApi.extensionDetails[extensionApiId];
|
|
|
|
const action = new BranchedAddonStudyAction();
|
|
const chooseBranchStub = sinon.stub(action, "chooseBranch");
|
|
chooseBranchStub.callsFake(async ({ branches }) =>
|
|
branches.find(b => b.slug === branch)
|
|
);
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
const study = await AddonStudies.get(recipe.id);
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"enroll",
|
|
"addon_study",
|
|
recipe.arguments.slug,
|
|
{
|
|
addonId,
|
|
addonVersion: "1.0",
|
|
branch,
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
Assert.deepEqual(
|
|
setExperimentActiveStub.args,
|
|
[
|
|
[
|
|
recipe.arguments.slug,
|
|
branch,
|
|
{ type: "normandy-addonstudy", enrollmentId: study.enrollmentId },
|
|
],
|
|
],
|
|
"setExperimentActive should be called"
|
|
);
|
|
|
|
const addon = await AddonManager.getAddonByID(addonId);
|
|
ok(addon, "The chosen branch's add-on should be installed");
|
|
is(
|
|
await AddonManager.getAddonByID(otherBranchAddonId),
|
|
null,
|
|
"The other branch's add-on should not be installed"
|
|
);
|
|
|
|
Assert.deepEqual(
|
|
study,
|
|
{
|
|
recipeId: recipe.id,
|
|
slug: recipe.arguments.slug,
|
|
userFacingName: recipe.arguments.userFacingName,
|
|
userFacingDescription: recipe.arguments.userFacingDescription,
|
|
addonId,
|
|
addonVersion: "1.0",
|
|
addonUrl: FIXTURE_ADDON_DETAILS[`normandydriver-${branch}-1.0`].url,
|
|
active: true,
|
|
branch,
|
|
studyStartDate: study.studyStartDate, // This is checked below
|
|
studyEndDate: null,
|
|
extensionApiId: extensionDetails.id,
|
|
extensionHash: extensionDetails.hash,
|
|
extensionHashAlgorithm: extensionDetails.hash_algorithm,
|
|
enrollmentId: study.enrollmentId,
|
|
temporaryErrorDeadline: null,
|
|
},
|
|
"the correct study data should be stored"
|
|
);
|
|
|
|
// cleanup
|
|
await safeUninstallAddon(addon);
|
|
Assert.deepEqual(
|
|
(await AddonManager.getAllAddons()).map(addon => addon.id),
|
|
initialAddonIds,
|
|
"all test add-ons are removed"
|
|
);
|
|
}
|
|
);
|
|
|
|
add_task(args => successEnrollBranchedTest({ ...args, branch: "a" }));
|
|
add_task(args => successEnrollBranchedTest({ ...args, branch: "b" }));
|
|
|
|
// If the enrolled branch no longer exists, unenroll
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
AddonStudies.withStudies([branchedAddonStudyFactory()]),
|
|
withSendEventSpy(),
|
|
withInstalledWebExtensionSafe(
|
|
{ id: FIXTURE_ADDON_ID, version: "1.0" },
|
|
{ expectUninstall: true }
|
|
),
|
|
async function unenrollIfBranchDisappears({
|
|
mockNormandyApi,
|
|
addonStudies: [study],
|
|
sendEventSpy,
|
|
installedWebExtensionSafe: { addonId },
|
|
}) {
|
|
const recipe = recipeFromStudy(study, {
|
|
arguments: {
|
|
branches: [
|
|
{
|
|
slug: "b", // different from enrolled study
|
|
ratio: 1,
|
|
extensionApiId: study.extensionApiId,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
mockNormandyApi.extensionDetails = {
|
|
[study.extensionApiId]: extensionDetailsFactory({
|
|
id: study.extensionApiId,
|
|
extension_id: study.addonId,
|
|
hash: study.extensionHash,
|
|
}),
|
|
};
|
|
const action = new BranchedAddonStudyAction();
|
|
const enrollSpy = sinon.spy(action, "enroll");
|
|
const unenrollSpy = sinon.spy(action, "unenroll");
|
|
const updateSpy = sinon.spy(action, "update");
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
ok(!enrollSpy.called, "Enroll should not be called");
|
|
ok(updateSpy.called, "Update should be called");
|
|
ok(unenrollSpy.called, "Unenroll should be called");
|
|
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"unenroll",
|
|
"addon_study",
|
|
study.name,
|
|
{
|
|
addonId,
|
|
addonVersion: study.addonVersion,
|
|
reason: "branch-removed",
|
|
branch: "a", // the original study branch
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
is(
|
|
await AddonManager.getAddonByID(addonId),
|
|
null,
|
|
"the add-on should be uninstalled"
|
|
);
|
|
|
|
const storedStudy = await AddonStudies.get(recipe.id);
|
|
ok(!storedStudy.active, "Study should be inactive");
|
|
ok(storedStudy.branch == "a", "Study's branch should not change");
|
|
ok(storedStudy.studyEndDate, "Study's end date should be set");
|
|
}
|
|
);
|
|
|
|
// Test that branches without an add-on can be enrolled and unenrolled succesfully.
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
ensureAddonCleanup(),
|
|
withMockNormandyApi(),
|
|
withSendEventSpy(),
|
|
AddonStudies.withStudies(),
|
|
async function noAddonBranches({ sendEventSpy }) {
|
|
const initialAddonIds = (await AddonManager.getAllAddons()).map(
|
|
addon => addon.id
|
|
);
|
|
|
|
const recipe = branchedAddonStudyRecipeFactory({
|
|
arguments: {
|
|
slug: "no-op-branch",
|
|
branches: [{ slug: "a", ratio: 1, extensionApiId: null }],
|
|
},
|
|
});
|
|
|
|
let action = new BranchedAddonStudyAction();
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
|
|
await action.finalize();
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
let study = await AddonStudies.get(recipe.id);
|
|
sendEventSpy.assertEvents([
|
|
[
|
|
"enroll",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
addonId: AddonStudies.NO_ADDON_MARKER,
|
|
addonVersion: AddonStudies.NO_ADDON_MARKER,
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
Assert.deepEqual(
|
|
(await AddonManager.getAllAddons()).map(addon => addon.id),
|
|
initialAddonIds,
|
|
"No add-on should be installed for the study"
|
|
);
|
|
|
|
Assert.deepEqual(
|
|
study,
|
|
{
|
|
recipeId: recipe.id,
|
|
slug: recipe.arguments.slug,
|
|
userFacingName: recipe.arguments.userFacingName,
|
|
userFacingDescription: recipe.arguments.userFacingDescription,
|
|
addonId: null,
|
|
addonVersion: null,
|
|
addonUrl: null,
|
|
active: true,
|
|
branch: "a",
|
|
studyStartDate: study.studyStartDate, // This is checked below
|
|
studyEndDate: null,
|
|
extensionApiId: null,
|
|
extensionHash: null,
|
|
extensionHashAlgorithm: null,
|
|
enrollmentId: study.enrollmentId,
|
|
temporaryErrorDeadline: null,
|
|
},
|
|
"the correct study data should be stored"
|
|
);
|
|
ok(study.studyStartDate, "studyStartDate should have a value");
|
|
NormandyTestUtils.isUuid(study.enrollmentId);
|
|
|
|
// Now unenroll
|
|
action = new BranchedAddonStudyAction();
|
|
await action.finalize();
|
|
is(action.lastError, null, "lastError should be null");
|
|
|
|
sendEventSpy.assertEvents([
|
|
// The event from before
|
|
[
|
|
"enroll",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
addonId: AddonStudies.NO_ADDON_MARKER,
|
|
addonVersion: AddonStudies.NO_ADDON_MARKER,
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
// And a new unenroll event
|
|
[
|
|
"unenroll",
|
|
"addon_study",
|
|
recipe.arguments.name,
|
|
{
|
|
addonId: AddonStudies.NO_ADDON_MARKER,
|
|
addonVersion: AddonStudies.NO_ADDON_MARKER,
|
|
branch: "a",
|
|
enrollmentId: study.enrollmentId,
|
|
},
|
|
],
|
|
]);
|
|
|
|
Assert.deepEqual(
|
|
(await AddonManager.getAllAddons()).map(addon => addon.id),
|
|
initialAddonIds,
|
|
"The set of add-ons should not change"
|
|
);
|
|
|
|
study = await AddonStudies.get(recipe.id);
|
|
Assert.deepEqual(
|
|
study,
|
|
{
|
|
recipeId: recipe.id,
|
|
slug: recipe.arguments.slug,
|
|
userFacingName: recipe.arguments.userFacingName,
|
|
userFacingDescription: recipe.arguments.userFacingDescription,
|
|
addonId: null,
|
|
addonVersion: null,
|
|
addonUrl: null,
|
|
active: false,
|
|
branch: "a",
|
|
studyStartDate: study.studyStartDate, // This is checked below
|
|
studyEndDate: study.studyEndDate, // This is checked below
|
|
extensionApiId: null,
|
|
extensionHash: null,
|
|
extensionHashAlgorithm: null,
|
|
enrollmentId: study.enrollmentId,
|
|
temporaryErrorDeadline: null,
|
|
},
|
|
"the correct study data should be stored"
|
|
);
|
|
ok(study.studyStartDate, "studyStartDate should have a value");
|
|
ok(study.studyEndDate, "studyEndDate should have a value");
|
|
NormandyTestUtils.isUuid(study.enrollmentId);
|
|
}
|
|
);
|
|
|
|
// Check that the appropriate set of suitabilities are considered temporary errors
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
async function test_temporary_errors_set_deadline() {
|
|
let suitabilities = [
|
|
{
|
|
suitability: BaseAction.suitability.SIGNATURE_ERROR,
|
|
isTemporaryError: true,
|
|
},
|
|
{
|
|
suitability: BaseAction.suitability.CAPABILITIES_MISMATCH,
|
|
isTemporaryError: false,
|
|
},
|
|
{
|
|
suitability: BaseAction.suitability.FILTER_MATCH,
|
|
isTemporaryError: false,
|
|
},
|
|
{
|
|
suitability: BaseAction.suitability.FILTER_MISMATCH,
|
|
isTemporaryError: false,
|
|
},
|
|
{
|
|
suitability: BaseAction.suitability.FILTER_ERROR,
|
|
isTemporaryError: true,
|
|
},
|
|
{
|
|
suitability: BaseAction.suitability.ARGUMENTS_INVALID,
|
|
isTemporaryError: false,
|
|
},
|
|
];
|
|
|
|
Assert.deepEqual(
|
|
suitabilities.map(({ suitability }) => suitability).sort(),
|
|
Array.from(Object.values(BaseAction.suitability)).sort(),
|
|
"This test covers all suitabilities"
|
|
);
|
|
|
|
// The action should set a deadline 1 week from now. To avoid intermittent
|
|
// failures, give this a generous bound of 2 hours on either side.
|
|
let now = Date.now();
|
|
let hour = 60 * 60 * 1000;
|
|
let expectedDeadline = now + 7 * 24 * hour;
|
|
let minDeadline = new Date(expectedDeadline - 2 * hour);
|
|
let maxDeadline = new Date(expectedDeadline + 2 * hour);
|
|
|
|
// For each suitability, build a decorator that sets up a suitabilty
|
|
// environment, and then call that decorator with a sub-test that asserts
|
|
// the suitability is handled correctly.
|
|
for (const { suitability, isTemporaryError } of suitabilities) {
|
|
const decorator = AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
slug: `test-for-suitability-${suitability}`,
|
|
}),
|
|
]);
|
|
await decorator(async ({ addonStudies: [study] }) => {
|
|
let action = new BranchedAddonStudyAction();
|
|
let recipe = recipeFromStudy(study);
|
|
await action.processRecipe(recipe, suitability);
|
|
let modifiedStudy = await AddonStudies.get(recipe.id);
|
|
|
|
if (isTemporaryError) {
|
|
ok(
|
|
// The constructor of this object is a Date, but is not the same as
|
|
// the Date that we have in our global scope, because it got sent
|
|
// through IndexedDB. Check the name of the constructor instead.
|
|
modifiedStudy.temporaryErrorDeadline.constructor.name == "Date",
|
|
`A temporary failure deadline should be set as a date for suitability ${suitability}`
|
|
);
|
|
let deadline = modifiedStudy.temporaryErrorDeadline;
|
|
ok(
|
|
deadline >= minDeadline && deadline <= maxDeadline,
|
|
`The temporary failure deadline should be in the expected range for ` +
|
|
`suitability ${suitability} (got ${deadline}, expected between ${minDeadline} and ${maxDeadline})`
|
|
);
|
|
} else {
|
|
ok(
|
|
!modifiedStudy.temporaryErrorDeadline,
|
|
`No temporary failure deadline should be set for suitability ${suitability}`
|
|
);
|
|
}
|
|
})();
|
|
}
|
|
}
|
|
);
|
|
|
|
// Check that if there is an existing deadline, temporary errors don't overwrite it
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
async function test_temporary_errors_dont_overwrite_deadline() {
|
|
let temporaryFailureSuitabilities = [
|
|
BaseAction.suitability.SIGNATURE_ERROR,
|
|
BaseAction.suitability.FILTER_ERROR,
|
|
];
|
|
|
|
// A deadline two hours in the future won't be hit during the test.
|
|
let now = Date.now();
|
|
let hour = 2 * 60 * 60 * 1000;
|
|
let unhitDeadline = new Date(now + hour);
|
|
|
|
// For each suitability, build a decorator that sets up a suitabilty
|
|
// environment, and then call that decorator with a sub-test that asserts
|
|
// the suitability is handled correctly.
|
|
for (const suitability of temporaryFailureSuitabilities) {
|
|
const decorator = AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
slug: `test-for-suitability-${suitability}`,
|
|
active: true,
|
|
temporaryErrorDeadline: unhitDeadline,
|
|
}),
|
|
]);
|
|
await decorator(async ({ addonStudies: [study] }) => {
|
|
let action = new BranchedAddonStudyAction();
|
|
let recipe = recipeFromStudy(study);
|
|
await action.processRecipe(recipe, suitability);
|
|
let modifiedStudy = await AddonStudies.get(recipe.id);
|
|
is(
|
|
modifiedStudy.temporaryErrorDeadline.toJSON(),
|
|
unhitDeadline.toJSON(),
|
|
`The temporary failure deadline should not be cleared for suitability ${suitability}`
|
|
);
|
|
})();
|
|
}
|
|
}
|
|
);
|
|
|
|
// Check that if the deadline is past, temporary errors end the study.
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
async function test_temporary_errors_hit_deadline() {
|
|
let temporaryFailureSuitabilities = [
|
|
BaseAction.suitability.SIGNATURE_ERROR,
|
|
BaseAction.suitability.FILTER_ERROR,
|
|
];
|
|
|
|
// Set a deadline of two hours in the past, so that the deadline is triggered.
|
|
let now = Date.now();
|
|
let hour = 60 * 60 * 1000;
|
|
let hitDeadline = new Date(now - 2 * hour);
|
|
|
|
// For each suitability, build a decorator that sets up a suitabilty
|
|
// environment, and then call that decorator with a sub-test that asserts
|
|
// the suitability is handled correctly.
|
|
for (const suitability of temporaryFailureSuitabilities) {
|
|
const decorator = AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
slug: `test-for-suitability-${suitability}`,
|
|
active: true,
|
|
temporaryErrorDeadline: hitDeadline,
|
|
}),
|
|
]);
|
|
await decorator(async ({ addonStudies: [study] }) => {
|
|
let action = new BranchedAddonStudyAction();
|
|
let recipe = recipeFromStudy(study);
|
|
await action.processRecipe(recipe, suitability);
|
|
let modifiedStudy = await AddonStudies.get(recipe.id);
|
|
ok(
|
|
!modifiedStudy.active,
|
|
`The study should end for suitability ${suitability}`
|
|
);
|
|
})();
|
|
}
|
|
}
|
|
);
|
|
|
|
// Check that non-temporary-error suitabilities clear the temporary deadline
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
async function test_non_temporary_error_clears_temporary_error_deadline() {
|
|
let suitabilitiesThatShouldClearDeadline = [
|
|
BaseAction.suitability.CAPABILITIES_MISMATCH,
|
|
BaseAction.suitability.FILTER_MATCH,
|
|
BaseAction.suitability.FILTER_MISMATCH,
|
|
BaseAction.suitability.ARGUMENTS_INVALID,
|
|
];
|
|
|
|
// Use a deadline in the past to demonstrate that even if the deadline has
|
|
// passed, only a temporary error suitability ends the study.
|
|
let now = Date.now();
|
|
let hour = 60 * 60 * 1000;
|
|
let hitDeadline = new Date(now - 2 * hour);
|
|
|
|
// For each suitability, build a decorator that sets up a suitabilty
|
|
// environment, and then call that decorator with a sub-test that asserts
|
|
// the suitability is handled correctly.
|
|
for (const suitability of suitabilitiesThatShouldClearDeadline) {
|
|
const decorator = AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
slug: `test-for-suitability-${suitability}`.toLocaleLowerCase(),
|
|
active: true,
|
|
temporaryErrorDeadline: hitDeadline,
|
|
}),
|
|
]);
|
|
await decorator(async ({ addonStudies: [study] }) => {
|
|
let action = new BranchedAddonStudyAction();
|
|
let recipe = recipeFromStudy(study);
|
|
await action.processRecipe(recipe, suitability);
|
|
let modifiedStudy = await AddonStudies.get(recipe.id);
|
|
ok(
|
|
!modifiedStudy.temporaryErrorDeadline,
|
|
`The temporary failure deadline should be cleared for suitabilitiy ${suitability}`
|
|
);
|
|
})();
|
|
}
|
|
}
|
|
);
|
|
|
|
// Check that invalid deadlines are reset
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
async function test_non_temporary_error_clears_temporary_error_deadline() {
|
|
let temporaryFailureSuitabilities = [
|
|
BaseAction.suitability.SIGNATURE_ERROR,
|
|
BaseAction.suitability.FILTER_ERROR,
|
|
];
|
|
|
|
// The action should set a deadline 1 week from now. To avoid intermittent
|
|
// failures, give this a generous bound of 2 hours on either side.
|
|
let invalidDeadline = new Date("not a valid date");
|
|
let now = Date.now();
|
|
let hour = 60 * 60 * 1000;
|
|
let expectedDeadline = now + 7 * 24 * hour;
|
|
let minDeadline = new Date(expectedDeadline - 2 * hour);
|
|
let maxDeadline = new Date(expectedDeadline + 2 * hour);
|
|
|
|
// For each suitability, build a decorator that sets up a suitabilty
|
|
// environment, and then call that decorator with a sub-test that asserts
|
|
// the suitability is handled correctly.
|
|
for (const suitability of temporaryFailureSuitabilities) {
|
|
const decorator = AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
slug: `test-for-suitability-${suitability}`.toLocaleLowerCase(),
|
|
active: true,
|
|
temporaryErrorDeadline: invalidDeadline,
|
|
}),
|
|
]);
|
|
await decorator(async ({ addonStudies: [study] }) => {
|
|
let action = new BranchedAddonStudyAction();
|
|
let recipe = recipeFromStudy(study);
|
|
await action.processRecipe(recipe, suitability);
|
|
is(action.lastError, null, "No errors should be reported");
|
|
let modifiedStudy = await AddonStudies.get(recipe.id);
|
|
ok(
|
|
modifiedStudy.temporaryErrorDeadline != invalidDeadline,
|
|
`The temporary failure deadline should be reset for suitabilitiy ${suitability}`
|
|
);
|
|
let deadline = new Date(modifiedStudy.temporaryErrorDeadline);
|
|
ok(
|
|
deadline >= minDeadline && deadline <= maxDeadline,
|
|
`The temporary failure deadline should be reset to a valid deadline for ${suitability}`
|
|
);
|
|
})();
|
|
}
|
|
}
|
|
);
|
|
|
|
// Check that an already unenrolled study doesn't try to unenroll again if
|
|
// the recipe doesn't apply the client anymore.
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
async function test_unenroll_when_already_expired() {
|
|
// Use a deadline that is already past
|
|
const now = new Date();
|
|
const hour = 1000 * 60 * 60;
|
|
const temporaryErrorDeadline = new Date(now - hour * 2).toJSON();
|
|
|
|
const suitabilitiesToCheck = Object.values(BaseAction.suitability);
|
|
|
|
const subtest = decorate(
|
|
AddonStudies.withStudies([
|
|
branchedAddonStudyFactory({
|
|
active: false,
|
|
temporaryErrorDeadline,
|
|
}),
|
|
]),
|
|
|
|
async ({ addonStudies: [study], suitability }) => {
|
|
const recipe = recipeFromStudy(study);
|
|
const action = new BranchedAddonStudyAction();
|
|
const unenrollSpy = sinon.spy(action.unenroll);
|
|
await action.processRecipe(recipe, suitability);
|
|
Assert.deepEqual(
|
|
unenrollSpy.args,
|
|
[],
|
|
`Stop should not be called for ${suitability}`
|
|
);
|
|
}
|
|
);
|
|
|
|
for (const suitability of suitabilitiesToCheck) {
|
|
await subtest({ suitability });
|
|
}
|
|
}
|
|
);
|
|
|
|
// If no recipes are received, it should be considered a temporary error
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
AddonStudies.withStudies([branchedAddonStudyFactory({ active: true })]),
|
|
withSpy(BranchedAddonStudyAction.prototype, "unenroll"),
|
|
withStub(BranchedAddonStudyAction.prototype, "_considerTemporaryError"),
|
|
async function testNoRecipes({
|
|
unenrollSpy,
|
|
_considerTemporaryErrorStub,
|
|
addonStudies: [study],
|
|
}) {
|
|
let action = new BranchedAddonStudyAction();
|
|
await action.finalize({ noRecipes: true });
|
|
|
|
Assert.deepEqual(unenrollSpy.args, [], "Unenroll should not be called");
|
|
Assert.deepEqual(
|
|
_considerTemporaryErrorStub.args,
|
|
[[{ study, reason: "no-recipes" }]],
|
|
"The experiment should accumulate a temporary error"
|
|
);
|
|
}
|
|
);
|
|
|
|
// If recipes are received, but the flag that none were received is set, an error should be thrown
|
|
decorate_task(
|
|
withStudiesEnabled(),
|
|
AddonStudies.withStudies([branchedAddonStudyFactory({ active: true })]),
|
|
async function testNoRecipes({ addonStudies: [study] }) {
|
|
let action = new BranchedAddonStudyAction();
|
|
let recipe = recipeFromStudy(study);
|
|
await action.processRecipe(recipe, BaseAction.suitability.FILTER_MISMATCH);
|
|
await action.finalize({ noRecipes: true });
|
|
ok(
|
|
action.lastError instanceof BranchedAddonStudyAction.BadNoRecipesArg,
|
|
"An error should be logged since some recipes were received"
|
|
);
|
|
}
|
|
);
|