fune/browser/components/newtab/lib/TopSitesFeed.jsm
Cristian Tuns b3bf09cc0d Backed out 6 changesets (bug 1816934, bug 1817182, bug 1817179, bug 1817183) for causing dt failures in browser_jsterm_autocomplete_null.js CLOSED TREE
Backed out changeset 17d4c013ed92 (bug 1817183)
Backed out changeset cfed8d9c23f3 (bug 1817183)
Backed out changeset 62fe2f589efe (bug 1817182)
Backed out changeset 557bd773fb85 (bug 1817179)
Backed out changeset 7f8a7865868b (bug 1816934)
Backed out changeset d6c1d4c0d2a0 (bug 1816934)
2023-02-17 10:51:33 -05:00

1412 lines
44 KiB
JavaScript

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const { XPCOMUtils } = ChromeUtils.importESModule(
"resource://gre/modules/XPCOMUtils.sys.mjs"
);
const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule(
"resource://activity-stream/common/Actions.sys.mjs"
);
const { TippyTopProvider } = ChromeUtils.import(
"resource://activity-stream/lib/TippyTopProvider.jsm"
);
const {
insertPinned,
TOP_SITES_MAX_SITES_PER_ROW,
} = ChromeUtils.importESModule(
"resource://activity-stream/common/Reducers.sys.mjs"
);
const { Dedupe } = ChromeUtils.importESModule(
"resource://activity-stream/common/Dedupe.sys.mjs"
);
const { shortURL } = ChromeUtils.import(
"resource://activity-stream/lib/ShortURL.jsm"
);
const { getDefaultOptions } = ChromeUtils.import(
"resource://activity-stream/lib/ActivityStreamStorage.jsm"
);
const {
CUSTOM_SEARCH_SHORTCUTS,
SEARCH_SHORTCUTS_EXPERIMENT,
SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF,
SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
checkHasSearchEngine,
getSearchProvider,
getSearchFormURL,
} = ChromeUtils.import("resource://activity-stream/lib/SearchShortcuts.jsm");
const lazy = {};
ChromeUtils.defineModuleGetter(
lazy,
"FilterAdult",
"resource://activity-stream/lib/FilterAdult.jsm"
);
ChromeUtils.defineModuleGetter(
lazy,
"LinksCache",
"resource://activity-stream/lib/LinksCache.jsm"
);
ChromeUtils.defineESModuleGetters(lazy, {
NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
Region: "resource://gre/modules/Region.sys.mjs",
});
ChromeUtils.defineModuleGetter(
lazy,
"Screenshots",
"resource://activity-stream/lib/Screenshots.jsm"
);
ChromeUtils.defineModuleGetter(
lazy,
"PageThumbs",
"resource://gre/modules/PageThumbs.jsm"
);
ChromeUtils.defineModuleGetter(
lazy,
"RemoteSettings",
"resource://services-settings/remote-settings.js"
);
XPCOMUtils.defineLazyGetter(lazy, "log", () => {
const { Logger } = ChromeUtils.import(
"resource://messaging-system/lib/Logger.jsm"
);
return new Logger("TopSitesFeed");
});
XPCOMUtils.defineLazyModuleGetters(lazy, {
NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
});
const DEFAULT_SITES_PREF = "default.sites";
const SHOWN_ON_NEWTAB_PREF = "feeds.topsites";
const DEFAULT_TOP_SITES = [];
const FRECENCY_THRESHOLD = 100 + 1; // 1 visit (skip first-run/one-time pages)
const MIN_FAVICON_SIZE = 96;
const CACHED_LINK_PROPS_TO_MIGRATE = ["screenshot", "customScreenshot"];
const PINNED_FAVICON_PROPS_TO_MIGRATE = [
"favicon",
"faviconRef",
"faviconSize",
];
const SECTION_ID = "topsites";
const ROWS_PREF = "topSitesRows";
const SHOW_SPONSORED_PREF = "showSponsoredTopSites";
const MAX_NUM_SPONSORED = 2;
// Search experiment stuff
const FILTER_DEFAULT_SEARCH_PREF = "improvesearch.noDefaultSearchTile";
const SEARCH_FILTERS = [
"google",
"search.yahoo",
"yahoo",
"bing",
"ask",
"duckduckgo",
];
const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting";
const DEFAULT_SITES_OVERRIDE_PREF =
"browser.newtabpage.activity-stream.default.sites";
const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment.";
// Mozilla Tiles Service (Contile) prefs
// Nimbus variable for the Contile integration. It falls back to the pref:
// `browser.topsites.contile.enabled`.
const NIMBUS_VARIABLE_CONTILE_ENABLED = "topSitesContileEnabled";
const CONTILE_ENDPOINT_PREF = "browser.topsites.contile.endpoint";
const CONTILE_UPDATE_INTERVAL = 15 * 60 * 1000; // 15 minutes
const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors";
function getShortURLForCurrentSearch() {
const url = shortURL({ url: Services.search.defaultEngine.searchForm });
return url;
}
class ContileIntegration {
constructor(topSitesFeed) {
this._topSitesFeed = topSitesFeed;
this._lastPeriodicUpdate = 0;
this._sites = [];
}
get sites() {
return this._sites;
}
periodicUpdate() {
let now = Date.now();
if (now - this._lastPeriodicUpdate >= CONTILE_UPDATE_INTERVAL) {
this._lastPeriodicUpdate = now;
this.refresh();
}
}
async refresh() {
let updateDefaultSites = await this._fetchSites();
if (updateDefaultSites) {
this._topSitesFeed._readDefaults();
}
}
/**
* Filter the tiles whose sponsor is on the Top Sites sponsor blocklist.
*
* @param {array} tiles
* An array of the tile objects
*/
_filterBlockedSponsors(tiles) {
const blocklist = JSON.parse(
Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
);
return tiles.filter(tile => !blocklist.includes(shortURL(tile)));
}
async _fetchSites() {
if (
!lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_CONTILE_ENABLED
) ||
!this._topSitesFeed.store.getState().Prefs.values[SHOW_SPONSORED_PREF]
) {
if (this._sites.length) {
this._sites = [];
return true;
}
return false;
}
try {
let url = Services.prefs.getStringPref(CONTILE_ENDPOINT_PREF);
const response = await fetch(url, { credentials: "omit" });
if (!response.ok) {
lazy.log.warn(
`Contile endpoint returned unexpected status: ${response.status}`
);
}
// Contile returns 204 indicating there is no content at the moment.
// If this happens, just return without signifying the change so that the
// existing tiles (`this._sites`) could retain. We might want to introduce
// other handling for this in the future.
if (response.status === 204) {
return false;
}
const body = await response.json();
if (body?.tiles && Array.isArray(body.tiles)) {
let { tiles } = body;
tiles = this._filterBlockedSponsors(tiles);
if (tiles.length > MAX_NUM_SPONSORED) {
lazy.log.warn(
`Contile provided more links than permitted. (${tiles.length} received, limit is ${MAX_NUM_SPONSORED})`
);
tiles.length = MAX_NUM_SPONSORED;
}
this._sites = tiles;
return true;
}
} catch (error) {
lazy.log.warn(
`Failed to fetch data from Contile server: ${error.message}`
);
}
return false;
}
}
class TopSitesFeed {
constructor() {
this._contile = new ContileIntegration(this);
this._tippyTopProvider = new TippyTopProvider();
XPCOMUtils.defineLazyGetter(
this,
"_currentSearchHostname",
getShortURLForCurrentSearch
);
this.dedupe = new Dedupe(this._dedupeKey);
this.frecentCache = new lazy.LinksCache(
lazy.NewTabUtils.activityStreamLinks,
"getTopSites",
CACHED_LINK_PROPS_TO_MIGRATE,
(oldOptions, newOptions) =>
// Refresh if no old options or requesting more items
!(oldOptions.numItems >= newOptions.numItems)
);
this.pinnedCache = new lazy.LinksCache(
lazy.NewTabUtils.pinnedLinks,
"links",
[...CACHED_LINK_PROPS_TO_MIGRATE, ...PINNED_FAVICON_PROPS_TO_MIGRATE]
);
lazy.PageThumbs.addExpirationFilter(this);
this._nimbusChangeListener = this._nimbusChangeListener.bind(this);
}
_nimbusChangeListener(event, reason) {
// The Nimbus API current doesn't specify the changed variable(s) in the
// listener callback, so we have to refresh unconditionally on every change
// of the `newtab` feature. It should be a manageable overhead given the
// current update cadence (6 hours) of Nimbus.
//
// Skip the experiment and rollout loading reasons since this feature has
// `isEarlyStartup` enabled, the feature variables are already available
// before the experiment or rollout loads.
if (
!["feature-experiment-loaded", "feature-rollout-loaded"].includes(reason)
) {
this._contile.refresh();
}
}
init() {
// If the feed was previously disabled PREFS_INITIAL_VALUES was never received
this._readDefaults({ isStartup: true });
this._storage = this.store.dbStorage.getDbTable("sectionPrefs");
this._contile.refresh();
Services.obs.addObserver(this, "browser-search-engine-modified");
Services.obs.addObserver(this, "browser-region-updated");
Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
lazy.NimbusFeatures.newtab.onUpdate(this._nimbusChangeListener);
}
uninit() {
lazy.PageThumbs.removeExpirationFilter(this);
Services.obs.removeObserver(this, "browser-search-engine-modified");
Services.obs.removeObserver(this, "browser-region-updated");
Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
lazy.NimbusFeatures.newtab.off(this._nimbusChangeListener);
}
observe(subj, topic, data) {
switch (topic) {
case "browser-search-engine-modified":
// We should update the current top sites if the search engine has been changed since
// the search engine that gets filtered out of top sites has changed.
// We also need to drop search shortcuts when their engine gets removed / hidden.
if (
data === "engine-default" &&
this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF]
) {
delete this._currentSearchHostname;
this._currentSearchHostname = getShortURLForCurrentSearch();
}
this.refresh({ broadcast: true });
break;
case "browser-region-updated":
this._readDefaults();
break;
case "nsPref:changed":
if (
data === REMOTE_SETTING_DEFAULTS_PREF ||
data === DEFAULT_SITES_OVERRIDE_PREF ||
data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH)
) {
this._readDefaults();
}
break;
}
}
_dedupeKey(site) {
return site && site.hostname;
}
/**
* _readDefaults - sets DEFAULT_TOP_SITES
*/
async _readDefaults({ isStartup = false } = {}) {
this._useRemoteSetting = false;
if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) {
this.refreshDefaults(
this.store.getState().Prefs.values[DEFAULT_SITES_PREF],
{ isStartup }
);
return;
}
// Try using default top sites from enterprise policies or tests. The pref
// is locked when set via enterprise policy. Tests have no default sites
// unless they set them via this pref.
if (
Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) ||
Cu.isInAutomation
) {
let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, "");
this.refreshDefaults(sites, { isStartup });
return;
}
// Clear out the array of any previous defaults.
DEFAULT_TOP_SITES.length = 0;
// Read defaults from contile.
const contileEnabled = lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_CONTILE_ENABLED
);
let hasContileTiles = false;
if (contileEnabled) {
let sponsoredPosition = 1;
for (let site of this._contile.sites) {
let hostname = shortURL(site);
let link = {
isDefault: true,
url: site.url,
hostname,
sendAttributionRequest: false,
label: site.name,
show_sponsored_label: hostname !== "yandex",
sponsored_position: sponsoredPosition++,
sponsored_click_url: site.click_url,
sponsored_impression_url: site.impression_url,
sponsored_tile_id: site.id,
};
if (site.image_url && site.image_size >= MIN_FAVICON_SIZE) {
// Only use the image from Contile if it's hi-res, otherwise, fallback
// to the built-in favicons.
link.favicon = site.image_url;
link.faviconSize = site.image_size;
}
DEFAULT_TOP_SITES.push(link);
}
hasContileTiles = sponsoredPosition > 1;
}
// Read defaults from remote settings.
this._useRemoteSetting = true;
let remoteSettingData = await this._getRemoteConfig();
const sponsoredBlocklist = JSON.parse(
Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
);
for (let siteData of remoteSettingData) {
let hostname = shortURL(siteData);
// Drop default sites when Contile already provided a sponsored one with
// the same host name.
if (
contileEnabled &&
DEFAULT_TOP_SITES.findIndex(site => site.hostname === hostname) > -1
) {
continue;
}
// Also drop those sponsored sites that were blocked by the user before
// with the same hostname.
if (
siteData.sponsored_position &&
sponsoredBlocklist.includes(hostname)
) {
continue;
}
let link = {
isDefault: true,
url: siteData.url,
hostname,
sendAttributionRequest: !!siteData.send_attribution_request,
};
if (siteData.url_urlbar_override) {
link.url_urlbar = siteData.url_urlbar_override;
}
if (siteData.title) {
link.label = siteData.title;
}
if (siteData.search_shortcut) {
link = await this.topSiteToSearchTopSite(link);
} else if (siteData.sponsored_position) {
if (contileEnabled && hasContileTiles) {
continue;
}
const {
sponsored_position,
sponsored_tile_id,
sponsored_impression_url,
sponsored_click_url,
} = siteData;
link = {
sponsored_position,
sponsored_tile_id,
sponsored_impression_url,
sponsored_click_url,
show_sponsored_label: link.hostname !== "yandex",
...link,
};
}
DEFAULT_TOP_SITES.push(link);
}
this.refresh({ broadcast: true, isStartup });
}
refreshDefaults(sites, { isStartup = false } = {}) {
// Clear out the array of any previous defaults
DEFAULT_TOP_SITES.length = 0;
// Add default sites if any based on the pref
if (sites) {
for (const url of sites.split(",")) {
const site = {
isDefault: true,
url,
};
site.hostname = shortURL(site);
DEFAULT_TOP_SITES.push(site);
}
}
this.refresh({ broadcast: true, isStartup });
}
async _getRemoteConfig(firstTime = true) {
if (!this._remoteConfig) {
this._remoteConfig = await lazy.RemoteSettings("top-sites");
this._remoteConfig.on("sync", () => {
this._readDefaults();
});
}
let result = [];
let failed = false;
try {
result = await this._remoteConfig.get();
} catch (ex) {
console.error(ex);
failed = true;
}
if (!result.length) {
console.error("Received empty top sites configuration!");
failed = true;
}
// If we failed, or the result is empty, try loading from the local dump.
if (firstTime && failed) {
await this._remoteConfig.db.clear();
// Now call this again.
return this._getRemoteConfig(false);
}
// Sort sites based on the "order" attribute.
result.sort((a, b) => a.order - b.order);
result = result.filter(topsite => {
// Filter by region.
if (topsite.exclude_regions?.includes(lazy.Region.home)) {
return false;
}
if (
topsite.include_regions?.length &&
!topsite.include_regions.includes(lazy.Region.home)
) {
return false;
}
// Filter by locale.
if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) {
return false;
}
if (
topsite.include_locales?.length &&
!topsite.include_locales.includes(Services.locale.appLocaleAsBCP47)
) {
return false;
}
// Filter by experiment.
// Exclude this top site if any of the specified experiments are running.
if (
topsite.exclude_experiments?.some(experimentID =>
Services.prefs.getBoolPref(
DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
false
)
)
) {
return false;
}
// Exclude this top site if none of the specified experiments are running.
if (
topsite.include_experiments?.length &&
topsite.include_experiments.every(
experimentID =>
!Services.prefs.getBoolPref(
DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
false
)
)
) {
return false;
}
return true;
});
return result;
}
filterForThumbnailExpiration(callback) {
const { rows } = this.store.getState().TopSites;
callback(
rows.reduce((acc, site) => {
acc.push(site.url);
if (site.customScreenshotURL) {
acc.push(site.customScreenshotURL);
}
return acc;
}, [])
);
}
/**
* shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine?
*
* @param {string} hostname a top site hostname, such as "amazon" or "foo"
* @returns {bool}
*/
shouldFilterSearchTile(hostname) {
if (
this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] &&
(SEARCH_FILTERS.includes(hostname) ||
hostname === this._currentSearchHostname)
) {
return true;
}
return false;
}
/**
* _maybeInsertSearchShortcuts - if the search shortcuts experiment is running,
* insert search shortcuts if needed
* @param {Array} plainPinnedSites (from the pinnedSitesCache)
* @returns {Boolean} Did we insert any search shortcuts?
*/
async _maybeInsertSearchShortcuts(plainPinnedSites) {
// Only insert shortcuts if the experiment is running
if (this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) {
// We don't want to insert shortcuts we've previously inserted
const prevInsertedShortcuts = this.store
.getState()
.Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",")
.filter(s => s); // Filter out empty strings
const newInsertedShortcuts = [];
let shouldPin = this._useRemoteSetting
? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname)
: this.store
.getState()
.Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF].split(",");
shouldPin = shouldPin
.map(getSearchProvider)
.filter(s => s && s.shortURL !== this._currentSearchHostname);
// If we've previously inserted all search shortcuts return early
if (
shouldPin.every(shortcut =>
prevInsertedShortcuts.includes(shortcut.shortURL)
)
) {
return false;
}
const numberOfSlots =
this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW;
// The plainPinnedSites array is populated with pinned sites at their
// respective indices, and null everywhere else, but is not always the
// right length
const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0);
const pinnedSites = [...plainPinnedSites].concat(
Array(emptySlots).fill(null)
);
const tryToInsertSearchShortcut = async shortcut => {
const nextAvailable = pinnedSites.indexOf(null);
// Only add a search shortcut if the site isn't already pinned, we
// haven't previously inserted it, there's space to pin it, and the
// search engine is available in Firefox
if (
!pinnedSites.find(s => s && shortURL(s) === shortcut.shortURL) &&
!prevInsertedShortcuts.includes(shortcut.shortURL) &&
nextAvailable > -1 &&
(await checkHasSearchEngine(shortcut.keyword))
) {
const site = await this.topSiteToSearchTopSite({ url: shortcut.url });
this._pinSiteAt(site, nextAvailable);
pinnedSites[nextAvailable] = site;
newInsertedShortcuts.push(shortcut.shortURL);
}
};
for (let shortcut of shouldPin) {
await tryToInsertSearchShortcut(shortcut);
}
if (newInsertedShortcuts.length) {
this.store.dispatch(
ac.SetPref(
SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
prevInsertedShortcuts.concat(newInsertedShortcuts).join(",")
)
);
return true;
}
}
return false;
}
// eslint-disable-next-line max-statements
async getLinksWithDefaults(isStartup = false) {
const prefValues = this.store.getState().Prefs.values;
const numItems = prefValues[ROWS_PREF] * TOP_SITES_MAX_SITES_PER_ROW;
const searchShortcutsExperiment = prefValues[SEARCH_SHORTCUTS_EXPERIMENT];
// We must wait for search services to initialize in order to access default
// search engine properties without triggering a synchronous initialization
await Services.search.init();
// Get all frecent sites from history.
let frecent = [];
const cache = await this.frecentCache.request({
// We need to overquery due to the top 5 alexa search + default search possibly being removed
numItems: numItems + SEARCH_FILTERS.length + 1,
topsiteFrecency: FRECENCY_THRESHOLD,
});
for (let link of cache) {
const hostname = shortURL(link);
if (!this.shouldFilterSearchTile(hostname)) {
frecent.push({
...(searchShortcutsExperiment
? await this.topSiteToSearchTopSite(link)
: link),
hostname,
});
}
}
// Get defaults.
let date = new Date();
let pad = number => number.toString().padStart(2, "0");
let yyyymmddhh =
String(date.getFullYear()) +
pad(date.getMonth() + 1) +
pad(date.getDate()) +
pad(date.getHours());
let notBlockedDefaultSites = [];
let sponsored = [];
for (let link of DEFAULT_TOP_SITES) {
// For sponsored Yandex links, default filtering is reversed: we only
// show them if Yandex is the default search engine.
if (link.sponsored_position && link.hostname === "yandex") {
if (link.hostname !== this._currentSearchHostname) {
continue;
}
} else if (this.shouldFilterSearchTile(link.hostname)) {
continue;
}
// Drop blocked default sites.
if (
lazy.NewTabUtils.blockedLinks.isBlocked({
url: link.url,
})
) {
continue;
}
// Process %YYYYMMDDHH% tag in the URL.
let url_end;
let url_start;
if (this._useRemoteSetting) {
[url_start, url_end] = link.url.split("%YYYYMMDDHH%");
}
if (typeof url_end === "string") {
link = {
...link,
// Save original URL without %YYYYMMDDHH% replaced so it can be
// blocked properly.
original_url: link.url,
url: url_start + yyyymmddhh + url_end,
};
if (link.url_urlbar) {
link.url_urlbar = link.url_urlbar.replace("%YYYYMMDDHH%", yyyymmddhh);
}
}
// If we've previously blocked a search shortcut, remove the default top site
// that matches the hostname
const searchProvider = getSearchProvider(shortURL(link));
if (
searchProvider &&
lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url })
) {
continue;
}
if (link.sponsored_position) {
if (!prefValues[SHOW_SPONSORED_PREF]) {
continue;
}
sponsored[link.sponsored_position - 1] = link;
// Unpin search shortcut if present for the sponsored link to be shown
// instead.
this._unpinSearchShortcut(link.hostname);
} else {
notBlockedDefaultSites.push(
searchShortcutsExperiment
? await this.topSiteToSearchTopSite(link)
: link
);
}
}
// Get pinned links augmented with desired properties
let plainPinned = await this.pinnedCache.request();
// Insert search shortcuts if we need to.
// _maybeInsertSearchShortcuts returns true if any search shortcuts are
// inserted, meaning we need to expire and refresh the pinnedCache
if (await this._maybeInsertSearchShortcuts(plainPinned)) {
this.pinnedCache.expire();
plainPinned = await this.pinnedCache.request();
}
const pinned = await Promise.all(
plainPinned.map(async link => {
if (!link) {
return link;
}
// Drop pinned search shortcuts when their engine has been removed / hidden.
if (link.searchTopSite) {
const searchProvider = getSearchProvider(shortURL(link));
if (
!searchProvider ||
!(await checkHasSearchEngine(searchProvider.keyword))
) {
return null;
}
}
// Copy all properties from a frecent link and add more
const finder = other => other.url === link.url;
// Remove frecent link's screenshot if pinned link has a custom one
const frecentSite = frecent.find(finder);
if (frecentSite && link.customScreenshotURL) {
delete frecentSite.screenshot;
}
// If the link is a frecent site, do not copy over 'isDefault', else check
// if the site is a default site
const copy = Object.assign(
{},
frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) },
link,
{ hostname: shortURL(link) },
{ searchTopSite: !!link.searchTopSite }
);
// Add in favicons if we don't already have it
if (!copy.favicon) {
try {
lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI(
await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy])
);
for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) {
copy.__sharedCache.updateLink(prop, copy[prop]);
}
} catch (e) {
// Some issue with favicon, so just continue without one
}
}
return copy;
})
);
// Remove any duplicates from frecent and default sites
const [
,
dedupedSponsored,
dedupedFrecent,
dedupedDefaults,
] = this.dedupe.group(pinned, sponsored, frecent, notBlockedDefaultSites);
const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults];
// Remove adult sites if we need to
const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned);
// Insert the original pinned sites into the deduped frecent and defaults.
let withPinned = insertPinned(checkedAdult, pinned);
// Insert sponsored sites at their desired position.
dedupedSponsored.forEach(link => {
if (!link) {
return;
}
let index = link.sponsored_position - 1;
if (index > withPinned.length) {
withPinned[index] = link;
} else {
withPinned.splice(index, 0, link);
}
});
// Remove excess items after we inserted sponsored ones.
withPinned = withPinned.slice(0, numItems);
// Now, get a tippy top icon, a rich icon, or screenshot for every item
for (const link of withPinned) {
if (link) {
// If there is a custom screenshot this is the only image we display
if (link.customScreenshotURL) {
this._fetchScreenshot(link, link.customScreenshotURL, isStartup);
} else if (link.searchTopSite && !link.isDefault) {
await this._attachTippyTopIconForSearchShortcut(link, link.label);
} else {
this._fetchIcon(link, isStartup);
}
// Remove internal properties that might be updated after dispatch
delete link.__sharedCache;
// Indicate that these links should get a frecency bonus when clicked
link.typedBonus = true;
}
}
this._linksWithDefaults = withPinned;
return withPinned;
}
/**
* Attach TippyTop icon to the given search shortcut
*
* Note that it queries the search form URL from search service For Yandex,
* and uses it to choose the best icon for its shortcut variants.
*
* @param {Object} link A link object with a `url` property
* @param {string} keyword Search keyword
*/
async _attachTippyTopIconForSearchShortcut(link, keyword) {
if (
["@\u044F\u043D\u0434\u0435\u043A\u0441", "@yandex"].includes(keyword)
) {
let site = { url: link.url };
site.url = (await getSearchFormURL(keyword)) || site.url;
this._tippyTopProvider.processSite(site);
link.tippyTopIcon = site.tippyTopIcon;
link.smallFavicon = site.smallFavicon;
link.backgroundColor = site.backgroundColor;
} else {
this._tippyTopProvider.processSite(link);
}
}
/**
* Refresh the top sites data for content.
* @param {bool} options.broadcast Should the update be broadcasted.
* @param {bool} options.isStartup Being called while TopSitesFeed is initting.
*/
async refresh(options = {}) {
if (!this._startedUp && !options.isStartup) {
// Initial refresh still pending.
return;
}
this._startedUp = true;
if (!this._tippyTopProvider.initialized) {
await this._tippyTopProvider.init();
}
const links = await this.getLinksWithDefaults({
isStartup: options.isStartup,
});
const newAction = { type: at.TOP_SITES_UPDATED, data: { links } };
let storedPrefs;
try {
storedPrefs = (await this._storage.get(SECTION_ID)) || {};
} catch (e) {
storedPrefs = {};
console.error("Problem getting stored prefs for TopSites");
}
newAction.data.pref = getDefaultOptions(storedPrefs);
if (options.isStartup) {
newAction.meta = {
isStartup: true,
};
}
if (options.broadcast) {
// Broadcast an update to all open content pages
this.store.dispatch(ac.BroadcastToContent(newAction));
} else {
// Don't broadcast only update the state and update the preloaded tab.
this.store.dispatch(ac.AlsoToPreloaded(newAction));
}
}
async updateCustomSearchShortcuts(isStartup = false) {
if (!this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) {
return;
}
if (!this._tippyTopProvider.initialized) {
await this._tippyTopProvider.init();
}
// Populate the state with available search shortcuts
let searchShortcuts = [];
for (const engine of await Services.search.getAppProvidedEngines()) {
const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s =>
engine.aliases.includes(s.keyword)
);
if (shortcut) {
let clone = { ...shortcut };
await this._attachTippyTopIconForSearchShortcut(clone, clone.keyword);
searchShortcuts.push(clone);
}
}
this.store.dispatch(
ac.BroadcastToContent({
type: at.UPDATE_SEARCH_SHORTCUTS,
data: { searchShortcuts },
meta: {
isStartup,
},
})
);
}
async topSiteToSearchTopSite(site) {
const searchProvider = getSearchProvider(shortURL(site));
if (
!searchProvider ||
!(await checkHasSearchEngine(searchProvider.keyword))
) {
return site;
}
return {
...site,
searchTopSite: true,
label: searchProvider.keyword,
};
}
/**
* Get an image for the link preferring tippy top, rich favicon, screenshots.
*/
async _fetchIcon(link, isStartup = false) {
// Nothing to do if we already have a rich icon from the page
if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) {
return;
}
// Nothing more to do if we can use a default tippy top icon
this._tippyTopProvider.processSite(link);
if (link.tippyTopIcon) {
return;
}
// Make a request for a better icon
this._requestRichIcon(link.url);
// Also request a screenshot if we don't have one yet
await this._fetchScreenshot(link, link.url, isStartup);
}
/**
* Fetch, cache and broadcast a screenshot for a specific topsite.
* @param link cached topsite object
* @param url where to fetch the image from
* @param isStartup Whether the screenshot is fetched while initting TopSitesFeed.
*/
async _fetchScreenshot(link, url, isStartup = false) {
// We shouldn't bother caching screenshots if they won't be shown.
if (
link.screenshot ||
!this.store.getState().Prefs.values[SHOWN_ON_NEWTAB_PREF]
) {
return;
}
await lazy.Screenshots.maybeCacheScreenshot(
link,
url,
"screenshot",
screenshot =>
this.store.dispatch(
ac.BroadcastToContent({
data: { screenshot, url: link.url },
type: at.SCREENSHOT_UPDATED,
meta: {
isStartup,
},
})
)
);
}
/**
* Dispatch screenshot preview to target or notify if request failed.
* @param customScreenshotURL {string} The URL used to capture the screenshot
* @param target {string} Id of content process where to dispatch the result
*/
async getScreenshotPreview(url, target) {
const preview = (await lazy.Screenshots.getScreenshotForURL(url)) || "";
this.store.dispatch(
ac.OnlyToOneContent(
{
data: { url, preview },
type: at.PREVIEW_RESPONSE,
},
target
)
);
}
_requestRichIcon(url) {
this.store.dispatch({
type: at.RICH_ICON_MISSING,
data: { url },
});
}
updateSectionPrefs(collapsed) {
this.store.dispatch(
ac.BroadcastToContent({
type: at.TOP_SITES_PREFS_UPDATED,
data: { pref: collapsed },
})
);
}
/**
* Inform others that top sites data has been updated due to pinned changes.
*/
_broadcastPinnedSitesUpdated() {
// Pinned data changed, so make sure we get latest
this.pinnedCache.expire();
// Refresh to update pinned sites with screenshots, trigger deduping, etc.
this.refresh({ broadcast: true });
}
/**
* Pin a site at a specific position saving only the desired keys.
* @param customScreenshotURL {string} User set URL of preview image for site
* @param label {string} User set string of custom site name
*/
async _pinSiteAt({ customScreenshotURL, label, url, searchTopSite }, index) {
const toPin = { url };
if (label) {
toPin.label = label;
}
if (customScreenshotURL) {
toPin.customScreenshotURL = customScreenshotURL;
}
if (searchTopSite) {
toPin.searchTopSite = searchTopSite;
}
lazy.NewTabUtils.pinnedLinks.pin(toPin, index);
await this._clearLinkCustomScreenshot({ customScreenshotURL, url });
}
async _clearLinkCustomScreenshot(site) {
// If screenshot url changed or was removed we need to update the cached link obj
if (site.customScreenshotURL !== undefined) {
const pinned = await this.pinnedCache.request();
const link = pinned.find(pin => pin && pin.url === site.url);
if (link && link.customScreenshotURL !== site.customScreenshotURL) {
link.__sharedCache.updateLink("screenshot", undefined);
}
}
}
/**
* Handle a pin action of a site to a position.
*/
async pin(action) {
let { site, index } = action.data;
index = this._adjustPinIndexForSponsoredLinks(site, index);
// If valid index provided, pin at that position
if (index >= 0) {
await this._pinSiteAt(site, index);
this._broadcastPinnedSitesUpdated();
} else {
// Bug 1458658. If the top site is being pinned from an 'Add a Top Site' option,
// then we want to make sure to unblock that link if it has previously been
// blocked. We know if the site has been added because the index will be -1.
if (index === -1) {
lazy.NewTabUtils.blockedLinks.unblock({ url: site.url });
this.frecentCache.expire();
}
this.insert(action);
}
}
/**
* Handle an unpin action of a site.
*/
unpin(action) {
const { site } = action.data;
lazy.NewTabUtils.pinnedLinks.unpin(site);
this._broadcastPinnedSitesUpdated();
}
unpinAllSearchShortcuts() {
Services.prefs.clearUserPref(
`browser.newtabpage.activity-stream.${SEARCH_SHORTCUTS_HAVE_PINNED_PREF}`
);
for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
if (pinnedLink && pinnedLink.searchTopSite) {
lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
}
}
this.pinnedCache.expire();
}
_unpinSearchShortcut(vendor) {
for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
if (
pinnedLink &&
pinnedLink.searchTopSite &&
shortURL(pinnedLink) === vendor
) {
lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
this.pinnedCache.expire();
const prevInsertedShortcuts = this.store
.getState()
.Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",");
this.store.dispatch(
ac.SetPref(
SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
prevInsertedShortcuts.filter(s => s !== vendor).join(",")
)
);
break;
}
}
}
/**
* Reduces the given pinning index by the number of preceding sponsored
* sites, to accomodate for sponsored sites pushing pinned ones to the side,
* effectively increasing their index again.
*/
_adjustPinIndexForSponsoredLinks(site, index) {
if (!this._linksWithDefaults) {
return index;
}
// Adjust insertion index for sponsored sites since their position is
// fixed.
let adjustedIndex = index;
for (let i = 0; i < index; i++) {
if (
this._linksWithDefaults[i]?.sponsored_position &&
this._linksWithDefaults[i]?.url !== site.url
) {
adjustedIndex--;
}
}
return adjustedIndex;
}
/**
* Insert a site to pin at a position shifting over any other pinned sites.
*/
_insertPin(site, originalIndex, draggedFromIndex) {
let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex);
// Don't insert any pins past the end of the visible top sites. Otherwise,
// we can end up with a bunch of pinned sites that can never be unpinned again
// from the UI.
const topSitesCount =
this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW;
if (index >= topSitesCount) {
return;
}
let pinned = lazy.NewTabUtils.pinnedLinks.links;
if (!pinned[index]) {
this._pinSiteAt(site, index);
} else {
pinned[draggedFromIndex] = null;
// Find the hole to shift the pinned site(s) towards. We shift towards the
// hole left by the site being dragged.
let holeIndex = index;
const indexStep = index > draggedFromIndex ? -1 : 1;
while (pinned[holeIndex]) {
holeIndex += indexStep;
}
if (holeIndex >= topSitesCount || holeIndex < 0) {
// There are no holes, so we will effectively unpin the last slot and shifting
// towards it. This only happens when adding a new top site to an already
// fully pinned grid.
holeIndex = topSitesCount - 1;
}
// Shift towards the hole.
const shiftingStep = holeIndex > index ? -1 : 1;
while (holeIndex !== index) {
const nextIndex = holeIndex + shiftingStep;
this._pinSiteAt(pinned[nextIndex], holeIndex);
holeIndex = nextIndex;
}
this._pinSiteAt(site, index);
}
}
/**
* Handle an insert (drop/add) action of a site.
*/
async insert(action) {
let { index } = action.data;
// Treat invalid pin index values (e.g., -1, undefined) as insert in the first position
if (!(index > 0)) {
index = 0;
}
// Inserting a top site pins it in the specified slot, pushing over any link already
// pinned in the slot (unless it's the last slot, then it replaces).
this._insertPin(
action.data.site,
index,
action.data.draggedFromIndex !== undefined
? action.data.draggedFromIndex
: this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW
);
await this._clearLinkCustomScreenshot(action.data.site);
this._broadcastPinnedSitesUpdated();
}
updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) {
// Unpin the deletedShortcuts.
deletedShortcuts.forEach(({ url }) => {
lazy.NewTabUtils.pinnedLinks.unpin({ url });
});
// Pin the addedShortcuts.
const numberOfSlots =
this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW;
addedShortcuts.forEach(shortcut => {
// Find first hole in pinnedLinks.
let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link);
if (
index < 0 &&
lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots
) {
// pinnedLinks can have less slots than the total available.
index = lazy.NewTabUtils.pinnedLinks.links.length;
}
if (index >= 0) {
lazy.NewTabUtils.pinnedLinks.pin(shortcut, index);
} else {
// No slots available, we need to do an insert in first slot and push over other pinned links.
this._insertPin(shortcut, 0, numberOfSlots);
}
});
this._broadcastPinnedSitesUpdated();
}
onAction(action) {
switch (action.type) {
case at.INIT:
this.init();
this.updateCustomSearchShortcuts(true /* isStartup */);
break;
case at.SYSTEM_TICK:
this.refresh({ broadcast: false });
this._contile.periodicUpdate();
break;
// All these actions mean we need new top sites
case at.PLACES_HISTORY_CLEARED:
case at.PLACES_LINKS_DELETED:
this.frecentCache.expire();
this.refresh({ broadcast: true });
break;
case at.PLACES_LINKS_CHANGED:
this.frecentCache.expire();
this.refresh({ broadcast: false });
break;
case at.PLACES_LINK_BLOCKED:
this.frecentCache.expire();
this.pinnedCache.expire();
this.refresh({ broadcast: true });
break;
case at.PREF_CHANGED:
switch (action.data.name) {
case DEFAULT_SITES_PREF:
if (!this._useRemoteSetting) {
this.refreshDefaults(action.data.value);
}
break;
case ROWS_PREF:
case FILTER_DEFAULT_SEARCH_PREF:
case SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF:
this.refresh({ broadcast: true });
break;
case SHOW_SPONSORED_PREF:
if (
lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_CONTILE_ENABLED
)
) {
this._contile.refresh();
} else {
this.refresh({ broadcast: true });
}
break;
case SEARCH_SHORTCUTS_EXPERIMENT:
if (action.data.value) {
this.updateCustomSearchShortcuts();
} else {
this.unpinAllSearchShortcuts();
}
this.refresh({ broadcast: true });
}
break;
case at.UPDATE_SECTION_PREFS:
if (action.data.id === SECTION_ID) {
this.updateSectionPrefs(action.data.value);
}
break;
case at.PREFS_INITIAL_VALUES:
if (!this._useRemoteSetting) {
this.refreshDefaults(action.data[DEFAULT_SITES_PREF]);
}
break;
case at.TOP_SITES_PIN:
this.pin(action);
break;
case at.TOP_SITES_UNPIN:
this.unpin(action);
break;
case at.TOP_SITES_INSERT:
this.insert(action);
break;
case at.PREVIEW_REQUEST:
this.getScreenshotPreview(action.data.url, action.meta.fromTarget);
break;
case at.UPDATE_PINNED_SEARCH_SHORTCUTS:
this.updatePinnedSearchShortcuts(action.data);
break;
case at.UNINIT:
this.uninit();
break;
}
}
}
const EXPORTED_SYMBOLS = [
"TopSitesFeed",
"DEFAULT_TOP_SITES",
"ContileIntegration",
];