1/* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4"use strict"; 5 6const { XPCOMUtils } = ChromeUtils.import( 7 "resource://gre/modules/XPCOMUtils.jsm" 8); 9ChromeUtils.defineModuleGetter( 10 this, 11 "NewTabUtils", 12 "resource://gre/modules/NewTabUtils.jsm" 13); 14ChromeUtils.defineModuleGetter( 15 this, 16 "RemoteSettings", 17 "resource://services-settings/remote-settings.js" 18); 19const { setTimeout, clearTimeout } = ChromeUtils.import( 20 "resource://gre/modules/Timer.jsm" 21); 22ChromeUtils.defineModuleGetter( 23 this, 24 "Services", 25 "resource://gre/modules/Services.jsm" 26); 27XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]); 28const { actionTypes: at, actionCreators: ac } = ChromeUtils.import( 29 "resource://activity-stream/common/Actions.jsm" 30); 31ChromeUtils.defineModuleGetter( 32 this, 33 "Region", 34 "resource://gre/modules/Region.jsm" 35); 36ChromeUtils.defineModuleGetter( 37 this, 38 "PersistentCache", 39 "resource://activity-stream/lib/PersistentCache.jsm" 40); 41 42const CACHE_KEY = "discovery_stream"; 43const LAYOUT_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes 44const STARTUP_CACHE_EXPIRE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week 45const COMPONENT_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes 46const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes 47const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour 48const MIN_PERSONALIZATION_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours 49const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server 50const FETCH_TIMEOUT = 45 * 1000; 51const PREF_CONFIG = "discoverystream.config"; 52const PREF_ENDPOINTS = "discoverystream.endpoints"; 53const PREF_IMPRESSION_ID = "browser.newtabpage.activity-stream.impressionId"; 54const PREF_ENABLED = "discoverystream.enabled"; 55const PREF_HARDCODED_BASIC_LAYOUT = "discoverystream.hardcoded-basic-layout"; 56const PREF_SPOCS_ENDPOINT = "discoverystream.spocs-endpoint"; 57const PREF_SPOCS_ENDPOINT_QUERY = "discoverystream.spocs-endpoint-query"; 58const PREF_REGION_BASIC_LAYOUT = "discoverystream.region-basic-layout"; 59const PREF_USER_TOPSTORIES = "feeds.section.topstories"; 60const PREF_SYSTEM_TOPSTORIES = "feeds.system.topstories"; 61const PREF_SPOCS_CLEAR_ENDPOINT = "discoverystream.endpointSpocsClear"; 62const PREF_SHOW_SPONSORED = "showSponsored"; 63const PREF_SPOC_IMPRESSIONS = "discoverystream.spoc.impressions"; 64const PREF_FLIGHT_BLOCKS = "discoverystream.flight.blocks"; 65const PREF_REC_IMPRESSIONS = "discoverystream.rec.impressions"; 66const PREF_COLLECTIONS_ENABLED = 67 "discoverystream.sponsored-collections.enabled"; 68const PREF_COLLECTION_DISMISSIBLE = "discoverystream.isCollectionDismissible"; 69const PREF_PERSONALIZATION = "discoverystream.personalization.enabled"; 70const PREF_PERSONALIZATION_OVERRIDE = 71 "discoverystream.personalization.override"; 72 73let getHardcodedLayout; 74 75this.DiscoveryStreamFeed = class DiscoveryStreamFeed { 76 constructor() { 77 // Internal state for checking if we've intialized all our data 78 this.loaded = false; 79 80 // Persistent cache for remote endpoint data. 81 this.cache = new PersistentCache(CACHE_KEY, true); 82 this.locale = Services.locale.appLocaleAsBCP47; 83 this._impressionId = this.getOrCreateImpressionId(); 84 // Internal in-memory cache for parsing json prefs. 85 this._prefCache = {}; 86 } 87 88 getOrCreateImpressionId() { 89 let impressionId = Services.prefs.getCharPref(PREF_IMPRESSION_ID, ""); 90 if (!impressionId) { 91 impressionId = String(Services.uuid.generateUUID()); 92 Services.prefs.setCharPref(PREF_IMPRESSION_ID, impressionId); 93 } 94 return impressionId; 95 } 96 97 finalLayoutEndpoint(url, apiKey) { 98 if (url.includes("$apiKey") && !apiKey) { 99 throw new Error( 100 `Layout Endpoint - An API key was specified but none configured: ${url}` 101 ); 102 } 103 return url.replace("$apiKey", apiKey); 104 } 105 106 get config() { 107 if (this._prefCache.config) { 108 return this._prefCache.config; 109 } 110 try { 111 this._prefCache.config = JSON.parse( 112 this.store.getState().Prefs.values[PREF_CONFIG] 113 ); 114 const layoutUrl = this._prefCache.config.layout_endpoint; 115 116 const apiKeyPref = this._prefCache.config.api_key_pref; 117 if (layoutUrl && apiKeyPref) { 118 const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); 119 this._prefCache.config.layout_endpoint = this.finalLayoutEndpoint( 120 layoutUrl, 121 apiKey 122 ); 123 } 124 } catch (e) { 125 // istanbul ignore next 126 this._prefCache.config = {}; 127 // istanbul ignore next 128 Cu.reportError( 129 `Could not parse preference. Try resetting ${PREF_CONFIG} in about:config. ${e}` 130 ); 131 } 132 this._prefCache.config.enabled = 133 this._prefCache.config.enabled && 134 this.store.getState().Prefs.values[PREF_ENABLED]; 135 136 return this._prefCache.config; 137 } 138 139 resetConfigDefauts() { 140 this.store.dispatch({ 141 type: at.CLEAR_PREF, 142 data: { 143 name: PREF_CONFIG, 144 }, 145 }); 146 } 147 148 get region() { 149 return Region.home; 150 } 151 152 get showSpocs() { 153 // Combine user-set sponsored opt-out with Mozilla-set config 154 return ( 155 this.store.getState().Prefs.values[PREF_SHOW_SPONSORED] && 156 this.config.show_spocs 157 ); 158 } 159 160 get showStories() { 161 // Combine user-set sponsored opt-out with Mozilla-set config 162 return ( 163 this.store.getState().Prefs.values[PREF_SYSTEM_TOPSTORIES] && 164 this.store.getState().Prefs.values[PREF_USER_TOPSTORIES] 165 ); 166 } 167 168 get personalized() { 169 // If both spocs and recs are not personalized, we might as well return false here. 170 const spocsPersonalized = this.store.getState().Prefs.values?.pocketConfig 171 ?.spocsPersonalized; 172 const recsPersonalized = this.store.getState().Prefs.values?.pocketConfig 173 ?.recsPersonalized; 174 const personalization = this.store.getState().Prefs.values[ 175 PREF_PERSONALIZATION 176 ]; 177 178 // There is a server sent flag to keep personalization on. 179 // If the server stops sending this, we turn personalization off, 180 // until the server starts returning the signal. 181 const overrideState = this.store.getState().Prefs.values[ 182 PREF_PERSONALIZATION_OVERRIDE 183 ]; 184 185 return ( 186 personalization && 187 !overrideState && 188 !!this.recommendationProvider && 189 (spocsPersonalized || recsPersonalized) 190 ); 191 } 192 193 get recommendationProvider() { 194 if (this._recommendationProvider) { 195 return this._recommendationProvider; 196 } 197 this._recommendationProvider = this.store.feeds.get( 198 "feeds.recommendationprovider" 199 ); 200 return this._recommendationProvider; 201 } 202 203 setupPrefs(isStartup = false) { 204 // Send the initial state of the pref on our reducer 205 this.store.dispatch( 206 ac.BroadcastToContent({ 207 type: at.DISCOVERY_STREAM_CONFIG_SETUP, 208 data: this.config, 209 meta: { 210 isStartup, 211 }, 212 }) 213 ); 214 this.store.dispatch( 215 ac.BroadcastToContent({ 216 type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE, 217 data: { 218 value: this.store.getState().Prefs.values[ 219 PREF_COLLECTION_DISMISSIBLE 220 ], 221 }, 222 meta: { 223 isStartup, 224 }, 225 }) 226 ); 227 } 228 229 uninitPrefs() { 230 // Reset in-memory cache 231 this._prefCache = {}; 232 } 233 234 async fetchFromEndpoint(rawEndpoint, options = {}) { 235 if (!rawEndpoint) { 236 Cu.reportError("Tried to fetch endpoint but none was configured."); 237 return null; 238 } 239 240 const apiKeyPref = this._prefCache.config.api_key_pref; 241 const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); 242 243 // The server somtimes returns this value already replaced, but we try this for two reasons: 244 // 1. Layout endpoints are not from the server. 245 // 2. Hardcoded layouts don't have this already done for us. 246 const endpoint = rawEndpoint 247 .replace("$apiKey", apiKey) 248 .replace("$locale", this.locale) 249 .replace("$region", this.region); 250 251 try { 252 // Make sure the requested endpoint is allowed 253 const allowed = this.store 254 .getState() 255 .Prefs.values[PREF_ENDPOINTS].split(","); 256 if (!allowed.some(prefix => endpoint.startsWith(prefix))) { 257 throw new Error(`Not one of allowed prefixes (${allowed})`); 258 } 259 260 const controller = new AbortController(); 261 const { signal } = controller; 262 263 const fetchPromise = fetch(endpoint, { 264 ...options, 265 credentials: "omit", 266 signal, 267 }); 268 // istanbul ignore next 269 const timeoutId = setTimeout(() => { 270 controller.abort(); 271 }, FETCH_TIMEOUT); 272 273 const response = await fetchPromise; 274 if (!response.ok) { 275 throw new Error(`Unexpected status (${response.status})`); 276 } 277 clearTimeout(timeoutId); 278 return response.json(); 279 } catch (error) { 280 Cu.reportError(`Failed to fetch ${endpoint}: ${error.message}`); 281 } 282 return null; 283 } 284 285 /** 286 * Returns true if data in the cache for a particular key has expired or is missing. 287 * @param {object} cachedData data returned from cache.get() 288 * @param {string} key a cache key 289 * @param {string?} url for "feed" only, the URL of the feed. 290 * @param {boolean} is this check done at initial browser load 291 */ 292 isExpired({ cachedData, key, url, isStartup }) { 293 const { layout, spocs, feeds } = cachedData; 294 const updateTimePerComponent = { 295 layout: LAYOUT_UPDATE_TIME, 296 spocs: SPOCS_FEEDS_UPDATE_TIME, 297 feed: COMPONENT_FEEDS_UPDATE_TIME, 298 }; 299 const EXPIRATION_TIME = isStartup 300 ? STARTUP_CACHE_EXPIRE_TIME 301 : updateTimePerComponent[key]; 302 switch (key) { 303 case "layout": 304 // This never needs to expire, as it's not expected to change. 305 if (this.config.hardcoded_layout) { 306 return false; 307 } 308 return !layout || !(Date.now() - layout.lastUpdated < EXPIRATION_TIME); 309 case "spocs": 310 return !spocs || !(Date.now() - spocs.lastUpdated < EXPIRATION_TIME); 311 case "feed": 312 return ( 313 !feeds || 314 !feeds[url] || 315 !(Date.now() - feeds[url].lastUpdated < EXPIRATION_TIME) 316 ); 317 default: 318 // istanbul ignore next 319 throw new Error(`${key} is not a valid key`); 320 } 321 } 322 323 async _checkExpirationPerComponent() { 324 const cachedData = (await this.cache.get()) || {}; 325 const { feeds } = cachedData; 326 return { 327 layout: this.isExpired({ cachedData, key: "layout" }), 328 spocs: this.isExpired({ cachedData, key: "spocs" }), 329 feeds: 330 !feeds || 331 Object.keys(feeds).some(url => 332 this.isExpired({ cachedData, key: "feed", url }) 333 ), 334 }; 335 } 336 337 /** 338 * Returns true if any data for the cached endpoints has expired or is missing. 339 */ 340 async checkIfAnyCacheExpired() { 341 const expirationPerComponent = await this._checkExpirationPerComponent(); 342 return ( 343 expirationPerComponent.layout || 344 expirationPerComponent.spocs || 345 expirationPerComponent.feeds 346 ); 347 } 348 349 async fetchLayout(isStartup) { 350 const cachedData = (await this.cache.get()) || {}; 351 let { layout } = cachedData; 352 if (this.isExpired({ cachedData, key: "layout", isStartup })) { 353 const layoutResponse = await this.fetchFromEndpoint( 354 this.config.layout_endpoint 355 ); 356 if (layoutResponse && layoutResponse.layout) { 357 layout = { 358 lastUpdated: Date.now(), 359 spocs: layoutResponse.spocs, 360 layout: layoutResponse.layout, 361 status: "success", 362 }; 363 364 await this.cache.set("layout", layout); 365 } else { 366 Cu.reportError("No response for response.layout prop"); 367 } 368 } 369 return layout; 370 } 371 372 updatePlacements(sendUpdate, layout, isStartup = false) { 373 const placements = []; 374 const placementsMap = {}; 375 for (const row of layout.filter(r => r.components && r.components.length)) { 376 for (const component of row.components) { 377 if (component.placement) { 378 // Throw away any dupes for the request. 379 if (!placementsMap[component.placement.name]) { 380 placementsMap[component.placement.name] = component.placement; 381 placements.push(component.placement); 382 } 383 } 384 } 385 } 386 if (placements.length) { 387 sendUpdate({ 388 type: at.DISCOVERY_STREAM_SPOCS_PLACEMENTS, 389 data: { placements }, 390 meta: { 391 isStartup, 392 }, 393 }); 394 } 395 } 396 397 /** 398 * Adds a query string to a URL. 399 * A query can be any string literal accepted by https://developer.mozilla.org/docs/Web/API/URLSearchParams 400 * Examples: "?foo=1&bar=2", "&foo=1&bar=2", "foo=1&bar=2", "?bar=2" or "bar=2" 401 */ 402 addEndpointQuery(url, query) { 403 if (!query) { 404 return url; 405 } 406 407 const urlObject = new URL(url); 408 const params = new URLSearchParams(query); 409 410 for (let [key, val] of params.entries()) { 411 urlObject.searchParams.append(key, val); 412 } 413 414 return urlObject.toString(); 415 } 416 417 parseSpocPositions(csvPositions) { 418 let spocPositions; 419 420 // Only accept parseable non-negative integers 421 try { 422 spocPositions = csvPositions.map(index => { 423 let parsedInt = parseInt(index, 10); 424 425 if (!isNaN(parsedInt) && parsedInt >= 0) { 426 return parsedInt; 427 } 428 429 throw new Error("Bad input"); 430 }); 431 } catch (e) { 432 // Catch spoc positions that are not numbers or negative, and do nothing. 433 // We have hard coded backup positions. 434 spocPositions = undefined; 435 } 436 437 return spocPositions; 438 } 439 440 async loadLayout(sendUpdate, isStartup) { 441 let layoutResp = {}; 442 let url = ""; 443 444 if (!this.config.hardcoded_layout) { 445 layoutResp = await this.fetchLayout(isStartup); 446 } 447 448 if (!layoutResp || !layoutResp.layout) { 449 const isBasicLayout = 450 this.config.hardcoded_basic_layout || 451 this.store.getState().Prefs.values[PREF_HARDCODED_BASIC_LAYOUT] || 452 this.store.getState().Prefs.values[PREF_REGION_BASIC_LAYOUT]; 453 454 const sponsoredCollectionsEnabled = this.store.getState().Prefs.values[ 455 PREF_COLLECTIONS_ENABLED 456 ]; 457 458 const pocketConfig = 459 this.store.getState().Prefs.values?.pocketConfig || {}; 460 461 let items = isBasicLayout ? 3 : 21; 462 if ( 463 pocketConfig.compactLayout || 464 pocketConfig.fourCardLayout || 465 pocketConfig.hybridLayout 466 ) { 467 items = isBasicLayout ? 4 : 24; 468 } 469 470 // Set a hardcoded layout if one is needed. 471 // Changing values in this layout in memory object is unnecessary. 472 layoutResp = getHardcodedLayout({ 473 items, 474 sponsoredCollectionsEnabled, 475 spocPositions: this.parseSpocPositions( 476 pocketConfig.spocPositions?.split(`,`) 477 ), 478 compactLayout: pocketConfig.compactLayout, 479 hybridLayout: pocketConfig.hybridLayout, 480 hideCardBackground: pocketConfig.hideCardBackground, 481 fourCardLayout: pocketConfig.fourCardLayout, 482 loadMore: pocketConfig.loadMore, 483 lastCardMessageEnabled: pocketConfig.lastCardMessageEnabled, 484 saveToPocketCard: pocketConfig.saveToPocketCard, 485 newFooterSection: pocketConfig.newFooterSection, 486 hideDescriptions: pocketConfig.hideDescriptions, 487 compactGrid: pocketConfig.compactGrid, 488 compactImages: pocketConfig.compactImages, 489 imageGradient: pocketConfig.imageGradient, 490 newSponsoredLabel: pocketConfig.newSponsoredLabel, 491 titleLines: pocketConfig.titleLines, 492 descLines: pocketConfig.descLines, 493 // For now essentialReadsHeader and editorsPicksHeader are English only. 494 essentialReadsHeader: 495 this.locale.startsWith("en-") && pocketConfig.essentialReadsHeader, 496 editorsPicksHeader: 497 this.locale.startsWith("en-") && pocketConfig.editorsPicksHeader, 498 readTime: pocketConfig.readTime, 499 }); 500 } 501 502 sendUpdate({ 503 type: at.DISCOVERY_STREAM_LAYOUT_UPDATE, 504 data: layoutResp, 505 meta: { 506 isStartup, 507 }, 508 }); 509 510 if (layoutResp.spocs) { 511 url = 512 this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] || 513 this.config.spocs_endpoint || 514 layoutResp.spocs.url; 515 516 const spocsEndpointQuery = this.store.getState().Prefs.values[ 517 PREF_SPOCS_ENDPOINT_QUERY 518 ]; 519 520 // For QA, testing, or debugging purposes, there may be a query string to add. 521 url = this.addEndpointQuery(url, spocsEndpointQuery); 522 523 if ( 524 url && 525 url !== this.store.getState().DiscoveryStream.spocs.spocs_endpoint 526 ) { 527 sendUpdate({ 528 type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT, 529 data: { 530 url, 531 }, 532 meta: { 533 isStartup, 534 }, 535 }); 536 this.updatePlacements(sendUpdate, layoutResp.layout, isStartup); 537 } 538 } 539 } 540 541 /** 542 * buildFeedPromise - Adds the promise result to newFeeds and 543 * pushes a promise to newsFeedsPromises. 544 * @param {Object} Has both newFeedsPromises (Array) and newFeeds (Object) 545 * @param {Boolean} isStartup We have different cache handling for startup. 546 * @returns {Function} We return a function so we can contain 547 * the scope for isStartup and the promises object. 548 * Combines feed results and promises for each component with a feed. 549 */ 550 buildFeedPromise( 551 { newFeedsPromises, newFeeds }, 552 isStartup = false, 553 sendUpdate 554 ) { 555 return component => { 556 const { url } = component.feed; 557 558 if (!newFeeds[url]) { 559 // We initially stub this out so we don't fetch dupes, 560 // we then fill in with the proper object inside the promise. 561 newFeeds[url] = {}; 562 const feedPromise = this.getComponentFeed(url, isStartup); 563 564 feedPromise 565 .then(feed => { 566 // If we stored the result of filter in feed cache as it happened, 567 // I think we could reduce doing this for cache fetches. 568 // Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606277 569 newFeeds[url] = this.filterRecommendations(feed); 570 sendUpdate({ 571 type: at.DISCOVERY_STREAM_FEED_UPDATE, 572 data: { 573 feed: newFeeds[url], 574 url, 575 }, 576 meta: { 577 isStartup, 578 }, 579 }); 580 }) 581 .catch( 582 /* istanbul ignore next */ error => { 583 Cu.reportError( 584 `Error trying to load component feed ${url}: ${error}` 585 ); 586 } 587 ); 588 newFeedsPromises.push(feedPromise); 589 } 590 }; 591 } 592 593 filterRecommendations(feed) { 594 if ( 595 feed && 596 feed.data && 597 feed.data.recommendations && 598 feed.data.recommendations.length 599 ) { 600 const { data: recommendations } = this.filterBlocked( 601 feed.data.recommendations 602 ); 603 return { 604 ...feed, 605 data: { 606 ...feed.data, 607 recommendations, 608 }, 609 }; 610 } 611 return feed; 612 } 613 614 /** 615 * reduceFeedComponents - Filters out components with no feeds, and combines 616 * all feeds on this component with the feeds from other components. 617 * @param {Boolean} isStartup We have different cache handling for startup. 618 * @returns {Function} We return a function so we can contain the scope for isStartup. 619 * Reduces feeds into promises and feed data. 620 */ 621 reduceFeedComponents(isStartup, sendUpdate) { 622 return (accumulator, row) => { 623 row.components 624 .filter(component => component && component.feed) 625 .forEach(this.buildFeedPromise(accumulator, isStartup, sendUpdate)); 626 return accumulator; 627 }; 628 } 629 630 /** 631 * buildFeedPromises - Filters out rows with no components, 632 * and gets us a promise for each unique feed. 633 * @param {Object} layout This is the Discovery Stream layout object. 634 * @param {Boolean} isStartup We have different cache handling for startup. 635 * @returns {Object} An object with newFeedsPromises (Array) and newFeeds (Object), 636 * we can Promise.all newFeedsPromises to get completed data in newFeeds. 637 */ 638 buildFeedPromises(layout, isStartup, sendUpdate) { 639 const initialData = { 640 newFeedsPromises: [], 641 newFeeds: {}, 642 }; 643 return layout 644 .filter(row => row && row.components) 645 .reduce(this.reduceFeedComponents(isStartup, sendUpdate), initialData); 646 } 647 648 async loadComponentFeeds(sendUpdate, isStartup = false) { 649 const { DiscoveryStream } = this.store.getState(); 650 651 if (!DiscoveryStream || !DiscoveryStream.layout) { 652 return; 653 } 654 655 // Reset the flag that indicates whether or not at least one API request 656 // was issued to fetch the component feed in `getComponentFeed()`. 657 this.componentFeedFetched = false; 658 const { newFeedsPromises, newFeeds } = this.buildFeedPromises( 659 DiscoveryStream.layout, 660 isStartup, 661 sendUpdate 662 ); 663 664 // Each promise has a catch already built in, so no need to catch here. 665 await Promise.all(newFeedsPromises); 666 667 if (this.componentFeedFetched) { 668 this.cleanUpTopRecImpressionPref(newFeeds); 669 } 670 await this.cache.set("feeds", newFeeds); 671 sendUpdate({ 672 type: at.DISCOVERY_STREAM_FEEDS_UPDATE, 673 meta: { 674 isStartup, 675 }, 676 }); 677 } 678 679 getPlacements() { 680 const { placements } = this.store.getState().DiscoveryStream.spocs; 681 // Backwards comp for before we had placements, assume just a single spocs placement. 682 if (!placements || !placements.length) { 683 return [{ name: "spocs" }]; 684 } 685 return placements; 686 } 687 688 // I wonder, can this be better as a reducer? 689 // See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717 690 placementsForEach(callback) { 691 this.getPlacements().forEach(callback); 692 } 693 694 // Bug 1567271 introduced meta data on a list of spocs. 695 // This involved moving the spocs array into an items prop. 696 // However, old data could still be returned, and cached data might also be old. 697 // For ths reason, we want to ensure if we don't find an items array, 698 // we use the previous array placement, and then stub out title and context to empty strings. 699 // We need to do this *after* both fresh fetches and cached data to reduce repetition. 700 normalizeSpocsItems(spocs) { 701 const items = spocs.items || spocs; 702 const title = spocs.title || ""; 703 const context = spocs.context || ""; 704 const sponsor = spocs.sponsor || ""; 705 // We do not stub sponsored_by_override with an empty string. It is an override, and an empty string 706 // explicitly means to override the client to display an empty string. 707 // An empty string is not an no op in this case. Undefined is the proper no op here. 708 const { sponsored_by_override } = spocs; 709 // Undefined is fine here. It's optional and only used by collections. 710 // If we leave it out, you get a collection that cannot be dismissed. 711 const { flight_id } = spocs; 712 return { 713 items, 714 title, 715 context, 716 sponsor, 717 sponsored_by_override, 718 ...(flight_id ? { flight_id } : {}), 719 }; 720 } 721 722 // This turns personalization on/off if the server sends the override command. 723 // The server sends a true signal to keep personalization on. So a malfunctioning 724 // server would more likely mistakenly turn off personalization, and not turn it on. 725 // This is safer, because the override is for cases where personalization is causing issues. 726 // So having it mistakenly go off is safe, but it mistakenly going on could be bad. 727 personalizationOverride(overrideCommand) { 728 // Are we currently in an override state. 729 // This is useful to know if we want to do a cleanup. 730 const overrideState = this.store.getState().Prefs.values[ 731 PREF_PERSONALIZATION_OVERRIDE 732 ]; 733 734 // Is this profile currently set to be personalized. 735 const personalization = this.store.getState().Prefs.values[ 736 PREF_PERSONALIZATION 737 ]; 738 739 // If we have an override command, profile is currently personalized, 740 // and is not currently being overridden, we can set the override pref. 741 if (overrideCommand && personalization && !overrideState) { 742 this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION_OVERRIDE, true)); 743 } 744 745 // This is if we need to revert an override and do cleanup. 746 // We do this if we are in an override state, 747 // but not currently receiving the override signal. 748 if (!overrideCommand && overrideState) { 749 this.store.dispatch({ 750 type: at.CLEAR_PREF, 751 data: { name: PREF_PERSONALIZATION_OVERRIDE }, 752 }); 753 } 754 } 755 756 updateSponsoredCollectionsPref(collectionEnabled = false) { 757 const currentState = this.store.getState().Prefs.values[ 758 PREF_COLLECTIONS_ENABLED 759 ]; 760 761 // If the current state does not match the new state, update the pref. 762 if (currentState !== collectionEnabled) { 763 this.store.dispatch( 764 ac.SetPref(PREF_COLLECTIONS_ENABLED, collectionEnabled) 765 ); 766 } 767 } 768 769 async loadSpocs(sendUpdate, isStartup) { 770 const cachedData = (await this.cache.get()) || {}; 771 let spocsState; 772 773 const { placements } = this.store.getState().DiscoveryStream.spocs; 774 775 if (this.showSpocs) { 776 spocsState = cachedData.spocs; 777 if (this.isExpired({ cachedData, key: "spocs", isStartup })) { 778 const endpoint = this.store.getState().DiscoveryStream.spocs 779 .spocs_endpoint; 780 781 const headers = new Headers(); 782 headers.append("content-type", "application/json"); 783 784 const apiKeyPref = this._prefCache.config.api_key_pref; 785 const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); 786 787 const spocsResponse = await this.fetchFromEndpoint(endpoint, { 788 method: "POST", 789 headers, 790 body: JSON.stringify({ 791 pocket_id: this._impressionId, 792 version: 2, 793 consumer_key: apiKey, 794 ...(placements.length ? { placements } : {}), 795 }), 796 }); 797 798 if (spocsResponse) { 799 spocsState = { 800 lastUpdated: Date.now(), 801 spocs: { 802 ...spocsResponse, 803 }, 804 }; 805 806 if (spocsResponse.settings && spocsResponse.settings.feature_flags) { 807 this.personalizationOverride( 808 // The server's old signal was for a version override. 809 // When we removed version 1, version 2 was now the defacto only version. 810 // Without a version 1, the override is now a command to turn off personalization. 811 !spocsResponse.settings.feature_flags.spoc_v2 812 ); 813 this.updateSponsoredCollectionsPref( 814 spocsResponse.settings.feature_flags.collections 815 ); 816 } 817 818 const spocsResultPromises = this.getPlacements().map( 819 async placement => { 820 const freshSpocs = spocsState.spocs[placement.name]; 821 822 if (!freshSpocs) { 823 return; 824 } 825 826 // spocs can be returns as an array, or an object with an items array. 827 // We want to normalize this so all our spocs have an items array. 828 // There can also be some meta data for title and context. 829 // This is mostly because of backwards compat. 830 const { 831 items: normalizedSpocsItems, 832 title, 833 context, 834 sponsor, 835 sponsored_by_override, 836 } = this.normalizeSpocsItems(freshSpocs); 837 838 if (!normalizedSpocsItems || !normalizedSpocsItems.length) { 839 // In the case of old data, we still want to ensure we normalize the data structure, 840 // even if it's empty. We expect the empty data to be an object with items array, 841 // and not just an empty array. 842 spocsState.spocs = { 843 ...spocsState.spocs, 844 [placement.name]: { 845 title, 846 context, 847 items: [], 848 }, 849 }; 850 return; 851 } 852 853 // Migrate flight_id 854 const { data: migratedSpocs } = this.migrateFlightId( 855 normalizedSpocsItems 856 ); 857 858 const { data: capResult } = this.frequencyCapSpocs(migratedSpocs); 859 860 const { data: blockedResults } = this.filterBlocked(capResult); 861 862 const { data: scoredResults } = await this.scoreItems( 863 blockedResults, 864 "spocs" 865 ); 866 867 spocsState.spocs = { 868 ...spocsState.spocs, 869 [placement.name]: { 870 title, 871 context, 872 sponsor, 873 sponsored_by_override, 874 items: scoredResults, 875 }, 876 }; 877 } 878 ); 879 await Promise.all(spocsResultPromises); 880 881 this.cleanUpFlightImpressionPref(spocsState.spocs); 882 await this.cache.set("spocs", { 883 lastUpdated: spocsState.lastUpdated, 884 spocs: spocsState.spocs, 885 }); 886 } else { 887 Cu.reportError("No response for spocs_endpoint prop"); 888 } 889 } 890 } 891 892 // Use good data if we have it, otherwise nothing. 893 // We can have no data if spocs set to off. 894 // We can have no data if request fails and there is no good cache. 895 // We want to send an update spocs or not, so client can render something. 896 spocsState = 897 spocsState && spocsState.spocs 898 ? spocsState 899 : { 900 lastUpdated: Date.now(), 901 spocs: {}, 902 }; 903 904 sendUpdate({ 905 type: at.DISCOVERY_STREAM_SPOCS_UPDATE, 906 data: { 907 lastUpdated: spocsState.lastUpdated, 908 spocs: spocsState.spocs, 909 }, 910 meta: { 911 isStartup, 912 }, 913 }); 914 } 915 916 async clearSpocs() { 917 const endpoint = this.store.getState().Prefs.values[ 918 PREF_SPOCS_CLEAR_ENDPOINT 919 ]; 920 if (!endpoint) { 921 return; 922 } 923 const headers = new Headers(); 924 headers.append("content-type", "application/json"); 925 926 await this.fetchFromEndpoint(endpoint, { 927 method: "DELETE", 928 headers, 929 body: JSON.stringify({ 930 pocket_id: this._impressionId, 931 }), 932 }); 933 } 934 935 /* 936 * This just re hydrates the provider from cache. 937 * We can call this on startup because it's generally fast. 938 * It reports to devtools the last time the data in the cache was updated. 939 */ 940 async loadPersonalizationScoresCache(isStartup = false) { 941 const cachedData = (await this.cache.get()) || {}; 942 const { personalization } = cachedData; 943 944 if (this.personalized && personalization && personalization.scores) { 945 this.recommendationProvider.setProvider(personalization.scores); 946 947 this.personalizationLastUpdated = personalization._timestamp; 948 949 this.store.dispatch( 950 ac.BroadcastToContent({ 951 type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED, 952 data: { 953 lastUpdated: this.personalizationLastUpdated, 954 }, 955 meta: { 956 isStartup, 957 }, 958 }) 959 ); 960 } 961 } 962 963 /* 964 * This creates a new recommendationProvider using fresh data, 965 * It's run on a last updated timer. This is the opposite of loadPersonalizationScoresCache. 966 * This is also much slower so we only trigger this in the background on idle-daily. 967 * It causes new profiles to pick up personalization slowly because the first time 968 * a new profile is run you don't have any old cache to use, so it needs to wait for the first 969 * idle-daily. Older profiles can rely on cache during the idle-daily gap. Idle-daily is 970 * usually run once every 24 hours. 971 */ 972 async updatePersonalizationScores() { 973 if ( 974 !this.personalized || 975 Date.now() - this.personalizationLastUpdated < 976 MIN_PERSONALIZATION_UPDATE_TIME 977 ) { 978 return; 979 } 980 981 this.recommendationProvider.setProvider(); 982 983 await this.recommendationProvider.init(); 984 985 const personalization = { scores: this.recommendationProvider.getScores() }; 986 this.personalizationLastUpdated = Date.now(); 987 988 this.store.dispatch( 989 ac.BroadcastToContent({ 990 type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED, 991 data: { 992 lastUpdated: this.personalizationLastUpdated, 993 }, 994 }) 995 ); 996 personalization._timestamp = this.personalizationLastUpdated; 997 this.cache.set("personalization", personalization); 998 } 999 1000 observe(subject, topic, data) { 1001 switch (topic) { 1002 case "idle-daily": 1003 this.updatePersonalizationScores(); 1004 break; 1005 } 1006 } 1007 1008 /* 1009 * This function is used to sort any type of story, both spocs and recs. 1010 * This uses hierarchical sorting, first sorting by priority, then by score within a priority. 1011 * This function could be sorting an array of spocs or an array of recs. 1012 * A rec would have priority undefined, and a spoc would probably have a priority set. 1013 * Priority is sorted ascending, so low numbers are the highest priority. 1014 * Score is sorted descending, so high numbers are the highest score. 1015 * Undefined priority values are considered the lowest priority. 1016 * A negative priority is considered the same as undefined, lowest priority. 1017 * A negative priority is unlikely and not currently supported or expected. 1018 * A negative score is a possible use case. 1019 */ 1020 sortItem(a, b) { 1021 // If the priorities are the same, sort based on score. 1022 // If both item priorities are undefined, 1023 // we can safely sort via score. 1024 if (a.priority === b.priority) { 1025 return b.score - a.score; 1026 } else if (!a.priority || a.priority <= 0) { 1027 // If priority is undefined or an unexpected value, 1028 // consider it lowest priority. 1029 return 1; 1030 } else if (!b.priority || b.priority <= 0) { 1031 // Also consider this case lowest priority. 1032 return -1; 1033 } 1034 // Our primary sort for items with priority. 1035 return a.priority - b.priority; 1036 } 1037 1038 async scoreItems(items, type) { 1039 const spocsPersonalized = this.store.getState().Prefs.values?.pocketConfig 1040 ?.spocsPersonalized; 1041 const recsPersonalized = this.store.getState().Prefs.values?.pocketConfig 1042 ?.recsPersonalized; 1043 const personalizedByType = 1044 type === "feed" ? recsPersonalized : spocsPersonalized; 1045 1046 const data = ( 1047 await Promise.all( 1048 items.map(item => this.scoreItem(item, personalizedByType)) 1049 ) 1050 ) 1051 // Sort by highest scores. 1052 .sort(this.sortItem); 1053 1054 return { data }; 1055 } 1056 1057 async scoreItem(item, personalizedByType) { 1058 item.score = item.item_score; 1059 if (item.score !== 0 && !item.score) { 1060 item.score = 1; 1061 } 1062 if (this.personalized && personalizedByType) { 1063 await this.recommendationProvider.calculateItemRelevanceScore(item); 1064 } 1065 return item; 1066 } 1067 1068 filterBlocked(data) { 1069 if (data && data.length) { 1070 let flights = this.readDataPref(PREF_FLIGHT_BLOCKS); 1071 const filteredItems = data.filter(item => { 1072 const blocked = 1073 NewTabUtils.blockedLinks.isBlocked({ url: item.url }) || 1074 flights[item.flight_id]; 1075 return !blocked; 1076 }); 1077 return { data: filteredItems }; 1078 } 1079 return { data }; 1080 } 1081 1082 // For backwards compatibility, older spoc endpoint don't have flight_id, 1083 // but instead had campaign_id we can use 1084 // 1085 // @param {Object} data An object that might have a SPOCS array. 1086 // @returns {Object} An object with a property `data` as the result. 1087 migrateFlightId(spocs) { 1088 if (spocs && spocs.length) { 1089 return { 1090 data: spocs.map(s => { 1091 return { 1092 ...s, 1093 ...(s.flight_id || s.campaign_id 1094 ? { 1095 flight_id: s.flight_id || s.campaign_id, 1096 } 1097 : {}), 1098 ...(s.caps 1099 ? { 1100 caps: { 1101 ...s.caps, 1102 flight: s.caps.flight || s.caps.campaign, 1103 }, 1104 } 1105 : {}), 1106 }; 1107 }), 1108 }; 1109 } 1110 return { data: spocs }; 1111 } 1112 1113 // Filter spocs based on frequency caps 1114 // 1115 // @param {Object} data An object that might have a SPOCS array. 1116 // @returns {Object} An object with a property `data` as the result, and a property 1117 // `filterItems` as the frequency capped items. 1118 frequencyCapSpocs(spocs) { 1119 if (spocs && spocs.length) { 1120 const impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS); 1121 const caps = []; 1122 const result = spocs.filter(s => { 1123 const isBelow = this.isBelowFrequencyCap(impressions, s); 1124 if (!isBelow) { 1125 caps.push(s); 1126 } 1127 return isBelow; 1128 }); 1129 // send caps to redux if any. 1130 if (caps.length) { 1131 this.store.dispatch({ 1132 type: at.DISCOVERY_STREAM_SPOCS_CAPS, 1133 data: caps, 1134 }); 1135 } 1136 return { data: result, filtered: caps }; 1137 } 1138 return { data: spocs, filtered: [] }; 1139 } 1140 1141 // Frequency caps are based on flight, which may include multiple spocs. 1142 // We currently support two types of frequency caps: 1143 // - lifetime: Indicates how many times spocs from a flight can be shown in total 1144 // - period: Indicates how many times spocs from a flight can be shown within a period 1145 // 1146 // So, for example, the feed configuration below defines that for flight 1 no more 1147 // than 5 spocs can be shown in total, and no more than 2 per hour. 1148 // "flight_id": 1, 1149 // "caps": { 1150 // "lifetime": 5, 1151 // "flight": { 1152 // "count": 2, 1153 // "period": 3600 1154 // } 1155 // } 1156 isBelowFrequencyCap(impressions, spoc) { 1157 const flightImpressions = impressions[spoc.flight_id]; 1158 if (!flightImpressions) { 1159 return true; 1160 } 1161 1162 const lifetime = spoc.caps && spoc.caps.lifetime; 1163 1164 const lifeTimeCap = Math.min( 1165 lifetime || MAX_LIFETIME_CAP, 1166 MAX_LIFETIME_CAP 1167 ); 1168 const lifeTimeCapExceeded = flightImpressions.length >= lifeTimeCap; 1169 if (lifeTimeCapExceeded) { 1170 return false; 1171 } 1172 1173 const flightCap = spoc.caps && spoc.caps.flight; 1174 if (flightCap) { 1175 const flightCapExceeded = 1176 flightImpressions.filter(i => Date.now() - i < flightCap.period * 1000) 1177 .length >= flightCap.count; 1178 return !flightCapExceeded; 1179 } 1180 return true; 1181 } 1182 1183 async retryFeed(feed) { 1184 const { url } = feed; 1185 const result = await this.getComponentFeed(url); 1186 const newFeed = this.filterRecommendations(result); 1187 this.store.dispatch( 1188 ac.BroadcastToContent({ 1189 type: at.DISCOVERY_STREAM_FEED_UPDATE, 1190 data: { 1191 feed: newFeed, 1192 url, 1193 }, 1194 }) 1195 ); 1196 } 1197 1198 async getComponentFeed(feedUrl, isStartup) { 1199 const cachedData = (await this.cache.get()) || {}; 1200 const { feeds } = cachedData; 1201 1202 let feed = feeds ? feeds[feedUrl] : null; 1203 if (this.isExpired({ cachedData, key: "feed", url: feedUrl, isStartup })) { 1204 const feedResponse = await this.fetchFromEndpoint(feedUrl); 1205 if (feedResponse) { 1206 const { data: scoredItems } = await this.scoreItems( 1207 feedResponse.recommendations, 1208 "feed" 1209 ); 1210 const { recsExpireTime } = feedResponse.settings; 1211 const recommendations = this.rotate(scoredItems, recsExpireTime); 1212 this.componentFeedFetched = true; 1213 feed = { 1214 lastUpdated: Date.now(), 1215 data: { 1216 settings: feedResponse.settings, 1217 recommendations, 1218 status: "success", 1219 }, 1220 }; 1221 } else { 1222 Cu.reportError("No response for feed"); 1223 } 1224 } 1225 1226 // If we have no feed at this point, both fetch and cache failed for some reason. 1227 return ( 1228 feed || { 1229 data: { 1230 status: "failed", 1231 }, 1232 } 1233 ); 1234 } 1235 1236 /** 1237 * Called at startup to update cached data in the background. 1238 */ 1239 async _maybeUpdateCachedData() { 1240 const expirationPerComponent = await this._checkExpirationPerComponent(); 1241 // Pass in `store.dispatch` to send the updates only to main 1242 if (expirationPerComponent.layout) { 1243 await this.loadLayout(this.store.dispatch); 1244 } 1245 if (expirationPerComponent.spocs) { 1246 await this.loadSpocs(this.store.dispatch); 1247 } 1248 if (expirationPerComponent.feeds) { 1249 await this.loadComponentFeeds(this.store.dispatch); 1250 } 1251 } 1252 1253 /** 1254 * @typedef {Object} RefreshAll 1255 * @property {boolean} updateOpenTabs - Sends updates to open tabs immediately if true, 1256 * updates in background if false 1257 * @property {boolean} isStartup - When the function is called at browser startup 1258 * 1259 * Refreshes layout, component feeds, and spocs in order if caches have expired. 1260 * @param {RefreshAll} options 1261 */ 1262 async refreshAll(options = {}) { 1263 const personalizationCacheLoadPromise = this.loadPersonalizationScoresCache( 1264 options.isStartup 1265 ); 1266 1267 const spocsPersonalized = this.store.getState().Prefs.values?.pocketConfig 1268 ?.spocsPersonalized; 1269 const recsPersonalized = this.store.getState().Prefs.values?.pocketConfig 1270 ?.recsPersonalized; 1271 1272 let expirationPerComponent = {}; 1273 if (this.personalized) { 1274 // We store this before we refresh content. 1275 // This way, we can know what and if something got updated, 1276 // so we can know to score the results. 1277 expirationPerComponent = await this._checkExpirationPerComponent(); 1278 } 1279 await this.refreshContent(options); 1280 1281 if (this.personalized) { 1282 // personalizationCacheLoadPromise is probably done, because of the refreshContent await above, 1283 // but to be sure, we should check that it's done, without making the parent function wait. 1284 personalizationCacheLoadPromise.then(() => { 1285 // If we don't have expired stories or feeds, we don't need to score after init. 1286 // If we do have expired stories, we want to score after init. 1287 // In both cases, we don't want these to block the parent function. 1288 // This is why we store the promise, and call then to do our scoring work. 1289 const initPromise = this.recommendationProvider.init(); 1290 initPromise.then(() => { 1291 // Both scoreFeeds and scoreSpocs are promises, 1292 // but they don't need to wait for each other. 1293 // We can just fire them and forget at this point. 1294 const { feeds, spocs } = this.store.getState().DiscoveryStream; 1295 if ( 1296 recsPersonalized && 1297 feeds.loaded && 1298 expirationPerComponent.feeds 1299 ) { 1300 this.scoreFeeds(feeds); 1301 } 1302 if ( 1303 spocsPersonalized && 1304 spocs.loaded && 1305 expirationPerComponent.spocs 1306 ) { 1307 this.scoreSpocs(spocs); 1308 } 1309 }); 1310 }); 1311 } 1312 } 1313 1314 async scoreFeeds(feedsState) { 1315 if (feedsState.data) { 1316 const feeds = {}; 1317 const feedsPromises = Object.keys(feedsState.data).map(url => { 1318 let feed = feedsState.data[url]; 1319 const feedPromise = this.scoreItems(feed.data.recommendations, "feed"); 1320 feedPromise.then(({ data: scoredItems }) => { 1321 const { recsExpireTime } = feed.data.settings; 1322 const recommendations = this.rotate(scoredItems, recsExpireTime); 1323 feed = { 1324 ...feed, 1325 data: { 1326 ...feed.data, 1327 recommendations, 1328 }, 1329 }; 1330 1331 feeds[url] = feed; 1332 1333 this.store.dispatch( 1334 ac.AlsoToPreloaded({ 1335 type: at.DISCOVERY_STREAM_FEED_UPDATE, 1336 data: { 1337 feed, 1338 url, 1339 }, 1340 }) 1341 ); 1342 }); 1343 return feedPromise; 1344 }); 1345 await Promise.all(feedsPromises); 1346 await this.cache.set("feeds", feeds); 1347 } 1348 } 1349 1350 async scoreSpocs(spocsState) { 1351 const spocsResultPromises = this.getPlacements().map(async placement => { 1352 const nextSpocs = spocsState.data[placement.name] || {}; 1353 const { items } = nextSpocs; 1354 1355 if (!items || !items.length) { 1356 return; 1357 } 1358 1359 const { data: scoreResult } = await this.scoreItems(items, "spocs"); 1360 1361 spocsState.data = { 1362 ...spocsState.data, 1363 [placement.name]: { 1364 ...nextSpocs, 1365 items: scoreResult, 1366 }, 1367 }; 1368 }); 1369 await Promise.all(spocsResultPromises); 1370 1371 // Update cache here so we don't need to re calculate scores on loads from cache. 1372 // Related Bug 1606276 1373 await this.cache.set("spocs", { 1374 lastUpdated: spocsState.lastUpdated, 1375 spocs: spocsState.data, 1376 }); 1377 this.store.dispatch( 1378 ac.AlsoToPreloaded({ 1379 type: at.DISCOVERY_STREAM_SPOCS_UPDATE, 1380 data: { 1381 lastUpdated: spocsState.lastUpdated, 1382 spocs: spocsState.data, 1383 }, 1384 }) 1385 ); 1386 } 1387 1388 async refreshContent(options = {}) { 1389 const { updateOpenTabs, isStartup } = options; 1390 1391 const dispatch = updateOpenTabs 1392 ? action => this.store.dispatch(ac.BroadcastToContent(action)) 1393 : this.store.dispatch; 1394 1395 await this.loadLayout(dispatch, isStartup); 1396 if (this.showStories) { 1397 await Promise.all([ 1398 this.loadSpocs(dispatch, isStartup).catch(error => 1399 Cu.reportError(`Error trying to load spocs feeds: ${error}`) 1400 ), 1401 this.loadComponentFeeds(dispatch, isStartup).catch(error => 1402 Cu.reportError(`Error trying to load component feeds: ${error}`) 1403 ), 1404 ]); 1405 if (isStartup) { 1406 await this._maybeUpdateCachedData(); 1407 } 1408 } 1409 } 1410 1411 // We have to rotate stories on the client so that 1412 // active stories are at the front of the list, followed by stories that have expired 1413 // impressions i.e. have been displayed for longer than recsExpireTime. 1414 rotate(recommendations, recsExpireTime) { 1415 const maxImpressionAge = Math.max( 1416 recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, 1417 DEFAULT_RECS_EXPIRE_TIME 1418 ); 1419 const impressions = this.readDataPref(PREF_REC_IMPRESSIONS); 1420 const expired = []; 1421 const active = []; 1422 for (const item of recommendations) { 1423 if ( 1424 impressions[item.id] && 1425 Date.now() - impressions[item.id] >= maxImpressionAge 1426 ) { 1427 expired.push(item); 1428 } else { 1429 active.push(item); 1430 } 1431 } 1432 return active.concat(expired); 1433 } 1434 1435 enableStories() { 1436 if (this.config.enabled && this.loaded) { 1437 // If stories are being re enabled, ensure we have stories. 1438 this.refreshAll({ updateOpenTabs: true }); 1439 } 1440 } 1441 1442 async enable() { 1443 await this.refreshAll({ updateOpenTabs: true, isStartup: true }); 1444 Services.obs.addObserver(this, "idle-daily"); 1445 this.loaded = true; 1446 } 1447 1448 async reset() { 1449 this.resetDataPrefs(); 1450 await this.resetCache(); 1451 if (this.loaded) { 1452 Services.obs.removeObserver(this, "idle-daily"); 1453 } 1454 this.resetState(); 1455 } 1456 1457 async resetCache() { 1458 await this.resetAllCache(); 1459 } 1460 1461 async resetContentCache() { 1462 await this.cache.set("layout", {}); 1463 await this.cache.set("feeds", {}); 1464 await this.cache.set("spocs", {}); 1465 } 1466 1467 async resetAllCache() { 1468 await this.resetContentCache(); 1469 await this.cache.set("personalization", {}); 1470 } 1471 1472 resetDataPrefs() { 1473 this.writeDataPref(PREF_SPOC_IMPRESSIONS, {}); 1474 this.writeDataPref(PREF_REC_IMPRESSIONS, {}); 1475 this.writeDataPref(PREF_FLIGHT_BLOCKS, {}); 1476 } 1477 1478 resetState() { 1479 // Reset reducer 1480 this.store.dispatch( 1481 ac.BroadcastToContent({ type: at.DISCOVERY_STREAM_LAYOUT_RESET }) 1482 ); 1483 this.store.dispatch( 1484 ac.BroadcastToContent({ 1485 type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE, 1486 data: { 1487 value: this.store.getState().Prefs.values[ 1488 PREF_COLLECTION_DISMISSIBLE 1489 ], 1490 }, 1491 }) 1492 ); 1493 this.personalizationLastUpdated = null; 1494 this.loaded = false; 1495 } 1496 1497 async onPrefChange() { 1498 // We always want to clear the cache/state if the pref has changed 1499 await this.reset(); 1500 if (this.config.enabled) { 1501 // Load data from all endpoints 1502 await this.enable(); 1503 } 1504 } 1505 1506 // This is a request to change the config from somewhere. 1507 // Can be from a spefic pref related to Discovery Stream, 1508 // or can be a generic request from an external feed that 1509 // something changed. 1510 configReset() { 1511 this._prefCache.config = null; 1512 this.store.dispatch( 1513 ac.BroadcastToContent({ 1514 type: at.DISCOVERY_STREAM_CONFIG_CHANGE, 1515 data: this.config, 1516 }) 1517 ); 1518 } 1519 1520 recordFlightImpression(flightId) { 1521 let impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS); 1522 1523 const timeStamps = impressions[flightId] || []; 1524 timeStamps.push(Date.now()); 1525 impressions = { ...impressions, [flightId]: timeStamps }; 1526 1527 this.writeDataPref(PREF_SPOC_IMPRESSIONS, impressions); 1528 } 1529 1530 recordTopRecImpressions(recId) { 1531 let impressions = this.readDataPref(PREF_REC_IMPRESSIONS); 1532 if (!impressions[recId]) { 1533 impressions = { ...impressions, [recId]: Date.now() }; 1534 this.writeDataPref(PREF_REC_IMPRESSIONS, impressions); 1535 } 1536 } 1537 1538 recordBlockFlightId(flightId) { 1539 const flights = this.readDataPref(PREF_FLIGHT_BLOCKS); 1540 if (!flights[flightId]) { 1541 flights[flightId] = 1; 1542 this.writeDataPref(PREF_FLIGHT_BLOCKS, flights); 1543 } 1544 } 1545 1546 cleanUpFlightImpressionPref(data) { 1547 let flightIds = []; 1548 this.placementsForEach(placement => { 1549 const newSpocs = data[placement.name]; 1550 if (!newSpocs) { 1551 return; 1552 } 1553 1554 const items = newSpocs.items || []; 1555 flightIds = [...flightIds, ...items.map(s => `${s.flight_id}`)]; 1556 }); 1557 if (flightIds && flightIds.length) { 1558 this.cleanUpImpressionPref( 1559 id => !flightIds.includes(id), 1560 PREF_SPOC_IMPRESSIONS 1561 ); 1562 } 1563 } 1564 1565 // Clean up rec impression pref by removing all stories that are no 1566 // longer part of the response. 1567 cleanUpTopRecImpressionPref(newFeeds) { 1568 // Need to build a single list of stories. 1569 const activeStories = Object.keys(newFeeds) 1570 .filter(currentValue => newFeeds[currentValue].data) 1571 .reduce((accumulator, currentValue) => { 1572 const { recommendations } = newFeeds[currentValue].data; 1573 return accumulator.concat(recommendations.map(i => `${i.id}`)); 1574 }, []); 1575 this.cleanUpImpressionPref( 1576 id => !activeStories.includes(id), 1577 PREF_REC_IMPRESSIONS 1578 ); 1579 } 1580 1581 writeDataPref(pref, impressions) { 1582 this.store.dispatch(ac.SetPref(pref, JSON.stringify(impressions))); 1583 } 1584 1585 readDataPref(pref) { 1586 const prefVal = this.store.getState().Prefs.values[pref]; 1587 return prefVal ? JSON.parse(prefVal) : {}; 1588 } 1589 1590 cleanUpImpressionPref(isExpired, pref) { 1591 const impressions = this.readDataPref(pref); 1592 let changed = false; 1593 1594 Object.keys(impressions).forEach(id => { 1595 if (isExpired(id)) { 1596 changed = true; 1597 delete impressions[id]; 1598 } 1599 }); 1600 1601 if (changed) { 1602 this.writeDataPref(pref, impressions); 1603 } 1604 } 1605 1606 onPocketConfigChanged() { 1607 // Update layout, and reload any off screen tabs. 1608 // This does not change any existing open tabs. 1609 // It also doesn't update any spoc or rec data, just the layout. 1610 const dispatch = action => this.store.dispatch(ac.AlsoToPreloaded(action)); 1611 this.loadLayout(dispatch, false); 1612 } 1613 1614 async onPrefChangedAction(action) { 1615 switch (action.data.name) { 1616 case PREF_CONFIG: 1617 case PREF_ENABLED: 1618 case PREF_HARDCODED_BASIC_LAYOUT: 1619 case PREF_SPOCS_ENDPOINT: 1620 case PREF_SPOCS_ENDPOINT_QUERY: 1621 case PREF_PERSONALIZATION: 1622 // This is a config reset directly related to Discovery Stream pref. 1623 this.configReset(); 1624 break; 1625 case PREF_COLLECTIONS_ENABLED: 1626 this.onPocketConfigChanged(); 1627 break; 1628 case PREF_USER_TOPSTORIES: 1629 case PREF_SYSTEM_TOPSTORIES: 1630 if (!action.data.value) { 1631 // Ensure we delete any remote data potentially related to spocs. 1632 this.clearSpocs(); 1633 } else { 1634 this.enableStories(); 1635 } 1636 break; 1637 // Check if spocs was disabled. Remove them if they were. 1638 case PREF_SHOW_SPONSORED: 1639 if (!action.data.value) { 1640 // Ensure we delete any remote data potentially related to spocs. 1641 this.clearSpocs(); 1642 } 1643 await this.loadSpocs(update => 1644 this.store.dispatch(ac.BroadcastToContent(update)) 1645 ); 1646 break; 1647 } 1648 } 1649 1650 async onAction(action) { 1651 switch (action.type) { 1652 case at.INIT: 1653 // During the initialization of Firefox: 1654 // 1. Set-up listeners and initialize the redux state for config; 1655 this.setupPrefs(true /* isStartup */); 1656 // 2. If config.enabled is true, start loading data. 1657 if (this.config.enabled) { 1658 await this.enable(); 1659 } 1660 break; 1661 case at.DISCOVERY_STREAM_DEV_SYSTEM_TICK: 1662 case at.SYSTEM_TICK: 1663 // Only refresh if we loaded once in .enable() 1664 if ( 1665 this.config.enabled && 1666 this.loaded && 1667 (await this.checkIfAnyCacheExpired()) 1668 ) { 1669 await this.refreshAll({ updateOpenTabs: false }); 1670 } 1671 break; 1672 case at.DISCOVERY_STREAM_DEV_IDLE_DAILY: 1673 Services.obs.notifyObservers(null, "idle-daily"); 1674 break; 1675 case at.DISCOVERY_STREAM_DEV_SYNC_RS: 1676 RemoteSettings.pollChanges(); 1677 break; 1678 case at.DISCOVERY_STREAM_DEV_EXPIRE_CACHE: 1679 // Personalization scores update at a slower interval than content, so in order to debug, 1680 // we want to be able to expire just content to trigger the earlier expire times. 1681 await this.resetContentCache(); 1682 break; 1683 case at.DISCOVERY_STREAM_CONFIG_SET_VALUE: 1684 // Use the original string pref to then set a value instead of 1685 // this.config which has some modifications 1686 this.store.dispatch( 1687 ac.SetPref( 1688 PREF_CONFIG, 1689 JSON.stringify({ 1690 ...JSON.parse(this.store.getState().Prefs.values[PREF_CONFIG]), 1691 [action.data.name]: action.data.value, 1692 }) 1693 ) 1694 ); 1695 break; 1696 1697 case at.DISCOVERY_STREAM_CONFIG_RESET: 1698 // This is a generic config reset likely related to an external feed pref. 1699 this.configReset(); 1700 break; 1701 case at.DISCOVERY_STREAM_CONFIG_RESET_DEFAULTS: 1702 this.resetConfigDefauts(); 1703 break; 1704 case at.DISCOVERY_STREAM_RETRY_FEED: 1705 this.retryFeed(action.data.feed); 1706 break; 1707 case at.DISCOVERY_STREAM_CONFIG_CHANGE: 1708 // When the config pref changes, load or unload data as needed. 1709 await this.onPrefChange(); 1710 break; 1711 case at.DISCOVERY_STREAM_IMPRESSION_STATS: 1712 if ( 1713 action.data.tiles && 1714 action.data.tiles[0] && 1715 action.data.tiles[0].id 1716 ) { 1717 this.recordTopRecImpressions(action.data.tiles[0].id); 1718 } 1719 break; 1720 case at.DISCOVERY_STREAM_SPOC_IMPRESSION: 1721 if (this.showSpocs) { 1722 this.recordFlightImpression(action.data.flightId); 1723 1724 // Apply frequency capping to SPOCs in the redux store, only update the 1725 // store if the SPOCs are changed. 1726 const spocsState = this.store.getState().DiscoveryStream.spocs; 1727 1728 let frequencyCapped = []; 1729 this.placementsForEach(placement => { 1730 const spocs = spocsState.data[placement.name]; 1731 if (!spocs || !spocs.items) { 1732 return; 1733 } 1734 1735 const { data: capResult, filtered } = this.frequencyCapSpocs( 1736 spocs.items 1737 ); 1738 frequencyCapped = [...frequencyCapped, ...filtered]; 1739 1740 spocsState.data = { 1741 ...spocsState.data, 1742 [placement.name]: { 1743 ...spocs, 1744 items: capResult, 1745 }, 1746 }; 1747 }); 1748 1749 if (frequencyCapped.length) { 1750 // Update cache here so we don't need to re calculate frequency caps on loads from cache. 1751 await this.cache.set("spocs", { 1752 lastUpdated: spocsState.lastUpdated, 1753 spocs: spocsState.data, 1754 }); 1755 1756 this.store.dispatch( 1757 ac.AlsoToPreloaded({ 1758 type: at.DISCOVERY_STREAM_SPOCS_UPDATE, 1759 data: { 1760 lastUpdated: spocsState.lastUpdated, 1761 spocs: spocsState.data, 1762 }, 1763 }) 1764 ); 1765 } 1766 } 1767 break; 1768 // This is fired from the browser, it has no concept of spocs, flight or pocket. 1769 // We match the blocked url with our available spoc urls to see if there is a match. 1770 // I suspect we *could* instead do this in BLOCK_URL but I'm not sure. 1771 case at.PLACES_LINK_BLOCKED: 1772 if (this.showSpocs) { 1773 let blockedItems = []; 1774 const spocsState = this.store.getState().DiscoveryStream.spocs; 1775 1776 this.placementsForEach(placement => { 1777 const spocs = spocsState.data[placement.name]; 1778 if (spocs && spocs.items && spocs.items.length) { 1779 const blockedResults = []; 1780 const blocks = spocs.items.filter(s => { 1781 const blocked = s.url === action.data.url; 1782 if (!blocked) { 1783 blockedResults.push(s); 1784 } 1785 return blocked; 1786 }); 1787 1788 blockedItems = [...blockedItems, ...blocks]; 1789 1790 spocsState.data = { 1791 ...spocsState.data, 1792 [placement.name]: { 1793 ...spocs, 1794 items: blockedResults, 1795 }, 1796 }; 1797 } 1798 }); 1799 1800 if (blockedItems.length) { 1801 // Update cache here so we don't need to re calculate blocks on loads from cache. 1802 await this.cache.set("spocs", { 1803 lastUpdated: spocsState.lastUpdated, 1804 spocs: spocsState.data, 1805 }); 1806 1807 // If we're blocking a spoc, we want open tabs to have 1808 // a slightly different treatment from future tabs. 1809 // AlsoToPreloaded updates the source data and preloaded tabs with a new spoc. 1810 // BroadcastToContent updates open tabs with a non spoc instead of a new spoc. 1811 this.store.dispatch( 1812 ac.AlsoToPreloaded({ 1813 type: at.DISCOVERY_STREAM_LINK_BLOCKED, 1814 data: action.data, 1815 }) 1816 ); 1817 this.store.dispatch( 1818 ac.BroadcastToContent({ 1819 type: at.DISCOVERY_STREAM_SPOC_BLOCKED, 1820 data: action.data, 1821 }) 1822 ); 1823 break; 1824 } 1825 } 1826 1827 this.store.dispatch( 1828 ac.BroadcastToContent({ 1829 type: at.DISCOVERY_STREAM_LINK_BLOCKED, 1830 data: action.data, 1831 }) 1832 ); 1833 break; 1834 case at.UNINIT: 1835 // When this feed is shutting down: 1836 this.uninitPrefs(); 1837 this._recommendationProvider = null; 1838 break; 1839 case at.BLOCK_URL: { 1840 // If we block a story that also has a flight_id 1841 // we want to record that as blocked too. 1842 // This is because a single flight might have slightly different urls. 1843 action.data.forEach(site => { 1844 const { flight_id } = site; 1845 if (flight_id) { 1846 this.recordBlockFlightId(flight_id); 1847 } 1848 }); 1849 break; 1850 } 1851 case at.PREF_CHANGED: 1852 await this.onPrefChangedAction(action); 1853 if (action.data.name === "pocketConfig") { 1854 await this.onPocketConfigChanged(action.data.value); 1855 } 1856 break; 1857 } 1858 } 1859}; 1860 1861/* This function generates a hardcoded layout each call. 1862 This is because modifying the original object would 1863 persist across pref changes and system_tick updates. 1864 1865 NOTE: There is some branching logic in the template. 1866 `items` How many items to include in the primary card grid. 1867 `spocPositions` Changes the position of spoc cards. 1868 `sponsoredCollectionsEnabled` Tuns on and off the sponsored collection section. 1869 `compactLayout` Changes cards to smaller more compact cards. 1870 `hybridLayout` Changes cards to smaller more compact cards only for specific breakpoints. 1871 `hideCardBackground` Removes Pocket card background and borders. 1872 `fourCardLayout` Enable four Pocket cards per row. 1873 `loadMore` Hide half the Pocket stories behind a load more button. 1874 `lastCardMessageEnabled` Shows a message card at the end of the feed. 1875 `newFooterSection` Changes the layout of the topics section. 1876 `saveToPocketCard` Cards have a save to Pocket button over their thumbnail on hover. 1877 `hideDescriptions` Hide or display descriptions for Pocket stories. 1878 `compactGrid` Reduce the number of pixels between the Pocket cards. 1879 `compactImages` Reduce the height on Pocket card images. 1880 `imageGradient` Add a gradient to the bottom of Pocket card images to blend the image in with the card. 1881 `newSponsoredLabel` Updates the sponsored label position to below the image. 1882 `titleLines` Changes the maximum number of lines a title can be for Pocket cards. 1883 `descLines` Changes the maximum number of lines a description can be for Pocket cards. 1884 `essentialReadsHeader` Updates the Pocket section header and title to say "Today’s Essential Reads", moves the "Recommended by Pocket" header to the right side. 1885 `editorsPicksHeader` Updates the Pocket section header and title to say "Editor’s Picks", if used with essentialReadsHeader, creates a second section 2 rows down for editorsPicks. 1886*/ 1887getHardcodedLayout = ({ 1888 items = 21, 1889 spocPositions = [2, 4, 11, 20], 1890 sponsoredCollectionsEnabled = false, 1891 compactLayout = false, 1892 hybridLayout = false, 1893 hideCardBackground = false, 1894 fourCardLayout = false, 1895 loadMore = false, 1896 lastCardMessageEnabled = false, 1897 newFooterSection = false, 1898 saveToPocketCard = false, 1899 hideDescriptions = true, 1900 compactGrid = false, 1901 compactImages = false, 1902 imageGradient = false, 1903 newSponsoredLabel = false, 1904 titleLines = 3, 1905 descLines = 3, 1906 essentialReadsHeader = false, 1907 editorsPicksHeader = false, 1908 readTime = false, 1909}) => ({ 1910 lastUpdate: Date.now(), 1911 spocs: { 1912 url: "https://spocs.getpocket.com/spocs", 1913 }, 1914 layout: [ 1915 { 1916 width: 12, 1917 components: [ 1918 { 1919 type: "TopSites", 1920 header: { 1921 title: { 1922 id: "newtab-section-header-topsites", 1923 }, 1924 }, 1925 properties: {}, 1926 }, 1927 ...(sponsoredCollectionsEnabled 1928 ? [ 1929 { 1930 type: "CollectionCardGrid", 1931 properties: { 1932 items: 3, 1933 }, 1934 header: { 1935 title: "", 1936 }, 1937 placement: { 1938 name: "sponsored-collection", 1939 ad_types: [3617], 1940 zone_ids: [217759, 218031], 1941 }, 1942 spocs: { 1943 probability: 1, 1944 positions: [ 1945 { 1946 index: 0, 1947 }, 1948 { 1949 index: 1, 1950 }, 1951 { 1952 index: 2, 1953 }, 1954 ], 1955 }, 1956 }, 1957 ] 1958 : []), 1959 { 1960 type: "Message", 1961 essentialReadsHeader, 1962 editorsPicksHeader, 1963 header: { 1964 title: { 1965 id: "newtab-section-header-pocket", 1966 values: { provider: "Pocket" }, 1967 }, 1968 subtitle: "", 1969 link_text: { 1970 id: "newtab-pocket-learn-more", 1971 }, 1972 link_url: "https://getpocket.com/firefox/new_tab_learn_more", 1973 icon: "chrome://global/skin/icons/pocket.svg", 1974 }, 1975 properties: {}, 1976 styles: { 1977 ".ds-message": "margin-bottom: -20px", 1978 }, 1979 }, 1980 { 1981 type: "CardGrid", 1982 properties: { 1983 items, 1984 hybridLayout, 1985 hideCardBackground: hideCardBackground || compactLayout, 1986 fourCardLayout: fourCardLayout || compactLayout, 1987 hideDescriptions: hideDescriptions || compactLayout, 1988 compactImages, 1989 imageGradient, 1990 newSponsoredLabel: newSponsoredLabel || compactLayout, 1991 titleLines: (compactLayout && 3) || titleLines, 1992 descLines, 1993 compactGrid, 1994 essentialReadsHeader, 1995 editorsPicksHeader, 1996 readTime: readTime || compactLayout, 1997 }, 1998 loadMore, 1999 lastCardMessageEnabled, 2000 saveToPocketCard, 2001 cta_variant: "link", 2002 header: { 2003 title: "", 2004 }, 2005 placement: { 2006 name: "spocs", 2007 ad_types: [3617], 2008 zone_ids: [217758, 217995], 2009 }, 2010 feed: { 2011 embed_reference: null, 2012 url: 2013 "https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=$locale®ion=$region&count=30", 2014 }, 2015 spocs: { 2016 probability: 1, 2017 positions: spocPositions.map(position => { 2018 return { index: position }; 2019 }), 2020 }, 2021 }, 2022 { 2023 type: "Navigation", 2024 newFooterSection, 2025 properties: { 2026 alignment: "left-align", 2027 links: [ 2028 { 2029 name: "Self improvement", 2030 url: 2031 "https://getpocket.com/explore/self-improvement?utm_source=pocket-newtab", 2032 }, 2033 { 2034 name: "Food", 2035 url: 2036 "https://getpocket.com/explore/food?utm_source=pocket-newtab", 2037 }, 2038 { 2039 name: "Entertainment", 2040 url: 2041 "https://getpocket.com/explore/entertainment?utm_source=pocket-newtab", 2042 }, 2043 { 2044 name: "Health & fitness", 2045 url: 2046 "https://getpocket.com/explore/health?utm_source=pocket-newtab", 2047 }, 2048 { 2049 name: "Science", 2050 url: 2051 "https://getpocket.com/explore/science?utm_source=pocket-newtab", 2052 }, 2053 { 2054 name: "More recommendations ›", 2055 url: "https://getpocket.com/explore?utm_source=pocket-newtab", 2056 }, 2057 ], 2058 extraLinks: [ 2059 { 2060 name: "Career", 2061 url: 2062 "https://getpocket.com/explore/career?utm_source=pocket-newtab", 2063 }, 2064 { 2065 name: "Technology", 2066 url: 2067 "https://getpocket.com/explore/technology?utm_source=pocket-newtab", 2068 }, 2069 ], 2070 privacyNoticeURL: { 2071 url: 2072 "https://www.mozilla.org/privacy/firefox/#suggest-relevant-content", 2073 title: { 2074 id: "newtab-section-menu-privacy-notice", 2075 }, 2076 }, 2077 }, 2078 header: { 2079 title: { 2080 id: "newtab-pocket-read-more", 2081 }, 2082 }, 2083 styles: { 2084 ".ds-navigation": "margin-top: -10px;", 2085 }, 2086 }, 2087 ...(newFooterSection 2088 ? [ 2089 { 2090 type: "PrivacyLink", 2091 properties: { 2092 url: "https://www.mozilla.org/privacy/firefox/", 2093 title: { 2094 id: "newtab-section-menu-privacy-notice", 2095 }, 2096 }, 2097 }, 2098 ] 2099 : []), 2100 ], 2101 }, 2102 ], 2103}); 2104 2105const EXPORTED_SYMBOLS = ["DiscoveryStreamFeed"]; 2106