1/* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5var EXPORTED_SYMBOLS = ["Utils"]; 6 7const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); 8const { XPCOMUtils } = ChromeUtils.import( 9 "resource://gre/modules/XPCOMUtils.jsm" 10); 11ChromeUtils.defineModuleGetter( 12 this, 13 "AppConstants", 14 "resource://gre/modules/AppConstants.jsm" 15); 16 17XPCOMUtils.defineLazyServiceGetter( 18 this, 19 "CaptivePortalService", 20 "@mozilla.org/network/captive-portal-service;1", 21 "nsICaptivePortalService" 22); 23XPCOMUtils.defineLazyServiceGetter( 24 this, 25 "gNetworkLinkService", 26 "@mozilla.org/network/network-link-service;1", 27 "nsINetworkLinkService" 28); 29 30XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]); 31 32// Create a new instance of the ConsoleAPI so we can control the maxLogLevel with a pref. 33// See LOG_LEVELS in Console.jsm. Common examples: "all", "debug", "info", "warn", "error". 34XPCOMUtils.defineLazyGetter(this, "log", () => { 35 const { ConsoleAPI } = ChromeUtils.import( 36 "resource://gre/modules/Console.jsm", 37 {} 38 ); 39 return new ConsoleAPI({ 40 maxLogLevel: "warn", 41 maxLogLevelPref: "services.settings.loglevel", 42 prefix: "services.settings", 43 }); 44}); 45 46XPCOMUtils.defineLazyPreferenceGetter( 47 this, 48 "gServerURL", 49 "services.settings.server" 50); 51 52function _isUndefined(value) { 53 return typeof value === "undefined"; 54} 55 56var Utils = { 57 get SERVER_URL() { 58 const env = Cc["@mozilla.org/process/environment;1"].getService( 59 Ci.nsIEnvironment 60 ); 61 const isXpcshell = env.exists("XPCSHELL_TEST_PROFILE_DIR"); 62 const isNotThunderbird = AppConstants.MOZ_APP_NAME != "thunderbird"; 63 return AppConstants.RELEASE_OR_BETA && 64 !Cu.isInAutomation && 65 !isXpcshell && 66 isNotThunderbird 67 ? "https://firefox.settings.services.mozilla.com/v1" 68 : gServerURL; 69 }, 70 71 CHANGES_PATH: "/buckets/monitor/collections/changes/changeset", 72 73 /** 74 * Logger instance. 75 */ 76 log, 77 78 /** 79 * Check if network is down. 80 * 81 * Note that if this returns false, it does not guarantee 82 * that network is up. 83 * 84 * @return {bool} Whether network is down or not. 85 */ 86 get isOffline() { 87 try { 88 return ( 89 Services.io.offline || 90 CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL || 91 !gNetworkLinkService.isLinkUp 92 ); 93 } catch (ex) { 94 log.warn("Could not determine network status.", ex); 95 } 96 return false; 97 }, 98 99 /** 100 * Check if local data exist for the specified client. 101 * 102 * @param {RemoteSettingsClient} client 103 * @return {bool} Whether it exists or not. 104 */ 105 async hasLocalData(client) { 106 const timestamp = await client.db.getLastModified(); 107 // Note: timestamp will be 0 if empty JSON dump is loaded. 108 return timestamp !== null; 109 }, 110 111 /** 112 * Check if we ship a JSON dump for the specified bucket and collection. 113 * 114 * @param {String} bucket 115 * @param {String} collection 116 * @return {bool} Whether it is present or not. 117 */ 118 async hasLocalDump(bucket, collection) { 119 try { 120 await fetch( 121 `resource://app/defaults/settings/${bucket}/${collection}.json` 122 ); 123 return true; 124 } catch (e) { 125 return false; 126 } 127 }, 128 129 /** 130 * Look up the last modification time of the JSON dump. 131 * 132 * @param {String} bucket 133 * @param {String} collection 134 * @return {int} The last modification time of the dump. -1 if non-existent. 135 */ 136 async getLocalDumpLastModified(bucket, collection) { 137 if (!this._dumpStats) { 138 if (!this._dumpStatsInitPromise) { 139 this._dumpStatsInitPromise = (async () => { 140 try { 141 let res = await fetch( 142 "resource://app/defaults/settings/last_modified.json" 143 ); 144 this._dumpStats = await res.json(); 145 } catch (e) { 146 log.warn(`Failed to load last_modified.json: ${e}`); 147 this._dumpStats = {}; 148 } 149 delete this._dumpStatsInitPromise; 150 })(); 151 } 152 await this._dumpStatsInitPromise; 153 } 154 const identifier = `${bucket}/${collection}`; 155 let lastModified = this._dumpStats[identifier]; 156 if (lastModified === undefined) { 157 try { 158 let res = await fetch( 159 `resource://app/defaults/settings/${bucket}/${collection}.json` 160 ); 161 let records = (await res.json()).data; 162 // Records in dumps are sorted by last_modified, newest first. 163 // https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304 164 lastModified = records[0]?.last_modified || 0; 165 } catch (e) { 166 lastModified = -1; 167 } 168 this._dumpStats[identifier] = lastModified; 169 } 170 return lastModified; 171 }, 172 173 /** 174 * Fetch the list of remote collections and their timestamp. 175 * ``` 176 * { 177 * "timestamp": 1486545678, 178 * "changes":[ 179 * { 180 * "host":"kinto-ota.dev.mozaws.net", 181 * "last_modified":1450717104423, 182 * "bucket":"blocklists", 183 * "collection":"certificates" 184 * }, 185 * ... 186 * ], 187 * "metadata": {} 188 * } 189 * ``` 190 * @param {String} serverUrl The server URL (eg. `https://server.org/v1`) 191 * @param {int} expectedTimestamp The timestamp that the server is supposed to return. 192 * We obtained it from the Megaphone notification payload, 193 * and we use it only for cache busting (Bug 1497159). 194 * @param {String} lastEtag (optional) The Etag of the latest poll to be matched 195 * by the server (eg. `"123456789"`). 196 * @param {Object} filters 197 */ 198 async fetchLatestChanges(serverUrl, options = {}) { 199 const { expectedTimestamp, lastEtag = "", filters = {} } = options; 200 201 let url = serverUrl + Utils.CHANGES_PATH; 202 const params = { 203 ...filters, 204 _expected: expectedTimestamp ?? 0, 205 }; 206 if (lastEtag != "") { 207 params._since = lastEtag; 208 } 209 if (params) { 210 url += 211 "?" + 212 Object.entries(params) 213 .map(([k, v]) => `${k}=${encodeURIComponent(v)}`) 214 .join("&"); 215 } 216 const response = await fetch(url); 217 218 if (response.status >= 500) { 219 throw new Error(`Server error ${response.status} ${response.statusText}`); 220 } 221 222 const is404FromCustomServer = 223 response.status == 404 && 224 Services.prefs.prefHasUserValue("services.settings.server"); 225 226 const ct = response.headers.get("Content-Type"); 227 if (!is404FromCustomServer && (!ct || !ct.includes("application/json"))) { 228 throw new Error(`Unexpected content-type "${ct}"`); 229 } 230 231 let payload; 232 try { 233 payload = await response.json(); 234 } catch (e) { 235 payload = e.message; 236 } 237 238 if (!payload.hasOwnProperty("changes")) { 239 // If the server is failing, the JSON response might not contain the 240 // expected data. For example, real server errors (Bug 1259145) 241 // or dummy local server for tests (Bug 1481348) 242 if (!is404FromCustomServer) { 243 throw new Error( 244 `Server error ${url} ${response.status} ${ 245 response.statusText 246 }: ${JSON.stringify(payload)}` 247 ); 248 } 249 } 250 251 const { changes = [], timestamp } = payload; 252 253 let serverTimeMillis = Date.parse(response.headers.get("Date")); 254 // Since the response is served via a CDN, the Date header value could have been cached. 255 const cacheAgeSeconds = response.headers.has("Age") 256 ? parseInt(response.headers.get("Age"), 10) 257 : 0; 258 serverTimeMillis += cacheAgeSeconds * 1000; 259 260 // Age of data (time between publication and now). 261 let lastModifiedMillis = Date.parse(response.headers.get("Last-Modified")); 262 const ageSeconds = (serverTimeMillis - lastModifiedMillis) / 1000; 263 264 // Check if the server asked the clients to back off. 265 let backoffSeconds; 266 if (response.headers.has("Backoff")) { 267 const value = parseInt(response.headers.get("Backoff"), 10); 268 if (!isNaN(value)) { 269 backoffSeconds = value; 270 } 271 } 272 273 return { 274 changes, 275 currentEtag: `"${timestamp}"`, 276 serverTimeMillis, 277 backoffSeconds, 278 ageSeconds, 279 }; 280 }, 281 282 /** 283 * Test if a single object matches all given filters. 284 * 285 * @param {Object} filters The filters object. 286 * @param {Object} entry The object to filter. 287 * @return {Boolean} 288 */ 289 filterObject(filters, entry) { 290 return Object.entries(filters).every(([filter, value]) => { 291 if (Array.isArray(value)) { 292 return value.some(candidate => candidate === entry[filter]); 293 } else if (typeof value === "object") { 294 return Utils.filterObject(value, entry[filter]); 295 } else if (!Object.prototype.hasOwnProperty.call(entry, filter)) { 296 console.error(`The property ${filter} does not exist`); 297 return false; 298 } 299 return entry[filter] === value; 300 }); 301 }, 302 303 /** 304 * Sorts records in a list according to a given ordering. 305 * 306 * @param {String} order The ordering, eg. `-last_modified`. 307 * @param {Array} list The collection to order. 308 * @return {Array} 309 */ 310 sortObjects(order, list) { 311 const hasDash = order[0] === "-"; 312 const field = hasDash ? order.slice(1) : order; 313 const direction = hasDash ? -1 : 1; 314 return list.slice().sort((a, b) => { 315 if (a[field] && _isUndefined(b[field])) { 316 return direction; 317 } 318 if (b[field] && _isUndefined(a[field])) { 319 return -direction; 320 } 321 if (_isUndefined(a[field]) && _isUndefined(b[field])) { 322 return 0; 323 } 324 return a[field] > b[field] ? direction : -direction; 325 }); 326 }, 327}; 328