1/* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5const { XPCOMUtils } = ChromeUtils.import( 6 "resource://gre/modules/XPCOMUtils.jsm" 7); 8const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); 9 10XPCOMUtils.defineLazyModuleGetters(this, { 11 AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm", 12 IDBHelpers: "resource://services-settings/IDBHelpers.jsm", 13 Utils: "resource://services-settings/Utils.jsm", 14 CommonUtils: "resource://services-common/utils.js", 15 ObjectUtils: "resource://gre/modules/ObjectUtils.jsm", 16}); 17XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log); 18 19var EXPORTED_SYMBOLS = ["Database"]; 20 21/** 22 * Database is a tiny wrapper with the objective 23 * of providing major kinto-offline-client collection API. 24 * (with the objective of getting rid of kinto-offline-client) 25 */ 26class Database { 27 constructor(identifier) { 28 ensureShutdownBlocker(); 29 this.identifier = identifier; 30 } 31 32 async list(options = {}) { 33 const { filters = {}, order = "" } = options; 34 let results = []; 35 try { 36 await executeIDB( 37 "records", 38 (store, rejectTransaction) => { 39 // Fast-path the (very common) no-filters case 40 if (ObjectUtils.isEmpty(filters)) { 41 const range = IDBKeyRange.only(this.identifier); 42 const request = store.index("cid").getAll(range); 43 request.onsuccess = e => { 44 results = e.target.result; 45 }; 46 return; 47 } 48 const request = store 49 .index("cid") 50 .openCursor(IDBKeyRange.only(this.identifier)); 51 const objFilters = transformSubObjectFilters(filters); 52 request.onsuccess = event => { 53 try { 54 const cursor = event.target.result; 55 if (cursor) { 56 const { value } = cursor; 57 if (Utils.filterObject(objFilters, value)) { 58 results.push(value); 59 } 60 cursor.continue(); 61 } 62 } catch (ex) { 63 rejectTransaction(ex); 64 } 65 }; 66 }, 67 { mode: "readonly" } 68 ); 69 } catch (e) { 70 throw new IDBHelpers.IndexedDBError(e, "list()", this.identifier); 71 } 72 // Remove IDB key field from results. 73 for (const result of results) { 74 delete result._cid; 75 } 76 return order ? Utils.sortObjects(order, results) : results; 77 } 78 79 async importChanges(metadata, timestamp, records = [], options = {}) { 80 const { clear = false } = options; 81 const _cid = this.identifier; 82 try { 83 await executeIDB( 84 ["collections", "timestamps", "records"], 85 (stores, rejectTransaction) => { 86 const [storeMetadata, storeTimestamps, storeRecords] = stores; 87 88 if (clear) { 89 // Our index is over the _cid and id fields. We want to remove 90 // all of the items in the collection for which the object was 91 // created, ie with _cid == this.identifier. 92 // We would like to just tell IndexedDB: 93 // store.index(IDBKeyRange.only(this.identifier)).delete(); 94 // to delete all records matching the first part of the 2-part key. 95 // Unfortunately such an API does not exist. 96 // While we could iterate over the index with a cursor, we'd do 97 // a roundtrip to PBackground for each item. Once you have 1000 98 // items, the result is very slow because of all the overhead of 99 // jumping between threads and serializing/deserializing. 100 // So instead, we tell the store to delete everything between 101 // "our" _cid identifier, and what would be the next identifier 102 // (via lexicographical sorting). Unfortunately there does not 103 // seem to be a way to specify bounds for all items that share 104 // the same first part of the key using just that first part, hence 105 // the use of the hypothetical [] for the second part of the end of 106 // the bounds. 107 storeRecords.delete( 108 IDBKeyRange.bound([_cid], [_cid, []], false, true) 109 ); 110 } 111 112 // Store or erase metadata. 113 if (metadata === null) { 114 storeMetadata.delete(_cid); 115 } else if (metadata) { 116 storeMetadata.put({ cid: _cid, metadata }); 117 } 118 // Store or erase timestamp. 119 if (timestamp === null) { 120 storeTimestamps.delete(_cid); 121 } else if (timestamp) { 122 storeTimestamps.put({ cid: _cid, value: timestamp }); 123 } 124 125 if (records.length == 0) { 126 return; 127 } 128 129 // Separate tombstones from creations/updates. 130 const toDelete = records.filter(r => r.deleted); 131 const toInsert = records.filter(r => !r.deleted); 132 console.debug( 133 `${_cid} ${toDelete.length} to delete, ${toInsert.length} to insert` 134 ); 135 // Delete local records for each tombstone. 136 IDBHelpers.bulkOperationHelper( 137 storeRecords, 138 { 139 reject: rejectTransaction, 140 completion() { 141 // Overwrite all other data. 142 IDBHelpers.bulkOperationHelper( 143 storeRecords, 144 { 145 reject: rejectTransaction, 146 }, 147 "put", 148 toInsert.map(item => ({ ...item, _cid })) 149 ); 150 }, 151 }, 152 "delete", 153 toDelete.map(item => [_cid, item.id]) 154 ); 155 }, 156 { desc: "importChanges() in " + _cid } 157 ); 158 } catch (e) { 159 throw new IDBHelpers.IndexedDBError(e, "importChanges()", _cid); 160 } 161 } 162 163 async getLastModified() { 164 let entry = null; 165 try { 166 await executeIDB( 167 "timestamps", 168 store => { 169 store.get(this.identifier).onsuccess = e => (entry = e.target.result); 170 }, 171 { mode: "readonly" } 172 ); 173 } catch (e) { 174 throw new IDBHelpers.IndexedDBError( 175 e, 176 "getLastModified()", 177 this.identifier 178 ); 179 } 180 if (!entry) { 181 return null; 182 } 183 // Some distributions where released with a modified dump that did not 184 // contain timestamps for last_modified. Work around this here, and return 185 // the timestamp as zero, so that the entries should get updated. 186 if (isNaN(entry.value)) { 187 console.warn(`Local timestamp is NaN for ${this.identifier}`); 188 return 0; 189 } 190 return entry.value; 191 } 192 193 async getMetadata() { 194 let entry = null; 195 try { 196 await executeIDB( 197 "collections", 198 store => { 199 store.get(this.identifier).onsuccess = e => (entry = e.target.result); 200 }, 201 { mode: "readonly" } 202 ); 203 } catch (e) { 204 throw new IDBHelpers.IndexedDBError(e, "getMetadata()", this.identifier); 205 } 206 return entry ? entry.metadata : null; 207 } 208 209 async getAttachment(attachmentId) { 210 let entry = null; 211 try { 212 await executeIDB( 213 "attachments", 214 store => { 215 store.get([this.identifier, attachmentId]).onsuccess = e => { 216 entry = e.target.result; 217 }; 218 }, 219 { mode: "readonly" } 220 ); 221 } catch (e) { 222 throw new IDBHelpers.IndexedDBError( 223 e, 224 "getAttachment()", 225 this.identifier 226 ); 227 } 228 return entry ? entry.attachment : null; 229 } 230 231 async saveAttachment(attachmentId, attachment) { 232 try { 233 await executeIDB( 234 "attachments", 235 store => { 236 if (attachment) { 237 store.put({ cid: this.identifier, attachmentId, attachment }); 238 } else { 239 store.delete([this.identifier, attachmentId]); 240 } 241 }, 242 { desc: "saveAttachment(" + attachmentId + ") in " + this.identifier } 243 ); 244 } catch (e) { 245 throw new IDBHelpers.IndexedDBError( 246 e, 247 "saveAttachment()", 248 this.identifier 249 ); 250 } 251 } 252 253 async clear() { 254 try { 255 await this.importChanges(null, null, [], { clear: true }); 256 } catch (e) { 257 throw new IDBHelpers.IndexedDBError(e, "clear()", this.identifier); 258 } 259 } 260 261 /* 262 * Methods used by unit tests. 263 */ 264 265 async create(record) { 266 if (!("id" in record)) { 267 record = { ...record, id: CommonUtils.generateUUID() }; 268 } 269 try { 270 await executeIDB( 271 "records", 272 store => { 273 store.add({ ...record, _cid: this.identifier }); 274 }, 275 { desc: "create() in " + this.identifier } 276 ); 277 } catch (e) { 278 throw new IDBHelpers.IndexedDBError(e, "create()", this.identifier); 279 } 280 return record; 281 } 282 283 async update(record) { 284 try { 285 await executeIDB( 286 "records", 287 store => { 288 store.put({ ...record, _cid: this.identifier }); 289 }, 290 { desc: "update() in " + this.identifier } 291 ); 292 } catch (e) { 293 throw new IDBHelpers.IndexedDBError(e, "update()", this.identifier); 294 } 295 } 296 297 async delete(recordId) { 298 try { 299 await executeIDB( 300 "records", 301 store => { 302 store.delete([this.identifier, recordId]); // [_cid, id] 303 }, 304 { desc: "delete() in " + this.identifier } 305 ); 306 } catch (e) { 307 throw new IDBHelpers.IndexedDBError(e, "delete()", this.identifier); 308 } 309 } 310} 311 312let gDB = null; 313let gDBPromise = null; 314 315/** 316 * This function attempts to ensure `gDB` points to a valid database value. 317 * If gDB is already a database, it will do no-op (but this may take a 318 * microtask or two). 319 * If opening the database fails, it will throw an IndexedDBError. 320 */ 321async function openIDB() { 322 // We can be called multiple times in a race; always ensure that when 323 // we complete, `gDB` is no longer null, but avoid doing the actual 324 // IndexedDB work more than once. 325 if (!gDBPromise) { 326 // Open and initialize/upgrade if needed. 327 gDBPromise = IDBHelpers.openIDB(); 328 } 329 let db = await gDBPromise; 330 if (!gDB) { 331 gDB = db; 332 } 333} 334 335const gPendingReadOnlyTransactions = new Set(); 336const gPendingWriteOperations = new Set(); 337/** 338 * Helper to wrap some IDBObjectStore operations into a promise. 339 * 340 * @param {IDBDatabase} db 341 * @param {String|String[]} storeNames - either a string or an array of strings. 342 * @param {function} callback 343 * @param {Object} options 344 * @param {String} options.mode 345 * @param {String} options.desc for shutdown tracking. 346 */ 347async function executeIDB(storeNames, callback, options = {}) { 348 if (!gDB) { 349 // Check if we're shutting down. Services.startup.shuttingDown will 350 // be true sooner, but is never true in xpcshell tests, so we check 351 // both that and a bool we set ourselves when `profile-before-change` 352 // starts. 353 if (gShutdownStarted || Services.startup.shuttingDown) { 354 throw new IDBHelpers.ShutdownError( 355 "The application is shutting down", 356 "execute()" 357 ); 358 } 359 await openIDB(); 360 } else { 361 // Even if we have a db, wait a tick to avoid making IndexedDB sad. 362 // We should be able to remove this once bug 1626935 is fixed. 363 await Promise.resolve(); 364 } 365 366 // Check for shutdown again as we've await'd something... 367 if (!gDB && (gShutdownStarted || Services.startup.shuttingDown)) { 368 throw new IDBHelpers.ShutdownError( 369 "The application is shutting down", 370 "execute()" 371 ); 372 } 373 374 // Start the actual transaction: 375 const { mode = "readwrite", desc = "" } = options; 376 let { promise, transaction } = IDBHelpers.executeIDB( 377 gDB, 378 storeNames, 379 mode, 380 callback, 381 desc 382 ); 383 384 // We track all readonly transactions and abort them at shutdown. 385 // We track all readwrite ones and await their completion at shutdown 386 // (to avoid dataloss when writes fail). 387 // We use a `.finally()` clause for this; it'll run the function irrespective 388 // of whether the promise resolves or rejects, and the promise it returns 389 // will resolve/reject with the same value. 390 let finishedFn; 391 if (mode == "readonly") { 392 gPendingReadOnlyTransactions.add(transaction); 393 finishedFn = () => gPendingReadOnlyTransactions.delete(transaction); 394 } else { 395 let obj = { promise, desc }; 396 gPendingWriteOperations.add(obj); 397 finishedFn = () => gPendingWriteOperations.delete(obj); 398 } 399 return promise.finally(finishedFn); 400} 401 402function makeNestedObjectFromArr(arr, val, nestedFiltersObj) { 403 const last = arr.length - 1; 404 return arr.reduce((acc, cv, i) => { 405 if (i === last) { 406 return (acc[cv] = val); 407 } else if (Object.prototype.hasOwnProperty.call(acc, cv)) { 408 return acc[cv]; 409 } 410 return (acc[cv] = {}); 411 }, nestedFiltersObj); 412} 413 414function transformSubObjectFilters(filtersObj) { 415 const transformedFilters = {}; 416 for (const [key, val] of Object.entries(filtersObj)) { 417 const keysArr = key.split("."); 418 makeNestedObjectFromArr(keysArr, val, transformedFilters); 419 } 420 return transformedFilters; 421} 422 423// We need to expose this wrapper function so we can test 424// shutdown handling. 425Database._executeIDB = executeIDB; 426 427let gShutdownStarted = false; 428// Test-only helper to be able to test shutdown multiple times: 429Database._cancelShutdown = () => { 430 gShutdownStarted = false; 431}; 432 433let gShutdownBlocker = false; 434Database._shutdownHandler = () => { 435 gShutdownStarted = true; 436 const NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR = 0x80660006; 437 // Duplicate the list (to avoid it being modified) and then 438 // abort all read-only transactions. 439 for (let transaction of Array.from(gPendingReadOnlyTransactions)) { 440 try { 441 transaction.abort(); 442 } catch (ex) { 443 // Ensure we don't throw/break, because either way we're in shutdown. 444 445 // In particular, `transaction.abort` can throw if the transaction 446 // is complete, ie if we manage to get called inbetween the 447 // transaction completing, and our completion handler being called 448 // to remove the item from the set. We don't care about that. 449 if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) { 450 // Report any other errors: 451 Cu.reportError(ex); 452 } 453 } 454 } 455 if (gDB) { 456 // This will return immediately; the actual close will happen once 457 // there are no more running transactions. 458 gDB.close(); 459 gDB = null; 460 } 461 gDBPromise = null; 462 return Promise.allSettled( 463 Array.from(gPendingWriteOperations).map(op => op.promise) 464 ); 465}; 466 467function ensureShutdownBlocker() { 468 if (gShutdownBlocker) { 469 return; 470 } 471 gShutdownBlocker = true; 472 AsyncShutdown.profileBeforeChange.addBlocker( 473 "RemoteSettingsClient - finish IDB access.", 474 Database._shutdownHandler, 475 { 476 fetchState() { 477 return Array.from(gPendingWriteOperations).map(op => op.desc); 478 }, 479 } 480 ); 481} 482