1/* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4"use strict"; 5 6/** 7 * FormHistory 8 * 9 * Used to store values that have been entered into forms which may later 10 * be used to automatically fill in the values when the form is visited again. 11 * 12 * search(terms, queryData, callback) 13 * Look up values that have been previously stored. 14 * terms - array of terms to return data for 15 * queryData - object that contains the query terms 16 * The query object contains properties for each search criteria to match, where the value 17 * of the property specifies the value that term must have. For example, 18 * { term1: value1, term2: value2 } 19 * callback - callback that is called when results are available or an error occurs. 20 * The callback is passed a result array containing each found entry. Each element in 21 * the array is an object containing a property for each search term specified by 'terms'. 22 * count(queryData, callback) 23 * Find the number of stored entries that match the given criteria. 24 * queryData - array of objects that indicate the query. See the search method for details. 25 * callback - callback that is called when results are available or an error occurs. 26 * The callback is passed the number of found entries. 27 * update(changes, callback) 28 * Write data to form history storage. 29 * changes - an array of changes to be made. If only one change is to be made, it 30 * may be passed as an object rather than a one-element array. 31 * Each change object is of the form: 32 * { op: operation, term1: value1, term2: value2, ... } 33 * Valid operations are: 34 * add - add a new entry 35 * update - update an existing entry 36 * remove - remove an entry 37 * bump - update the last accessed time on an entry 38 * The terms specified allow matching of one or more specific entries. If no terms 39 * are specified then all entries are matched. This means that { op: "remove" } is 40 * used to remove all entries and clear the form history. 41 * callback - callback that is called when results have been stored. 42 * getAutoCompeteResults(searchString, params, callback) 43 * Retrieve an array of form history values suitable for display in an autocomplete list. 44 * Returns an mozIStoragePendingStatement that can be used to cancel the operation if 45 * needed. 46 * searchString - the string to search for, typically the entered value of a textbox 47 * params - zero or more filter arguments: 48 * fieldname - form field name 49 * agedWeight 50 * bucketSize 51 * expiryDate 52 * maxTimeGroundings 53 * timeGroupingSize 54 * prefixWeight 55 * boundaryWeight 56 * callback - callback that is called with the array of results. Each result in the array 57 * is an object with four arguments: 58 * text, textLowerCase, frecency, totalScore 59 * schemaVersion 60 * This property holds the version of the database schema 61 * 62 * Terms: 63 * guid - entry identifier. For 'add', a guid will be generated. 64 * fieldname - form field name 65 * value - form value 66 * timesUsed - the number of times the entry has been accessed 67 * firstUsed - the time the the entry was first created 68 * lastUsed - the time the entry was last accessed 69 * firstUsedStart - search for entries created after or at this time 70 * firstUsedEnd - search for entries created before or at this time 71 * lastUsedStart - search for entries last accessed after or at this time 72 * lastUsedEnd - search for entries last accessed before or at this time 73 * newGuid - a special case valid only for 'update' and allows the guid for 74 * an existing record to be updated. The 'guid' term is the only 75 * other term which can be used (ie, you can not also specify a 76 * fieldname, value etc) and indicates the guid of the existing 77 * record that should be updated. 78 * 79 * In all of the above methods, the callback argument should be an object with 80 * handleResult(result), handleFailure(error) and handleCompletion(reason) functions. 81 * For search and getAutoCompeteResults, result is an object containing the desired 82 * properties. For count, result is the integer count. For, update, handleResult is 83 * not called. For handleCompletion, reason is either 0 if successful or 1 if 84 * an error occurred. 85 */ 86 87var EXPORTED_SYMBOLS = ["FormHistory"]; 88 89ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm"); 90ChromeUtils.import("resource://gre/modules/Services.jsm"); 91ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); 92 93XPCOMUtils.defineLazyServiceGetter(this, "uuidService", 94 "@mozilla.org/uuid-generator;1", 95 "nsIUUIDGenerator"); 96ChromeUtils.defineModuleGetter(this, "AsyncShutdown", 97 "resource://gre/modules/AsyncShutdown.jsm"); 98ChromeUtils.defineModuleGetter(this, "OS", 99 "resource://gre/modules/osfile.jsm"); 100ChromeUtils.defineModuleGetter(this, "Sqlite", 101 "resource://gre/modules/Sqlite.jsm"); 102 103 104const DB_SCHEMA_VERSION = 4; 105const DAY_IN_MS = 86400000; // 1 day in milliseconds 106const MAX_SEARCH_TOKENS = 10; 107const NOOP = function noop() {}; 108const DB_FILENAME = "formhistory.sqlite"; 109 110var supportsDeletedTable = AppConstants.platform == "android"; 111 112var Prefs = { 113 initialized: false, 114 115 get debug() { this.ensureInitialized(); return this._debug; }, 116 get enabled() { this.ensureInitialized(); return this._enabled; }, 117 get expireDays() { this.ensureInitialized(); return this._expireDays; }, 118 119 ensureInitialized() { 120 if (this.initialized) { 121 return; 122 } 123 124 this.initialized = true; 125 126 this._debug = Services.prefs.getBoolPref("browser.formfill.debug"); 127 this._enabled = Services.prefs.getBoolPref("browser.formfill.enable"); 128 this._expireDays = Services.prefs.getIntPref("browser.formfill.expire_days"); 129 }, 130}; 131 132function log(aMessage) { 133 if (Prefs.debug) { 134 Services.console.logStringMessage("FormHistory: " + aMessage); 135 } 136} 137 138function sendNotification(aType, aData) { 139 if (typeof aData == "string") { 140 let strWrapper = Cc["@mozilla.org/supports-string;1"] 141 .createInstance(Ci.nsISupportsString); 142 strWrapper.data = aData; 143 aData = strWrapper; 144 } else if (typeof aData == "number") { 145 let intWrapper = Cc["@mozilla.org/supports-PRInt64;1"] 146 .createInstance(Ci.nsISupportsPRInt64); 147 intWrapper.data = aData; 148 aData = intWrapper; 149 } else if (aData) { 150 throw Components.Exception("Invalid type " + (typeof aType) + " passed to sendNotification", 151 Cr.NS_ERROR_ILLEGAL_VALUE); 152 } 153 154 Services.obs.notifyObservers(aData, "satchel-storage-changed", aType); 155} 156 157/** 158 * Current database schema 159 */ 160 161const dbSchema = { 162 tables: { 163 moz_formhistory: { 164 "id": "INTEGER PRIMARY KEY", 165 "fieldname": "TEXT NOT NULL", 166 "value": "TEXT NOT NULL", 167 "timesUsed": "INTEGER", 168 "firstUsed": "INTEGER", 169 "lastUsed": "INTEGER", 170 "guid": "TEXT", 171 }, 172 moz_deleted_formhistory: { 173 "id": "INTEGER PRIMARY KEY", 174 "timeDeleted": "INTEGER", 175 "guid": "TEXT", 176 }, 177 }, 178 indices: { 179 moz_formhistory_index: { 180 table: "moz_formhistory", 181 columns: ["fieldname"], 182 }, 183 moz_formhistory_lastused_index: { 184 table: "moz_formhistory", 185 columns: ["lastUsed"], 186 }, 187 moz_formhistory_guid_index: { 188 table: "moz_formhistory", 189 columns: ["guid"], 190 }, 191 }, 192}; 193 194/** 195 * Validating and processing API querying data 196 */ 197 198const validFields = [ 199 "fieldname", 200 "value", 201 "timesUsed", 202 "firstUsed", 203 "lastUsed", 204 "guid", 205]; 206 207const searchFilters = [ 208 "firstUsedStart", 209 "firstUsedEnd", 210 "lastUsedStart", 211 "lastUsedEnd", 212]; 213 214function validateOpData(aData, aDataType) { 215 let thisValidFields = validFields; 216 // A special case to update the GUID - in this case there can be a 'newGuid' 217 // field and of the normally valid fields, only 'guid' is accepted. 218 if (aDataType == "Update" && "newGuid" in aData) { 219 thisValidFields = ["guid", "newGuid"]; 220 } 221 for (let field in aData) { 222 if (field != "op" && !thisValidFields.includes(field)) { 223 throw Components.Exception( 224 aDataType + " query contains an unrecognized field: " + field, 225 Cr.NS_ERROR_ILLEGAL_VALUE); 226 } 227 } 228 return aData; 229} 230 231function validateSearchData(aData, aDataType) { 232 for (let field in aData) { 233 if (field != "op" && !validFields.includes(field) && !searchFilters.includes(field)) { 234 throw Components.Exception( 235 aDataType + " query contains an unrecognized field: " + field, 236 Cr.NS_ERROR_ILLEGAL_VALUE); 237 } 238 } 239} 240 241function makeQueryPredicates(aQueryData, delimiter = " AND ") { 242 return Object.keys(aQueryData).map(field => { 243 switch (field) { 244 case "firstUsedStart": { 245 return "firstUsed >= :" + field; 246 } 247 case "firstUsedEnd": { 248 return "firstUsed <= :" + field; 249 } 250 case "lastUsedStart": { 251 return "lastUsed >= :" + field; 252 } 253 case "lastUsedEnd": { 254 return "lastUsed <= :" + field; 255 } 256 } 257 258 return field + " = :" + field; 259 }).join(delimiter); 260} 261 262function generateGUID() { 263 // string like: "{f60d9eac-9421-4abc-8491-8e8322b063d4}" 264 let uuid = uuidService.generateUUID().toString(); 265 let raw = ""; // A string with the low bytes set to random values 266 let bytes = 0; 267 for (let i = 1; bytes < 12; i += 2) { 268 // Skip dashes 269 if (uuid[i] == "-") { 270 i++; 271 } 272 let hexVal = parseInt(uuid[i] + uuid[i + 1], 16); 273 raw += String.fromCharCode(hexVal); 274 bytes++; 275 } 276 return btoa(raw); 277} 278 279var Migrators = { 280 /* 281 * Updates the DB schema to v3 (bug 506402). 282 * Adds deleted form history table. 283 */ 284 async dbAsyncMigrateToVersion4(conn) { 285 const TABLE_NAME = "moz_deleted_formhistory"; 286 let tableExists = await conn.tableExists(TABLE_NAME); 287 if (!tableExists) { 288 let table = dbSchema.tables[TABLE_NAME]; 289 let tSQL = Object.keys(table).map(col => [col, table[col]].join(" ")).join(", "); 290 await conn.execute(`CREATE TABLE ${TABLE_NAME} (${tSQL})`); 291 } 292 }, 293}; 294 295/** 296 * @typedef {Object} InsertQueryData 297 * @property {Object} updatedChange 298 * A change requested by FormHistory. 299 * @property {String} query 300 * The insert query string. 301 */ 302 303/** 304 * Prepares a query and some default parameters when inserting an entry 305 * to the database. 306 * 307 * @param {Object} change 308 * The change requested by FormHistory. 309 * @param {number} now 310 * The current timestamp in microseconds. 311 * @returns {InsertQueryData} 312 * The query information needed to pass along to the database. 313 */ 314function prepareInsertQuery(change, now) { 315 let updatedChange = Object.assign({}, change); 316 let query = "INSERT INTO moz_formhistory " + 317 "(fieldname, value, timesUsed, firstUsed, lastUsed, guid) " + 318 "VALUES (:fieldname, :value, :timesUsed, :firstUsed, :lastUsed, :guid)"; 319 updatedChange.timesUsed = updatedChange.timesUsed || 1; 320 updatedChange.firstUsed = updatedChange.firstUsed || now; 321 updatedChange.lastUsed = updatedChange.lastUsed || now; 322 323 return { 324 updatedChange, 325 query, 326 }; 327} 328 329// There is a fieldname / value uniqueness constraint that's at this time 330// only enforced at this level. This Map maps fieldnames => values that 331// are in the process of being inserted into the database so that we know 332// not to try to insert the same ones on top. Attempts to do so will be 333// ignored. 334var InProgressInserts = { 335 _inProgress: new Map(), 336 337 add(fieldname, value) { 338 let fieldnameSet = this._inProgress.get(fieldname); 339 if (!fieldnameSet) { 340 this._inProgress.set(fieldname, new Set([value])); 341 return true; 342 } 343 344 if (!fieldnameSet.has(value)) { 345 fieldnameSet.add(value); 346 return true; 347 } 348 349 return false; 350 }, 351 352 clear(fieldnamesAndValues) { 353 for (let [fieldname, value] of fieldnamesAndValues) { 354 let fieldnameSet = this._inProgress.get(fieldname); 355 if (fieldnameSet && 356 fieldnameSet.delete(value) && 357 fieldnameSet.size == 0) { 358 this._inProgress.delete(fieldname); 359 } 360 } 361 }, 362}; 363 364/** 365 * Constructs and executes database statements from a pre-processed list of 366 * inputted changes. 367 * 368 * @param {Array.<Object>} aChanges changes to form history 369 * @param {Object} aPreparedHandlers 370 */ 371// XXX This should be split up and the complexity reduced. 372// eslint-disable-next-line complexity 373async function updateFormHistoryWrite(aChanges, aPreparedHandlers) { 374 log("updateFormHistoryWrite " + aChanges.length); 375 376 // pass 'now' down so that every entry in the batch has the same timestamp 377 let now = Date.now() * 1000; 378 let queries = []; 379 let notifications = []; 380 let adds = []; 381 let conn = await FormHistory.db; 382 383 for (let change of aChanges) { 384 let operation = change.op; 385 delete change.op; 386 switch (operation) { 387 case "remove": { 388 log("Remove from form history " + change); 389 let queryTerms = makeQueryPredicates(change); 390 391 // Fetch the GUIDs we are going to delete. 392 try { 393 let query = "SELECT guid FROM moz_formhistory"; 394 if (queryTerms) { 395 query += " WHERE " + queryTerms; 396 } 397 398 await conn.executeCached(query, change, row => { 399 notifications.push(["formhistory-remove", row.getResultByName("guid")]); 400 }); 401 } catch (e) { 402 log("Error getting guids from moz_formhistory: " + e); 403 } 404 405 if (supportsDeletedTable) { 406 log("Moving to deleted table " + change); 407 let query = "INSERT INTO moz_deleted_formhistory (guid, timeDeleted)"; 408 409 // TODO: Add these items to the deleted items table once we've sorted 410 // out the issues from bug 756701 411 if (change.guid || queryTerms) { 412 query += 413 change.guid ? " VALUES (:guid, :timeDeleted)" 414 : " SELECT guid, :timeDeleted FROM moz_formhistory WHERE " + queryTerms; 415 change.timeDeleted = now; 416 queries.push({ query, params: Object.assign({}, change) }); 417 } 418 419 if ("timeDeleted" in change) { 420 delete change.timeDeleted; 421 } 422 } 423 424 let query = "DELETE FROM moz_formhistory"; 425 if (queryTerms) { 426 log("removeEntries"); 427 query += " WHERE " + queryTerms; 428 } else { 429 log("removeAllEntries"); 430 // Not specifying any fields means we should remove all entries. We 431 // won't need to modify the query in this case. 432 } 433 434 queries.push({ query, params: change }); 435 break; 436 } 437 case "update": { 438 log("Update form history " + change); 439 let guid = change.guid; 440 delete change.guid; 441 // a special case for updating the GUID - the new value can be 442 // specified in newGuid. 443 if (change.newGuid) { 444 change.guid = change.newGuid; 445 delete change.newGuid; 446 } 447 448 let query = "UPDATE moz_formhistory SET "; 449 let queryTerms = makeQueryPredicates(change, ", "); 450 if (!queryTerms) { 451 throw Components.Exception("Update query must define fields to modify.", 452 Cr.NS_ERROR_ILLEGAL_VALUE); 453 } 454 query += queryTerms + " WHERE guid = :existing_guid"; 455 change.existing_guid = guid; 456 queries.push({ query, params: change }); 457 notifications.push(["formhistory-update", guid]); 458 break; 459 } 460 case "bump": { 461 log("Bump form history " + change); 462 if (change.guid) { 463 let query = "UPDATE moz_formhistory " + 464 "SET timesUsed = timesUsed + 1, lastUsed = :lastUsed WHERE guid = :guid"; 465 let queryParams = { 466 lastUsed: now, 467 guid: change.guid, 468 }; 469 470 queries.push({ query, params: queryParams }); 471 notifications.push(["formhistory-update", change.guid]); 472 } else { 473 if (!InProgressInserts.add(change.fieldname, change.value)) { 474 // This updateFormHistoryWrite call, or a previous one, is already 475 // going to add this fieldname / value pair, so we can ignore this. 476 continue; 477 } 478 adds.push([change.fieldname, change.value]); 479 change.guid = generateGUID(); 480 let { query, updatedChange } = prepareInsertQuery(change, now); 481 queries.push({ query, params: updatedChange }); 482 notifications.push(["formhistory-add", updatedChange.guid]); 483 } 484 break; 485 } 486 case "add": { 487 if (!InProgressInserts.add(change.fieldname, change.value)) { 488 // This updateFormHistoryWrite call, or a previous one, is already 489 // going to add this fieldname / value pair, so we can ignore this. 490 continue; 491 } 492 adds.push([change.fieldname, change.value]); 493 494 log("Add to form history " + change); 495 if (!change.guid) { 496 change.guid = generateGUID(); 497 } 498 499 let { query, updatedChange } = prepareInsertQuery(change, now); 500 queries.push({ query, params: updatedChange }); 501 notifications.push(["formhistory-add", updatedChange.guid]); 502 break; 503 } 504 default: { 505 // We should've already guaranteed that change.op is one of the above 506 throw Components.Exception("Invalid operation " + operation, 507 Cr.NS_ERROR_ILLEGAL_VALUE); 508 } 509 } 510 } 511 512 try { 513 await runUpdateQueries(conn, queries); 514 for (let [notification, param] of notifications) { 515 // We're either sending a GUID or nothing at all. 516 sendNotification(notification, param); 517 } 518 519 aPreparedHandlers.handleCompletion(0); 520 } catch (e) { 521 aPreparedHandlers.handleError(e); 522 aPreparedHandlers.handleCompletion(1); 523 } finally { 524 InProgressInserts.clear(adds); 525 } 526} 527 528/** 529 * Runs queries for an update operation to the database. This 530 * is separated out from updateFormHistoryWrite to avoid shutdown 531 * leaks where the handlers passed to updateFormHistoryWrite would 532 * leak from the closure around the executeTransaction function. 533 * 534 * @param {SqliteConnection} conn the database connection 535 * @param {Object} queries query string and param pairs generated 536 * by updateFormHistoryWrite 537 */ 538async function runUpdateQueries(conn, queries) { 539 await conn.executeTransaction(async () => { 540 for (let { query, params } of queries) { 541 await conn.executeCached(query, params); 542 } 543 }); 544} 545 546/** 547 * Functions that expire entries in form history and shrinks database 548 * afterwards as necessary initiated by expireOldEntries. 549 */ 550 551/** 552 * Removes entries from database. 553 * 554 * @param {number} aExpireTime expiration timestamp 555 * @param {number} aBeginningCount numer of entries at first 556 */ 557function expireOldEntriesDeletion(aExpireTime, aBeginningCount) { 558 log("expireOldEntriesDeletion(" + aExpireTime + "," + aBeginningCount + ")"); 559 560 FormHistory.update([{ 561 op: "remove", 562 lastUsedEnd: aExpireTime, 563 }], { 564 handleCompletion() { 565 expireOldEntriesVacuum(aExpireTime, aBeginningCount); 566 }, 567 handleError(aError) { 568 log("expireOldEntriesDeletionFailure"); 569 }, 570 }); 571} 572 573/** 574 * Counts number of entries removed and shrinks database as necessary. 575 * 576 * @param {number} aExpireTime expiration timestamp 577 * @param {number} aBeginningCount number of entries at first 578 */ 579function expireOldEntriesVacuum(aExpireTime, aBeginningCount) { 580 FormHistory.count({}, { 581 handleResult(aEndingCount) { 582 if (aBeginningCount - aEndingCount > 500) { 583 log("expireOldEntriesVacuum"); 584 585 FormHistory.db.then(async conn => { 586 try { 587 await conn.executeCached("VACUUM"); 588 } catch (e) { 589 log("expireVacuumError"); 590 } 591 }); 592 } 593 594 sendNotification("formhistory-expireoldentries", aExpireTime); 595 }, 596 handleError(aError) { 597 log("expireEndCountFailure"); 598 }, 599 }); 600} 601 602 603/** 604 * Database creation and access. Used by FormHistory and some of the 605 * utility functions, but is not exposed to the outside world. 606 * @class 607 */ 608var DB = { 609 // Once we establish a database connection, we have to hold a reference 610 // to it so that it won't get GC'd. 611 _instance: null, 612 // MAX_ATTEMPTS is how many times we'll try to establish a connection 613 // or migrate a database before giving up. 614 MAX_ATTEMPTS: 2, 615 616 /** String representing where the FormHistory database is on the filesystem */ 617 get path() { 618 return OS.Path.join(OS.Constants.Path.profileDir, DB_FILENAME); 619 }, 620 621 /** 622 * Sets up and returns a connection to the FormHistory database. The 623 * connection also registers itself with AsyncShutdown so that the 624 * connection is closed on when the profile-before-change observer 625 * notification is fired. 626 * 627 * @returns {Promise} 628 * @resolves An Sqlite.jsm connection to the database. 629 * @rejects If connecting to the database, or migrating the database 630 * failed after MAX_ATTEMPTS attempts (where each attempt 631 * backs up and deletes the old database), this will reject 632 * with the Sqlite.jsm error. 633 */ 634 get conn() { 635 delete this.conn; 636 let conn = new Promise(async (resolve, reject) => { 637 try { 638 this._instance = await this._establishConn(); 639 } catch (e) { 640 log("Failed to establish database connection: " + e); 641 reject(e); 642 return; 643 } 644 645 resolve(this._instance); 646 }); 647 648 return this.conn = conn; 649 }, 650 651 // Private functions 652 653 /** 654 * Tries to connect to the Sqlite database at this.path, and then 655 * migrates the database as necessary. If any of the steps to do this 656 * fail, this function should re-enter itself with an incremented 657 * attemptNum so that another attempt can be made after backing up 658 * and deleting the old database. 659 * 660 * @async 661 * @param {number} attemptNum 662 * The optional number of the attempt that is being made to connect 663 * to the database. Defaults to 0. 664 * @returns {Promise} 665 * @resolves An Sqlite.jsm connection to the database. 666 * @rejects After MAX_ATTEMPTS, this will reject with the Sqlite.jsm 667 * error. 668 */ 669 async _establishConn(attemptNum = 0) { 670 log(`Establishing database connection - attempt # ${attemptNum}`); 671 let conn; 672 try { 673 conn = await Sqlite.openConnection({ path: this.path }); 674 Sqlite.shutdown.addBlocker( 675 "Closing FormHistory database.", () => conn.close()); 676 } catch (e) { 677 // Bug 1423729 - We should check the reason for the connection failure, 678 // in case this is due to the disk being full or the database file being 679 // inaccessible due to third-party software (like anti-virus software). 680 // In that case, we should probably fail right away. 681 if (attemptNum < this.MAX_ATTEMPTS) { 682 log("Establishing connection failed."); 683 await this._failover(conn); 684 return this._establishConn(++attemptNum); 685 } 686 687 if (conn) { 688 await conn.close(); 689 } 690 log("Establishing connection failed too many times. Giving up."); 691 throw e; 692 } 693 694 try { 695 let dbVersion = parseInt(await conn.getSchemaVersion(), 10); 696 697 // Case 1: Database is up to date and we're ready to go. 698 if (dbVersion == DB_SCHEMA_VERSION) { 699 return conn; 700 } 701 702 // Case 2: Downgrade 703 if (dbVersion > DB_SCHEMA_VERSION) { 704 log("Downgrading to version " + DB_SCHEMA_VERSION); 705 // User's DB is newer. Sanity check that our expected columns are 706 // present, and if so mark the lower version and merrily continue 707 // on. If the columns are borked, something is wrong so blow away 708 // the DB and start from scratch. [Future incompatible upgrades 709 // should switch to a different table or file.] 710 if (!(await this._expectedColumnsPresent(conn))) { 711 throw Components.Exception("DB is missing expected columns", 712 Cr.NS_ERROR_FILE_CORRUPTED); 713 } 714 715 // Change the stored version to the current version. If the user 716 // runs the newer code again, it will see the lower version number 717 // and re-upgrade (to fixup any entries the old code added). 718 await conn.setSchemaVersion(DB_SCHEMA_VERSION); 719 return conn; 720 } 721 722 // Case 3: Very old database that cannot be migrated. 723 // 724 // When FormHistory is released, we will no longer support the various 725 // schema versions prior to this release that nsIFormHistory2 once did. 726 // We'll throw an NS_ERROR_FILE_CORRUPTED, which should cause us to wipe 727 // out this DB and create a new one (unless this is our MAX_ATTEMPTS 728 // attempt). 729 if (dbVersion > 0 && dbVersion < 3) { 730 throw Components.Exception("DB version is unsupported.", 731 Cr.NS_ERROR_FILE_CORRUPTED); 732 } 733 734 if (dbVersion == 0) { 735 // Case 4: New database 736 await conn.executeTransaction(async () => { 737 log("Creating DB -- tables"); 738 for (let name in dbSchema.tables) { 739 let table = dbSchema.tables[name]; 740 let tSQL = Object.keys(table).map(col => [col, table[col]].join(" ")).join(", "); 741 log("Creating table " + name + " with " + tSQL); 742 await conn.execute(`CREATE TABLE ${name} (${tSQL})`); 743 } 744 745 log("Creating DB -- indices"); 746 for (let name in dbSchema.indices) { 747 let index = dbSchema.indices[name]; 748 let statement = "CREATE INDEX IF NOT EXISTS " + name + " ON " + index.table + 749 "(" + index.columns.join(", ") + ")"; 750 await conn.execute(statement); 751 } 752 }); 753 } else { 754 // Case 5: Old database requiring a migration 755 await conn.executeTransaction(async () => { 756 for (let v = dbVersion + 1; v <= DB_SCHEMA_VERSION; v++) { 757 log("Upgrading to version " + v + "..."); 758 await Migrators["dbAsyncMigrateToVersion" + v](conn); 759 } 760 }); 761 } 762 763 await conn.setSchemaVersion(DB_SCHEMA_VERSION); 764 765 return conn; 766 } catch (e) { 767 if (e.result != Cr.NS_ERROR_FILE_CORRUPTED) { 768 throw e; 769 } 770 771 if (attemptNum < this.MAX_ATTEMPTS) { 772 log("Setting up database failed."); 773 await this._failover(conn); 774 return this._establishConn(++attemptNum); 775 } 776 777 if (conn) { 778 await conn.close(); 779 } 780 781 log("Setting up database failed too many times. Giving up."); 782 783 throw e; 784 } 785 }, 786 787 /** 788 * Closes a connection to the database, then backs up the database before 789 * deleting it. 790 * 791 * @async 792 * @param {SqliteConnection | null} conn 793 * The connection to the database that we failed to establish or 794 * migrate. 795 * @throws If any file operations fail. 796 */ 797 async _failover(conn) { 798 log("Cleaning up DB file - close & remove & backup."); 799 if (conn) { 800 await conn.close(); 801 } 802 let backupFile = this.path + ".corrupt"; 803 let { file, path: uniquePath } = 804 await OS.File.openUnique(backupFile, { humanReadable: true }); 805 await file.close(); 806 await OS.File.copy(this.path, uniquePath); 807 await OS.File.remove(this.path); 808 log("Completed DB cleanup."); 809 }, 810 811 /** 812 * Tests that a database connection contains the tables that we expect. 813 * 814 * @async 815 * @param {SqliteConnection | null} conn 816 * The connection to the database that we're testing. 817 * @returns {Promise} 818 * @resolves true if all expected columns are present. 819 */ 820 async _expectedColumnsPresent(conn) { 821 for (let name in dbSchema.tables) { 822 let table = dbSchema.tables[name]; 823 let query = "SELECT " + 824 Object.keys(table).join(", ") + 825 " FROM " + name; 826 try { 827 await conn.execute(query, null, (row, cancel) => { 828 // One row is enough to let us know this worked. 829 cancel(); 830 }); 831 } catch (e) { 832 return false; 833 } 834 } 835 836 log("Verified that expected columns are present in DB."); 837 return true; 838 }, 839}; 840 841this.FormHistory = { 842 get db() { 843 return DB.conn; 844 }, 845 846 get enabled() { 847 return Prefs.enabled; 848 }, 849 850 _prepareHandlers(handlers) { 851 let defaultHandlers = { 852 handleResult: NOOP, 853 handleError: NOOP, 854 handleCompletion: NOOP, 855 }; 856 857 if (!handlers) { 858 return defaultHandlers; 859 } 860 861 if (handlers.handleResult) { 862 defaultHandlers.handleResult = handlers.handleResult; 863 } 864 if (handlers.handleError) { 865 defaultHandlers.handleError = handlers.handleError; 866 } 867 if (handlers.handleCompletion) { 868 defaultHandlers.handleCompletion = handlers.handleCompletion; 869 } 870 871 return defaultHandlers; 872 }, 873 874 search(aSelectTerms, aSearchData, aRowFuncOrHandlers) { 875 // if no terms selected, select everything 876 if (!aSelectTerms) { 877 aSelectTerms = validFields; 878 } 879 validateSearchData(aSearchData, "Search"); 880 881 let query = "SELECT " + aSelectTerms.join(", ") + " FROM moz_formhistory"; 882 let queryTerms = makeQueryPredicates(aSearchData); 883 if (queryTerms) { 884 query += " WHERE " + queryTerms; 885 } 886 887 let handlers; 888 889 if (typeof aRowFuncOrHandlers == "function") { 890 handlers = this._prepareHandlers(); 891 handlers.handleResult = aRowFuncOrHandlers; 892 } else if (typeof aRowFuncOrHandlers == "object") { 893 handlers = this._prepareHandlers(aRowFuncOrHandlers); 894 } 895 896 let allResults = []; 897 898 return new Promise((resolve, reject) => { 899 this.db.then(async conn => { 900 try { 901 await conn.executeCached(query, aSearchData, row => { 902 let result = {}; 903 for (let field of aSelectTerms) { 904 result[field] = row.getResultByName(field); 905 } 906 907 if (handlers) { 908 handlers.handleResult(result); 909 } else { 910 allResults.push(result); 911 } 912 }); 913 if (handlers) { 914 handlers.handleCompletion(0); 915 } 916 resolve(allResults); 917 } catch (e) { 918 if (handlers) { 919 handlers.handleError(e); 920 handlers.handleCompletion(1); 921 } 922 reject(e); 923 } 924 }); 925 }); 926 }, 927 928 count(aSearchData, aHandlers) { 929 validateSearchData(aSearchData, "Count"); 930 931 let query = "SELECT COUNT(*) AS numEntries FROM moz_formhistory"; 932 let queryTerms = makeQueryPredicates(aSearchData); 933 if (queryTerms) { 934 query += " WHERE " + queryTerms; 935 } 936 937 let handlers = this._prepareHandlers(aHandlers); 938 939 return new Promise((resolve, reject) => { 940 this.db.then(async conn => { 941 try { 942 let rows = await conn.executeCached(query, aSearchData); 943 let count = rows[0].getResultByName("numEntries"); 944 handlers.handleResult(count); 945 handlers.handleCompletion(0); 946 resolve(count); 947 } catch (e) { 948 handlers.handleError(e); 949 handlers.handleCompletion(1); 950 reject(e); 951 } 952 }); 953 }); 954 }, 955 956 update(aChanges, aHandlers) { 957 // Used to keep track of how many searches have been started. When that number 958 // are finished, updateFormHistoryWrite can be called. 959 let numSearches = 0; 960 let completedSearches = 0; 961 let searchFailed = false; 962 963 function validIdentifier(change) { 964 // The identifier is only valid if one of either the guid 965 // or the (fieldname/value) are set (so an X-OR) 966 return Boolean(change.guid) != Boolean(change.fieldname && change.value); 967 } 968 969 if (!("length" in aChanges)) { 970 aChanges = [aChanges]; 971 } 972 973 let handlers = this._prepareHandlers(aHandlers); 974 975 let isRemoveOperation = aChanges.every(change => change && change.op && change.op == "remove"); 976 if (!Prefs.enabled && !isRemoveOperation) { 977 handlers.handleError({ 978 message: "Form history is disabled, only remove operations are allowed", 979 result: Ci.mozIStorageError.MISUSE, 980 }); 981 handlers.handleCompletion(1); 982 return; 983 } 984 985 for (let change of aChanges) { 986 switch (change.op) { 987 case "remove": 988 validateSearchData(change, "Remove"); 989 continue; 990 case "update": 991 if (validIdentifier(change)) { 992 validateOpData(change, "Update"); 993 if (change.guid) { 994 continue; 995 } 996 } else { 997 throw Components.Exception( 998 "update op='update' does not correctly reference a entry.", 999 Cr.NS_ERROR_ILLEGAL_VALUE); 1000 } 1001 break; 1002 case "bump": 1003 if (validIdentifier(change)) { 1004 validateOpData(change, "Bump"); 1005 if (change.guid) { 1006 continue; 1007 } 1008 } else { 1009 throw Components.Exception( 1010 "update op='bump' does not correctly reference a entry.", 1011 Cr.NS_ERROR_ILLEGAL_VALUE); 1012 } 1013 break; 1014 case "add": 1015 if (change.fieldname && change.value) { 1016 validateOpData(change, "Add"); 1017 } else { 1018 throw Components.Exception( 1019 "update op='add' must have a fieldname and a value.", 1020 Cr.NS_ERROR_ILLEGAL_VALUE); 1021 } 1022 break; 1023 default: 1024 throw Components.Exception( 1025 "update does not recognize op='" + change.op + "'", 1026 Cr.NS_ERROR_ILLEGAL_VALUE); 1027 } 1028 1029 numSearches++; 1030 let changeToUpdate = change; 1031 FormHistory.search( 1032 ["guid"], 1033 { 1034 fieldname: change.fieldname, 1035 value: change.value, 1036 }, { 1037 foundResult: false, 1038 handleResult(aResult) { 1039 if (this.foundResult) { 1040 log("Database contains multiple entries with the same fieldname/value pair."); 1041 handlers.handleError({ 1042 message: 1043 "Database contains multiple entries with the same fieldname/value pair.", 1044 result: 19, // Constraint violation 1045 }); 1046 1047 searchFailed = true; 1048 return; 1049 } 1050 1051 this.foundResult = true; 1052 changeToUpdate.guid = aResult.guid; 1053 }, 1054 1055 handleError(aError) { 1056 handlers.handleError(aError); 1057 }, 1058 1059 handleCompletion(aReason) { 1060 completedSearches++; 1061 if (completedSearches == numSearches) { 1062 if (!aReason && !searchFailed) { 1063 updateFormHistoryWrite(aChanges, handlers); 1064 } else { 1065 handlers.handleCompletion(1); 1066 } 1067 } 1068 }, 1069 }); 1070 } 1071 1072 if (numSearches == 0) { 1073 // We don't have to wait for any statements to return. 1074 updateFormHistoryWrite(aChanges, handlers); 1075 } 1076 }, 1077 1078 getAutoCompleteResults(searchString, params, aHandlers) { 1079 // only do substring matching when the search string contains more than one character 1080 let searchTokens; 1081 let where = ""; 1082 let boundaryCalc = ""; 1083 1084 if (searchString.length >= 1) { 1085 params.valuePrefix = searchString + "%"; 1086 } 1087 1088 if (searchString.length > 1) { 1089 searchTokens = searchString.split(/\s+/); 1090 1091 // build up the word boundary and prefix match bonus calculation 1092 boundaryCalc = "MAX(1, :prefixWeight * (value LIKE :valuePrefix ESCAPE '/') + ("; 1093 // for each word, calculate word boundary weights for the SELECT clause and 1094 // add word to the WHERE clause of the query 1095 let tokenCalc = []; 1096 let searchTokenCount = Math.min(searchTokens.length, MAX_SEARCH_TOKENS); 1097 for (let i = 0; i < searchTokenCount; i++) { 1098 let escapedToken = searchTokens[i]; 1099 params["tokenBegin" + i] = escapedToken + "%"; 1100 params["tokenBoundary" + i] = "% " + escapedToken + "%"; 1101 params["tokenContains" + i] = "%" + escapedToken + "%"; 1102 1103 tokenCalc.push("(value LIKE :tokenBegin" + i + " ESCAPE '/') + " + 1104 "(value LIKE :tokenBoundary" + i + " ESCAPE '/')"); 1105 where += "AND (value LIKE :tokenContains" + i + " ESCAPE '/') "; 1106 } 1107 // add more weight if we have a traditional prefix match and 1108 // multiply boundary bonuses by boundary weight 1109 boundaryCalc += tokenCalc.join(" + ") + ") * :boundaryWeight)"; 1110 } else if (searchString.length == 1) { 1111 where = "AND (value LIKE :valuePrefix ESCAPE '/') "; 1112 boundaryCalc = "1"; 1113 delete params.prefixWeight; 1114 delete params.boundaryWeight; 1115 } else { 1116 where = ""; 1117 boundaryCalc = "1"; 1118 delete params.prefixWeight; 1119 delete params.boundaryWeight; 1120 } 1121 1122 params.now = Date.now() * 1000; // convert from ms to microseconds 1123 1124 let handlers = this._prepareHandlers(aHandlers); 1125 1126 /* Three factors in the frecency calculation for an entry (in order of use in calculation): 1127 * 1) average number of times used - items used more are ranked higher 1128 * 2) how recently it was last used - items used recently are ranked higher 1129 * 3) additional weight for aged entries surviving expiry - these entries are relevant 1130 * since they have been used multiple times over a large time span so rank them higher 1131 * The score is then divided by the bucket size and we round the result so that entries 1132 * with a very similar frecency are bucketed together with an alphabetical sort. This is 1133 * to reduce the amount of moving around by entries while typing. 1134 */ 1135 1136 let query = "/* do not warn (bug 496471): can't use an index */ " + 1137 "SELECT value, guid, " + 1138 "ROUND( " + 1139 "timesUsed / MAX(1.0, (lastUsed - firstUsed) / :timeGroupingSize) * " + 1140 "MAX(1.0, :maxTimeGroupings - (:now - lastUsed) / :timeGroupingSize) * " + 1141 "MAX(1.0, :agedWeight * (firstUsed < :expiryDate)) / " + 1142 ":bucketSize " + 1143 ", 3) AS frecency, " + 1144 boundaryCalc + " AS boundaryBonuses " + 1145 "FROM moz_formhistory " + 1146 "WHERE fieldname=:fieldname " + where + 1147 "ORDER BY ROUND(frecency * boundaryBonuses) DESC, UPPER(value) ASC"; 1148 1149 let cancelled = false; 1150 1151 let cancellableQuery = { 1152 cancel() { 1153 cancelled = true; 1154 }, 1155 }; 1156 1157 this.db.then(async conn => { 1158 try { 1159 await conn.executeCached(query, params, (row, cancel) => { 1160 if (cancelled) { 1161 cancel(); 1162 return; 1163 } 1164 1165 let value = row.getResultByName("value"); 1166 let guid = row.getResultByName("guid"); 1167 let frecency = row.getResultByName("frecency"); 1168 let entry = { 1169 text: value, 1170 guid, 1171 textLowerCase: value.toLowerCase(), 1172 frecency, 1173 totalScore: Math.round(frecency * row.getResultByName("boundaryBonuses")), 1174 }; 1175 handlers.handleResult(entry); 1176 }); 1177 handlers.handleCompletion(0); 1178 } catch (e) { 1179 handlers.handleError(e); 1180 handlers.handleCompletion(1); 1181 } 1182 }); 1183 1184 return cancellableQuery; 1185 }, 1186 1187 // This is used only so that the test can verify deleted table support. 1188 get _supportsDeletedTable() { 1189 return supportsDeletedTable; 1190 }, 1191 set _supportsDeletedTable(val) { 1192 supportsDeletedTable = val; 1193 }, 1194 1195 // The remaining methods are called by FormHistoryStartup.js 1196 updatePrefs() { 1197 Prefs.initialized = false; 1198 }, 1199 1200 expireOldEntries() { 1201 log("expireOldEntries"); 1202 1203 // Determine how many days of history we're supposed to keep. 1204 // Calculate expireTime in microseconds 1205 let expireTime = (Date.now() - Prefs.expireDays * DAY_IN_MS) * 1000; 1206 1207 sendNotification("formhistory-beforeexpireoldentries", expireTime); 1208 1209 FormHistory.count({}, { 1210 handleResult(aBeginningCount) { 1211 expireOldEntriesDeletion(expireTime, aBeginningCount); 1212 }, 1213 handleError(aError) { 1214 log("expireStartCountFailure"); 1215 }, 1216 }); 1217 }, 1218}; 1219 1220// Prevent add-ons from redefining this API 1221Object.freeze(FormHistory); 1222