1/* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4"use strict"; 5 6/** 7 * FormHistory 8 * 9 * Used to store values that have been entered into forms which may later 10 * be used to automatically fill in the values when the form is visited again. 11 * 12 * search(terms, queryData, callback) 13 * Look up values that have been previously stored. 14 * terms - array of terms to return data for 15 * queryData - object that contains the query terms 16 * The query object contains properties for each search criteria to match, where the value 17 * of the property specifies the value that term must have. For example, 18 * { term1: value1, term2: value2 } 19 * callback - callback that is called when results are available or an error occurs. 20 * The callback is passed a result array containing each found entry. Each element in 21 * the array is an object containing a property for each search term specified by 'terms'. 22 * count(queryData, callback) 23 * Find the number of stored entries that match the given criteria. 24 * queryData - array of objects that indicate the query. See the search method for details. 25 * callback - callback that is called when results are available or an error occurs. 26 * The callback is passed the number of found entries. 27 * update(changes, callback) 28 * Write data to form history storage. 29 * changes - an array of changes to be made. If only one change is to be made, it 30 * may be passed as an object rather than a one-element array. 31 * Each change object is of the form: 32 * { op: operation, term1: value1, term2: value2, ... } 33 * Valid operations are: 34 * add - add a new entry 35 * update - update an existing entry 36 * remove - remove an entry 37 * bump - update the last accessed time on an entry 38 * The terms specified allow matching of one or more specific entries. If no terms 39 * are specified then all entries are matched. This means that { op: "remove" } is 40 * used to remove all entries and clear the form history. 41 * callback - callback that is called when results have been stored. 42 * getAutoCompeteResults(searchString, params, callback) 43 * Retrieve an array of form history values suitable for display in an autocomplete list. 44 * Returns an mozIStoragePendingStatement that can be used to cancel the operation if 45 * needed. 46 * searchString - the string to search for, typically the entered value of a textbox 47 * params - zero or more filter arguments: 48 * fieldname - form field name 49 * agedWeight 50 * bucketSize 51 * expiryDate 52 * maxTimeGroundings 53 * timeGroupingSize 54 * prefixWeight 55 * boundaryWeight 56 * source 57 * callback - callback that is called with the array of results. Each result in the array 58 * is an object with four arguments: 59 * text, textLowerCase, frecency, totalScore 60 * schemaVersion 61 * This property holds the version of the database schema 62 * 63 * Terms: 64 * guid - entry identifier. For 'add', a guid will be generated. 65 * fieldname - form field name 66 * value - form value 67 * timesUsed - the number of times the entry has been accessed 68 * firstUsed - the time the the entry was first created 69 * lastUsed - the time the entry was last accessed 70 * firstUsedStart - search for entries created after or at this time 71 * firstUsedEnd - search for entries created before or at this time 72 * lastUsedStart - search for entries last accessed after or at this time 73 * lastUsedEnd - search for entries last accessed before or at this time 74 * newGuid - a special case valid only for 'update' and allows the guid for 75 * an existing record to be updated. The 'guid' term is the only 76 * other term which can be used (ie, you can not also specify a 77 * fieldname, value etc) and indicates the guid of the existing 78 * record that should be updated. 79 * 80 * In all of the above methods, the callback argument should be an object with 81 * handleResult(result), handleFailure(error) and handleCompletion(reason) functions. 82 * For search and getAutoCompeteResults, result is an object containing the desired 83 * properties. For count, result is the integer count. For, update, handleResult is 84 * not called. For handleCompletion, reason is either 0 if successful or 1 if 85 * an error occurred. 86 */ 87 88var EXPORTED_SYMBOLS = ["FormHistory"]; 89 90const { XPCOMUtils } = ChromeUtils.import( 91 "resource://gre/modules/XPCOMUtils.jsm" 92); 93const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); 94const { AppConstants } = ChromeUtils.import( 95 "resource://gre/modules/AppConstants.jsm" 96); 97 98XPCOMUtils.defineLazyServiceGetter( 99 this, 100 "uuidService", 101 "@mozilla.org/uuid-generator;1", 102 "nsIUUIDGenerator" 103); 104ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm"); 105ChromeUtils.defineModuleGetter( 106 this, 107 "Sqlite", 108 "resource://gre/modules/Sqlite.jsm" 109); 110 111const DB_SCHEMA_VERSION = 5; 112const DAY_IN_MS = 86400000; // 1 day in milliseconds 113const MAX_SEARCH_TOKENS = 10; 114const NOOP = function noop() {}; 115const DB_FILENAME = "formhistory.sqlite"; 116 117var supportsDeletedTable = AppConstants.platform == "android"; 118 119var Prefs = { 120 initialized: false, 121 122 get debug() { 123 this.ensureInitialized(); 124 return this._debug; 125 }, 126 get enabled() { 127 this.ensureInitialized(); 128 return this._enabled; 129 }, 130 get expireDays() { 131 this.ensureInitialized(); 132 return this._expireDays; 133 }, 134 135 ensureInitialized() { 136 if (this.initialized) { 137 return; 138 } 139 140 this.initialized = true; 141 142 this._debug = Services.prefs.getBoolPref("browser.formfill.debug"); 143 this._enabled = Services.prefs.getBoolPref("browser.formfill.enable"); 144 this._expireDays = Services.prefs.getIntPref( 145 "browser.formfill.expire_days" 146 ); 147 }, 148}; 149 150function log(aMessage) { 151 if (Prefs.debug) { 152 Services.console.logStringMessage("FormHistory: " + aMessage); 153 } 154} 155 156function sendNotification(aType, aData) { 157 if (typeof aData == "string") { 158 let strWrapper = Cc["@mozilla.org/supports-string;1"].createInstance( 159 Ci.nsISupportsString 160 ); 161 strWrapper.data = aData; 162 aData = strWrapper; 163 } else if (typeof aData == "number") { 164 let intWrapper = Cc["@mozilla.org/supports-PRInt64;1"].createInstance( 165 Ci.nsISupportsPRInt64 166 ); 167 intWrapper.data = aData; 168 aData = intWrapper; 169 } else if (aData) { 170 throw Components.Exception( 171 "Invalid type " + typeof aType + " passed to sendNotification", 172 Cr.NS_ERROR_ILLEGAL_VALUE 173 ); 174 } 175 176 Services.obs.notifyObservers(aData, "satchel-storage-changed", aType); 177} 178 179/** 180 * Current database schema 181 */ 182 183const dbSchema = { 184 tables: { 185 moz_formhistory: { 186 id: "INTEGER PRIMARY KEY", 187 fieldname: "TEXT NOT NULL", 188 value: "TEXT NOT NULL", 189 timesUsed: "INTEGER", 190 firstUsed: "INTEGER", 191 lastUsed: "INTEGER", 192 guid: "TEXT", 193 }, 194 moz_deleted_formhistory: { 195 id: "INTEGER PRIMARY KEY", 196 timeDeleted: "INTEGER", 197 guid: "TEXT", 198 }, 199 moz_sources: { 200 id: "INTEGER PRIMARY KEY", 201 source: "TEXT NOT NULL", 202 }, 203 moz_history_to_sources: { 204 history_id: "INTEGER", 205 source_id: "INTEGER", 206 SQL: ` 207 PRIMARY KEY (history_id, source_id), 208 FOREIGN KEY (history_id) REFERENCES moz_formhistory(id) ON DELETE CASCADE, 209 FOREIGN KEY (source_id) REFERENCES moz_sources(id) ON DELETE CASCADE 210 `, 211 }, 212 }, 213 indices: { 214 moz_formhistory_index: { 215 table: "moz_formhistory", 216 columns: ["fieldname"], 217 }, 218 moz_formhistory_lastused_index: { 219 table: "moz_formhistory", 220 columns: ["lastUsed"], 221 }, 222 moz_formhistory_guid_index: { 223 table: "moz_formhistory", 224 columns: ["guid"], 225 }, 226 }, 227}; 228 229/** 230 * Validating and processing API querying data 231 */ 232 233const validFields = [ 234 "fieldname", 235 "firstUsed", 236 "guid", 237 "lastUsed", 238 "source", 239 "timesUsed", 240 "value", 241]; 242 243const searchFilters = [ 244 "firstUsedStart", 245 "firstUsedEnd", 246 "lastUsedStart", 247 "lastUsedEnd", 248 "source", 249]; 250 251function validateOpData(aData, aDataType) { 252 let thisValidFields = validFields; 253 // A special case to update the GUID - in this case there can be a 'newGuid' 254 // field and of the normally valid fields, only 'guid' is accepted. 255 if (aDataType == "Update" && "newGuid" in aData) { 256 thisValidFields = ["guid", "newGuid"]; 257 } 258 for (let field in aData) { 259 if (field != "op" && !thisValidFields.includes(field)) { 260 throw Components.Exception( 261 aDataType + " query contains an unrecognized field: " + field, 262 Cr.NS_ERROR_ILLEGAL_VALUE 263 ); 264 } 265 } 266 return aData; 267} 268 269function validateSearchData(aData, aDataType) { 270 for (let field in aData) { 271 if ( 272 field != "op" && 273 !validFields.includes(field) && 274 !searchFilters.includes(field) 275 ) { 276 throw Components.Exception( 277 aDataType + " query contains an unrecognized field: " + field, 278 Cr.NS_ERROR_ILLEGAL_VALUE 279 ); 280 } 281 } 282} 283 284function makeQueryPredicates(aQueryData, delimiter = " AND ") { 285 let params = {}; 286 let queryTerms = Object.keys(aQueryData) 287 .filter(field => aQueryData[field] !== undefined) 288 .map(field => { 289 params[field] = aQueryData[field]; 290 switch (field) { 291 case "firstUsedStart": { 292 return "firstUsed >= :" + field; 293 } 294 case "firstUsedEnd": { 295 return "firstUsed <= :" + field; 296 } 297 case "lastUsedStart": { 298 return "lastUsed >= :" + field; 299 } 300 case "lastUsedEnd": { 301 return "lastUsed <= :" + field; 302 } 303 case "source": { 304 return `EXISTS( 305 SELECT 1 FROM moz_history_to_sources 306 JOIN moz_sources s ON s.id = source_id 307 WHERE source = :${field} 308 AND history_id = moz_formhistory.id 309 )`; 310 } 311 } 312 return field + " = :" + field; 313 }) 314 .join(delimiter); 315 return { queryTerms, params }; 316} 317 318function generateGUID() { 319 // string like: "{f60d9eac-9421-4abc-8491-8e8322b063d4}" 320 let uuid = uuidService.generateUUID().toString(); 321 let raw = ""; // A string with the low bytes set to random values 322 let bytes = 0; 323 for (let i = 1; bytes < 12; i += 2) { 324 // Skip dashes 325 if (uuid[i] == "-") { 326 i++; 327 } 328 let hexVal = parseInt(uuid[i] + uuid[i + 1], 16); 329 raw += String.fromCharCode(hexVal); 330 bytes++; 331 } 332 return btoa(raw); 333} 334 335var Migrators = { 336 // Bug 506402 - Adds deleted form history table. 337 async dbAsyncMigrateToVersion4(conn) { 338 const tableName = "moz_deleted_formhistory"; 339 let tableExists = await conn.tableExists(tableName); 340 if (!tableExists) { 341 await createTable(conn, tableName); 342 } 343 }, 344 345 // Bug 1654862 - Adds sources and moz_history_to_sources tables. 346 async dbAsyncMigrateToVersion5(conn) { 347 if (!(await conn.tableExists("moz_sources"))) { 348 for (let tableName of ["moz_history_to_sources", "moz_sources"]) { 349 await createTable(conn, tableName); 350 } 351 } 352 }, 353}; 354 355/** 356 * @typedef {Object} InsertQueryData 357 * @property {Object} updatedChange 358 * A change requested by FormHistory. 359 * @property {String} query 360 * The insert query string. 361 */ 362 363/** 364 * Prepares a query and some default parameters when inserting an entry 365 * to the database. 366 * 367 * @param {Object} change 368 * The change requested by FormHistory. 369 * @param {number} now 370 * The current timestamp in microseconds. 371 * @returns {InsertQueryData} 372 * The query information needed to pass along to the database. 373 */ 374function prepareInsertQuery(change, now) { 375 let params = {}; 376 for (let key of new Set([ 377 ...Object.keys(change), 378 // These must always be NOT NULL. 379 "firstUsed", 380 "lastUsed", 381 "timesUsed", 382 ])) { 383 switch (key) { 384 case "fieldname": 385 case "guid": 386 case "value": 387 params[key] = change[key]; 388 break; 389 case "firstUsed": 390 case "lastUsed": 391 params[key] = change[key] || now; 392 break; 393 case "timesUsed": 394 params[key] = change[key] || 1; 395 break; 396 default: 397 // Skip unnecessary properties. 398 } 399 } 400 401 return { 402 query: ` 403 INSERT INTO moz_formhistory 404 (fieldname, value, timesUsed, firstUsed, lastUsed, guid) 405 VALUES (:fieldname, :value, :timesUsed, :firstUsed, :lastUsed, :guid)`, 406 params, 407 }; 408} 409 410// There is a fieldname / value uniqueness constraint that's at this time 411// only enforced at this level. This Map maps fieldnames => values that 412// are in the process of being inserted into the database so that we know 413// not to try to insert the same ones on top. Attempts to do so will be 414// ignored. 415var InProgressInserts = { 416 _inProgress: new Map(), 417 418 add(fieldname, value) { 419 let fieldnameSet = this._inProgress.get(fieldname); 420 if (!fieldnameSet) { 421 this._inProgress.set(fieldname, new Set([value])); 422 return true; 423 } 424 425 if (!fieldnameSet.has(value)) { 426 fieldnameSet.add(value); 427 return true; 428 } 429 430 return false; 431 }, 432 433 clear(fieldnamesAndValues) { 434 for (let [fieldname, value] of fieldnamesAndValues) { 435 let fieldnameSet = this._inProgress.get(fieldname); 436 if ( 437 fieldnameSet && 438 fieldnameSet.delete(value) && 439 fieldnameSet.size == 0 440 ) { 441 this._inProgress.delete(fieldname); 442 } 443 } 444 }, 445}; 446 447function getAddSourceToGuidQueries(source, guid) { 448 return [ 449 { 450 query: `INSERT OR IGNORE INTO moz_sources (source) VALUES (:source)`, 451 params: { source }, 452 }, 453 { 454 query: ` 455 INSERT OR IGNORE INTO moz_history_to_sources (history_id, source_id) 456 VALUES( 457 (SELECT id FROM moz_formhistory WHERE guid = :guid), 458 (SELECT id FROM moz_sources WHERE source = :source) 459 ) 460 `, 461 params: { guid, source }, 462 }, 463 ]; 464} 465 466/** 467 * Constructs and executes database statements from a pre-processed list of 468 * inputted changes. 469 * 470 * @param {Array.<Object>} aChanges changes to form history 471 * @param {Object} aPreparedHandlers 472 */ 473// XXX This should be split up and the complexity reduced. 474// eslint-disable-next-line complexity 475async function updateFormHistoryWrite(aChanges, aPreparedHandlers) { 476 log("updateFormHistoryWrite " + aChanges.length); 477 478 // pass 'now' down so that every entry in the batch has the same timestamp 479 let now = Date.now() * 1000; 480 let queries = []; 481 let notifications = []; 482 let adds = []; 483 let conn = await FormHistory.db; 484 485 for (let change of aChanges) { 486 let operation = change.op; 487 delete change.op; 488 switch (operation) { 489 case "remove": { 490 log("Remove from form history " + change); 491 let { queryTerms, params } = makeQueryPredicates(change); 492 493 // If source is defined, we only remove the source relation, if the 494 // consumer intends to remove the value from everywhere, then they 495 // should not pass source. This gives full control to the caller. 496 if (change.source) { 497 await conn.executeCached( 498 `DELETE FROM moz_history_to_sources 499 WHERE source_id = ( 500 SELECT id FROM moz_sources WHERE source = :source 501 ) 502 AND history_id = ( 503 SELECT id FROM moz_formhistory WHERE ${queryTerms} 504 ) 505 `, 506 params 507 ); 508 break; 509 } 510 511 // Fetch the GUIDs we are going to delete. 512 try { 513 let query = "SELECT guid FROM moz_formhistory"; 514 if (queryTerms) { 515 query += " WHERE " + queryTerms; 516 } 517 518 await conn.executeCached(query, params, row => { 519 notifications.push([ 520 "formhistory-remove", 521 row.getResultByName("guid"), 522 ]); 523 }); 524 } catch (e) { 525 log("Error getting guids from moz_formhistory: " + e); 526 } 527 528 if (supportsDeletedTable) { 529 log("Moving to deleted table " + change); 530 let query = "INSERT INTO moz_deleted_formhistory (guid, timeDeleted)"; 531 532 // TODO: Add these items to the deleted items table once we've sorted 533 // out the issues from bug 756701 534 if (change.guid || queryTerms) { 535 query += change.guid 536 ? " VALUES (:guid, :timeDeleted)" 537 : " SELECT guid, :timeDeleted FROM moz_formhistory WHERE " + 538 queryTerms; 539 queries.push({ 540 query, 541 params: Object.assign({ timeDeleted: now }, params), 542 }); 543 } 544 } 545 546 let query = "DELETE FROM moz_formhistory"; 547 if (queryTerms) { 548 log("removeEntries"); 549 query += " WHERE " + queryTerms; 550 } else { 551 log("removeAllEntries"); 552 // Not specifying any fields means we should remove all entries. We 553 // won't need to modify the query in this case. 554 } 555 556 queries.push({ query, params }); 557 // Expire orphan sources. 558 queries.push({ 559 query: ` 560 DELETE FROM moz_sources WHERE id NOT IN ( 561 SELECT DISTINCT source_id FROM moz_history_to_sources 562 )`, 563 }); 564 break; 565 } 566 case "update": { 567 log("Update form history " + change); 568 let guid = change.guid; 569 delete change.guid; 570 // a special case for updating the GUID - the new value can be 571 // specified in newGuid. 572 if (change.newGuid) { 573 change.guid = change.newGuid; 574 delete change.newGuid; 575 } 576 577 let query = "UPDATE moz_formhistory SET "; 578 let { queryTerms, params } = makeQueryPredicates(change, ", "); 579 if (!queryTerms) { 580 throw Components.Exception( 581 "Update query must define fields to modify.", 582 Cr.NS_ERROR_ILLEGAL_VALUE 583 ); 584 } 585 query += queryTerms + " WHERE guid = :existing_guid"; 586 queries.push({ 587 query, 588 params: Object.assign({ existing_guid: guid }, params), 589 }); 590 591 notifications.push(["formhistory-update", guid]); 592 593 // Source is ignored for "update" operations, since it's not really 594 // common to change the source of a value, and anyway currently this is 595 // mostly used to update guids. 596 break; 597 } 598 case "bump": { 599 log("Bump form history " + change); 600 if (change.guid) { 601 let query = 602 "UPDATE moz_formhistory " + 603 "SET timesUsed = timesUsed + 1, lastUsed = :lastUsed WHERE guid = :guid"; 604 let queryParams = { 605 lastUsed: now, 606 guid: change.guid, 607 }; 608 609 queries.push({ query, params: queryParams }); 610 notifications.push(["formhistory-update", change.guid]); 611 } else { 612 if (!InProgressInserts.add(change.fieldname, change.value)) { 613 // This updateFormHistoryWrite call, or a previous one, is already 614 // going to add this fieldname / value pair, so we can ignore this. 615 continue; 616 } 617 adds.push([change.fieldname, change.value]); 618 change.guid = generateGUID(); 619 let { query, params } = prepareInsertQuery(change, now); 620 queries.push({ query, params }); 621 notifications.push(["formhistory-add", params.guid]); 622 } 623 624 if (change.source) { 625 queries = queries.concat( 626 getAddSourceToGuidQueries(change.source, change.guid) 627 ); 628 } 629 break; 630 } 631 case "add": { 632 if (!InProgressInserts.add(change.fieldname, change.value)) { 633 // This updateFormHistoryWrite call, or a previous one, is already 634 // going to add this fieldname / value pair, so we can ignore this. 635 continue; 636 } 637 adds.push([change.fieldname, change.value]); 638 639 log("Add to form history " + change); 640 if (!change.guid) { 641 change.guid = generateGUID(); 642 } 643 644 let { query, params } = prepareInsertQuery(change, now); 645 queries.push({ query, params }); 646 647 notifications.push(["formhistory-add", params.guid]); 648 649 if (change.source) { 650 queries = queries.concat( 651 getAddSourceToGuidQueries(change.source, change.guid) 652 ); 653 } 654 break; 655 } 656 default: { 657 // We should've already guaranteed that change.op is one of the above 658 throw Components.Exception( 659 "Invalid operation " + operation, 660 Cr.NS_ERROR_ILLEGAL_VALUE 661 ); 662 } 663 } 664 } 665 666 try { 667 await runUpdateQueries(conn, queries); 668 for (let [notification, param] of notifications) { 669 // We're either sending a GUID or nothing at all. 670 sendNotification(notification, param); 671 } 672 673 aPreparedHandlers.handleCompletion(0); 674 } catch (e) { 675 aPreparedHandlers.handleError(e); 676 aPreparedHandlers.handleCompletion(1); 677 } finally { 678 InProgressInserts.clear(adds); 679 } 680} 681 682/** 683 * Runs queries for an update operation to the database. This 684 * is separated out from updateFormHistoryWrite to avoid shutdown 685 * leaks where the handlers passed to updateFormHistoryWrite would 686 * leak from the closure around the executeTransaction function. 687 * 688 * @param {SqliteConnection} conn the database connection 689 * @param {Object} queries query string and param pairs generated 690 * by updateFormHistoryWrite 691 */ 692async function runUpdateQueries(conn, queries) { 693 await conn.executeTransaction(async () => { 694 for (let { query, params } of queries) { 695 await conn.executeCached(query, params); 696 } 697 }); 698} 699 700/** 701 * Functions that expire entries in form history and shrinks database 702 * afterwards as necessary initiated by expireOldEntries. 703 */ 704 705/** 706 * Removes entries from database. 707 * 708 * @param {number} aExpireTime expiration timestamp 709 * @param {number} aBeginningCount numer of entries at first 710 */ 711function expireOldEntriesDeletion(aExpireTime, aBeginningCount) { 712 log("expireOldEntriesDeletion(" + aExpireTime + "," + aBeginningCount + ")"); 713 714 FormHistory.update( 715 [ 716 { 717 op: "remove", 718 lastUsedEnd: aExpireTime, 719 }, 720 ], 721 { 722 handleCompletion() { 723 expireOldEntriesVacuum(aExpireTime, aBeginningCount); 724 }, 725 handleError(aError) { 726 log("expireOldEntriesDeletionFailure"); 727 }, 728 } 729 ); 730} 731 732/** 733 * Counts number of entries removed and shrinks database as necessary. 734 * 735 * @param {number} aExpireTime expiration timestamp 736 * @param {number} aBeginningCount number of entries at first 737 */ 738function expireOldEntriesVacuum(aExpireTime, aBeginningCount) { 739 FormHistory.count( 740 {}, 741 { 742 handleResult(aEndingCount) { 743 if (aBeginningCount - aEndingCount > 500) { 744 log("expireOldEntriesVacuum"); 745 746 FormHistory.db.then(async conn => { 747 try { 748 await conn.executeCached("VACUUM"); 749 } catch (e) { 750 log("expireVacuumError"); 751 } 752 }); 753 } 754 755 sendNotification("formhistory-expireoldentries", aExpireTime); 756 }, 757 handleError(aError) { 758 log("expireEndCountFailure"); 759 }, 760 } 761 ); 762} 763 764async function createTable(conn, tableName) { 765 let table = dbSchema.tables[tableName]; 766 let columns = Object.keys(table) 767 .filter(col => col != "SQL") 768 .map(col => [col, table[col]].join(" ")) 769 .join(", "); 770 let no_rowid = Object.keys(table).includes("id") ? "" : "WITHOUT ROWID"; 771 log("Creating table " + tableName + " with " + columns); 772 await conn.execute( 773 `CREATE TABLE ${tableName} ( 774 ${columns} 775 ${table.SQL ? "," + table.SQL : ""} 776 ) ${no_rowid}` 777 ); 778} 779 780/** 781 * Database creation and access. Used by FormHistory and some of the 782 * utility functions, but is not exposed to the outside world. 783 * @class 784 */ 785var DB = { 786 // Once we establish a database connection, we have to hold a reference 787 // to it so that it won't get GC'd. 788 _instance: null, 789 // MAX_ATTEMPTS is how many times we'll try to establish a connection 790 // or migrate a database before giving up. 791 MAX_ATTEMPTS: 2, 792 793 /** String representing where the FormHistory database is on the filesystem */ 794 get path() { 795 return OS.Path.join(OS.Constants.Path.profileDir, DB_FILENAME); 796 }, 797 798 /** 799 * Sets up and returns a connection to the FormHistory database. The 800 * connection also registers itself with AsyncShutdown so that the 801 * connection is closed on when the profile-before-change observer 802 * notification is fired. 803 * 804 * @returns {Promise} 805 * @resolves An Sqlite.jsm connection to the database. 806 * @rejects If connecting to the database, or migrating the database 807 * failed after MAX_ATTEMPTS attempts (where each attempt 808 * backs up and deletes the old database), this will reject 809 * with the Sqlite.jsm error. 810 */ 811 get conn() { 812 delete this.conn; 813 let conn = (async () => { 814 try { 815 this._instance = await this._establishConn(); 816 } catch (e) { 817 log("Failed to establish database connection: " + e); 818 throw e; 819 } 820 821 return this._instance; 822 })(); 823 824 return (this.conn = conn); 825 }, 826 827 // Private functions 828 829 /** 830 * Tries to connect to the Sqlite database at this.path, and then 831 * migrates the database as necessary. If any of the steps to do this 832 * fail, this function should re-enter itself with an incremented 833 * attemptNum so that another attempt can be made after backing up 834 * and deleting the old database. 835 * 836 * @async 837 * @param {number} attemptNum 838 * The optional number of the attempt that is being made to connect 839 * to the database. Defaults to 0. 840 * @returns {Promise} 841 * @resolves An Sqlite.jsm connection to the database. 842 * @rejects After MAX_ATTEMPTS, this will reject with the Sqlite.jsm 843 * error. 844 */ 845 async _establishConn(attemptNum = 0) { 846 log(`Establishing database connection - attempt # ${attemptNum}`); 847 let conn; 848 try { 849 conn = await Sqlite.openConnection({ path: this.path }); 850 Sqlite.shutdown.addBlocker("Closing FormHistory database.", () => 851 conn.close() 852 ); 853 } catch (e) { 854 // Bug 1423729 - We should check the reason for the connection failure, 855 // in case this is due to the disk being full or the database file being 856 // inaccessible due to third-party software (like anti-virus software). 857 // In that case, we should probably fail right away. 858 if (attemptNum < this.MAX_ATTEMPTS) { 859 log("Establishing connection failed."); 860 await this._failover(conn); 861 return this._establishConn(++attemptNum); 862 } 863 864 if (conn) { 865 await conn.close(); 866 } 867 log("Establishing connection failed too many times. Giving up."); 868 throw e; 869 } 870 871 try { 872 // Enable foreign keys support. 873 await conn.execute("PRAGMA foreign_keys = ON"); 874 875 let dbVersion = parseInt(await conn.getSchemaVersion(), 10); 876 877 // Case 1: Database is up to date and we're ready to go. 878 if (dbVersion == DB_SCHEMA_VERSION) { 879 return conn; 880 } 881 882 // Case 2: Downgrade 883 if (dbVersion > DB_SCHEMA_VERSION) { 884 log("Downgrading to version " + DB_SCHEMA_VERSION); 885 // User's DB is newer. Sanity check that our expected columns are 886 // present, and if so mark the lower version and merrily continue 887 // on. If the columns are borked, something is wrong so blow away 888 // the DB and start from scratch. [Future incompatible upgrades 889 // should switch to a different table or file.] 890 if (!(await this._expectedColumnsPresent(conn))) { 891 throw Components.Exception( 892 "DB is missing expected columns", 893 Cr.NS_ERROR_FILE_CORRUPTED 894 ); 895 } 896 897 // Change the stored version to the current version. If the user 898 // runs the newer code again, it will see the lower version number 899 // and re-upgrade (to fixup any entries the old code added). 900 await conn.setSchemaVersion(DB_SCHEMA_VERSION); 901 return conn; 902 } 903 904 // Case 3: Very old database that cannot be migrated. 905 // 906 // When FormHistory is released, we will no longer support the various 907 // schema versions prior to this release that nsIFormHistory2 once did. 908 // We'll throw an NS_ERROR_FILE_CORRUPTED, which should cause us to wipe 909 // out this DB and create a new one (unless this is our MAX_ATTEMPTS 910 // attempt). 911 if (dbVersion > 0 && dbVersion < 3) { 912 throw Components.Exception( 913 "DB version is unsupported.", 914 Cr.NS_ERROR_FILE_CORRUPTED 915 ); 916 } 917 918 if (dbVersion == 0) { 919 // Case 4: New database 920 await conn.executeTransaction(async () => { 921 log("Creating DB -- tables"); 922 for (let name in dbSchema.tables) { 923 await createTable(conn, name); 924 } 925 926 log("Creating DB -- indices"); 927 for (let name in dbSchema.indices) { 928 let index = dbSchema.indices[name]; 929 let statement = 930 "CREATE INDEX IF NOT EXISTS " + 931 name + 932 " ON " + 933 index.table + 934 "(" + 935 index.columns.join(", ") + 936 ")"; 937 await conn.execute(statement); 938 } 939 }); 940 } else { 941 // Case 5: Old database requiring a migration 942 await conn.executeTransaction(async () => { 943 for (let v = dbVersion + 1; v <= DB_SCHEMA_VERSION; v++) { 944 log("Upgrading to version " + v + "..."); 945 await Migrators["dbAsyncMigrateToVersion" + v](conn); 946 } 947 }); 948 } 949 950 await conn.setSchemaVersion(DB_SCHEMA_VERSION); 951 952 return conn; 953 } catch (e) { 954 if (e.result != Cr.NS_ERROR_FILE_CORRUPTED) { 955 throw e; 956 } 957 958 if (attemptNum < this.MAX_ATTEMPTS) { 959 log("Setting up database failed."); 960 await this._failover(conn); 961 return this._establishConn(++attemptNum); 962 } 963 964 if (conn) { 965 await conn.close(); 966 } 967 968 log("Setting up database failed too many times. Giving up."); 969 970 throw e; 971 } 972 }, 973 974 /** 975 * Closes a connection to the database, then backs up the database before 976 * deleting it. 977 * 978 * @async 979 * @param {SqliteConnection | null} conn 980 * The connection to the database that we failed to establish or 981 * migrate. 982 * @throws If any file operations fail. 983 */ 984 async _failover(conn) { 985 log("Cleaning up DB file - close & remove & backup."); 986 if (conn) { 987 await conn.close(); 988 } 989 let backupFile = this.path + ".corrupt"; 990 let { file, path: uniquePath } = await OS.File.openUnique(backupFile, { 991 humanReadable: true, 992 }); 993 await file.close(); 994 await OS.File.copy(this.path, uniquePath); 995 await OS.File.remove(this.path); 996 log("Completed DB cleanup."); 997 }, 998 999 /** 1000 * Tests that a database connection contains the tables that we expect. 1001 * 1002 * @async 1003 * @param {SqliteConnection | null} conn 1004 * The connection to the database that we're testing. 1005 * @returns {Promise} 1006 * @resolves true if all expected columns are present. 1007 */ 1008 async _expectedColumnsPresent(conn) { 1009 for (let name in dbSchema.tables) { 1010 let table = dbSchema.tables[name]; 1011 let columns = Object.keys(table).filter(col => col != "SQL"); 1012 let query = "SELECT " + columns.join(", ") + " FROM " + name; 1013 try { 1014 await conn.execute(query, null, (row, cancel) => { 1015 // One row is enough to let us know this worked. 1016 cancel(); 1017 }); 1018 } catch (e) { 1019 return false; 1020 } 1021 } 1022 1023 log("Verified that expected columns are present in DB."); 1024 return true; 1025 }, 1026}; 1027 1028this.FormHistory = { 1029 get db() { 1030 return DB.conn; 1031 }, 1032 1033 get enabled() { 1034 return Prefs.enabled; 1035 }, 1036 1037 _prepareHandlers(handlers) { 1038 let defaultHandlers = { 1039 handleResult: NOOP, 1040 handleError: NOOP, 1041 handleCompletion: NOOP, 1042 }; 1043 1044 if (!handlers) { 1045 return defaultHandlers; 1046 } 1047 1048 if (handlers.handleResult) { 1049 defaultHandlers.handleResult = handlers.handleResult; 1050 } 1051 if (handlers.handleError) { 1052 defaultHandlers.handleError = handlers.handleError; 1053 } 1054 if (handlers.handleCompletion) { 1055 defaultHandlers.handleCompletion = handlers.handleCompletion; 1056 } 1057 1058 return defaultHandlers; 1059 }, 1060 1061 search(aSelectTerms, aSearchData, aRowFuncOrHandlers) { 1062 // if no terms selected, select everything 1063 if (!aSelectTerms) { 1064 // Source is not a valid column in moz_formhistory. 1065 aSelectTerms = validFields.filter(f => f != "source"); 1066 } 1067 1068 validateSearchData(aSearchData, "Search"); 1069 1070 let query = "SELECT " + aSelectTerms.join(", ") + " FROM moz_formhistory"; 1071 let { queryTerms, params } = makeQueryPredicates(aSearchData); 1072 if (queryTerms) { 1073 query += " WHERE " + queryTerms; 1074 } 1075 1076 let handlers; 1077 1078 if (typeof aRowFuncOrHandlers == "function") { 1079 handlers = this._prepareHandlers(); 1080 handlers.handleResult = aRowFuncOrHandlers; 1081 } else if (typeof aRowFuncOrHandlers == "object") { 1082 handlers = this._prepareHandlers(aRowFuncOrHandlers); 1083 } 1084 1085 let allResults = []; 1086 1087 return new Promise((resolve, reject) => { 1088 this.db.then(async conn => { 1089 try { 1090 await conn.executeCached(query, params, row => { 1091 let result = {}; 1092 for (let field of aSelectTerms) { 1093 result[field] = row.getResultByName(field); 1094 } 1095 1096 if (handlers) { 1097 handlers.handleResult(result); 1098 } else { 1099 allResults.push(result); 1100 } 1101 }); 1102 if (handlers) { 1103 handlers.handleCompletion(0); 1104 } 1105 resolve(allResults); 1106 } catch (e) { 1107 if (handlers) { 1108 handlers.handleError(e); 1109 handlers.handleCompletion(1); 1110 } 1111 reject(e); 1112 } 1113 }); 1114 }); 1115 }, 1116 1117 count(aSearchData, aHandlers) { 1118 validateSearchData(aSearchData, "Count"); 1119 1120 let query = "SELECT COUNT(*) AS numEntries FROM moz_formhistory"; 1121 let { queryTerms, params } = makeQueryPredicates(aSearchData); 1122 if (queryTerms) { 1123 query += " WHERE " + queryTerms; 1124 } 1125 1126 let handlers = this._prepareHandlers(aHandlers); 1127 1128 return new Promise((resolve, reject) => { 1129 this.db.then(async conn => { 1130 try { 1131 let rows = await conn.executeCached(query, params); 1132 let count = rows[0].getResultByName("numEntries"); 1133 handlers.handleResult(count); 1134 handlers.handleCompletion(0); 1135 resolve(count); 1136 } catch (e) { 1137 handlers.handleError(e); 1138 handlers.handleCompletion(1); 1139 reject(e); 1140 } 1141 }); 1142 }); 1143 }, 1144 1145 update(aChanges, aHandlers) { 1146 // Used to keep track of how many searches have been started. When that number 1147 // are finished, updateFormHistoryWrite can be called. 1148 let numSearches = 0; 1149 let completedSearches = 0; 1150 let searchFailed = false; 1151 1152 function validIdentifier(change) { 1153 // The identifier is only valid if one of either the guid 1154 // or the (fieldname/value) are set (so an X-OR) 1155 return Boolean(change.guid) != Boolean(change.fieldname && change.value); 1156 } 1157 1158 if (!("length" in aChanges)) { 1159 aChanges = [aChanges]; 1160 } 1161 1162 let handlers = this._prepareHandlers(aHandlers); 1163 1164 let isRemoveOperation = aChanges.every( 1165 change => change && change.op && change.op == "remove" 1166 ); 1167 if (!Prefs.enabled && !isRemoveOperation) { 1168 handlers.handleError({ 1169 message: "Form history is disabled, only remove operations are allowed", 1170 result: Ci.mozIStorageError.MISUSE, 1171 }); 1172 handlers.handleCompletion(1); 1173 return; 1174 } 1175 1176 for (let change of aChanges) { 1177 switch (change.op) { 1178 case "remove": 1179 validateSearchData(change, "Remove"); 1180 continue; 1181 case "update": 1182 if (validIdentifier(change)) { 1183 validateOpData(change, "Update"); 1184 if (change.guid) { 1185 continue; 1186 } 1187 } else { 1188 throw Components.Exception( 1189 "update op='update' does not correctly reference a entry.", 1190 Cr.NS_ERROR_ILLEGAL_VALUE 1191 ); 1192 } 1193 break; 1194 case "bump": 1195 if (validIdentifier(change)) { 1196 validateOpData(change, "Bump"); 1197 if (change.guid) { 1198 continue; 1199 } 1200 } else { 1201 throw Components.Exception( 1202 "update op='bump' does not correctly reference a entry.", 1203 Cr.NS_ERROR_ILLEGAL_VALUE 1204 ); 1205 } 1206 break; 1207 case "add": 1208 if (change.fieldname && change.value) { 1209 validateOpData(change, "Add"); 1210 } else { 1211 throw Components.Exception( 1212 "update op='add' must have a fieldname and a value.", 1213 Cr.NS_ERROR_ILLEGAL_VALUE 1214 ); 1215 } 1216 break; 1217 default: 1218 throw Components.Exception( 1219 "update does not recognize op='" + change.op + "'", 1220 Cr.NS_ERROR_ILLEGAL_VALUE 1221 ); 1222 } 1223 1224 numSearches++; 1225 let changeToUpdate = change; 1226 FormHistory.search( 1227 ["guid"], 1228 { 1229 fieldname: change.fieldname, 1230 value: change.value, 1231 }, 1232 { 1233 foundResult: false, 1234 handleResult(aResult) { 1235 if (this.foundResult) { 1236 log( 1237 "Database contains multiple entries with the same fieldname/value pair." 1238 ); 1239 handlers.handleError({ 1240 message: 1241 "Database contains multiple entries with the same fieldname/value pair.", 1242 result: 19, // Constraint violation 1243 }); 1244 1245 searchFailed = true; 1246 return; 1247 } 1248 1249 this.foundResult = true; 1250 changeToUpdate.guid = aResult.guid; 1251 }, 1252 1253 handleError(aError) { 1254 handlers.handleError(aError); 1255 }, 1256 1257 handleCompletion(aReason) { 1258 completedSearches++; 1259 if (completedSearches == numSearches) { 1260 if (!aReason && !searchFailed) { 1261 updateFormHistoryWrite(aChanges, handlers); 1262 } else { 1263 handlers.handleCompletion(1); 1264 } 1265 } 1266 }, 1267 } 1268 ); 1269 } 1270 1271 if (numSearches == 0) { 1272 // We don't have to wait for any statements to return. 1273 updateFormHistoryWrite(aChanges, handlers); 1274 } 1275 }, 1276 1277 getAutoCompleteResults(searchString, params, aHandlers) { 1278 // only do substring matching when the search string contains more than one character 1279 let searchTokens; 1280 let where = ""; 1281 let boundaryCalc = ""; 1282 1283 if (searchString.length >= 1) { 1284 params.valuePrefix = searchString + "%"; 1285 } 1286 1287 if (searchString.length > 1) { 1288 searchTokens = searchString.split(/\s+/); 1289 1290 // build up the word boundary and prefix match bonus calculation 1291 boundaryCalc = 1292 "MAX(1, :prefixWeight * (value LIKE :valuePrefix ESCAPE '/') + ("; 1293 // for each word, calculate word boundary weights for the SELECT clause and 1294 // add word to the WHERE clause of the query 1295 let tokenCalc = []; 1296 let searchTokenCount = Math.min(searchTokens.length, MAX_SEARCH_TOKENS); 1297 for (let i = 0; i < searchTokenCount; i++) { 1298 let escapedToken = searchTokens[i]; 1299 params["tokenBegin" + i] = escapedToken + "%"; 1300 params["tokenBoundary" + i] = "% " + escapedToken + "%"; 1301 params["tokenContains" + i] = "%" + escapedToken + "%"; 1302 1303 tokenCalc.push( 1304 "(value LIKE :tokenBegin" + 1305 i + 1306 " ESCAPE '/') + " + 1307 "(value LIKE :tokenBoundary" + 1308 i + 1309 " ESCAPE '/')" 1310 ); 1311 where += "AND (value LIKE :tokenContains" + i + " ESCAPE '/') "; 1312 } 1313 // add more weight if we have a traditional prefix match and 1314 // multiply boundary bonuses by boundary weight 1315 boundaryCalc += tokenCalc.join(" + ") + ") * :boundaryWeight)"; 1316 } else if (searchString.length == 1) { 1317 where = "AND (value LIKE :valuePrefix ESCAPE '/') "; 1318 boundaryCalc = "1"; 1319 delete params.prefixWeight; 1320 delete params.boundaryWeight; 1321 } else { 1322 where = ""; 1323 boundaryCalc = "1"; 1324 delete params.prefixWeight; 1325 delete params.boundaryWeight; 1326 } 1327 1328 params.now = Date.now() * 1000; // convert from ms to microseconds 1329 1330 if (params.source) { 1331 where += `AND EXISTS( 1332 SELECT 1 FROM moz_history_to_sources 1333 JOIN moz_sources s ON s.id = source_id 1334 WHERE source = :source 1335 AND history_id = moz_formhistory.id 1336 )`; 1337 } 1338 1339 let handlers = this._prepareHandlers(aHandlers); 1340 1341 /* Three factors in the frecency calculation for an entry (in order of use in calculation): 1342 * 1) average number of times used - items used more are ranked higher 1343 * 2) how recently it was last used - items used recently are ranked higher 1344 * 3) additional weight for aged entries surviving expiry - these entries are relevant 1345 * since they have been used multiple times over a large time span so rank them higher 1346 * The score is then divided by the bucket size and we round the result so that entries 1347 * with a very similar frecency are bucketed together with an alphabetical sort. This is 1348 * to reduce the amount of moving around by entries while typing. 1349 */ 1350 1351 let query = 1352 "/* do not warn (bug 496471): can't use an index */ " + 1353 "SELECT value, guid, " + 1354 "ROUND( " + 1355 "timesUsed / MAX(1.0, (lastUsed - firstUsed) / :timeGroupingSize) * " + 1356 "MAX(1.0, :maxTimeGroupings - (:now - lastUsed) / :timeGroupingSize) * " + 1357 "MAX(1.0, :agedWeight * (firstUsed < :expiryDate)) / " + 1358 ":bucketSize " + 1359 ", 3) AS frecency, " + 1360 boundaryCalc + 1361 " AS boundaryBonuses " + 1362 "FROM moz_formhistory " + 1363 "WHERE fieldname=:fieldname " + 1364 where + 1365 "ORDER BY ROUND(frecency * boundaryBonuses) DESC, UPPER(value) ASC"; 1366 1367 let cancelled = false; 1368 1369 let cancellableQuery = { 1370 cancel() { 1371 cancelled = true; 1372 }, 1373 }; 1374 1375 this.db.then(async conn => { 1376 try { 1377 await conn.executeCached(query, params, (row, cancel) => { 1378 if (cancelled) { 1379 cancel(); 1380 return; 1381 } 1382 1383 let value = row.getResultByName("value"); 1384 let guid = row.getResultByName("guid"); 1385 let frecency = row.getResultByName("frecency"); 1386 let entry = { 1387 text: value, 1388 guid, 1389 textLowerCase: value.toLowerCase(), 1390 frecency, 1391 totalScore: Math.round( 1392 frecency * row.getResultByName("boundaryBonuses") 1393 ), 1394 }; 1395 handlers.handleResult(entry); 1396 }); 1397 handlers.handleCompletion(0); 1398 } catch (e) { 1399 handlers.handleError(e); 1400 handlers.handleCompletion(1); 1401 } 1402 }); 1403 1404 return cancellableQuery; 1405 }, 1406 1407 // This is used only so that the test can verify deleted table support. 1408 get _supportsDeletedTable() { 1409 return supportsDeletedTable; 1410 }, 1411 set _supportsDeletedTable(val) { 1412 supportsDeletedTable = val; 1413 }, 1414 1415 // The remaining methods are called by FormHistoryStartup.js 1416 updatePrefs() { 1417 Prefs.initialized = false; 1418 }, 1419 1420 expireOldEntries() { 1421 log("expireOldEntries"); 1422 1423 // Determine how many days of history we're supposed to keep. 1424 // Calculate expireTime in microseconds 1425 let expireTime = (Date.now() - Prefs.expireDays * DAY_IN_MS) * 1000; 1426 1427 sendNotification("formhistory-beforeexpireoldentries", expireTime); 1428 1429 FormHistory.count( 1430 {}, 1431 { 1432 handleResult(aBeginningCount) { 1433 expireOldEntriesDeletion(expireTime, aBeginningCount); 1434 }, 1435 handleError(aError) { 1436 log("expireStartCountFailure"); 1437 }, 1438 } 1439 ); 1440 }, 1441}; 1442 1443// Prevent add-ons from redefining this API 1444Object.freeze(FormHistory); 1445