1/* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4"use strict"; 5 6/** 7 * FormHistory 8 * 9 * Used to store values that have been entered into forms which may later 10 * be used to automatically fill in the values when the form is visited again. 11 * 12 * search(terms, queryData, callback) 13 * Look up values that have been previously stored. 14 * terms - array of terms to return data for 15 * queryData - object that contains the query terms 16 * The query object contains properties for each search criteria to match, where the value 17 * of the property specifies the value that term must have. For example, 18 * { term1: value1, term2: value2 } 19 * callback - callback that is called when results are available or an error occurs. 20 * The callback is passed a result array containing each found entry. Each element in 21 * the array is an object containing a property for each search term specified by 'terms'. 22 * count(queryData, callback) 23 * Find the number of stored entries that match the given criteria. 24 * queryData - array of objects that indicate the query. See the search method for details. 25 * callback - callback that is called when results are available or an error occurs. 26 * The callback is passed the number of found entries. 27 * update(changes, callback) 28 * Write data to form history storage. 29 * changes - an array of changes to be made. If only one change is to be made, it 30 * may be passed as an object rather than a one-element array. 31 * Each change object is of the form: 32 * { op: operation, term1: value1, term2: value2, ... } 33 * Valid operations are: 34 * add - add a new entry 35 * update - update an existing entry 36 * remove - remove an entry 37 * bump - update the last accessed time on an entry 38 * The terms specified allow matching of one or more specific entries. If no terms 39 * are specified then all entries are matched. This means that { op: "remove" } is 40 * used to remove all entries and clear the form history. 41 * callback - callback that is called when results have been stored. 42 * getAutoCompeteResults(searchString, params, callback) 43 * Retrieve an array of form history values suitable for display in an autocomplete list. 44 * Returns an mozIStoragePendingStatement that can be used to cancel the operation if 45 * needed. 46 * searchString - the string to search for, typically the entered value of a textbox 47 * params - zero or more filter arguments: 48 * fieldname - form field name 49 * agedWeight 50 * bucketSize 51 * expiryDate 52 * maxTimeGroundings 53 * timeGroupingSize 54 * prefixWeight 55 * boundaryWeight 56 * source 57 * callback - callback that is called with the array of results. Each result in the array 58 * is an object with four arguments: 59 * text, textLowerCase, frecency, totalScore 60 * schemaVersion 61 * This property holds the version of the database schema 62 * 63 * Terms: 64 * guid - entry identifier. For 'add', a guid will be generated. 65 * fieldname - form field name 66 * value - form value 67 * timesUsed - the number of times the entry has been accessed 68 * firstUsed - the time the the entry was first created 69 * lastUsed - the time the entry was last accessed 70 * firstUsedStart - search for entries created after or at this time 71 * firstUsedEnd - search for entries created before or at this time 72 * lastUsedStart - search for entries last accessed after or at this time 73 * lastUsedEnd - search for entries last accessed before or at this time 74 * newGuid - a special case valid only for 'update' and allows the guid for 75 * an existing record to be updated. The 'guid' term is the only 76 * other term which can be used (ie, you can not also specify a 77 * fieldname, value etc) and indicates the guid of the existing 78 * record that should be updated. 79 * 80 * In all of the above methods, the callback argument should be an object with 81 * handleResult(result), handleFailure(error) and handleCompletion(reason) functions. 82 * For search and getAutoCompeteResults, result is an object containing the desired 83 * properties. For count, result is the integer count. For, update, handleResult is 84 * not called. For handleCompletion, reason is either 0 if successful or 1 if 85 * an error occurred. 86 */ 87 88var EXPORTED_SYMBOLS = ["FormHistory"]; 89 90const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); 91const { AppConstants } = ChromeUtils.import( 92 "resource://gre/modules/AppConstants.jsm" 93); 94 95ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm"); 96ChromeUtils.defineModuleGetter( 97 this, 98 "Sqlite", 99 "resource://gre/modules/Sqlite.jsm" 100); 101 102const DB_SCHEMA_VERSION = 5; 103const DAY_IN_MS = 86400000; // 1 day in milliseconds 104const MAX_SEARCH_TOKENS = 10; 105const NOOP = function noop() {}; 106const DB_FILENAME = "formhistory.sqlite"; 107 108var supportsDeletedTable = AppConstants.platform == "android"; 109 110var Prefs = { 111 initialized: false, 112 113 get debug() { 114 this.ensureInitialized(); 115 return this._debug; 116 }, 117 get enabled() { 118 this.ensureInitialized(); 119 return this._enabled; 120 }, 121 get expireDays() { 122 this.ensureInitialized(); 123 return this._expireDays; 124 }, 125 126 ensureInitialized() { 127 if (this.initialized) { 128 return; 129 } 130 131 this.initialized = true; 132 133 this._debug = Services.prefs.getBoolPref("browser.formfill.debug"); 134 this._enabled = Services.prefs.getBoolPref("browser.formfill.enable"); 135 this._expireDays = Services.prefs.getIntPref( 136 "browser.formfill.expire_days" 137 ); 138 }, 139}; 140 141function log(aMessage) { 142 if (Prefs.debug) { 143 Services.console.logStringMessage("FormHistory: " + aMessage); 144 } 145} 146 147function sendNotification(aType, aData) { 148 if (typeof aData == "string") { 149 let strWrapper = Cc["@mozilla.org/supports-string;1"].createInstance( 150 Ci.nsISupportsString 151 ); 152 strWrapper.data = aData; 153 aData = strWrapper; 154 } else if (typeof aData == "number") { 155 let intWrapper = Cc["@mozilla.org/supports-PRInt64;1"].createInstance( 156 Ci.nsISupportsPRInt64 157 ); 158 intWrapper.data = aData; 159 aData = intWrapper; 160 } else if (aData) { 161 throw Components.Exception( 162 "Invalid type " + typeof aType + " passed to sendNotification", 163 Cr.NS_ERROR_ILLEGAL_VALUE 164 ); 165 } 166 167 Services.obs.notifyObservers(aData, "satchel-storage-changed", aType); 168} 169 170/** 171 * Current database schema 172 */ 173 174const dbSchema = { 175 tables: { 176 moz_formhistory: { 177 id: "INTEGER PRIMARY KEY", 178 fieldname: "TEXT NOT NULL", 179 value: "TEXT NOT NULL", 180 timesUsed: "INTEGER", 181 firstUsed: "INTEGER", 182 lastUsed: "INTEGER", 183 guid: "TEXT", 184 }, 185 moz_deleted_formhistory: { 186 id: "INTEGER PRIMARY KEY", 187 timeDeleted: "INTEGER", 188 guid: "TEXT", 189 }, 190 moz_sources: { 191 id: "INTEGER PRIMARY KEY", 192 source: "TEXT NOT NULL", 193 }, 194 moz_history_to_sources: { 195 history_id: "INTEGER", 196 source_id: "INTEGER", 197 SQL: ` 198 PRIMARY KEY (history_id, source_id), 199 FOREIGN KEY (history_id) REFERENCES moz_formhistory(id) ON DELETE CASCADE, 200 FOREIGN KEY (source_id) REFERENCES moz_sources(id) ON DELETE CASCADE 201 `, 202 }, 203 }, 204 indices: { 205 moz_formhistory_index: { 206 table: "moz_formhistory", 207 columns: ["fieldname"], 208 }, 209 moz_formhistory_lastused_index: { 210 table: "moz_formhistory", 211 columns: ["lastUsed"], 212 }, 213 moz_formhistory_guid_index: { 214 table: "moz_formhistory", 215 columns: ["guid"], 216 }, 217 }, 218}; 219 220/** 221 * Validating and processing API querying data 222 */ 223 224const validFields = [ 225 "fieldname", 226 "firstUsed", 227 "guid", 228 "lastUsed", 229 "source", 230 "timesUsed", 231 "value", 232]; 233 234const searchFilters = [ 235 "firstUsedStart", 236 "firstUsedEnd", 237 "lastUsedStart", 238 "lastUsedEnd", 239 "source", 240]; 241 242function validateOpData(aData, aDataType) { 243 let thisValidFields = validFields; 244 // A special case to update the GUID - in this case there can be a 'newGuid' 245 // field and of the normally valid fields, only 'guid' is accepted. 246 if (aDataType == "Update" && "newGuid" in aData) { 247 thisValidFields = ["guid", "newGuid"]; 248 } 249 for (let field in aData) { 250 if (field != "op" && !thisValidFields.includes(field)) { 251 throw Components.Exception( 252 aDataType + " query contains an unrecognized field: " + field, 253 Cr.NS_ERROR_ILLEGAL_VALUE 254 ); 255 } 256 } 257 return aData; 258} 259 260function validateSearchData(aData, aDataType) { 261 for (let field in aData) { 262 if ( 263 field != "op" && 264 !validFields.includes(field) && 265 !searchFilters.includes(field) 266 ) { 267 throw Components.Exception( 268 aDataType + " query contains an unrecognized field: " + field, 269 Cr.NS_ERROR_ILLEGAL_VALUE 270 ); 271 } 272 } 273} 274 275function makeQueryPredicates(aQueryData, delimiter = " AND ") { 276 let params = {}; 277 let queryTerms = Object.keys(aQueryData) 278 .filter(field => aQueryData[field] !== undefined) 279 .map(field => { 280 params[field] = aQueryData[field]; 281 switch (field) { 282 case "firstUsedStart": { 283 return "firstUsed >= :" + field; 284 } 285 case "firstUsedEnd": { 286 return "firstUsed <= :" + field; 287 } 288 case "lastUsedStart": { 289 return "lastUsed >= :" + field; 290 } 291 case "lastUsedEnd": { 292 return "lastUsed <= :" + field; 293 } 294 case "source": { 295 return `EXISTS( 296 SELECT 1 FROM moz_history_to_sources 297 JOIN moz_sources s ON s.id = source_id 298 WHERE source = :${field} 299 AND history_id = moz_formhistory.id 300 )`; 301 } 302 } 303 return field + " = :" + field; 304 }) 305 .join(delimiter); 306 return { queryTerms, params }; 307} 308 309function generateGUID() { 310 // string like: "{f60d9eac-9421-4abc-8491-8e8322b063d4}" 311 let uuid = Services.uuid.generateUUID().toString(); 312 let raw = ""; // A string with the low bytes set to random values 313 let bytes = 0; 314 for (let i = 1; bytes < 12; i += 2) { 315 // Skip dashes 316 if (uuid[i] == "-") { 317 i++; 318 } 319 let hexVal = parseInt(uuid[i] + uuid[i + 1], 16); 320 raw += String.fromCharCode(hexVal); 321 bytes++; 322 } 323 return btoa(raw); 324} 325 326var Migrators = { 327 // Bug 506402 - Adds deleted form history table. 328 async dbAsyncMigrateToVersion4(conn) { 329 const tableName = "moz_deleted_formhistory"; 330 let tableExists = await conn.tableExists(tableName); 331 if (!tableExists) { 332 await createTable(conn, tableName); 333 } 334 }, 335 336 // Bug 1654862 - Adds sources and moz_history_to_sources tables. 337 async dbAsyncMigrateToVersion5(conn) { 338 if (!(await conn.tableExists("moz_sources"))) { 339 for (let tableName of ["moz_history_to_sources", "moz_sources"]) { 340 await createTable(conn, tableName); 341 } 342 } 343 }, 344}; 345 346/** 347 * @typedef {Object} InsertQueryData 348 * @property {Object} updatedChange 349 * A change requested by FormHistory. 350 * @property {String} query 351 * The insert query string. 352 */ 353 354/** 355 * Prepares a query and some default parameters when inserting an entry 356 * to the database. 357 * 358 * @param {Object} change 359 * The change requested by FormHistory. 360 * @param {number} now 361 * The current timestamp in microseconds. 362 * @returns {InsertQueryData} 363 * The query information needed to pass along to the database. 364 */ 365function prepareInsertQuery(change, now) { 366 let params = {}; 367 for (let key of new Set([ 368 ...Object.keys(change), 369 // These must always be NOT NULL. 370 "firstUsed", 371 "lastUsed", 372 "timesUsed", 373 ])) { 374 switch (key) { 375 case "fieldname": 376 case "guid": 377 case "value": 378 params[key] = change[key]; 379 break; 380 case "firstUsed": 381 case "lastUsed": 382 params[key] = change[key] || now; 383 break; 384 case "timesUsed": 385 params[key] = change[key] || 1; 386 break; 387 default: 388 // Skip unnecessary properties. 389 } 390 } 391 392 return { 393 query: ` 394 INSERT INTO moz_formhistory 395 (fieldname, value, timesUsed, firstUsed, lastUsed, guid) 396 VALUES (:fieldname, :value, :timesUsed, :firstUsed, :lastUsed, :guid)`, 397 params, 398 }; 399} 400 401// There is a fieldname / value uniqueness constraint that's at this time 402// only enforced at this level. This Map maps fieldnames => values that 403// are in the process of being inserted into the database so that we know 404// not to try to insert the same ones on top. Attempts to do so will be 405// ignored. 406var InProgressInserts = { 407 _inProgress: new Map(), 408 409 add(fieldname, value) { 410 let fieldnameSet = this._inProgress.get(fieldname); 411 if (!fieldnameSet) { 412 this._inProgress.set(fieldname, new Set([value])); 413 return true; 414 } 415 416 if (!fieldnameSet.has(value)) { 417 fieldnameSet.add(value); 418 return true; 419 } 420 421 return false; 422 }, 423 424 clear(fieldnamesAndValues) { 425 for (let [fieldname, value] of fieldnamesAndValues) { 426 let fieldnameSet = this._inProgress.get(fieldname); 427 if ( 428 fieldnameSet && 429 fieldnameSet.delete(value) && 430 fieldnameSet.size == 0 431 ) { 432 this._inProgress.delete(fieldname); 433 } 434 } 435 }, 436}; 437 438function getAddSourceToGuidQueries(source, guid) { 439 return [ 440 { 441 query: `INSERT OR IGNORE INTO moz_sources (source) VALUES (:source)`, 442 params: { source }, 443 }, 444 { 445 query: ` 446 INSERT OR IGNORE INTO moz_history_to_sources (history_id, source_id) 447 VALUES( 448 (SELECT id FROM moz_formhistory WHERE guid = :guid), 449 (SELECT id FROM moz_sources WHERE source = :source) 450 ) 451 `, 452 params: { guid, source }, 453 }, 454 ]; 455} 456 457/** 458 * Constructs and executes database statements from a pre-processed list of 459 * inputted changes. 460 * 461 * @param {Array.<Object>} aChanges changes to form history 462 * @param {Object} aPreparedHandlers 463 */ 464// XXX This should be split up and the complexity reduced. 465// eslint-disable-next-line complexity 466async function updateFormHistoryWrite(aChanges, aPreparedHandlers) { 467 log("updateFormHistoryWrite " + aChanges.length); 468 469 // pass 'now' down so that every entry in the batch has the same timestamp 470 let now = Date.now() * 1000; 471 let queries = []; 472 let notifications = []; 473 let adds = []; 474 let conn = await FormHistory.db; 475 476 for (let change of aChanges) { 477 let operation = change.op; 478 delete change.op; 479 switch (operation) { 480 case "remove": { 481 log("Remove from form history " + change); 482 let { queryTerms, params } = makeQueryPredicates(change); 483 484 // If source is defined, we only remove the source relation, if the 485 // consumer intends to remove the value from everywhere, then they 486 // should not pass source. This gives full control to the caller. 487 if (change.source) { 488 await conn.executeCached( 489 `DELETE FROM moz_history_to_sources 490 WHERE source_id = ( 491 SELECT id FROM moz_sources WHERE source = :source 492 ) 493 AND history_id = ( 494 SELECT id FROM moz_formhistory WHERE ${queryTerms} 495 ) 496 `, 497 params 498 ); 499 break; 500 } 501 502 // Fetch the GUIDs we are going to delete. 503 try { 504 let query = "SELECT guid FROM moz_formhistory"; 505 if (queryTerms) { 506 query += " WHERE " + queryTerms; 507 } 508 509 await conn.executeCached(query, params, row => { 510 notifications.push([ 511 "formhistory-remove", 512 row.getResultByName("guid"), 513 ]); 514 }); 515 } catch (e) { 516 log("Error getting guids from moz_formhistory: " + e); 517 } 518 519 if (supportsDeletedTable) { 520 log("Moving to deleted table " + change); 521 let query = "INSERT INTO moz_deleted_formhistory (guid, timeDeleted)"; 522 523 // TODO: Add these items to the deleted items table once we've sorted 524 // out the issues from bug 756701 525 if (change.guid || queryTerms) { 526 query += change.guid 527 ? " VALUES (:guid, :timeDeleted)" 528 : " SELECT guid, :timeDeleted FROM moz_formhistory WHERE " + 529 queryTerms; 530 queries.push({ 531 query, 532 params: Object.assign({ timeDeleted: now }, params), 533 }); 534 } 535 } 536 537 let query = "DELETE FROM moz_formhistory"; 538 if (queryTerms) { 539 log("removeEntries"); 540 query += " WHERE " + queryTerms; 541 } else { 542 log("removeAllEntries"); 543 // Not specifying any fields means we should remove all entries. We 544 // won't need to modify the query in this case. 545 } 546 547 queries.push({ query, params }); 548 // Expire orphan sources. 549 queries.push({ 550 query: ` 551 DELETE FROM moz_sources WHERE id NOT IN ( 552 SELECT DISTINCT source_id FROM moz_history_to_sources 553 )`, 554 }); 555 break; 556 } 557 case "update": { 558 log("Update form history " + change); 559 let guid = change.guid; 560 delete change.guid; 561 // a special case for updating the GUID - the new value can be 562 // specified in newGuid. 563 if (change.newGuid) { 564 change.guid = change.newGuid; 565 delete change.newGuid; 566 } 567 568 let query = "UPDATE moz_formhistory SET "; 569 let { queryTerms, params } = makeQueryPredicates(change, ", "); 570 if (!queryTerms) { 571 throw Components.Exception( 572 "Update query must define fields to modify.", 573 Cr.NS_ERROR_ILLEGAL_VALUE 574 ); 575 } 576 query += queryTerms + " WHERE guid = :existing_guid"; 577 queries.push({ 578 query, 579 params: Object.assign({ existing_guid: guid }, params), 580 }); 581 582 notifications.push(["formhistory-update", guid]); 583 584 // Source is ignored for "update" operations, since it's not really 585 // common to change the source of a value, and anyway currently this is 586 // mostly used to update guids. 587 break; 588 } 589 case "bump": { 590 log("Bump form history " + change); 591 if (change.guid) { 592 let query = 593 "UPDATE moz_formhistory " + 594 "SET timesUsed = timesUsed + 1, lastUsed = :lastUsed WHERE guid = :guid"; 595 let queryParams = { 596 lastUsed: now, 597 guid: change.guid, 598 }; 599 600 queries.push({ query, params: queryParams }); 601 notifications.push(["formhistory-update", change.guid]); 602 } else { 603 if (!InProgressInserts.add(change.fieldname, change.value)) { 604 // This updateFormHistoryWrite call, or a previous one, is already 605 // going to add this fieldname / value pair, so we can ignore this. 606 continue; 607 } 608 adds.push([change.fieldname, change.value]); 609 change.guid = generateGUID(); 610 let { query, params } = prepareInsertQuery(change, now); 611 queries.push({ query, params }); 612 notifications.push(["formhistory-add", params.guid]); 613 } 614 615 if (change.source) { 616 queries = queries.concat( 617 getAddSourceToGuidQueries(change.source, change.guid) 618 ); 619 } 620 break; 621 } 622 case "add": { 623 if (!InProgressInserts.add(change.fieldname, change.value)) { 624 // This updateFormHistoryWrite call, or a previous one, is already 625 // going to add this fieldname / value pair, so we can ignore this. 626 continue; 627 } 628 adds.push([change.fieldname, change.value]); 629 630 log("Add to form history " + change); 631 if (!change.guid) { 632 change.guid = generateGUID(); 633 } 634 635 let { query, params } = prepareInsertQuery(change, now); 636 queries.push({ query, params }); 637 638 notifications.push(["formhistory-add", params.guid]); 639 640 if (change.source) { 641 queries = queries.concat( 642 getAddSourceToGuidQueries(change.source, change.guid) 643 ); 644 } 645 break; 646 } 647 default: { 648 // We should've already guaranteed that change.op is one of the above 649 throw Components.Exception( 650 "Invalid operation " + operation, 651 Cr.NS_ERROR_ILLEGAL_VALUE 652 ); 653 } 654 } 655 } 656 657 try { 658 await runUpdateQueries(conn, queries); 659 for (let [notification, param] of notifications) { 660 // We're either sending a GUID or nothing at all. 661 sendNotification(notification, param); 662 } 663 664 aPreparedHandlers.handleCompletion(0); 665 } catch (e) { 666 aPreparedHandlers.handleError(e); 667 aPreparedHandlers.handleCompletion(1); 668 } finally { 669 InProgressInserts.clear(adds); 670 } 671} 672 673/** 674 * Runs queries for an update operation to the database. This 675 * is separated out from updateFormHistoryWrite to avoid shutdown 676 * leaks where the handlers passed to updateFormHistoryWrite would 677 * leak from the closure around the executeTransaction function. 678 * 679 * @param {SqliteConnection} conn the database connection 680 * @param {Object} queries query string and param pairs generated 681 * by updateFormHistoryWrite 682 */ 683async function runUpdateQueries(conn, queries) { 684 await conn.executeTransaction(async () => { 685 for (let { query, params } of queries) { 686 await conn.executeCached(query, params); 687 } 688 }); 689} 690 691/** 692 * Functions that expire entries in form history and shrinks database 693 * afterwards as necessary initiated by expireOldEntries. 694 */ 695 696/** 697 * Removes entries from database. 698 * 699 * @param {number} aExpireTime expiration timestamp 700 * @param {number} aBeginningCount numer of entries at first 701 */ 702function expireOldEntriesDeletion(aExpireTime, aBeginningCount) { 703 log("expireOldEntriesDeletion(" + aExpireTime + "," + aBeginningCount + ")"); 704 705 FormHistory.update( 706 [ 707 { 708 op: "remove", 709 lastUsedEnd: aExpireTime, 710 }, 711 ], 712 { 713 handleCompletion() { 714 expireOldEntriesVacuum(aExpireTime, aBeginningCount); 715 }, 716 handleError(aError) { 717 log("expireOldEntriesDeletionFailure"); 718 }, 719 } 720 ); 721} 722 723/** 724 * Counts number of entries removed and shrinks database as necessary. 725 * 726 * @param {number} aExpireTime expiration timestamp 727 * @param {number} aBeginningCount number of entries at first 728 */ 729function expireOldEntriesVacuum(aExpireTime, aBeginningCount) { 730 FormHistory.count( 731 {}, 732 { 733 handleResult(aEndingCount) { 734 if (aBeginningCount - aEndingCount > 500) { 735 log("expireOldEntriesVacuum"); 736 737 FormHistory.db.then(async conn => { 738 try { 739 await conn.executeCached("VACUUM"); 740 } catch (e) { 741 log("expireVacuumError"); 742 } 743 }); 744 } 745 746 sendNotification("formhistory-expireoldentries", aExpireTime); 747 }, 748 handleError(aError) { 749 log("expireEndCountFailure"); 750 }, 751 } 752 ); 753} 754 755async function createTable(conn, tableName) { 756 let table = dbSchema.tables[tableName]; 757 let columns = Object.keys(table) 758 .filter(col => col != "SQL") 759 .map(col => [col, table[col]].join(" ")) 760 .join(", "); 761 let no_rowid = Object.keys(table).includes("id") ? "" : "WITHOUT ROWID"; 762 log("Creating table " + tableName + " with " + columns); 763 await conn.execute( 764 `CREATE TABLE ${tableName} ( 765 ${columns} 766 ${table.SQL ? "," + table.SQL : ""} 767 ) ${no_rowid}` 768 ); 769} 770 771/** 772 * Database creation and access. Used by FormHistory and some of the 773 * utility functions, but is not exposed to the outside world. 774 * @class 775 */ 776var DB = { 777 // Once we establish a database connection, we have to hold a reference 778 // to it so that it won't get GC'd. 779 _instance: null, 780 // MAX_ATTEMPTS is how many times we'll try to establish a connection 781 // or migrate a database before giving up. 782 MAX_ATTEMPTS: 2, 783 784 /** String representing where the FormHistory database is on the filesystem */ 785 get path() { 786 return OS.Path.join(OS.Constants.Path.profileDir, DB_FILENAME); 787 }, 788 789 /** 790 * Sets up and returns a connection to the FormHistory database. The 791 * connection also registers itself with AsyncShutdown so that the 792 * connection is closed on when the profile-before-change observer 793 * notification is fired. 794 * 795 * @returns {Promise} 796 * @resolves An Sqlite.jsm connection to the database. 797 * @rejects If connecting to the database, or migrating the database 798 * failed after MAX_ATTEMPTS attempts (where each attempt 799 * backs up and deletes the old database), this will reject 800 * with the Sqlite.jsm error. 801 */ 802 get conn() { 803 delete this.conn; 804 let conn = (async () => { 805 try { 806 this._instance = await this._establishConn(); 807 } catch (e) { 808 log("Failed to establish database connection: " + e); 809 throw e; 810 } 811 812 return this._instance; 813 })(); 814 815 return (this.conn = conn); 816 }, 817 818 // Private functions 819 820 /** 821 * Tries to connect to the Sqlite database at this.path, and then 822 * migrates the database as necessary. If any of the steps to do this 823 * fail, this function should re-enter itself with an incremented 824 * attemptNum so that another attempt can be made after backing up 825 * and deleting the old database. 826 * 827 * @async 828 * @param {number} attemptNum 829 * The optional number of the attempt that is being made to connect 830 * to the database. Defaults to 0. 831 * @returns {Promise} 832 * @resolves An Sqlite.jsm connection to the database. 833 * @rejects After MAX_ATTEMPTS, this will reject with the Sqlite.jsm 834 * error. 835 */ 836 async _establishConn(attemptNum = 0) { 837 log(`Establishing database connection - attempt # ${attemptNum}`); 838 let conn; 839 try { 840 conn = await Sqlite.openConnection({ path: this.path }); 841 Sqlite.shutdown.addBlocker("Closing FormHistory database.", () => 842 conn.close() 843 ); 844 } catch (e) { 845 // Bug 1423729 - We should check the reason for the connection failure, 846 // in case this is due to the disk being full or the database file being 847 // inaccessible due to third-party software (like anti-virus software). 848 // In that case, we should probably fail right away. 849 if (attemptNum < this.MAX_ATTEMPTS) { 850 log("Establishing connection failed."); 851 await this._failover(conn); 852 return this._establishConn(++attemptNum); 853 } 854 855 if (conn) { 856 await conn.close(); 857 } 858 log("Establishing connection failed too many times. Giving up."); 859 throw e; 860 } 861 862 try { 863 // Enable foreign keys support. 864 await conn.execute("PRAGMA foreign_keys = ON"); 865 866 let dbVersion = parseInt(await conn.getSchemaVersion(), 10); 867 868 // Case 1: Database is up to date and we're ready to go. 869 if (dbVersion == DB_SCHEMA_VERSION) { 870 return conn; 871 } 872 873 // Case 2: Downgrade 874 if (dbVersion > DB_SCHEMA_VERSION) { 875 log("Downgrading to version " + DB_SCHEMA_VERSION); 876 // User's DB is newer. Sanity check that our expected columns are 877 // present, and if so mark the lower version and merrily continue 878 // on. If the columns are borked, something is wrong so blow away 879 // the DB and start from scratch. [Future incompatible upgrades 880 // should switch to a different table or file.] 881 if (!(await this._expectedColumnsPresent(conn))) { 882 throw Components.Exception( 883 "DB is missing expected columns", 884 Cr.NS_ERROR_FILE_CORRUPTED 885 ); 886 } 887 888 // Change the stored version to the current version. If the user 889 // runs the newer code again, it will see the lower version number 890 // and re-upgrade (to fixup any entries the old code added). 891 await conn.setSchemaVersion(DB_SCHEMA_VERSION); 892 return conn; 893 } 894 895 // Case 3: Very old database that cannot be migrated. 896 // 897 // When FormHistory is released, we will no longer support the various 898 // schema versions prior to this release that nsIFormHistory2 once did. 899 // We'll throw an NS_ERROR_FILE_CORRUPTED, which should cause us to wipe 900 // out this DB and create a new one (unless this is our MAX_ATTEMPTS 901 // attempt). 902 if (dbVersion > 0 && dbVersion < 3) { 903 throw Components.Exception( 904 "DB version is unsupported.", 905 Cr.NS_ERROR_FILE_CORRUPTED 906 ); 907 } 908 909 if (dbVersion == 0) { 910 // Case 4: New database 911 await conn.executeTransaction(async () => { 912 log("Creating DB -- tables"); 913 for (let name in dbSchema.tables) { 914 await createTable(conn, name); 915 } 916 917 log("Creating DB -- indices"); 918 for (let name in dbSchema.indices) { 919 let index = dbSchema.indices[name]; 920 let statement = 921 "CREATE INDEX IF NOT EXISTS " + 922 name + 923 " ON " + 924 index.table + 925 "(" + 926 index.columns.join(", ") + 927 ")"; 928 await conn.execute(statement); 929 } 930 }); 931 } else { 932 // Case 5: Old database requiring a migration 933 await conn.executeTransaction(async () => { 934 for (let v = dbVersion + 1; v <= DB_SCHEMA_VERSION; v++) { 935 log("Upgrading to version " + v + "..."); 936 await Migrators["dbAsyncMigrateToVersion" + v](conn); 937 } 938 }); 939 } 940 941 await conn.setSchemaVersion(DB_SCHEMA_VERSION); 942 943 return conn; 944 } catch (e) { 945 if (e.result != Cr.NS_ERROR_FILE_CORRUPTED) { 946 throw e; 947 } 948 949 if (attemptNum < this.MAX_ATTEMPTS) { 950 log("Setting up database failed."); 951 await this._failover(conn); 952 return this._establishConn(++attemptNum); 953 } 954 955 if (conn) { 956 await conn.close(); 957 } 958 959 log("Setting up database failed too many times. Giving up."); 960 961 throw e; 962 } 963 }, 964 965 /** 966 * Closes a connection to the database, then backs up the database before 967 * deleting it. 968 * 969 * @async 970 * @param {SqliteConnection | null} conn 971 * The connection to the database that we failed to establish or 972 * migrate. 973 * @throws If any file operations fail. 974 */ 975 async _failover(conn) { 976 log("Cleaning up DB file - close & remove & backup."); 977 if (conn) { 978 await conn.close(); 979 } 980 let backupFile = this.path + ".corrupt"; 981 let { file, path: uniquePath } = await OS.File.openUnique(backupFile, { 982 humanReadable: true, 983 }); 984 await file.close(); 985 await OS.File.copy(this.path, uniquePath); 986 await OS.File.remove(this.path); 987 log("Completed DB cleanup."); 988 }, 989 990 /** 991 * Tests that a database connection contains the tables that we expect. 992 * 993 * @async 994 * @param {SqliteConnection | null} conn 995 * The connection to the database that we're testing. 996 * @returns {Promise} 997 * @resolves true if all expected columns are present. 998 */ 999 async _expectedColumnsPresent(conn) { 1000 for (let name in dbSchema.tables) { 1001 let table = dbSchema.tables[name]; 1002 let columns = Object.keys(table).filter(col => col != "SQL"); 1003 let query = "SELECT " + columns.join(", ") + " FROM " + name; 1004 try { 1005 await conn.execute(query, null, (row, cancel) => { 1006 // One row is enough to let us know this worked. 1007 cancel(); 1008 }); 1009 } catch (e) { 1010 return false; 1011 } 1012 } 1013 1014 log("Verified that expected columns are present in DB."); 1015 return true; 1016 }, 1017}; 1018 1019this.FormHistory = { 1020 get db() { 1021 return DB.conn; 1022 }, 1023 1024 get enabled() { 1025 return Prefs.enabled; 1026 }, 1027 1028 _prepareHandlers(handlers) { 1029 let defaultHandlers = { 1030 handleResult: NOOP, 1031 handleError: NOOP, 1032 handleCompletion: NOOP, 1033 }; 1034 1035 if (!handlers) { 1036 return defaultHandlers; 1037 } 1038 1039 if (handlers.handleResult) { 1040 defaultHandlers.handleResult = handlers.handleResult; 1041 } 1042 if (handlers.handleError) { 1043 defaultHandlers.handleError = handlers.handleError; 1044 } 1045 if (handlers.handleCompletion) { 1046 defaultHandlers.handleCompletion = handlers.handleCompletion; 1047 } 1048 1049 return defaultHandlers; 1050 }, 1051 1052 search(aSelectTerms, aSearchData, aRowFuncOrHandlers) { 1053 // if no terms selected, select everything 1054 if (!aSelectTerms) { 1055 // Source is not a valid column in moz_formhistory. 1056 aSelectTerms = validFields.filter(f => f != "source"); 1057 } 1058 1059 validateSearchData(aSearchData, "Search"); 1060 1061 let query = "SELECT " + aSelectTerms.join(", ") + " FROM moz_formhistory"; 1062 let { queryTerms, params } = makeQueryPredicates(aSearchData); 1063 if (queryTerms) { 1064 query += " WHERE " + queryTerms; 1065 } 1066 1067 let handlers; 1068 1069 if (typeof aRowFuncOrHandlers == "function") { 1070 handlers = this._prepareHandlers(); 1071 handlers.handleResult = aRowFuncOrHandlers; 1072 } else if (typeof aRowFuncOrHandlers == "object") { 1073 handlers = this._prepareHandlers(aRowFuncOrHandlers); 1074 } 1075 1076 let allResults = []; 1077 1078 return new Promise((resolve, reject) => { 1079 this.db.then(async conn => { 1080 try { 1081 await conn.executeCached(query, params, row => { 1082 let result = {}; 1083 for (let field of aSelectTerms) { 1084 result[field] = row.getResultByName(field); 1085 } 1086 1087 if (handlers) { 1088 handlers.handleResult(result); 1089 } else { 1090 allResults.push(result); 1091 } 1092 }); 1093 if (handlers) { 1094 handlers.handleCompletion(0); 1095 } 1096 resolve(allResults); 1097 } catch (e) { 1098 if (handlers) { 1099 handlers.handleError(e); 1100 handlers.handleCompletion(1); 1101 } 1102 reject(e); 1103 } 1104 }); 1105 }); 1106 }, 1107 1108 count(aSearchData, aHandlers) { 1109 validateSearchData(aSearchData, "Count"); 1110 1111 let query = "SELECT COUNT(*) AS numEntries FROM moz_formhistory"; 1112 let { queryTerms, params } = makeQueryPredicates(aSearchData); 1113 if (queryTerms) { 1114 query += " WHERE " + queryTerms; 1115 } 1116 1117 let handlers = this._prepareHandlers(aHandlers); 1118 1119 return new Promise((resolve, reject) => { 1120 this.db.then(async conn => { 1121 try { 1122 let rows = await conn.executeCached(query, params); 1123 let count = rows[0].getResultByName("numEntries"); 1124 handlers.handleResult(count); 1125 handlers.handleCompletion(0); 1126 resolve(count); 1127 } catch (e) { 1128 handlers.handleError(e); 1129 handlers.handleCompletion(1); 1130 reject(e); 1131 } 1132 }); 1133 }); 1134 }, 1135 1136 update(aChanges, aHandlers) { 1137 // Used to keep track of how many searches have been started. When that number 1138 // are finished, updateFormHistoryWrite can be called. 1139 let numSearches = 0; 1140 let completedSearches = 0; 1141 let searchFailed = false; 1142 1143 function validIdentifier(change) { 1144 // The identifier is only valid if one of either the guid 1145 // or the (fieldname/value) are set (so an X-OR) 1146 return Boolean(change.guid) != Boolean(change.fieldname && change.value); 1147 } 1148 1149 if (!("length" in aChanges)) { 1150 aChanges = [aChanges]; 1151 } 1152 1153 let handlers = this._prepareHandlers(aHandlers); 1154 1155 let isRemoveOperation = aChanges.every( 1156 change => change && change.op && change.op == "remove" 1157 ); 1158 if (!Prefs.enabled && !isRemoveOperation) { 1159 handlers.handleError({ 1160 message: "Form history is disabled, only remove operations are allowed", 1161 result: Ci.mozIStorageError.MISUSE, 1162 }); 1163 handlers.handleCompletion(1); 1164 return; 1165 } 1166 1167 for (let change of aChanges) { 1168 switch (change.op) { 1169 case "remove": 1170 validateSearchData(change, "Remove"); 1171 continue; 1172 case "update": 1173 if (validIdentifier(change)) { 1174 validateOpData(change, "Update"); 1175 if (change.guid) { 1176 continue; 1177 } 1178 } else { 1179 throw Components.Exception( 1180 "update op='update' does not correctly reference a entry.", 1181 Cr.NS_ERROR_ILLEGAL_VALUE 1182 ); 1183 } 1184 break; 1185 case "bump": 1186 if (validIdentifier(change)) { 1187 validateOpData(change, "Bump"); 1188 if (change.guid) { 1189 continue; 1190 } 1191 } else { 1192 throw Components.Exception( 1193 "update op='bump' does not correctly reference a entry.", 1194 Cr.NS_ERROR_ILLEGAL_VALUE 1195 ); 1196 } 1197 break; 1198 case "add": 1199 if (change.fieldname && change.value) { 1200 validateOpData(change, "Add"); 1201 } else { 1202 throw Components.Exception( 1203 "update op='add' must have a fieldname and a value.", 1204 Cr.NS_ERROR_ILLEGAL_VALUE 1205 ); 1206 } 1207 break; 1208 default: 1209 throw Components.Exception( 1210 "update does not recognize op='" + change.op + "'", 1211 Cr.NS_ERROR_ILLEGAL_VALUE 1212 ); 1213 } 1214 1215 numSearches++; 1216 let changeToUpdate = change; 1217 FormHistory.search( 1218 ["guid"], 1219 { 1220 fieldname: change.fieldname, 1221 value: change.value, 1222 }, 1223 { 1224 foundResult: false, 1225 handleResult(aResult) { 1226 if (this.foundResult) { 1227 log( 1228 "Database contains multiple entries with the same fieldname/value pair." 1229 ); 1230 handlers.handleError({ 1231 message: 1232 "Database contains multiple entries with the same fieldname/value pair.", 1233 result: 19, // Constraint violation 1234 }); 1235 1236 searchFailed = true; 1237 return; 1238 } 1239 1240 this.foundResult = true; 1241 changeToUpdate.guid = aResult.guid; 1242 }, 1243 1244 handleError(aError) { 1245 handlers.handleError(aError); 1246 }, 1247 1248 handleCompletion(aReason) { 1249 completedSearches++; 1250 if (completedSearches == numSearches) { 1251 if (!aReason && !searchFailed) { 1252 updateFormHistoryWrite(aChanges, handlers); 1253 } else { 1254 handlers.handleCompletion(1); 1255 } 1256 } 1257 }, 1258 } 1259 ); 1260 } 1261 1262 if (numSearches == 0) { 1263 // We don't have to wait for any statements to return. 1264 updateFormHistoryWrite(aChanges, handlers); 1265 } 1266 }, 1267 1268 getAutoCompleteResults(searchString, params, aHandlers) { 1269 // only do substring matching when the search string contains more than one character 1270 let searchTokens; 1271 let where = ""; 1272 let boundaryCalc = ""; 1273 1274 if (searchString.length >= 1) { 1275 params.valuePrefix = searchString + "%"; 1276 } 1277 1278 if (searchString.length > 1) { 1279 searchTokens = searchString.split(/\s+/); 1280 1281 // build up the word boundary and prefix match bonus calculation 1282 boundaryCalc = 1283 "MAX(1, :prefixWeight * (value LIKE :valuePrefix ESCAPE '/') + ("; 1284 // for each word, calculate word boundary weights for the SELECT clause and 1285 // add word to the WHERE clause of the query 1286 let tokenCalc = []; 1287 let searchTokenCount = Math.min(searchTokens.length, MAX_SEARCH_TOKENS); 1288 for (let i = 0; i < searchTokenCount; i++) { 1289 let escapedToken = searchTokens[i]; 1290 params["tokenBegin" + i] = escapedToken + "%"; 1291 params["tokenBoundary" + i] = "% " + escapedToken + "%"; 1292 params["tokenContains" + i] = "%" + escapedToken + "%"; 1293 1294 tokenCalc.push( 1295 "(value LIKE :tokenBegin" + 1296 i + 1297 " ESCAPE '/') + " + 1298 "(value LIKE :tokenBoundary" + 1299 i + 1300 " ESCAPE '/')" 1301 ); 1302 where += "AND (value LIKE :tokenContains" + i + " ESCAPE '/') "; 1303 } 1304 // add more weight if we have a traditional prefix match and 1305 // multiply boundary bonuses by boundary weight 1306 boundaryCalc += tokenCalc.join(" + ") + ") * :boundaryWeight)"; 1307 } else if (searchString.length == 1) { 1308 where = "AND (value LIKE :valuePrefix ESCAPE '/') "; 1309 boundaryCalc = "1"; 1310 delete params.prefixWeight; 1311 delete params.boundaryWeight; 1312 } else { 1313 where = ""; 1314 boundaryCalc = "1"; 1315 delete params.prefixWeight; 1316 delete params.boundaryWeight; 1317 } 1318 1319 params.now = Date.now() * 1000; // convert from ms to microseconds 1320 1321 if (params.source) { 1322 where += `AND EXISTS( 1323 SELECT 1 FROM moz_history_to_sources 1324 JOIN moz_sources s ON s.id = source_id 1325 WHERE source = :source 1326 AND history_id = moz_formhistory.id 1327 )`; 1328 } 1329 1330 let handlers = this._prepareHandlers(aHandlers); 1331 1332 /* Three factors in the frecency calculation for an entry (in order of use in calculation): 1333 * 1) average number of times used - items used more are ranked higher 1334 * 2) how recently it was last used - items used recently are ranked higher 1335 * 3) additional weight for aged entries surviving expiry - these entries are relevant 1336 * since they have been used multiple times over a large time span so rank them higher 1337 * The score is then divided by the bucket size and we round the result so that entries 1338 * with a very similar frecency are bucketed together with an alphabetical sort. This is 1339 * to reduce the amount of moving around by entries while typing. 1340 */ 1341 1342 let query = 1343 "/* do not warn (bug 496471): can't use an index */ " + 1344 "SELECT value, guid, " + 1345 "ROUND( " + 1346 "timesUsed / MAX(1.0, (lastUsed - firstUsed) / :timeGroupingSize) * " + 1347 "MAX(1.0, :maxTimeGroupings - (:now - lastUsed) / :timeGroupingSize) * " + 1348 "MAX(1.0, :agedWeight * (firstUsed < :expiryDate)) / " + 1349 ":bucketSize " + 1350 ", 3) AS frecency, " + 1351 boundaryCalc + 1352 " AS boundaryBonuses " + 1353 "FROM moz_formhistory " + 1354 "WHERE fieldname=:fieldname " + 1355 where + 1356 "ORDER BY ROUND(frecency * boundaryBonuses) DESC, UPPER(value) ASC"; 1357 1358 let cancelled = false; 1359 1360 let cancellableQuery = { 1361 cancel() { 1362 cancelled = true; 1363 }, 1364 }; 1365 1366 this.db.then(async conn => { 1367 try { 1368 await conn.executeCached(query, params, (row, cancel) => { 1369 if (cancelled) { 1370 cancel(); 1371 return; 1372 } 1373 1374 let value = row.getResultByName("value"); 1375 let guid = row.getResultByName("guid"); 1376 let frecency = row.getResultByName("frecency"); 1377 let entry = { 1378 text: value, 1379 guid, 1380 textLowerCase: value.toLowerCase(), 1381 frecency, 1382 totalScore: Math.round( 1383 frecency * row.getResultByName("boundaryBonuses") 1384 ), 1385 }; 1386 handlers.handleResult(entry); 1387 }); 1388 handlers.handleCompletion(0); 1389 } catch (e) { 1390 handlers.handleError(e); 1391 handlers.handleCompletion(1); 1392 } 1393 }); 1394 1395 return cancellableQuery; 1396 }, 1397 1398 // This is used only so that the test can verify deleted table support. 1399 get _supportsDeletedTable() { 1400 return supportsDeletedTable; 1401 }, 1402 set _supportsDeletedTable(val) { 1403 supportsDeletedTable = val; 1404 }, 1405 1406 // The remaining methods are called by FormHistoryStartup.js 1407 updatePrefs() { 1408 Prefs.initialized = false; 1409 }, 1410 1411 expireOldEntries() { 1412 log("expireOldEntries"); 1413 1414 // Determine how many days of history we're supposed to keep. 1415 // Calculate expireTime in microseconds 1416 let expireTime = (Date.now() - Prefs.expireDays * DAY_IN_MS) * 1000; 1417 1418 sendNotification("formhistory-beforeexpireoldentries", expireTime); 1419 1420 FormHistory.count( 1421 {}, 1422 { 1423 handleResult(aBeginningCount) { 1424 expireOldEntriesDeletion(expireTime, aBeginningCount); 1425 }, 1426 handleError(aError) { 1427 log("expireStartCountFailure"); 1428 }, 1429 } 1430 ); 1431 }, 1432}; 1433 1434// Prevent add-ons from redefining this API 1435Object.freeze(FormHistory); 1436