1/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- 2 * vim: sw=2 ts=2 sts=2 expandtab filetype=javascript 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7const BYTES_PER_MEBIBYTE = 1048576; 8const MS_PER_DAY = 86400000; 9// Threshold value for removeOldCorruptDBs. 10// Corrupt DBs older than this value are removed. 11const CORRUPT_DB_RETAIN_DAYS = 14; 12 13// Seconds between maintenance runs. 14const MAINTENANCE_INTERVAL_SECONDS = 7 * 86400; 15 16const { ComponentUtils } = ChromeUtils.import( 17 "resource://gre/modules/ComponentUtils.jsm" 18); 19const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); 20const { XPCOMUtils } = ChromeUtils.import( 21 "resource://gre/modules/XPCOMUtils.jsm" 22); 23 24XPCOMUtils.defineLazyModuleGetters(this, { 25 OS: "resource://gre/modules/osfile.jsm", 26 PlacesUtils: "resource://gre/modules/PlacesUtils.jsm", 27 Sqlite: "resource://gre/modules/Sqlite.jsm", 28}); 29 30var EXPORTED_SYMBOLS = ["PlacesDBUtils", "PlacesDBUtilsIdleMaintenance"]; 31 32var PlacesDBUtils = { 33 _isShuttingDown: false, 34 35 _clearTaskQueue: false, 36 clearPendingTasks() { 37 PlacesDBUtils._clearTaskQueue = true; 38 }, 39 40 /** 41 * Executes integrity check and common maintenance tasks. 42 * 43 * @return a Map[taskName(String) -> Object]. The Object has the following properties: 44 * - succeeded: boolean 45 * - logs: an array of strings containing the messages logged by the task. 46 */ 47 async maintenanceOnIdle() { 48 let tasks = [ 49 this.checkIntegrity, 50 this.invalidateCaches, 51 this.checkCoherence, 52 this._refreshUI, 53 this.originFrecencyStats, 54 this.incrementalVacuum, 55 this.removeOldCorruptDBs, 56 ]; 57 let telemetryStartTime = Date.now(); 58 let taskStatusMap = await PlacesDBUtils.runTasks(tasks); 59 60 Services.prefs.setIntPref( 61 "places.database.lastMaintenance", 62 parseInt(Date.now() / 1000) 63 ); 64 Services.telemetry 65 .getHistogramById("PLACES_IDLE_MAINTENANCE_TIME_MS") 66 .add(Date.now() - telemetryStartTime); 67 return taskStatusMap; 68 }, 69 70 /** 71 * Executes integrity check, common and advanced maintenance tasks (like 72 * expiration and vacuum). Will also collect statistics on the database. 73 * 74 * Note: although this function isn't actually async, we keep it async to 75 * allow us to maintain a simple, consistent API for the tasks within this object. 76 * 77 * @return {Promise} 78 * A promise that resolves with a Map[taskName(String) -> Object]. 79 * The Object has the following properties: 80 * - succeeded: boolean 81 * - logs: an array of strings containing the messages logged by the task. 82 */ 83 async checkAndFixDatabase() { 84 let tasks = [ 85 this.checkIntegrity, 86 this.invalidateCaches, 87 this.checkCoherence, 88 this.expire, 89 this.originFrecencyStats, 90 this.vacuum, 91 this.stats, 92 this._refreshUI, 93 ]; 94 return PlacesDBUtils.runTasks(tasks); 95 }, 96 97 /** 98 * Forces a full refresh of Places views. 99 * 100 * Note: although this function isn't actually async, we keep it async to 101 * allow us to maintain a simple, consistent API for the tasks within this object. 102 * 103 * @returns {Array} An empty array. 104 */ 105 async _refreshUI() { 106 PlacesObservers.notifyListeners([new PlacesPurgeCaches()]); 107 return []; 108 }, 109 110 /** 111 * Checks integrity and tries to fix the database through a reindex. 112 * 113 * @return {Promise} resolves if database is sane or is made sane. 114 * @resolves to an array of logs for this task. 115 * @rejects if we're unable to fix corruption or unable to check status. 116 */ 117 async checkIntegrity() { 118 let logs = []; 119 120 async function check(dbName) { 121 try { 122 await integrity(dbName); 123 logs.push(`The ${dbName} database is sane`); 124 } catch (ex) { 125 PlacesDBUtils.clearPendingTasks(); 126 if (ex.result == Cr.NS_ERROR_FILE_CORRUPTED) { 127 logs.push(`The ${dbName} database is corrupt`); 128 Services.prefs.setCharPref( 129 "places.database.replaceDatabaseOnStartup", 130 dbName 131 ); 132 throw new Error( 133 `Unable to fix corruption, ${dbName} will be replaced on next startup` 134 ); 135 } 136 throw new Error(`Unable to check ${dbName} integrity: ${ex}`); 137 } 138 } 139 140 await check("places.sqlite"); 141 await check("favicons.sqlite"); 142 143 return logs; 144 }, 145 146 invalidateCaches() { 147 let logs = []; 148 return PlacesUtils.withConnectionWrapper( 149 "PlacesDBUtils: invalidate caches", 150 async db => { 151 let idsWithStaleGuidsRows = await db.execute( 152 `SELECT id FROM moz_bookmarks 153 WHERE guid IS NULL OR 154 NOT IS_VALID_GUID(guid) OR 155 (type = :bookmark_type AND fk IS NULL) OR 156 (type <> :bookmark_type AND fk NOT NULL) OR 157 type IS NULL`, 158 { bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK } 159 ); 160 for (let row of idsWithStaleGuidsRows) { 161 let id = row.getResultByName("id"); 162 PlacesUtils.invalidateCachedGuidFor(id); 163 } 164 logs.push("The caches have been invalidated"); 165 return logs; 166 } 167 ).catch(ex => { 168 PlacesDBUtils.clearPendingTasks(); 169 throw new Error("Unable to invalidate caches"); 170 }); 171 }, 172 173 /** 174 * Checks data coherence and tries to fix most common errors. 175 * 176 * @return {Promise} resolves when coherence is checked. 177 * @resolves to an array of logs for this task. 178 * @rejects if database is not coherent. 179 */ 180 async checkCoherence() { 181 let logs = []; 182 let stmts = await PlacesDBUtils._getCoherenceStatements(); 183 let coherenceCheck = true; 184 await PlacesUtils.withConnectionWrapper( 185 "PlacesDBUtils: coherence check:", 186 db => 187 db.executeTransaction(async () => { 188 for (let { query, params } of stmts) { 189 try { 190 await db.execute(query, params || null); 191 } catch (ex) { 192 Cu.reportError(ex); 193 coherenceCheck = false; 194 } 195 } 196 }) 197 ); 198 199 if (coherenceCheck) { 200 logs.push("The database is coherent"); 201 } else { 202 PlacesDBUtils.clearPendingTasks(); 203 throw new Error("Unable to complete the coherence check"); 204 } 205 return logs; 206 }, 207 208 /** 209 * Runs incremental vacuum on databases supporting it. 210 * 211 * @return {Promise} resolves when done. 212 * @resolves to an array of logs for this task. 213 * @rejects if we were unable to vacuum. 214 */ 215 async incrementalVacuum() { 216 let logs = []; 217 return PlacesUtils.withConnectionWrapper( 218 "PlacesDBUtils: incrementalVacuum", 219 async db => { 220 let count = ( 221 await db.execute("PRAGMA favicons.freelist_count") 222 )[0].getResultByIndex(0); 223 if (count < 10) { 224 logs.push( 225 `The favicons database has only ${count} free pages, not vacuuming.` 226 ); 227 } else { 228 logs.push( 229 `The favicons database has ${count} free pages, vacuuming.` 230 ); 231 await db.execute("PRAGMA favicons.incremental_vacuum"); 232 count = ( 233 await db.execute("PRAGMA favicons.freelist_count") 234 )[0].getResultByIndex(0); 235 logs.push( 236 `The database has been vacuumed and has now ${count} free pages.` 237 ); 238 } 239 return logs; 240 } 241 ).catch(ex => { 242 PlacesDBUtils.clearPendingTasks(); 243 throw new Error( 244 "Unable to incrementally vacuum the favicons database " + ex 245 ); 246 }); 247 }, 248 249 async _getCoherenceStatements() { 250 let cleanupStatements = [ 251 // MOZ_PLACES 252 // L.1 remove duplicate URLs. 253 // This task uses a temp table of potential dupes, and a trigger to remove 254 // them. It runs first because it relies on subsequent tasks to clean up 255 // orphaned foreign key references. The task works like this: first, we 256 // insert all rows with the same hash into the temp table. This lets 257 // SQLite use the `url_hash` index for scanning `moz_places`. Hashes 258 // aren't unique, so two different URLs might have the same hash. To find 259 // the actual dupes, we use a unique constraint on the URL in the temp 260 // table. If that fails, we bump the dupe count. Then, we delete all dupes 261 // from the table. This fires the cleanup trigger, which updates all 262 // foreign key references to point to one of the duplicate Places, then 263 // deletes the others. 264 { 265 query: `CREATE TEMP TABLE IF NOT EXISTS moz_places_dupes_temp( 266 id INTEGER PRIMARY KEY 267 , hash INTEGER NOT NULL 268 , url TEXT UNIQUE NOT NULL 269 , count INTEGER NOT NULL DEFAULT 0 270 )`, 271 }, 272 { 273 query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_places_remove_dupes_temp_trigger 274 AFTER DELETE ON moz_places_dupes_temp 275 FOR EACH ROW 276 BEGIN 277 /* Reassign history visits. */ 278 UPDATE moz_historyvisits SET 279 place_id = OLD.id 280 WHERE place_id IN (SELECT id FROM moz_places 281 WHERE id <> OLD.id AND 282 url_hash = OLD.hash AND 283 url = OLD.url); 284 285 /* Merge autocomplete history entries. */ 286 INSERT INTO moz_inputhistory(place_id, input, use_count) 287 SELECT OLD.id, a.input, a.use_count 288 FROM moz_inputhistory a 289 JOIN moz_places h ON h.id = a.place_id 290 WHERE h.id <> OLD.id AND 291 h.url_hash = OLD.hash AND 292 h.url = OLD.url 293 ON CONFLICT(place_id, input) DO UPDATE SET 294 place_id = excluded.place_id, 295 use_count = use_count + excluded.use_count; 296 297 /* Merge page annos, ignoring annos with the same name that are 298 already set on the destination. */ 299 INSERT OR IGNORE INTO moz_annos(id, place_id, anno_attribute_id, 300 content, flags, expiration, type, 301 dateAdded, lastModified) 302 SELECT (SELECT k.id FROM moz_annos k 303 WHERE k.place_id = OLD.id AND 304 k.anno_attribute_id = a.anno_attribute_id), OLD.id, 305 a.anno_attribute_id, a.content, a.flags, a.expiration, a.type, 306 a.dateAdded, a.lastModified 307 FROM moz_annos a 308 JOIN moz_places h ON h.id = a.place_id 309 WHERE h.id <> OLD.id AND 310 url_hash = OLD.hash AND 311 url = OLD.url; 312 313 /* Reassign bookmarks, and bump the Sync change counter just in case 314 we have new keywords. */ 315 UPDATE moz_bookmarks SET 316 fk = OLD.id, 317 syncChangeCounter = syncChangeCounter + 1 318 WHERE fk IN (SELECT id FROM moz_places 319 WHERE url_hash = OLD.hash AND 320 url = OLD.url); 321 322 /* Reassign keywords. */ 323 UPDATE moz_keywords SET 324 place_id = OLD.id 325 WHERE place_id IN (SELECT id FROM moz_places 326 WHERE id <> OLD.id AND 327 url_hash = OLD.hash AND 328 url = OLD.url); 329 330 /* Now that we've updated foreign key references, drop the 331 conflicting source. */ 332 DELETE FROM moz_places 333 WHERE id <> OLD.id AND 334 url_hash = OLD.hash AND 335 url = OLD.url; 336 337 /* Recalculate frecency for the destination. */ 338 UPDATE moz_places SET 339 frecency = calculate_frecency(id) 340 WHERE id = OLD.id; 341 342 /* Trigger frecency updates for affected origins. */ 343 DELETE FROM moz_updateoriginsupdate_temp; 344 END`, 345 }, 346 { 347 query: `INSERT INTO moz_places_dupes_temp(id, hash, url, count) 348 SELECT h.id, h.url_hash, h.url, 1 349 FROM moz_places h 350 JOIN (SELECT url_hash FROM moz_places 351 GROUP BY url_hash 352 HAVING count(*) > 1) d ON d.url_hash = h.url_hash 353 ON CONFLICT(url) DO UPDATE SET 354 count = count + 1`, 355 }, 356 { query: `DELETE FROM moz_places_dupes_temp WHERE count > 1` }, 357 { query: `DROP TABLE moz_places_dupes_temp` }, 358 359 // MOZ_ANNO_ATTRIBUTES 360 // A.1 remove obsolete annotations from moz_annos. 361 // The 'weave0' idiom exploits character ordering (0 follows /) to 362 // efficiently select all annos with a 'weave/' prefix. 363 { 364 query: `DELETE FROM moz_annos 365 WHERE type = 4 OR anno_attribute_id IN ( 366 SELECT id FROM moz_anno_attributes 367 WHERE name = 'downloads/destinationFileName' OR 368 name BETWEEN 'weave/' AND 'weave0' 369 )`, 370 }, 371 372 // A.2 remove obsolete annotations from moz_items_annos. 373 { 374 query: `DELETE FROM moz_items_annos 375 WHERE type = 4 OR anno_attribute_id IN ( 376 SELECT id FROM moz_anno_attributes 377 WHERE name = 'sync/children' 378 OR name = 'placesInternal/GUID' 379 OR name BETWEEN 'weave/' AND 'weave0' 380 )`, 381 }, 382 383 // A.3 remove unused attributes. 384 { 385 query: `DELETE FROM moz_anno_attributes WHERE id IN ( 386 SELECT id FROM moz_anno_attributes n 387 WHERE NOT EXISTS 388 (SELECT id FROM moz_annos WHERE anno_attribute_id = n.id LIMIT 1) 389 AND NOT EXISTS 390 (SELECT id FROM moz_items_annos WHERE anno_attribute_id = n.id LIMIT 1) 391 )`, 392 }, 393 394 // MOZ_ANNOS 395 // B.1 remove annos with an invalid attribute 396 { 397 query: `DELETE FROM moz_annos WHERE id IN ( 398 SELECT id FROM moz_annos a 399 WHERE NOT EXISTS 400 (SELECT id FROM moz_anno_attributes 401 WHERE id = a.anno_attribute_id LIMIT 1) 402 )`, 403 }, 404 405 // B.2 remove orphan annos 406 { 407 query: `DELETE FROM moz_annos WHERE id IN ( 408 SELECT id FROM moz_annos a 409 WHERE NOT EXISTS 410 (SELECT id FROM moz_places WHERE id = a.place_id LIMIT 1) 411 )`, 412 }, 413 414 // D.1 remove items that are not uri bookmarks from tag containers 415 { 416 query: `DELETE FROM moz_bookmarks WHERE guid NOT IN ( 417 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 418 ) AND id IN ( 419 SELECT b.id FROM moz_bookmarks b 420 WHERE b.parent IN 421 (SELECT id FROM moz_bookmarks WHERE parent = :tags_folder) 422 AND b.type <> :bookmark_type 423 )`, 424 params: { 425 tags_folder: PlacesUtils.tagsFolderId, 426 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 427 rootGuid: PlacesUtils.bookmarks.rootGuid, 428 menuGuid: PlacesUtils.bookmarks.menuGuid, 429 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 430 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 431 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 432 }, 433 }, 434 435 // D.2 remove empty tags 436 { 437 query: `DELETE FROM moz_bookmarks WHERE guid NOT IN ( 438 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 439 ) AND id IN ( 440 SELECT b.id FROM moz_bookmarks b 441 WHERE b.id IN 442 (SELECT id FROM moz_bookmarks WHERE parent = :tags_folder) 443 AND NOT EXISTS 444 (SELECT id from moz_bookmarks WHERE parent = b.id LIMIT 1) 445 )`, 446 params: { 447 tags_folder: PlacesUtils.tagsFolderId, 448 rootGuid: PlacesUtils.bookmarks.rootGuid, 449 menuGuid: PlacesUtils.bookmarks.menuGuid, 450 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 451 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 452 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 453 }, 454 }, 455 456 // D.3 move orphan items to unsorted folder 457 { 458 query: `UPDATE moz_bookmarks SET 459 parent = (SELECT id FROM moz_bookmarks WHERE guid = :unfiledGuid) 460 WHERE guid NOT IN ( 461 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 462 ) AND id IN ( 463 SELECT b.id FROM moz_bookmarks b 464 WHERE NOT EXISTS 465 (SELECT id FROM moz_bookmarks WHERE id = b.parent LIMIT 1) 466 )`, 467 params: { 468 rootGuid: PlacesUtils.bookmarks.rootGuid, 469 menuGuid: PlacesUtils.bookmarks.menuGuid, 470 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 471 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 472 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 473 }, 474 }, 475 476 // D.4 Insert tombstones for any synced items with the wrong type. 477 // Sync doesn't support changing the type of an existing item while 478 // keeping its GUID. To avoid confusing other clients, we insert 479 // tombstones for all synced items with the wrong type, so that we 480 // can reupload them with the correct type and a new GUID. 481 { 482 query: `INSERT OR IGNORE INTO moz_bookmarks_deleted(guid, dateRemoved) 483 SELECT guid, :dateRemoved 484 FROM moz_bookmarks 485 WHERE syncStatus <> :syncStatus AND 486 ((type IN (:folder_type, :separator_type) AND 487 fk NOTNULL) OR 488 (type = :bookmark_type AND 489 fk IS NULL) OR 490 type IS NULL)`, 491 params: { 492 dateRemoved: PlacesUtils.toPRTime(new Date()), 493 syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW, 494 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 495 folder_type: PlacesUtils.bookmarks.TYPE_FOLDER, 496 separator_type: PlacesUtils.bookmarks.TYPE_SEPARATOR, 497 }, 498 }, 499 500 // D.5 fix wrong item types 501 // Folders and separators should not have an fk. 502 // If they have a valid fk, convert them to bookmarks, and give them new 503 // GUIDs. If the item has children, we'll move them to the unfiled root 504 // in D.8. If the `fk` doesn't exist in `moz_places`, we'll remove the 505 // item in D.9. 506 { 507 query: `UPDATE moz_bookmarks 508 SET guid = GENERATE_GUID(), 509 type = :bookmark_type 510 WHERE guid NOT IN ( 511 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 512 ) AND id IN ( 513 SELECT id FROM moz_bookmarks b 514 WHERE type IN (:folder_type, :separator_type) 515 AND fk NOTNULL 516 )`, 517 params: { 518 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 519 folder_type: PlacesUtils.bookmarks.TYPE_FOLDER, 520 separator_type: PlacesUtils.bookmarks.TYPE_SEPARATOR, 521 rootGuid: PlacesUtils.bookmarks.rootGuid, 522 menuGuid: PlacesUtils.bookmarks.menuGuid, 523 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 524 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 525 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 526 }, 527 }, 528 529 // D.6 fix wrong item types 530 // Bookmarks should have an fk, if they don't have any, convert them to 531 // folders. 532 { 533 query: `UPDATE moz_bookmarks 534 SET guid = GENERATE_GUID(), 535 type = :folder_type 536 WHERE guid NOT IN ( 537 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 538 ) AND id IN ( 539 SELECT id FROM moz_bookmarks b 540 WHERE type = :bookmark_type 541 AND fk IS NULL 542 )`, 543 params: { 544 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 545 folder_type: PlacesUtils.bookmarks.TYPE_FOLDER, 546 rootGuid: PlacesUtils.bookmarks.rootGuid, 547 menuGuid: PlacesUtils.bookmarks.menuGuid, 548 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 549 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 550 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 551 }, 552 }, 553 554 // D.7 fix wrong item types 555 // `moz_bookmarks.type` doesn't have a NOT NULL constraint, so it's 556 // possible for an item to not have a type (bug 1586427). 557 { 558 query: `UPDATE moz_bookmarks 559 SET guid = GENERATE_GUID(), 560 type = CASE WHEN fk NOT NULL THEN :bookmark_type ELSE :folder_type END 561 WHERE guid NOT IN ( 562 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 563 ) AND type IS NULL`, 564 params: { 565 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 566 folder_type: PlacesUtils.bookmarks.TYPE_FOLDER, 567 rootGuid: PlacesUtils.bookmarks.rootGuid, 568 menuGuid: PlacesUtils.bookmarks.menuGuid, 569 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 570 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 571 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 572 }, 573 }, 574 575 // D.8 fix wrong parents 576 // Items cannot have separators or other bookmarks 577 // as parent, if they have bad parent move them to unsorted bookmarks. 578 { 579 query: `UPDATE moz_bookmarks SET 580 parent = (SELECT id FROM moz_bookmarks WHERE guid = :unfiledGuid) 581 WHERE guid NOT IN ( 582 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 583 ) AND id IN ( 584 SELECT id FROM moz_bookmarks b 585 WHERE EXISTS 586 (SELECT id FROM moz_bookmarks WHERE id = b.parent 587 AND type IN (:bookmark_type, :separator_type) 588 LIMIT 1) 589 )`, 590 params: { 591 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 592 separator_type: PlacesUtils.bookmarks.TYPE_SEPARATOR, 593 rootGuid: PlacesUtils.bookmarks.rootGuid, 594 menuGuid: PlacesUtils.bookmarks.menuGuid, 595 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 596 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 597 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 598 }, 599 }, 600 601 // D.9 remove items without a valid place 602 // We've already converted folders with an `fk` to bookmarks in D.5, 603 // and bookmarks without an `fk` to folders in D.6. However, the `fk` 604 // might not reference an existing `moz_places.id`, even if it's 605 // NOT NULL. This statement takes care of those. 606 { 607 query: `DELETE FROM moz_bookmarks AS b 608 WHERE b.guid NOT IN ( 609 :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ 610 ) AND b.fk NOT NULL 611 AND b.type = :bookmark_type 612 AND NOT EXISTS (SELECT 1 FROM moz_places h WHERE h.id = b.fk)`, 613 params: { 614 bookmark_type: PlacesUtils.bookmarks.TYPE_BOOKMARK, 615 rootGuid: PlacesUtils.bookmarks.rootGuid, 616 menuGuid: PlacesUtils.bookmarks.menuGuid, 617 toolbarGuid: PlacesUtils.bookmarks.toolbarGuid, 618 unfiledGuid: PlacesUtils.bookmarks.unfiledGuid, 619 tagsGuid: PlacesUtils.bookmarks.tagsGuid, 620 }, 621 }, 622 623 // D.10 recalculate positions 624 // This requires multiple related statements. 625 // We can detect a folder with bad position values comparing the sum of 626 // all distinct position values (+1 since position is 0-based) with the 627 // triangular numbers obtained by the number of children (n). 628 // SUM(DISTINCT position + 1) == (n * (n + 1) / 2). 629 // id is not a PRIMARY KEY on purpose, since we need a rowid that 630 // increments monotonically. 631 { 632 query: `CREATE TEMP TABLE IF NOT EXISTS moz_bm_reindex_temp ( 633 id INTEGER 634 , parent INTEGER 635 , position INTEGER 636 )`, 637 }, 638 { 639 query: `INSERT INTO moz_bm_reindex_temp 640 SELECT id, parent, 0 641 FROM moz_bookmarks b 642 WHERE parent IN ( 643 SELECT parent 644 FROM moz_bookmarks 645 GROUP BY parent 646 HAVING (SUM(DISTINCT position + 1) - (count(*) * (count(*) + 1) / 2)) <> 0 647 ) 648 ORDER BY parent ASC, position ASC, ROWID ASC`, 649 }, 650 { 651 query: `CREATE INDEX IF NOT EXISTS moz_bm_reindex_temp_index 652 ON moz_bm_reindex_temp(parent)`, 653 }, 654 { 655 query: `UPDATE moz_bm_reindex_temp SET position = ( 656 ROWID - (SELECT MIN(t.ROWID) FROM moz_bm_reindex_temp t 657 WHERE t.parent = moz_bm_reindex_temp.parent) 658 )`, 659 }, 660 { 661 query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_reindex_temp_trigger 662 BEFORE DELETE ON moz_bm_reindex_temp 663 FOR EACH ROW 664 BEGIN 665 UPDATE moz_bookmarks SET position = OLD.position WHERE id = OLD.id; 666 END`, 667 }, 668 { query: `DELETE FROM moz_bm_reindex_temp` }, 669 { query: `DROP INDEX moz_bm_reindex_temp_index` }, 670 { query: `DROP TRIGGER moz_bm_reindex_temp_trigger` }, 671 { query: `DROP TABLE moz_bm_reindex_temp` }, 672 673 // D.12 Fix empty-named tags. 674 // Tags were allowed to have empty names due to a UI bug. Fix them by 675 // replacing their title with "(notitle)", and bumping the change counter 676 // for all bookmarks with the fixed tags. 677 { 678 query: `UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1 679 WHERE fk IN (SELECT b.fk FROM moz_bookmarks b 680 JOIN moz_bookmarks p ON p.id = b.parent 681 WHERE length(p.title) = 0 AND p.type = :folder_type AND 682 p.parent = :tags_folder)`, 683 params: { 684 folder_type: PlacesUtils.bookmarks.TYPE_FOLDER, 685 tags_folder: PlacesUtils.tagsFolderId, 686 }, 687 }, 688 { 689 query: `UPDATE moz_bookmarks SET title = :empty_title 690 WHERE length(title) = 0 AND type = :folder_type 691 AND parent = :tags_folder`, 692 params: { 693 empty_title: "(notitle)", 694 folder_type: PlacesUtils.bookmarks.TYPE_FOLDER, 695 tags_folder: PlacesUtils.tagsFolderId, 696 }, 697 }, 698 699 // MOZ_ICONS 700 // E.1 remove orphan icon entries. 701 { 702 query: `DELETE FROM moz_pages_w_icons WHERE page_url_hash NOT IN ( 703 SELECT url_hash FROM moz_places 704 )`, 705 }, 706 707 // Remove icons whose origin is not in moz_origins, unless referenced. 708 { 709 query: `DELETE FROM moz_icons WHERE id IN ( 710 SELECT id FROM moz_icons WHERE root = 0 711 UNION ALL 712 SELECT id FROM moz_icons 713 WHERE root = 1 714 AND get_host_and_port(icon_url) NOT IN (SELECT host FROM moz_origins) 715 AND fixup_url(get_host_and_port(icon_url)) NOT IN (SELECT host FROM moz_origins) 716 EXCEPT 717 SELECT icon_id FROM moz_icons_to_pages 718 )`, 719 }, 720 721 // MOZ_HISTORYVISITS 722 // F.1 remove orphan visits 723 { 724 query: `DELETE FROM moz_historyvisits WHERE id IN ( 725 SELECT id FROM moz_historyvisits v 726 WHERE NOT EXISTS 727 (SELECT id FROM moz_places WHERE id = v.place_id LIMIT 1) 728 )`, 729 }, 730 731 // MOZ_INPUTHISTORY 732 // G.1 remove orphan input history 733 { 734 query: `DELETE FROM moz_inputhistory WHERE place_id IN ( 735 SELECT place_id FROM moz_inputhistory i 736 WHERE NOT EXISTS 737 (SELECT id FROM moz_places WHERE id = i.place_id LIMIT 1) 738 )`, 739 }, 740 741 // MOZ_ITEMS_ANNOS 742 // H.1 remove item annos with an invalid attribute 743 { 744 query: `DELETE FROM moz_items_annos WHERE id IN ( 745 SELECT id FROM moz_items_annos t 746 WHERE NOT EXISTS 747 (SELECT id FROM moz_anno_attributes 748 WHERE id = t.anno_attribute_id LIMIT 1) 749 )`, 750 }, 751 752 // H.2 remove orphan item annos 753 { 754 query: `DELETE FROM moz_items_annos WHERE id IN ( 755 SELECT id FROM moz_items_annos t 756 WHERE NOT EXISTS 757 (SELECT id FROM moz_bookmarks WHERE id = t.item_id LIMIT 1) 758 )`, 759 }, 760 761 // MOZ_KEYWORDS 762 // I.1 remove unused keywords 763 { 764 query: `DELETE FROM moz_keywords WHERE id IN ( 765 SELECT id FROM moz_keywords k 766 WHERE NOT EXISTS 767 (SELECT 1 FROM moz_places h WHERE k.place_id = h.id) 768 )`, 769 }, 770 771 // MOZ_PLACES 772 // L.2 recalculate visit_count and last_visit_date 773 { 774 query: `UPDATE moz_places 775 SET visit_count = (SELECT count(*) FROM moz_historyvisits 776 WHERE place_id = moz_places.id AND visit_type NOT IN (0,4,7,8,9)), 777 last_visit_date = (SELECT MAX(visit_date) FROM moz_historyvisits 778 WHERE place_id = moz_places.id) 779 WHERE id IN ( 780 SELECT h.id FROM moz_places h 781 WHERE visit_count <> (SELECT count(*) FROM moz_historyvisits v 782 WHERE v.place_id = h.id AND visit_type NOT IN (0,4,7,8,9)) 783 OR last_visit_date <> (SELECT MAX(visit_date) FROM moz_historyvisits v 784 WHERE v.place_id = h.id) 785 )`, 786 }, 787 788 // L.3 recalculate hidden for redirects. 789 { 790 query: `UPDATE moz_places 791 SET hidden = 1 792 WHERE id IN ( 793 SELECT h.id FROM moz_places h 794 JOIN moz_historyvisits src ON src.place_id = h.id 795 JOIN moz_historyvisits dst ON dst.from_visit = src.id AND dst.visit_type IN (5,6) 796 LEFT JOIN moz_bookmarks on fk = h.id AND fk ISNULL 797 GROUP BY src.place_id HAVING count(*) = visit_count 798 )`, 799 }, 800 801 // L.4 recalculate foreign_count. 802 { 803 query: `UPDATE moz_places SET foreign_count = 804 (SELECT count(*) FROM moz_bookmarks WHERE fk = moz_places.id ) + 805 (SELECT count(*) FROM moz_keywords WHERE place_id = moz_places.id )`, 806 }, 807 808 // L.5 recalculate missing hashes. 809 { 810 query: `UPDATE moz_places SET url_hash = hash(url) WHERE url_hash = 0`, 811 }, 812 813 // L.6 fix invalid Place GUIDs. 814 { 815 query: `UPDATE moz_places 816 SET guid = GENERATE_GUID() 817 WHERE guid IS NULL OR 818 NOT IS_VALID_GUID(guid)`, 819 }, 820 821 // MOZ_BOOKMARKS 822 // S.1 fix invalid GUIDs for synced bookmarks. 823 // This requires multiple related statements. 824 // First, we insert tombstones for all synced bookmarks with invalid 825 // GUIDs, so that we can delete them on the server. Second, we add a 826 // temporary trigger to bump the change counter for the parents of any 827 // items we update, since Sync stores the list of child GUIDs on the 828 // parent. Finally, we assign new GUIDs for all items with missing and 829 // invalid GUIDs, bump their change counters, and reset their sync 830 // statuses to NEW so that they're considered for deduping. 831 { 832 query: `INSERT OR IGNORE INTO moz_bookmarks_deleted(guid, dateRemoved) 833 SELECT guid, :dateRemoved 834 FROM moz_bookmarks 835 WHERE syncStatus <> :syncStatus AND 836 guid NOT NULL AND 837 NOT IS_VALID_GUID(guid)`, 838 params: { 839 dateRemoved: PlacesUtils.toPRTime(new Date()), 840 syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW, 841 }, 842 }, 843 { 844 query: `UPDATE moz_bookmarks 845 SET guid = GENERATE_GUID(), 846 syncStatus = :syncStatus 847 WHERE guid IS NULL OR 848 NOT IS_VALID_GUID(guid)`, 849 params: { 850 syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW, 851 }, 852 }, 853 854 // S.2 drop tombstones for bookmarks that aren't deleted. 855 { 856 query: `DELETE FROM moz_bookmarks_deleted 857 WHERE guid IN (SELECT guid FROM moz_bookmarks)`, 858 }, 859 860 // S.3 set missing added and last modified dates. 861 { 862 query: `UPDATE moz_bookmarks 863 SET dateAdded = COALESCE(NULLIF(dateAdded, 0), NULLIF(lastModified, 0), NULLIF(( 864 SELECT MIN(visit_date) FROM moz_historyvisits 865 WHERE place_id = fk 866 ), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000), 867 lastModified = COALESCE(NULLIF(lastModified, 0), NULLIF(dateAdded, 0), NULLIF(( 868 SELECT MAX(visit_date) FROM moz_historyvisits 869 WHERE place_id = fk 870 ), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000) 871 WHERE NULLIF(dateAdded, 0) IS NULL OR 872 NULLIF(lastModified, 0) IS NULL`, 873 }, 874 875 // S.4 reset added dates that are ahead of last modified dates. 876 { 877 query: `UPDATE moz_bookmarks 878 SET dateAdded = lastModified 879 WHERE dateAdded > lastModified`, 880 }, 881 ]; 882 883 // Create triggers for updating Sync metadata. The "sync change" trigger 884 // bumps the parent's change counter when we update a GUID or move an item 885 // to a different folder, since Sync stores the list of child GUIDs on the 886 // parent. The "sync tombstone" trigger inserts tombstones for deleted 887 // synced bookmarks. 888 cleanupStatements.unshift({ 889 query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_change_temp_trigger 890 AFTER UPDATE OF guid, parent, position ON moz_bookmarks 891 FOR EACH ROW 892 BEGIN 893 UPDATE moz_bookmarks 894 SET syncChangeCounter = syncChangeCounter + 1 895 WHERE id IN (OLD.parent, NEW.parent, NEW.id); 896 END`, 897 }); 898 cleanupStatements.unshift({ 899 query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_tombstone_temp_trigger 900 AFTER DELETE ON moz_bookmarks 901 FOR EACH ROW WHEN OLD.guid NOT NULL AND 902 OLD.syncStatus <> 1 903 BEGIN 904 UPDATE moz_bookmarks 905 SET syncChangeCounter = syncChangeCounter + 1 906 WHERE id = OLD.parent; 907 908 INSERT INTO moz_bookmarks_deleted(guid, dateRemoved) 909 VALUES(OLD.guid, STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000); 910 END`, 911 }); 912 cleanupStatements.push({ 913 query: `DROP TRIGGER moz_bm_sync_change_temp_trigger`, 914 }); 915 cleanupStatements.push({ 916 query: `DROP TRIGGER moz_bm_sync_tombstone_temp_trigger`, 917 }); 918 919 return cleanupStatements; 920 }, 921 922 /** 923 * Tries to vacuum the database. 924 * 925 * Note: although this function isn't actually async, we keep it async to 926 * allow us to maintain a simple, consistent API for the tasks within this object. 927 * 928 * @return {Promise} resolves when database is vacuumed. 929 * @resolves to an array of logs for this task. 930 * @rejects if we are unable to vacuum database. 931 */ 932 async vacuum() { 933 let logs = []; 934 let placesDbPath = OS.Path.join( 935 OS.Constants.Path.profileDir, 936 "places.sqlite" 937 ); 938 let info = await OS.File.stat(placesDbPath); 939 logs.push(`Initial database size is ${parseInt(info.size / 1024)}KiB`); 940 return PlacesUtils.withConnectionWrapper( 941 "PlacesDBUtils: vacuum", 942 async db => { 943 await db.execute("VACUUM"); 944 logs.push("The database has been vacuumed"); 945 info = await OS.File.stat(placesDbPath); 946 logs.push(`Final database size is ${parseInt(info.size / 1024)}KiB`); 947 return logs; 948 } 949 ).catch(() => { 950 PlacesDBUtils.clearPendingTasks(); 951 throw new Error("Unable to vacuum database"); 952 }); 953 }, 954 955 /** 956 * Forces a full expiration on the database. 957 * 958 * Note: although this function isn't actually async, we keep it async to 959 * allow us to maintain a simple, consistent API for the tasks within this object. 960 * 961 * @return {Promise} resolves when the database in cleaned up. 962 * @resolves to an array of logs for this task. 963 */ 964 async expire() { 965 let logs = []; 966 967 let expiration = Cc["@mozilla.org/places/expiration;1"].getService( 968 Ci.nsIObserver 969 ); 970 971 let returnPromise = new Promise(res => { 972 let observer = (subject, topic, data) => { 973 Services.obs.removeObserver(observer, topic); 974 logs.push("Database cleaned up"); 975 res(logs); 976 }; 977 Services.obs.addObserver(observer, PlacesUtils.TOPIC_EXPIRATION_FINISHED); 978 }); 979 980 // Force an orphans expiration step. 981 expiration.observe(null, "places-debug-start-expiration", 0); 982 return returnPromise; 983 }, 984 985 /** 986 * Collects statistical data on the database. 987 * 988 * @return {Promise} resolves when statistics are collected. 989 * @resolves to an array of logs for this task. 990 * @rejects if we are unable to collect stats for some reason. 991 */ 992 async stats() { 993 let logs = []; 994 let placesDbPath = OS.Path.join( 995 OS.Constants.Path.profileDir, 996 "places.sqlite" 997 ); 998 let info = await OS.File.stat(placesDbPath); 999 logs.push(`Places.sqlite size is ${parseInt(info.size / 1024)}KiB`); 1000 let faviconsDbPath = OS.Path.join( 1001 OS.Constants.Path.profileDir, 1002 "favicons.sqlite" 1003 ); 1004 info = await OS.File.stat(faviconsDbPath); 1005 logs.push(`Favicons.sqlite size is ${parseInt(info.size / 1024)}KiB`); 1006 1007 // Execute each step async. 1008 let pragmas = [ 1009 "user_version", 1010 "page_size", 1011 "cache_size", 1012 "journal_mode", 1013 "synchronous", 1014 ].map(p => `pragma_${p}`); 1015 let pragmaQuery = `SELECT * FROM ${pragmas.join(", ")}`; 1016 await PlacesUtils.withConnectionWrapper( 1017 "PlacesDBUtils: pragma for stats", 1018 async db => { 1019 let row = (await db.execute(pragmaQuery))[0]; 1020 for (let i = 0; i != pragmas.length; i++) { 1021 logs.push(`${pragmas[i]} is ${row.getResultByIndex(i)}`); 1022 } 1023 } 1024 ).catch(() => { 1025 logs.push("Could not set pragma for stat collection"); 1026 }); 1027 1028 // Get maximum number of unique URIs. 1029 try { 1030 let limitURIs = await Cc["@mozilla.org/places/expiration;1"] 1031 .getService(Ci.nsISupports) 1032 .wrappedJSObject.getPagesLimit(); 1033 logs.push( 1034 "History can store a maximum of " + limitURIs + " unique pages" 1035 ); 1036 } catch (ex) {} 1037 1038 let query = "SELECT name FROM sqlite_master WHERE type = :type"; 1039 let params = {}; 1040 let _getTableCount = async tableName => { 1041 let db = await PlacesUtils.promiseDBConnection(); 1042 let rows = await db.execute(`SELECT count(*) FROM ${tableName}`); 1043 logs.push( 1044 `Table ${tableName} has ${rows[0].getResultByIndex(0)} records` 1045 ); 1046 }; 1047 1048 try { 1049 params.type = "table"; 1050 let db = await PlacesUtils.promiseDBConnection(); 1051 await db.execute(query, params, r => 1052 _getTableCount(r.getResultByIndex(0)) 1053 ); 1054 1055 params.type = "index"; 1056 await db.execute(query, params, r => { 1057 logs.push(`Index ${r.getResultByIndex(0)}`); 1058 }); 1059 1060 params.type = "trigger"; 1061 await db.execute(query, params, r => { 1062 logs.push(`Trigger ${r.getResultByIndex(0)}`); 1063 }); 1064 } catch (ex) { 1065 throw new Error("Unable to collect stats."); 1066 } 1067 1068 return logs; 1069 }, 1070 1071 /** 1072 * Recalculates statistical data on the origin frecencies in the database. 1073 * 1074 * @return {Promise} resolves when statistics are collected. 1075 */ 1076 originFrecencyStats() { 1077 return new Promise(resolve => { 1078 PlacesUtils.history.recalculateOriginFrecencyStats(() => 1079 resolve(["Recalculated origin frecency stats"]) 1080 ); 1081 }); 1082 }, 1083 1084 /** 1085 * Collects telemetry data and reports it to Telemetry. 1086 * 1087 * Note: although this function isn't actually async, we keep it async to 1088 * allow us to maintain a simple, consistent API for the tasks within this object. 1089 * 1090 */ 1091 async telemetry() { 1092 // This will be populated with one integer property for each probe result, 1093 // using the histogram name as key. 1094 let probeValues = {}; 1095 1096 // The following array contains an ordered list of entries that are 1097 // processed to collect telemetry data. Each entry has these properties: 1098 // 1099 // histogram: Name of the telemetry histogram to update. 1100 // query: This is optional. If present, contains a database command 1101 // that will be executed asynchronously, and whose result will 1102 // be added to the telemetry histogram. 1103 // callback: This is optional. If present, contains a function that must 1104 // return the value that will be added to the telemetry 1105 // histogram. If a query is also present, its result is passed 1106 // as the first argument of the function. If the function 1107 // raises an exception, no data is added to the histogram. 1108 // 1109 // Since all queries are executed in order by the database backend, the 1110 // callbacks can also use the result of previous queries stored in the 1111 // probeValues object. 1112 let probes = [ 1113 { 1114 histogram: "PLACES_PAGES_COUNT", 1115 query: "SELECT count(*) FROM moz_places", 1116 }, 1117 1118 { 1119 histogram: "PLACES_BOOKMARKS_COUNT", 1120 query: `SELECT count(*) FROM moz_bookmarks b 1121 JOIN moz_bookmarks t ON t.id = b.parent 1122 AND t.parent <> :tags_folder 1123 WHERE b.type = :type_bookmark`, 1124 params: { 1125 tags_folder: PlacesUtils.tagsFolderId, 1126 type_bookmark: PlacesUtils.bookmarks.TYPE_BOOKMARK, 1127 }, 1128 }, 1129 1130 { 1131 histogram: "PLACES_TAGS_COUNT", 1132 query: `SELECT count(*) FROM moz_bookmarks 1133 WHERE parent = :tags_folder`, 1134 params: { 1135 tags_folder: PlacesUtils.tagsFolderId, 1136 }, 1137 }, 1138 1139 { 1140 histogram: "PLACES_KEYWORDS_COUNT", 1141 query: "SELECT count(*) FROM moz_keywords", 1142 }, 1143 1144 { 1145 histogram: "PLACES_SORTED_BOOKMARKS_PERC", 1146 query: `SELECT IFNULL(ROUND(( 1147 SELECT count(*) FROM moz_bookmarks b 1148 JOIN moz_bookmarks t ON t.id = b.parent 1149 AND t.parent <> :tags_folder AND t.parent > :places_root 1150 WHERE b.type = :type_bookmark 1151 ) * 100 / ( 1152 SELECT count(*) FROM moz_bookmarks b 1153 JOIN moz_bookmarks t ON t.id = b.parent 1154 AND t.parent <> :tags_folder 1155 WHERE b.type = :type_bookmark 1156 )), 0)`, 1157 params: { 1158 places_root: PlacesUtils.placesRootId, 1159 tags_folder: PlacesUtils.tagsFolderId, 1160 type_bookmark: PlacesUtils.bookmarks.TYPE_BOOKMARK, 1161 }, 1162 }, 1163 1164 { 1165 histogram: "PLACES_TAGGED_BOOKMARKS_PERC", 1166 query: `SELECT IFNULL(ROUND(( 1167 SELECT count(*) FROM moz_bookmarks b 1168 JOIN moz_bookmarks t ON t.id = b.parent 1169 AND t.parent = :tags_folder 1170 ) * 100 / ( 1171 SELECT count(*) FROM moz_bookmarks b 1172 JOIN moz_bookmarks t ON t.id = b.parent 1173 AND t.parent <> :tags_folder 1174 WHERE b.type = :type_bookmark 1175 )), 0)`, 1176 params: { 1177 tags_folder: PlacesUtils.tagsFolderId, 1178 type_bookmark: PlacesUtils.bookmarks.TYPE_BOOKMARK, 1179 }, 1180 }, 1181 1182 { 1183 histogram: "PLACES_DATABASE_FILESIZE_MB", 1184 async callback() { 1185 let placesDbPath = OS.Path.join( 1186 OS.Constants.Path.profileDir, 1187 "places.sqlite" 1188 ); 1189 let info = await OS.File.stat(placesDbPath); 1190 return parseInt(info.size / BYTES_PER_MEBIBYTE); 1191 }, 1192 }, 1193 1194 { 1195 histogram: "PLACES_DATABASE_PAGESIZE_B", 1196 query: "PRAGMA page_size /* PlacesDBUtils.jsm PAGESIZE_B */", 1197 }, 1198 1199 { 1200 histogram: "PLACES_DATABASE_SIZE_PER_PAGE_B", 1201 query: "PRAGMA page_count", 1202 callback(aDbPageCount) { 1203 // Note that the database file size would not be meaningful for this 1204 // calculation, because the file grows in fixed-size chunks. 1205 let dbPageSize = probeValues.PLACES_DATABASE_PAGESIZE_B; 1206 let placesPageCount = probeValues.PLACES_PAGES_COUNT; 1207 return Math.round((dbPageSize * aDbPageCount) / placesPageCount); 1208 }, 1209 }, 1210 1211 { 1212 histogram: "PLACES_DATABASE_FAVICONS_FILESIZE_MB", 1213 async callback() { 1214 let faviconsDbPath = OS.Path.join( 1215 OS.Constants.Path.profileDir, 1216 "favicons.sqlite" 1217 ); 1218 let info = await OS.File.stat(faviconsDbPath); 1219 return parseInt(info.size / BYTES_PER_MEBIBYTE); 1220 }, 1221 }, 1222 1223 { 1224 histogram: "PLACES_ANNOS_BOOKMARKS_COUNT", 1225 query: "SELECT count(*) FROM moz_items_annos", 1226 }, 1227 1228 { 1229 histogram: "PLACES_ANNOS_PAGES_COUNT", 1230 query: "SELECT count(*) FROM moz_annos", 1231 }, 1232 1233 { 1234 histogram: "PLACES_MAINTENANCE_DAYSFROMLAST", 1235 callback() { 1236 try { 1237 let lastMaintenance = Services.prefs.getIntPref( 1238 "places.database.lastMaintenance" 1239 ); 1240 let nowSeconds = parseInt(Date.now() / 1000); 1241 return parseInt((nowSeconds - lastMaintenance) / 86400); 1242 } catch (ex) { 1243 return 60; 1244 } 1245 }, 1246 }, 1247 ]; 1248 1249 for (let probe of probes) { 1250 let val; 1251 if ("query" in probe) { 1252 let db = await PlacesUtils.promiseDBConnection(); 1253 val = ( 1254 await db.execute(probe.query, probe.params || {}) 1255 )[0].getResultByIndex(0); 1256 } 1257 // Report the result of the probe through Telemetry. 1258 // The resulting promise cannot reject. 1259 if ("callback" in probe) { 1260 val = await probe.callback(val); 1261 } 1262 probeValues[probe.histogram] = val; 1263 Services.telemetry.getHistogramById(probe.histogram).add(val); 1264 } 1265 }, 1266 1267 /** 1268 * Remove old and useless places.sqlite.corrupt files. 1269 * 1270 * @resolves to an array of logs for this task. 1271 * 1272 */ 1273 async removeOldCorruptDBs() { 1274 let logs = []; 1275 logs.push( 1276 "> Cleanup profile from places.sqlite.corrupt files older than " + 1277 CORRUPT_DB_RETAIN_DAYS + 1278 " days." 1279 ); 1280 let re = /^places\.sqlite(-\d)?\.corrupt$/; 1281 let currentTime = Date.now(); 1282 let iterator = new OS.File.DirectoryIterator(OS.Constants.Path.profileDir); 1283 try { 1284 await iterator.forEach(async entry => { 1285 let lastModificationDate; 1286 if (!entry.isDir && !entry.isSymLink && re.test(entry.name)) { 1287 if ("winLastWriteDate" in entry) { 1288 // Under Windows, additional information allows us to sort files immediately 1289 // without having to perform additional I/O. 1290 lastModificationDate = entry.winLastWriteDate.getTime(); 1291 } else { 1292 // Under other OSes, we need to call OS.File.stat 1293 let info = await OS.File.stat(entry.path); 1294 lastModificationDate = info.lastModificationDate.getTime(); 1295 } 1296 try { 1297 // Convert milliseconds to days. 1298 let days = Math.ceil( 1299 (currentTime - lastModificationDate) / MS_PER_DAY 1300 ); 1301 if (days >= CORRUPT_DB_RETAIN_DAYS || days < 0) { 1302 await OS.File.remove(entry.path); 1303 } 1304 } catch (error) { 1305 logs.push("Could not remove file: " + entry.path, error); 1306 } 1307 } 1308 }); 1309 } catch (error) { 1310 logs.push("removeOldCorruptDBs failed", error); 1311 } finally { 1312 iterator.close(); 1313 } 1314 return logs; 1315 }, 1316 1317 /** 1318 * Runs a list of tasks, returning a Map when done. 1319 * 1320 * @param tasks 1321 * Array of tasks to be executed, in form of pointers to methods in 1322 * this module. 1323 * @return {Promise} 1324 * A promise that resolves with a Map[taskName(String) -> Object]. 1325 * The Object has the following properties: 1326 * - succeeded: boolean 1327 * - logs: an array of strings containing the messages logged by the task 1328 */ 1329 async runTasks(tasks) { 1330 if (!this._registeredShutdownObserver) { 1331 this._registeredShutdownObserver = true; 1332 PlacesUtils.registerShutdownFunction(() => { 1333 this._isShuttingDown = true; 1334 }); 1335 } 1336 PlacesDBUtils._clearTaskQueue = false; 1337 let tasksMap = new Map(); 1338 for (let task of tasks) { 1339 if (PlacesDBUtils._isShuttingDown) { 1340 tasksMap.set(task.name, { 1341 succeeded: false, 1342 logs: ["Shutting down, will not schedule the task."], 1343 }); 1344 continue; 1345 } 1346 1347 if (PlacesDBUtils._clearTaskQueue) { 1348 tasksMap.set(task.name, { 1349 succeeded: false, 1350 logs: ["The task queue was cleared by an error in another task."], 1351 }); 1352 continue; 1353 } 1354 1355 let result = await task() 1356 .then((logs = [`${task.name} complete`]) => ({ succeeded: true, logs })) 1357 .catch(err => ({ succeeded: false, logs: [err.message] })); 1358 tasksMap.set(task.name, result); 1359 } 1360 return tasksMap; 1361 }, 1362}; 1363 1364async function integrity(dbName) { 1365 async function check(db) { 1366 let row; 1367 await db.execute("PRAGMA integrity_check", null, (r, cancel) => { 1368 row = r; 1369 cancel(); 1370 }); 1371 return row.getResultByIndex(0) === "ok"; 1372 } 1373 1374 // Create a new connection for this check, so we can operate independently 1375 // from a broken Places service. 1376 // openConnection returns an exception with .result == Cr.NS_ERROR_FILE_CORRUPTED, 1377 // we should do the same everywhere we want maintenance to try replacing the 1378 // database on next startup. 1379 let path = OS.Path.join(OS.Constants.Path.profileDir, dbName); 1380 let db = await Sqlite.openConnection({ path }); 1381 try { 1382 if (await check(db)) { 1383 return; 1384 } 1385 1386 // We stopped due to an integrity corruption, try to fix it if possible. 1387 // First, try to reindex, this often fixes simple indices problems. 1388 try { 1389 await db.execute("REINDEX"); 1390 } catch (ex) { 1391 throw new Components.Exception( 1392 "Impossible to reindex database", 1393 Cr.NS_ERROR_FILE_CORRUPTED 1394 ); 1395 } 1396 1397 // Check again. 1398 if (!(await check(db))) { 1399 throw new Components.Exception( 1400 "The database is still corrupt", 1401 Cr.NS_ERROR_FILE_CORRUPTED 1402 ); 1403 } 1404 } finally { 1405 await db.close(); 1406 } 1407} 1408 1409function PlacesDBUtilsIdleMaintenance() {} 1410 1411PlacesDBUtilsIdleMaintenance.prototype = { 1412 observe(subject, topic, data) { 1413 switch (topic) { 1414 case "idle-daily": 1415 // Once a week run places.sqlite maintenance tasks. 1416 let lastMaintenance = Services.prefs.getIntPref( 1417 "places.database.lastMaintenance", 1418 0 1419 ); 1420 let nowSeconds = parseInt(Date.now() / 1000); 1421 if (lastMaintenance < nowSeconds - MAINTENANCE_INTERVAL_SECONDS) { 1422 PlacesDBUtils.maintenanceOnIdle(); 1423 } 1424 break; 1425 default: 1426 throw new Error("Trying to handle an unknown category."); 1427 } 1428 }, 1429 _xpcom_factory: ComponentUtils.generateSingletonFactory( 1430 PlacesDBUtilsIdleMaintenance 1431 ), 1432 classID: Components.ID("d38926e0-29c1-11eb-8588-0800200c9a66"), 1433 QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), 1434}; 1435