1 /* This Source Code Form is subject to the terms of the Mozilla Public
2  * License, v. 2.0. If a copy of the MPL was not distributed with this
3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4 
5 #include "mozilla/ArrayUtils.h"
6 #include "mozilla/Attributes.h"
7 #include "mozilla/DebugOnly.h"
8 #include "mozilla/ScopeExit.h"
9 #include "mozilla/SpinEventLoopUntil.h"
10 #include "mozilla/JSONWriter.h"
11 
12 #include "Database.h"
13 
14 #include "nsIInterfaceRequestorUtils.h"
15 #include "nsIFile.h"
16 
17 #include "nsNavBookmarks.h"
18 #include "nsNavHistory.h"
19 #include "nsPlacesTables.h"
20 #include "nsPlacesIndexes.h"
21 #include "nsPlacesTriggers.h"
22 #include "nsPlacesMacros.h"
23 #include "nsVariant.h"
24 #include "SQLFunctions.h"
25 #include "Helpers.h"
26 #include "nsFaviconService.h"
27 
28 #include "nsAppDirectoryServiceDefs.h"
29 #include "nsDirectoryServiceUtils.h"
30 #include "prenv.h"
31 #include "prsystem.h"
32 #include "nsPrintfCString.h"
33 #include "mozilla/Preferences.h"
34 #include "mozilla/Services.h"
35 #include "mozilla/Unused.h"
36 #include "mozIStorageService.h"
37 #include "prtime.h"
38 
39 #include "nsXULAppAPI.h"
40 
41 // Time between corrupt database backups.
42 #define RECENT_BACKUP_TIME_MICROSEC (int64_t)86400 * PR_USEC_PER_SEC  // 24H
43 
44 // Filename of the database.
45 #define DATABASE_FILENAME u"places.sqlite"_ns
46 // Filename of the icons database.
47 #define DATABASE_FAVICONS_FILENAME u"favicons.sqlite"_ns
48 
49 // Set to the database file name when it was found corrupt by a previous
50 // maintenance run.
51 #define PREF_FORCE_DATABASE_REPLACEMENT \
52   "places.database.replaceDatabaseOnStartup"
53 
54 // Whether on corruption we should try to fix the database by cloning it.
55 #define PREF_DATABASE_CLONEONCORRUPTION "places.database.cloneOnCorruption"
56 
57 // Set to specify the size of the places database growth increments in kibibytes
58 #define PREF_GROWTH_INCREMENT_KIB "places.database.growthIncrementKiB"
59 
60 // Set to disable the default robust storage and use volatile, in-memory
61 // storage without robust transaction flushing guarantees. This makes
62 // SQLite use much less I/O at the cost of losing data when things crash.
63 // The pref is only honored if an environment variable is set. The env
64 // variable is intentionally named something scary to help prevent someone
65 // from thinking it is a useful performance optimization they should enable.
66 #define PREF_DISABLE_DURABILITY "places.database.disableDurability"
67 
68 #define PREF_PREVIEWS_ENABLED "places.previews.enabled"
69 
70 #define ENV_ALLOW_CORRUPTION \
71   "ALLOW_PLACES_DATABASE_TO_LOSE_DATA_AND_BECOME_CORRUPT"
72 
73 #define PREF_MIGRATE_V52_ORIGIN_FRECENCIES \
74   "places.database.migrateV52OriginFrecencies"
75 
76 // Maximum size for the WAL file.
77 // For performance reasons this should be as large as possible, so that more
78 // transactions can fit into it, and the checkpoint cost is paid less often.
79 // At the same time, since we use synchronous = NORMAL, an fsync happens only
80 // at checkpoint time, so we don't want the WAL to grow too much and risk to
81 // lose all the contained transactions on a crash.
82 #define DATABASE_MAX_WAL_BYTES 2048000
83 
84 // Since exceeding the journal limit will cause a truncate, we allow a slightly
85 // larger limit than DATABASE_MAX_WAL_BYTES to reduce the number of truncates.
86 // This is the number of bytes the journal can grow over the maximum wal size
87 // before being truncated.
88 #define DATABASE_JOURNAL_OVERHEAD_BYTES 2048000
89 
90 #define BYTES_PER_KIBIBYTE 1024
91 
92 // How much time Sqlite can wait before returning a SQLITE_BUSY error.
93 #define DATABASE_BUSY_TIMEOUT_MS 100
94 
95 // This annotation is no longer used & is obsolete, but here for migration.
96 #define LAST_USED_ANNO "bookmarkPropertiesDialog/folderLastUsed"_ns
97 // This is key in the meta table that the LAST_USED_ANNO is migrated to.
98 #define LAST_USED_FOLDERS_META_KEY "places/bookmarks/edit/lastusedfolder"_ns
99 
100 // We use a fixed title for the mobile root to avoid marking the database as
101 // corrupt if we can't look up the localized title in the string bundle. Sync
102 // sets the title to the localized version when it creates the left pane query.
103 #define MOBILE_ROOT_TITLE "mobile"
104 
105 // Legacy item annotation used by the old Sync engine.
106 #define SYNC_PARENT_ANNO "sync/parent"
107 
108 using namespace mozilla;
109 
110 namespace mozilla {
111 namespace places {
112 
113 namespace {
114 
115 ////////////////////////////////////////////////////////////////////////////////
116 //// Helpers
117 
118 /**
119  * Get the filename for a corrupt database.
120  */
getCorruptFilename(const nsString & aDbFilename)121 nsString getCorruptFilename(const nsString& aDbFilename) {
122   return aDbFilename + u".corrupt"_ns;
123 }
124 /**
125  * Get the filename for a recover database.
126  */
getRecoverFilename(const nsString & aDbFilename)127 nsString getRecoverFilename(const nsString& aDbFilename) {
128   return aDbFilename + u".recover"_ns;
129 }
130 
131 /**
132  * Checks whether exists a corrupt database file created not longer than
133  * RECENT_BACKUP_TIME_MICROSEC ago.
134  */
isRecentCorruptFile(const nsCOMPtr<nsIFile> & aCorruptFile)135 bool isRecentCorruptFile(const nsCOMPtr<nsIFile>& aCorruptFile) {
136   MOZ_ASSERT(NS_IsMainThread());
137   bool fileExists = false;
138   if (NS_FAILED(aCorruptFile->Exists(&fileExists)) || !fileExists) {
139     return false;
140   }
141   PRTime lastMod = 0;
142   if (NS_FAILED(aCorruptFile->GetLastModifiedTime(&lastMod)) || lastMod <= 0 ||
143       (PR_Now() - lastMod) > RECENT_BACKUP_TIME_MICROSEC) {
144     return false;
145   }
146   return true;
147 }
148 
149 /**
150  * Sets the connection journal mode to one of the JOURNAL_* types.
151  *
152  * @param aDBConn
153  *        The database connection.
154  * @param aJournalMode
155  *        One of the JOURNAL_* types.
156  * @returns the current journal mode.
157  * @note this may return a different journal mode than the required one, since
158  *       setting it may fail.
159  */
SetJournalMode(nsCOMPtr<mozIStorageConnection> & aDBConn,enum JournalMode aJournalMode)160 enum JournalMode SetJournalMode(nsCOMPtr<mozIStorageConnection>& aDBConn,
161                                 enum JournalMode aJournalMode) {
162   MOZ_ASSERT(NS_IsMainThread());
163   nsAutoCString journalMode;
164   switch (aJournalMode) {
165     default:
166       MOZ_FALLTHROUGH_ASSERT("Trying to set an unknown journal mode.");
167       // Fall through to the default DELETE journal.
168     case JOURNAL_DELETE:
169       journalMode.AssignLiteral("delete");
170       break;
171     case JOURNAL_TRUNCATE:
172       journalMode.AssignLiteral("truncate");
173       break;
174     case JOURNAL_MEMORY:
175       journalMode.AssignLiteral("memory");
176       break;
177     case JOURNAL_WAL:
178       journalMode.AssignLiteral("wal");
179       break;
180   }
181 
182   nsCOMPtr<mozIStorageStatement> statement;
183   nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA journal_mode = ");
184   query.Append(journalMode);
185   aDBConn->CreateStatement(query, getter_AddRefs(statement));
186   NS_ENSURE_TRUE(statement, JOURNAL_DELETE);
187 
188   bool hasResult = false;
189   if (NS_SUCCEEDED(statement->ExecuteStep(&hasResult)) && hasResult &&
190       NS_SUCCEEDED(statement->GetUTF8String(0, journalMode))) {
191     if (journalMode.EqualsLiteral("delete")) {
192       return JOURNAL_DELETE;
193     }
194     if (journalMode.EqualsLiteral("truncate")) {
195       return JOURNAL_TRUNCATE;
196     }
197     if (journalMode.EqualsLiteral("memory")) {
198       return JOURNAL_MEMORY;
199     }
200     if (journalMode.EqualsLiteral("wal")) {
201       return JOURNAL_WAL;
202     }
203     MOZ_ASSERT(false, "Got an unknown journal mode.");
204   }
205 
206   return JOURNAL_DELETE;
207 }
208 
CreateRoot(nsCOMPtr<mozIStorageConnection> & aDBConn,const nsCString & aRootName,const nsCString & aGuid,const nsCString & titleString,const int32_t position,int64_t & newId)209 nsresult CreateRoot(nsCOMPtr<mozIStorageConnection>& aDBConn,
210                     const nsCString& aRootName, const nsCString& aGuid,
211                     const nsCString& titleString, const int32_t position,
212                     int64_t& newId) {
213   MOZ_ASSERT(NS_IsMainThread());
214 
215   // A single creation timestamp for all roots so that the root folder's
216   // last modification time isn't earlier than its childrens' creation time.
217   static PRTime timestamp = 0;
218   if (!timestamp) timestamp = RoundedPRNow();
219 
220   // Create a new bookmark folder for the root.
221   nsCOMPtr<mozIStorageStatement> stmt;
222   nsresult rv = aDBConn->CreateStatement(
223       nsLiteralCString(
224           "INSERT INTO moz_bookmarks "
225           "(type, position, title, dateAdded, lastModified, guid, parent, "
226           "syncChangeCounter, syncStatus) "
227           "VALUES (:item_type, :item_position, :item_title,"
228           ":date_added, :last_modified, :guid, "
229           "IFNULL((SELECT id FROM moz_bookmarks WHERE parent = 0), 0), "
230           "1, :sync_status)"),
231       getter_AddRefs(stmt));
232   if (NS_FAILED(rv)) return rv;
233 
234   rv = stmt->BindInt32ByName("item_type"_ns,
235                              nsINavBookmarksService::TYPE_FOLDER);
236   if (NS_FAILED(rv)) return rv;
237   rv = stmt->BindInt32ByName("item_position"_ns, position);
238   if (NS_FAILED(rv)) return rv;
239   rv = stmt->BindUTF8StringByName("item_title"_ns, titleString);
240   if (NS_FAILED(rv)) return rv;
241   rv = stmt->BindInt64ByName("date_added"_ns, timestamp);
242   if (NS_FAILED(rv)) return rv;
243   rv = stmt->BindInt64ByName("last_modified"_ns, timestamp);
244   if (NS_FAILED(rv)) return rv;
245   rv = stmt->BindUTF8StringByName("guid"_ns, aGuid);
246   if (NS_FAILED(rv)) return rv;
247   rv = stmt->BindInt32ByName("sync_status"_ns,
248                              nsINavBookmarksService::SYNC_STATUS_NEW);
249   if (NS_FAILED(rv)) return rv;
250   rv = stmt->Execute();
251   if (NS_FAILED(rv)) return rv;
252 
253   newId = nsNavBookmarks::sLastInsertedItemId;
254   return NS_OK;
255 }
256 
SetupDurability(nsCOMPtr<mozIStorageConnection> & aDBConn,int32_t aDBPageSize)257 nsresult SetupDurability(nsCOMPtr<mozIStorageConnection>& aDBConn,
258                          int32_t aDBPageSize) {
259   nsresult rv;
260   if (PR_GetEnv(ENV_ALLOW_CORRUPTION) &&
261       Preferences::GetBool(PREF_DISABLE_DURABILITY, false)) {
262     // Volatile storage was requested. Use the in-memory journal (no
263     // filesystem I/O) and don't sync the filesystem after writing.
264     SetJournalMode(aDBConn, JOURNAL_MEMORY);
265     rv = aDBConn->ExecuteSimpleSQL("PRAGMA synchronous = OFF"_ns);
266     NS_ENSURE_SUCCESS(rv, rv);
267   } else {
268     // Be sure to set journal mode after page_size.  WAL would prevent the
269     // change otherwise.
270     if (JOURNAL_WAL == SetJournalMode(aDBConn, JOURNAL_WAL)) {
271       // Set the WAL journal size limit.
272       int32_t checkpointPages =
273           static_cast<int32_t>(DATABASE_MAX_WAL_BYTES / aDBPageSize);
274       nsAutoCString checkpointPragma("PRAGMA wal_autocheckpoint = ");
275       checkpointPragma.AppendInt(checkpointPages);
276       rv = aDBConn->ExecuteSimpleSQL(checkpointPragma);
277       NS_ENSURE_SUCCESS(rv, rv);
278     } else {
279       // Ignore errors, if we fail here the database could be considered corrupt
280       // and we won't be able to go on, even if it's just matter of a bogus file
281       // system.  The default mode (DELETE) will be fine in such a case.
282       (void)SetJournalMode(aDBConn, JOURNAL_TRUNCATE);
283 
284       // Set synchronous to FULL to ensure maximum data integrity, even in
285       // case of crashes or unclean shutdowns.
286       rv = aDBConn->ExecuteSimpleSQL("PRAGMA synchronous = FULL"_ns);
287       NS_ENSURE_SUCCESS(rv, rv);
288     }
289   }
290 
291   // The journal is usually free to grow for performance reasons, but it never
292   // shrinks back.  Since the space taken may be problematic, limit its size.
293   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
294   journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES +
295                               DATABASE_JOURNAL_OVERHEAD_BYTES);
296   (void)aDBConn->ExecuteSimpleSQL(journalSizePragma);
297 
298   // Grow places in |growthIncrementKiB| increments to limit fragmentation on
299   // disk. By default, it's 5 MB.
300   int32_t growthIncrementKiB =
301       Preferences::GetInt(PREF_GROWTH_INCREMENT_KIB, 5 * BYTES_PER_KIBIBYTE);
302   if (growthIncrementKiB > 0) {
303     (void)aDBConn->SetGrowthIncrement(growthIncrementKiB * BYTES_PER_KIBIBYTE,
304                                       ""_ns);
305   }
306   return NS_OK;
307 }
308 
AttachDatabase(nsCOMPtr<mozIStorageConnection> & aDBConn,const nsACString & aPath,const nsACString & aName)309 nsresult AttachDatabase(nsCOMPtr<mozIStorageConnection>& aDBConn,
310                         const nsACString& aPath, const nsACString& aName) {
311   nsCOMPtr<mozIStorageStatement> stmt;
312   nsresult rv = aDBConn->CreateStatement("ATTACH DATABASE :path AS "_ns + aName,
313                                          getter_AddRefs(stmt));
314   NS_ENSURE_SUCCESS(rv, rv);
315   rv = stmt->BindUTF8StringByName("path"_ns, aPath);
316   NS_ENSURE_SUCCESS(rv, rv);
317   rv = stmt->Execute();
318   NS_ENSURE_SUCCESS(rv, rv);
319 
320   // The journal limit must be set apart for each database.
321   nsAutoCString journalSizePragma("PRAGMA favicons.journal_size_limit = ");
322   journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES +
323                               DATABASE_JOURNAL_OVERHEAD_BYTES);
324   Unused << aDBConn->ExecuteSimpleSQL(journalSizePragma);
325 
326   return NS_OK;
327 }
328 
329 }  // namespace
330 
331 ////////////////////////////////////////////////////////////////////////////////
332 //// Database
333 
PLACES_FACTORY_SINGLETON_IMPLEMENTATION(Database,gDatabase)334 PLACES_FACTORY_SINGLETON_IMPLEMENTATION(Database, gDatabase)
335 
336 NS_IMPL_ISUPPORTS(Database, nsIObserver, nsISupportsWeakReference)
337 
338 Database::Database()
339     : mMainThreadStatements(mMainConn),
340       mMainThreadAsyncStatements(mMainConn),
341       mAsyncThreadStatements(mMainConn),
342       mDBPageSize(0),
343       mDatabaseStatus(nsINavHistoryService::DATABASE_STATUS_OK),
344       mClosed(false),
345       mClientsShutdown(new ClientsShutdownBlocker()),
346       mConnectionShutdown(new ConnectionShutdownBlocker(this)),
347       mMaxUrlLength(0),
348       mCacheObservers(TOPIC_PLACES_INIT_COMPLETE),
349       mRootId(-1),
350       mMenuRootId(-1),
351       mTagsRootId(-1),
352       mUnfiledRootId(-1),
353       mToolbarRootId(-1),
354       mMobileRootId(-1) {
355   MOZ_ASSERT(!XRE_IsContentProcess(),
356              "Cannot instantiate Places in the content process");
357   // Attempting to create two instances of the service?
358   MOZ_ASSERT(!gDatabase);
359   gDatabase = this;
360 }
361 
362 already_AddRefed<nsIAsyncShutdownClient>
GetProfileChangeTeardownPhase()363 Database::GetProfileChangeTeardownPhase() {
364   nsCOMPtr<nsIAsyncShutdownService> asyncShutdownSvc =
365       services::GetAsyncShutdownService();
366   MOZ_ASSERT(asyncShutdownSvc);
367   if (NS_WARN_IF(!asyncShutdownSvc)) {
368     return nullptr;
369   }
370 
371   // Consumers of Places should shutdown before us, at profile-change-teardown.
372   nsCOMPtr<nsIAsyncShutdownClient> shutdownPhase;
373   DebugOnly<nsresult> rv =
374       asyncShutdownSvc->GetProfileChangeTeardown(getter_AddRefs(shutdownPhase));
375   MOZ_ASSERT(NS_SUCCEEDED(rv));
376   return shutdownPhase.forget();
377 }
378 
379 already_AddRefed<nsIAsyncShutdownClient>
GetProfileBeforeChangePhase()380 Database::GetProfileBeforeChangePhase() {
381   nsCOMPtr<nsIAsyncShutdownService> asyncShutdownSvc =
382       services::GetAsyncShutdownService();
383   MOZ_ASSERT(asyncShutdownSvc);
384   if (NS_WARN_IF(!asyncShutdownSvc)) {
385     return nullptr;
386   }
387 
388   // Consumers of Places should shutdown before us, at profile-change-teardown.
389   nsCOMPtr<nsIAsyncShutdownClient> shutdownPhase;
390   DebugOnly<nsresult> rv =
391       asyncShutdownSvc->GetProfileBeforeChange(getter_AddRefs(shutdownPhase));
392   MOZ_ASSERT(NS_SUCCEEDED(rv));
393   return shutdownPhase.forget();
394 }
395 
396 Database::~Database() = default;
397 
GetAsyncStatement(const nsACString & aQuery)398 already_AddRefed<mozIStorageAsyncStatement> Database::GetAsyncStatement(
399     const nsACString& aQuery) {
400   if (PlacesShutdownBlocker::sIsStarted || NS_FAILED(EnsureConnection())) {
401     return nullptr;
402   }
403 
404   MOZ_ASSERT(NS_IsMainThread());
405   return mMainThreadAsyncStatements.GetCachedStatement(aQuery);
406 }
407 
GetStatement(const nsACString & aQuery)408 already_AddRefed<mozIStorageStatement> Database::GetStatement(
409     const nsACString& aQuery) {
410   if (PlacesShutdownBlocker::sIsStarted) {
411     return nullptr;
412   }
413   if (NS_IsMainThread()) {
414     if (NS_FAILED(EnsureConnection())) {
415       return nullptr;
416     }
417     return mMainThreadStatements.GetCachedStatement(aQuery);
418   }
419   // In the async case, the connection must have been started on the main-thread
420   // already.
421   MOZ_ASSERT(mMainConn);
422   return mAsyncThreadStatements.GetCachedStatement(aQuery);
423 }
424 
GetClientsShutdown()425 already_AddRefed<nsIAsyncShutdownClient> Database::GetClientsShutdown() {
426   if (mClientsShutdown) return mClientsShutdown->GetClient();
427   return nullptr;
428 }
429 
GetConnectionShutdown()430 already_AddRefed<nsIAsyncShutdownClient> Database::GetConnectionShutdown() {
431   if (mConnectionShutdown) return mConnectionShutdown->GetClient();
432   return nullptr;
433 }
434 
435 // static
GetDatabase()436 already_AddRefed<Database> Database::GetDatabase() {
437   if (PlacesShutdownBlocker::sIsStarted) {
438     return nullptr;
439   }
440   return GetSingleton();
441 }
442 
Init()443 nsresult Database::Init() {
444   MOZ_ASSERT(NS_IsMainThread());
445 
446   // DO NOT FAIL HERE, otherwise we would never break the cycle between this
447   // object and the shutdown blockers, causing unexpected leaks.
448 
449   {
450     // First of all Places clients should block profile-change-teardown.
451     nsCOMPtr<nsIAsyncShutdownClient> shutdownPhase =
452         GetProfileChangeTeardownPhase();
453     MOZ_ASSERT(shutdownPhase);
454     if (shutdownPhase) {
455       DebugOnly<nsresult> rv = shutdownPhase->AddBlocker(
456           static_cast<nsIAsyncShutdownBlocker*>(mClientsShutdown.get()),
457           NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__, u""_ns);
458       MOZ_ASSERT(NS_SUCCEEDED(rv));
459     }
460   }
461 
462   {
463     // Then connection closing should block profile-before-change.
464     nsCOMPtr<nsIAsyncShutdownClient> shutdownPhase =
465         GetProfileBeforeChangePhase();
466     MOZ_ASSERT(shutdownPhase);
467     if (shutdownPhase) {
468       DebugOnly<nsresult> rv = shutdownPhase->AddBlocker(
469           static_cast<nsIAsyncShutdownBlocker*>(mConnectionShutdown.get()),
470           NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__, u""_ns);
471       MOZ_ASSERT(NS_SUCCEEDED(rv));
472     }
473   }
474 
475   // Finally observe profile shutdown notifications.
476   nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
477   if (os) {
478     (void)os->AddObserver(this, TOPIC_PROFILE_CHANGE_TEARDOWN, true);
479   }
480   return NS_OK;
481 }
482 
EnsureConnection()483 nsresult Database::EnsureConnection() {
484   // Run this only once.
485   if (mMainConn ||
486       mDatabaseStatus == nsINavHistoryService::DATABASE_STATUS_LOCKED) {
487     return NS_OK;
488   }
489   // Don't try to create a database too late.
490   if (PlacesShutdownBlocker::sIsStarted) {
491     return NS_ERROR_FAILURE;
492   }
493 
494   MOZ_ASSERT(NS_IsMainThread(),
495              "Database initialization must happen on the main-thread");
496 
497   {
498     bool initSucceeded = false;
499     auto notify = MakeScopeExit([&]() {
500       // If the database connection cannot be opened, it may just be locked
501       // by third parties.  Set a locked state.
502       if (!initSucceeded) {
503         mMainConn = nullptr;
504         mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_LOCKED;
505       }
506       // Notify at the next tick, to avoid re-entrancy problems.
507       NS_DispatchToMainThread(
508           NewRunnableMethod("places::Database::EnsureConnection()", this,
509                             &Database::NotifyConnectionInitalized));
510     });
511 
512     nsCOMPtr<mozIStorageService> storage =
513         do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID);
514     NS_ENSURE_STATE(storage);
515 
516     nsCOMPtr<nsIFile> profileDir;
517     nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
518                                          getter_AddRefs(profileDir));
519     NS_ENSURE_SUCCESS(rv, rv);
520 
521     nsCOMPtr<nsIFile> databaseFile;
522     rv = profileDir->Clone(getter_AddRefs(databaseFile));
523     NS_ENSURE_SUCCESS(rv, rv);
524     rv = databaseFile->Append(DATABASE_FILENAME);
525     NS_ENSURE_SUCCESS(rv, rv);
526     bool databaseExisted = false;
527     rv = databaseFile->Exists(&databaseExisted);
528     NS_ENSURE_SUCCESS(rv, rv);
529 
530     nsAutoString corruptDbName;
531     if (NS_SUCCEEDED(Preferences::GetString(PREF_FORCE_DATABASE_REPLACEMENT,
532                                             corruptDbName)) &&
533         !corruptDbName.IsEmpty()) {
534       // If this pref is set, maintenance required a database replacement, due
535       // to integrity corruption. Be sure to clear the pref to avoid handling it
536       // more than once.
537       (void)Preferences::ClearUser(PREF_FORCE_DATABASE_REPLACEMENT);
538 
539       // The database is corrupt, backup and replace it with a new one.
540       nsCOMPtr<nsIFile> fileToBeReplaced;
541       bool fileExists = false;
542       if (NS_SUCCEEDED(profileDir->Clone(getter_AddRefs(fileToBeReplaced))) &&
543           NS_SUCCEEDED(fileToBeReplaced->Exists(&fileExists)) && fileExists) {
544         rv = BackupAndReplaceDatabaseFile(storage, corruptDbName, true, false);
545         NS_ENSURE_SUCCESS(rv, rv);
546       }
547     }
548 
549     // Open the database file.  If it does not exist a new one will be created.
550     // Use an unshared connection, it will consume more memory but avoid shared
551     // cache contentions across threads.
552     rv = storage->OpenUnsharedDatabase(databaseFile,
553                                        mozIStorageService::CONNECTION_DEFAULT,
554                                        getter_AddRefs(mMainConn));
555     if (NS_SUCCEEDED(rv) && !databaseExisted) {
556       mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_CREATE;
557     } else if (rv == NS_ERROR_FILE_CORRUPTED) {
558       // The database is corrupt, backup and replace it with a new one.
559       rv = BackupAndReplaceDatabaseFile(storage, DATABASE_FILENAME, true, true);
560       // Fallback to catch-all handler.
561     }
562     NS_ENSURE_SUCCESS(rv, rv);
563 
564     // Initialize the database schema.  In case of failure the existing schema
565     // is is corrupt or incoherent, thus the database should be replaced.
566     bool databaseMigrated = false;
567     rv = SetupDatabaseConnection(storage);
568     bool shouldTryToCloneDb = true;
569     if (NS_SUCCEEDED(rv)) {
570       // Failing to initialize the schema may indicate a corruption.
571       rv = InitSchema(&databaseMigrated);
572       if (NS_FAILED(rv)) {
573         // Cloning the db on a schema migration may not be a good idea, since we
574         // may end up cloning the schema problems.
575         shouldTryToCloneDb = false;
576         if (rv == NS_ERROR_STORAGE_BUSY || rv == NS_ERROR_FILE_IS_LOCKED ||
577             rv == NS_ERROR_FILE_NO_DEVICE_SPACE ||
578             rv == NS_ERROR_OUT_OF_MEMORY) {
579           // The database is not corrupt, though some migration step failed.
580           // This may be caused by concurrent use of sync and async Storage APIs
581           // or by a system issue.
582           // The best we can do is trying again. If it should still fail, Places
583           // won't work properly and will be handled as LOCKED.
584           rv = InitSchema(&databaseMigrated);
585           if (NS_FAILED(rv)) {
586             rv = NS_ERROR_FILE_IS_LOCKED;
587           }
588         } else {
589           rv = NS_ERROR_FILE_CORRUPTED;
590         }
591       }
592     }
593     if (NS_WARN_IF(NS_FAILED(rv))) {
594       if (rv != NS_ERROR_FILE_IS_LOCKED) {
595         mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_CORRUPT;
596       }
597       // Some errors may not indicate a database corruption, for those cases we
598       // just bail out without throwing away a possibly valid places.sqlite.
599       if (rv == NS_ERROR_FILE_CORRUPTED) {
600         // Since we don't know which database is corrupt, we must replace both.
601         rv = BackupAndReplaceDatabaseFile(storage, DATABASE_FAVICONS_FILENAME,
602                                           false, false);
603         NS_ENSURE_SUCCESS(rv, rv);
604         rv = BackupAndReplaceDatabaseFile(storage, DATABASE_FILENAME,
605                                           shouldTryToCloneDb, true);
606         NS_ENSURE_SUCCESS(rv, rv);
607         // Try to initialize the new database again.
608         rv = SetupDatabaseConnection(storage);
609         NS_ENSURE_SUCCESS(rv, rv);
610         rv = InitSchema(&databaseMigrated);
611       }
612       // Bail out if we couldn't fix the database.
613       NS_ENSURE_SUCCESS(rv, rv);
614     }
615 
616     if (databaseMigrated) {
617       mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_UPGRADED;
618     }
619 
620     // Initialize here all the items that are not part of the on-disk database,
621     // like views, temp triggers or temp tables.  The database should not be
622     // considered corrupt if any of the following fails.
623 
624     rv = InitTempEntities();
625     NS_ENSURE_SUCCESS(rv, rv);
626 
627     rv = CheckRoots();
628     NS_ENSURE_SUCCESS(rv, rv);
629 
630     initSucceeded = true;
631   }
632   return NS_OK;
633 }
634 
NotifyConnectionInitalized()635 nsresult Database::NotifyConnectionInitalized() {
636   MOZ_ASSERT(NS_IsMainThread());
637   // Notify about Places initialization.
638   nsCOMArray<nsIObserver> entries;
639   mCacheObservers.GetEntries(entries);
640   for (int32_t idx = 0; idx < entries.Count(); ++idx) {
641     MOZ_ALWAYS_SUCCEEDS(
642         entries[idx]->Observe(nullptr, TOPIC_PLACES_INIT_COMPLETE, nullptr));
643   }
644   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
645   if (obs) {
646     MOZ_ALWAYS_SUCCEEDS(
647         obs->NotifyObservers(nullptr, TOPIC_PLACES_INIT_COMPLETE, nullptr));
648   }
649   return NS_OK;
650 }
651 
EnsureFaviconsDatabaseAttached(const nsCOMPtr<mozIStorageService> & aStorage)652 nsresult Database::EnsureFaviconsDatabaseAttached(
653     const nsCOMPtr<mozIStorageService>& aStorage) {
654   MOZ_ASSERT(NS_IsMainThread());
655 
656   nsCOMPtr<nsIFile> databaseFile;
657   NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
658                          getter_AddRefs(databaseFile));
659   NS_ENSURE_STATE(databaseFile);
660   nsresult rv = databaseFile->Append(DATABASE_FAVICONS_FILENAME);
661   NS_ENSURE_SUCCESS(rv, rv);
662   nsString iconsPath;
663   rv = databaseFile->GetPath(iconsPath);
664   NS_ENSURE_SUCCESS(rv, rv);
665 
666   bool fileExists = false;
667   if (NS_SUCCEEDED(databaseFile->Exists(&fileExists)) && fileExists) {
668     return AttachDatabase(mMainConn, NS_ConvertUTF16toUTF8(iconsPath),
669                           "favicons"_ns);
670   }
671 
672   // Open the database file, this will also create it.
673   nsCOMPtr<mozIStorageConnection> conn;
674   rv = aStorage->OpenUnsharedDatabase(databaseFile,
675                                       mozIStorageService::CONNECTION_DEFAULT,
676                                       getter_AddRefs(conn));
677   NS_ENSURE_SUCCESS(rv, rv);
678 
679   {
680     // Ensure we'll close the connection when done.
681     auto cleanup = MakeScopeExit([&]() {
682       // We cannot use AsyncClose() here, because by the time we try to ATTACH
683       // this database, its transaction could be still be running and that would
684       // cause the ATTACH query to fail.
685       MOZ_ALWAYS_TRUE(NS_SUCCEEDED(conn->Close()));
686     });
687 
688     // Enable incremental vacuum for this database. Since it will contain even
689     // large blobs and can be cleared with history, it's worth to have it.
690     // Note that it will be necessary to manually use PRAGMA incremental_vacuum.
691     rv = conn->ExecuteSimpleSQL("PRAGMA auto_vacuum = INCREMENTAL"_ns);
692     NS_ENSURE_SUCCESS(rv, rv);
693 
694 #if !defined(HAVE_64BIT_BUILD)
695     // Ensure that temp tables are held in memory, not on disk, on 32 bit
696     // platforms.
697     rv = conn->ExecuteSimpleSQL("PRAGMA temp_store = MEMORY"_ns);
698     NS_ENSURE_SUCCESS(rv, rv);
699 #endif
700 
701     int32_t defaultPageSize;
702     rv = conn->GetDefaultPageSize(&defaultPageSize);
703     NS_ENSURE_SUCCESS(rv, rv);
704     rv = SetupDurability(conn, defaultPageSize);
705     NS_ENSURE_SUCCESS(rv, rv);
706 
707     // We are going to update the database, so everything from now on should be
708     // in a transaction for performances.
709     mozStorageTransaction transaction(conn, false);
710     // XXX Handle the error, bug 1696133.
711     Unused << NS_WARN_IF(NS_FAILED(transaction.Start()));
712     rv = conn->ExecuteSimpleSQL(CREATE_MOZ_ICONS);
713     NS_ENSURE_SUCCESS(rv, rv);
714     rv = conn->ExecuteSimpleSQL(CREATE_IDX_MOZ_ICONS_ICONURLHASH);
715     NS_ENSURE_SUCCESS(rv, rv);
716     rv = conn->ExecuteSimpleSQL(CREATE_MOZ_PAGES_W_ICONS);
717     NS_ENSURE_SUCCESS(rv, rv);
718     rv = conn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PAGES_W_ICONS_ICONURLHASH);
719     NS_ENSURE_SUCCESS(rv, rv);
720     rv = conn->ExecuteSimpleSQL(CREATE_MOZ_ICONS_TO_PAGES);
721     NS_ENSURE_SUCCESS(rv, rv);
722     rv = transaction.Commit();
723     NS_ENSURE_SUCCESS(rv, rv);
724 
725     // The scope exit will take care of closing the connection.
726   }
727 
728   rv = AttachDatabase(mMainConn, NS_ConvertUTF16toUTF8(iconsPath),
729                       "favicons"_ns);
730   NS_ENSURE_SUCCESS(rv, rv);
731 
732   return NS_OK;
733 }
734 
BackupAndReplaceDatabaseFile(nsCOMPtr<mozIStorageService> & aStorage,const nsString & aDbFilename,bool aTryToClone,bool aReopenConnection)735 nsresult Database::BackupAndReplaceDatabaseFile(
736     nsCOMPtr<mozIStorageService>& aStorage, const nsString& aDbFilename,
737     bool aTryToClone, bool aReopenConnection) {
738   MOZ_ASSERT(NS_IsMainThread());
739 
740   if (aDbFilename.Equals(DATABASE_FILENAME)) {
741     mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_CORRUPT;
742   } else {
743     // Due to OS file lockings, attached databases can't be cloned properly,
744     // otherwise trying to reattach them later would fail.
745     aTryToClone = false;
746   }
747 
748   nsCOMPtr<nsIFile> profDir;
749   nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
750                                        getter_AddRefs(profDir));
751   NS_ENSURE_SUCCESS(rv, rv);
752   nsCOMPtr<nsIFile> databaseFile;
753   rv = profDir->Clone(getter_AddRefs(databaseFile));
754   NS_ENSURE_SUCCESS(rv, rv);
755   rv = databaseFile->Append(aDbFilename);
756   NS_ENSURE_SUCCESS(rv, rv);
757 
758   // If we already failed in the last 24 hours avoid to create another corrupt
759   // file, since doing so, in some situation, could cause us to create a new
760   // corrupt file at every try to access any Places service.  That is bad
761   // because it would quickly fill the user's disk space without any notice.
762   nsCOMPtr<nsIFile> corruptFile;
763   rv = profDir->Clone(getter_AddRefs(corruptFile));
764   NS_ENSURE_SUCCESS(rv, rv);
765   nsString corruptFilename = getCorruptFilename(aDbFilename);
766   rv = corruptFile->Append(corruptFilename);
767   NS_ENSURE_SUCCESS(rv, rv);
768   if (!isRecentCorruptFile(corruptFile)) {
769     // Ensure we never create more than one corrupt file.
770     nsCOMPtr<nsIFile> corruptFile;
771     rv = profDir->Clone(getter_AddRefs(corruptFile));
772     NS_ENSURE_SUCCESS(rv, rv);
773     nsString corruptFilename = getCorruptFilename(aDbFilename);
774     rv = corruptFile->Append(corruptFilename);
775     NS_ENSURE_SUCCESS(rv, rv);
776     rv = corruptFile->Remove(false);
777     if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
778         rv != NS_ERROR_FILE_NOT_FOUND) {
779       return rv;
780     }
781 
782     nsCOMPtr<nsIFile> backup;
783     Unused << aStorage->BackupDatabaseFile(databaseFile, corruptFilename,
784                                            profDir, getter_AddRefs(backup));
785   }
786 
787   // If anything fails from this point on, we have a stale connection or
788   // database file, and there's not much more we can do.
789   // The only thing we can try to do is to replace the database on the next
790   // startup, and report the problem through telemetry.
791   {
792     enum eCorruptDBReplaceStage : int8_t {
793       stage_closing = 0,
794       stage_removing,
795       stage_reopening,
796       stage_replaced,
797       stage_cloning,
798       stage_cloned
799     };
800     eCorruptDBReplaceStage stage = stage_closing;
801     auto guard = MakeScopeExit([&]() {
802       if (stage != stage_replaced) {
803         // Reaching this point means the database is corrupt and we failed to
804         // replace it.  For this session part of the application related to
805         // bookmarks and history will misbehave.  The frontend may show a
806         // "locked" notification to the user though.
807         // Set up a pref to try replacing the database at the next startup.
808         Preferences::SetString(PREF_FORCE_DATABASE_REPLACEMENT, aDbFilename);
809       }
810       // Report the corruption through telemetry.
811       Telemetry::Accumulate(
812           Telemetry::PLACES_DATABASE_CORRUPTION_HANDLING_STAGE,
813           static_cast<int8_t>(stage));
814     });
815 
816     // Close database connection if open.
817     if (mMainConn) {
818       rv = mMainConn->SpinningSynchronousClose();
819       NS_ENSURE_SUCCESS(rv, rv);
820       mMainConn = nullptr;
821     }
822 
823     // Remove the broken database.
824     stage = stage_removing;
825     rv = databaseFile->Remove(false);
826     if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
827         rv != NS_ERROR_FILE_NOT_FOUND) {
828       return rv;
829     }
830 
831     // Create a new database file and try to clone tables from the corrupt one.
832     bool cloned = false;
833     if (aTryToClone &&
834         Preferences::GetBool(PREF_DATABASE_CLONEONCORRUPTION, true)) {
835       stage = stage_cloning;
836       rv = TryToCloneTablesFromCorruptDatabase(aStorage, databaseFile);
837       if (NS_SUCCEEDED(rv)) {
838         // If we cloned successfully, we should not consider the database
839         // corrupt anymore, otherwise we could reimport default bookmarks.
840         mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_OK;
841         cloned = true;
842       }
843     }
844 
845     if (aReopenConnection) {
846       // Use an unshared connection, it will consume more memory but avoid
847       // shared cache contentions across threads.
848       stage = stage_reopening;
849       rv = aStorage->OpenUnsharedDatabase(
850           databaseFile, mozIStorageService::CONNECTION_DEFAULT,
851           getter_AddRefs(mMainConn));
852       NS_ENSURE_SUCCESS(rv, rv);
853     }
854 
855     stage = cloned ? stage_cloned : stage_replaced;
856   }
857 
858   return NS_OK;
859 }
860 
TryToCloneTablesFromCorruptDatabase(const nsCOMPtr<mozIStorageService> & aStorage,const nsCOMPtr<nsIFile> & aDatabaseFile)861 nsresult Database::TryToCloneTablesFromCorruptDatabase(
862     const nsCOMPtr<mozIStorageService>& aStorage,
863     const nsCOMPtr<nsIFile>& aDatabaseFile) {
864   MOZ_ASSERT(NS_IsMainThread());
865 
866   nsAutoString filename;
867   nsresult rv = aDatabaseFile->GetLeafName(filename);
868 
869   nsCOMPtr<nsIFile> corruptFile;
870   rv = aDatabaseFile->Clone(getter_AddRefs(corruptFile));
871   NS_ENSURE_SUCCESS(rv, rv);
872   rv = corruptFile->SetLeafName(getCorruptFilename(filename));
873   NS_ENSURE_SUCCESS(rv, rv);
874   nsAutoString path;
875   rv = corruptFile->GetPath(path);
876   NS_ENSURE_SUCCESS(rv, rv);
877 
878   nsCOMPtr<nsIFile> recoverFile;
879   rv = aDatabaseFile->Clone(getter_AddRefs(recoverFile));
880   NS_ENSURE_SUCCESS(rv, rv);
881   rv = recoverFile->SetLeafName(getRecoverFilename(filename));
882   NS_ENSURE_SUCCESS(rv, rv);
883   // Ensure there's no previous recover file.
884   rv = recoverFile->Remove(false);
885   if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
886       rv != NS_ERROR_FILE_NOT_FOUND) {
887     return rv;
888   }
889 
890   nsCOMPtr<mozIStorageConnection> conn;
891   auto guard = MakeScopeExit([&]() {
892     if (conn) {
893       Unused << conn->Close();
894     }
895     Unused << recoverFile->Remove(false);
896   });
897 
898   rv = aStorage->OpenUnsharedDatabase(recoverFile,
899                                       mozIStorageService::CONNECTION_DEFAULT,
900                                       getter_AddRefs(conn));
901   NS_ENSURE_SUCCESS(rv, rv);
902   rv = AttachDatabase(conn, NS_ConvertUTF16toUTF8(path), "corrupt"_ns);
903   NS_ENSURE_SUCCESS(rv, rv);
904 
905   mozStorageTransaction transaction(conn, false);
906 
907   // XXX Handle the error, bug 1696133.
908   Unused << NS_WARN_IF(NS_FAILED(transaction.Start()));
909 
910   // Copy the schema version.
911   nsCOMPtr<mozIStorageStatement> stmt;
912   (void)conn->CreateStatement("PRAGMA corrupt.user_version"_ns,
913                               getter_AddRefs(stmt));
914   NS_ENSURE_TRUE(stmt, NS_ERROR_OUT_OF_MEMORY);
915   bool hasResult;
916   rv = stmt->ExecuteStep(&hasResult);
917   NS_ENSURE_SUCCESS(rv, rv);
918   int32_t schemaVersion = stmt->AsInt32(0);
919   rv = conn->SetSchemaVersion(schemaVersion);
920   NS_ENSURE_SUCCESS(rv, rv);
921 
922   // Recreate the tables.
923   rv = conn->CreateStatement(
924       nsLiteralCString(
925           "SELECT name, sql FROM corrupt.sqlite_master "
926           "WHERE type = 'table' AND name BETWEEN 'moz_' AND 'moza'"),
927       getter_AddRefs(stmt));
928   NS_ENSURE_SUCCESS(rv, rv);
929   while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
930     nsAutoCString name;
931     rv = stmt->GetUTF8String(0, name);
932     NS_ENSURE_SUCCESS(rv, rv);
933     nsAutoCString query;
934     rv = stmt->GetUTF8String(1, query);
935     NS_ENSURE_SUCCESS(rv, rv);
936     rv = conn->ExecuteSimpleSQL(query);
937     NS_ENSURE_SUCCESS(rv, rv);
938     // Copy the table contents.
939     rv = conn->ExecuteSimpleSQL("INSERT INTO main."_ns + name +
940                                 " SELECT * FROM corrupt."_ns + name);
941     if (NS_FAILED(rv)) {
942       rv = conn->ExecuteSimpleSQL("INSERT INTO main."_ns + name +
943                                   " SELECT * FROM corrupt."_ns + name +
944                                   " ORDER BY rowid DESC"_ns);
945     }
946     NS_ENSURE_SUCCESS(rv, rv);
947   }
948 
949   // Recreate the indices.  Doing this after data addition is faster.
950   rv = conn->CreateStatement(
951       nsLiteralCString(
952           "SELECT sql FROM corrupt.sqlite_master "
953           "WHERE type <> 'table' AND name BETWEEN 'moz_' AND 'moza'"),
954       getter_AddRefs(stmt));
955   NS_ENSURE_SUCCESS(rv, rv);
956   hasResult = false;
957   while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
958     nsAutoCString query;
959     rv = stmt->GetUTF8String(0, query);
960     NS_ENSURE_SUCCESS(rv, rv);
961     rv = conn->ExecuteSimpleSQL(query);
962     NS_ENSURE_SUCCESS(rv, rv);
963   }
964   rv = stmt->Finalize();
965   NS_ENSURE_SUCCESS(rv, rv);
966 
967   rv = transaction.Commit();
968   NS_ENSURE_SUCCESS(rv, rv);
969 
970   Unused << conn->Close();
971   conn = nullptr;
972   rv = recoverFile->RenameTo(nullptr, filename);
973   NS_ENSURE_SUCCESS(rv, rv);
974   Unused << corruptFile->Remove(false);
975 
976   guard.release();
977   return NS_OK;
978 }
979 
SetupDatabaseConnection(nsCOMPtr<mozIStorageService> & aStorage)980 nsresult Database::SetupDatabaseConnection(
981     nsCOMPtr<mozIStorageService>& aStorage) {
982   MOZ_ASSERT(NS_IsMainThread());
983 
984   // Using immediate transactions allows the main connection to retry writes
985   // that fail with `SQLITE_BUSY` because a cloned connection has locked the
986   // database for writing.
987   nsresult rv = mMainConn->SetDefaultTransactionType(
988       mozIStorageConnection::TRANSACTION_IMMEDIATE);
989   NS_ENSURE_SUCCESS(rv, rv);
990 
991   // WARNING: any statement executed before setting the journal mode must be
992   // finalized, since SQLite doesn't allow changing the journal mode if there
993   // is any outstanding statement.
994 
995   {
996     // Get the page size.  This may be different than the default if the
997     // database file already existed with a different page size.
998     nsCOMPtr<mozIStorageStatement> statement;
999     rv = mMainConn->CreateStatement(
1000         nsLiteralCString(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
1001         getter_AddRefs(statement));
1002     NS_ENSURE_SUCCESS(rv, rv);
1003     bool hasResult = false;
1004     rv = statement->ExecuteStep(&hasResult);
1005     NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FILE_CORRUPTED);
1006     rv = statement->GetInt32(0, &mDBPageSize);
1007     NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && mDBPageSize > 0,
1008                    NS_ERROR_FILE_CORRUPTED);
1009   }
1010 
1011 #if !defined(HAVE_64BIT_BUILD)
1012   // Ensure that temp tables are held in memory, not on disk, on 32 bit
1013   // platforms.
1014   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1015       MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA temp_store = MEMORY"));
1016   NS_ENSURE_SUCCESS(rv, rv);
1017 #endif
1018 
1019   rv = SetupDurability(mMainConn, mDBPageSize);
1020   NS_ENSURE_SUCCESS(rv, rv);
1021 
1022   nsAutoCString busyTimeoutPragma("PRAGMA busy_timeout = ");
1023   busyTimeoutPragma.AppendInt(DATABASE_BUSY_TIMEOUT_MS);
1024   (void)mMainConn->ExecuteSimpleSQL(busyTimeoutPragma);
1025 
1026   // Enable FOREIGN KEY support. This is a strict requirement.
1027   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1028       MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA foreign_keys = ON"));
1029   NS_ENSURE_SUCCESS(rv, NS_ERROR_FILE_CORRUPTED);
1030 #ifdef DEBUG
1031   {
1032     // There are a few cases where setting foreign_keys doesn't work:
1033     //  * in the middle of a multi-statement transaction
1034     //  * if the SQLite library in use doesn't support them
1035     // Since we need foreign_keys, let's at least assert in debug mode.
1036     nsCOMPtr<mozIStorageStatement> stmt;
1037     mMainConn->CreateStatement("PRAGMA foreign_keys"_ns, getter_AddRefs(stmt));
1038     bool hasResult = false;
1039     if (stmt && NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
1040       int32_t fkState = stmt->AsInt32(0);
1041       MOZ_ASSERT(fkState, "Foreign keys should be enabled");
1042     }
1043   }
1044 #endif
1045 
1046   // Attach the favicons database to the main connection.
1047   rv = EnsureFaviconsDatabaseAttached(aStorage);
1048   if (NS_FAILED(rv)) {
1049     // The favicons database may be corrupt. Try to replace and reattach it.
1050     nsCOMPtr<nsIFile> iconsFile;
1051     rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
1052                                 getter_AddRefs(iconsFile));
1053     NS_ENSURE_SUCCESS(rv, rv);
1054     rv = iconsFile->Append(DATABASE_FAVICONS_FILENAME);
1055     NS_ENSURE_SUCCESS(rv, rv);
1056     rv = iconsFile->Remove(false);
1057     if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
1058         rv != NS_ERROR_FILE_NOT_FOUND) {
1059       return rv;
1060     }
1061     rv = EnsureFaviconsDatabaseAttached(aStorage);
1062     NS_ENSURE_SUCCESS(rv, rv);
1063   }
1064 
1065   // Create favicons temp entities.
1066   rv = mMainConn->ExecuteSimpleSQL(CREATE_ICONS_AFTERINSERT_TRIGGER);
1067   NS_ENSURE_SUCCESS(rv, rv);
1068 
1069   // We use our functions during migration, so initialize them now.
1070   rv = InitFunctions();
1071   NS_ENSURE_SUCCESS(rv, rv);
1072 
1073   return NS_OK;
1074 }
1075 
InitSchema(bool * aDatabaseMigrated)1076 nsresult Database::InitSchema(bool* aDatabaseMigrated) {
1077   MOZ_ASSERT(NS_IsMainThread());
1078   *aDatabaseMigrated = false;
1079 
1080   // Get the database schema version.
1081   int32_t currentSchemaVersion;
1082   nsresult rv = mMainConn->GetSchemaVersion(&currentSchemaVersion);
1083   NS_ENSURE_SUCCESS(rv, rv);
1084   bool databaseInitialized = currentSchemaVersion > 0;
1085 
1086   if (databaseInitialized && currentSchemaVersion == DATABASE_SCHEMA_VERSION) {
1087     // The database is up to date and ready to go.
1088     return NS_OK;
1089   }
1090 
1091   auto guard = MakeScopeExit([&]() {
1092     // These run at the end of the migration, out of the transaction,
1093     // regardless of its success.
1094     MigrateV52OriginFrecencies();
1095   });
1096 
1097   // We are going to update the database, so everything from now on should be in
1098   // a transaction for performances.
1099   mozStorageTransaction transaction(mMainConn, false);
1100 
1101   // XXX Handle the error, bug 1696133.
1102   Unused << NS_WARN_IF(NS_FAILED(transaction.Start()));
1103 
1104   if (databaseInitialized) {
1105     // Migration How-to:
1106     //
1107     // 1. increment PLACES_SCHEMA_VERSION.
1108     // 2. implement a method that performs upgrade to your version from the
1109     //    previous one.
1110     //
1111     // NOTE: The downgrade process is pretty much complicated by the fact old
1112     //       versions cannot know what a new version is going to implement.
1113     //       The only thing we will do for downgrades is setting back the schema
1114     //       version, so that next upgrades will run again the migration step.
1115 
1116     if (currentSchemaVersion < DATABASE_SCHEMA_VERSION) {
1117       *aDatabaseMigrated = true;
1118 
1119       if (currentSchemaVersion < 43) {
1120         // These are versions older than Firefox 60 ESR that are not supported
1121         // anymore.  In this case it's safer to just replace the database.
1122         return NS_ERROR_FILE_CORRUPTED;
1123       }
1124 
1125       // Firefox 60 uses schema version 43. - This is an ESR.
1126 
1127       if (currentSchemaVersion < 44) {
1128         rv = MigrateV44Up();
1129         NS_ENSURE_SUCCESS(rv, rv);
1130       }
1131 
1132       if (currentSchemaVersion < 45) {
1133         rv = MigrateV45Up();
1134         NS_ENSURE_SUCCESS(rv, rv);
1135       }
1136 
1137       if (currentSchemaVersion < 46) {
1138         rv = MigrateV46Up();
1139         NS_ENSURE_SUCCESS(rv, rv);
1140       }
1141 
1142       if (currentSchemaVersion < 47) {
1143         rv = MigrateV47Up();
1144         NS_ENSURE_SUCCESS(rv, rv);
1145       }
1146 
1147       // Firefox 61 uses schema version 47.
1148 
1149       if (currentSchemaVersion < 48) {
1150         rv = MigrateV48Up();
1151         NS_ENSURE_SUCCESS(rv, rv);
1152       }
1153 
1154       if (currentSchemaVersion < 49) {
1155         rv = MigrateV49Up();
1156         NS_ENSURE_SUCCESS(rv, rv);
1157       }
1158 
1159       if (currentSchemaVersion < 50) {
1160         rv = MigrateV50Up();
1161         NS_ENSURE_SUCCESS(rv, rv);
1162       }
1163 
1164       if (currentSchemaVersion < 51) {
1165         rv = MigrateV51Up();
1166         NS_ENSURE_SUCCESS(rv, rv);
1167       }
1168 
1169       if (currentSchemaVersion < 52) {
1170         rv = MigrateV52Up();
1171         NS_ENSURE_SUCCESS(rv, rv);
1172       }
1173 
1174       // Firefox 62 uses schema version 52.
1175       // Firefox 68 uses schema version 52. - This is an ESR.
1176 
1177       if (currentSchemaVersion < 53) {
1178         rv = MigrateV53Up();
1179         NS_ENSURE_SUCCESS(rv, rv);
1180       }
1181 
1182       // Firefox 69 uses schema version 53
1183       // Firefox 78 uses schema version 53 - This is an ESR.
1184 
1185       if (currentSchemaVersion < 54) {
1186         rv = MigrateV54Up();
1187         NS_ENSURE_SUCCESS(rv, rv);
1188       }
1189 
1190       // Firefox 81 uses schema version 54
1191 
1192       if (currentSchemaVersion < 55) {
1193         rv = MigrateV55Up();
1194         NS_ENSURE_SUCCESS(rv, rv);
1195       }
1196 
1197       if (currentSchemaVersion < 56) {
1198         rv = MigrateV56Up();
1199         NS_ENSURE_SUCCESS(rv, rv);
1200       }
1201 
1202       if (currentSchemaVersion < 57) {
1203         rv = MigrateV57Up();
1204         NS_ENSURE_SUCCESS(rv, rv);
1205       }
1206 
1207       // Firefox 91 uses schema version 57
1208 
1209       if (currentSchemaVersion < 58) {
1210         rv = MigrateV58Up();
1211         NS_ENSURE_SUCCESS(rv, rv);
1212       }
1213 
1214       // Firefox 92 uses schema version 58
1215 
1216       if (currentSchemaVersion < 59) {
1217         rv = MigrateV59Up();
1218         NS_ENSURE_SUCCESS(rv, rv);
1219       }
1220 
1221       // Firefox 94 uses schema version 59
1222 
1223       if (currentSchemaVersion < 60) {
1224         rv = MigrateV60Up();
1225         NS_ENSURE_SUCCESS(rv, rv);
1226       }
1227 
1228       // Firefox 96 uses schema version 60
1229 
1230       if (currentSchemaVersion < 61) {
1231         rv = MigrateV61Up();
1232         NS_ENSURE_SUCCESS(rv, rv);
1233       }
1234 
1235       if (currentSchemaVersion < 62) {
1236         rv = MigrateV62Up();
1237         NS_ENSURE_SUCCESS(rv, rv);
1238       }
1239 
1240       // Firefox 97 uses schema version 62
1241 
1242       if (currentSchemaVersion < 63) {
1243         rv = MigrateV63Up();
1244         NS_ENSURE_SUCCESS(rv, rv);
1245       }
1246 
1247       // Firefox 98 uses schema version 63
1248 
1249       if (currentSchemaVersion < 64) {
1250         rv = MigrateV64Up();
1251         NS_ENSURE_SUCCESS(rv, rv);
1252       }
1253 
1254       // Firefox 99 uses schema version 64
1255 
1256       // Schema Upgrades must add migration code here.
1257       // >>> IMPORTANT! <<<
1258       // NEVER MIX UP SYNC AND ASYNC EXECUTION IN MIGRATORS, YOU MAY LOCK THE
1259       // CONNECTION AND CAUSE FURTHER STEPS TO FAIL.
1260       // In case, set a bool and do the async work in the ScopeExit guard just
1261       // before the migration steps.
1262     }
1263   } else {
1264     // This is a new database, so we have to create all the tables and indices.
1265 
1266     // moz_origins.
1267     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ORIGINS);
1268     NS_ENSURE_SUCCESS(rv, rv);
1269 
1270     // moz_places.
1271     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES);
1272     NS_ENSURE_SUCCESS(rv, rv);
1273     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_URL_HASH);
1274     NS_ENSURE_SUCCESS(rv, rv);
1275     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_REVHOST);
1276     NS_ENSURE_SUCCESS(rv, rv);
1277     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_VISITCOUNT);
1278     NS_ENSURE_SUCCESS(rv, rv);
1279     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_FRECENCY);
1280     NS_ENSURE_SUCCESS(rv, rv);
1281     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_LASTVISITDATE);
1282     NS_ENSURE_SUCCESS(rv, rv);
1283     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_GUID);
1284     NS_ENSURE_SUCCESS(rv, rv);
1285     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_ORIGIN_ID);
1286     NS_ENSURE_SUCCESS(rv, rv);
1287 
1288     // moz_historyvisits.
1289     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_HISTORYVISITS);
1290     NS_ENSURE_SUCCESS(rv, rv);
1291     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_PLACEDATE);
1292     NS_ENSURE_SUCCESS(rv, rv);
1293     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_FROMVISIT);
1294     NS_ENSURE_SUCCESS(rv, rv);
1295     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_VISITDATE);
1296     NS_ENSURE_SUCCESS(rv, rv);
1297 
1298     // moz_inputhistory.
1299     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_INPUTHISTORY);
1300     NS_ENSURE_SUCCESS(rv, rv);
1301 
1302     // moz_bookmarks.
1303     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_BOOKMARKS);
1304     NS_ENSURE_SUCCESS(rv, rv);
1305     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_BOOKMARKS_DELETED);
1306     NS_ENSURE_SUCCESS(rv, rv);
1307     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_PLACETYPE);
1308     NS_ENSURE_SUCCESS(rv, rv);
1309     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_PARENTPOSITION);
1310     NS_ENSURE_SUCCESS(rv, rv);
1311     rv =
1312         mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_PLACELASTMODIFIED);
1313     NS_ENSURE_SUCCESS(rv, rv);
1314     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_DATEADDED);
1315     NS_ENSURE_SUCCESS(rv, rv);
1316     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_GUID);
1317     NS_ENSURE_SUCCESS(rv, rv);
1318 
1319     // moz_keywords.
1320     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_KEYWORDS);
1321     NS_ENSURE_SUCCESS(rv, rv);
1322     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_KEYWORDS_PLACEPOSTDATA);
1323     NS_ENSURE_SUCCESS(rv, rv);
1324 
1325     // moz_anno_attributes.
1326     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ANNO_ATTRIBUTES);
1327     NS_ENSURE_SUCCESS(rv, rv);
1328 
1329     // moz_annos.
1330     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ANNOS);
1331     NS_ENSURE_SUCCESS(rv, rv);
1332     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_ANNOS_PLACEATTRIBUTE);
1333     NS_ENSURE_SUCCESS(rv, rv);
1334 
1335     // moz_items_annos.
1336     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ITEMS_ANNOS);
1337     NS_ENSURE_SUCCESS(rv, rv);
1338     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_ITEMSANNOS_PLACEATTRIBUTE);
1339     NS_ENSURE_SUCCESS(rv, rv);
1340 
1341     // moz_meta.
1342     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_META);
1343     NS_ENSURE_SUCCESS(rv, rv);
1344 
1345     // moz_places_metadata
1346     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA);
1347     NS_ENSURE_SUCCESS(rv, rv);
1348     rv = mMainConn->ExecuteSimpleSQL(
1349         CREATE_IDX_MOZ_PLACES_METADATA_PLACECREATED);
1350     NS_ENSURE_SUCCESS(rv, rv);
1351     rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_METADATA_REFERRER);
1352     NS_ENSURE_SUCCESS(rv, rv);
1353 
1354     // moz_places_metadata_search_queries
1355     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SEARCH_QUERIES);
1356     NS_ENSURE_SUCCESS(rv, rv);
1357 
1358     // moz_places_metadata_snapshots
1359     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SNAPSHOTS);
1360     NS_ENSURE_SUCCESS(rv, rv);
1361     rv = mMainConn->ExecuteSimpleSQL(
1362         CREATE_IDX_MOZ_PLACES_METADATA_SNAPSHOTS_PINNNED);
1363     NS_ENSURE_SUCCESS(rv, rv);
1364 
1365     // moz_places_metadata_snapshots_extra
1366     rv =
1367         mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SNAPSHOTS_EXTRA);
1368     NS_ENSURE_SUCCESS(rv, rv);
1369     rv = mMainConn->ExecuteSimpleSQL(
1370         CREATE_IDX_MOZ_PLACES_METADATA_SNAPSHOTS_EXTRA_TYPE);
1371     NS_ENSURE_SUCCESS(rv, rv);
1372 
1373     // moz_places_metadata_snapshots_groups
1374     rv = mMainConn->ExecuteSimpleSQL(
1375         CREATE_MOZ_PLACES_METADATA_SNAPSHOTS_GROUPS);
1376     NS_ENSURE_SUCCESS(rv, rv);
1377     // moz_places_metadata_groups_to_snapshots
1378     rv = mMainConn->ExecuteSimpleSQL(
1379         CREATE_MOZ_PLACES_METADATA_GROUPS_TO_SNAPSHOTS);
1380     NS_ENSURE_SUCCESS(rv, rv);
1381 
1382     // moz_session_metadata
1383     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_SESSION_METADATA);
1384     NS_ENSURE_SUCCESS(rv, rv);
1385 
1386     // moz_session_to_places
1387     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_SESSION_TO_PLACES);
1388     NS_ENSURE_SUCCESS(rv, rv);
1389 
1390     // moz_previews_tombstones
1391     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PREVIEWS_TOMBSTONES);
1392     NS_ENSURE_SUCCESS(rv, rv);
1393 
1394     // The bookmarks roots get initialized in CheckRoots().
1395   }
1396 
1397   // Set the schema version to the current one.
1398   rv = mMainConn->SetSchemaVersion(DATABASE_SCHEMA_VERSION);
1399   NS_ENSURE_SUCCESS(rv, rv);
1400 
1401   rv = transaction.Commit();
1402   NS_ENSURE_SUCCESS(rv, rv);
1403 
1404   // ANY FAILURE IN THIS METHOD WILL CAUSE US TO MARK THE DATABASE AS CORRUPT
1405   // AND TRY TO REPLACE IT.
1406   // DO NOT PUT HERE ANYTHING THAT IS NOT RELATED TO INITIALIZATION OR MODIFYING
1407   // THE DISK DATABASE.
1408 
1409   return NS_OK;
1410 }
1411 
CheckRoots()1412 nsresult Database::CheckRoots() {
1413   MOZ_ASSERT(NS_IsMainThread());
1414 
1415   // If the database has just been created, skip straight to the part where
1416   // we create the roots.
1417   if (mDatabaseStatus == nsINavHistoryService::DATABASE_STATUS_CREATE) {
1418     return EnsureBookmarkRoots(0, /* shouldReparentRoots */ false);
1419   }
1420 
1421   nsCOMPtr<mozIStorageStatement> stmt;
1422   nsresult rv = mMainConn->CreateStatement(
1423       nsLiteralCString("SELECT guid, id, position, parent FROM moz_bookmarks "
1424                        "WHERE guid IN ( "
1425                        "'" ROOT_GUID "', '" MENU_ROOT_GUID
1426                        "', '" TOOLBAR_ROOT_GUID "', "
1427                        "'" TAGS_ROOT_GUID "', '" UNFILED_ROOT_GUID
1428                        "', '" MOBILE_ROOT_GUID "' )"),
1429       getter_AddRefs(stmt));
1430   NS_ENSURE_SUCCESS(rv, rv);
1431 
1432   bool hasResult;
1433   nsAutoCString guid;
1434   int32_t maxPosition = 0;
1435   bool shouldReparentRoots = false;
1436   while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
1437     rv = stmt->GetUTF8String(0, guid);
1438     NS_ENSURE_SUCCESS(rv, rv);
1439 
1440     int64_t parentId = stmt->AsInt64(3);
1441 
1442     if (guid.EqualsLiteral(ROOT_GUID)) {
1443       mRootId = stmt->AsInt64(1);
1444       shouldReparentRoots |= parentId != 0;
1445     } else {
1446       maxPosition = std::max(stmt->AsInt32(2), maxPosition);
1447 
1448       if (guid.EqualsLiteral(MENU_ROOT_GUID)) {
1449         mMenuRootId = stmt->AsInt64(1);
1450       } else if (guid.EqualsLiteral(TOOLBAR_ROOT_GUID)) {
1451         mToolbarRootId = stmt->AsInt64(1);
1452       } else if (guid.EqualsLiteral(TAGS_ROOT_GUID)) {
1453         mTagsRootId = stmt->AsInt64(1);
1454       } else if (guid.EqualsLiteral(UNFILED_ROOT_GUID)) {
1455         mUnfiledRootId = stmt->AsInt64(1);
1456       } else if (guid.EqualsLiteral(MOBILE_ROOT_GUID)) {
1457         mMobileRootId = stmt->AsInt64(1);
1458       }
1459       shouldReparentRoots |= parentId != mRootId;
1460     }
1461   }
1462 
1463   rv = EnsureBookmarkRoots(maxPosition + 1, shouldReparentRoots);
1464   NS_ENSURE_SUCCESS(rv, rv);
1465 
1466   return NS_OK;
1467 }
1468 
EnsureBookmarkRoots(const int32_t startPosition,bool shouldReparentRoots)1469 nsresult Database::EnsureBookmarkRoots(const int32_t startPosition,
1470                                        bool shouldReparentRoots) {
1471   MOZ_ASSERT(NS_IsMainThread());
1472 
1473   nsresult rv;
1474 
1475   if (mRootId < 1) {
1476     // The first root's title is an empty string.
1477     rv = CreateRoot(mMainConn, "places"_ns, "root________"_ns, ""_ns, 0,
1478                     mRootId);
1479 
1480     if (NS_FAILED(rv)) return rv;
1481   }
1482 
1483   int32_t position = startPosition;
1484 
1485   // For the other roots, the UI doesn't rely on the value in the database, so
1486   // just set it to something simple to make it easier for humans to read.
1487   if (mMenuRootId < 1) {
1488     rv = CreateRoot(mMainConn, "menu"_ns, "menu________"_ns, "menu"_ns,
1489                     position, mMenuRootId);
1490     if (NS_FAILED(rv)) return rv;
1491     position++;
1492   }
1493 
1494   if (mToolbarRootId < 1) {
1495     rv = CreateRoot(mMainConn, "toolbar"_ns, "toolbar_____"_ns, "toolbar"_ns,
1496                     position, mToolbarRootId);
1497     if (NS_FAILED(rv)) return rv;
1498     position++;
1499   }
1500 
1501   if (mTagsRootId < 1) {
1502     rv = CreateRoot(mMainConn, "tags"_ns, "tags________"_ns, "tags"_ns,
1503                     position, mTagsRootId);
1504     if (NS_FAILED(rv)) return rv;
1505     position++;
1506   }
1507 
1508   if (mUnfiledRootId < 1) {
1509     rv = CreateRoot(mMainConn, "unfiled"_ns, "unfiled_____"_ns, "unfiled"_ns,
1510                     position, mUnfiledRootId);
1511     if (NS_FAILED(rv)) return rv;
1512     position++;
1513   }
1514 
1515   if (mMobileRootId < 1) {
1516     int64_t mobileRootId = CreateMobileRoot();
1517     if (mobileRootId <= 0) return NS_ERROR_FAILURE;
1518     {
1519       nsCOMPtr<mozIStorageStatement> mobileRootSyncStatusStmt;
1520       rv = mMainConn->CreateStatement(
1521           nsLiteralCString("UPDATE moz_bookmarks SET syncStatus = "
1522                            ":sync_status WHERE id = :id"),
1523           getter_AddRefs(mobileRootSyncStatusStmt));
1524       if (NS_FAILED(rv)) return rv;
1525 
1526       rv = mobileRootSyncStatusStmt->BindInt32ByName(
1527           "sync_status"_ns, nsINavBookmarksService::SYNC_STATUS_NEW);
1528       if (NS_FAILED(rv)) return rv;
1529       rv = mobileRootSyncStatusStmt->BindInt64ByName("id"_ns, mobileRootId);
1530       if (NS_FAILED(rv)) return rv;
1531 
1532       rv = mobileRootSyncStatusStmt->Execute();
1533       if (NS_FAILED(rv)) return rv;
1534 
1535       mMobileRootId = mobileRootId;
1536     }
1537   }
1538 
1539   if (!shouldReparentRoots) {
1540     return NS_OK;
1541   }
1542 
1543   // At least one root had the wrong parent, so we need to ensure that
1544   // all roots are parented correctly, fix their positions, and bump the
1545   // Sync change counter.
1546   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1547       "CREATE TEMP TRIGGER moz_ensure_bookmark_roots_trigger "
1548       "AFTER UPDATE OF parent ON moz_bookmarks FOR EACH ROW "
1549       "WHEN OLD.parent <> NEW.parent "
1550       "BEGIN "
1551       "UPDATE moz_bookmarks SET "
1552       "syncChangeCounter = syncChangeCounter + 1 "
1553       "WHERE id IN (OLD.parent, NEW.parent, NEW.id); "
1554 
1555       "UPDATE moz_bookmarks SET "
1556       "position = position - 1 "
1557       "WHERE parent = OLD.parent AND position >= OLD.position; "
1558 
1559       // Fix the positions of the root's old siblings. Since we've already
1560       // moved the root, we need to exclude it from the subquery.
1561       "UPDATE moz_bookmarks SET "
1562       "position = IFNULL((SELECT MAX(position) + 1 FROM moz_bookmarks "
1563       "WHERE parent = NEW.parent AND "
1564       "id <> NEW.id), 0)"
1565       "WHERE id = NEW.id; "
1566       "END"));
1567   if (NS_FAILED(rv)) return rv;
1568   auto guard = MakeScopeExit([&]() {
1569     Unused << mMainConn->ExecuteSimpleSQL(
1570         "DROP TRIGGER moz_ensure_bookmark_roots_trigger"_ns);
1571   });
1572 
1573   nsCOMPtr<mozIStorageStatement> reparentStmt;
1574   rv = mMainConn->CreateStatement(
1575       nsLiteralCString(
1576           "UPDATE moz_bookmarks SET "
1577           "parent = CASE id WHEN :root_id THEN 0 ELSE :root_id END "
1578           "WHERE id IN (:root_id, :menu_root_id, :toolbar_root_id, "
1579           ":tags_root_id, "
1580           ":unfiled_root_id, :mobile_root_id)"),
1581       getter_AddRefs(reparentStmt));
1582   if (NS_FAILED(rv)) return rv;
1583 
1584   rv = reparentStmt->BindInt64ByName("root_id"_ns, mRootId);
1585   if (NS_FAILED(rv)) return rv;
1586   rv = reparentStmt->BindInt64ByName("menu_root_id"_ns, mMenuRootId);
1587   if (NS_FAILED(rv)) return rv;
1588   rv = reparentStmt->BindInt64ByName("toolbar_root_id"_ns, mToolbarRootId);
1589   if (NS_FAILED(rv)) return rv;
1590   rv = reparentStmt->BindInt64ByName("tags_root_id"_ns, mTagsRootId);
1591   if (NS_FAILED(rv)) return rv;
1592   rv = reparentStmt->BindInt64ByName("unfiled_root_id"_ns, mUnfiledRootId);
1593   if (NS_FAILED(rv)) return rv;
1594   rv = reparentStmt->BindInt64ByName("mobile_root_id"_ns, mMobileRootId);
1595   if (NS_FAILED(rv)) return rv;
1596 
1597   rv = reparentStmt->Execute();
1598   if (NS_FAILED(rv)) return rv;
1599 
1600   return NS_OK;
1601 }
1602 
InitFunctions()1603 nsresult Database::InitFunctions() {
1604   MOZ_ASSERT(NS_IsMainThread());
1605 
1606   nsresult rv = GetUnreversedHostFunction::create(mMainConn);
1607   NS_ENSURE_SUCCESS(rv, rv);
1608   rv = MatchAutoCompleteFunction::create(mMainConn);
1609   NS_ENSURE_SUCCESS(rv, rv);
1610   rv = CalculateFrecencyFunction::create(mMainConn);
1611   NS_ENSURE_SUCCESS(rv, rv);
1612   rv = GenerateGUIDFunction::create(mMainConn);
1613   NS_ENSURE_SUCCESS(rv, rv);
1614   rv = IsValidGUIDFunction::create(mMainConn);
1615   NS_ENSURE_SUCCESS(rv, rv);
1616   rv = FixupURLFunction::create(mMainConn);
1617   NS_ENSURE_SUCCESS(rv, rv);
1618   rv = StoreLastInsertedIdFunction::create(mMainConn);
1619   NS_ENSURE_SUCCESS(rv, rv);
1620   rv = HashFunction::create(mMainConn);
1621   NS_ENSURE_SUCCESS(rv, rv);
1622   rv = GetQueryParamFunction::create(mMainConn);
1623   NS_ENSURE_SUCCESS(rv, rv);
1624   rv = GetPrefixFunction::create(mMainConn);
1625   NS_ENSURE_SUCCESS(rv, rv);
1626   rv = GetHostAndPortFunction::create(mMainConn);
1627   NS_ENSURE_SUCCESS(rv, rv);
1628   rv = StripPrefixAndUserinfoFunction::create(mMainConn);
1629   NS_ENSURE_SUCCESS(rv, rv);
1630   rv = IsFrecencyDecayingFunction::create(mMainConn);
1631   NS_ENSURE_SUCCESS(rv, rv);
1632   rv = NoteSyncChangeFunction::create(mMainConn);
1633   NS_ENSURE_SUCCESS(rv, rv);
1634   rv = InvalidateDaysOfHistoryFunction::create(mMainConn);
1635   NS_ENSURE_SUCCESS(rv, rv);
1636   rv = MD5HexFunction::create(mMainConn);
1637   NS_ENSURE_SUCCESS(rv, rv);
1638 
1639   return NS_OK;
1640 }
1641 
InitTempEntities()1642 nsresult Database::InitTempEntities() {
1643   MOZ_ASSERT(NS_IsMainThread());
1644 
1645   nsresult rv =
1646       mMainConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERINSERT_TRIGGER);
1647   NS_ENSURE_SUCCESS(rv, rv);
1648   rv = mMainConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERDELETE_TRIGGER);
1649   NS_ENSURE_SUCCESS(rv, rv);
1650 
1651   // Add the triggers that update the moz_origins table as necessary.
1652   rv = mMainConn->ExecuteSimpleSQL(CREATE_UPDATEORIGINSINSERT_TEMP);
1653   NS_ENSURE_SUCCESS(rv, rv);
1654   rv = mMainConn->ExecuteSimpleSQL(
1655       CREATE_UPDATEORIGINSINSERT_AFTERDELETE_TRIGGER);
1656   NS_ENSURE_SUCCESS(rv, rv);
1657   rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_AFTERINSERT_TRIGGER);
1658   NS_ENSURE_SUCCESS(rv, rv);
1659   rv = mMainConn->ExecuteSimpleSQL(CREATE_UPDATEORIGINSDELETE_TEMP);
1660   NS_ENSURE_SUCCESS(rv, rv);
1661   rv = mMainConn->ExecuteSimpleSQL(
1662       CREATE_UPDATEORIGINSDELETE_AFTERDELETE_TRIGGER);
1663   NS_ENSURE_SUCCESS(rv, rv);
1664 
1665   if (Preferences::GetBool(PREF_PREVIEWS_ENABLED, false)) {
1666     rv = mMainConn->ExecuteSimpleSQL(
1667         CREATE_PLACES_AFTERDELETE_WPREVIEWS_TRIGGER);
1668     NS_ENSURE_SUCCESS(rv, rv);
1669   } else {
1670     rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_AFTERDELETE_TRIGGER);
1671     NS_ENSURE_SUCCESS(rv, rv);
1672   }
1673 
1674   rv = mMainConn->ExecuteSimpleSQL(CREATE_UPDATEORIGINSUPDATE_TEMP);
1675   NS_ENSURE_SUCCESS(rv, rv);
1676   rv = mMainConn->ExecuteSimpleSQL(
1677       CREATE_UPDATEORIGINSUPDATE_AFTERDELETE_TRIGGER);
1678   NS_ENSURE_SUCCESS(rv, rv);
1679   rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_AFTERUPDATE_FRECENCY_TRIGGER);
1680   NS_ENSURE_SUCCESS(rv, rv);
1681 
1682   rv = mMainConn->ExecuteSimpleSQL(
1683       CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERDELETE_TRIGGER);
1684   NS_ENSURE_SUCCESS(rv, rv);
1685   rv = mMainConn->ExecuteSimpleSQL(
1686       CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERINSERT_TRIGGER);
1687   NS_ENSURE_SUCCESS(rv, rv);
1688   rv = mMainConn->ExecuteSimpleSQL(
1689       CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERUPDATE_TRIGGER);
1690   NS_ENSURE_SUCCESS(rv, rv);
1691 
1692   rv = mMainConn->ExecuteSimpleSQL(
1693       CREATE_KEYWORDS_FOREIGNCOUNT_AFTERDELETE_TRIGGER);
1694   NS_ENSURE_SUCCESS(rv, rv);
1695   rv = mMainConn->ExecuteSimpleSQL(
1696       CREATE_KEYWORDS_FOREIGNCOUNT_AFTERINSERT_TRIGGER);
1697   NS_ENSURE_SUCCESS(rv, rv);
1698   rv = mMainConn->ExecuteSimpleSQL(
1699       CREATE_KEYWORDS_FOREIGNCOUNT_AFTERUPDATE_TRIGGER);
1700   NS_ENSURE_SUCCESS(rv, rv);
1701   rv =
1702       mMainConn->ExecuteSimpleSQL(CREATE_BOOKMARKS_DELETED_AFTERINSERT_TRIGGER);
1703   NS_ENSURE_SUCCESS(rv, rv);
1704   rv =
1705       mMainConn->ExecuteSimpleSQL(CREATE_BOOKMARKS_DELETED_AFTERDELETE_TRIGGER);
1706   NS_ENSURE_SUCCESS(rv, rv);
1707 
1708   rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_METADATA_AFTERINSERT_TRIGGER);
1709   NS_ENSURE_SUCCESS(rv, rv);
1710   rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_METADATA_AFTERDELETE_TRIGGER);
1711   NS_ENSURE_SUCCESS(rv, rv);
1712 
1713   return NS_OK;
1714 }
1715 
MigrateV44Up()1716 nsresult Database::MigrateV44Up() {
1717   // We need to remove any non-builtin roots and their descendants.
1718 
1719   // Install a temp trigger to clean up linked tables when the main
1720   // bookmarks are deleted.
1721   nsresult rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1722       "CREATE TEMP TRIGGER moz_migrate_bookmarks_trigger "
1723       "AFTER DELETE ON moz_bookmarks FOR EACH ROW "
1724       "BEGIN "
1725       // Insert tombstones.
1726       "INSERT OR IGNORE INTO moz_bookmarks_deleted (guid, dateRemoved) "
1727       "VALUES (OLD.guid, strftime('%s', 'now', 'localtime', 'utc') * 1000000); "
1728       // Remove old annotations for the bookmarks.
1729       "DELETE FROM moz_items_annos "
1730       "WHERE item_id = OLD.id; "
1731       // Decrease the foreign_count in moz_places.
1732       "UPDATE moz_places "
1733       "SET foreign_count = foreign_count - 1 "
1734       "WHERE id = OLD.fk; "
1735       "END "));
1736   if (NS_FAILED(rv)) return rv;
1737 
1738   // This trigger listens for moz_places deletes, and updates moz_annos and
1739   // moz_keywords accordingly.
1740   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1741       "CREATE TEMP TRIGGER moz_migrate_annos_trigger "
1742       "AFTER UPDATE ON moz_places FOR EACH ROW "
1743       // Only remove from moz_places if we don't have any remaining keywords
1744       // pointing to this place, and it hasn't been visited. Note: orphan
1745       // keywords are tidied up below.
1746       "WHEN NEW.visit_count = 0 AND "
1747       " NEW.foreign_count = (SELECT COUNT(*) FROM moz_keywords WHERE place_id "
1748       "= NEW.id) "
1749       "BEGIN "
1750       // No more references to the place, so we can delete the place itself.
1751       "DELETE FROM moz_places "
1752       "WHERE id = NEW.id; "
1753       // Delete annotations relating to the place.
1754       "DELETE FROM moz_annos "
1755       "WHERE place_id = NEW.id; "
1756       // Delete keywords relating to the place.
1757       "DELETE FROM moz_keywords "
1758       "WHERE place_id = NEW.id; "
1759       "END "));
1760   if (NS_FAILED(rv)) return rv;
1761 
1762   // Listens to moz_keyword deletions, to ensure moz_places gets the
1763   // foreign_count updated corrrectly.
1764   rv = mMainConn->ExecuteSimpleSQL(
1765       nsLiteralCString("CREATE TEMP TRIGGER moz_migrate_keyword_trigger "
1766                        "AFTER DELETE ON moz_keywords FOR EACH ROW "
1767                        "BEGIN "
1768                        // If we remove a keyword, then reduce the foreign_count.
1769                        "UPDATE moz_places "
1770                        "SET foreign_count = foreign_count - 1 "
1771                        "WHERE id = OLD.place_id; "
1772                        "END "));
1773   if (NS_FAILED(rv)) return rv;
1774 
1775   // First of all, find the non-builtin roots.
1776   nsCOMPtr<mozIStorageStatement> deleteStmt;
1777   rv = mMainConn->CreateStatement(
1778       nsLiteralCString("WITH RECURSIVE "
1779                        "itemsToRemove(id, guid) AS ( "
1780                        "SELECT b.id, b.guid FROM moz_bookmarks b "
1781                        "JOIN moz_bookmarks p ON b.parent = p.id "
1782                        "WHERE p.guid = 'root________' AND "
1783                        "b.guid NOT IN ('menu________', 'toolbar_____', "
1784                        "'tags________', 'unfiled_____', 'mobile______') "
1785                        "UNION ALL "
1786                        "SELECT b.id, b.guid FROM moz_bookmarks b "
1787                        "JOIN itemsToRemove d ON d.id = b.parent "
1788                        "WHERE b.guid NOT IN ('menu________', 'toolbar_____', "
1789                        "'tags________', 'unfiled_____', 'mobile______') "
1790                        ") "
1791                        "DELETE FROM moz_bookmarks "
1792                        "WHERE id IN (SELECT id FROM itemsToRemove) "),
1793       getter_AddRefs(deleteStmt));
1794   if (NS_FAILED(rv)) return rv;
1795 
1796   rv = deleteStmt->Execute();
1797   if (NS_FAILED(rv)) return rv;
1798 
1799   // Before we remove the triggers, check for keywords attached to places which
1800   // no longer have a bookmark to them. We do this before removing the triggers,
1801   // so that we can make use of the keyword trigger to update the counts in
1802   // moz_places.
1803   rv = mMainConn->ExecuteSimpleSQL(
1804       nsLiteralCString("DELETE FROM moz_keywords WHERE place_id IN ( "
1805                        "SELECT h.id FROM moz_keywords k "
1806                        "JOIN moz_places h ON h.id = k.place_id "
1807                        "GROUP BY place_id HAVING h.foreign_count = count(*) "
1808                        ")"));
1809   if (NS_FAILED(rv)) return rv;
1810 
1811   // Now remove the temp triggers.
1812   rv = mMainConn->ExecuteSimpleSQL(
1813       "DROP TRIGGER moz_migrate_bookmarks_trigger "_ns);
1814   if (NS_FAILED(rv)) return rv;
1815   rv =
1816       mMainConn->ExecuteSimpleSQL("DROP TRIGGER moz_migrate_annos_trigger "_ns);
1817   if (NS_FAILED(rv)) return rv;
1818   rv = mMainConn->ExecuteSimpleSQL(
1819       "DROP TRIGGER moz_migrate_keyword_trigger "_ns);
1820   if (NS_FAILED(rv)) return rv;
1821 
1822   // Cleanup any orphan annotation attributes.
1823   rv = mMainConn->ExecuteSimpleSQL(
1824       nsLiteralCString("DELETE FROM moz_anno_attributes WHERE id IN ( "
1825                        "SELECT id FROM moz_anno_attributes n "
1826                        "EXCEPT "
1827                        "SELECT DISTINCT anno_attribute_id FROM moz_annos "
1828                        "EXCEPT "
1829                        "SELECT DISTINCT anno_attribute_id FROM moz_items_annos "
1830                        ")"));
1831   if (NS_FAILED(rv)) return rv;
1832 
1833   return NS_OK;
1834 }
1835 
MigrateV45Up()1836 nsresult Database::MigrateV45Up() {
1837   nsCOMPtr<mozIStorageStatement> metaTableStmt;
1838   nsresult rv = mMainConn->CreateStatement("SELECT 1 FROM moz_meta"_ns,
1839                                            getter_AddRefs(metaTableStmt));
1840   if (NS_FAILED(rv)) {
1841     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_META);
1842     NS_ENSURE_SUCCESS(rv, rv);
1843   }
1844   return NS_OK;
1845 }
1846 
MigrateV46Up()1847 nsresult Database::MigrateV46Up() {
1848   // Convert the existing queries. For simplicity we assume the user didn't
1849   // edit these queries, and just do a 1:1 conversion.
1850   nsresult rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1851       "UPDATE moz_places "
1852       "SET url = IFNULL('place:tag=' || ( "
1853       "SELECT title FROM moz_bookmarks "
1854       "WHERE id = CAST(get_query_param(substr(url, 7), 'folder') AS INT) "
1855       "), url) "
1856       "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND "
1857       "hash('place', 'prefix_hi') "
1858       "AND url LIKE '%type=7%' "
1859       "AND EXISTS(SELECT 1 FROM moz_bookmarks "
1860       "WHERE id = CAST(get_query_param(substr(url, 7), 'folder') AS INT)) "));
1861 
1862   // Recalculate hashes for all tag queries.
1863   rv = mMainConn->ExecuteSimpleSQL(
1864       nsLiteralCString("UPDATE moz_places SET url_hash = hash(url) "
1865                        "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND "
1866                        "hash('place', 'prefix_hi') "
1867                        "AND url LIKE '%tag=%' "));
1868   NS_ENSURE_SUCCESS(rv, rv);
1869 
1870   // Update Sync fields for all tag queries.
1871   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1872       "UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1 "
1873       "WHERE fk IN ( "
1874       "SELECT id FROM moz_places "
1875       "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND "
1876       "hash('place', 'prefix_hi') "
1877       "AND url LIKE '%tag=%' "
1878       ") "));
1879   NS_ENSURE_SUCCESS(rv, rv);
1880   return NS_OK;
1881 }
1882 
MigrateV47Up()1883 nsresult Database::MigrateV47Up() {
1884   // v46 may have mistakenly set some url to NULL, we must fix those.
1885   // Since the original url was an invalid query, we replace NULLs with an
1886   // empty query.
1887   nsresult rv = mMainConn->ExecuteSimpleSQL(
1888       nsLiteralCString("UPDATE moz_places "
1889                        "SET url = 'place:excludeItems=1', url_hash = "
1890                        "hash('place:excludeItems=1') "
1891                        "WHERE url ISNULL "));
1892   NS_ENSURE_SUCCESS(rv, rv);
1893   // Update Sync fields for these queries.
1894   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1895       "UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1 "
1896       "WHERE fk IN ( "
1897       "SELECT id FROM moz_places "
1898       "WHERE url_hash = hash('place:excludeItems=1') "
1899       "AND url = 'place:excludeItems=1' "
1900       ") "));
1901   NS_ENSURE_SUCCESS(rv, rv);
1902   return NS_OK;
1903 }
1904 
MigrateV48Up()1905 nsresult Database::MigrateV48Up() {
1906   // Create and populate moz_origins.
1907   nsCOMPtr<mozIStorageStatement> stmt;
1908   nsresult rv = mMainConn->CreateStatement("SELECT * FROM moz_origins; "_ns,
1909                                            getter_AddRefs(stmt));
1910   if (NS_FAILED(rv)) {
1911     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ORIGINS);
1912     NS_ENSURE_SUCCESS(rv, rv);
1913   }
1914   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1915       "INSERT OR IGNORE INTO moz_origins (prefix, host, frecency) "
1916       "SELECT get_prefix(url), get_host_and_port(url), -1 "
1917       "FROM moz_places; "));
1918   NS_ENSURE_SUCCESS(rv, rv);
1919 
1920   // Add and populate moz_places.origin_id.
1921   rv = mMainConn->CreateStatement("SELECT origin_id FROM moz_places; "_ns,
1922                                   getter_AddRefs(stmt));
1923   if (NS_FAILED(rv)) {
1924     rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1925         "ALTER TABLE moz_places "
1926         "ADD COLUMN origin_id INTEGER REFERENCES moz_origins(id); "));
1927     NS_ENSURE_SUCCESS(rv, rv);
1928   }
1929   rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_ORIGIN_ID);
1930   NS_ENSURE_SUCCESS(rv, rv);
1931   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
1932       "UPDATE moz_places "
1933       "SET origin_id = ( "
1934       "SELECT id FROM moz_origins "
1935       "WHERE prefix = get_prefix(url) AND host = get_host_and_port(url) "
1936       "); "));
1937   NS_ENSURE_SUCCESS(rv, rv);
1938 
1939   // From this point on, nobody should use moz_hosts again.  Empty it so that we
1940   // don't leak the user's history, but don't remove it yet so that the user can
1941   // downgrade.
1942   // This can fail, if moz_hosts doesn't exist anymore, that is what happens in
1943   // case of downgrade+upgrade.
1944   Unused << mMainConn->ExecuteSimpleSQL("DELETE FROM moz_hosts; "_ns);
1945 
1946   return NS_OK;
1947 }
1948 
MigrateV49Up()1949 nsresult Database::MigrateV49Up() {
1950   // These hidden preferences were added along with the v48 migration as part of
1951   // the frecency stats implementation but are now replaced with entries in the
1952   // moz_meta table.
1953   Unused << Preferences::ClearUser("places.frecency.stats.count");
1954   Unused << Preferences::ClearUser("places.frecency.stats.sum");
1955   Unused << Preferences::ClearUser("places.frecency.stats.sumOfSquares");
1956   return NS_OK;
1957 }
1958 
MigrateV50Up()1959 nsresult Database::MigrateV50Up() {
1960   // Convert the existing queries. We don't have REGEX available, so the
1961   // simplest thing to do is to pull the urls out, and process them manually.
1962   nsCOMPtr<mozIStorageStatement> stmt;
1963   nsresult rv = mMainConn->CreateStatement(
1964       nsLiteralCString("SELECT id, url FROM moz_places "
1965                        "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND "
1966                        "hash('place', 'prefix_hi') "
1967                        "AND url LIKE '%folder=%' "),
1968       getter_AddRefs(stmt));
1969   if (NS_FAILED(rv)) return rv;
1970 
1971   AutoTArray<std::pair<int64_t, nsCString>, 32> placeURLs;
1972 
1973   bool hasMore = false;
1974   nsCString url;
1975   while (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) {
1976     int64_t placeId;
1977     rv = stmt->GetInt64(0, &placeId);
1978     if (NS_FAILED(rv)) return rv;
1979     rv = stmt->GetUTF8String(1, url);
1980     if (NS_FAILED(rv)) return rv;
1981 
1982     // XXX(Bug 1631371) Check if this should use a fallible operation as it
1983     // pretended earlier.
1984     placeURLs.AppendElement(std::make_pair(placeId, url));
1985   }
1986 
1987   if (placeURLs.IsEmpty()) {
1988     return NS_OK;
1989   }
1990 
1991   int64_t placeId;
1992   for (uint32_t i = 0; i < placeURLs.Length(); ++i) {
1993     placeId = placeURLs[i].first;
1994     url = placeURLs[i].second;
1995 
1996     rv = ConvertOldStyleQuery(url);
1997     // Something bad happened, and we can't convert it, so just continue.
1998     if (NS_WARN_IF(NS_FAILED(rv))) {
1999       continue;
2000     }
2001 
2002     nsCOMPtr<mozIStorageStatement> updateStmt;
2003     rv = mMainConn->CreateStatement(
2004         nsLiteralCString("UPDATE moz_places "
2005                          "SET url = :url, url_hash = hash(:url) "
2006                          "WHERE id = :placeId "),
2007         getter_AddRefs(updateStmt));
2008     if (NS_FAILED(rv)) return rv;
2009 
2010     rv = URIBinder::Bind(updateStmt, "url"_ns, url);
2011     if (NS_FAILED(rv)) return rv;
2012     rv = updateStmt->BindInt64ByName("placeId"_ns, placeId);
2013     if (NS_FAILED(rv)) return rv;
2014 
2015     rv = updateStmt->Execute();
2016     if (NS_FAILED(rv)) return rv;
2017 
2018     // Update Sync fields for these queries.
2019     nsCOMPtr<mozIStorageStatement> syncStmt;
2020     rv = mMainConn->CreateStatement(
2021         nsLiteralCString("UPDATE moz_bookmarks SET syncChangeCounter = "
2022                          "syncChangeCounter + 1 "
2023                          "WHERE fk = :placeId "),
2024         getter_AddRefs(syncStmt));
2025     if (NS_FAILED(rv)) return rv;
2026 
2027     rv = syncStmt->BindInt64ByName("placeId"_ns, placeId);
2028     if (NS_FAILED(rv)) return rv;
2029 
2030     rv = syncStmt->Execute();
2031     if (NS_FAILED(rv)) return rv;
2032   }
2033 
2034   return NS_OK;
2035 }
2036 
2037 struct StringWriteFunc : public JSONWriteFunc {
2038   nsCString& mCString;
StringWriteFuncmozilla::places::StringWriteFunc2039   explicit StringWriteFunc(nsCString& aCString) : mCString(aCString) {}
Writemozilla::places::StringWriteFunc2040   void Write(const Span<const char>& aStr) override { mCString.Append(aStr); }
2041 };
2042 
MigrateV51Up()2043 nsresult Database::MigrateV51Up() {
2044   nsCOMPtr<mozIStorageStatement> stmt;
2045   nsresult rv = mMainConn->CreateStatement(
2046       nsLiteralCString("SELECT b.guid FROM moz_anno_attributes n "
2047                        "JOIN moz_items_annos a ON n.id = a.anno_attribute_id "
2048                        "JOIN moz_bookmarks b ON a.item_id = b.id "
2049                        "WHERE n.name = :anno_name ORDER BY a.content DESC"),
2050       getter_AddRefs(stmt));
2051   if (NS_FAILED(rv)) {
2052     MOZ_ASSERT(false,
2053                "Should succeed unless item annotations table has been removed");
2054     return NS_OK;
2055   };
2056 
2057   rv = stmt->BindUTF8StringByName("anno_name"_ns, LAST_USED_ANNO);
2058   NS_ENSURE_SUCCESS(rv, rv);
2059 
2060   nsAutoCString json;
2061   JSONWriter jw{MakeUnique<StringWriteFunc>(json)};
2062   jw.StartArrayProperty(nullptr, JSONWriter::SingleLineStyle);
2063 
2064   bool hasAtLeastOne = false;
2065   bool hasMore = false;
2066   uint32_t length;
2067   while (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) {
2068     hasAtLeastOne = true;
2069     const char* stmtString = stmt->AsSharedUTF8String(0, &length);
2070     jw.StringElement(Span<const char>(stmtString, length));
2071   }
2072   jw.EndArray();
2073 
2074   // If we don't have any, just abort early and save the extra work.
2075   if (!hasAtLeastOne) {
2076     return NS_OK;
2077   }
2078 
2079   rv = mMainConn->CreateStatement(
2080       nsLiteralCString("INSERT OR REPLACE INTO moz_meta "
2081                        "VALUES (:key, :value) "),
2082       getter_AddRefs(stmt));
2083   NS_ENSURE_SUCCESS(rv, rv);
2084 
2085   rv = stmt->BindUTF8StringByName("key"_ns, LAST_USED_FOLDERS_META_KEY);
2086   NS_ENSURE_SUCCESS(rv, rv);
2087   rv = stmt->BindUTF8StringByName("value"_ns, json);
2088   NS_ENSURE_SUCCESS(rv, rv);
2089   rv = stmt->Execute();
2090   NS_ENSURE_SUCCESS(rv, rv);
2091 
2092   // Clean up the now redundant annotations.
2093   rv = mMainConn->CreateStatement(
2094       nsLiteralCString(
2095           "DELETE FROM moz_items_annos WHERE anno_attribute_id = "
2096           "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name) "),
2097       getter_AddRefs(stmt));
2098   NS_ENSURE_SUCCESS(rv, rv);
2099   rv = stmt->BindUTF8StringByName("anno_name"_ns, LAST_USED_ANNO);
2100   NS_ENSURE_SUCCESS(rv, rv);
2101   rv = stmt->Execute();
2102   NS_ENSURE_SUCCESS(rv, rv);
2103 
2104   rv = mMainConn->CreateStatement(
2105       nsLiteralCString(
2106           "DELETE FROM moz_anno_attributes WHERE name = :anno_name "),
2107       getter_AddRefs(stmt));
2108   NS_ENSURE_SUCCESS(rv, rv);
2109   rv = stmt->BindUTF8StringByName("anno_name"_ns, LAST_USED_ANNO);
2110   NS_ENSURE_SUCCESS(rv, rv);
2111   rv = stmt->Execute();
2112   NS_ENSURE_SUCCESS(rv, rv);
2113 
2114   return NS_OK;
2115 }
2116 
2117 namespace {
2118 
2119 class MigrateV52OriginFrecenciesRunnable final : public Runnable {
2120  public:
2121   NS_DECL_NSIRUNNABLE
2122   explicit MigrateV52OriginFrecenciesRunnable(mozIStorageConnection* aDBConn);
2123 
2124  private:
2125   nsCOMPtr<mozIStorageConnection> mDBConn;
2126 };
2127 
MigrateV52OriginFrecenciesRunnable(mozIStorageConnection * aDBConn)2128 MigrateV52OriginFrecenciesRunnable::MigrateV52OriginFrecenciesRunnable(
2129     mozIStorageConnection* aDBConn)
2130     : Runnable("places::MigrateV52OriginFrecenciesRunnable"),
2131       mDBConn(aDBConn) {}
2132 
2133 NS_IMETHODIMP
Run()2134 MigrateV52OriginFrecenciesRunnable::Run() {
2135   if (NS_IsMainThread()) {
2136     // Migration done.  Clear the pref.
2137     Unused << Preferences::ClearUser(PREF_MIGRATE_V52_ORIGIN_FRECENCIES);
2138 
2139     // Now that frecencies have been migrated, recalculate the origin frecency
2140     // stats.
2141     nsNavHistory* navHistory = nsNavHistory::GetHistoryService();
2142     NS_ENSURE_STATE(navHistory);
2143     nsresult rv = navHistory->RecalculateOriginFrecencyStats(nullptr);
2144     NS_ENSURE_SUCCESS(rv, rv);
2145 
2146     return NS_OK;
2147   }
2148 
2149   // We do the work in chunks, or the wal journal may grow too much.
2150   nsresult rv = mDBConn->ExecuteSimpleSQL(nsLiteralCString(
2151       "UPDATE moz_origins "
2152       "SET frecency = ( "
2153       "SELECT CAST(TOTAL(frecency) AS INTEGER) "
2154       "FROM moz_places "
2155       "WHERE frecency > 0 AND moz_places.origin_id = moz_origins.id "
2156       ") "
2157       "WHERE id IN ( "
2158       "SELECT id "
2159       "FROM moz_origins "
2160       "WHERE frecency < 0 "
2161       "LIMIT 400 "
2162       ") "));
2163   NS_ENSURE_SUCCESS(rv, rv);
2164 
2165   nsCOMPtr<mozIStorageStatement> selectStmt;
2166   rv = mDBConn->CreateStatement(nsLiteralCString("SELECT 1 "
2167                                                  "FROM moz_origins "
2168                                                  "WHERE frecency < 0 "
2169                                                  "LIMIT 1 "),
2170                                 getter_AddRefs(selectStmt));
2171   NS_ENSURE_SUCCESS(rv, rv);
2172   bool hasResult = false;
2173   rv = selectStmt->ExecuteStep(&hasResult);
2174   NS_ENSURE_SUCCESS(rv, rv);
2175   if (hasResult) {
2176     // There are more results to handle. Re-dispatch to the same thread for the
2177     // next chunk.
2178     return NS_DispatchToCurrentThread(this);
2179   }
2180 
2181   // Re-dispatch to the main-thread to flip the migration pref.
2182   return NS_DispatchToMainThread(this);
2183 }
2184 
2185 }  // namespace
2186 
MigrateV52OriginFrecencies()2187 void Database::MigrateV52OriginFrecencies() {
2188   MOZ_ASSERT(NS_IsMainThread());
2189 
2190   if (!Preferences::GetBool(PREF_MIGRATE_V52_ORIGIN_FRECENCIES)) {
2191     // The migration has already been completed.
2192     return;
2193   }
2194 
2195   RefPtr<MigrateV52OriginFrecenciesRunnable> runnable(
2196       new MigrateV52OriginFrecenciesRunnable(mMainConn));
2197   nsCOMPtr<nsIEventTarget> target(do_GetInterface(mMainConn));
2198   MOZ_ASSERT(target);
2199   if (target) {
2200     Unused << target->Dispatch(runnable, NS_DISPATCH_NORMAL);
2201   }
2202 }
2203 
MigrateV52Up()2204 nsresult Database::MigrateV52Up() {
2205   // Before this migration, moz_origin.frecency is the max frecency of all
2206   // places with the origin.  After this migration, it's the sum of frecencies
2207   // of all places with the origin.
2208   //
2209   // Setting this pref will cause InitSchema to begin async migration, via
2210   // MigrateV52OriginFrecencies.  When that migration is done, origin frecency
2211   // stats are recalculated (see MigrateV52OriginFrecenciesRunnable::Run).
2212   Unused << Preferences::SetBool(PREF_MIGRATE_V52_ORIGIN_FRECENCIES, true);
2213 
2214   // Set all origin frecencies to -1 so that MigrateV52OriginFrecenciesRunnable
2215   // will migrate them.
2216   nsresult rv =
2217       mMainConn->ExecuteSimpleSQL("UPDATE moz_origins SET frecency = -1 "_ns);
2218   NS_ENSURE_SUCCESS(rv, rv);
2219 
2220   // This migration also renames these moz_meta keys that keep track of frecency
2221   // stats.  (That happens when stats are recalculated.)  Delete the old ones.
2222   rv =
2223       mMainConn->ExecuteSimpleSQL(nsLiteralCString("DELETE FROM moz_meta "
2224                                                    "WHERE key IN ( "
2225                                                    "'frecency_count', "
2226                                                    "'frecency_sum', "
2227                                                    "'frecency_sum_of_squares' "
2228                                                    ") "));
2229   NS_ENSURE_SUCCESS(rv, rv);
2230 
2231   return NS_OK;
2232 }
2233 
MigrateV53Up()2234 nsresult Database::MigrateV53Up() {
2235   nsCOMPtr<mozIStorageStatement> stmt;
2236   nsresult rv = mMainConn->CreateStatement("SELECT 1 FROM moz_items_annos"_ns,
2237                                            getter_AddRefs(stmt));
2238   if (NS_FAILED(rv)) {
2239     // Likely we removed the table.
2240     return NS_OK;
2241   }
2242 
2243   // Remove all item annotations but SYNC_PARENT_ANNO.
2244   rv = mMainConn->CreateStatement(
2245       nsLiteralCString(
2246           "DELETE FROM moz_items_annos "
2247           "WHERE anno_attribute_id NOT IN ( "
2248           "  SELECT id FROM moz_anno_attributes WHERE name = :anno_name "
2249           ") "),
2250       getter_AddRefs(stmt));
2251   NS_ENSURE_SUCCESS(rv, rv);
2252   rv = stmt->BindUTF8StringByName("anno_name"_ns,
2253                                   nsLiteralCString(SYNC_PARENT_ANNO));
2254   NS_ENSURE_SUCCESS(rv, rv);
2255   rv = stmt->Execute();
2256   NS_ENSURE_SUCCESS(rv, rv);
2257 
2258   rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString(
2259       "DELETE FROM moz_anno_attributes WHERE id IN ( "
2260       "  SELECT id FROM moz_anno_attributes "
2261       "  EXCEPT "
2262       "  SELECT DISTINCT anno_attribute_id FROM moz_annos "
2263       "  EXCEPT "
2264       "  SELECT DISTINCT anno_attribute_id FROM moz_items_annos "
2265       ")"));
2266   NS_ENSURE_SUCCESS(rv, rv);
2267 
2268   return NS_OK;
2269 }
2270 
MigrateV54Up()2271 nsresult Database::MigrateV54Up() {
2272   // Add an expiration column to moz_icons_to_pages.
2273   nsCOMPtr<mozIStorageStatement> stmt;
2274   nsresult rv = mMainConn->CreateStatement(
2275       "SELECT expire_ms FROM moz_icons_to_pages"_ns, getter_AddRefs(stmt));
2276   if (NS_FAILED(rv)) {
2277     rv = mMainConn->ExecuteSimpleSQL(
2278         "ALTER TABLE moz_icons_to_pages "
2279         "ADD COLUMN expire_ms INTEGER NOT NULL DEFAULT 0 "_ns);
2280     NS_ENSURE_SUCCESS(rv, rv);
2281   }
2282 
2283   // Set all the zero-ed entries as expired today, they won't be removed until
2284   // the next related page load.
2285   rv = mMainConn->ExecuteSimpleSQL(
2286       "UPDATE moz_icons_to_pages "
2287       "SET expire_ms = strftime('%s','now','localtime','start "
2288       "of day','utc') * 1000 "
2289       "WHERE expire_ms = 0 "_ns);
2290   NS_ENSURE_SUCCESS(rv, rv);
2291 
2292   return NS_OK;
2293 }
2294 
MigrateV55Up()2295 nsresult Database::MigrateV55Up() {
2296   // Add places metadata tables.
2297   nsCOMPtr<mozIStorageStatement> stmt;
2298   nsresult rv = mMainConn->CreateStatement(
2299       "SELECT id FROM moz_places_metadata"_ns, getter_AddRefs(stmt));
2300   if (NS_FAILED(rv)) {
2301     // Create the tables.
2302     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA);
2303     NS_ENSURE_SUCCESS(rv, rv);
2304     // moz_places_metadata_search_queries.
2305     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SEARCH_QUERIES);
2306     NS_ENSURE_SUCCESS(rv, rv);
2307   }
2308 
2309   return NS_OK;
2310 }
2311 
MigrateV56Up()2312 nsresult Database::MigrateV56Up() {
2313   // Add places metadata (place_id, created_at) index.
2314   return mMainConn->ExecuteSimpleSQL(
2315       CREATE_IDX_MOZ_PLACES_METADATA_PLACECREATED);
2316 }
2317 
MigrateV57Up()2318 nsresult Database::MigrateV57Up() {
2319   // Add the scrolling columns to the metadata.
2320   nsCOMPtr<mozIStorageStatement> stmt;
2321   nsresult rv = mMainConn->CreateStatement(
2322       "SELECT scrolling_time FROM moz_places_metadata"_ns,
2323       getter_AddRefs(stmt));
2324   if (NS_FAILED(rv)) {
2325     rv = mMainConn->ExecuteSimpleSQL(
2326         "ALTER TABLE moz_places_metadata "
2327         "ADD COLUMN scrolling_time INTEGER NOT NULL DEFAULT 0 "_ns);
2328     NS_ENSURE_SUCCESS(rv, rv);
2329   }
2330 
2331   rv = mMainConn->CreateStatement(
2332       "SELECT scrolling_distance FROM moz_places_metadata"_ns,
2333       getter_AddRefs(stmt));
2334   if (NS_FAILED(rv)) {
2335     rv = mMainConn->ExecuteSimpleSQL(
2336         "ALTER TABLE moz_places_metadata "
2337         "ADD COLUMN scrolling_distance INTEGER NOT NULL DEFAULT 0 "_ns);
2338     NS_ENSURE_SUCCESS(rv, rv);
2339   }
2340   return NS_OK;
2341 }
2342 
MigrateV58Up()2343 nsresult Database::MigrateV58Up() {
2344   // Add metadata snapshots tables if necessary.
2345   nsCOMPtr<mozIStorageStatement> stmt;
2346   nsresult rv = mMainConn->CreateStatement(
2347       "SELECT id FROM moz_places_metadata_snapshots"_ns, getter_AddRefs(stmt));
2348   if (NS_FAILED(rv)) {
2349     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SNAPSHOTS);
2350     NS_ENSURE_SUCCESS(rv, rv);
2351     rv =
2352         mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SNAPSHOTS_EXTRA);
2353     NS_ENSURE_SUCCESS(rv, rv);
2354     rv = mMainConn->ExecuteSimpleSQL(
2355         CREATE_MOZ_PLACES_METADATA_SNAPSHOTS_GROUPS);
2356     NS_ENSURE_SUCCESS(rv, rv);
2357     rv = mMainConn->ExecuteSimpleSQL(
2358         CREATE_MOZ_PLACES_METADATA_GROUPS_TO_SNAPSHOTS);
2359     NS_ENSURE_SUCCESS(rv, rv);
2360   }
2361   return NS_OK;
2362 }
2363 
MigrateV59Up()2364 nsresult Database::MigrateV59Up() {
2365   // Add metadata snapshots tables if necessary.
2366   nsCOMPtr<mozIStorageStatement> stmt;
2367   nsresult rv = mMainConn->CreateStatement(
2368       "SELECT id FROM moz_session_metadata"_ns, getter_AddRefs(stmt));
2369   if (NS_FAILED(rv)) {
2370     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_SESSION_METADATA);
2371     NS_ENSURE_SUCCESS(rv, rv);
2372     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_SESSION_TO_PLACES);
2373     NS_ENSURE_SUCCESS(rv, rv);
2374   }
2375   return NS_OK;
2376 }
2377 
MigrateV60Up()2378 nsresult Database::MigrateV60Up() {
2379   // Add the site_name column to moz_places.
2380   nsCOMPtr<mozIStorageStatement> stmt;
2381   nsresult rv = mMainConn->CreateStatement(
2382       "SELECT site_name FROM moz_places"_ns, getter_AddRefs(stmt));
2383   if (NS_FAILED(rv)) {
2384     rv = mMainConn->ExecuteSimpleSQL(
2385         "ALTER TABLE moz_places ADD COLUMN site_name TEXT"_ns);
2386     NS_ENSURE_SUCCESS(rv, rv);
2387   }
2388   return NS_OK;
2389 }
2390 
MigrateV61Up()2391 nsresult Database::MigrateV61Up() {
2392   // Add previews tombstones table if necessary.
2393   nsCOMPtr<mozIStorageStatement> stmt;
2394   nsresult rv = mMainConn->CreateStatement(
2395       "SELECT hash FROM moz_previews_tombstones"_ns, getter_AddRefs(stmt));
2396   if (NS_FAILED(rv)) {
2397     rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PREVIEWS_TOMBSTONES);
2398     NS_ENSURE_SUCCESS(rv, rv);
2399   }
2400   return NS_OK;
2401 }
2402 
MigrateV62Up()2403 nsresult Database::MigrateV62Up() {
2404   // Add builder columns if necessary.
2405   nsCOMPtr<mozIStorageStatement> stmt;
2406   nsresult rv = mMainConn->CreateStatement(
2407       "SELECT builder FROM moz_places_metadata_snapshots_groups"_ns,
2408       getter_AddRefs(stmt));
2409   if (NS_FAILED(rv)) {
2410     rv = mMainConn->ExecuteSimpleSQL(
2411         "ALTER TABLE moz_places_metadata_snapshots_groups "
2412         "ADD COLUMN builder TEXT NOT NULL "_ns);
2413     NS_ENSURE_SUCCESS(rv, rv);
2414     rv = mMainConn->ExecuteSimpleSQL(
2415         "ALTER TABLE moz_places_metadata_snapshots_groups "
2416         "ADD COLUMN builder_data TEXT "_ns);
2417     NS_ENSURE_SUCCESS(rv, rv);
2418   }
2419 
2420   // Add indexes if necessary.
2421   rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_METADATA_REFERRER);
2422   NS_ENSURE_SUCCESS(rv, rv);
2423   rv = mMainConn->ExecuteSimpleSQL(
2424       CREATE_IDX_MOZ_PLACES_METADATA_SNAPSHOTS_PINNNED);
2425   NS_ENSURE_SUCCESS(rv, rv);
2426   rv = mMainConn->ExecuteSimpleSQL(
2427       CREATE_IDX_MOZ_PLACES_METADATA_SNAPSHOTS_EXTRA_TYPE);
2428   NS_ENSURE_SUCCESS(rv, rv);
2429 
2430   return NS_OK;
2431 }
2432 
MigrateV63Up()2433 nsresult Database::MigrateV63Up() {
2434   // Add title column to snapshots if necessary.
2435   nsCOMPtr<mozIStorageStatement> stmt;
2436   nsresult rv = mMainConn->CreateStatement(
2437       "SELECT title FROM moz_places_metadata_snapshots"_ns,
2438       getter_AddRefs(stmt));
2439   if (NS_FAILED(rv)) {
2440     rv = mMainConn->ExecuteSimpleSQL(
2441         "ALTER TABLE moz_places_metadata_snapshots "
2442         "ADD COLUMN title TEXT "_ns);
2443     NS_ENSURE_SUCCESS(rv, rv);
2444   }
2445 
2446   return NS_OK;
2447 }
2448 
MigrateV64Up()2449 nsresult Database::MigrateV64Up() {
2450   // Add hidden column to snapshot groups if necessary.
2451   nsCOMPtr<mozIStorageStatement> stmt;
2452   nsresult rv = mMainConn->CreateStatement(
2453       "SELECT hidden FROM moz_places_metadata_snapshots_groups"_ns,
2454       getter_AddRefs(stmt));
2455   if (NS_FAILED(rv)) {
2456     rv = mMainConn->ExecuteSimpleSQL(
2457         "ALTER TABLE moz_places_metadata_snapshots_groups "
2458         "ADD COLUMN hidden INTEGER DEFAULT 0 NOT NULL "_ns);
2459     NS_ENSURE_SUCCESS(rv, rv);
2460   }
2461 
2462   return NS_OK;
2463 }
2464 
ConvertOldStyleQuery(nsCString & aURL)2465 nsresult Database::ConvertOldStyleQuery(nsCString& aURL) {
2466   AutoTArray<QueryKeyValuePair, 8> tokens;
2467   nsresult rv = TokenizeQueryString(aURL, &tokens);
2468   NS_ENSURE_SUCCESS(rv, rv);
2469 
2470   AutoTArray<QueryKeyValuePair, 8> newTokens;
2471   bool invalid = false;
2472   nsAutoCString guid;
2473 
2474   for (uint32_t j = 0; j < tokens.Length(); ++j) {
2475     const QueryKeyValuePair& kvp = tokens[j];
2476 
2477     if (!kvp.key.EqualsLiteral("folder")) {
2478       // XXX(Bug 1631371) Check if this should use a fallible operation as it
2479       // pretended earlier.
2480       newTokens.AppendElement(kvp);
2481       continue;
2482     }
2483 
2484     int64_t itemId = kvp.value.ToInteger(&rv);
2485     if (NS_SUCCEEDED(rv)) {
2486       // We have the folder's ID, now to find its GUID.
2487       nsCOMPtr<mozIStorageStatement> stmt;
2488       nsresult rv = mMainConn->CreateStatement(
2489           nsLiteralCString("SELECT guid FROM moz_bookmarks "
2490                            "WHERE id = :itemId "),
2491           getter_AddRefs(stmt));
2492       if (NS_FAILED(rv)) return rv;
2493 
2494       rv = stmt->BindInt64ByName("itemId"_ns, itemId);
2495       if (NS_FAILED(rv)) return rv;
2496 
2497       bool hasMore = false;
2498       if (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) {
2499         rv = stmt->GetUTF8String(0, guid);
2500         if (NS_FAILED(rv)) return rv;
2501       }
2502     } else if (kvp.value.EqualsLiteral("PLACES_ROOT")) {
2503       guid = nsLiteralCString(ROOT_GUID);
2504     } else if (kvp.value.EqualsLiteral("BOOKMARKS_MENU")) {
2505       guid = nsLiteralCString(MENU_ROOT_GUID);
2506     } else if (kvp.value.EqualsLiteral("TAGS")) {
2507       guid = nsLiteralCString(TAGS_ROOT_GUID);
2508     } else if (kvp.value.EqualsLiteral("UNFILED_BOOKMARKS")) {
2509       guid = nsLiteralCString(UNFILED_ROOT_GUID);
2510     } else if (kvp.value.EqualsLiteral("TOOLBAR")) {
2511       guid = nsLiteralCString(TOOLBAR_ROOT_GUID);
2512     } else if (kvp.value.EqualsLiteral("MOBILE_BOOKMARKS")) {
2513       guid = nsLiteralCString(MOBILE_ROOT_GUID);
2514     }
2515 
2516     QueryKeyValuePair* newPair;
2517     if (guid.IsEmpty()) {
2518       // This is invalid, so we'll change this key/value pair to something else
2519       // so that the query remains a valid url.
2520       newPair = new QueryKeyValuePair("invalidOldParentId"_ns, kvp.value);
2521       invalid = true;
2522     } else {
2523       newPair = new QueryKeyValuePair("parent"_ns, guid);
2524     }
2525     // XXX(Bug 1631371) Check if this should use a fallible operation as it
2526     // pretended earlier.
2527     newTokens.AppendElement(*newPair);
2528     delete newPair;
2529   }
2530 
2531   if (invalid) {
2532     // One or more of the folders don't exist, replace with an empty query.
2533     newTokens.AppendElement(QueryKeyValuePair("excludeItems"_ns, "1"_ns));
2534   }
2535 
2536   TokensToQueryString(newTokens, aURL);
2537   return NS_OK;
2538 }
2539 
CreateMobileRoot()2540 int64_t Database::CreateMobileRoot() {
2541   MOZ_ASSERT(NS_IsMainThread());
2542 
2543   // Create the mobile root, ignoring conflicts if one already exists (for
2544   // example, if the user downgraded to an earlier release channel).
2545   nsCOMPtr<mozIStorageStatement> createStmt;
2546   nsresult rv = mMainConn->CreateStatement(
2547       nsLiteralCString(
2548           "INSERT OR IGNORE INTO moz_bookmarks "
2549           "(type, title, dateAdded, lastModified, guid, position, parent) "
2550           "SELECT :item_type, :item_title, :timestamp, :timestamp, :guid, "
2551           "IFNULL((SELECT MAX(position) + 1 FROM moz_bookmarks p WHERE "
2552           "p.parent = b.id), 0), b.id "
2553           "FROM moz_bookmarks b WHERE b.parent = 0"),
2554       getter_AddRefs(createStmt));
2555   if (NS_FAILED(rv)) return -1;
2556 
2557   rv = createStmt->BindInt32ByName("item_type"_ns,
2558                                    nsINavBookmarksService::TYPE_FOLDER);
2559   if (NS_FAILED(rv)) return -1;
2560   rv = createStmt->BindUTF8StringByName("item_title"_ns,
2561                                         nsLiteralCString(MOBILE_ROOT_TITLE));
2562   if (NS_FAILED(rv)) return -1;
2563   rv = createStmt->BindInt64ByName("timestamp"_ns, RoundedPRNow());
2564   if (NS_FAILED(rv)) return -1;
2565   rv = createStmt->BindUTF8StringByName("guid"_ns,
2566                                         nsLiteralCString(MOBILE_ROOT_GUID));
2567   if (NS_FAILED(rv)) return -1;
2568 
2569   rv = createStmt->Execute();
2570   if (NS_FAILED(rv)) return -1;
2571 
2572   // Find the mobile root ID. We can't use the last inserted ID because the
2573   // root might already exist, and we ignore on conflict.
2574   nsCOMPtr<mozIStorageStatement> findIdStmt;
2575   rv = mMainConn->CreateStatement(
2576       "SELECT id FROM moz_bookmarks WHERE guid = :guid"_ns,
2577       getter_AddRefs(findIdStmt));
2578   if (NS_FAILED(rv)) return -1;
2579 
2580   rv = findIdStmt->BindUTF8StringByName("guid"_ns,
2581                                         nsLiteralCString(MOBILE_ROOT_GUID));
2582   if (NS_FAILED(rv)) return -1;
2583 
2584   bool hasResult = false;
2585   rv = findIdStmt->ExecuteStep(&hasResult);
2586   if (NS_FAILED(rv) || !hasResult) return -1;
2587 
2588   int64_t rootId;
2589   rv = findIdStmt->GetInt64(0, &rootId);
2590   if (NS_FAILED(rv)) return -1;
2591 
2592   return rootId;
2593 }
2594 
Shutdown()2595 void Database::Shutdown() {
2596   // As the last step in the shutdown path, finalize the database handle.
2597   MOZ_ASSERT(NS_IsMainThread());
2598   MOZ_ASSERT(!mClosed);
2599 
2600   // Break cycles with the shutdown blockers.
2601   mClientsShutdown = nullptr;
2602   nsCOMPtr<mozIStorageCompletionCallback> connectionShutdown =
2603       std::move(mConnectionShutdown);
2604 
2605   if (!mMainConn) {
2606     // The connection has never been initialized. Just mark it as closed.
2607     mClosed = true;
2608     (void)connectionShutdown->Complete(NS_OK, nullptr);
2609     return;
2610   }
2611 
2612 #ifdef DEBUG
2613   {
2614     bool hasResult;
2615     nsCOMPtr<mozIStorageStatement> stmt;
2616 
2617     // Sanity check for missing guids.
2618     nsresult rv =
2619         mMainConn->CreateStatement(nsLiteralCString("SELECT 1 "
2620                                                     "FROM moz_places "
2621                                                     "WHERE guid IS NULL "),
2622                                    getter_AddRefs(stmt));
2623     MOZ_ASSERT(NS_SUCCEEDED(rv));
2624     rv = stmt->ExecuteStep(&hasResult);
2625     MOZ_ASSERT(NS_SUCCEEDED(rv));
2626     MOZ_ASSERT(!hasResult, "Found a page without a GUID!");
2627     rv = mMainConn->CreateStatement(nsLiteralCString("SELECT 1 "
2628                                                      "FROM moz_bookmarks "
2629                                                      "WHERE guid IS NULL "),
2630                                     getter_AddRefs(stmt));
2631     MOZ_ASSERT(NS_SUCCEEDED(rv));
2632     rv = stmt->ExecuteStep(&hasResult);
2633     MOZ_ASSERT(NS_SUCCEEDED(rv));
2634     MOZ_ASSERT(!hasResult, "Found a bookmark without a GUID!");
2635 
2636     // Sanity check for unrounded dateAdded and lastModified values (bug
2637     // 1107308).
2638     rv = mMainConn->CreateStatement(
2639         nsLiteralCString(
2640             "SELECT 1 "
2641             "FROM moz_bookmarks "
2642             "WHERE dateAdded % 1000 > 0 OR lastModified % 1000 > 0 LIMIT 1"),
2643         getter_AddRefs(stmt));
2644     MOZ_ASSERT(NS_SUCCEEDED(rv));
2645     rv = stmt->ExecuteStep(&hasResult);
2646     MOZ_ASSERT(NS_SUCCEEDED(rv));
2647     MOZ_ASSERT(!hasResult, "Found unrounded dates!");
2648 
2649     // Sanity check url_hash
2650     rv = mMainConn->CreateStatement(
2651         "SELECT 1 FROM moz_places WHERE url_hash = 0"_ns, getter_AddRefs(stmt));
2652     MOZ_ASSERT(NS_SUCCEEDED(rv));
2653     rv = stmt->ExecuteStep(&hasResult);
2654     MOZ_ASSERT(NS_SUCCEEDED(rv));
2655     MOZ_ASSERT(!hasResult, "Found a place without a hash!");
2656 
2657     // Sanity check unique urls
2658     rv = mMainConn->CreateStatement(
2659         nsLiteralCString(
2660             "SELECT 1 FROM moz_places GROUP BY url HAVING count(*) > 1 "),
2661         getter_AddRefs(stmt));
2662     MOZ_ASSERT(NS_SUCCEEDED(rv));
2663     rv = stmt->ExecuteStep(&hasResult);
2664     MOZ_ASSERT(NS_SUCCEEDED(rv));
2665     MOZ_ASSERT(!hasResult, "Found a duplicate url!");
2666 
2667     // Sanity check NULL urls
2668     rv = mMainConn->CreateStatement(
2669         "SELECT 1 FROM moz_places WHERE url ISNULL "_ns, getter_AddRefs(stmt));
2670     MOZ_ASSERT(NS_SUCCEEDED(rv));
2671     rv = stmt->ExecuteStep(&hasResult);
2672     MOZ_ASSERT(NS_SUCCEEDED(rv));
2673     MOZ_ASSERT(!hasResult, "Found a NULL url!");
2674   }
2675 #endif
2676 
2677   mMainThreadStatements.FinalizeStatements();
2678   mMainThreadAsyncStatements.FinalizeStatements();
2679 
2680   RefPtr<FinalizeStatementCacheProxy<mozIStorageStatement>> event =
2681       new FinalizeStatementCacheProxy<mozIStorageStatement>(
2682           mAsyncThreadStatements, NS_ISUPPORTS_CAST(nsIObserver*, this));
2683   DispatchToAsyncThread(event);
2684 
2685   mClosed = true;
2686 
2687   // Execute PRAGMA optimized as last step, this will ensure proper database
2688   // performance across restarts.
2689   nsCOMPtr<mozIStoragePendingStatement> ps;
2690   MOZ_ALWAYS_SUCCEEDS(mMainConn->ExecuteSimpleSQLAsync(
2691       "PRAGMA optimize(0x02)"_ns, nullptr, getter_AddRefs(ps)));
2692 
2693   (void)mMainConn->AsyncClose(connectionShutdown);
2694   mMainConn = nullptr;
2695 }
2696 
2697 ////////////////////////////////////////////////////////////////////////////////
2698 //// nsIObserver
2699 
2700 NS_IMETHODIMP
Observe(nsISupports * aSubject,const char * aTopic,const char16_t * aData)2701 Database::Observe(nsISupports* aSubject, const char* aTopic,
2702                   const char16_t* aData) {
2703   MOZ_ASSERT(NS_IsMainThread());
2704   if (strcmp(aTopic, TOPIC_PROFILE_CHANGE_TEARDOWN) == 0) {
2705     // Tests simulating shutdown may cause multiple notifications.
2706     if (PlacesShutdownBlocker::sIsStarted) {
2707       return NS_OK;
2708     }
2709 
2710     nsCOMPtr<nsIObserverService> os = services::GetObserverService();
2711     NS_ENSURE_STATE(os);
2712 
2713     // If shutdown happens in the same mainthread loop as init, observers could
2714     // handle the places-init-complete notification after xpcom-shutdown, when
2715     // the connection does not exist anymore.  Removing those observers would
2716     // be less expensive but may cause their RemoveObserver calls to throw.
2717     // Thus notify the topic now, so they stop listening for it.
2718     nsCOMPtr<nsISimpleEnumerator> e;
2719     if (NS_SUCCEEDED(os->EnumerateObservers(TOPIC_PLACES_INIT_COMPLETE,
2720                                             getter_AddRefs(e))) &&
2721         e) {
2722       bool hasMore = false;
2723       while (NS_SUCCEEDED(e->HasMoreElements(&hasMore)) && hasMore) {
2724         nsCOMPtr<nsISupports> supports;
2725         if (NS_SUCCEEDED(e->GetNext(getter_AddRefs(supports)))) {
2726           nsCOMPtr<nsIObserver> observer = do_QueryInterface(supports);
2727           (void)observer->Observe(observer, TOPIC_PLACES_INIT_COMPLETE,
2728                                   nullptr);
2729         }
2730       }
2731     }
2732 
2733     // Notify all Places users that we are about to shutdown.
2734     (void)os->NotifyObservers(nullptr, TOPIC_PLACES_SHUTDOWN, nullptr);
2735   } else if (strcmp(aTopic, TOPIC_SIMULATE_PLACES_SHUTDOWN) == 0) {
2736     // This notification is (and must be) only used by tests that are trying
2737     // to simulate Places shutdown out of the normal shutdown path.
2738 
2739     // Tests simulating shutdown may cause re-entrance.
2740     if (PlacesShutdownBlocker::sIsStarted) {
2741       return NS_OK;
2742     }
2743 
2744     // We are simulating a shutdown, so invoke the shutdown blockers,
2745     // wait for them, then proceed with connection shutdown.
2746     // Since we are already going through shutdown, but it's not the real one,
2747     // we won't need to block the real one anymore, so we can unblock it.
2748     {
2749       nsCOMPtr<nsIAsyncShutdownClient> shutdownPhase =
2750           GetProfileChangeTeardownPhase();
2751       if (shutdownPhase) {
2752         shutdownPhase->RemoveBlocker(mClientsShutdown.get());
2753       }
2754       (void)mClientsShutdown->BlockShutdown(nullptr);
2755     }
2756 
2757     // Spin the events loop until the clients are done.
2758     // Note, this is just for tests, specifically test_clearHistory_shutdown.js
2759     SpinEventLoopUntil("places:Database::Observe(SIMULATE_PLACES_SHUTDOWN)"_ns,
2760                        [&]() {
2761                          return mClientsShutdown->State() ==
2762                                 PlacesShutdownBlocker::States::RECEIVED_DONE;
2763                        });
2764 
2765     {
2766       nsCOMPtr<nsIAsyncShutdownClient> shutdownPhase =
2767           GetProfileBeforeChangePhase();
2768       if (shutdownPhase) {
2769         shutdownPhase->RemoveBlocker(mConnectionShutdown.get());
2770       }
2771       (void)mConnectionShutdown->BlockShutdown(nullptr);
2772     }
2773   }
2774   return NS_OK;
2775 }
2776 
2777 }  // namespace places
2778 }  // namespace mozilla
2779