1 /*
2 * Copyright (C) 2008, Nokia <ivan.frade@nokia.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 #include "config.h"
21
22 #include <string.h>
23 #include <stdlib.h>
24 #include <regex.h>
25 #include <zlib.h>
26 #include <locale.h>
27 #include <time.h>
28 #include <unistd.h>
29 #include <sys/types.h>
30 #include <stdio.h>
31 #include <fcntl.h>
32 #include <errno.h>
33
34 #include <glib/gstdio.h>
35
36 #include <libtracker-common/tracker-common.h>
37 #include <libtracker-common/tracker-parser.h>
38
39 #if HAVE_TRACKER_FTS
40 #include <libtracker-fts/tracker-fts.h>
41 #endif
42
43 #include "tracker-db-journal.h"
44 #include "tracker-db-manager.h"
45 #include "tracker-db-interface-sqlite.h"
46 #include "tracker-db-interface.h"
47 #include "tracker-data-manager.h"
48
49 #define UNKNOWN_STATUS 0.5
50
51 /* ZLib buffer settings */
52 #define ZLIB_BUF_SIZE 8192
53
54 #define MAX_INTERFACES_PER_CPU 16
55 #define MAX_INTERFACES (MAX_INTERFACES_PER_CPU * g_get_num_processors ())
56
57 /* Required minimum space needed to create databases (5Mb) */
58 #define TRACKER_DB_MIN_REQUIRED_SPACE 5242880
59
60 /* Default memory settings for databases */
61 #define TRACKER_DB_PAGE_SIZE_DONT_SET -1
62
63 /* Set current database version we are working with */
64 #define TRACKER_DB_VERSION_NOW TRACKER_DB_VERSION_0_15_2
65 #define TRACKER_DB_VERSION_FILE "db-version.txt"
66 #define TRACKER_DB_LOCALE_FILE "db-locale.txt"
67
68 #define TRACKER_VACUUM_CHECK_SIZE ((goffset) 4 * 1024 * 1024 * 1024) /* 4GB */
69
70 #define IN_USE_FILENAME ".meta.isrunning"
71
72 #define PARSER_VERSION_FILENAME "parser-version.txt"
73
74 #define TOSTRING1(x) #x
75 #define TOSTRING(x) TOSTRING1(x)
76 #define TRACKER_PARSER_VERSION_STRING TOSTRING(TRACKER_PARSER_VERSION)
77
78 typedef enum {
79 TRACKER_DB_VERSION_UNKNOWN, /* Unknown */
80 TRACKER_DB_VERSION_0_6_6, /* before indexer-split */
81 TRACKER_DB_VERSION_0_6_90, /* after indexer-split */
82 TRACKER_DB_VERSION_0_6_91, /* stable release */
83 TRACKER_DB_VERSION_0_6_92, /* current TRUNK */
84 TRACKER_DB_VERSION_0_7_0, /* vstore branch */
85 TRACKER_DB_VERSION_0_7_4, /* nothing special */
86 TRACKER_DB_VERSION_0_7_12, /* nmo ontology */
87 TRACKER_DB_VERSION_0_7_13, /* coalesce & writeback */
88 TRACKER_DB_VERSION_0_7_17, /* mlo ontology */
89 TRACKER_DB_VERSION_0_7_20, /* nco im ontology */
90 TRACKER_DB_VERSION_0_7_21, /* named graphs/localtime */
91 TRACKER_DB_VERSION_0_7_22, /* fts-limits branch */
92 TRACKER_DB_VERSION_0_7_28, /* RC1 + mto + nco:url */
93 TRACKER_DB_VERSION_0_8_0, /* stable release */
94 TRACKER_DB_VERSION_0_9_0, /* unstable release */
95 TRACKER_DB_VERSION_0_9_8, /* affiliation cardinality + volumes */
96 TRACKER_DB_VERSION_0_9_15, /* mtp:hidden */
97 TRACKER_DB_VERSION_0_9_16, /* Fix for NB#184823 */
98 TRACKER_DB_VERSION_0_9_19, /* collation */
99 TRACKER_DB_VERSION_0_9_21, /* Fix for NB#186055 */
100 TRACKER_DB_VERSION_0_9_24, /* nmo:PhoneMessage class */
101 TRACKER_DB_VERSION_0_9_34, /* ontology cache */
102 TRACKER_DB_VERSION_0_9_38, /* nie:url an inverse functional property */
103 TRACKER_DB_VERSION_0_15_2 /* fts4 */
104 } TrackerDBVersion;
105
106 typedef struct {
107 TrackerDBInterface *iface;
108 TrackerDBInterface *wal_iface;
109 const gchar *file;
110 const gchar *name;
111 gchar *abs_filename;
112 gint cache_size;
113 gint page_size;
114 gboolean attached;
115 gboolean is_index;
116 guint64 mtime;
117 } TrackerDBDefinition;
118
119 static TrackerDBDefinition db_base = {
120 NULL, NULL,
121 "meta.db",
122 "meta",
123 NULL,
124 TRACKER_DB_CACHE_SIZE_DEFAULT,
125 8192,
126 FALSE,
127 FALSE,
128 0
129 };
130
131 struct _TrackerDBManager {
132 TrackerDBDefinition db;
133 gboolean locations_initialized;
134 gchar *data_dir;
135 gchar *user_data_dir;
136 gchar *in_use_filename;
137 GFile *cache_location;
138 GFile *data_location;
139 TrackerDBManagerFlags flags;
140 guint s_cache_size;
141 guint u_cache_size;
142
143 gpointer vtab_data;
144
145 GWeakRef iface_data;
146
147 GAsyncQueue *interfaces;
148 GThread *wal_thread;
149 };
150
151 static gboolean db_exec_no_reply (TrackerDBInterface *iface,
152 const gchar *query,
153 ...);
154 static TrackerDBInterface *tracker_db_manager_create_db_interface (TrackerDBManager *db_manager,
155 gboolean readonly,
156 GError **error);
157 static void db_remove_locale_file (TrackerDBManager *db_manager);
158
159 static TrackerDBInterface * init_writable_db_interface (TrackerDBManager *db_manager);
160
161 static gboolean
db_exec_no_reply(TrackerDBInterface * iface,const gchar * query,...)162 db_exec_no_reply (TrackerDBInterface *iface,
163 const gchar *query,
164 ...)
165 {
166 va_list args;
167
168 va_start (args, query);
169 tracker_db_interface_execute_vquery (iface, NULL, query, args);
170 va_end (args);
171
172 return TRUE;
173 }
174
175 TrackerDBManagerFlags
tracker_db_manager_get_flags(TrackerDBManager * db_manager,guint * select_cache_size,guint * update_cache_size)176 tracker_db_manager_get_flags (TrackerDBManager *db_manager,
177 guint *select_cache_size,
178 guint *update_cache_size)
179 {
180 if (select_cache_size)
181 *select_cache_size = db_manager->s_cache_size;
182
183 if (update_cache_size)
184 *update_cache_size = db_manager->u_cache_size;
185
186 return db_manager->flags;
187 }
188
189 static void
db_set_params(TrackerDBInterface * iface,gint cache_size,gint page_size,gboolean readonly,GError ** error)190 db_set_params (TrackerDBInterface *iface,
191 gint cache_size,
192 gint page_size,
193 gboolean readonly,
194 GError **error)
195 {
196 GError *internal_error = NULL;
197 TrackerDBStatement *stmt;
198
199 #ifdef DISABLE_JOURNAL
200 tracker_db_interface_execute_query (iface, NULL, "PRAGMA synchronous = NORMAL;");
201 #else
202 tracker_db_interface_execute_query (iface, NULL, "PRAGMA synchronous = OFF;");
203 #endif /* DISABLE_JOURNAL */
204 tracker_db_interface_execute_query (iface, NULL, "PRAGMA encoding = \"UTF-8\"");
205 tracker_db_interface_execute_query (iface, NULL, "PRAGMA auto_vacuum = 0;");
206
207 if (readonly) {
208 tracker_db_interface_execute_query (iface, NULL, "PRAGMA temp_store = MEMORY;");
209 } else {
210 tracker_db_interface_execute_query (iface, NULL, "PRAGMA temp_store = FILE;");
211 }
212
213 stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE,
214 &internal_error,
215 "PRAGMA journal_mode = WAL;");
216
217 if (internal_error) {
218 g_info ("Can't set journal mode to WAL: '%s'",
219 internal_error->message);
220 g_propagate_error (error, internal_error);
221 } else {
222 TrackerDBCursor *cursor;
223
224 cursor = tracker_db_statement_start_cursor (stmt, NULL);
225 if (tracker_db_cursor_iter_next (cursor, NULL, NULL)) {
226 if (g_ascii_strcasecmp (tracker_db_cursor_get_string (cursor, 0, NULL), "WAL") != 0) {
227 g_set_error (error,
228 TRACKER_DB_INTERFACE_ERROR,
229 TRACKER_DB_OPEN_ERROR,
230 "Can't set journal mode to WAL");
231 }
232 }
233 g_object_unref (cursor);
234 }
235
236 if (stmt) {
237 g_object_unref (stmt);
238 }
239
240 /* disable autocheckpoint */
241 tracker_db_interface_execute_query (iface, NULL, "PRAGMA wal_autocheckpoint = 0");
242
243 tracker_db_interface_execute_query (iface, NULL, "PRAGMA journal_size_limit = 10240000");
244
245 if (page_size != TRACKER_DB_PAGE_SIZE_DONT_SET) {
246 g_info (" Setting page size to %d", page_size);
247 tracker_db_interface_execute_query (iface, NULL, "PRAGMA page_size = %d", page_size);
248 }
249
250 tracker_db_interface_execute_query (iface, NULL, "PRAGMA cache_size = %d", cache_size);
251 g_info (" Setting cache size to %d", cache_size);
252 }
253
254 void
tracker_db_manager_remove_all(TrackerDBManager * db_manager)255 tracker_db_manager_remove_all (TrackerDBManager *db_manager)
256 {
257 gchar *filename;
258
259 g_info ("Removing all database/storage files");
260
261 g_info (" Removing database:'%s'", db_manager->db.abs_filename);
262 g_unlink (db_manager->db.abs_filename);
263
264 /* also delete shm and wal helper files */
265 filename = g_strdup_printf ("%s-shm", db_manager->db.abs_filename);
266 g_unlink (filename);
267 g_free (filename);
268
269 filename = g_strdup_printf ("%s-wal", db_manager->db.abs_filename);
270 g_unlink (filename);
271 g_free (filename);
272
273 /* Remove locale file also */
274 db_remove_locale_file (db_manager);
275 }
276
277 static TrackerDBVersion
db_get_version(TrackerDBManager * db_manager)278 db_get_version (TrackerDBManager *db_manager)
279 {
280 TrackerDBVersion version;
281 gchar *filename;
282
283 filename = g_build_filename (db_manager->data_dir, TRACKER_DB_VERSION_FILE, NULL);
284
285 if (G_LIKELY (g_file_test (filename, G_FILE_TEST_EXISTS))) {
286 gchar *contents;
287
288 /* Check version is correct */
289 if (G_LIKELY (g_file_get_contents (filename, &contents, NULL, NULL))) {
290 if (contents && strlen (contents) <= 2) {
291 version = atoi (contents);
292 } else {
293 g_info (" Version file content size is either 0 or bigger than expected");
294
295 version = TRACKER_DB_VERSION_UNKNOWN;
296 }
297
298 g_free (contents);
299 } else {
300 g_info (" Could not get content of file '%s'", filename);
301
302 version = TRACKER_DB_VERSION_UNKNOWN;
303 }
304 } else {
305 g_info (" Could not find database version file:'%s'", filename);
306 g_info (" Current databases are either old or no databases are set up yet");
307
308 version = TRACKER_DB_VERSION_UNKNOWN;
309 }
310
311 g_free (filename);
312
313 return version;
314 }
315
316 void
tracker_db_manager_create_version_file(TrackerDBManager * db_manager)317 tracker_db_manager_create_version_file (TrackerDBManager *db_manager)
318 {
319 GError *error = NULL;
320 gchar *filename;
321 gchar *str;
322
323 filename = g_build_filename (db_manager->data_dir, TRACKER_DB_VERSION_FILE, NULL);
324 g_info (" Creating version file '%s'", filename);
325
326 str = g_strdup_printf ("%d", TRACKER_DB_VERSION_NOW);
327
328 if (!g_file_set_contents (filename, str, -1, &error)) {
329 g_info (" Could not set file contents, %s",
330 error ? error->message : "no error given");
331 g_clear_error (&error);
332 }
333
334 g_free (str);
335 g_free (filename);
336 }
337
338 void
tracker_db_manager_remove_version_file(TrackerDBManager * db_manager)339 tracker_db_manager_remove_version_file (TrackerDBManager *db_manager)
340 {
341 gchar *filename;
342
343 filename = g_build_filename (db_manager->data_dir, TRACKER_DB_VERSION_FILE, NULL);
344 g_info (" Removing db-version file:'%s'", filename);
345 g_unlink (filename);
346 g_free (filename);
347 }
348
349 static void
db_remove_locale_file(TrackerDBManager * db_manager)350 db_remove_locale_file (TrackerDBManager *db_manager)
351 {
352 gchar *filename;
353
354 filename = g_build_filename (db_manager->data_dir, TRACKER_DB_LOCALE_FILE, NULL);
355 g_info (" Removing db-locale file:'%s'", filename);
356 g_unlink (filename);
357 g_free (filename);
358 }
359
360 static gchar *
db_get_locale(TrackerDBManager * db_manager)361 db_get_locale (TrackerDBManager *db_manager)
362 {
363 gchar *locale = NULL;
364 gchar *filename;
365
366 filename = g_build_filename (db_manager->data_dir, TRACKER_DB_LOCALE_FILE, NULL);
367
368 if (G_LIKELY (g_file_test (filename, G_FILE_TEST_EXISTS))) {
369 gchar *contents;
370
371 /* Check locale is correct */
372 if (G_LIKELY (g_file_get_contents (filename, &contents, NULL, NULL))) {
373 if (contents && strlen (contents) == 0) {
374 g_critical (" Empty locale file found at '%s'", filename);
375 g_free (contents);
376 } else {
377 /* Re-use contents */
378 locale = contents;
379 }
380 } else {
381 g_critical (" Could not get content of file '%s'", filename);
382 }
383 } else {
384 /* expected when restoring from backup, always recreate indices */
385 g_info (" Could not find database locale file:'%s'", filename);
386 locale = g_strdup ("unknown");
387 }
388
389 g_free (filename);
390
391 return locale;
392 }
393
394 static void
db_set_locale(TrackerDBManager * db_manager,const gchar * locale)395 db_set_locale (TrackerDBManager *db_manager,
396 const gchar *locale)
397 {
398 GError *error = NULL;
399 gchar *filename;
400 gchar *str;
401
402 filename = g_build_filename (db_manager->data_dir, TRACKER_DB_LOCALE_FILE, NULL);
403 g_info (" Creating locale file '%s'", filename);
404
405 str = g_strdup_printf ("%s", locale ? locale : "");
406
407 if (!g_file_set_contents (filename, str, -1, &error)) {
408 g_info (" Could not set file contents, %s",
409 error ? error->message : "no error given");
410 g_clear_error (&error);
411 }
412
413 g_free (str);
414 g_free (filename);
415 }
416
417 gboolean
tracker_db_manager_locale_changed(TrackerDBManager * db_manager,GError ** error)418 tracker_db_manager_locale_changed (TrackerDBManager *db_manager,
419 GError **error)
420 {
421 gchar *db_locale;
422 gchar *current_locale;
423 gboolean changed;
424
425 /* As a special case, we allow calling this API function before
426 * tracker_data_manager_init() has been called, so it can be used
427 * to check for locale mismatches for initializing the database.
428 */
429 tracker_db_manager_ensure_locations (db_manager, db_manager->cache_location, db_manager->data_location);
430
431 /* Get current collation locale */
432 current_locale = tracker_locale_get (TRACKER_LOCALE_COLLATE);
433
434 /* Get db locale */
435 db_locale = db_get_locale (db_manager);
436
437 /* If they are different, recreate indexes. Note that having
438 * both to NULL is actually valid, they would default to
439 * the unicode collation without locale-specific stuff. */
440 if (g_strcmp0 (db_locale, current_locale) != 0) {
441 g_set_error (error,
442 TRACKER_DB_INTERFACE_ERROR,
443 TRACKER_DB_OPEN_ERROR,
444 "Locale change detected (DB:%s, User/App:%s)",
445 db_locale, current_locale);
446 changed = TRUE;
447 } else {
448 g_info ("Current and DB locales match: '%s'", db_locale);
449 changed = FALSE;
450 }
451
452 g_free (db_locale);
453 g_free (current_locale);
454
455 return changed;
456 }
457
458 void
tracker_db_manager_set_current_locale(TrackerDBManager * db_manager)459 tracker_db_manager_set_current_locale (TrackerDBManager *db_manager)
460 {
461 gchar *current_locale;
462
463 /* Get current collation locale */
464 current_locale = tracker_locale_get (TRACKER_LOCALE_COLLATE);
465 g_info ("Saving DB locale as: '%s'", current_locale);
466 db_set_locale (db_manager, current_locale);
467 g_free (current_locale);
468 }
469
470 static void
db_manager_analyze(TrackerDBManager * db_manager,TrackerDBInterface * iface)471 db_manager_analyze (TrackerDBManager *db_manager,
472 TrackerDBInterface *iface)
473 {
474 guint64 current_mtime;
475
476 current_mtime = tracker_file_get_mtime (db_manager->db.abs_filename);
477
478 if (current_mtime > db_manager->db.mtime) {
479 g_info (" Analyzing DB:'%s'", db_manager->db.name);
480 db_exec_no_reply (iface, "ANALYZE %s.Services", db_manager->db.name);
481
482 /* Remember current mtime for future */
483 db_manager->db.mtime = current_mtime;
484 } else {
485 g_info (" Not updating DB:'%s', no changes since last optimize", db_manager->db.name);
486 }
487 }
488
489 static void
db_recreate_all(TrackerDBManager * db_manager,GError ** error)490 db_recreate_all (TrackerDBManager *db_manager,
491 GError **error)
492 {
493 gchar *locale;
494 GError *internal_error = NULL;
495
496 /* We call an internal version of this function here
497 * because at the time 'initialized' = FALSE and that
498 * will cause errors and do nothing.
499 */
500 g_info ("Cleaning up database files for reindex");
501
502 tracker_db_manager_remove_all (db_manager);
503
504 /* Now create the databases and close them */
505 g_info ("Creating database files, this may take a few moments...");
506
507 db_manager->db.iface = tracker_db_manager_create_db_interface (db_manager, FALSE, &internal_error);
508 if (internal_error) {
509 g_propagate_error (error, internal_error);
510 return;
511 }
512
513 g_clear_object (&db_manager->db.iface);
514 g_clear_object (&db_manager->db.wal_iface);
515
516 locale = tracker_locale_get (TRACKER_LOCALE_COLLATE);
517 /* Initialize locale file */
518 db_set_locale (db_manager, locale);
519 g_free (locale);
520 }
521
522 void
tracker_db_manager_ensure_locations(TrackerDBManager * db_manager,GFile * cache_location,GFile * data_location)523 tracker_db_manager_ensure_locations (TrackerDBManager *db_manager,
524 GFile *cache_location,
525 GFile *data_location)
526 {
527 gchar *dir;
528
529 if (db_manager->locations_initialized) {
530 return;
531 }
532
533 db_manager->locations_initialized = TRUE;
534 db_manager->data_dir = g_file_get_path (cache_location);
535
536 /* For DISABLE_JOURNAL case we should use g_get_user_data_dir here. For now
537 * keeping this as-is */
538 db_manager->user_data_dir = g_file_get_path (data_location);
539
540 db_manager->db = db_base;
541
542 dir = g_file_get_path (cache_location);
543 db_manager->db.abs_filename = g_build_filename (dir, db_manager->db.file, NULL);
544 g_free (dir);
545 }
546
547 static void
perform_recreate(TrackerDBManager * db_manager,gboolean * first_time,GError ** error)548 perform_recreate (TrackerDBManager *db_manager,
549 gboolean *first_time,
550 GError **error)
551 {
552 GError *internal_error = NULL;
553
554 if (first_time) {
555 *first_time = TRUE;
556 }
557
558 g_clear_object (&db_manager->db.iface);
559 g_clear_object (&db_manager->db.wal_iface);
560
561 if (!tracker_file_system_has_enough_space (db_manager->data_dir, TRACKER_DB_MIN_REQUIRED_SPACE, TRUE)) {
562 g_set_error (error,
563 TRACKER_DB_INTERFACE_ERROR,
564 TRACKER_DB_OPEN_ERROR,
565 "Filesystem has not enough space");
566 return;
567 }
568
569 db_recreate_all (db_manager, &internal_error);
570
571 if (internal_error) {
572 g_propagate_error (error, internal_error);
573 }
574 }
575
576 TrackerDBManager *
tracker_db_manager_new(TrackerDBManagerFlags flags,GFile * cache_location,GFile * data_location,gboolean * first_time,gboolean restoring_backup,gboolean shared_cache,guint select_cache_size,guint update_cache_size,TrackerBusyCallback busy_callback,gpointer busy_user_data,const gchar * busy_operation,GObject * iface_data,gpointer vtab_data,GError ** error)577 tracker_db_manager_new (TrackerDBManagerFlags flags,
578 GFile *cache_location,
579 GFile *data_location,
580 gboolean *first_time,
581 gboolean restoring_backup,
582 gboolean shared_cache,
583 guint select_cache_size,
584 guint update_cache_size,
585 TrackerBusyCallback busy_callback,
586 gpointer busy_user_data,
587 const gchar *busy_operation,
588 GObject *iface_data,
589 gpointer vtab_data,
590 GError **error)
591 {
592 TrackerDBManager *db_manager;
593 TrackerDBVersion version;
594 gboolean need_reindex;
595 int in_use_file;
596 gboolean loaded = FALSE;
597 TrackerDBInterface *resources_iface;
598 GError *internal_error = NULL;
599
600 if (!cache_location || !data_location) {
601 g_set_error (error,
602 TRACKER_DATA_ONTOLOGY_ERROR,
603 TRACKER_DATA_UNSUPPORTED_LOCATION,
604 "All data storage and ontology locations must be provided");
605 return NULL;
606 }
607
608 db_manager = g_new0 (TrackerDBManager, 1);
609 db_manager->vtab_data = vtab_data;
610
611 /* First set defaults for return values */
612 if (first_time) {
613 *first_time = FALSE;
614 }
615
616 need_reindex = FALSE;
617
618 /* Set up locations */
619 g_info ("Setting database locations");
620
621 db_manager->flags = flags;
622 db_manager->s_cache_size = select_cache_size;
623 db_manager->u_cache_size = update_cache_size;
624 db_manager->interfaces = g_async_queue_new_full (g_object_unref);
625
626 g_set_object (&db_manager->cache_location, cache_location);
627 g_set_object (&db_manager->data_location, data_location);
628 g_weak_ref_init (&db_manager->iface_data, iface_data);
629
630 tracker_db_manager_ensure_locations (db_manager, cache_location, data_location);
631 db_manager->in_use_filename = g_build_filename (db_manager->user_data_dir,
632 IN_USE_FILENAME,
633 NULL);
634
635 /* Don't do need_reindex checks for readonly (direct-access) */
636 if ((flags & TRACKER_DB_MANAGER_READONLY) == 0) {
637
638 /* Make sure the directories exist */
639 g_info ("Checking database directories exist");
640
641 g_mkdir_with_parents (db_manager->data_dir, 00755);
642 g_mkdir_with_parents (db_manager->user_data_dir, 00755);
643
644 g_info ("Checking database version");
645
646 version = db_get_version (db_manager);
647
648 if (version < TRACKER_DB_VERSION_NOW) {
649 g_info (" A reindex will be forced");
650 need_reindex = TRUE;
651 }
652
653 if (need_reindex) {
654 tracker_db_manager_create_version_file (db_manager);
655 }
656 }
657
658 g_info ("Checking whether database files exist");
659
660 /* Check we have the database in place, if it is
661 * missing, we reindex.
662 *
663 * There's no need to check for files not existing (for
664 * reindex) if reindexing is already needed.
665 */
666 if (!need_reindex &&
667 !g_file_test (db_manager->db.abs_filename, G_FILE_TEST_EXISTS)) {
668 if ((flags & TRACKER_DB_MANAGER_READONLY) == 0) {
669 g_info ("Could not find database file:'%s', reindex will be forced", db_manager->db.abs_filename);
670 need_reindex = TRUE;
671 } else {
672 g_set_error (error,
673 TRACKER_DB_INTERFACE_ERROR,
674 TRACKER_DB_OPEN_ERROR,
675 "Could not find database file:'%s'.", db_manager->db.abs_filename);
676
677 tracker_db_manager_free (db_manager);
678 return NULL;
679 }
680 }
681
682 db_manager->locations_initialized = TRUE;
683
684 /* Don't do remove-dbs for readonly (direct-access) */
685 if ((flags & TRACKER_DB_MANAGER_READONLY) == 0) {
686
687 /* If we are just initializing to remove the databases,
688 * return here.
689 */
690 if ((flags & TRACKER_DB_MANAGER_REMOVE_ALL) != 0) {
691 return db_manager;
692 }
693 }
694
695 /* Set general database options */
696 if (shared_cache) {
697 g_info ("Enabling database shared cache");
698 tracker_db_interface_sqlite_enable_shared_cache ();
699 }
700
701 /* Should we reindex? If so, just remove all databases files,
702 * NOT the paths, note, that these paths are also used for
703 * other things like the nfs lock file.
704 */
705 if (flags & TRACKER_DB_MANAGER_FORCE_REINDEX || need_reindex) {
706
707 if (flags & TRACKER_DB_MANAGER_READONLY) {
708 /* no reindexing supported in read-only mode (direct access) */
709
710 g_set_error (error,
711 TRACKER_DB_INTERFACE_ERROR,
712 TRACKER_DB_OPEN_ERROR,
713 "No reindexing supported in read-only mode (direct access)");
714
715 tracker_db_manager_free (db_manager);
716 return NULL;
717 }
718
719 perform_recreate (db_manager, first_time, &internal_error);
720
721 if (internal_error) {
722 g_propagate_error (error, internal_error);
723 tracker_db_manager_free (db_manager);
724 return NULL;
725 }
726
727 /* Load databases */
728 g_info ("Loading databases files...");
729
730 } else if ((flags & TRACKER_DB_MANAGER_READONLY) == 0) {
731 /* do not do shutdown check for read-only mode (direct access) */
732 gboolean must_recreate = FALSE;
733
734 /* Load databases */
735 g_info ("Loading databases files...");
736
737 #ifndef DISABLE_JOURNAL
738 must_recreate = !tracker_db_journal_reader_verify_last (data_location,
739 NULL);
740 #endif /* DISABLE_JOURNAL */
741
742 if (!must_recreate && g_file_test (db_manager->in_use_filename, G_FILE_TEST_EXISTS)) {
743 gsize size = 0;
744 struct stat st;
745 TrackerDBStatement *stmt;
746 #ifndef DISABLE_JOURNAL
747 gchar *busy_status;
748 #endif /* DISABLE_JOURNAL */
749
750 g_info ("Didn't shut down cleanly last time, doing integrity checks");
751
752 if (g_stat (db_manager->db.abs_filename, &st) == 0) {
753 size = st.st_size;
754 }
755
756 /* Size is 1 when using echo > file.db, none of our databases
757 * are only one byte in size even initually. */
758
759 if (size <= 1) {
760 if (!restoring_backup) {
761 must_recreate = TRUE;
762 } else {
763 g_set_error (error,
764 TRACKER_DB_INTERFACE_ERROR,
765 TRACKER_DB_OPEN_ERROR,
766 "Corrupt db file");
767 tracker_db_manager_free (db_manager);
768 return NULL;
769 }
770 }
771
772 if (!must_recreate) {
773 db_manager->db.iface = tracker_db_manager_create_db_interface (db_manager, FALSE, &internal_error);
774
775 if (internal_error) {
776 /* If this already doesn't succeed, then surely the file is
777 * corrupt. No need to check for integrity anymore. */
778 if (!restoring_backup) {
779 g_clear_error (&internal_error);
780 must_recreate = TRUE;
781 } else {
782 g_propagate_error (error, internal_error);
783 tracker_db_manager_free (db_manager);
784 return NULL;
785 }
786 }
787 }
788
789 if (!must_recreate) {
790 db_manager->db.mtime = tracker_file_get_mtime (db_manager->db.abs_filename);
791
792 loaded = TRUE;
793
794 #ifndef DISABLE_JOURNAL
795 /* Report OPERATION - STATUS */
796 busy_status = g_strdup_printf ("%s - %s",
797 busy_operation,
798 "Integrity checking");
799 busy_callback (busy_status, 0, busy_user_data);
800 g_free (busy_status);
801
802 stmt = tracker_db_interface_create_statement (db_manager->db.iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE,
803 &internal_error,
804 "PRAGMA integrity_check(1)");
805
806 if (internal_error != NULL) {
807 if (internal_error->domain == TRACKER_DB_INTERFACE_ERROR &&
808 internal_error->code == TRACKER_DB_QUERY_ERROR) {
809 must_recreate = TRUE;
810 } else {
811 g_critical ("%s", internal_error->message);
812 }
813 g_error_free (internal_error);
814 internal_error = NULL;
815 } else {
816 TrackerDBCursor *cursor = NULL;
817
818 if (stmt) {
819 cursor = tracker_db_statement_start_cursor (stmt, NULL);
820 g_object_unref (stmt);
821 } else {
822 g_critical ("Can't create stmt for integrity_check, no error given");
823 }
824
825 if (cursor) {
826 if (tracker_db_cursor_iter_next (cursor, NULL, NULL)) {
827 if (g_strcmp0 (tracker_db_cursor_get_string (cursor, 0, NULL), "ok") != 0) {
828 must_recreate = TRUE;
829 }
830 }
831 g_object_unref (cursor);
832 }
833 }
834 #endif /* DISABLE_JOURNAL */
835 }
836
837 if (!must_recreate) {
838 /* ensure that database has been initialized by an earlier tracker-store start
839 by checking whether Resource table exists */
840 stmt = tracker_db_interface_create_statement (db_manager->db.iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE,
841 &internal_error,
842 "SELECT 1 FROM Resource");
843 if (internal_error != NULL) {
844 if (!restoring_backup) {
845 must_recreate = TRUE;
846 g_error_free (internal_error);
847 internal_error = NULL;
848 }
849 } else {
850 g_object_unref (stmt);
851 }
852 }
853 }
854
855 if (must_recreate) {
856 g_info ("Database severely damaged. We will recreate it"
857 #ifndef DISABLE_JOURNAL
858 " and replay the journal if available.");
859 #else
860 ".");
861 #endif /* DISABLE_JOURNAL */
862
863 perform_recreate (db_manager, first_time, &internal_error);
864 if (internal_error) {
865 g_propagate_error (error, internal_error);
866 return FALSE;
867 }
868 loaded = FALSE;
869 } else {
870 if (internal_error) {
871 g_propagate_error (error, internal_error);
872 return FALSE;
873 }
874 }
875 }
876
877 if (!loaded) {
878 db_manager->db.mtime = tracker_file_get_mtime (db_manager->db.abs_filename);
879 }
880
881 if ((flags & TRACKER_DB_MANAGER_READONLY) == 0) {
882 /* do not create in-use file for read-only mode (direct access) */
883 in_use_file = g_open (db_manager->in_use_filename,
884 O_WRONLY | O_APPEND | O_CREAT | O_SYNC,
885 S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
886
887 if (in_use_file >= 0) {
888 fsync (in_use_file);
889 close (in_use_file);
890 }
891 }
892
893 resources_iface = tracker_db_manager_create_db_interface (db_manager,
894 TRUE, &internal_error);
895
896 if (internal_error) {
897 if ((!restoring_backup) && (flags & TRACKER_DB_MANAGER_READONLY) == 0) {
898 GError *new_error = NULL;
899
900 perform_recreate (db_manager, first_time, &new_error);
901 if (!new_error) {
902 resources_iface = tracker_db_manager_create_db_interface (db_manager, TRUE,
903 &internal_error);
904 } else {
905 /* Most serious error is the recreate one here */
906 g_clear_error (&internal_error);
907 g_propagate_error (error, new_error);
908 tracker_db_manager_free (db_manager);
909 return NULL;
910 }
911 } else {
912 g_propagate_error (error, internal_error);
913 tracker_db_manager_free (db_manager);
914 return NULL;
915 }
916 }
917
918 g_clear_object (&resources_iface);
919
920 return db_manager;
921 }
922
923 void
tracker_db_manager_free(TrackerDBManager * db_manager)924 tracker_db_manager_free (TrackerDBManager *db_manager)
925 {
926 gboolean readonly = (db_manager->flags & TRACKER_DB_MANAGER_READONLY) != 0;
927
928 g_async_queue_unref (db_manager->interfaces);
929 g_free (db_manager->db.abs_filename);
930
931 if (db_manager->wal_thread)
932 g_thread_join (db_manager->wal_thread);
933
934 g_clear_object (&db_manager->db.wal_iface);
935
936 if (db_manager->db.iface) {
937 if (!readonly)
938 tracker_db_interface_sqlite_wal_checkpoint (db_manager->db.iface, TRUE, NULL);
939 g_object_unref (db_manager->db.iface);
940 }
941
942 g_weak_ref_clear (&db_manager->iface_data);
943
944 g_free (db_manager->data_dir);
945 g_free (db_manager->user_data_dir);
946
947 if (!readonly) {
948 /* do not delete in-use file for read-only mode (direct access) */
949 g_unlink (db_manager->in_use_filename);
950 }
951
952 g_free (db_manager->in_use_filename);
953 g_free (db_manager);
954 }
955
956 void
tracker_db_manager_optimize(TrackerDBManager * db_manager)957 tracker_db_manager_optimize (TrackerDBManager *db_manager)
958 {
959 gboolean dbs_are_open = FALSE;
960 TrackerDBInterface *iface;
961
962 g_info ("Optimizing database...");
963
964 g_info (" Checking database is not in use");
965
966 iface = tracker_db_manager_get_writable_db_interface (db_manager);
967
968 /* Check if any connections are open? */
969 if (G_OBJECT (iface)->ref_count > 1) {
970 g_info (" database is still in use with %d references!",
971 G_OBJECT (iface)->ref_count);
972
973 dbs_are_open = TRUE;
974 }
975
976 if (dbs_are_open) {
977 g_info (" Not optimizing database, still in use with > 1 reference");
978 return;
979 }
980
981 /* Optimize the metadata database */
982 db_manager_analyze (db_manager, iface);
983 }
984
985 const gchar *
tracker_db_manager_get_file(TrackerDBManager * db_manager)986 tracker_db_manager_get_file (TrackerDBManager *db_manager)
987 {
988 return db_manager->db.abs_filename;
989 }
990
991 static TrackerDBInterface *
tracker_db_manager_create_db_interface(TrackerDBManager * db_manager,gboolean readonly,GError ** error)992 tracker_db_manager_create_db_interface (TrackerDBManager *db_manager,
993 gboolean readonly,
994 GError **error)
995 {
996 TrackerDBInterface *connection;
997 GError *internal_error = NULL;
998 TrackerDBInterfaceFlags flags = 0;
999
1000 if (readonly)
1001 flags |= TRACKER_DB_INTERFACE_READONLY;
1002 if (db_manager->flags & TRACKER_DB_MANAGER_ENABLE_MUTEXES)
1003 flags |= TRACKER_DB_INTERFACE_USE_MUTEX;
1004
1005 connection = tracker_db_interface_sqlite_new (db_manager->db.abs_filename,
1006 flags,
1007 &internal_error);
1008 if (internal_error) {
1009 g_propagate_error (error, internal_error);
1010 return NULL;
1011 }
1012
1013 tracker_db_interface_set_user_data (connection,
1014 g_weak_ref_get (&db_manager->iface_data),
1015 g_object_unref);
1016
1017 tracker_db_interface_init_vtabs (connection, db_manager->vtab_data);
1018
1019 db_set_params (connection,
1020 db_manager->db.cache_size,
1021 db_manager->db.page_size,
1022 readonly,
1023 &internal_error);
1024
1025 if (internal_error) {
1026 g_propagate_error (error, internal_error);
1027 g_object_unref (connection);
1028 return NULL;
1029 }
1030
1031 tracker_db_interface_set_max_stmt_cache_size (connection,
1032 TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT,
1033 db_manager->s_cache_size);
1034
1035 if (!readonly) {
1036 tracker_db_interface_set_max_stmt_cache_size (connection,
1037 TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
1038 db_manager->u_cache_size);
1039 }
1040
1041 return connection;
1042 }
1043
1044 /**
1045 * tracker_db_manager_get_db_interface:
1046 *
1047 * Request a database connection to the database
1048 *
1049 * The caller must NOT g_object_unref the result
1050 *
1051 * returns: (callee-owns): a database connection
1052 **/
1053 TrackerDBInterface *
tracker_db_manager_get_db_interface(TrackerDBManager * db_manager)1054 tracker_db_manager_get_db_interface (TrackerDBManager *db_manager)
1055 {
1056 GError *internal_error = NULL;
1057 TrackerDBInterface *interface;
1058
1059 /* The interfaces never actually leave the async queue,
1060 * we use it as a thread synchronized LRU, which doesn't
1061 * mean the interface found has no other active cursors,
1062 * in which case we either optionally create a new
1063 * TrackerDBInterface, or resign to sharing the obtained
1064 * one with other threads (thus getting increased contention
1065 * in the interface lock).
1066 */
1067 g_async_queue_lock (db_manager->interfaces);
1068 interface = g_async_queue_try_pop_unlocked (db_manager->interfaces);
1069
1070 if (interface && tracker_db_interface_get_is_used (interface) &&
1071 g_async_queue_length_unlocked (db_manager->interfaces) < MAX_INTERFACES) {
1072 /* Put it back and go at creating a new one */
1073 g_async_queue_push_front_unlocked (db_manager->interfaces, interface);
1074 interface = NULL;
1075 }
1076
1077 if (!interface) {
1078 /* Create a new one to satisfy the request */
1079 interface = tracker_db_manager_create_db_interface (db_manager,
1080 TRUE, &internal_error);
1081
1082 if (interface) {
1083 #if HAVE_TRACKER_FTS
1084 tracker_data_manager_init_fts (interface, FALSE);
1085 #endif
1086 } else {
1087 if (g_async_queue_length_unlocked (db_manager->interfaces) == 0) {
1088 g_critical ("Error opening database: %s", internal_error->message);
1089 g_error_free (internal_error);
1090 g_async_queue_unlock (db_manager->interfaces);
1091 return NULL;
1092 } else {
1093 g_error_free (internal_error);
1094 /* Fetch the first interface back. Oh well */
1095 interface = g_async_queue_try_pop_unlocked (db_manager->interfaces);
1096 }
1097 }
1098 }
1099
1100 g_async_queue_push_unlocked (db_manager->interfaces, interface);
1101 g_async_queue_unlock (db_manager->interfaces);
1102
1103 return interface;
1104 }
1105
1106 static void
wal_checkpoint(TrackerDBInterface * iface,gboolean blocking)1107 wal_checkpoint (TrackerDBInterface *iface,
1108 gboolean blocking)
1109 {
1110 GError *error = NULL;
1111
1112 g_debug ("Checkpointing database...");
1113
1114 tracker_db_interface_sqlite_wal_checkpoint (iface, blocking,
1115 blocking ? &error : NULL);
1116
1117 if (error) {
1118 g_warning ("Error in %s WAL checkpoint: %s",
1119 blocking ? "blocking" : "deferred",
1120 error->message);
1121 g_error_free (error);
1122 }
1123
1124 g_debug ("Checkpointing complete");
1125 }
1126
1127 static gpointer
wal_checkpoint_thread(gpointer data)1128 wal_checkpoint_thread (gpointer data)
1129 {
1130 TrackerDBManager *db_manager = data;
1131
1132 if (!db_manager->db.wal_iface)
1133 db_manager->db.wal_iface = init_writable_db_interface (db_manager);
1134
1135 wal_checkpoint (db_manager->db.wal_iface, FALSE);
1136
1137 return NULL;
1138 }
1139
1140 static void
wal_hook(TrackerDBInterface * iface,gint n_pages,gpointer user_data)1141 wal_hook (TrackerDBInterface *iface,
1142 gint n_pages,
1143 gpointer user_data)
1144 {
1145 TrackerDBManager *db_manager = user_data;
1146
1147 /* Ensure there is only one WAL checkpoint at a time */
1148 if (db_manager->wal_thread)
1149 g_thread_join (db_manager->wal_thread);
1150
1151 if (n_pages >= 10000) {
1152 /* Do immediate checkpointing (blocking updates) to
1153 * prevent excessive WAL file growth.
1154 */
1155 wal_checkpoint (iface, TRUE);
1156 } else {
1157 /* Defer non-blocking checkpoint to thread */
1158 db_manager->wal_thread = g_thread_try_new ("wal-checkpoint", wal_checkpoint_thread,
1159 db_manager, NULL);
1160 }
1161 }
1162
1163 static TrackerDBInterface *
init_writable_db_interface(TrackerDBManager * db_manager)1164 init_writable_db_interface (TrackerDBManager *db_manager)
1165 {
1166 TrackerDBInterface *iface;
1167 GError *error = NULL;
1168 gboolean readonly;
1169
1170 /* Honor anyway the DBManager readonly flag */
1171 readonly = (db_manager->flags & TRACKER_DB_MANAGER_READONLY) != 0;
1172 iface = tracker_db_manager_create_db_interface (db_manager, readonly, &error);
1173 if (error) {
1174 g_critical ("Error opening readwrite database: %s", error->message);
1175 g_error_free (error);
1176 }
1177
1178 return iface;
1179 }
1180
1181 TrackerDBInterface *
tracker_db_manager_get_writable_db_interface(TrackerDBManager * db_manager)1182 tracker_db_manager_get_writable_db_interface (TrackerDBManager *db_manager)
1183 {
1184 if (db_manager->db.iface == NULL) {
1185 db_manager->db.iface = init_writable_db_interface (db_manager);
1186
1187 if (db_manager->db.iface &&
1188 (db_manager->flags & TRACKER_DB_MANAGER_READONLY) == 0) {
1189 tracker_db_interface_sqlite_wal_hook (db_manager->db.iface,
1190 wal_hook, db_manager);
1191 }
1192 }
1193
1194 return db_manager->db.iface;
1195 }
1196
1197 /**
1198 * tracker_db_manager_has_enough_space:
1199 *
1200 * Checks whether the file system, where the database files are stored,
1201 * has enough free space to allow modifications.
1202 *
1203 * returns: TRUE if there is enough space, FALSE otherwise
1204 **/
1205 gboolean
tracker_db_manager_has_enough_space(TrackerDBManager * db_manager)1206 tracker_db_manager_has_enough_space (TrackerDBManager *db_manager)
1207 {
1208 return tracker_file_system_has_enough_space (db_manager->data_dir, TRACKER_DB_MIN_REQUIRED_SPACE, FALSE);
1209 }
1210
1211 inline static gchar *
get_parser_version_filename(TrackerDBManager * db_manager)1212 get_parser_version_filename (TrackerDBManager *db_manager)
1213 {
1214 return g_build_filename (db_manager->data_dir,
1215 PARSER_VERSION_FILENAME,
1216 NULL);
1217 }
1218
1219
1220 gboolean
tracker_db_manager_get_tokenizer_changed(TrackerDBManager * db_manager)1221 tracker_db_manager_get_tokenizer_changed (TrackerDBManager *db_manager)
1222 {
1223 gchar *filename, *version;
1224 gboolean changed = TRUE;
1225
1226 filename = get_parser_version_filename (db_manager);
1227
1228 if (g_file_get_contents (filename, &version, NULL, NULL)) {
1229 changed = strcmp (version, TRACKER_PARSER_VERSION_STRING) != 0;
1230 g_free (version);
1231 }
1232
1233 g_free (filename);
1234
1235 return changed;
1236 }
1237
1238 void
tracker_db_manager_tokenizer_update(TrackerDBManager * db_manager)1239 tracker_db_manager_tokenizer_update (TrackerDBManager *db_manager)
1240 {
1241 GError *error = NULL;
1242 gchar *filename;
1243
1244 filename = get_parser_version_filename (db_manager);
1245
1246 if (!g_file_set_contents (filename, TRACKER_PARSER_VERSION_STRING, -1, &error)) {
1247 g_warning ("The file '%s' could not be rewritten by Tracker and "
1248 "should be deleted manually. Not doing so will result "
1249 "in Tracker rebuilding its FTS tokens on every startup. "
1250 "The error received was: '%s'", filename, error->message);
1251 g_error_free (error);
1252 }
1253
1254 g_free (filename);
1255 }
1256
1257 void
tracker_db_manager_check_perform_vacuum(TrackerDBManager * db_manager)1258 tracker_db_manager_check_perform_vacuum (TrackerDBManager *db_manager)
1259 {
1260 TrackerDBInterface *iface;
1261
1262 if (tracker_file_get_size (db_manager->db.abs_filename) < TRACKER_VACUUM_CHECK_SIZE)
1263 return;
1264
1265 iface = tracker_db_manager_get_writable_db_interface (db_manager);
1266 tracker_db_interface_execute_query (iface, NULL, "VACUUM");
1267 }
1268