1 /*
2 * Copyright (C) 2009, Nokia <ivan.frade@nokia.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 #include "config.h"
21
22 #include <libtracker-common/tracker-common.h>
23
24 #include "tracker-crawler.h"
25 #include "tracker-miner-fs.h"
26 #include "tracker-monitor.h"
27 #include "tracker-utils.h"
28 #include "tracker-priority-queue.h"
29 #include "tracker-task-pool.h"
30 #include "tracker-sparql-buffer.h"
31 #include "tracker-file-notifier.h"
32
33 /* If defined will print the tree from GNode while running */
34 #ifdef CRAWLED_TREE_ENABLE_TRACE
35 #warning Tree debugging traces enabled
36 #endif /* CRAWLED_TREE_ENABLE_TRACE */
37
38 /* If defined will print push/pop actions on queues */
39 #ifdef EVENT_QUEUE_ENABLE_TRACE
40 #warning Event Queue traces enabled
41 #define EVENT_QUEUE_LOG_PREFIX "[Event Queues] "
42 #define EVENT_QUEUE_STATUS_TIMEOUT_SECS 30
43 #define trace_eq(message, ...) g_debug (EVENT_QUEUE_LOG_PREFIX message, ##__VA_ARGS__)
44 #define trace_eq_action(pushed, queue_name, position, gfile1, gfile2, reason) \
45 do { \
46 gchar *uri1 = g_file_get_uri (gfile1); \
47 gchar *uri2 = gfile2 ? g_file_get_uri (gfile2) : NULL; \
48 g_debug ("%s%s '%s%s%s' %s %s of queue '%s'%s%s", \
49 EVENT_QUEUE_LOG_PREFIX, \
50 pushed ? "Pushed" : "Popped", \
51 uri1, \
52 uri2 ? "->" : "", \
53 uri2 ? uri2 : "", \
54 pushed ? "to" : "from", \
55 position, \
56 queue_name, \
57 reason ? ": " : "", \
58 reason ? reason : ""); \
59 g_free (uri1); \
60 g_free (uri2); \
61 } while (0)
62 #define trace_eq_push_tail(queue_name, gfile, reason) \
63 trace_eq_action (TRUE, queue_name, "tail", gfile, NULL, reason)
64 #define trace_eq_push_head(queue_name, gfile, reason) \
65 trace_eq_action (TRUE, queue_name, "head", gfile, NULL, reason)
66 #define trace_eq_push_tail_2(queue_name, gfile1, gfile2, reason) \
67 trace_eq_action (TRUE, queue_name, "tail", gfile1, gfile2, reason)
68 #define trace_eq_push_head_2(queue_name, gfile1, gfile2, reason) \
69 trace_eq_action (TRUE, queue_name, "head", gfile1, gfile2, reason)
70 #define trace_eq_pop_head(queue_name, gfile) \
71 trace_eq_action (FALSE, queue_name, "head", gfile, NULL, NULL)
72 #define trace_eq_pop_head_2(queue_name, gfile1, gfile2) \
73 trace_eq_action (FALSE, queue_name, "head", gfile1, gfile2, NULL)
74 static gboolean miner_fs_queues_status_trace_timeout_cb (gpointer data);
75 #else
76 #define trace_eq(...)
77 #define trace_eq_push_tail(...)
78 #define trace_eq_push_head(...)
79 #define trace_eq_push_tail_2(...)
80 #define trace_eq_push_head_2(...)
81 #define trace_eq_pop_head(...)
82 #define trace_eq_pop_head_2(...)
83 #endif /* EVENT_QUEUE_ENABLE_TRACE */
84
85 /* Default processing pool limits to be set */
86 #define DEFAULT_WAIT_POOL_LIMIT 1
87 #define DEFAULT_READY_POOL_LIMIT 1
88
89 /* Put tasks processing at a lower priority so other events
90 * (timeouts, monitor events, etc...) are guaranteed to be
91 * dispatched promptly.
92 */
93 #define TRACKER_TASK_PRIORITY G_PRIORITY_DEFAULT_IDLE + 10
94
95 #define MAX_SIMULTANEOUS_ITEMS 64
96
97 /**
98 * SECTION:tracker-miner-fs
99 * @short_description: Abstract base class for filesystem miners
100 * @include: libtracker-miner/tracker-miner.h
101 *
102 * #TrackerMinerFS is an abstract base class for miners that collect data
103 * from a filesystem where parent/child relationships need to be
104 * inserted into the database correctly with queue management.
105 *
106 * All the filesystem crawling and monitoring is abstracted away,
107 * leaving to implementations the decisions of what directories/files
108 * should it process, and the actual data extraction.
109 *
110 * Example creating a TrackerMinerFS with our own file system root and
111 * data provider.
112 *
113 * First create our class and base it on TrackerMinerFS:
114 * |[
115 * G_DEFINE_TYPE_WITH_CODE (MyMinerFiles, my_miner_files, TRACKER_TYPE_MINER_FS,
116 * G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE,
117 * my_miner_files_initable_iface_init))
118 * ]|
119 *
120 * Later in our class creation function, we are supplying the
121 * arguments we want. In this case, the 'root' is a #GFile pointing to
122 * a root URI location (for example 'file:///') and 'data_provider' is a
123 * #TrackerDataProvider used to enumerate 'root' and return children it
124 * finds. If 'data_provider' is %NULL (the default), then a
125 * #TrackerFileDataProvider is created automatically.
126 * |[
127 * // Note that only 'name' is mandatory
128 * miner = g_initable_new (MY_TYPE_MINER_FILES,
129 * NULL,
130 * error,
131 * "name", "MyMinerFiles",
132 * "root", root,
133 * "data-provider", data_provider,
134 * "processing-pool-wait-limit", 10,
135 * "processing-pool-ready-limit", 100,
136 * NULL);
137 * ]|
138 **/
139
140 typedef struct {
141 guint16 type;
142 guint attributes_update : 1;
143 GFile *file;
144 GFile *dest_file;
145 } QueueEvent;
146
147 typedef struct {
148 GFile *file;
149 gchar *urn;
150 gint priority;
151 GCancellable *cancellable;
152 TrackerMiner *miner;
153 } UpdateProcessingTaskContext;
154
155 struct _TrackerMinerFSPrivate {
156 TrackerPriorityQueue *items;
157
158 guint item_queues_handler_id;
159 GFile *item_queue_blocker;
160
161 #ifdef EVENT_QUEUE_ENABLE_TRACE
162 guint queue_status_timeout_id;
163 #endif /* EVENT_QUEUE_ENABLE_TRACE */
164
165 /* Root / tree / index */
166 GFile *root;
167 TrackerIndexingTree *indexing_tree;
168 TrackerFileNotifier *file_notifier;
169 TrackerDataProvider *data_provider;
170
171 /* Sparql insertion tasks */
172 TrackerTaskPool *task_pool;
173 TrackerSparqlBuffer *sparql_buffer;
174 guint sparql_buffer_limit;
175
176 /* File properties */
177 GQuark quark_recursive_removal;
178
179 /* Properties */
180 gdouble throttle;
181
182 /* Status */
183 GTimer *timer;
184 GTimer *extraction_timer;
185
186 guint been_started : 1; /* TRUE if miner has been started */
187 guint been_crawled : 1; /* TRUE if initial crawling has been
188 * done */
189 guint shown_totals : 1; /* TRUE if totals have been shown */
190 guint is_paused : 1; /* TRUE if miner is paused */
191
192 guint timer_stopped : 1; /* TRUE if main timer is stopped */
193 guint extraction_timer_stopped : 1; /* TRUE if the extraction
194 * timer is stopped */
195
196 GHashTable *roots_to_notify; /* Used to signal indexing
197 * trees finished */
198
199 /*
200 * Statistics
201 */
202
203 /* How many we found during crawling and how many were black
204 * listed (ignored). Reset to 0 when processing stops. */
205 guint total_directories_found;
206 guint total_directories_ignored;
207 guint total_files_found;
208 guint total_files_ignored;
209
210 /* How many we indexed and how many had errors indexing. */
211 guint total_files_processed;
212 guint total_files_notified;
213 guint total_files_notified_error;
214 };
215
216 typedef enum {
217 QUEUE_NONE,
218 QUEUE_CREATED,
219 QUEUE_UPDATED,
220 QUEUE_DELETED,
221 QUEUE_MOVED,
222 QUEUE_WAIT,
223 } QueueState;
224
225 typedef enum {
226 QUEUE_ACTION_NONE = 0,
227 QUEUE_ACTION_DELETE_FIRST = 1 << 0,
228 QUEUE_ACTION_DELETE_SECOND = 1 << 1,
229 } QueueCoalesceAction;
230
231 enum {
232 PROCESS_FILE,
233 PROCESS_FILE_ATTRIBUTES,
234 FINISHED,
235 FINISHED_ROOT,
236 REMOVE_FILE,
237 REMOVE_CHILDREN,
238 MOVE_FILE,
239 LAST_SIGNAL
240 };
241
242 enum {
243 PROP_0,
244 PROP_THROTTLE,
245 PROP_ROOT,
246 PROP_WAIT_POOL_LIMIT,
247 PROP_READY_POOL_LIMIT,
248 PROP_DATA_PROVIDER
249 };
250
251 static void miner_fs_initable_iface_init (GInitableIface *iface);
252
253 static void fs_finalize (GObject *object);
254 static void fs_constructed (GObject *object);
255 static void fs_set_property (GObject *object,
256 guint prop_id,
257 const GValue *value,
258 GParamSpec *pspec);
259 static void fs_get_property (GObject *object,
260 guint prop_id,
261 GValue *value,
262 GParamSpec *pspec);
263
264 static void miner_started (TrackerMiner *miner);
265 static void miner_stopped (TrackerMiner *miner);
266 static void miner_paused (TrackerMiner *miner);
267 static void miner_resumed (TrackerMiner *miner);
268
269 static void indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree,
270 GFile *directory,
271 gpointer user_data);
272 static void file_notifier_file_created (TrackerFileNotifier *notifier,
273 GFile *file,
274 gpointer user_data);
275 static void file_notifier_file_deleted (TrackerFileNotifier *notifier,
276 GFile *file,
277 gpointer user_data);
278 static void file_notifier_file_updated (TrackerFileNotifier *notifier,
279 GFile *file,
280 gboolean attributes_only,
281 gpointer user_data);
282 static void file_notifier_file_moved (TrackerFileNotifier *notifier,
283 GFile *source,
284 GFile *dest,
285 gpointer user_data);
286 static void file_notifier_directory_started (TrackerFileNotifier *notifier,
287 GFile *directory,
288 gpointer user_data);
289 static void file_notifier_directory_finished (TrackerFileNotifier *notifier,
290 GFile *directory,
291 guint directories_found,
292 guint directories_ignored,
293 guint files_found,
294 guint files_ignored,
295 gpointer user_data);
296 static void file_notifier_finished (TrackerFileNotifier *notifier,
297 gpointer user_data);
298
299 static void item_queue_handlers_set_up (TrackerMinerFS *fs);
300
301 static void task_pool_cancel_foreach (gpointer data,
302 gpointer user_data);
303 static void task_pool_limit_reached_notify_cb (GObject *object,
304 GParamSpec *pspec,
305 gpointer user_data);
306
307 static void miner_fs_queue_event (TrackerMinerFS *fs,
308 QueueEvent *event,
309 guint priority);
310
311 static GQuark quark_last_queue_event = 0;
312 static GInitableIface* miner_fs_initable_parent_iface;
313 static guint signals[LAST_SIGNAL] = { 0, };
314
315 /**
316 * tracker_miner_fs_error_quark:
317 *
318 * Gives the caller the #GQuark used to identify #TrackerMinerFS errors
319 * in #GError structures. The #GQuark is used as the domain for the error.
320 *
321 * Returns: the #GQuark used for the domain of a #GError.
322 *
323 * Since: 1.2
324 **/
325 G_DEFINE_QUARK (TrackerMinerFSError, tracker_miner_fs_error)
326
327 G_DEFINE_ABSTRACT_TYPE_WITH_CODE (TrackerMinerFS, tracker_miner_fs, TRACKER_TYPE_MINER,
328 G_ADD_PRIVATE (TrackerMinerFS)
329 G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE,
330 miner_fs_initable_iface_init));
331
332 static void
tracker_miner_fs_class_init(TrackerMinerFSClass * klass)333 tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
334 {
335 GObjectClass *object_class = G_OBJECT_CLASS (klass);
336 TrackerMinerClass *miner_class = TRACKER_MINER_CLASS (klass);
337
338 object_class->finalize = fs_finalize;
339 object_class->constructed = fs_constructed;
340 object_class->set_property = fs_set_property;
341 object_class->get_property = fs_get_property;
342
343 miner_class->started = miner_started;
344 miner_class->stopped = miner_stopped;
345 miner_class->paused = miner_paused;
346 miner_class->resumed = miner_resumed;
347
348 g_object_class_install_property (object_class,
349 PROP_THROTTLE,
350 g_param_spec_double ("throttle",
351 "Throttle",
352 "Modifier for the indexing speed, 0 is max speed",
353 0, 1, 0,
354 G_PARAM_READWRITE));
355 g_object_class_install_property (object_class,
356 PROP_ROOT,
357 g_param_spec_object ("root",
358 "Root",
359 "Top level URI for our indexing tree and file notify clases",
360 G_TYPE_FILE,
361 G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
362 g_object_class_install_property (object_class,
363 PROP_WAIT_POOL_LIMIT,
364 g_param_spec_uint ("processing-pool-wait-limit",
365 "Processing pool limit for WAIT tasks",
366 "Maximum number of files that can be concurrently "
367 "processed by the upper layer",
368 1, G_MAXUINT, DEFAULT_WAIT_POOL_LIMIT,
369 G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
370 g_object_class_install_property (object_class,
371 PROP_READY_POOL_LIMIT,
372 g_param_spec_uint ("processing-pool-ready-limit",
373 "Processing pool limit for READY tasks",
374 "Maximum number of SPARQL updates that can be merged "
375 "in a single connection to the store",
376 1, G_MAXUINT, DEFAULT_READY_POOL_LIMIT,
377 G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
378 g_object_class_install_property (object_class,
379 PROP_DATA_PROVIDER,
380 g_param_spec_object ("data-provider",
381 "Data provider",
382 "Data provider populating data, e.g. like GFileEnumerator",
383 TRACKER_TYPE_DATA_PROVIDER,
384 G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
385
386 /**
387 * TrackerMinerFS::process-file:
388 * @miner_fs: the #TrackerMinerFS
389 * @file: a #GFile
390 * @builder: a #TrackerSparqlBuilder
391 * @cancellable: a #GCancellable
392 *
393 * The ::process-file signal is emitted whenever a file should
394 * be processed, and it's metadata extracted.
395 *
396 * @builder is the #TrackerSparqlBuilder where all sparql updates
397 * to be performed for @file will be appended.
398 *
399 * This signal allows both synchronous and asynchronous extraction,
400 * in the synchronous case @cancellable can be safely ignored. In
401 * either case, on successful metadata extraction, implementations
402 * must call tracker_miner_fs_notify_finish() to indicate that
403 * processing has finished on @file, so the miner can execute
404 * the SPARQL updates and continue processing other files.
405 *
406 * Returns: %TRUE if the file is accepted for processing,
407 * %FALSE if the file should be ignored.
408 *
409 * Since: 0.8
410 **/
411 signals[PROCESS_FILE] =
412 g_signal_new ("process-file",
413 G_OBJECT_CLASS_TYPE (object_class),
414 G_SIGNAL_RUN_LAST,
415 G_STRUCT_OFFSET (TrackerMinerFSClass, process_file),
416 NULL, NULL,
417 NULL,
418 G_TYPE_BOOLEAN,
419 2, G_TYPE_FILE, G_TYPE_TASK);
420
421 /**
422 * TrackerMinerFS::process-file-attributes:
423 * @miner_fs: the #TrackerMinerFS
424 * @file: a #GFile
425 * @builder: a #TrackerSparqlBuilder
426 * @cancellable: a #GCancellable
427 *
428 * The ::process-file-attributes signal is emitted whenever a file should
429 * be processed, but only the attribute-related metadata extracted.
430 *
431 * @builder is the #TrackerSparqlBuilder where all sparql updates
432 * to be performed for @file will be appended. For the properties being
433 * updated, the DELETE statements should be included as well.
434 *
435 * This signal allows both synchronous and asynchronous extraction,
436 * in the synchronous case @cancellable can be safely ignored. In
437 * either case, on successful metadata extraction, implementations
438 * must call tracker_miner_fs_notify_finish() to indicate that
439 * processing has finished on @file, so the miner can execute
440 * the SPARQL updates and continue processing other files.
441 *
442 * Returns: %TRUE if the file is accepted for processing,
443 * %FALSE if the file should be ignored.
444 *
445 * Since: 0.10
446 **/
447 signals[PROCESS_FILE_ATTRIBUTES] =
448 g_signal_new ("process-file-attributes",
449 G_OBJECT_CLASS_TYPE (object_class),
450 G_SIGNAL_RUN_LAST,
451 G_STRUCT_OFFSET (TrackerMinerFSClass, process_file_attributes),
452 NULL, NULL,
453 NULL,
454 G_TYPE_BOOLEAN,
455 2, G_TYPE_FILE, G_TYPE_TASK);
456
457 /**
458 * TrackerMinerFS::finished:
459 * @miner_fs: the #TrackerMinerFS
460 * @elapsed: elapsed time since mining was started
461 * @directories_found: number of directories found
462 * @directories_ignored: number of ignored directories
463 * @files_found: number of files found
464 * @files_ignored: number of ignored files
465 *
466 * The ::finished signal is emitted when @miner_fs has finished
467 * all pending processing.
468 *
469 * Since: 0.8
470 **/
471 signals[FINISHED] =
472 g_signal_new ("finished",
473 G_TYPE_FROM_CLASS (object_class),
474 G_SIGNAL_RUN_LAST,
475 G_STRUCT_OFFSET (TrackerMinerFSClass, finished),
476 NULL, NULL,
477 NULL,
478 G_TYPE_NONE,
479 5,
480 G_TYPE_DOUBLE,
481 G_TYPE_UINT,
482 G_TYPE_UINT,
483 G_TYPE_UINT,
484 G_TYPE_UINT);
485
486 /**
487 * TrackerMinerFS::finished-root:
488 * @miner_fs: the #TrackerMinerFS
489 * @file: a #GFile
490 *
491 * The ::finished-crawl signal is emitted when @miner_fs has
492 * finished finding all resources that need to be indexed
493 * with the root location of @file. At this point, it's likely
494 * many are still in the queue to be added to the database,
495 * but this gives some indication that a location is
496 * processed.
497 *
498 * Since: 1.2
499 **/
500 signals[FINISHED_ROOT] =
501 g_signal_new ("finished-root",
502 G_TYPE_FROM_CLASS (object_class),
503 G_SIGNAL_RUN_LAST,
504 G_STRUCT_OFFSET (TrackerMinerFSClass, finished_root),
505 NULL, NULL,
506 NULL,
507 G_TYPE_NONE,
508 1,
509 G_TYPE_FILE);
510
511 /**
512 * TrackerMinerFS::remove-file:
513 * @miner_fs: the #TrackerMinerFS
514 * @file: a #GFile
515 * @children_only: #TRUE if only the children of @file are to be deleted
516 * @builder: a #TrackerSparqlBuilder
517 *
518 * The ::remove-file signal will be emitted on files that need removal
519 * according to the miner configuration (either the files themselves are
520 * deleted, or the directory/contents no longer need inspection according
521 * to miner configuration and their location.
522 *
523 * This operation is always assumed to be recursive, the @children_only
524 * argument will be %TRUE if for any reason the topmost directory needs
525 * to stay (e.g. moved from a recursively indexed directory tree to a
526 * non-recursively indexed location).
527 *
528 * The @builder argument can be used to provide additional SPARQL
529 * deletes and updates necessary around the deletion of those items. If
530 * the return value of this signal is %TRUE, @builder is expected to
531 * contain all relevant deletes for this operation.
532 *
533 * If the return value of this signal is %FALSE, the miner will apply
534 * its default behavior, which is deleting all triples that correspond
535 * to the affected URIs.
536 *
537 * Returns: %TRUE if @builder contains all the necessary operations to
538 * delete the affected resources, %FALSE to let the miner
539 * implicitly handle the deletion.
540 *
541 * Since: 1.8
542 **/
543 signals[REMOVE_FILE] =
544 g_signal_new ("remove-file",
545 G_TYPE_FROM_CLASS (object_class),
546 G_SIGNAL_RUN_LAST,
547 G_STRUCT_OFFSET (TrackerMinerFSClass, remove_file),
548 NULL, NULL, NULL,
549 G_TYPE_STRING,
550 1, G_TYPE_FILE);
551
552 signals[REMOVE_CHILDREN] =
553 g_signal_new ("remove-children",
554 G_TYPE_FROM_CLASS (object_class),
555 G_SIGNAL_RUN_LAST,
556 G_STRUCT_OFFSET (TrackerMinerFSClass, remove_children),
557 NULL, NULL, NULL,
558 G_TYPE_STRING,
559 1, G_TYPE_FILE);
560
561 signals[MOVE_FILE] =
562 g_signal_new ("move-file",
563 G_TYPE_FROM_CLASS (object_class),
564 G_SIGNAL_RUN_LAST,
565 G_STRUCT_OFFSET (TrackerMinerFSClass, move_file),
566 NULL, NULL, NULL,
567 G_TYPE_STRING,
568 3, G_TYPE_FILE, G_TYPE_FILE, G_TYPE_BOOLEAN);
569
570 quark_last_queue_event = g_quark_from_static_string ("tracker-last-queue-event");
571 }
572
573 static void
tracker_miner_fs_init(TrackerMinerFS * object)574 tracker_miner_fs_init (TrackerMinerFS *object)
575 {
576 TrackerMinerFSPrivate *priv;
577
578 object->priv = tracker_miner_fs_get_instance_private (object);
579
580 priv = object->priv;
581
582 priv->timer = g_timer_new ();
583 priv->extraction_timer = g_timer_new ();
584
585 g_timer_stop (priv->timer);
586 g_timer_stop (priv->extraction_timer);
587
588 priv->timer_stopped = TRUE;
589 priv->extraction_timer_stopped = TRUE;
590
591 priv->items = tracker_priority_queue_new ();
592
593 #ifdef EVENT_QUEUE_ENABLE_TRACE
594 priv->queue_status_timeout_id = g_timeout_add_seconds (EVENT_QUEUE_STATUS_TIMEOUT_SECS,
595 miner_fs_queues_status_trace_timeout_cb,
596 object);
597 #endif /* PROCESSING_POOL_ENABLE_TRACE */
598
599 /* Create processing pools */
600 priv->task_pool = tracker_task_pool_new (DEFAULT_WAIT_POOL_LIMIT);
601 g_signal_connect (priv->task_pool, "notify::limit-reached",
602 G_CALLBACK (task_pool_limit_reached_notify_cb), object);
603
604 priv->quark_recursive_removal = g_quark_from_static_string ("tracker-recursive-removal");
605
606 priv->roots_to_notify = g_hash_table_new_full (g_file_hash,
607 (GEqualFunc) g_file_equal,
608 g_object_unref,
609 NULL);
610 }
611
612 static gboolean
miner_fs_initable_init(GInitable * initable,GCancellable * cancellable,GError ** error)613 miner_fs_initable_init (GInitable *initable,
614 GCancellable *cancellable,
615 GError **error)
616 {
617 TrackerMinerFSPrivate *priv;
618 guint limit;
619
620 if (!miner_fs_initable_parent_iface->init (initable, cancellable, error)) {
621 return FALSE;
622 }
623
624 priv = TRACKER_MINER_FS (initable)->priv;
625
626 g_object_get (initable, "processing-pool-ready-limit", &limit, NULL);
627 priv->sparql_buffer = tracker_sparql_buffer_new (tracker_miner_get_connection (TRACKER_MINER (initable)),
628 limit);
629
630 if (!priv->sparql_buffer) {
631 g_set_error (error,
632 tracker_miner_fs_error_quark (),
633 TRACKER_MINER_FS_ERROR_INIT,
634 "Could not create TrackerSparqlBuffer needed to process resources");
635 return FALSE;
636 }
637
638 g_signal_connect (priv->sparql_buffer, "notify::limit-reached",
639 G_CALLBACK (task_pool_limit_reached_notify_cb),
640 initable);
641
642 if (!priv->indexing_tree) {
643 g_set_error (error,
644 tracker_miner_fs_error_quark (),
645 TRACKER_MINER_FS_ERROR_INIT,
646 "Could not create TrackerIndexingTree needed to manage content indexed");
647 return FALSE;
648 }
649
650 g_signal_connect (priv->indexing_tree, "directory-removed",
651 G_CALLBACK (indexing_tree_directory_removed),
652 initable);
653
654 /* Create the file notifier */
655 priv->file_notifier = tracker_file_notifier_new (priv->indexing_tree,
656 priv->data_provider,
657 tracker_miner_get_connection (TRACKER_MINER (initable)));
658
659 if (!priv->file_notifier) {
660 g_set_error (error,
661 tracker_miner_fs_error_quark (),
662 TRACKER_MINER_FS_ERROR_INIT,
663 "Could not create TrackerFileNotifier needed to signal new resources to be indexed");
664 return FALSE;
665 }
666
667 g_signal_connect (priv->file_notifier, "file-created",
668 G_CALLBACK (file_notifier_file_created),
669 initable);
670 g_signal_connect (priv->file_notifier, "file-updated",
671 G_CALLBACK (file_notifier_file_updated),
672 initable);
673 g_signal_connect (priv->file_notifier, "file-deleted",
674 G_CALLBACK (file_notifier_file_deleted),
675 initable);
676 g_signal_connect (priv->file_notifier, "file-moved",
677 G_CALLBACK (file_notifier_file_moved),
678 initable);
679 g_signal_connect (priv->file_notifier, "directory-started",
680 G_CALLBACK (file_notifier_directory_started),
681 initable);
682 g_signal_connect (priv->file_notifier, "directory-finished",
683 G_CALLBACK (file_notifier_directory_finished),
684 initable);
685 g_signal_connect (priv->file_notifier, "finished",
686 G_CALLBACK (file_notifier_finished),
687 initable);
688
689 return TRUE;
690 }
691
692 static void
miner_fs_initable_iface_init(GInitableIface * iface)693 miner_fs_initable_iface_init (GInitableIface *iface)
694 {
695 miner_fs_initable_parent_iface = g_type_interface_peek_parent (iface);
696 iface->init = miner_fs_initable_init;
697 }
698
699 static QueueEvent *
queue_event_new(TrackerMinerFSEventType type,GFile * file)700 queue_event_new (TrackerMinerFSEventType type,
701 GFile *file)
702 {
703 QueueEvent *event;
704
705 g_assert (type != TRACKER_MINER_FS_EVENT_MOVED);
706
707 event = g_new0 (QueueEvent, 1);
708 event->type = type;
709 g_set_object (&event->file, file);
710
711 return event;
712 }
713
714 static QueueEvent *
queue_event_moved_new(GFile * source,GFile * dest)715 queue_event_moved_new (GFile *source,
716 GFile *dest)
717 {
718 QueueEvent *event;
719
720 event = g_new0 (QueueEvent, 1);
721 event->type = TRACKER_MINER_FS_EVENT_MOVED;
722 g_set_object (&event->dest_file, dest);
723 g_set_object (&event->file, source);
724
725 return event;
726 }
727
728 static GList *
queue_event_get_last_event_node(QueueEvent * event)729 queue_event_get_last_event_node (QueueEvent *event)
730 {
731 return g_object_get_qdata (G_OBJECT (event->file),
732 quark_last_queue_event);
733 }
734
735 static void
queue_event_save_node(QueueEvent * event,GList * node)736 queue_event_save_node (QueueEvent *event,
737 GList *node)
738 {
739 g_assert (node->data == event);
740 g_object_set_qdata (G_OBJECT (event->file),
741 quark_last_queue_event, node);
742 }
743
744 static void
queue_event_dispose_node(QueueEvent * event)745 queue_event_dispose_node (QueueEvent *event)
746 {
747 GList *node;
748
749 node = queue_event_get_last_event_node (event);
750
751 if (node && node->data == event) {
752 g_object_steal_qdata (G_OBJECT (event->file),
753 quark_last_queue_event);
754 }
755 }
756
757 static void
queue_event_free(QueueEvent * event)758 queue_event_free (QueueEvent *event)
759 {
760 queue_event_dispose_node (event);
761
762 g_clear_object (&event->dest_file);
763 g_clear_object (&event->file);
764 g_free (event);
765 }
766
767 static QueueCoalesceAction
queue_event_coalesce(const QueueEvent * first,const QueueEvent * second,QueueEvent ** replacement)768 queue_event_coalesce (const QueueEvent *first,
769 const QueueEvent *second,
770 QueueEvent **replacement)
771 {
772 *replacement = NULL;
773
774 if (first->type == TRACKER_MINER_FS_EVENT_CREATED) {
775 if ((second->type == TRACKER_MINER_FS_EVENT_UPDATED ||
776 second->type == TRACKER_MINER_FS_EVENT_CREATED) &&
777 first->file == second->file) {
778 return QUEUE_ACTION_DELETE_SECOND;
779 } else if (second->type == TRACKER_MINER_FS_EVENT_MOVED &&
780 first->file == second->file) {
781 *replacement = queue_event_new (TRACKER_MINER_FS_EVENT_CREATED,
782 second->dest_file);
783 return (QUEUE_ACTION_DELETE_FIRST |
784 QUEUE_ACTION_DELETE_SECOND);
785 } else if (second->type == TRACKER_MINER_FS_EVENT_DELETED &&
786 first->file == second->file) {
787 /* We can't be sure that "create" is replacing a file
788 * here. Preserve the second event just in case.
789 */
790 return QUEUE_ACTION_DELETE_FIRST;
791 }
792 } else if (first->type == TRACKER_MINER_FS_EVENT_UPDATED) {
793 if (second->type == TRACKER_MINER_FS_EVENT_UPDATED &&
794 first->file == second->file) {
795 if (first->attributes_update && !second->attributes_update)
796 return QUEUE_ACTION_DELETE_FIRST;
797 else
798 return QUEUE_ACTION_DELETE_SECOND;
799 } else if (second->type == TRACKER_MINER_FS_EVENT_DELETED &&
800 first->file == second->file) {
801 return QUEUE_ACTION_DELETE_FIRST;
802 }
803 } else if (first->type == TRACKER_MINER_FS_EVENT_MOVED) {
804 if (second->type == TRACKER_MINER_FS_EVENT_MOVED &&
805 first->dest_file == second->file) {
806 if (first->file != second->dest_file) {
807 *replacement = queue_event_moved_new (first->file,
808 second->dest_file);
809 }
810
811 return (QUEUE_ACTION_DELETE_FIRST |
812 QUEUE_ACTION_DELETE_SECOND);
813 } else if (second->type == TRACKER_MINER_FS_EVENT_DELETED &&
814 first->dest_file == second->file) {
815 *replacement = queue_event_new (TRACKER_MINER_FS_EVENT_DELETED,
816 first->file);
817 return (QUEUE_ACTION_DELETE_FIRST |
818 QUEUE_ACTION_DELETE_SECOND);
819 }
820 } else if (first->type == TRACKER_MINER_FS_EVENT_DELETED &&
821 second->type == TRACKER_MINER_FS_EVENT_DELETED) {
822 return QUEUE_ACTION_DELETE_SECOND;
823 }
824
825 return QUEUE_ACTION_NONE;
826 }
827
828 static gboolean
queue_event_is_descendant(QueueEvent * event,GFile * prefix)829 queue_event_is_descendant (QueueEvent *event,
830 GFile *prefix)
831 {
832 return g_file_has_prefix (event->file, prefix);
833 }
834
835 static gboolean
queue_event_is_equal_or_descendant(QueueEvent * event,GFile * prefix)836 queue_event_is_equal_or_descendant (QueueEvent *event,
837 GFile *prefix)
838 {
839 return (g_file_equal (event->file, prefix) ||
840 g_file_has_prefix (event->file, prefix));
841 }
842
843 static void
fs_finalize(GObject * object)844 fs_finalize (GObject *object)
845 {
846 TrackerMinerFSPrivate *priv;
847
848 priv = TRACKER_MINER_FS (object)->priv;
849
850 g_timer_destroy (priv->timer);
851 g_timer_destroy (priv->extraction_timer);
852
853 if (priv->item_queues_handler_id) {
854 g_source_remove (priv->item_queues_handler_id);
855 priv->item_queues_handler_id = 0;
856 }
857
858 if (priv->item_queue_blocker) {
859 g_object_unref (priv->item_queue_blocker);
860 }
861
862 if (priv->file_notifier) {
863 tracker_file_notifier_stop (priv->file_notifier);
864 }
865
866 /* Cancel every pending task */
867 tracker_task_pool_foreach (priv->task_pool,
868 task_pool_cancel_foreach,
869 NULL);
870 g_object_unref (priv->task_pool);
871
872 if (priv->sparql_buffer) {
873 g_object_unref (priv->sparql_buffer);
874 }
875
876 tracker_priority_queue_foreach (priv->items,
877 (GFunc) queue_event_free,
878 NULL);
879 tracker_priority_queue_unref (priv->items);
880
881 g_object_unref (priv->root);
882
883 if (priv->indexing_tree) {
884 g_object_unref (priv->indexing_tree);
885 }
886
887 if (priv->file_notifier) {
888 g_object_unref (priv->file_notifier);
889 }
890
891 if (priv->roots_to_notify) {
892 g_hash_table_unref (priv->roots_to_notify);
893
894 /* Just in case we end up using this AFTER finalize, not expected */
895 priv->roots_to_notify = NULL;
896 }
897
898 #ifdef EVENT_QUEUE_ENABLE_TRACE
899 if (priv->queue_status_timeout_id)
900 g_source_remove (priv->queue_status_timeout_id);
901 #endif /* PROCESSING_POOL_ENABLE_TRACE */
902
903 G_OBJECT_CLASS (tracker_miner_fs_parent_class)->finalize (object);
904 }
905
906 static void
fs_constructed(GObject * object)907 fs_constructed (GObject *object)
908 {
909 TrackerMinerFSPrivate *priv;
910
911 /* NOTE: We have to do this in this order because initables
912 * are called _AFTER_ constructed and for subclasses that are
913 * not initables we don't have any other way than to chain
914 * constructed and root/indexing tree must exist at that
915 * point.
916 *
917 * If priv->indexing_tree is NULL after this function, the
918 * initiable functions will fail and this class will not be
919 * created anyway.
920 */
921 G_OBJECT_CLASS (tracker_miner_fs_parent_class)->constructed (object);
922
923 priv = TRACKER_MINER_FS (object)->priv;
924
925 /* Create root if one didn't exist */
926 if (priv->root == NULL) {
927 /* We default to file:/// */
928 priv->root = g_file_new_for_uri ("file:///");
929 }
930
931 /* Create indexing tree */
932 priv->indexing_tree = tracker_indexing_tree_new_with_root (priv->root);
933 }
934
935 static void
fs_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)936 fs_set_property (GObject *object,
937 guint prop_id,
938 const GValue *value,
939 GParamSpec *pspec)
940 {
941 TrackerMinerFS *fs = TRACKER_MINER_FS (object);
942
943 switch (prop_id) {
944 case PROP_THROTTLE:
945 tracker_miner_fs_set_throttle (TRACKER_MINER_FS (object),
946 g_value_get_double (value));
947 break;
948 case PROP_ROOT:
949 /* We expect this to only occur once, on object construct */
950 fs->priv->root = g_value_dup_object (value);
951 break;
952 case PROP_WAIT_POOL_LIMIT:
953 tracker_task_pool_set_limit (fs->priv->task_pool,
954 g_value_get_uint (value));
955 break;
956 case PROP_READY_POOL_LIMIT:
957 fs->priv->sparql_buffer_limit = g_value_get_uint (value);
958
959 if (fs->priv->sparql_buffer) {
960 tracker_task_pool_set_limit (TRACKER_TASK_POOL (fs->priv->sparql_buffer),
961 fs->priv->sparql_buffer_limit);
962 }
963 break;
964 case PROP_DATA_PROVIDER:
965 fs->priv->data_provider = g_value_dup_object (value);
966 break;
967 default:
968 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
969 break;
970 }
971 }
972
973 static void
fs_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)974 fs_get_property (GObject *object,
975 guint prop_id,
976 GValue *value,
977 GParamSpec *pspec)
978 {
979 TrackerMinerFS *fs;
980
981 fs = TRACKER_MINER_FS (object);
982
983 switch (prop_id) {
984 case PROP_THROTTLE:
985 g_value_set_double (value, fs->priv->throttle);
986 break;
987 case PROP_ROOT:
988 g_value_set_object (value, fs->priv->root);
989 break;
990 case PROP_WAIT_POOL_LIMIT:
991 g_value_set_uint (value, tracker_task_pool_get_limit (fs->priv->task_pool));
992 break;
993 case PROP_READY_POOL_LIMIT:
994 g_value_set_uint (value, fs->priv->sparql_buffer_limit);
995 break;
996 case PROP_DATA_PROVIDER:
997 g_value_set_object (value, fs->priv->data_provider);
998 break;
999 default:
1000 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1001 break;
1002 }
1003 }
1004
1005 static void
task_pool_limit_reached_notify_cb(GObject * object,GParamSpec * pspec,gpointer user_data)1006 task_pool_limit_reached_notify_cb (GObject *object,
1007 GParamSpec *pspec,
1008 gpointer user_data)
1009 {
1010 if (!tracker_task_pool_limit_reached (TRACKER_TASK_POOL (object))) {
1011 item_queue_handlers_set_up (TRACKER_MINER_FS (user_data));
1012 }
1013 }
1014
1015 static void
miner_started(TrackerMiner * miner)1016 miner_started (TrackerMiner *miner)
1017 {
1018 TrackerMinerFS *fs;
1019
1020 fs = TRACKER_MINER_FS (miner);
1021
1022 fs->priv->been_started = TRUE;
1023
1024 g_info ("Initializing");
1025
1026 g_object_set (miner,
1027 "progress", 0.0,
1028 "status", "Initializing",
1029 "remaining-time", 0,
1030 NULL);
1031
1032 tracker_file_notifier_start (fs->priv->file_notifier);
1033 }
1034
1035 static void
miner_stopped(TrackerMiner * miner)1036 miner_stopped (TrackerMiner *miner)
1037 {
1038 g_info ("Idle");
1039
1040 g_object_set (miner,
1041 "progress", 1.0,
1042 "status", "Idle",
1043 "remaining-time", -1,
1044 NULL);
1045 }
1046
1047 static void
miner_paused(TrackerMiner * miner)1048 miner_paused (TrackerMiner *miner)
1049 {
1050 TrackerMinerFS *fs;
1051
1052 fs = TRACKER_MINER_FS (miner);
1053
1054 fs->priv->is_paused = TRUE;
1055
1056 tracker_file_notifier_stop (fs->priv->file_notifier);
1057
1058 if (fs->priv->item_queues_handler_id) {
1059 g_source_remove (fs->priv->item_queues_handler_id);
1060 fs->priv->item_queues_handler_id = 0;
1061 }
1062 }
1063
1064 static void
miner_resumed(TrackerMiner * miner)1065 miner_resumed (TrackerMiner *miner)
1066 {
1067 TrackerMinerFS *fs;
1068
1069 fs = TRACKER_MINER_FS (miner);
1070
1071 fs->priv->is_paused = FALSE;
1072
1073 tracker_file_notifier_start (fs->priv->file_notifier);
1074
1075 /* Only set up queue handler if we have items waiting to be
1076 * processed.
1077 */
1078 if (tracker_miner_fs_has_items_to_process (fs)) {
1079 item_queue_handlers_set_up (fs);
1080 }
1081 }
1082
1083 static void
notify_roots_finished(TrackerMinerFS * fs,gboolean check_queues)1084 notify_roots_finished (TrackerMinerFS *fs,
1085 gboolean check_queues)
1086 {
1087 GHashTableIter iter;
1088 gpointer key, value;
1089
1090 if (check_queues &&
1091 fs->priv->roots_to_notify &&
1092 g_hash_table_size (fs->priv->roots_to_notify) < 2) {
1093 /* Technically, if there is only one root, it's
1094 * pointless to do anything before the FINISHED (not
1095 * FINISHED_ROOT) signal is emitted. In that
1096 * situation we calls function first anyway with
1097 * check_queues=FALSE so we still notify roots. This
1098 * is really just for efficiency.
1099 */
1100 return;
1101 } else if (fs->priv->roots_to_notify == NULL ||
1102 g_hash_table_size (fs->priv->roots_to_notify) < 1) {
1103 /* Nothing to do */
1104 return;
1105 }
1106
1107 g_hash_table_iter_init (&iter, fs->priv->roots_to_notify);
1108 while (g_hash_table_iter_next (&iter, &key, &value)) {
1109 GFile *root = key;
1110
1111 /* Check if any content for root is still in the queue
1112 * to be processed. This is only called each time a
1113 * container/folder has been added to Tracker (so not
1114 * too frequently)
1115 */
1116 if (check_queues &&
1117 tracker_priority_queue_find (fs->priv->items, NULL, (GEqualFunc) queue_event_is_descendant, root)) {
1118 continue;
1119 }
1120
1121 /* Signal root is finished */
1122 g_signal_emit (fs, signals[FINISHED_ROOT], 0, root);
1123
1124 /* Remove from hash table */
1125 g_hash_table_iter_remove (&iter);
1126 }
1127 }
1128
1129 static void
process_print_stats(TrackerMinerFS * fs)1130 process_print_stats (TrackerMinerFS *fs)
1131 {
1132 /* Only do this the first time, otherwise the results are
1133 * likely to be inaccurate. Devices can be added or removed so
1134 * we can't assume stats are correct.
1135 */
1136 if (!fs->priv->shown_totals) {
1137 fs->priv->shown_totals = TRUE;
1138
1139 g_info ("--------------------------------------------------");
1140 g_info ("Total directories : %d (%d ignored)",
1141 fs->priv->total_directories_found,
1142 fs->priv->total_directories_ignored);
1143 g_info ("Total files : %d (%d ignored)",
1144 fs->priv->total_files_found,
1145 fs->priv->total_files_ignored);
1146 #if 0
1147 g_info ("Total monitors : %d",
1148 tracker_monitor_get_count (fs->priv->monitor));
1149 #endif
1150 g_info ("Total processed : %d (%d notified, %d with error)",
1151 fs->priv->total_files_processed,
1152 fs->priv->total_files_notified,
1153 fs->priv->total_files_notified_error);
1154 g_info ("--------------------------------------------------\n");
1155 }
1156 }
1157
1158 static void
process_stop(TrackerMinerFS * fs)1159 process_stop (TrackerMinerFS *fs)
1160 {
1161 /* Now we have finished crawling, print stats and enable monitor events */
1162 process_print_stats (fs);
1163
1164 g_timer_stop (fs->priv->timer);
1165 g_timer_stop (fs->priv->extraction_timer);
1166
1167 fs->priv->timer_stopped = TRUE;
1168 fs->priv->extraction_timer_stopped = TRUE;
1169
1170 g_info ("Idle");
1171
1172 g_object_set (fs,
1173 "progress", 1.0,
1174 "status", "Idle",
1175 "remaining-time", 0,
1176 NULL);
1177
1178 /* Make sure we signal _ALL_ roots as finished before the
1179 * main FINISHED signal
1180 */
1181 notify_roots_finished (fs, FALSE);
1182
1183 g_signal_emit (fs, signals[FINISHED], 0,
1184 g_timer_elapsed (fs->priv->timer, NULL),
1185 fs->priv->total_directories_found,
1186 fs->priv->total_directories_ignored,
1187 fs->priv->total_files_found,
1188 fs->priv->total_files_ignored);
1189
1190 g_timer_stop (fs->priv->timer);
1191 g_timer_stop (fs->priv->extraction_timer);
1192
1193 fs->priv->total_directories_found = 0;
1194 fs->priv->total_directories_ignored = 0;
1195 fs->priv->total_files_found = 0;
1196 fs->priv->total_files_ignored = 0;
1197
1198 fs->priv->been_crawled = TRUE;
1199 }
1200
1201 static gboolean
item_queue_is_blocked_by_file(TrackerMinerFS * fs,GFile * file)1202 item_queue_is_blocked_by_file (TrackerMinerFS *fs,
1203 GFile *file)
1204 {
1205 g_return_val_if_fail (G_IS_FILE (file), FALSE);
1206
1207 if (fs->priv->item_queue_blocker != NULL &&
1208 (fs->priv->item_queue_blocker == file ||
1209 g_file_equal (fs->priv->item_queue_blocker, file))) {
1210 return TRUE;
1211 }
1212
1213 return FALSE;
1214 }
1215
1216 static void
sparql_buffer_task_finished_cb(GObject * object,GAsyncResult * result,gpointer user_data)1217 sparql_buffer_task_finished_cb (GObject *object,
1218 GAsyncResult *result,
1219 gpointer user_data)
1220 {
1221 TrackerMinerFS *fs;
1222 TrackerMinerFSPrivate *priv;
1223 TrackerTask *task;
1224 GFile *task_file;
1225 gboolean recursive;
1226 GError *error = NULL;
1227
1228 fs = user_data;
1229 priv = fs->priv;
1230
1231 task = tracker_sparql_buffer_push_finish (TRACKER_SPARQL_BUFFER (object),
1232 result, &error);
1233
1234 if (error) {
1235 g_critical ("Could not execute sparql: %s", error->message);
1236 priv->total_files_notified_error++;
1237 g_error_free (error);
1238 }
1239
1240 task_file = tracker_task_get_file (task);
1241
1242 recursive = GPOINTER_TO_INT (g_object_steal_qdata (G_OBJECT (task_file),
1243 priv->quark_recursive_removal));
1244 tracker_file_notifier_invalidate_file_iri (priv->file_notifier, task_file, recursive);
1245
1246 if (item_queue_is_blocked_by_file (fs, task_file)) {
1247 g_object_unref (priv->item_queue_blocker);
1248 priv->item_queue_blocker = NULL;
1249 }
1250
1251 if (priv->item_queue_blocker != NULL) {
1252 if (tracker_task_pool_get_size (TRACKER_TASK_POOL (object)) > 0) {
1253 tracker_sparql_buffer_flush (TRACKER_SPARQL_BUFFER (object),
1254 "Item queue still blocked after flush");
1255
1256 /* Check if we've finished inserting for given prefixes ... */
1257 notify_roots_finished (fs, TRUE);
1258 }
1259 } else {
1260 item_queue_handlers_set_up (fs);
1261 }
1262
1263 tracker_task_unref (task);
1264 }
1265
1266 static UpdateProcessingTaskContext *
update_processing_task_context_new(TrackerMiner * miner,gint priority,const gchar * urn,GCancellable * cancellable)1267 update_processing_task_context_new (TrackerMiner *miner,
1268 gint priority,
1269 const gchar *urn,
1270 GCancellable *cancellable)
1271 {
1272 UpdateProcessingTaskContext *ctxt;
1273
1274 ctxt = g_slice_new0 (UpdateProcessingTaskContext);
1275 ctxt->miner = miner;
1276 ctxt->urn = g_strdup (urn);
1277 ctxt->priority = priority;
1278
1279 if (cancellable) {
1280 ctxt->cancellable = g_object_ref (cancellable);
1281 }
1282
1283 return ctxt;
1284 }
1285
1286 static void
update_processing_task_context_free(UpdateProcessingTaskContext * ctxt)1287 update_processing_task_context_free (UpdateProcessingTaskContext *ctxt)
1288 {
1289 g_free (ctxt->urn);
1290
1291 if (ctxt->cancellable) {
1292 g_object_unref (ctxt->cancellable);
1293 }
1294
1295 g_slice_free (UpdateProcessingTaskContext, ctxt);
1296 }
1297
1298 static void
on_signal_gtask_complete(GObject * source,GAsyncResult * res,gpointer user_data)1299 on_signal_gtask_complete (GObject *source,
1300 GAsyncResult *res,
1301 gpointer user_data)
1302 {
1303 TrackerMinerFS *fs = TRACKER_MINER_FS (source);
1304 TrackerTask *task, *sparql_task = NULL;
1305 UpdateProcessingTaskContext *ctxt;
1306 GError *error = NULL;
1307 GFile *file = user_data;
1308 gchar *uri, *sparql;
1309
1310 sparql = g_task_propagate_pointer (G_TASK (res), &error);
1311 g_object_unref (res);
1312
1313 task = tracker_task_pool_find (fs->priv->task_pool, file);
1314 g_assert (task != NULL);
1315
1316 ctxt = tracker_task_get_data (task);
1317 uri = g_file_get_uri (file);
1318
1319 if (error) {
1320 g_message ("Could not process '%s': %s", uri, error->message);
1321 g_error_free (error);
1322
1323 fs->priv->total_files_notified_error++;
1324 } else {
1325 fs->priv->total_files_notified++;
1326
1327 if (ctxt->urn) {
1328 /* The SPARQL builder will already contain the necessary
1329 * DELETE statements for the properties being updated */
1330 g_debug ("Updating item '%s' with urn '%s'",
1331 uri,
1332 ctxt->urn);
1333 } else {
1334 g_debug ("Creating new item '%s'", uri);
1335 }
1336
1337 sparql_task = tracker_sparql_task_new_take_sparql_str (file, sparql);
1338 }
1339
1340 if (sparql_task) {
1341 tracker_sparql_buffer_push (fs->priv->sparql_buffer,
1342 sparql_task,
1343 ctxt->priority,
1344 sparql_buffer_task_finished_cb,
1345 fs);
1346
1347 if (item_queue_is_blocked_by_file (fs, file)) {
1348 tracker_sparql_buffer_flush (fs->priv->sparql_buffer, "Current file is blocking item queue");
1349
1350 /* Check if we've finished inserting for given prefixes ... */
1351 notify_roots_finished (fs, TRUE);
1352 }
1353
1354 /* We can let go of our reference here because the
1355 * sparql buffer takes its own reference when adding
1356 * it to the task pool.
1357 */
1358 tracker_task_unref (sparql_task);
1359 } else {
1360 if (item_queue_is_blocked_by_file (fs, file)) {
1361 /* Make sure that we don't stall the item queue, although we could
1362 * expect the file to be reenqueued until the loop detector makes
1363 * us drop it since we were specifically waiting for it to complete.
1364 */
1365 g_object_unref (fs->priv->item_queue_blocker);
1366 fs->priv->item_queue_blocker = NULL;
1367 item_queue_handlers_set_up (fs);
1368 }
1369 }
1370
1371 /* Last reference is kept by the pool, removing the task from
1372 * the pool cleans up the task too!
1373 *
1374 * NOTE that calling this any earlier actually causes invalid
1375 * reads because the task frees up the
1376 * UpdateProcessingTaskContext and GFile.
1377 */
1378 tracker_task_pool_remove (fs->priv->task_pool, task);
1379
1380 if (tracker_miner_fs_has_items_to_process (fs) == FALSE &&
1381 tracker_task_pool_get_size (TRACKER_TASK_POOL (fs->priv->task_pool)) == 0) {
1382 /* We need to run this one more time to trigger process_stop() */
1383 item_queue_handlers_set_up (fs);
1384 }
1385
1386 g_free (uri);
1387 }
1388
1389 static gboolean
item_add_or_update(TrackerMinerFS * fs,GFile * file,gint priority,gboolean attributes_update)1390 item_add_or_update (TrackerMinerFS *fs,
1391 GFile *file,
1392 gint priority,
1393 gboolean attributes_update)
1394 {
1395 TrackerMinerFSPrivate *priv;
1396 UpdateProcessingTaskContext *ctxt;
1397 GCancellable *cancellable;
1398 gboolean processing;
1399 TrackerTask *task;
1400 const gchar *urn;
1401 gchar *uri;
1402 GTask *gtask;
1403
1404 priv = fs->priv;
1405
1406 cancellable = g_cancellable_new ();
1407 g_object_ref (file);
1408
1409 urn = tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
1410 file, FALSE);
1411
1412 /* Create task and add it to the pool as a WAIT task (we need to extract
1413 * the file metadata and such) */
1414 ctxt = update_processing_task_context_new (TRACKER_MINER (fs),
1415 priority,
1416 urn,
1417 cancellable);
1418 task = tracker_task_new (file, ctxt,
1419 (GDestroyNotify) update_processing_task_context_free);
1420
1421 tracker_task_pool_add (priv->task_pool, task);
1422 tracker_task_unref (task);
1423
1424 /* Call ::process-file to see if we handle this resource or not */
1425 uri = g_file_get_uri (file);
1426
1427 gtask = g_task_new (fs, ctxt->cancellable, on_signal_gtask_complete, file);
1428
1429 if (!attributes_update) {
1430 g_debug ("Processing file '%s'...", uri);
1431 g_signal_emit (fs, signals[PROCESS_FILE], 0,
1432 file, gtask,
1433 &processing);
1434 } else {
1435 g_debug ("Processing attributes in file '%s'...", uri);
1436 g_signal_emit (fs, signals[PROCESS_FILE_ATTRIBUTES], 0,
1437 file, gtask,
1438 &processing);
1439 }
1440
1441 if (!processing) {
1442 GError *error;
1443
1444 error = g_error_new (tracker_miner_fs_error_quark (),
1445 TRACKER_MINER_FS_ERROR_INIT,
1446 "TrackerMinerFS::process-file returned FALSE");
1447 g_task_return_error (gtask, error);
1448 } else {
1449 fs->priv->total_files_processed++;
1450 }
1451
1452 g_free (uri);
1453 g_object_unref (file);
1454 g_object_unref (cancellable);
1455
1456 return !tracker_task_pool_limit_reached (priv->task_pool);
1457 }
1458
1459 static gboolean
item_remove(TrackerMinerFS * fs,GFile * file,gboolean only_children,GString * task_sparql)1460 item_remove (TrackerMinerFS *fs,
1461 GFile *file,
1462 gboolean only_children,
1463 GString *task_sparql)
1464 {
1465 gchar *uri, *sparql;
1466 guint signal_num;
1467
1468 uri = g_file_get_uri (file);
1469
1470 g_debug ("Removing item: '%s' (Deleted from filesystem or no longer monitored)",
1471 uri);
1472
1473 g_object_set_qdata (G_OBJECT (file),
1474 fs->priv->quark_recursive_removal,
1475 GINT_TO_POINTER (TRUE));
1476
1477 /* Call the implementation to generate a SPARQL update for the removal. */
1478 signal_num = only_children ? REMOVE_CHILDREN : REMOVE_FILE;
1479 g_signal_emit (fs, signals[signal_num], 0, file, &sparql);
1480
1481 if (sparql && sparql[0] != '\0') {
1482 g_string_append (task_sparql, sparql);
1483 g_string_append (task_sparql, ";\n");
1484 }
1485
1486 g_free (sparql);
1487 g_free (uri);
1488
1489 return TRUE;
1490 }
1491
1492 static gboolean
item_move(TrackerMinerFS * fs,GFile * dest_file,GFile * source_file,GString * dest_task_sparql,GString * source_task_sparql)1493 item_move (TrackerMinerFS *fs,
1494 GFile *dest_file,
1495 GFile *source_file,
1496 GString *dest_task_sparql,
1497 GString *source_task_sparql)
1498 {
1499 gchar *uri, *source_uri, *sparql;
1500 GFileInfo *file_info;
1501 const gchar *source_iri;
1502 gboolean source_exists;
1503 TrackerDirectoryFlags source_flags, flags;
1504 gboolean recursive;
1505
1506 uri = g_file_get_uri (dest_file);
1507 source_uri = g_file_get_uri (source_file);
1508
1509 /* FIXME: Should check the _NO_STAT on TrackerDirectoryFlags first! */
1510 file_info = g_file_query_info (dest_file,
1511 G_FILE_ATTRIBUTE_STANDARD_TYPE,
1512 G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
1513 NULL, NULL);
1514
1515 /* Get 'source' ID */
1516 source_iri = tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
1517 source_file, TRUE);
1518 source_exists = (source_iri != NULL);
1519
1520 if (!file_info) {
1521 gboolean retval;
1522
1523 if (source_exists) {
1524 /* Destination file has gone away, ignore dest file and remove source if any */
1525 retval = item_remove (fs, source_file, FALSE, source_task_sparql);
1526 } else {
1527 /* Destination file went away, and source wasn't indexed either */
1528 retval = TRUE;
1529 }
1530
1531 g_free (source_uri);
1532 g_free (uri);
1533
1534 return retval;
1535 } else if (!source_exists) {
1536 gboolean retval;
1537
1538 /* The source file might not be indexed yet (eg. temporary save
1539 * files that are immediately renamed to the definitive path).
1540 * Deal with those as newly added items.
1541 */
1542 g_debug ("Source file '%s' not yet in store, indexing '%s' "
1543 "from scratch", source_uri, uri);
1544
1545 retval = item_add_or_update (fs, dest_file, G_PRIORITY_DEFAULT, FALSE);
1546
1547 g_free (source_uri);
1548 g_free (uri);
1549 g_object_unref (file_info);
1550
1551 return retval;
1552 }
1553
1554 g_debug ("Moving item from '%s' to '%s'",
1555 source_uri,
1556 uri);
1557
1558 tracker_indexing_tree_get_root (fs->priv->indexing_tree, source_file, &source_flags);
1559 tracker_indexing_tree_get_root (fs->priv->indexing_tree, dest_file, &flags);
1560 recursive = ((source_flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0 &&
1561 (flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0 &&
1562 g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY);
1563
1564 /* Delete destination item from store if any */
1565 item_remove (fs, dest_file, FALSE, dest_task_sparql);
1566
1567 /* If the original location is recursive, but the destination location
1568 * is not, remove all children.
1569 */
1570 if (!recursive &&
1571 (source_flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0)
1572 item_remove (fs, source_file, TRUE, source_task_sparql);
1573
1574 g_signal_emit (fs, signals[MOVE_FILE], 0, dest_file, source_file, recursive, &sparql);
1575
1576 if (sparql && sparql[0] != '\0') {
1577 /* This is treated as a task on dest_file */
1578 g_string_append (dest_task_sparql, sparql);
1579 g_string_append (dest_task_sparql, ";\n");
1580 }
1581
1582 g_free (sparql);
1583 g_free (uri);
1584 g_free (source_uri);
1585 g_object_unref (file_info);
1586
1587 return TRUE;
1588 }
1589
1590 static gboolean
should_wait(TrackerMinerFS * fs,GFile * file)1591 should_wait (TrackerMinerFS *fs,
1592 GFile *file)
1593 {
1594 GFile *parent;
1595
1596 /* Is the item already being processed? */
1597 if (tracker_task_pool_find (fs->priv->task_pool, file) ||
1598 tracker_task_pool_find (TRACKER_TASK_POOL (fs->priv->sparql_buffer), file)) {
1599 /* Yes, a previous event on same item currently
1600 * being processed */
1601 fs->priv->item_queue_blocker = g_object_ref (file);
1602 return TRUE;
1603 }
1604
1605 /* Is the item's parent being processed right now? */
1606 parent = g_file_get_parent (file);
1607 if (parent) {
1608 if (tracker_task_pool_find (fs->priv->task_pool, parent) ||
1609 tracker_task_pool_find (TRACKER_TASK_POOL (fs->priv->sparql_buffer), parent)) {
1610 /* Yes, a previous event on the parent of this item
1611 * currently being processed */
1612 fs->priv->item_queue_blocker = parent;
1613 return TRUE;
1614 }
1615
1616 g_object_unref (parent);
1617 }
1618 return FALSE;
1619 }
1620
1621 static gboolean
item_queue_get_next_file(TrackerMinerFS * fs,GFile ** file,GFile ** source_file,TrackerMinerFSEventType * type,gint * priority_out,gboolean * attributes_update)1622 item_queue_get_next_file (TrackerMinerFS *fs,
1623 GFile **file,
1624 GFile **source_file,
1625 TrackerMinerFSEventType *type,
1626 gint *priority_out,
1627 gboolean *attributes_update)
1628 {
1629 QueueEvent *event;
1630 gint priority;
1631
1632 *file = NULL;
1633 *source_file = NULL;
1634
1635 if (tracker_file_notifier_is_active (fs->priv->file_notifier) ||
1636 tracker_task_pool_limit_reached (fs->priv->task_pool) ||
1637 tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
1638 if (tracker_task_pool_get_size (fs->priv->task_pool) == 0) {
1639 fs->priv->extraction_timer_stopped = TRUE;
1640 g_timer_stop (fs->priv->extraction_timer);
1641 }
1642
1643 /* There are still pending items to crawl,
1644 * or extract pool limit is reached
1645 */
1646 return FALSE;
1647 }
1648
1649 event = tracker_priority_queue_peek (fs->priv->items, &priority);
1650
1651 if (event) {
1652 if (should_wait (fs, event->file) ||
1653 (event->dest_file && should_wait (fs, event->dest_file))) {
1654 return FALSE;
1655 }
1656
1657 if (event->type == TRACKER_MINER_FS_EVENT_MOVED) {
1658 g_set_object (file, event->dest_file);
1659 g_set_object (source_file, event->file);
1660 } else {
1661 g_set_object (file, event->file);
1662 }
1663
1664 *type = event->type;
1665 *priority_out = priority;
1666 *attributes_update = event->attributes_update;
1667
1668 queue_event_free (event);
1669 tracker_priority_queue_pop (fs->priv->items, NULL);
1670 }
1671
1672 return TRUE;
1673 }
1674
1675 static gdouble
item_queue_get_progress(TrackerMinerFS * fs,guint * n_items_processed,guint * n_items_remaining)1676 item_queue_get_progress (TrackerMinerFS *fs,
1677 guint *n_items_processed,
1678 guint *n_items_remaining)
1679 {
1680 guint items_to_process = 0;
1681 guint items_total = 0;
1682
1683 items_to_process += tracker_priority_queue_get_length (fs->priv->items);
1684
1685 items_total += fs->priv->total_directories_found;
1686 items_total += fs->priv->total_files_found;
1687
1688 if (n_items_processed) {
1689 *n_items_processed = ((items_total >= items_to_process) ?
1690 (items_total - items_to_process) : 0);
1691 }
1692
1693 if (n_items_remaining) {
1694 *n_items_remaining = items_to_process;
1695 }
1696
1697 if (items_total == 0 ||
1698 items_to_process == 0 ||
1699 items_to_process > items_total) {
1700 return 1.0;
1701 }
1702
1703 return (gdouble) (items_total - items_to_process) / items_total;
1704 }
1705
1706 /* Add a task to the processing pool to update stored information on 'file'.
1707 *
1708 * This function takes ownership of the 'sparql' string.
1709 */
1710 static void
push_task(TrackerMinerFS * fs,GFile * file,gchar * sparql)1711 push_task (TrackerMinerFS *fs,
1712 GFile *file,
1713 gchar *sparql)
1714 {
1715 TrackerTask *task;
1716
1717 task = tracker_sparql_task_new_take_sparql_str (file, sparql);
1718 tracker_sparql_buffer_push (fs->priv->sparql_buffer,
1719 task,
1720 G_PRIORITY_DEFAULT,
1721 sparql_buffer_task_finished_cb,
1722 fs);
1723 tracker_task_unref (task);
1724 }
1725
1726 static gboolean
miner_handle_next_item(TrackerMinerFS * fs)1727 miner_handle_next_item (TrackerMinerFS *fs)
1728 {
1729 GFile *file = NULL;
1730 GFile *source_file = NULL;
1731 GFile *parent;
1732 GTimeVal time_now;
1733 static GTimeVal time_last = { 0 };
1734 gboolean keep_processing = TRUE;
1735 gboolean attributes_update = FALSE;
1736 TrackerMinerFSEventType type;
1737 gint priority = 0;
1738 GString *task_sparql = NULL;
1739 GString *source_task_sparql = NULL;
1740
1741 if (fs->priv->timer_stopped) {
1742 g_timer_start (fs->priv->timer);
1743 fs->priv->timer_stopped = FALSE;
1744 }
1745
1746 if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
1747 /* Task pool is full, give it a break */
1748 return FALSE;
1749 }
1750
1751 if (!item_queue_get_next_file (fs, &file, &source_file, &type, &priority, &attributes_update)) {
1752 /* We should flush the processing pool buffer here, because
1753 * if there was a previous task on the same file we want to
1754 * process now, we want it to get finished before we can go
1755 * on with the queues... */
1756 tracker_sparql_buffer_flush (fs->priv->sparql_buffer,
1757 "Queue handlers WAIT");
1758
1759 /* Check if we've finished inserting for given prefixes ... */
1760 notify_roots_finished (fs, TRUE);
1761
1762 /* Items are still being processed, so wait until
1763 * the processing pool is cleared before starting with
1764 * the next directories batch.
1765 */
1766 return FALSE;
1767 }
1768
1769 if (file == NULL) {
1770 g_timer_stop (fs->priv->extraction_timer);
1771 fs->priv->extraction_timer_stopped = TRUE;
1772 } else if (fs->priv->extraction_timer_stopped) {
1773 g_timer_continue (fs->priv->extraction_timer);
1774 fs->priv->extraction_timer_stopped = FALSE;
1775 }
1776
1777 /* Update progress, but don't spam it. */
1778 g_get_current_time (&time_now);
1779
1780 if ((time_now.tv_sec - time_last.tv_sec) >= 1) {
1781 guint items_processed, items_remaining;
1782 gdouble progress_now;
1783 static gdouble progress_last = 0.0;
1784 static gint info_last = 0;
1785 gdouble seconds_elapsed, extraction_elapsed;
1786
1787 time_last = time_now;
1788
1789 /* Update progress? */
1790 progress_now = item_queue_get_progress (fs,
1791 &items_processed,
1792 &items_remaining);
1793 seconds_elapsed = g_timer_elapsed (fs->priv->timer, NULL);
1794 extraction_elapsed = g_timer_elapsed (fs->priv->extraction_timer, NULL);
1795
1796 if (!tracker_file_notifier_is_active (fs->priv->file_notifier)) {
1797 gchar *status;
1798 gint remaining_time;
1799
1800 g_object_get (fs, "status", &status, NULL);
1801
1802 /* Compute remaining time */
1803 remaining_time = (gint)tracker_seconds_estimate (extraction_elapsed,
1804 items_processed,
1805 items_remaining);
1806
1807 /* CLAMP progress so it doesn't go back below
1808 * 2% (which we use for crawling)
1809 */
1810 if (g_strcmp0 (status, "Processing…") != 0) {
1811 /* Don't spam this */
1812 g_info ("Processing…");
1813 g_object_set (fs,
1814 "status", "Processing…",
1815 "progress", CLAMP (progress_now, 0.02, 1.00),
1816 "remaining-time", remaining_time,
1817 NULL);
1818 } else {
1819 g_object_set (fs,
1820 "progress", CLAMP (progress_now, 0.02, 1.00),
1821 "remaining-time", remaining_time,
1822 NULL);
1823 }
1824
1825 g_free (status);
1826 }
1827
1828 if (++info_last >= 5 &&
1829 (gint) (progress_last * 100) != (gint) (progress_now * 100)) {
1830 gchar *str1, *str2;
1831
1832 info_last = 0;
1833 progress_last = progress_now;
1834
1835 /* Log estimated remaining time */
1836 str1 = tracker_seconds_estimate_to_string (extraction_elapsed,
1837 TRUE,
1838 items_processed,
1839 items_remaining);
1840 str2 = tracker_seconds_to_string (seconds_elapsed, TRUE);
1841
1842 g_info ("Processed %u/%u, estimated %s left, %s elapsed",
1843 items_processed,
1844 items_processed + items_remaining,
1845 str1,
1846 str2);
1847
1848 g_free (str2);
1849 g_free (str1);
1850 }
1851 }
1852
1853 if (file == NULL) {
1854 if (!tracker_file_notifier_is_active (fs->priv->file_notifier) &&
1855 tracker_task_pool_get_size (fs->priv->task_pool) == 0) {
1856 if (tracker_task_pool_get_size (TRACKER_TASK_POOL (fs->priv->sparql_buffer)) == 0) {
1857 /* Print stats and signal finished */
1858 process_stop (fs);
1859 } else {
1860 /* Flush any possible pending update here */
1861 tracker_sparql_buffer_flush (fs->priv->sparql_buffer,
1862 "Queue handlers NONE");
1863
1864 /* Check if we've finished inserting for given prefixes ... */
1865 notify_roots_finished (fs, TRUE);
1866 }
1867 }
1868
1869 /* No more files left to process */
1870 return FALSE;
1871 }
1872
1873 /* Handle queues */
1874 switch (type) {
1875 case TRACKER_MINER_FS_EVENT_MOVED:
1876 task_sparql = g_string_new ("");
1877 source_task_sparql = g_string_new ("");
1878 keep_processing = item_move (fs, file, source_file, task_sparql, source_task_sparql);
1879 break;
1880 case TRACKER_MINER_FS_EVENT_DELETED:
1881 task_sparql = g_string_new ("");
1882 keep_processing = item_remove (fs, file, FALSE, task_sparql);
1883 break;
1884 case TRACKER_MINER_FS_EVENT_CREATED:
1885 case TRACKER_MINER_FS_EVENT_UPDATED:
1886 parent = g_file_get_parent (file);
1887
1888 if (!parent ||
1889 tracker_indexing_tree_file_is_root (fs->priv->indexing_tree, file) ||
1890 !tracker_indexing_tree_get_root (fs->priv->indexing_tree, file, NULL) ||
1891 tracker_file_notifier_get_file_iri (fs->priv->file_notifier, parent, TRUE)) {
1892 keep_processing = item_add_or_update (fs, file, priority, attributes_update);
1893 } else {
1894 gchar *uri;
1895
1896 /* We got an event on a file that has not its parent indexed
1897 * even though it should. Given item_queue_get_next_file()
1898 * above should return FALSE whenever the parent file is
1899 * being processed, this means the parent is neither
1900 * being processed nor indexed, no good.
1901 *
1902 * Bail out in these cases by removing all queued files
1903 * inside the missing file. Whatever it was, it shall
1904 * hopefully be fixed on next index.
1905 */
1906 uri = g_file_get_uri (parent);
1907 g_warning ("Parent '%s' not indexed yet", uri);
1908 g_free (uri);
1909
1910 tracker_priority_queue_foreach_remove (fs->priv->items,
1911 (GEqualFunc) queue_event_is_equal_or_descendant,
1912 parent,
1913 (GDestroyNotify) queue_event_free);
1914 keep_processing = TRUE;
1915 }
1916
1917 if (parent) {
1918 g_object_unref (parent);
1919 }
1920
1921 break;
1922 default:
1923 g_assert_not_reached ();
1924 }
1925
1926 if (source_task_sparql) {
1927 if (source_task_sparql->len == 0) {
1928 g_string_free (source_task_sparql, TRUE);
1929 } else {
1930 push_task (fs, source_file, g_string_free (source_task_sparql, FALSE));
1931 }
1932 }
1933
1934 if (task_sparql) {
1935 if (task_sparql->len == 0) {
1936 g_string_free (task_sparql, TRUE);
1937 } else {
1938 push_task (fs, file, g_string_free (task_sparql, FALSE));
1939 }
1940 }
1941
1942 if (!tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
1943 item_queue_handlers_set_up (fs);
1944 }
1945
1946 if (file) {
1947 g_object_unref (file);
1948 }
1949
1950 if (source_file) {
1951 g_object_unref (source_file);
1952 }
1953
1954 return keep_processing;
1955 }
1956
1957 static gboolean
item_queue_handlers_cb(gpointer user_data)1958 item_queue_handlers_cb (gpointer user_data)
1959 {
1960 TrackerMinerFS *fs = user_data;
1961 gboolean retval = FALSE;
1962 gint i;
1963
1964 for (i = 0; i < MAX_SIMULTANEOUS_ITEMS; i++) {
1965 retval = miner_handle_next_item (fs);
1966 if (retval == FALSE)
1967 break;
1968 }
1969
1970 if (retval == FALSE) {
1971 fs->priv->item_queues_handler_id = 0;
1972 }
1973
1974 return retval;
1975 }
1976
1977 static guint
_tracker_idle_add(TrackerMinerFS * fs,GSourceFunc func,gpointer user_data)1978 _tracker_idle_add (TrackerMinerFS *fs,
1979 GSourceFunc func,
1980 gpointer user_data)
1981 {
1982 guint interval;
1983
1984 interval = TRACKER_CRAWLER_MAX_TIMEOUT_INTERVAL * fs->priv->throttle;
1985
1986 if (interval == 0) {
1987 return g_idle_add_full (TRACKER_TASK_PRIORITY, func, user_data, NULL);
1988 } else {
1989 return g_timeout_add_full (TRACKER_TASK_PRIORITY, interval, func, user_data, NULL);
1990 }
1991 }
1992
1993 static void
item_queue_handlers_set_up(TrackerMinerFS * fs)1994 item_queue_handlers_set_up (TrackerMinerFS *fs)
1995 {
1996 trace_eq ("Setting up queue handlers...");
1997 if (fs->priv->item_queues_handler_id != 0) {
1998 trace_eq (" cancelled: already one active");
1999 return;
2000 }
2001
2002 if (fs->priv->is_paused) {
2003 trace_eq (" cancelled: paused");
2004 return;
2005 }
2006
2007 if (fs->priv->item_queue_blocker) {
2008 trace_eq (" cancelled: item queue blocked waiting for file '%s'",
2009 g_file_get_uri (fs->priv->item_queue_blocker));
2010 return;
2011 }
2012
2013 /* Already processing max number of sparql updates */
2014 if (tracker_task_pool_limit_reached (fs->priv->task_pool)) {
2015 trace_eq (" cancelled: pool limit reached (tasks: %u (max %u)",
2016 tracker_task_pool_get_size (fs->priv->task_pool),
2017 tracker_task_pool_get_limit (fs->priv->task_pool));
2018 return;
2019 }
2020
2021 if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
2022 trace_eq (" cancelled: pool limit reached (sparql buffer: %u)",
2023 tracker_task_pool_get_limit (TRACKER_TASK_POOL (fs->priv->sparql_buffer)));
2024 return;
2025 }
2026
2027 if (!tracker_file_notifier_is_active (fs->priv->file_notifier)) {
2028 gchar *status;
2029 gdouble progress;
2030
2031 g_object_get (fs,
2032 "progress", &progress,
2033 "status", &status,
2034 NULL);
2035
2036 /* Don't spam this */
2037 if (progress > 0.01 && g_strcmp0 (status, "Processing…") != 0) {
2038 g_info ("Processing…");
2039 g_object_set (fs, "status", "Processing…", NULL);
2040 }
2041
2042 g_free (status);
2043 }
2044
2045 trace_eq (" scheduled in idle");
2046 fs->priv->item_queues_handler_id =
2047 _tracker_idle_add (fs,
2048 item_queue_handlers_cb,
2049 fs);
2050 }
2051
2052 static gboolean
should_check_file(TrackerMinerFS * fs,GFile * file,gboolean is_dir)2053 should_check_file (TrackerMinerFS *fs,
2054 GFile *file,
2055 gboolean is_dir)
2056 {
2057 GFileType file_type;
2058
2059 file_type = (is_dir) ? G_FILE_TYPE_DIRECTORY : G_FILE_TYPE_REGULAR;
2060 return tracker_indexing_tree_file_is_indexable (fs->priv->indexing_tree,
2061 file, file_type);
2062 }
2063
2064 static gint
miner_fs_get_queue_priority(TrackerMinerFS * fs,GFile * file)2065 miner_fs_get_queue_priority (TrackerMinerFS *fs,
2066 GFile *file)
2067 {
2068 TrackerDirectoryFlags flags;
2069
2070 tracker_indexing_tree_get_root (fs->priv->indexing_tree,
2071 file, &flags);
2072
2073 return (flags & TRACKER_DIRECTORY_FLAG_PRIORITY) ?
2074 G_PRIORITY_HIGH : G_PRIORITY_DEFAULT;
2075 }
2076
2077 static void
miner_fs_queue_event(TrackerMinerFS * fs,QueueEvent * event,guint priority)2078 miner_fs_queue_event (TrackerMinerFS *fs,
2079 QueueEvent *event,
2080 guint priority)
2081 {
2082 GList *old = NULL, *link = NULL;
2083
2084 if (event->type == TRACKER_MINER_FS_EVENT_MOVED) {
2085 /* Remove all children of the dest location from being processed. */
2086 tracker_priority_queue_foreach_remove (fs->priv->items,
2087 (GEqualFunc) queue_event_is_equal_or_descendant,
2088 event->dest_file,
2089 (GDestroyNotify) queue_event_free);
2090 }
2091
2092 old = queue_event_get_last_event_node (event);
2093
2094 if (old) {
2095 QueueCoalesceAction action;
2096 QueueEvent *replacement = NULL;
2097
2098 action = queue_event_coalesce (old->data, event, &replacement);
2099
2100 if (action & QUEUE_ACTION_DELETE_FIRST) {
2101 queue_event_free (old->data);
2102 tracker_priority_queue_remove_node (fs->priv->items,
2103 old);
2104 }
2105
2106 if (action & QUEUE_ACTION_DELETE_SECOND) {
2107 queue_event_free (event);
2108 event = NULL;
2109 }
2110
2111 if (replacement)
2112 event = replacement;
2113 }
2114
2115 if (event) {
2116 if (event->type == TRACKER_MINER_FS_EVENT_DELETED) {
2117 /* Remove all children of this file from being processed. */
2118 tracker_priority_queue_foreach_remove (fs->priv->items,
2119 (GEqualFunc) queue_event_is_equal_or_descendant,
2120 event->file,
2121 (GDestroyNotify) queue_event_free);
2122 }
2123
2124 /* Ensure IRI is cached */
2125 tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
2126 event->file, TRUE);
2127
2128 link = tracker_priority_queue_add (fs->priv->items, event, priority);
2129 queue_event_save_node (event, link);
2130 item_queue_handlers_set_up (fs);
2131 }
2132 }
2133
2134 static gboolean
filter_event(TrackerMinerFS * fs,TrackerMinerFSEventType type,GFile * file,GFile * source_file)2135 filter_event (TrackerMinerFS *fs,
2136 TrackerMinerFSEventType type,
2137 GFile *file,
2138 GFile *source_file)
2139 {
2140 TrackerMinerFSClass *klass = TRACKER_MINER_FS_GET_CLASS (fs);
2141
2142 if (!klass->filter_event)
2143 return FALSE;
2144
2145 return klass->filter_event (fs, type, file, source_file);
2146 }
2147
2148 static void
file_notifier_file_created(TrackerFileNotifier * notifier,GFile * file,gpointer user_data)2149 file_notifier_file_created (TrackerFileNotifier *notifier,
2150 GFile *file,
2151 gpointer user_data)
2152 {
2153 TrackerMinerFS *fs = user_data;
2154 QueueEvent *event;
2155
2156 if (filter_event (fs, TRACKER_MINER_FS_EVENT_CREATED, file, NULL))
2157 return;
2158
2159 event = queue_event_new (TRACKER_MINER_FS_EVENT_CREATED, file);
2160 miner_fs_queue_event (fs, event, miner_fs_get_queue_priority (fs, file));
2161 }
2162
2163 static void
file_notifier_file_deleted(TrackerFileNotifier * notifier,GFile * file,gpointer user_data)2164 file_notifier_file_deleted (TrackerFileNotifier *notifier,
2165 GFile *file,
2166 gpointer user_data)
2167 {
2168 TrackerMinerFS *fs = user_data;
2169 QueueEvent *event;
2170
2171 if (filter_event (fs, TRACKER_MINER_FS_EVENT_DELETED, file, NULL))
2172 return;
2173
2174 if (tracker_file_notifier_get_file_type (notifier, file) == G_FILE_TYPE_DIRECTORY) {
2175 /* Cancel all pending tasks on files inside the path given by file */
2176 tracker_task_pool_foreach (fs->priv->task_pool,
2177 task_pool_cancel_foreach,
2178 file);
2179 }
2180
2181 event = queue_event_new (TRACKER_MINER_FS_EVENT_DELETED, file);
2182 miner_fs_queue_event (fs, event, miner_fs_get_queue_priority (fs, file));
2183 }
2184
2185 static void
file_notifier_file_updated(TrackerFileNotifier * notifier,GFile * file,gboolean attributes_only,gpointer user_data)2186 file_notifier_file_updated (TrackerFileNotifier *notifier,
2187 GFile *file,
2188 gboolean attributes_only,
2189 gpointer user_data)
2190 {
2191 TrackerMinerFS *fs = user_data;
2192 QueueEvent *event;
2193
2194 if (!attributes_only &&
2195 filter_event (fs, TRACKER_MINER_FS_EVENT_UPDATED, file, NULL))
2196 return;
2197
2198 event = queue_event_new (TRACKER_MINER_FS_EVENT_UPDATED, file);
2199 event->attributes_update = attributes_only;
2200 miner_fs_queue_event (fs, event, miner_fs_get_queue_priority (fs, file));
2201 }
2202
2203 static void
file_notifier_file_moved(TrackerFileNotifier * notifier,GFile * source,GFile * dest,gpointer user_data)2204 file_notifier_file_moved (TrackerFileNotifier *notifier,
2205 GFile *source,
2206 GFile *dest,
2207 gpointer user_data)
2208 {
2209 TrackerMinerFS *fs = user_data;
2210 QueueEvent *event;
2211
2212 if (filter_event (fs, TRACKER_MINER_FS_EVENT_MOVED, dest, source))
2213 return;
2214
2215 event = queue_event_moved_new (source, dest);
2216 miner_fs_queue_event (fs, event, miner_fs_get_queue_priority (fs, source));
2217 }
2218
2219 static void
file_notifier_directory_started(TrackerFileNotifier * notifier,GFile * directory,gpointer user_data)2220 file_notifier_directory_started (TrackerFileNotifier *notifier,
2221 GFile *directory,
2222 gpointer user_data)
2223 {
2224 TrackerMinerFS *fs = user_data;
2225 TrackerDirectoryFlags flags;
2226 gchar *str, *uri;
2227
2228 uri = g_file_get_uri (directory);
2229 tracker_indexing_tree_get_root (fs->priv->indexing_tree,
2230 directory, &flags);
2231
2232 if ((flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0) {
2233 str = g_strdup_printf ("Crawling recursively directory '%s'", uri);
2234 } else {
2235 str = g_strdup_printf ("Crawling single directory '%s'", uri);
2236 }
2237
2238 if (fs->priv->timer_stopped) {
2239 g_timer_start (fs->priv->timer);
2240 fs->priv->timer_stopped = FALSE;
2241 }
2242
2243 if (fs->priv->extraction_timer_stopped) {
2244 g_timer_start (fs->priv->timer);
2245 fs->priv->extraction_timer_stopped = FALSE;
2246 }
2247
2248 /* Always set the progress here to at least 1%, and the remaining time
2249 * to -1 as we cannot guess during crawling (we don't know how many directories
2250 * we will find) */
2251 g_object_set (fs,
2252 "progress", 0.01,
2253 "status", str,
2254 "remaining-time", -1,
2255 NULL);
2256 g_free (str);
2257 g_free (uri);
2258 }
2259
2260 static void
file_notifier_directory_finished(TrackerFileNotifier * notifier,GFile * directory,guint directories_found,guint directories_ignored,guint files_found,guint files_ignored,gpointer user_data)2261 file_notifier_directory_finished (TrackerFileNotifier *notifier,
2262 GFile *directory,
2263 guint directories_found,
2264 guint directories_ignored,
2265 guint files_found,
2266 guint files_ignored,
2267 gpointer user_data)
2268 {
2269 TrackerMinerFS *fs = user_data;
2270 gchar *str, *uri;
2271
2272 /* Update stats */
2273 fs->priv->total_directories_found += directories_found;
2274 fs->priv->total_directories_ignored += directories_ignored;
2275 fs->priv->total_files_found += files_found;
2276 fs->priv->total_files_ignored += files_ignored;
2277
2278 uri = g_file_get_uri (directory);
2279 str = g_strdup_printf ("Crawl finished for directory '%s'", uri);
2280
2281 g_object_set (fs,
2282 "progress", 0.01,
2283 "status", str,
2284 "remaining-time", -1,
2285 NULL);
2286
2287 g_free (str);
2288 g_free (uri);
2289
2290 if (directories_found == 0 &&
2291 files_found == 0) {
2292 /* Signal now because we have nothing to index */
2293 g_signal_emit (fs, signals[FINISHED_ROOT], 0, directory);
2294 } else {
2295 /* Add root to list we want to be notified about when
2296 * finished indexing! */
2297 g_hash_table_replace (fs->priv->roots_to_notify,
2298 g_object_ref (directory),
2299 GUINT_TO_POINTER(time(NULL)));
2300 }
2301 }
2302
2303 static void
file_notifier_finished(TrackerFileNotifier * notifier,gpointer user_data)2304 file_notifier_finished (TrackerFileNotifier *notifier,
2305 gpointer user_data)
2306 {
2307 TrackerMinerFS *fs = user_data;
2308
2309 if (!tracker_miner_fs_has_items_to_process (fs)) {
2310 g_info ("Finished all tasks");
2311 process_stop (fs);
2312 } else {
2313 item_queue_handlers_set_up (fs);
2314 }
2315 }
2316
2317
2318 #ifdef CRAWLED_TREE_ENABLE_TRACE
2319
2320 static gboolean
print_file_tree(GNode * node,gpointer user_data)2321 print_file_tree (GNode *node,
2322 gpointer user_data)
2323 {
2324 gchar *name;
2325 gint i;
2326
2327 name = g_file_get_basename (node->data);
2328
2329 /* Indentation */
2330 for (i = g_node_depth (node) - 1; i > 0; i--) {
2331 g_print (" ");
2332 }
2333
2334 g_print ("%s\n", name);
2335 g_free (name);
2336
2337 return FALSE;
2338 }
2339
2340 #endif /* CRAWLED_TREE_ENABLE_TRACE */
2341
2342 static void
task_pool_cancel_foreach(gpointer data,gpointer user_data)2343 task_pool_cancel_foreach (gpointer data,
2344 gpointer user_data)
2345 {
2346 TrackerTask *task = data;
2347 GFile *file = user_data;
2348 GFile *task_file;
2349 UpdateProcessingTaskContext *ctxt;
2350
2351 ctxt = tracker_task_get_data (task);
2352 task_file = tracker_task_get_file (task);
2353
2354 if (ctxt &&
2355 ctxt->cancellable &&
2356 (!file ||
2357 (g_file_equal (task_file, file) ||
2358 g_file_has_prefix (task_file, file)))) {
2359 g_cancellable_cancel (ctxt->cancellable);
2360 }
2361 }
2362
2363 static void
indexing_tree_directory_removed(TrackerIndexingTree * indexing_tree,GFile * directory,gpointer user_data)2364 indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree,
2365 GFile *directory,
2366 gpointer user_data)
2367 {
2368 TrackerMinerFS *fs = user_data;
2369 TrackerMinerFSPrivate *priv = fs->priv;
2370 GTimer *timer = g_timer_new ();
2371
2372 /* Cancel all pending tasks on files inside the path given by file */
2373 tracker_task_pool_foreach (priv->task_pool,
2374 task_pool_cancel_foreach,
2375 directory);
2376
2377 g_debug (" Cancelled processing pool tasks at %f\n", g_timer_elapsed (timer, NULL));
2378
2379 /* Remove anything contained in the removed directory
2380 * from all relevant processing queues.
2381 */
2382 tracker_priority_queue_foreach_remove (priv->items,
2383 (GEqualFunc) queue_event_is_equal_or_descendant,
2384 directory,
2385 (GDestroyNotify) queue_event_free);
2386
2387 g_debug (" Removed files at %f\n", g_timer_elapsed (timer, NULL));
2388 g_timer_destroy (timer);
2389 }
2390
2391 static gboolean
check_file_parents(TrackerMinerFS * fs,GFile * file)2392 check_file_parents (TrackerMinerFS *fs,
2393 GFile *file)
2394 {
2395 GFile *parent, *root;
2396 GList *parents = NULL, *p;
2397 QueueEvent *event;
2398
2399 parent = g_file_get_parent (file);
2400
2401 if (!parent) {
2402 return FALSE;
2403 }
2404
2405 root = tracker_indexing_tree_get_root (fs->priv->indexing_tree,
2406 parent, NULL);
2407 if (!root) {
2408 g_object_unref (parent);
2409 return FALSE;
2410 }
2411
2412 /* Add parent directories until we're past the config dir */
2413 while (parent &&
2414 !g_file_has_prefix (root, parent)) {
2415 parents = g_list_prepend (parents, parent);
2416 parent = g_file_get_parent (parent);
2417 }
2418
2419 /* Last parent fetched is not added to the list */
2420 if (parent) {
2421 g_object_unref (parent);
2422 }
2423
2424 for (p = parents; p; p = p->next) {
2425 event = queue_event_new (TRACKER_MINER_FS_EVENT_UPDATED, p->data);
2426 miner_fs_queue_event (fs, event, miner_fs_get_queue_priority (fs, p->data));
2427 g_object_unref (p->data);
2428 }
2429
2430 g_list_free (parents);
2431
2432 return TRUE;
2433 }
2434
2435 /**
2436 * tracker_miner_fs_check_file:
2437 * @fs: a #TrackerMinerFS
2438 * @file: #GFile for the file to check
2439 * @priority: the priority of the check task
2440 * @check_parents: whether to check parents and eligibility or not
2441 *
2442 * Tells the filesystem miner to check and index a file at
2443 * a given priority, this file must be part of the usual
2444 * crawling directories of #TrackerMinerFS. See
2445 * tracker_indexing_tree_add().
2446 *
2447 * Since: 0.10
2448 **/
2449 void
tracker_miner_fs_check_file(TrackerMinerFS * fs,GFile * file,gint priority,gboolean check_parents)2450 tracker_miner_fs_check_file (TrackerMinerFS *fs,
2451 GFile *file,
2452 gint priority,
2453 gboolean check_parents)
2454 {
2455 gboolean should_process = TRUE;
2456 QueueEvent *event;
2457 gchar *uri;
2458
2459 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
2460 g_return_if_fail (G_IS_FILE (file));
2461
2462 if (check_parents) {
2463 should_process = should_check_file (fs, file, FALSE);
2464 }
2465
2466 uri = g_file_get_uri (file);
2467
2468 g_debug ("%s:'%s' (FILE) (requested by application)",
2469 should_process ? "Found " : "Ignored",
2470 uri);
2471
2472 if (should_process) {
2473 if (check_parents && !check_file_parents (fs, file)) {
2474 return;
2475 }
2476
2477 trace_eq_push_tail ("UPDATED", file, "Requested by application");
2478 tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
2479 file, TRUE);
2480
2481 event = queue_event_new (TRACKER_MINER_FS_EVENT_UPDATED, file);
2482 miner_fs_queue_event (fs, event, priority);
2483 }
2484
2485 g_free (uri);
2486 }
2487
2488 /**
2489 * tracker_miner_fs_notify_finish:
2490 * @fs: a #TrackerMinerFS
2491 * @task: a #GTask obtained in a #TrackerMinerFS signal/vmethod
2492 * @sparql: (nullable): Resulting sparql for the given operation, or %NULL if
2493 * there is an error
2494 * @error: a #GError with the error that happened during processing, or %NULL.
2495 *
2496 * Notifies @fs that all processing on @file has been finished, if any error
2497 * happened during file data processing, it should be passed in @error, else
2498 * @sparql should contain correct SPARQL representing the operation in
2499 * particular.
2500 *
2501 * This function is expected to be called in reaction to all #TrackerMinerFS
2502 * signals
2503 **/
2504 void
tracker_miner_fs_notify_finish(TrackerMinerFS * fs,GTask * task,const gchar * sparql,GError * error)2505 tracker_miner_fs_notify_finish (TrackerMinerFS *fs,
2506 GTask *task,
2507 const gchar *sparql,
2508 GError *error)
2509 {
2510 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
2511 g_return_if_fail (G_IS_TASK (task));
2512 g_return_if_fail (sparql || error);
2513
2514 if (error)
2515 g_task_return_error (task, error);
2516 else
2517 g_task_return_pointer (task, g_strdup (sparql), g_free);
2518 }
2519
2520 /**
2521 * tracker_miner_fs_set_throttle:
2522 * @fs: a #TrackerMinerFS
2523 * @throttle: a double between 0.0 and 1.0
2524 *
2525 * Tells the filesystem miner to throttle its operations. A value of
2526 * 0.0 means no throttling at all, so the miner will perform
2527 * operations at full speed, 1.0 is the slowest value. With a value of
2528 * 1.0, the @fs is typically waiting one full second before handling
2529 * the next batch of queued items to be processed.
2530 *
2531 * Since: 0.8
2532 **/
2533 void
tracker_miner_fs_set_throttle(TrackerMinerFS * fs,gdouble throttle)2534 tracker_miner_fs_set_throttle (TrackerMinerFS *fs,
2535 gdouble throttle)
2536 {
2537 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
2538
2539 throttle = CLAMP (throttle, 0, 1);
2540
2541 if (fs->priv->throttle == throttle) {
2542 return;
2543 }
2544
2545 fs->priv->throttle = throttle;
2546
2547 /* Update timeouts */
2548 if (fs->priv->item_queues_handler_id != 0) {
2549 g_source_remove (fs->priv->item_queues_handler_id);
2550
2551 fs->priv->item_queues_handler_id =
2552 _tracker_idle_add (fs,
2553 item_queue_handlers_cb,
2554 fs);
2555 }
2556 }
2557
2558 /**
2559 * tracker_miner_fs_get_throttle:
2560 * @fs: a #TrackerMinerFS
2561 *
2562 * Gets the current throttle value, see
2563 * tracker_miner_fs_set_throttle() for more details.
2564 *
2565 * Returns: a double representing a value between 0.0 and 1.0.
2566 *
2567 * Since: 0.8
2568 **/
2569 gdouble
tracker_miner_fs_get_throttle(TrackerMinerFS * fs)2570 tracker_miner_fs_get_throttle (TrackerMinerFS *fs)
2571 {
2572 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), 0);
2573
2574 return fs->priv->throttle;
2575 }
2576
2577 /**
2578 * tracker_miner_fs_get_urn:
2579 * @fs: a #TrackerMinerFS
2580 * @file: a #GFile obtained in #TrackerMinerFS::process-file
2581 *
2582 * If the item exists in the store, this function retrieves
2583 * the URN for a #GFile being currently processed.
2584
2585 * If @file is not being currently processed by @fs, or doesn't
2586 * exist in the store yet, %NULL will be returned.
2587 *
2588 * Returns: (transfer none) (nullable): The URN containing the data associated to @file,
2589 * or %NULL.
2590 *
2591 * Since: 0.8
2592 **/
2593 const gchar *
tracker_miner_fs_get_urn(TrackerMinerFS * fs,GFile * file)2594 tracker_miner_fs_get_urn (TrackerMinerFS *fs,
2595 GFile *file)
2596 {
2597 TrackerTask *task;
2598
2599 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
2600 g_return_val_if_fail (G_IS_FILE (file), NULL);
2601
2602 /* Check if found in currently processed data */
2603 task = tracker_task_pool_find (fs->priv->task_pool, file);
2604
2605 if (!task) {
2606 gchar *uri;
2607
2608 uri = g_file_get_uri (file);
2609
2610 g_critical ("File '%s' is not being currently processed, "
2611 "so the URN cannot be retrieved.", uri);
2612 g_free (uri);
2613
2614 return NULL;
2615 } else {
2616 UpdateProcessingTaskContext *ctxt;
2617
2618 /* We are only storing the URN in the created/updated tasks */
2619 ctxt = tracker_task_get_data (task);
2620
2621 if (!ctxt) {
2622 gchar *uri;
2623
2624 uri = g_file_get_uri (file);
2625 g_critical ("File '%s' is being processed, but not as a "
2626 "CREATED/UPDATED task, so cannot get URN",
2627 uri);
2628 g_free (uri);
2629 return NULL;
2630 }
2631
2632 return ctxt->urn;
2633 }
2634 }
2635
2636 /**
2637 * tracker_miner_fs_query_urn:
2638 * @fs: a #TrackerMinerFS
2639 * @file: a #GFile
2640 *
2641 * If the item exists in the store, this function retrieves
2642 * the URN of the given #GFile
2643
2644 * If @file doesn't exist in the store yet, %NULL will be returned.
2645 *
2646 * Returns: (transfer full): A newly allocated string with the URN containing the data associated
2647 * to @file, or %NULL.
2648 *
2649 * Since: 0.10
2650 **/
2651 gchar *
tracker_miner_fs_query_urn(TrackerMinerFS * fs,GFile * file)2652 tracker_miner_fs_query_urn (TrackerMinerFS *fs,
2653 GFile *file)
2654 {
2655 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
2656 g_return_val_if_fail (G_IS_FILE (file), NULL);
2657
2658 return g_strdup (tracker_file_notifier_get_file_iri (fs->priv->file_notifier, file, TRUE));
2659 }
2660
2661 /**
2662 * tracker_miner_fs_has_items_to_process:
2663 * @fs: a #TrackerMinerFS
2664 *
2665 * The @fs keeps many priority queus for content it is processing.
2666 * This function returns %TRUE if the sum of all (or any) priority
2667 * queues is more than 0. This includes items deleted, created,
2668 * updated, moved or being written back.
2669 *
2670 * Returns: %TRUE if there are items to process in the internal
2671 * queues, otherwise %FALSE.
2672 *
2673 * Since: 0.10
2674 **/
2675 gboolean
tracker_miner_fs_has_items_to_process(TrackerMinerFS * fs)2676 tracker_miner_fs_has_items_to_process (TrackerMinerFS *fs)
2677 {
2678 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
2679
2680 if (tracker_file_notifier_is_active (fs->priv->file_notifier) ||
2681 !tracker_priority_queue_is_empty (fs->priv->items)) {
2682 return TRUE;
2683 }
2684
2685 return FALSE;
2686 }
2687
2688 /**
2689 * tracker_miner_fs_get_indexing_tree:
2690 * @fs: a #TrackerMinerFS
2691 *
2692 * Returns the #TrackerIndexingTree which determines
2693 * what files/directories are indexed by @fs
2694 *
2695 * Returns: (transfer none): The #TrackerIndexingTree
2696 * holding the indexing configuration
2697 **/
2698 TrackerIndexingTree *
tracker_miner_fs_get_indexing_tree(TrackerMinerFS * fs)2699 tracker_miner_fs_get_indexing_tree (TrackerMinerFS *fs)
2700 {
2701 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
2702
2703 return fs->priv->indexing_tree;
2704 }
2705
2706 /**
2707 * tracker_miner_fs_get_data_provider:
2708 * @fs: a #TrackerMinerFS
2709 *
2710 * Returns the #TrackerDataProvider implementation, which is being used
2711 * to supply #GFile and #GFileInfo content to Tracker.
2712 *
2713 * Returns: (transfer none): The #TrackerDataProvider supplying content
2714 *
2715 * Since: 1.2
2716 **/
2717 TrackerDataProvider *
tracker_miner_fs_get_data_provider(TrackerMinerFS * fs)2718 tracker_miner_fs_get_data_provider (TrackerMinerFS *fs)
2719 {
2720 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
2721
2722 return fs->priv->data_provider;
2723 }
2724
2725 #ifdef EVENT_QUEUE_ENABLE_TRACE
2726
2727 static void
trace_events_foreach(gpointer data,gpointer fs)2728 trace_events_foreach (gpointer data,
2729 gpointer fs)
2730 {
2731 QueueEvent *event = data;
2732 gchar *uri, *dest_uri = NULL;
2733
2734 uri = g_file_get_uri (event->file);
2735 if (event->dest_file)
2736 dest_uri = g_file_get_uri (event->dest_file);
2737
2738 trace_eq ("(%d) '%s' '%s'",
2739 event->type, uri, dest_uri);
2740
2741 g_free (dest_uri);
2742 g_free (uri);
2743 }
2744
2745 static gboolean
miner_fs_queues_status_trace_timeout_cb(gpointer data)2746 miner_fs_queues_status_trace_timeout_cb (gpointer data)
2747 {
2748 TrackerMinerFS *fs = data;
2749
2750 trace_eq ("(%s) Queue '%s' has %u elements:",
2751 G_OBJECT_TYPE_NAME (fs),
2752 queue_name,
2753 tracker_priority_queue_get_length (queue));
2754 tracker_priority_queue_foreach (queue,
2755 trace_events_foreach,
2756 fs);
2757
2758 return G_SOURCE_CONTINUE;
2759 }
2760
2761 #endif /* EVENT_QUEUE_ENABLE_TRACE */
2762