1 /*
2 * GNOME Online Miners - crawls through your online content
3 * Copyright (c) 2013 Red Hat, Inc.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 * 02110-1301, USA.
19 *
20 * Author: Marek Chalupa <mchalupa@redhat.com>
21 */
22
23 #include "config.h"
24
25 #include <goa/goa.h>
26 #include <grilo.h>
27
28 #include "gom-flickr-miner.h"
29 #include "gom-utils.h"
30
31 #define MINER_IDENTIFIER "gd:flickr:miner:3c63f509-23e8-4283-8aed-154bb55ef07b"
32
33 G_DEFINE_TYPE (GomFlickrMiner, gom_flickr_miner, GOM_TYPE_MINER)
34
35 struct _GomFlickrMinerPrivate {
36 GQueue *boxes;
37 };
38
39 typedef enum {
40 OP_FETCH_ALL,
41 OP_CREATE_HIEARCHY
42 } OpType;
43
44 typedef struct {
45 GrlMedia *media;
46 GrlMedia *parent;
47 } FlickrEntry;
48
49 typedef struct {
50 FlickrEntry *parent_entry;
51 GCancellable *cancellable;
52 GHashTable *previous_resources;
53 GMainLoop *loop;
54 GomAccountMinerJob *job;
55 GrlSource *source;
56 TrackerSparqlConnection *connection;
57 const gchar *datasource_urn;
58 const gchar *source_id;
59 } SyncData;
60
61 static void account_miner_job_browse_container (GomAccountMinerJob *job,
62 TrackerSparqlConnection *connection,
63 GHashTable *previous_resources,
64 const gchar *datasource_urn,
65 FlickrEntry *entry,
66 GCancellable *cancellable);
67
68 static FlickrEntry *
create_entry(GrlMedia * media,GrlMedia * parent)69 create_entry (GrlMedia *media, GrlMedia *parent)
70 {
71 FlickrEntry *entry;
72
73 entry = g_slice_new0 (FlickrEntry);
74
75 entry->media = (media != NULL) ? g_object_ref (media) : NULL;
76 entry->parent = (parent != NULL) ? g_object_ref (parent) : NULL;
77
78 return entry;
79 }
80
81 static void
free_entry(FlickrEntry * entry)82 free_entry (FlickrEntry *entry)
83 {
84 g_clear_object (&entry->media);
85 g_clear_object (&entry->parent);
86 g_slice_free (FlickrEntry, entry);
87 }
88
89 static GrlOperationOptions *
get_grl_options(GrlSource * source)90 get_grl_options (GrlSource *source)
91 {
92 GrlCaps *caps;
93 GrlOperationOptions *opts = NULL;
94
95 caps = grl_source_get_caps (source, GRL_OP_BROWSE);
96 opts = grl_operation_options_new (caps);
97
98 g_return_val_if_fail (opts != NULL, NULL);
99
100 grl_operation_options_set_resolution_flags (opts, GRL_RESOLVE_FAST_ONLY);
101
102 return opts;
103 }
104
105 static gboolean
account_miner_job_process_entry(GomAccountMinerJob * job,TrackerSparqlConnection * connection,GHashTable * previous_resources,const gchar * datasource_urn,OpType op_type,FlickrEntry * entry,GCancellable * cancellable,GError ** error)106 account_miner_job_process_entry (GomAccountMinerJob *job,
107 TrackerSparqlConnection *connection,
108 GHashTable *previous_resources,
109 const gchar *datasource_urn,
110 OpType op_type,
111 FlickrEntry *entry,
112 GCancellable *cancellable,
113 GError **error)
114 {
115 GDateTime *created_time, *modification_date;
116 gchar *contact_resource;
117 gchar *mime;
118 gchar *resource = NULL;
119 gchar *date, *identifier;
120 const gchar *class = NULL, *id;
121 const gchar *url;
122 gboolean resource_exists, mtime_changed;
123 gint64 new_mtime;
124
125 if (op_type == OP_CREATE_HIEARCHY && entry->parent == NULL && !grl_media_is_container (entry->media))
126 return TRUE;
127
128 id = grl_media_get_id (entry->media);
129 identifier = g_strdup_printf ("%sflickr:%s",
130 grl_media_is_container (entry->media) ?
131 "photos:collection:" : "",
132 id);
133
134 /* remove from the list of the previous resources */
135 g_hash_table_remove (previous_resources, identifier);
136
137 if (grl_media_is_container (entry->media))
138 class = "nfo:DataContainer";
139 else
140 class = "nmm:Photo";
141
142 resource = gom_tracker_sparql_connection_ensure_resource
143 (connection,
144 cancellable, error,
145 &resource_exists,
146 datasource_urn, identifier,
147 "nfo:RemoteDataObject", class, NULL);
148
149 if (*error != NULL)
150 goto out;
151
152 gom_tracker_update_datasource (connection, datasource_urn,
153 resource_exists, identifier, resource,
154 cancellable, error);
155
156 if (*error != NULL)
157 goto out;
158
159 if (entry->parent != NULL)
160 {
161 gchar *parent_resource_urn, *parent_identifier;
162 const gchar *parent_id;
163
164 parent_identifier = g_strconcat ("photos:collection:flickr:",
165 grl_media_get_id (entry->parent) , NULL);
166 parent_resource_urn = gom_tracker_sparql_connection_ensure_resource
167 (connection, cancellable, error,
168 NULL,
169 datasource_urn, parent_identifier,
170 "nfo:RemoteDataObject", "nfo:DataContainer", NULL);
171 g_free (parent_identifier);
172
173 if (*error != NULL)
174 goto out;
175
176 gom_tracker_sparql_connection_insert_or_replace_triple
177 (connection,
178 cancellable, error,
179 datasource_urn, resource,
180 "nie:isPartOf", parent_resource_urn);
181 g_free (parent_resource_urn);
182
183 if (*error != NULL)
184 goto out;
185 }
186
187 gom_tracker_sparql_connection_insert_or_replace_triple
188 (connection,
189 cancellable, error,
190 datasource_urn, resource,
191 "nie:title", grl_media_get_title (entry->media));
192
193 if (*error != NULL)
194 goto out;
195
196 if (op_type == OP_CREATE_HIEARCHY)
197 goto out;
198
199 /* only GRL_METADATA_KEY_CREATION_DATE is
200 * implemented, GRL_METADATA_KEY_MODIFICATION_DATE is not
201 */
202 created_time = modification_date = grl_media_get_creation_date (entry->media);
203 new_mtime = g_date_time_to_unix (modification_date);
204 mtime_changed = gom_tracker_update_mtime (connection, new_mtime,
205 resource_exists, identifier, resource,
206 cancellable, error);
207
208 if (*error != NULL)
209 goto out;
210
211 /* avoid updating the DB if the entry already exists and has not
212 * been modified since our last run.
213 */
214 if (!mtime_changed)
215 goto out;
216
217 /* the resource changed - just set all the properties again */
218 if (created_time != NULL)
219 {
220 date = gom_iso8601_from_timestamp (g_date_time_to_unix (created_time));
221 gom_tracker_sparql_connection_insert_or_replace_triple
222 (connection,
223 cancellable, error,
224 datasource_urn, resource,
225 "nie:contentCreated", date);
226 g_free (date);
227 }
228
229 if (*error != NULL)
230 goto out;
231
232 url = grl_media_get_url (entry->media);
233 gom_tracker_sparql_connection_insert_or_replace_triple
234 (connection,
235 cancellable, error,
236 datasource_urn, resource,
237 "nie:url", url);
238
239 if (*error != NULL)
240 goto out;
241
242 gom_tracker_sparql_connection_insert_or_replace_triple
243 (connection,
244 cancellable, error,
245 datasource_urn, resource,
246 "nie:description", grl_media_get_description (entry->media));
247
248 if (*error != NULL)
249 goto out;
250
251 mime = g_content_type_guess (url, NULL, 0, NULL);
252 if (mime != NULL)
253 {
254 gom_tracker_sparql_connection_insert_or_replace_triple
255 (connection,
256 cancellable, error,
257 datasource_urn, resource,
258 "nie:mimeType", mime);
259 g_free (mime);
260
261 if (*error != NULL)
262 goto out;
263 }
264
265 contact_resource = gom_tracker_utils_ensure_contact_resource
266 (connection,
267 cancellable, error,
268 datasource_urn, grl_media_get_author (entry->media));
269
270 if (*error != NULL)
271 goto out;
272
273 gom_tracker_sparql_connection_insert_or_replace_triple
274 (connection,
275 cancellable, error,
276 datasource_urn, resource,
277 "nco:creator", contact_resource);
278 g_free (contact_resource);
279
280 if (*error != NULL)
281 goto out;
282
283 out:
284 g_free (resource);
285 g_free (identifier);
286
287 if (*error != NULL)
288 return FALSE;
289
290 return TRUE;
291 }
292
293 static void
source_browse_cb(GrlSource * source,guint operation_id,GrlMedia * media,guint remaining,gpointer user_data,const GError * error)294 source_browse_cb (GrlSource *source,
295 guint operation_id,
296 GrlMedia *media,
297 guint remaining,
298 gpointer user_data,
299 const GError *error)
300 {
301 GError *local_error = NULL;
302 SyncData *data = (SyncData *) user_data;
303 GomFlickrMiner *self = GOM_FLICKR_MINER (data->job->miner);
304
305 if (error != NULL)
306 {
307 g_warning ("Unable to browse source %p: %s", source, error->message);
308 return;
309 }
310
311 if (media != NULL)
312 {
313 FlickrEntry *entry;
314
315 entry = create_entry (media, data->parent_entry->media);
316 account_miner_job_process_entry (data->job,
317 data->connection,
318 data->previous_resources,
319 data->datasource_urn,
320 OP_CREATE_HIEARCHY,
321 entry,
322 data->cancellable,
323 &local_error);
324 if (local_error != NULL)
325 {
326 g_warning ("Unable to process entry %p: %s", media, local_error->message);
327 g_error_free (local_error);
328 }
329
330 if (grl_media_is_container (media))
331 g_queue_push_tail (self->priv->boxes, entry);
332 else
333 free_entry (entry);
334 }
335
336 if (remaining == 0)
337 g_main_loop_quit (data->loop);
338 }
339
340 static void
account_miner_job_browse_container(GomAccountMinerJob * job,TrackerSparqlConnection * connection,GHashTable * previous_resources,const gchar * datasource_urn,FlickrEntry * entry,GCancellable * cancellable)341 account_miner_job_browse_container (GomAccountMinerJob *job,
342 TrackerSparqlConnection *connection,
343 GHashTable *previous_resources,
344 const gchar *datasource_urn,
345 FlickrEntry *entry,
346 GCancellable *cancellable)
347 {
348 GMainContext *context;
349 GrlSource *source;
350 GrlOperationOptions *opts;
351 const GList *keys;
352 SyncData data;
353
354 data.cancellable = cancellable;
355 data.connection = connection;
356 data.datasource_urn = datasource_urn;
357 data.parent_entry = entry;
358 data.job = job;
359 data.previous_resources = previous_resources;
360
361 context = g_main_context_new ();
362 g_main_context_push_thread_default (context);
363 data.loop = g_main_loop_new (context, FALSE);
364
365 source = GRL_SOURCE (g_hash_table_lookup (data.job->services, "photos"));
366
367 keys = grl_source_supported_keys (source);
368 opts = get_grl_options (source);
369
370 grl_source_browse (source,
371 entry->media,
372 keys,
373 opts,
374 source_browse_cb,
375 &data);
376 g_main_loop_run (data.loop);
377
378 g_object_unref (opts);
379 g_main_loop_unref (data.loop);
380 g_main_context_pop_thread_default (context);
381 g_main_context_unref (context);
382 }
383
384 static void
source_search_cb(GrlSource * source,guint operation_id,GrlMedia * media,guint remaining,gpointer user_data,const GError * error)385 source_search_cb (GrlSource *source,
386 guint operation_id,
387 GrlMedia *media,
388 guint remaining,
389 gpointer user_data,
390 const GError *error)
391 {
392 GError *local_error = NULL;
393 SyncData *data = (SyncData *) user_data;
394
395 if (error != NULL)
396 {
397 g_warning ("Unable to search source %p: %s", source, error->message);
398 return;
399 }
400
401 if (media != NULL)
402 {
403 FlickrEntry *entry;
404
405 entry = create_entry (media, NULL);
406 account_miner_job_process_entry (data->job,
407 data->connection,
408 data->previous_resources,
409 data->datasource_urn,
410 OP_FETCH_ALL,
411 entry,
412 data->cancellable,
413 &local_error);
414 if (local_error != NULL)
415 {
416 g_warning ("Unable to process entry %p: %s", media, local_error->message);
417 g_error_free (local_error);
418 }
419
420 free_entry (entry);
421 }
422
423 if (remaining == 0)
424 g_main_loop_quit (data->loop);
425 }
426
427 static void
query_flickr(GomAccountMinerJob * job,TrackerSparqlConnection * connection,GHashTable * previous_resources,const gchar * datasource_urn,GCancellable * cancellable,GError ** error)428 query_flickr (GomAccountMinerJob *job,
429 TrackerSparqlConnection *connection,
430 GHashTable *previous_resources,
431 const gchar *datasource_urn,
432 GCancellable *cancellable,
433 GError **error)
434 {
435 GomFlickrMiner *self = GOM_FLICKR_MINER (job->miner);
436 GomFlickrMinerPrivate *priv = self->priv;
437 FlickrEntry *entry;
438 const GList *keys;
439 GMainContext *context;
440 GrlOperationOptions *opts;
441 GrlSource *source;
442 SyncData data;
443
444 source = GRL_SOURCE (g_hash_table_lookup (job->services, "photos"));
445 if (source == NULL)
446 {
447 /* FIXME: use proper #defines and enumerated types */
448 g_set_error (error,
449 g_quark_from_static_string ("gom-error"),
450 0,
451 "Can not query without a service");
452 return;
453 }
454
455 /* grl_source_browse does not fetch photos that are not part of a
456 * set. So, use grl_source_search to fetch all photos and then allot
457 * each photo to any set that it might be a part of.
458 */
459
460 data.cancellable = cancellable;
461 data.connection = connection;
462 data.datasource_urn = datasource_urn;
463 data.job = job;
464 data.previous_resources = previous_resources;
465 context = g_main_context_new ();
466 g_main_context_push_thread_default (context);
467 data.loop = g_main_loop_new (context, FALSE);
468
469 keys = grl_source_supported_keys (source);
470 opts = get_grl_options (source);
471 grl_source_search (source, NULL, keys, opts, source_search_cb, &data);
472 g_main_loop_run (data.loop);
473
474 g_object_unref (opts);
475 g_main_loop_unref (data.loop);
476 g_main_context_pop_thread_default (context);
477 g_main_context_unref (context);
478
479 entry = create_entry (NULL, NULL);
480 account_miner_job_browse_container (job, connection, previous_resources, datasource_urn, entry, cancellable);
481 free_entry (entry);
482
483 while (!g_queue_is_empty (priv->boxes))
484 {
485 entry = (FlickrEntry *) g_queue_pop_head (priv->boxes);
486 account_miner_job_browse_container (job, connection, previous_resources, datasource_urn, entry, cancellable);
487 free_entry (entry);
488 }
489 }
490
491 static void
source_added_cb(GrlRegistry * registry,GrlSource * source,gpointer user_data)492 source_added_cb (GrlRegistry *registry, GrlSource *source, gpointer user_data)
493 {
494 SyncData *data = (SyncData *) user_data;
495 gchar *source_id;
496
497 g_object_get (source, "source-id", &source_id, NULL);
498 if (g_strcmp0 (source_id, data->source_id) != 0)
499 goto out;
500
501 data->source = g_object_ref (source);
502 g_main_loop_quit (data->loop);
503
504 out:
505 g_free (source_id);
506 }
507
508 static GHashTable *
create_services(GomMiner * self,GoaObject * object)509 create_services (GomMiner *self,
510 GoaObject *object)
511 {
512 GHashTable *services;
513 GoaAccount *acc;
514 GrlRegistry *registry;
515 GrlSource *source = NULL;
516 gchar *source_id = NULL;
517
518 services = g_hash_table_new_full (g_str_hash, g_str_equal,
519 NULL, (GDestroyNotify) g_object_unref);
520
521 acc = goa_object_peek_account (object);
522 if (acc == NULL)
523 goto out;
524
525 if (gom_miner_supports_type (self, "photos"))
526 {
527 source_id = g_strdup_printf ("grl-flickr-%s", goa_account_get_id (acc));
528
529 registry = grl_registry_get_default ();
530
531 g_debug ("Looking for source %s", source_id);
532 source = grl_registry_lookup_source (registry, source_id);
533 if (source == NULL)
534 {
535 GMainContext *context;
536 SyncData data;
537
538 context = g_main_context_get_thread_default ();
539 data.loop = g_main_loop_new (context, FALSE);
540 data.source_id = source_id;
541
542 g_signal_connect (registry, "source-added", G_CALLBACK (source_added_cb), &data);
543 g_main_loop_run (data.loop);
544 g_main_loop_unref (data.loop);
545
546 /* we steal the ref from data */
547 source = data.source;
548 }
549 else
550 {
551 /* freeing job calls unref upon this object */
552 g_object_ref (source);
553 }
554 g_free (source_id);
555 g_hash_table_insert (services, "photos", source);
556 }
557
558 out:
559 return services;
560 }
561
562 static void
gom_flickr_miner_finalize(GObject * object)563 gom_flickr_miner_finalize (GObject *object)
564 {
565 GomFlickrMiner *self = GOM_FLICKR_MINER (object);
566
567 g_queue_free_full (self->priv->boxes, (GDestroyNotify) free_entry);
568
569 G_OBJECT_CLASS (gom_flickr_miner_parent_class)->finalize (object);
570 }
571
572 static void
gom_flickr_miner_init(GomFlickrMiner * self)573 gom_flickr_miner_init (GomFlickrMiner *self)
574 {
575 self->priv = G_TYPE_INSTANCE_GET_PRIVATE (self, GOM_TYPE_FLICKR_MINER, GomFlickrMinerPrivate);
576 self->priv->boxes = g_queue_new ();
577 }
578
579 static void
gom_flickr_miner_class_init(GomFlickrMinerClass * klass)580 gom_flickr_miner_class_init (GomFlickrMinerClass *klass)
581 {
582 GObjectClass *oclass = G_OBJECT_CLASS (klass);
583 GomMinerClass *miner_class = GOM_MINER_CLASS (klass);
584 GrlRegistry *registry;
585 GError *error = NULL;
586
587 oclass->finalize = gom_flickr_miner_finalize;
588
589 miner_class->goa_provider_type = "flickr";
590 miner_class->miner_identifier = MINER_IDENTIFIER;
591 miner_class->version = 1;
592
593 miner_class->create_services = create_services;
594 miner_class->query = query_flickr;
595
596 grl_init (NULL, NULL);
597 registry = grl_registry_get_default ();
598 grl_registry_load_all_plugins (registry, FALSE, &error);
599
600 if (error != NULL || !grl_registry_activate_plugin_by_id (registry, "grl-flickr", &error))
601 {
602 g_error ("%s", error->message);
603 g_error_free (error);
604 }
605
606 g_type_class_add_private (klass, sizeof (GomFlickrMinerPrivate));
607 }
608