1 /* This file is part of GEGL.
2 *
3 * This library is free software; you can redistribute it and/or
4 * modify it under the terms of the GNU Lesser General Public
5 * License as published by the Free Software Foundation; either
6 * version 3 of the License, or (at your option) any later version.
7 *
8 * This library is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * Lesser General Public License for more details.
12 *
13 * You should have received a copy of the GNU Lesser General Public
14 * License along with GEGL; if not, see <https://www.gnu.org/licenses/>.
15 *
16 * Copyright 2006, 2007, 2008 Øyvind Kolås <pippin@gimp.org>
17 * 2012 Ville Sokk <ville.sokk@gmail.com>
18 */
19
20 /* GeglTileBackendFile stores tiles of a GeglBuffer on disk. There are
21 * two versions of the class. This one uses regular I/O calls in a
22 * separate thread (shared between instances of the class) that performs
23 * all file operations except reading and opening. Communication between
24 * the main gegl thread and the writer thread is performed using a
25 * queue. The writer thread sleeps if the queue is empty. If an entry is
26 * read and the tile is in the queue then its data is copied from the
27 * queue instead of read from disk. There are two locks, queue_mutex and
28 * write_mutex. The first one is used to append to the queue or read from
29 * it, the second one to completely stop the writer thread from working
30 * (to remove/change queue entries).
31 */
32
33 #include "config.h"
34
35 #include <gio/gio.h>
36 #include <sys/types.h>
37 #include <sys/stat.h>
38 #include <fcntl.h>
39 #include <unistd.h>
40 #include <string.h>
41 #include <errno.h>
42
43 #include <glib-object.h>
44 #include <glib/gprintf.h>
45 #include <glib/gstdio.h>
46
47 #include "gegl-buffer.h"
48 #include "gegl-buffer-backend.h"
49 #include "gegl-tile-backend.h"
50 #include "gegl-tile-backend-file.h"
51 #include "gegl-buffer-index.h"
52 #include "gegl-buffer-swap.h"
53 #include "gegl-buffer-types.h"
54 #include "gegl-debug.h"
55 #include "gegl-buffer-config.h"
56
57
58 #ifndef HAVE_FSYNC
59
60 #ifdef G_OS_WIN32
61 #define fsync _commit
62 #endif
63
64 #endif
65
66 #ifdef G_OS_WIN32
67 #define BINARY_FLAG O_BINARY
68 #else
69 #define BINARY_FLAG 0
70 #endif
71
72 struct _GeglTileBackendFile
73 {
74 GeglTileBackend parent_instance;
75
76 /* the path to our buffer */
77 gchar *path;
78
79 /* the file exist (and we've thus been able to initialize i and o,
80 * the utility call ensure_exist() should be called before any code
81 * using i and o)
82 */
83 gboolean exist;
84
85 /* total size of file */
86 guint total;
87
88 /* hashtable containing all entries of buffer, the index is written
89 * to the swapfile conforming to the structures laid out in
90 * gegl-buffer-index.h
91 */
92 GHashTable *index;
93
94 /* list of offsets to tiles that are free */
95 GSList *free_list;
96
97 /* offset to next pre allocated tile slot */
98 guint next_pre_alloc;
99
100 /* revision of last index sync, for cooperated sharing of a buffer
101 * file
102 */
103 guint32 rev;
104
105 /* a local copy of the header that will be written to the file, in a
106 * multiple user per buffer scenario, the flags in the header might
107 * be used for locking/signalling
108 */
109 GeglBufferHeader header;
110
111 /* cached offsets of the file handles to avoid lseek syscall if possible */
112 gint in_offset;
113 gint out_offset;
114
115 /* current offset, used when writing the index */
116 gint offset;
117
118 /* when writing buffer blocks the writer keeps one block unwritten
119 * at all times to be able to keep track of the ->next offsets in
120 * the blocks.
121 */
122 GeglFileBackendEntry *in_holding;
123
124 /* loading buffer */
125 GList *tiles;
126
127 /* GFile refering to our buffer */
128 GFile *file;
129
130 GFileMonitor *monitor;
131
132 /* number of write operations in the queue for this file */
133 gint pending_ops;
134
135 /* used for waiting on writes to the file to be finished */
136 GCond cond;
137
138 /* for writing */
139 int o;
140
141 /* for reading */
142 int i;
143 };
144
145
146 static void gegl_tile_backend_file_ensure_exist (GeglTileBackendFile *self);
147 static gboolean gegl_tile_backend_file_write_block (GeglTileBackendFile *self,
148 GeglFileBackendEntry *block);
149 static void gegl_tile_backend_file_dbg_alloc (int size);
150 static void gegl_tile_backend_file_dbg_dealloc (int size);
151
152
153 G_DEFINE_TYPE (GeglTileBackendFile, gegl_tile_backend_file, GEGL_TYPE_TILE_BACKEND)
154
155 static GObjectClass * parent_class = NULL;
156
157 /* this debugging is across all buffers */
158 static gint allocs = 0;
159 static gint file_size = 0;
160 static gint peak_allocs = 0;
161 static gint peak_file_size = 0;
162
163 static GQueue queue = G_QUEUE_INIT;
164 static GMutex mutex = { 0, };
165 static GCond queue_cond = { 0, };
166 static GCond max_cond = { 0, };
167 static gint queue_size = 0;
168 static GeglFileBackendThreadParams *in_progress;
169
170
171 static void
gegl_tile_backend_file_finish_writing(GeglTileBackendFile * self)172 gegl_tile_backend_file_finish_writing (GeglTileBackendFile *self)
173 {
174 g_mutex_lock (&mutex);
175 while (self->pending_ops != 0)
176 g_cond_wait (&self->cond, &mutex);
177 g_mutex_unlock (&mutex);
178 }
179
180 static void
gegl_tile_backend_file_push_queue(GeglFileBackendThreadParams * params)181 gegl_tile_backend_file_push_queue (GeglFileBackendThreadParams *params)
182 {
183 g_mutex_lock (&mutex);
184
185 /* block if the queue has gotten too big */
186 while (queue_size > gegl_buffer_config ()->queue_size)
187 g_cond_wait (&max_cond, &mutex);
188
189 params->file->pending_ops += 1;
190 g_queue_push_tail (&queue, params);
191
192 if (params->entry)
193 {
194 if (params->operation == OP_WRITE)
195 {
196 params->entry->tile_link = g_queue_peek_tail_link (&queue);
197 queue_size += params->length + sizeof (GList) +
198 sizeof (GeglFileBackendThreadParams);
199 }
200 else /* OP_WRITE_BLOCK */
201 params->entry->block_link = g_queue_peek_tail_link (&queue);
202 }
203
204 /* wake up the writer thread */
205 g_cond_signal (&queue_cond);
206
207 g_mutex_unlock (&mutex);
208 }
209
210 static inline void
gegl_tile_backend_file_write(GeglFileBackendThreadParams * params)211 gegl_tile_backend_file_write (GeglFileBackendThreadParams *params)
212 {
213 gint to_be_written = params->length;
214 gint fd = params->file->o;
215 goffset offset = params->offset;
216
217 if (params->file->out_offset != params->offset)
218 {
219 if (lseek (fd, offset, SEEK_SET) < 0)
220 {
221 g_warning ("unable to seek to tile in buffer: %s", g_strerror (errno));
222 return;
223 }
224 params->file->out_offset = params->offset;
225 }
226
227 while (to_be_written > 0)
228 {
229 gint wrote;
230 wrote = write (fd,
231 params->source + params->length - to_be_written,
232 to_be_written);
233 if (wrote <= 0)
234 {
235 g_message ("unable to write tile data to self: "
236 "%s (%d/%d bytes written)",
237 g_strerror (errno), wrote, to_be_written);
238 break;
239 }
240
241 to_be_written -= wrote;
242 params->file->out_offset += wrote;;
243 }
244
245 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "writer thread wrote at %i", (gint)offset);
246 }
247
248 static gpointer
gegl_tile_backend_file_writer_thread(gpointer ignored)249 gegl_tile_backend_file_writer_thread (gpointer ignored)
250 {
251 while (TRUE)
252 {
253 GeglFileBackendThreadParams *params;
254
255 g_mutex_lock (&mutex);
256
257 while (g_queue_is_empty (&queue))
258 g_cond_wait (&queue_cond, &mutex);
259
260 params = (GeglFileBackendThreadParams *)g_queue_pop_head (&queue);
261 if (params->entry)
262 {
263 in_progress = params;
264 if (params->operation == OP_WRITE)
265 params->entry->tile_link = NULL;
266 else /* OP_WRITE_BLOCK */
267 params->entry->block_link = NULL;
268 }
269 g_mutex_unlock (&mutex);
270
271 switch (params->operation)
272 {
273 case OP_WRITE:
274 gegl_tile_backend_file_write (params);
275 break;
276 case OP_WRITE_BLOCK:
277 gegl_tile_backend_file_write (params);
278 break;
279 case OP_TRUNCATE:
280 if (ftruncate (params->file->o, params->length) != 0)
281 g_warning ("failed to resize file: %s", g_strerror (errno));
282 break;
283 case OP_SYNC:
284 fsync (params->file->o);
285 break;
286 }
287
288 g_mutex_lock (&mutex);
289 in_progress = NULL;
290
291 /* the file maybe waiting for its file operations to finish */
292 params->file->pending_ops -= 1;
293 if (params->file->pending_ops == 0)
294 g_cond_signal (¶ms->file->cond);
295
296 if (params->operation == OP_WRITE)
297 {
298 queue_size -= params->length + sizeof (GList) +
299 sizeof (GeglFileBackendThreadParams);
300 g_free (params->source);
301
302 /* unblock the main thread if the queue had gotten too big */
303 if (queue_size < gegl_buffer_config ()->queue_size)
304 g_cond_signal (&max_cond);
305 }
306
307 g_free (params);
308
309 g_mutex_unlock (&mutex);
310 }
311
312 return NULL;
313 }
314
315 static void
gegl_tile_backend_file_entry_read(GeglTileBackendFile * self,GeglFileBackendEntry * entry,guchar * dest)316 gegl_tile_backend_file_entry_read (GeglTileBackendFile *self,
317 GeglFileBackendEntry *entry,
318 guchar *dest)
319 {
320 gint tile_size = gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self));
321 gint to_be_read = tile_size;
322 goffset offset = entry->tile->offset;
323
324 gegl_tile_backend_file_ensure_exist (self);
325
326 if (entry->tile_link || in_progress)
327 {
328 GeglFileBackendThreadParams *queued_op = NULL;
329 g_mutex_lock (&mutex);
330
331 if (entry->tile_link)
332 queued_op = entry->tile_link->data;
333 else if (in_progress && in_progress->entry == entry &&
334 in_progress->operation == OP_WRITE)
335 queued_op = in_progress;
336
337 if (queued_op)
338 {
339 memcpy (dest, queued_op->source, to_be_read);
340 g_mutex_unlock (&mutex);
341
342 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "read entry %i,%i,%i from queue", entry->tile->x, entry->tile->y, entry->tile->z);
343
344 return;
345 }
346
347 g_mutex_unlock (&mutex);
348 }
349
350 if (self->in_offset != offset)
351 {
352 if (lseek (self->i, offset, SEEK_SET) < 0)
353 {
354 g_warning ("unable to seek to tile in buffer: %s", g_strerror (errno));
355 return;
356 }
357 self->in_offset = offset;
358 }
359
360 while (to_be_read > 0)
361 {
362 GError *error = NULL;
363 gint byte_read;
364
365 byte_read = read (self->i, dest + tile_size - to_be_read, to_be_read);
366 if (byte_read <= 0)
367 {
368 g_message ("unable to read tile data from self: "
369 "%s (%d/%d bytes read) %s",
370 g_strerror (errno), byte_read, to_be_read, error?error->message:"--");
371 return;
372 }
373 to_be_read -= byte_read;
374 self->in_offset += byte_read;
375 }
376
377 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "read entry %i,%i,%i at %i", entry->tile->x, entry->tile->y, entry->tile->z, (gint)offset);
378 }
379
380 static inline void
gegl_tile_backend_file_entry_write(GeglTileBackendFile * self,GeglFileBackendEntry * entry,guchar * source)381 gegl_tile_backend_file_entry_write (GeglTileBackendFile *self,
382 GeglFileBackendEntry *entry,
383 guchar *source)
384 {
385 GeglFileBackendThreadParams *params;
386 gint length = gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self));
387 guchar *new_source;
388
389 gegl_tile_backend_file_ensure_exist (self);
390
391 if (entry->tile_link)
392 {
393 g_mutex_lock (&mutex);
394
395 if (entry->tile_link)
396 {
397 params = entry->tile_link->data;
398 memcpy (params->source, source, length);
399 g_mutex_unlock (&mutex);
400
401 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "overwrote queue entry %i,%i,%i at %i", entry->tile->x, entry->tile->y, entry->tile->z, (gint)entry->tile->offset);
402
403 return;
404 }
405
406 g_mutex_unlock (&mutex);
407 }
408
409 new_source = g_malloc (length);
410 memcpy (new_source, source, length);
411
412 params = g_new0 (GeglFileBackendThreadParams, 1);
413 params->operation = OP_WRITE;
414 params->length = length;
415 params->offset = entry->tile->offset;
416 params->file = self;
417 params->source = new_source;
418 params->entry = entry;
419
420 gegl_tile_backend_file_push_queue (params);
421
422 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "pushed entry write %i,%i,%i at %i", entry->tile->x, entry->tile->y, entry->tile->z, (gint)entry->tile->offset);
423 }
424
425 static GeglFileBackendEntry *
gegl_tile_backend_file_file_entry_create(gint x,gint y,gint z)426 gegl_tile_backend_file_file_entry_create (gint x,
427 gint y,
428 gint z)
429 {
430 GeglFileBackendEntry *entry = g_new0 (GeglFileBackendEntry, 1);
431
432 entry->tile = gegl_tile_entry_new (x, y, z);
433 entry->tile_link = NULL;
434 entry->block_link = NULL;
435
436 return entry;
437 }
438
439 static inline GeglFileBackendEntry *
gegl_tile_backend_file_file_entry_new(GeglTileBackendFile * self)440 gegl_tile_backend_file_file_entry_new (GeglTileBackendFile *self)
441 {
442 GeglFileBackendEntry *entry = gegl_tile_backend_file_file_entry_create (0,0,0);
443
444 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "Creating new entry");
445
446 gegl_tile_backend_file_ensure_exist (self);
447
448 if (self->free_list)
449 {
450 guint64 offset = *(guint64*)self->free_list->data;
451
452 entry->tile->offset = offset;
453 self->free_list = g_slist_remove (self->free_list, self->free_list->data);
454
455 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, " set offset %i from free list", ((gint)entry->tile->offset));
456 }
457 else
458 {
459 gint tile_size = gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self));
460
461 entry->tile->offset = self->next_pre_alloc;
462 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, " set offset %i (next allocation)", (gint)entry->tile->offset);
463 self->next_pre_alloc += tile_size;
464
465 if (self->next_pre_alloc >= self->total) /* automatic growing ensuring that
466 we have room for next allocation..
467 */
468 {
469 GeglFileBackendThreadParams *params = g_new0 (GeglFileBackendThreadParams, 1);
470
471 self->total = self->total + 32 * tile_size;
472 params->operation = OP_TRUNCATE;
473 params->file = self;
474 params->length = self->total;
475
476 gegl_tile_backend_file_push_queue (params);
477
478 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "pushed truncate to %i bytes", (gint)self->total);
479
480 self->in_offset = self->out_offset = -1;
481 }
482 }
483 gegl_tile_backend_file_dbg_alloc (gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self)));
484 return entry;
485 }
486
487 static void
gegl_tile_backend_file_file_entry_destroy(GeglTileBackendFile * self,GeglFileBackendEntry * entry)488 gegl_tile_backend_file_file_entry_destroy (GeglTileBackendFile *self,
489 GeglFileBackendEntry *entry)
490 {
491 guint64 *offset = g_new (guint64, 1);
492 *offset = entry->tile->offset;
493
494 if (entry->tile_link || entry->block_link)
495 {
496 gint i;
497 GList *link;
498
499 g_mutex_lock (&mutex);
500
501 for (i = 0, link = entry->tile_link;
502 i < 2;
503 i++, link = entry->block_link)
504 {
505 if (link)
506 {
507 GeglFileBackendThreadParams *queued_op = link->data;
508 queued_op->file->pending_ops -= 1;
509 g_queue_delete_link (&queue, link);
510 g_free (queued_op->source);
511 g_free (queued_op);
512 }
513 }
514
515 g_mutex_unlock (&mutex);
516 }
517
518 self->free_list = g_slist_prepend (self->free_list, offset);
519 g_hash_table_remove (self->index, entry);
520
521 gegl_tile_backend_file_dbg_dealloc (gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self)));
522
523 g_free (entry->tile);
524 g_free (entry);
525 }
526
527 static gboolean
gegl_tile_backend_file_write_header(GeglTileBackendFile * self)528 gegl_tile_backend_file_write_header (GeglTileBackendFile *self)
529 {
530 GeglFileBackendThreadParams *params = g_new0 (GeglFileBackendThreadParams, 1);
531 guchar *new_source = g_malloc (256);
532 GeglRectangle roi = gegl_tile_backend_get_extent ((GeglTileBackend *)self);
533
534 gegl_tile_backend_file_ensure_exist (self);
535
536 self->header.x = roi.x;
537 self->header.y = roi.y;
538 self->header.width = roi.width;
539 self->header.height = roi.height;
540
541 memcpy (new_source, &(self->header), 256);
542
543 params->operation = OP_WRITE;
544 params->source = new_source;
545 params->offset = 0;
546 params->length = 256;
547 params->file = self;
548
549 gegl_tile_backend_file_push_queue (params);
550 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "pushed header write, next=%i", (gint)self->header.next);
551
552 params = g_new0 (GeglFileBackendThreadParams, 1);
553 params->operation = OP_SYNC;
554 params->file = self;
555
556 gegl_tile_backend_file_push_queue (params);
557 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "pushed sync of %s", self->path);
558
559 return TRUE;
560 }
561
562 static gboolean
gegl_tile_backend_file_write_block(GeglTileBackendFile * self,GeglFileBackendEntry * item)563 gegl_tile_backend_file_write_block (GeglTileBackendFile *self,
564 GeglFileBackendEntry *item)
565 {
566 gegl_tile_backend_file_ensure_exist (self);
567 if (self->in_holding)
568 {
569 GeglFileBackendThreadParams *params;
570 GeglBufferBlock *block = &(self->in_holding->tile->block);
571 guint64 next_allocation = self->offset + block->length;
572 gint length = block->length;
573 guchar *new_source;
574
575 /* update the next offset pointer in the previous block */
576 if (item == NULL)
577 /* the previous block was the last block */
578 block->next = 0;
579 else
580 block->next = next_allocation;
581
582 /* XXX: should promiscuosuly try to compress here as well,. if revisions
583 are not matching..
584 */
585
586 if (self->in_holding->block_link)
587 {
588 g_mutex_lock (&mutex);
589
590 if (self->in_holding->block_link)
591 {
592 params = self->in_holding->block_link->data;
593 params->offset = self->offset;
594 memcpy (params->source, block, length);
595 g_mutex_unlock (&mutex);
596
597 self->offset = next_allocation;
598 self->in_holding = item;
599 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "Overwrote queue block: length:%i flags:%i next:%i at offset %i",
600 block->length,
601 block->flags,
602 (gint)block->next,
603 (gint)self->offset);
604 return TRUE;
605 }
606
607 g_mutex_unlock (&mutex);
608 }
609
610 params = g_new0 (GeglFileBackendThreadParams, 1);
611 new_source = g_malloc (length);
612
613 memcpy (new_source, block, length);
614
615 params->operation = OP_WRITE_BLOCK;
616 params->length = length;
617 params->file = self;
618 params->offset = self->offset;
619 params->source = new_source;
620 params->entry = self->in_holding;
621
622 gegl_tile_backend_file_push_queue (params);
623
624 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "Pushed write of block: length:%i flags:%i next:%i at offset %i",
625 block->length,
626 block->flags,
627 (gint)block->next,
628 (gint)self->offset);
629
630 self->offset = next_allocation;
631 }
632 else
633 {
634 /* we're setting up for the first write */
635 self->offset = self->next_pre_alloc; /* start writing header at end
636 * of file, worry about writing
637 * header inside free list later
638 */
639 }
640 self->in_holding = item;
641
642 return TRUE;
643 }
644
645 void
gegl_tile_backend_file_stats(void)646 gegl_tile_backend_file_stats (void)
647 {
648 g_warning ("leaked: %i chunks (%f mb) peak: %i (%i bytes %fmb))",
649 allocs, file_size / 1024 / 1024.0,
650 peak_allocs, peak_file_size, peak_file_size / 1024 / 1024.0);
651 }
652
653 static void
gegl_tile_backend_file_dbg_alloc(gint size)654 gegl_tile_backend_file_dbg_alloc (gint size)
655 {
656 allocs++;
657 file_size += size;
658 if (allocs > peak_allocs)
659 peak_allocs = allocs;
660 if (file_size > peak_file_size)
661 peak_file_size = file_size;
662 }
663
664 static void
gegl_tile_backend_file_dbg_dealloc(gint size)665 gegl_tile_backend_file_dbg_dealloc (gint size)
666 {
667 allocs--;
668 file_size -= size;
669 }
670
671 static inline GeglFileBackendEntry *
gegl_tile_backend_file_lookup_entry(GeglTileBackendFile * self,gint x,gint y,gint z)672 gegl_tile_backend_file_lookup_entry (GeglTileBackendFile *self,
673 gint x,
674 gint y,
675 gint z)
676 {
677 GeglFileBackendEntry *ret = NULL;
678 GeglFileBackendEntry *key = gegl_tile_backend_file_file_entry_create (x,y,z);
679 ret = g_hash_table_lookup (self->index, key);
680 g_free (key->tile);
681 g_free (key);
682 return ret;
683 }
684
685 /* this is the only place that actually should
686 * instantiate tiles, when the cache is large enough
687 * that should make sure we don't hit this function
688 * too often.
689 */
690 static GeglTile *
gegl_tile_backend_file_get_tile(GeglTileSource * self,gint x,gint y,gint z)691 gegl_tile_backend_file_get_tile (GeglTileSource *self,
692 gint x,
693 gint y,
694 gint z)
695 {
696 GeglTileBackend *backend;
697 GeglTileBackendFile *tile_backend_file;
698 GeglFileBackendEntry *entry;
699 GeglTile *tile = NULL;
700 gint tile_size;
701
702 backend = GEGL_TILE_BACKEND (self);
703 tile_backend_file = GEGL_TILE_BACKEND_FILE (backend);
704 entry = gegl_tile_backend_file_lookup_entry (tile_backend_file, x, y, z);
705
706 if (!entry)
707 return NULL;
708
709 tile_size = gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self));
710 tile = gegl_tile_new (tile_size);
711 gegl_tile_set_rev (tile, entry->tile->rev);
712 gegl_tile_mark_as_stored (tile);
713
714 gegl_tile_backend_file_entry_read (tile_backend_file, entry, gegl_tile_get_data (tile));
715 return tile;
716 }
717
718 static gpointer
gegl_tile_backend_file_set_tile(GeglTileSource * self,GeglTile * tile,gint x,gint y,gint z)719 gegl_tile_backend_file_set_tile (GeglTileSource *self,
720 GeglTile *tile,
721 gint x,
722 gint y,
723 gint z)
724 {
725 GeglTileBackend *backend;
726 GeglTileBackendFile *tile_backend_file;
727 GeglFileBackendEntry *entry;
728
729 backend = GEGL_TILE_BACKEND (self);
730 tile_backend_file = GEGL_TILE_BACKEND_FILE (backend);
731 entry = gegl_tile_backend_file_lookup_entry (tile_backend_file, x, y, z);
732
733 if (entry == NULL)
734 {
735 entry = gegl_tile_backend_file_file_entry_new (tile_backend_file);
736 entry->tile->x = x;
737 entry->tile->y = y;
738 entry->tile->z = z;
739 g_hash_table_insert (tile_backend_file->index, entry, entry);
740 }
741 entry->tile->rev = gegl_tile_get_rev (tile);
742
743 gegl_tile_backend_file_entry_write (tile_backend_file, entry, gegl_tile_get_data (tile));
744 gegl_tile_mark_as_stored (tile);
745 return NULL;
746 }
747
748 static gpointer
gegl_tile_backend_file_void_tile(GeglTileSource * self,GeglTile * tile,gint x,gint y,gint z)749 gegl_tile_backend_file_void_tile (GeglTileSource *self,
750 GeglTile *tile,
751 gint x,
752 gint y,
753 gint z)
754 {
755 GeglTileBackend *backend;
756 GeglTileBackendFile *tile_backend_file;
757 GeglFileBackendEntry *entry;
758
759 backend = GEGL_TILE_BACKEND (self);
760 tile_backend_file = GEGL_TILE_BACKEND_FILE (backend);
761 entry = gegl_tile_backend_file_lookup_entry (tile_backend_file, x, y, z);
762
763 if (entry != NULL)
764 {
765 gegl_tile_backend_file_file_entry_destroy (tile_backend_file, entry);
766 }
767
768 return NULL;
769 }
770
771 static gpointer
gegl_tile_backend_file_exist_tile(GeglTileSource * self,GeglTile * tile,gint x,gint y,gint z)772 gegl_tile_backend_file_exist_tile (GeglTileSource *self,
773 GeglTile *tile,
774 gint x,
775 gint y,
776 gint z)
777 {
778 GeglTileBackend *backend;
779 GeglTileBackendFile *tile_backend_file;
780 GeglFileBackendEntry *entry;
781
782 backend = GEGL_TILE_BACKEND (self);
783 tile_backend_file = GEGL_TILE_BACKEND_FILE (backend);
784 entry = gegl_tile_backend_file_lookup_entry (tile_backend_file, x, y, z);
785
786 return entry!=NULL?((gpointer)0x1):NULL;
787 }
788
789 static gpointer
gegl_tile_backend_file_flush(GeglTileSource * source,GeglTile * tile,gint x,gint y,gint z)790 gegl_tile_backend_file_flush (GeglTileSource *source,
791 GeglTile *tile,
792 gint x,
793 gint y,
794 gint z)
795 {
796 GeglTileBackend *backend;
797 GeglTileBackendFile *self;
798 GList *tiles;
799
800 backend = GEGL_TILE_BACKEND (source);
801 self = GEGL_TILE_BACKEND_FILE (backend);
802
803 gegl_tile_backend_file_ensure_exist (self);
804
805 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "flushing %s", self->path);
806
807 self->header.rev ++;
808 self->header.next = self->next_pre_alloc; /* this is the offset
809 we start handing
810 out headers from*/
811 tiles = g_hash_table_get_keys (self->index);
812
813 if (tiles == NULL)
814 self->header.next = 0;
815 else
816 {
817 GList *iter;
818 for (iter = tiles; iter; iter = iter->next)
819 {
820 GeglFileBackendEntry *item = iter->data;
821
822 gegl_tile_backend_file_write_block (self, item);
823 }
824 gegl_tile_backend_file_write_block (self, NULL); /* terminate the index */
825 g_list_free (tiles);
826 }
827
828 gegl_tile_backend_file_write_header (self);
829
830 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "flushed %s", self->path);
831
832 return (gpointer)0xf0f;
833 }
834
835 enum
836 {
837 PROP_0,
838 PROP_PATH
839 };
840
841 static gpointer
gegl_tile_backend_file_command(GeglTileSource * self,GeglTileCommand command,gint x,gint y,gint z,gpointer data)842 gegl_tile_backend_file_command (GeglTileSource *self,
843 GeglTileCommand command,
844 gint x,
845 gint y,
846 gint z,
847 gpointer data)
848 {
849 switch (command)
850 {
851 case GEGL_TILE_GET:
852 return gegl_tile_backend_file_get_tile (self, x, y, z);
853 case GEGL_TILE_SET:
854 return gegl_tile_backend_file_set_tile (self, data, x, y, z);
855
856 case GEGL_TILE_IDLE:
857 return NULL; /* we could perhaps lazily be writing indexes
858 * at some intervals, making it work as an
859 * autosave for the buffer?
860 */
861
862 case GEGL_TILE_VOID:
863 return gegl_tile_backend_file_void_tile (self, data, x, y, z);
864
865 case GEGL_TILE_EXIST:
866 return gegl_tile_backend_file_exist_tile (self, data, x, y, z);
867 case GEGL_TILE_FLUSH:
868 return gegl_tile_backend_file_flush (self, data, x, y, z);
869
870 default:
871 break;
872 }
873
874 return gegl_tile_backend_command (GEGL_TILE_BACKEND (self),
875 command, x, y, z, data);
876 }
877
878 static void
gegl_tile_backend_file_set_property(GObject * object,guint property_id,const GValue * value,GParamSpec * pspec)879 gegl_tile_backend_file_set_property (GObject *object,
880 guint property_id,
881 const GValue *value,
882 GParamSpec *pspec)
883 {
884 GeglTileBackendFile *self = GEGL_TILE_BACKEND_FILE (object);
885
886 switch (property_id)
887 {
888 case PROP_PATH:
889 if (self->path)
890 g_free (self->path);
891 self->path = g_value_dup_string (value);
892 break;
893
894 default:
895 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
896 break;
897 }
898 }
899
900 static void
gegl_tile_backend_file_get_property(GObject * object,guint property_id,GValue * value,GParamSpec * pspec)901 gegl_tile_backend_file_get_property (GObject *object,
902 guint property_id,
903 GValue *value,
904 GParamSpec *pspec)
905 {
906 GeglTileBackendFile *self = GEGL_TILE_BACKEND_FILE (object);
907
908 switch (property_id)
909 {
910 case PROP_PATH:
911 g_value_set_string (value, self->path);
912 break;
913
914 default:
915 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
916 break;
917 }
918 }
919
920 static void
gegl_tile_backend_file_free_free_list(GeglTileBackendFile * self)921 gegl_tile_backend_file_free_free_list (GeglTileBackendFile *self)
922 {
923 GSList *iter = self->free_list;
924
925 for (; iter; iter = iter->next)
926 g_free (iter->data);
927
928 g_slist_free (self->free_list);
929
930 self->free_list = NULL;
931 }
932
933 static void
gegl_tile_backend_file_finalize(GObject * object)934 gegl_tile_backend_file_finalize (GObject *object)
935 {
936 GeglTileBackendFile *self = (GeglTileBackendFile *) object;
937
938 if (self->index)
939 {
940 GList *tiles = g_hash_table_get_keys (self->index);
941
942 if (tiles != NULL)
943 {
944 GList *iter;
945
946 for (iter = tiles; iter; iter = iter->next)
947 gegl_tile_backend_file_file_entry_destroy (self, iter->data);
948 }
949
950 g_list_free (tiles);
951
952 g_hash_table_unref (self->index);
953 }
954
955 if (self->exist)
956 {
957 gegl_tile_backend_file_finish_writing (self);
958 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "finalizing buffer %s", self->path);
959
960 if (self->i != -1)
961 {
962 close (self->i);
963 self->i = -1;
964 }
965 if (self->o != -1)
966 {
967 close (self->o);
968 self->o = -1;
969 }
970 }
971
972 if (self->free_list)
973 gegl_tile_backend_file_free_free_list (self);
974
975 if (self->path)
976 {
977 if (gegl_buffer_swap_has_file (self->path))
978 gegl_buffer_swap_remove_file (self->path);
979
980 g_free (self->path);
981 }
982
983 if (self->monitor)
984 {
985 g_file_monitor_cancel (self->monitor);
986 g_object_unref (self->monitor);
987 }
988
989 if (self->file)
990 g_object_unref (self->file);
991
992 g_cond_clear (&self->cond);
993
994 G_OBJECT_CLASS (parent_class)->finalize (object);
995 }
996
997 static guint
gegl_tile_backend_file_hashfunc(gconstpointer key)998 gegl_tile_backend_file_hashfunc (gconstpointer key)
999 {
1000 const GeglBufferTile *e = ((GeglFileBackendEntry *)key)->tile;
1001 guint hash;
1002 gint i;
1003 gint srcA = e->x;
1004 gint srcB = e->y;
1005 gint srcC = e->z;
1006
1007 /* interleave the 10 least significant bits of all coordinates,
1008 * this gives us Z-order / morton order of the space and should
1009 * work well as a hash
1010 */
1011 hash = 0;
1012 for (i = 9; i >= 0; i--)
1013 {
1014 #define ADD_BIT(bit) do { hash |= (((bit) != 0) ? 1 : 0); hash <<= 1; } while (0)
1015 ADD_BIT (srcA & (1 << i));
1016 ADD_BIT (srcB & (1 << i));
1017 ADD_BIT (srcC & (1 << i));
1018 #undef ADD_BIT
1019 }
1020 return hash;
1021 }
1022
1023 static gboolean
gegl_tile_backend_file_equalfunc(gconstpointer a,gconstpointer b)1024 gegl_tile_backend_file_equalfunc (gconstpointer a,
1025 gconstpointer b)
1026 {
1027 const GeglBufferTile *ea = ((GeglFileBackendEntry*)a)->tile;
1028 const GeglBufferTile *eb = ((GeglFileBackendEntry*)b)->tile;
1029
1030 if (ea->x == eb->x &&
1031 ea->y == eb->y &&
1032 ea->z == eb->z)
1033 return TRUE;
1034
1035 return FALSE;
1036 }
1037
1038
1039 static void
gegl_tile_backend_file_load_index(GeglTileBackendFile * self,gboolean block)1040 gegl_tile_backend_file_load_index (GeglTileBackendFile *self,
1041 gboolean block)
1042 {
1043 GeglBufferHeader new_header;
1044 GList *iter;
1045 GeglTileBackend *backend;
1046 goffset offset = 0;
1047 goffset max = 0;
1048 gint tile_size;
1049
1050 /* compute total from and next pre alloc by monitoring tiles as they
1051 * are added here
1052 */
1053 /* reload header */
1054 new_header = gegl_buffer_read_header (self->i, &offset)->header;
1055
1056 while (new_header.flags & GEGL_FLAG_LOCKED)
1057 {
1058 g_usleep (50000);
1059 new_header = gegl_buffer_read_header (self->i, &offset)->header;
1060 }
1061
1062 if (new_header.rev == self->header.rev)
1063 {
1064 GEGL_NOTE(GEGL_DEBUG_TILE_BACKEND, "header not changed: %s", self->path);
1065 return;
1066 }
1067 else
1068 {
1069 self->header = new_header;
1070 GEGL_NOTE(GEGL_DEBUG_TILE_BACKEND, "loading index: %s", self->path);
1071 }
1072
1073 tile_size = gegl_tile_backend_get_tile_size (GEGL_TILE_BACKEND (self));
1074 offset = self->header.next;
1075 self->tiles = gegl_buffer_read_index (self->i, &offset);
1076 self->in_offset = self->out_offset = -1;
1077 backend = GEGL_TILE_BACKEND (self);
1078
1079 for (iter = self->tiles; iter; iter=iter->next)
1080 {
1081 GeglBufferItem *item = iter->data;
1082 GeglFileBackendEntry *new;
1083 GeglFileBackendEntry *existing =
1084 gegl_tile_backend_file_lookup_entry (self, item->tile.x, item->tile.y, item->tile.z);
1085
1086 if (item->tile.offset > max)
1087 max = item->tile.offset + tile_size;
1088
1089 if (existing)
1090 {
1091 if (existing->tile->rev == item->tile.rev)
1092 {
1093 g_assert (existing->tile->offset == item->tile.offset);
1094 *existing->tile = item->tile;
1095 g_free (item);
1096 continue;
1097 }
1098 else
1099 {
1100 GeglTileStorage *storage =
1101 (void*)gegl_tile_backend_peek_storage (backend);
1102 GeglRectangle rect;
1103 g_hash_table_remove (self->index, existing);
1104
1105 gegl_tile_source_refetch (GEGL_TILE_SOURCE (storage),
1106 existing->tile->x,
1107 existing->tile->y,
1108 existing->tile->z);
1109
1110 if (existing->tile->z == 0)
1111 {
1112 rect.width = self->header.tile_width;
1113 rect.height = self->header.tile_height;
1114 rect.x = existing->tile->x * self->header.tile_width;
1115 rect.y = existing->tile->y * self->header.tile_height;
1116 }
1117 g_free (existing->tile);
1118 g_free (existing);
1119
1120 g_signal_emit_by_name (storage, "changed", &rect, NULL);
1121 }
1122 }
1123 new = gegl_tile_backend_file_file_entry_create (0, 0, 0);
1124 new->tile = iter->data;
1125 g_hash_table_insert (self->index, new, new);
1126 }
1127 g_list_free (self->tiles);
1128 gegl_tile_backend_file_free_free_list (self);
1129 self->next_pre_alloc = max; /* if bigger than own? */
1130 self->total = max;
1131 self->tiles = NULL;
1132 }
1133
1134 static void
gegl_tile_backend_file_file_changed(GFileMonitor * monitor,GFile * file,GFile * other_file,GFileMonitorEvent event_type,GeglTileBackendFile * self)1135 gegl_tile_backend_file_file_changed (GFileMonitor *monitor,
1136 GFile *file,
1137 GFile *other_file,
1138 GFileMonitorEvent event_type,
1139 GeglTileBackendFile *self)
1140 {
1141 if (event_type == G_FILE_MONITOR_EVENT_CHANGED)
1142 {
1143 gegl_tile_backend_file_load_index (self, TRUE);
1144 self->in_offset = self->out_offset = -1;
1145 }
1146 }
1147
1148 static void
gegl_tile_backend_file_constructed(GObject * object)1149 gegl_tile_backend_file_constructed (GObject *object)
1150 {
1151 GeglTileBackendFile *self = GEGL_TILE_BACKEND_FILE (object);
1152 GeglTileBackend *backend = GEGL_TILE_BACKEND (object);
1153
1154 G_OBJECT_CLASS (parent_class)->constructed (object);
1155
1156 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "constructing file backend: %s", self->path);
1157
1158 self->file = g_file_new_for_commandline_arg (self->path);
1159 self->i = self->o = -1;
1160 self->index = g_hash_table_new (gegl_tile_backend_file_hashfunc,
1161 gegl_tile_backend_file_equalfunc);
1162 self->pending_ops = 0;
1163 g_cond_init (&self->cond);
1164
1165 /* If the file already exists open it, assuming it is a GeglBuffer. */
1166 if (g_access (self->path, F_OK) != -1)
1167 {
1168 goffset offset = 0;
1169
1170 /* Install a monitor for changes to the file in case other applications
1171 * might be writing to the buffer
1172 */
1173 self->monitor = g_file_monitor_file (self->file, G_FILE_MONITOR_NONE,
1174 NULL, NULL);
1175 g_signal_connect (self->monitor, "changed",
1176 G_CALLBACK (gegl_tile_backend_file_file_changed),
1177 self);
1178
1179 self->o = g_open (self->path, O_RDWR|O_CREAT|BINARY_FLAG, 0770);
1180 if (self->o == -1)
1181 {
1182 /* Try again but this time with only read access. This is
1183 * a quick-fix for make distcheck, where img_cmp fails
1184 * when it opens a GeglBuffer file in the source tree
1185 * (which is read-only).
1186 */
1187 self->o = g_open (self->path, O_RDONLY|BINARY_FLAG, 0770);
1188
1189 if (self->o == -1)
1190 g_warning ("%s: Could not open '%s': %s", G_STRFUNC, self->path, g_strerror (errno));
1191 }
1192 self->i = g_open (self->path, O_RDONLY|BINARY_FLAG, 0);
1193
1194 self->header = gegl_buffer_read_header (self->i, &offset)->header;
1195 self->header.rev = self->header.rev -1;
1196
1197 /* we are overriding all of the work of the actual constructor here,
1198 * a really evil hack :d
1199 */
1200 backend->priv->tile_width = self->header.tile_width;
1201 backend->priv->tile_height = self->header.tile_height;
1202 backend->priv->format = babl_format (self->header.description);
1203 backend->priv->px_size = babl_format_get_bytes_per_pixel (backend->priv->format);
1204 backend->priv->tile_size = backend->priv->tile_width *
1205 backend->priv->tile_height *
1206 backend->priv->px_size;
1207 backend->priv->extent = (GeglRectangle) {self->header.x,
1208 self->header.y,
1209 self->header.width,
1210 self->header.height};
1211
1212 /* insert each of the entries into the hash table */
1213 gegl_tile_backend_file_load_index (self, TRUE);
1214 self->exist = TRUE;
1215 g_assert (self->i != -1);
1216 g_assert (self->o != -1);
1217
1218 /* to autoflush gegl_buffer_set */
1219
1220 /* XXX: poking at internals, icky */
1221 backend->priv->shared = TRUE;
1222 }
1223 else
1224 {
1225 self->exist = FALSE; /* this is also the default, the file will be created on demand */
1226 }
1227
1228 g_assert (self->file);
1229
1230 gegl_tile_backend_set_flush_on_destroy (backend, FALSE);
1231 }
1232
1233 static void
gegl_tile_backend_file_ensure_exist(GeglTileBackendFile * self)1234 gegl_tile_backend_file_ensure_exist (GeglTileBackendFile *self)
1235 {
1236 if (!self->exist)
1237 {
1238 GeglTileBackend *backend;
1239 self->exist = TRUE;
1240
1241 backend = GEGL_TILE_BACKEND (self);
1242
1243 GEGL_NOTE (GEGL_DEBUG_TILE_BACKEND, "creating swapfile %s", self->path);
1244
1245 self->o = g_open (self->path, O_RDWR|O_CREAT|BINARY_FLAG, 0770);
1246 if (self->o == -1)
1247 g_warning ("%s: Could not open '%s': %s", G_STRFUNC, self->path, g_strerror (errno));
1248
1249 self->next_pre_alloc = 256; /* reserved space for header */
1250 self->total = 256; /* reserved space for header */
1251 self->in_offset = self->out_offset = 0;
1252 self->pending_ops = 0;
1253 gegl_buffer_header_init (&self->header,
1254 backend->priv->tile_width,
1255 backend->priv->tile_height,
1256 backend->priv->px_size,
1257 backend->priv->format);
1258 gegl_tile_backend_file_write_header (self);
1259 self->i = g_open (self->path, O_RDONLY|BINARY_FLAG, 0);
1260
1261 g_assert (self->i != -1);
1262 g_assert (self->o != -1);
1263 }
1264 }
1265
1266 static void
gegl_tile_backend_file_class_init(GeglTileBackendFileClass * klass)1267 gegl_tile_backend_file_class_init (GeglTileBackendFileClass *klass)
1268 {
1269 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
1270
1271 parent_class = g_type_class_peek_parent (klass);
1272
1273 gobject_class->get_property = gegl_tile_backend_file_get_property;
1274 gobject_class->set_property = gegl_tile_backend_file_set_property;
1275 gobject_class->constructed = gegl_tile_backend_file_constructed;
1276 gobject_class->finalize = gegl_tile_backend_file_finalize;
1277
1278 g_cond_init (&queue_cond);
1279 g_cond_init (&max_cond);
1280 g_mutex_init (&mutex);
1281 g_thread_new ("GeglTileBackendFile async writer thread",
1282 gegl_tile_backend_file_writer_thread, NULL);
1283
1284 GEGL_BUFFER_STRUCT_CHECK_PADDING;
1285
1286 g_object_class_install_property (gobject_class, PROP_PATH,
1287 g_param_spec_string ("path",
1288 "path",
1289 "The base path for this backing file for a buffer",
1290 NULL,
1291 G_PARAM_CONSTRUCT_ONLY |
1292 G_PARAM_READWRITE |
1293 G_PARAM_STATIC_STRINGS));
1294 }
1295
1296 static void
gegl_tile_backend_file_init(GeglTileBackendFile * self)1297 gegl_tile_backend_file_init (GeglTileBackendFile *self)
1298 {
1299 ((GeglTileSource*)self)->command = gegl_tile_backend_file_command;
1300 self->path = NULL;
1301 self->file = NULL;
1302 self->i = -1;
1303 self->o = -1;
1304 self->index = NULL;
1305 self->free_list = NULL;
1306 self->next_pre_alloc = 256; /* reserved space for header */
1307 self->total = 256; /* reserved space for header */
1308 self->pending_ops = 0;
1309 }
1310
1311 gboolean
gegl_tile_backend_file_try_lock(GeglTileBackendFile * self)1312 gegl_tile_backend_file_try_lock (GeglTileBackendFile *self)
1313 {
1314 GeglBufferHeader new_header;
1315
1316 new_header = gegl_buffer_read_header (self->i, NULL)->header;
1317 if (new_header.flags & GEGL_FLAG_LOCKED)
1318 {
1319 return FALSE;
1320 }
1321 self->header.flags += GEGL_FLAG_LOCKED;
1322 gegl_tile_backend_file_write_header (self);
1323
1324 return TRUE;
1325 }
1326
1327 gboolean
gegl_tile_backend_file_unlock(GeglTileBackendFile * self)1328 gegl_tile_backend_file_unlock (GeglTileBackendFile *self)
1329 {
1330 if (!(self->header.flags & GEGL_FLAG_LOCKED))
1331 {
1332 g_warning ("tried to unlock unlocked buffer");
1333 return FALSE;
1334 }
1335 self->header.flags -= GEGL_FLAG_LOCKED;
1336 gegl_tile_backend_file_write_header (self);
1337
1338 /* wait until all writes to this file are finished before handing it over
1339 to another process */
1340 gegl_tile_backend_file_finish_writing (self);
1341
1342 return TRUE;
1343 }
1344