1 /*
2 For general Scribus (>=1.3.2) copyright and licensing information please refer
3 to the COPYING file provided with the program. Following this notice may exist
4 a copyright and/or license notice that predates the release of Scribus 1.3.2
5 for which a new license (GPL+exception) is in place.
6 */
7 /***************************************************************************
8 copyright : (C) 2010 by Marcus Holland-Moritz
9 email : scribus@mhxnet.de
10 ***************************************************************************/
11
12 /***************************************************************************
13 * *
14 * This program is free software; you can redistribute it and/or modify *
15 * it under the terms of the GNU General Public License as published by *
16 * the Free Software Foundation; either version 2 of the License, or *
17 * (at your option) any later version. *
18 * *
19 ***************************************************************************/
20
21 #include <QCryptographicHash>
22 #include <QXmlStreamWriter>
23 #include <QXmlStreamReader>
24 #include <QByteArray>
25 #include <QDateTime>
26 #include <QDir>
27 #include <QFile>
28 #include <QFileInfo>
29
30 #include "sclockedfile.h"
31 #include "scimagecacheproxy.h"
32 #include "scimagecachemanager.h"
33 #include "scimagecachewriteaction.h"
34 #include "scpaths.h"
35 #include "util_file.h"
36
37 #if defined(DEBUG_SCIMAGECACHE)
38 #define SC_DEBUG_FILE 1
39 #else
40 #define SC_DEBUG_FILE 0
41 #endif
42 #include "scdebug.h"
43
44 // MD5 has been chosen as a hash algorithm as it less prone to collisions than MD4,
45 // but at the same time twice as fast to compute as SHA-1. Furthermore, it's 32 bits
46 // shorter than SHA-1, making the filenames at least a little shorter.
47
48 namespace {
49 const QString CACHEFILE_VERSION("1");
50 const QCryptographicHash::Algorithm HASH_ALGORITHM = QCryptographicHash::Md5;
51 const int CACHEDIR_LEVELS = 2;
52 const char * const imageFormat = "PNG";
53
absolutePath(const QString & fn)54 inline QString absolutePath(const QString & fn)
55 {
56 return ScImageCacheManager::absolutePath(fn);
57 }
58 }
59
60 const QString ScImageCacheProxy::metaSuffix("xml");
61 const QString ScImageCacheProxy::referenceSuffix("ref");
62 const QString ScImageCacheProxy::imageSuffix("png");
63
ScImageCacheProxy(const QString & fn)64 ScImageCacheProxy::ScImageCacheProxy(const QString & fn)
65 : m_filename(fn), m_isEnabled(ScImageCacheManager::instance().enabled())
66 {
67 if (!m_isEnabled)
68 return;
69
70 QFileInfo imfo(m_filename);
71
72 if (imfo.exists())
73 {
74 addMetadata("version", CACHEFILE_VERSION);
75 addMetadata("path", m_filename);
76 addMetadata("size", QString::number(imfo.size()));
77 addMetadata("lastModifiedUTC", imfo.lastModified().toUTC().toString(Qt::ISODate));
78 }
79 }
80
~ScImageCacheProxy()81 ScImageCacheProxy::~ScImageCacheProxy()
82 {
83 // nothing :)
84 }
85
addMetadata(const QString & key,const QString & value)86 void ScImageCacheProxy::addMetadata(const QString & key, const QString & value)
87 {
88 m_metadata[key] = value;
89 }
90
addModifier(const QString & key,const QString & value)91 void ScImageCacheProxy::addModifier(const QString & key, const QString & value)
92 {
93 m_modifier[key] = value;
94 m_metanameCache.clear();
95 }
96
delModifier(const QString & key)97 void ScImageCacheProxy::delModifier(const QString & key)
98 {
99 m_modifier.remove(key);
100 m_metanameCache.clear();
101 }
102
addInfo(const QString & key,const QString & value)103 void ScImageCacheProxy::addInfo(const QString & key, const QString & value)
104 {
105 m_imginfo[key] = value;
106 }
107
getInfo(const QString & key) const108 QString ScImageCacheProxy::getInfo(const QString & key) const
109 {
110 return m_imginfo[key];
111 }
112
imageFile(const QString & base)113 QString ScImageCacheProxy::imageFile(const QString & base)
114 {
115 return base + "." + imageSuffix;
116 }
117
referenceFile(const QString & base)118 QString ScImageCacheProxy::referenceFile(const QString & base)
119 {
120 return base + "." + referenceSuffix;
121 }
122
getBaseName(const QString & metafile)123 QString ScImageCacheProxy::getBaseName(const QString & metafile)
124 {
125 QString base;
126 return loadMetadata(metafile, nullptr, nullptr, nullptr, &base) ? base : QString();
127 }
128
loadMetadata(ScLockedFile * file,MetaMap * meta,MetaMap * mod,MetaMap * info,QString * base)129 bool ScImageCacheProxy::loadMetadata(ScLockedFile *file, MetaMap *meta, MetaMap *mod, MetaMap *info, QString *base)
130 {
131 QXmlStreamReader xml(file->io());
132
133 bool baseFound = false;
134 bool metaFound = false;
135 bool modFound = false;
136 bool infoFound = false;
137
138 while (!xml.atEnd())
139 {
140 if (xml.readNext() == QXmlStreamReader::StartElement)
141 {
142 QXmlStreamAttributes attr = xml.attributes();
143
144 if (xml.name() == "cache")
145 {
146 if (attr.hasAttribute("base"))
147 {
148 if (base)
149 *base = attr.value("base").toString();
150
151 baseFound = true;
152 }
153 }
154 else if (xml.name() == "metadata")
155 {
156 if (meta)
157 {
158 meta->clear();
159
160 foreach (QXmlStreamAttribute a, attr)
161 (*meta)[a.name().toString()] = a.value().toString();
162 }
163
164 metaFound = true;
165 }
166 else if (xml.name() == "modifier")
167 {
168 if (mod)
169 {
170 mod->clear();
171
172 foreach (QXmlStreamAttribute a, attr)
173 (*mod)[a.name().toString()] = a.value().toString();
174 }
175
176 modFound = true;
177 }
178 else if (xml.name() == "imginfo")
179 {
180 if (info)
181 {
182 info->clear();
183
184 foreach (QXmlStreamAttribute a, attr)
185 (*info)[a.name().toString()] = a.value().toString();
186 }
187
188 infoFound = true;
189 }
190 }
191 }
192
193 if (xml.hasError())
194 {
195 scDebug() << "error parsing" << file->name() << xml.errorString() << "in line" << xml.lineNumber() << "column" << xml.columnNumber();
196 return false;
197 }
198
199 if (!baseFound) scDebug() << "base not found";
200 if (!metaFound) scDebug() << "meta not found";
201 if (!modFound) scDebug() << "mod not found";
202 if (!infoFound) scDebug() << "info not found";
203
204 return baseFound && metaFound && modFound && infoFound;
205 }
206
loadMetadata(const QString & fn,MetaMap * meta,MetaMap * mod,MetaMap * info,QString * base)207 bool ScImageCacheProxy::loadMetadata(const QString & fn, MetaMap *meta, MetaMap *mod, MetaMap *info, QString *base)
208 {
209 ScLockedFileRO file(absolutePath(fn));
210 if (!file.open())
211 {
212 scDebug() << "failed to open" << fn;
213 return false;
214 }
215 return loadMetadata(&file, meta, mod, info, base);
216 }
217
loadMetadata(MetaMap * meta,MetaMap * mod,MetaMap * info,QString * base) const218 bool ScImageCacheProxy::loadMetadata(MetaMap *meta, MetaMap *mod, MetaMap *info, QString *base) const
219 {
220 return loadMetadata(metaName(), meta, mod, info, base);
221 }
222
saveMetadata(ScLockedFile * file,const MetaMap & meta,const MetaMap & mod,const MetaMap & info,const QString & base)223 void ScImageCacheProxy::saveMetadata(ScLockedFile *file, const MetaMap & meta, const MetaMap & mod, const MetaMap & info, const QString & base)
224 {
225 QXmlStreamWriter xml(file->io());
226
227 xml.setAutoFormatting(true);
228 xml.writeStartDocument();
229 xml.writeStartElement("cache");
230 xml.writeAttribute("base", base);
231 xml.writeStartElement("metadata");
232 for (MetaMap::const_iterator i = meta.constBegin(); i != meta.constEnd(); i++)
233 xml.writeAttribute(i.key(), i.value());
234 xml.writeEndElement();
235 xml.writeStartElement("modifier");
236 for (MetaMap::const_iterator i = mod.constBegin(); i != mod.constEnd(); i++)
237 xml.writeAttribute(i.key(), i.value());
238 xml.writeEndElement();
239 xml.writeStartElement("imginfo");
240 for (MetaMap::const_iterator i = info.constBegin(); i != info.constEnd(); i++)
241 xml.writeAttribute(i.key(), i.value());
242 xml.writeEndElement();
243 xml.writeEndElement();
244 xml.writeEndDocument();
245 }
246
canUseCachedImage() const247 bool ScImageCacheProxy::canUseCachedImage() const
248 {
249 if (!enabled())
250 return false;
251
252 MetaMap cmeta; // cached metadata
253 MetaMap cmod; // cached modifiers
254 QString base;
255
256 if (m_metadata.isEmpty())
257 {
258 scDebug() << "cannot use cached image, no metadata";
259 return false;
260 }
261
262 if (!loadMetadata(&cmeta, &cmod, nullptr, &base))
263 {
264 scDebug() << "cannot use cached image, load metadata failed";
265 return false;
266 }
267
268 QString fn = absolutePath(imageFile(base));
269 QFileInfo info(fn);
270
271 if (!info.exists())
272 return false;
273
274 if (cmeta.size() != m_metadata.size())
275 return false;
276
277 if (cmod.size() != m_modifier.size())
278 return false;
279
280 for (MetaMap::const_iterator i = m_metadata.constBegin(); i != m_metadata.constEnd(); i++)
281 if (cmeta[i.key()] != i.value())
282 return false;
283
284 for (MetaMap::const_iterator i = m_modifier.constBegin(); i != m_modifier.constEnd(); i++)
285 if (cmod[i.key()] != i.value())
286 return false;
287
288 return true;
289 }
290
addDirLevels(QString name)291 QString ScImageCacheProxy::addDirLevels(QString name)
292 {
293 Q_ASSERT(name.size() > CACHEDIR_LEVELS);
294 if (name.size() <= CACHEDIR_LEVELS)
295 {
296 scDebug() << "BUG: invalid name" << name << "passed to addDirLevels";
297 return QString();
298 }
299 for (int i = CACHEDIR_LEVELS; i > 0; i--)
300 name.insert(i, '/');
301 return name;
302 }
303
imageBaseName(const QImage & image) const304 QString ScImageCacheProxy::imageBaseName(const QImage & image) const
305 {
306 if (!m_metadata.contains("size"))
307 {
308 scDebug() << "size not present in metadata";
309 return QString();
310 }
311 QCryptographicHash hash(HASH_ALGORITHM);
312 for (int i = 0; i < image.height(); i++)
313 hash.addData(reinterpret_cast<const char *>(image.scanLine(i)), image.bytesPerLine());
314 return addDirLevels(hash.result().toHex()) + "-" + m_metadata["size"];
315 }
316
metaName() const317 const QString & ScImageCacheProxy::metaName() const
318 {
319 if (m_metanameCache.isEmpty())
320 {
321 QCryptographicHash hash(HASH_ALGORITHM);
322 hash.addData(m_filename.toUtf8());
323 for (MetaMap::const_iterator i = m_modifier.constBegin(); i != m_modifier.constEnd(); i++)
324 {
325 hash.addData(i.key().toUtf8());
326 hash.addData(i.value().toUtf8());
327 }
328 m_metanameCache = addDirLevels(hash.result().toHex()) + "." + metaSuffix;
329 }
330 return m_metanameCache;
331 }
332
createCacheDir()333 bool ScImageCacheProxy::createCacheDir()
334 {
335 QString cachedir = ScPaths::imageCacheDir();
336 QDir cdir(cachedir);
337
338 if (!cdir.exists())
339 {
340 scDebug() << "creating" << cachedir;
341 if (!cdir.mkpath(cachedir))
342 {
343 scDebug() << "could not create" << cachedir;
344 return false;
345 }
346 }
347
348 return true;
349 }
350
load(QImage & image)351 bool ScImageCacheProxy::load(QImage & image)
352 {
353 if (!enabled())
354 return false;
355
356 QString base;
357
358 if (!loadMetadata(&m_metadata, &m_modifier, &m_imginfo, &base))
359 {
360 scDebug() << "could not load metadata for" << m_filename;
361 return false;
362 }
363
364 QString fn = absolutePath(imageFile(base));
365
366 if (!image.load(fn))
367 {
368 scDebug() << "could not load cached image for" << m_filename;
369 return false;
370 }
371
372 scDebug() << "successfully loaded" << m_filename << "from" << fn;
373 return true;
374 }
375
save(const QImage & image)376 bool ScImageCacheProxy::save(const QImage & image)
377 {
378 if (!enabled())
379 return false;
380
381 scDebug() << "saving" << m_filename << "to cache";
382
383 Q_ASSERT(!m_metadata.isEmpty());
384 Q_ASSERT(!m_imginfo.isEmpty());
385
386 if (m_metadata.isEmpty())
387 {
388 scDebug() << "BUG: attempt to save cache without metadata";
389 return false;
390 }
391
392 if (m_imginfo.isEmpty())
393 {
394 scDebug() << "BUG: attempt to save cache without image info";
395 return false;
396 }
397
398 if (!createCacheDir())
399 return false;
400
401 // The cache write lock does not prevent other instances from writing to
402 // the cache. It only prevents other instances from setting a master lock.
403
404 ScImageCacheWriteAction action;
405
406 if (!action.start())
407 return false;
408
409 // Computing the imageBaseName is rather longish, so do it before locking
410 // the files in order to keep the lock time as short as possible.
411
412 QString base = imageBaseName(image);
413
414 Q_ASSERT(!base.isEmpty());
415
416 if (base.isEmpty())
417 {
418 scDebug() << "BUG: could not create image base name";
419 return false;
420 }
421
422 scDebug() << "storing as base" << base;
423
424 QString refName = base + "." + referenceSuffix;
425 QString imgName = base + "." + imageSuffix;
426 QString oldBase;
427 QString oldRefName;
428 QString oldImgName;
429 bool haveOldRef = false;
430
431 ScLockedFileRW meta(absolutePath(metaName()));
432 ScLockedFileRW ref(absolutePath(refName));
433 ScLockedFileRW img(absolutePath(imgName));
434 ScLockedFileRW oldRef;
435
436 if (!meta.createPath())
437 {
438 scDebug() << "could not create path for" << meta.name();
439 return false;
440 }
441
442 if (!ref.createPath())
443 {
444 scDebug() << "could not create path for" << ref.name();
445 return false;
446 }
447
448 // Try to acquire necessary locks. Locks will be automatically cleaned up
449 // upon destruction of the lock object, so we can safely return at any time.
450
451 if (!action.add(metaName()))
452 {
453 scDebug() << "could not add lock for" << metaName();
454 return false;
455 }
456
457 // This is a bit tricky... if the meta file already exists, it will most
458 // probably point to a different reference file. We need to access this
459 // "old" reference file as well in order to decrement its reference count.
460
461 if (meta.exists())
462 {
463 if (!loadMetadata(nullptr, nullptr, nullptr, &oldBase))
464 {
465 scDebug() << "could not read metadata from" << meta.name();
466 return false;
467 }
468
469 oldRefName = oldBase + "." + referenceSuffix;
470 oldImgName = oldBase + "." + imageSuffix;
471
472 if (oldBase != base)
473 {
474 oldRef.setName(absolutePath(oldRefName));
475
476 if (!action.add(oldRefName))
477 {
478 scDebug() << "could not add" << oldRefName << "to action";
479 return false;
480 }
481 if (!action.add(oldImgName))
482 {
483 scDebug() << "could not add" << oldImgName << "to action";
484 return false;
485 }
486
487 haveOldRef = oldRef.exists();
488
489 if (!haveOldRef)
490 oldRef.unlock();
491 }
492 }
493
494 if (!action.add(refName))
495 {
496 scDebug() << "could not add" << refName << "to action";
497 return false;
498 }
499
500 if (!action.add(imgName))
501 {
502 scDebug() << "could not add" << imgName << "to action";
503 return false;
504 }
505
506 // The meta and reference files have both been locked now, so we're safe to
507 // write to the cache. Locking the reference file implicitly also locks the
508 // image file. We can also safely open all files already, as they are only
509 // temporary files and don't conflict with other files in the cache.
510
511 // cases:
512 // * completely new entry, none of the files exist
513 // - create image file
514 // - create reference file with refcount 1
515 // - update meta file
516 // * new meta file, but reference file exists
517 // - increment reference count
518 // - update meta file
519 // - keep image
520 // * old meta file exists and reference files are identical
521 // - keep reference file
522 // - update meta file
523 // - keep image
524 // * old meta file exists and reference files differ
525 // - decrement reference count of old reference file
526 // - continue as above
527
528 // Open the metafile. If this fails, everything else is quite useless.
529
530 if (!meta.open())
531 {
532 scDebug() << "could not open meta file" << meta.name();
533 return false;
534 }
535
536 // Update the reference files if necessary.
537
538 if (haveOldRef)
539 {
540 // we don't care if this fails
541 // if there's any problem, the next cache cleanup will detect it
542 unrefImage(&oldRef, oldImgName);
543 }
544
545 if (oldBase != base)
546 {
547 if (!refImage(&ref))
548 {
549 scDebug() << "could not reference new image" << ref.name();
550 return false;
551 }
552 }
553
554 // Write image file if necessary. Existing image files are *never* re-written
555 // under the assumption that there are no collisions.
556
557 if (img.exists())
558 {
559 scDebug() << "cached image for" << m_filename << "already exists in" << img.name();
560 }
561 else
562 {
563 if (!img.open())
564 {
565 scDebug() << "could not open image file" << img.name();
566 return false;
567 }
568 int level = ScImageCacheManager::instance().compressionLevel();
569 level = level < 0 ? level : 10*(9 - level);
570 scDebug() << "compressing" << imageFormat << "image, quality =" << level;
571 if (!image.save(img.io(), imageFormat, level))
572 {
573 scDebug() << "could not save image" << img.name();
574 return false;
575 }
576
577 img.commit();
578
579 scDebug() << "successfully stored" << m_filename << "in cache as" << img.name();
580 }
581
582 // Save the metadata.
583
584 saveMetadata(&meta, m_metadata, m_modifier, m_imginfo, base);
585 meta.commit();
586
587 // Explicit commit will also trigger access file update
588
589 action.commit();
590
591 return true;
592 }
593
loadRef(ScLockedFile * file,int & refcount)594 bool ScImageCacheProxy::loadRef(ScLockedFile *file, int & refcount)
595 {
596 QXmlStreamReader xml(file->io());
597 bool refcountFound = false;
598
599 while (!xml.atEnd())
600 {
601 if (xml.readNext() == QXmlStreamReader::StartElement)
602 {
603 QXmlStreamAttributes attr = xml.attributes();
604
605 if (xml.name() == "reference")
606 if (attr.hasAttribute("count"))
607 refcount = attr.value("count").toString().toInt(&refcountFound);
608 }
609 }
610
611 if (xml.hasError())
612 {
613 scDebug() << "error parsing" << file->name() << xml.errorString() << "in line" << xml.lineNumber() << "column" << xml.columnNumber();
614 return false;
615 }
616
617 return refcountFound;
618 }
619
saveRef(ScLockedFile * file,int refcount)620 void ScImageCacheProxy::saveRef(ScLockedFile *file, int refcount)
621 {
622 QXmlStreamWriter xml(file->io());
623
624 xml.setAutoFormatting(true);
625 xml.writeStartDocument();
626 xml.writeStartElement("reference");
627 xml.writeAttribute("count", QString::number(refcount));
628 xml.writeEndElement();
629 xml.writeEndDocument();
630 }
631
getRefCount(const QString & reffile,int & refcount)632 bool ScImageCacheProxy::getRefCount(const QString & reffile, int & refcount)
633 {
634 return getRefCountAbs(absolutePath(reffile), refcount);
635 }
636
getRefCountAbs(const QString & reffile,int & refcount)637 bool ScImageCacheProxy::getRefCountAbs(const QString & reffile, int & refcount)
638 {
639 ScLockedFileRO ro(reffile);
640 if (!ro.open())
641 {
642 scDebug() << "could not open reference file" << ro.name();
643 return false;
644 }
645 if (!loadRef(&ro, refcount))
646 {
647 scDebug() << "could not read reference file" << ro.name();
648 return false;
649 }
650 return true;
651 }
652
fixRefCount(const QString & reffile,int refcount)653 bool ScImageCacheProxy::fixRefCount(const QString & reffile, int refcount)
654 {
655 ScLockedFileRW rw(absolutePath(reffile));
656 if (!rw.open())
657 {
658 scDebug() << "could not open reference file" << rw.name();
659 return false;
660 }
661 saveRef(&rw, refcount);
662 return rw.commit();
663 }
664
removeCacheEntry(const QString & metafile,bool haveMasterLock)665 bool ScImageCacheProxy::removeCacheEntry(const QString & metafile, bool haveMasterLock)
666 {
667 ScImageCacheWriteAction action(haveMasterLock);
668
669 if (!action.start())
670 return false;
671
672 ScLockedFileRW meta(absolutePath(metafile));
673
674 if (!action.add(metafile))
675 {
676 scDebug() << "could not add" << metafile;
677 return false;
678 }
679
680 QString base = getBaseName(metafile);
681
682 meta.remove();
683
684 if (base.isEmpty())
685 {
686 scDebug() << "empty basename in" << metafile;
687 }
688 else
689 {
690 QString reffile = referenceFile(base);
691 QString imgfile = imageFile(base);
692
693 if (!action.add(reffile))
694 {
695 scDebug() << "could not add" << reffile;
696 return false;
697 }
698
699 if (!action.add(imgfile))
700 {
701 scDebug() << "could not add" << imgfile;
702 return false;
703 }
704
705 ScLockedFileRW ref(absolutePath(reffile));
706
707 // we don't care if these fail
708 // if there's any problem, the next cache cleanup will detect it
709 unrefImage(&ref, absolutePath(imgfile));
710 }
711
712 action.commit();
713
714 return true;
715 }
716
refImage(ScLockedFile * file)717 bool ScImageCacheProxy::refImage(ScLockedFile *file)
718 {
719 int refcount = 0;
720
721 if (file->exists() && !getRefCountAbs(file->name(), refcount))
722 return false;
723
724 refcount++;
725
726 if (!file->open())
727 {
728 scDebug() << "could not open reference file for writing" << file->name();
729 return false;
730 }
731
732 saveRef(file, refcount);
733
734 return file->commit();
735 }
736
unrefImage(ScLockedFile * file,const QString & imageName)737 bool ScImageCacheProxy::unrefImage(ScLockedFile *file, const QString & imageName)
738 {
739 int refcount = 0;
740
741 if (file->exists())
742 {
743 if (!getRefCountAbs(file->name(), refcount))
744 return false;
745 }
746 else
747 {
748 // could also happen if someone else is messing with the cache
749 scDebug() << "BUG: attempt to unref non-existent reference file" << file->name();
750 return false;
751 }
752
753 refcount--;
754
755 if (refcount == 0)
756 {
757 bool rv = true;
758
759 scDebug() << "refcount dropped to zero for" << file->name();
760
761 if (!file->remove())
762 {
763 scDebug() << "could not remove reference file" << file->name();
764 rv = false;
765 }
766
767 if (QFile::exists(imageName) && !QFile::remove(imageName))
768 {
769 scDebug() << "could not remove image file" << imageName;
770 rv = false;
771 }
772
773 return rv;
774 }
775
776 if (!file->open())
777 {
778 scDebug() << "could not open reference file for writing" << file->name();
779 return false;
780 }
781
782 saveRef(file, refcount);
783
784 return file->commit();
785 }
786
touch() const787 bool ScImageCacheProxy::touch() const
788 {
789 scDebug() << "touching metafile" << metaName();
790 return touchFile(absolutePath(metaName()));
791 }
792