1 #include "pictureloader.h"
2
3 #include "carddatabase.h"
4 #include "main.h"
5 #include "settingscache.h"
6 #include "thememanager.h"
7
8 #include <QApplication>
9 #include <QCryptographicHash>
10 #include <QDebug>
11 #include <QDir>
12 #include <QDirIterator>
13 #include <QFile>
14 #include <QImageReader>
15 #include <QNetworkAccessManager>
16 #include <QNetworkReply>
17 #include <QNetworkRequest>
18 #include <QPainter>
19 #include <QPixmapCache>
20 #include <QSet>
21 #include <QSvgRenderer>
22 #include <QThread>
23 #include <QUrl>
24 #include <algorithm>
25 #include <utility>
26
27 // never cache more than 300 cards at once for a single deck
28 #define CACHED_CARD_PER_DECK_MAX 300
29
PictureToLoad(CardInfoPtr _card)30 PictureToLoad::PictureToLoad(CardInfoPtr _card)
31 : card(std::move(_card)), urlTemplates(SettingsCache::instance().downloads().getAllURLs())
32 {
33 if (card) {
34 for (const auto &set : card->getSets()) {
35 sortedSets << set.getPtr();
36 }
37 if (sortedSets.empty()) {
38 sortedSets << CardSet::newInstance("", "", "", QDate());
39 }
40 std::sort(sortedSets.begin(), sortedSets.end(), SetDownloadPriorityComparator());
41 // The first time called, nextSet will also populate the Urls for the first set.
42 nextSet();
43 }
44 }
45
populateSetUrls()46 void PictureToLoad::populateSetUrls()
47 {
48 /* currentSetUrls is a list, populated each time a new set is requested for a particular card
49 and Urls are removed from it as a download is attempted from each one. Custom Urls for
50 a set are given higher priority, so should be placed first in the list. */
51 currentSetUrls.clear();
52
53 if (card && currentSet) {
54 QString setCustomURL = card->getCustomPicURL(currentSet->getShortName());
55
56 if (!setCustomURL.isEmpty()) {
57 currentSetUrls.append(setCustomURL);
58 }
59 }
60
61 for (const QString &urlTemplate : urlTemplates) {
62 QString transformedUrl = transformUrl(urlTemplate);
63
64 if (!transformedUrl.isEmpty()) {
65 currentSetUrls.append(transformedUrl);
66 }
67 }
68
69 /* Call nextUrl to make sure currentUrl is up-to-date
70 but we don't need the result here. */
71 (void)nextUrl();
72 }
73
nextSet()74 bool PictureToLoad::nextSet()
75 {
76 if (!sortedSets.isEmpty()) {
77 currentSet = sortedSets.takeFirst();
78 populateSetUrls();
79 return true;
80 }
81 currentSet = {};
82 return false;
83 }
84
nextUrl()85 bool PictureToLoad::nextUrl()
86 {
87 if (!currentSetUrls.isEmpty()) {
88 currentUrl = currentSetUrls.takeFirst();
89 return true;
90 }
91 currentUrl = QString();
92 return false;
93 }
94
getSetName() const95 QString PictureToLoad::getSetName() const
96 {
97 if (currentSet) {
98 return currentSet->getCorrectedShortName();
99 } else {
100 return QString();
101 }
102 }
103
104 // Card back returned by gatherer when card is not found
105 QStringList PictureLoaderWorker::md5Blacklist = QStringList() << "db0c48db407a907c16ade38de048a441";
106
PictureLoaderWorker()107 PictureLoaderWorker::PictureLoaderWorker()
108 : QObject(nullptr), picsPath(SettingsCache::instance().getPicsPath()),
109 customPicsPath(SettingsCache::instance().getCustomPicsPath()),
110 picDownload(SettingsCache::instance().getPicDownload()), downloadRunning(false), loadQueueRunning(false)
111 {
112 connect(this, SIGNAL(startLoadQueue()), this, SLOT(processLoadQueue()), Qt::QueuedConnection);
113 connect(&SettingsCache::instance(), SIGNAL(picsPathChanged()), this, SLOT(picsPathChanged()));
114 connect(&SettingsCache::instance(), SIGNAL(picDownloadChanged()), this, SLOT(picDownloadChanged()));
115
116 networkManager = new QNetworkAccessManager(this);
117 connect(networkManager, SIGNAL(finished(QNetworkReply *)), this, SLOT(picDownloadFinished(QNetworkReply *)));
118
119 pictureLoaderThread = new QThread;
120 pictureLoaderThread->start(QThread::LowPriority);
121 moveToThread(pictureLoaderThread);
122 }
123
~PictureLoaderWorker()124 PictureLoaderWorker::~PictureLoaderWorker()
125 {
126 pictureLoaderThread->deleteLater();
127 }
128
processLoadQueue()129 void PictureLoaderWorker::processLoadQueue()
130 {
131 if (loadQueueRunning) {
132 return;
133 }
134
135 loadQueueRunning = true;
136 while (true) {
137 mutex.lock();
138 if (loadQueue.isEmpty()) {
139 mutex.unlock();
140 loadQueueRunning = false;
141 return;
142 }
143 cardBeingLoaded = loadQueue.takeFirst();
144 mutex.unlock();
145
146 QString setName = cardBeingLoaded.getSetName();
147 QString cardName = cardBeingLoaded.getCard()->getName();
148 QString correctedCardName = cardBeingLoaded.getCard()->getCorrectedName();
149
150 qDebug() << "PictureLoader: [card: " << cardName << " set: " << setName << "]: Trying to load picture";
151
152 if (cardImageExistsOnDisk(setName, correctedCardName)) {
153 continue;
154 }
155
156 if (picDownload) {
157 qDebug() << "PictureLoader: [card: " << cardName << " set: " << setName
158 << "]: Picture not found on disk, trying to download";
159 cardsToDownload.append(cardBeingLoaded);
160 cardBeingLoaded.clear();
161 if (!downloadRunning) {
162 startNextPicDownload();
163 }
164 } else {
165 if (cardBeingLoaded.nextSet()) {
166 qDebug() << "PictureLoader: [card: " << cardName << " set: " << setName
167 << "]: Picture NOT found and download disabled, moving to next "
168 "set (new set: "
169 << setName << " card: " << cardName << ")";
170 mutex.lock();
171 loadQueue.prepend(cardBeingLoaded);
172 cardBeingLoaded.clear();
173 mutex.unlock();
174 } else {
175 qDebug() << "PictureLoader: [card: " << cardName << " set: " << setName
176 << "]: Picture NOT found, download disabled, no more sets to "
177 "try: BAILING OUT (old set: "
178 << setName << " card: " << cardName << ")";
179 imageLoaded(cardBeingLoaded.getCard(), QImage());
180 }
181 }
182 }
183 }
184
cardImageExistsOnDisk(QString & setName,QString & correctedCardname)185 bool PictureLoaderWorker::cardImageExistsOnDisk(QString &setName, QString &correctedCardname)
186 {
187 QImage image;
188 QImageReader imgReader;
189 imgReader.setDecideFormatFromContent(true);
190 QList<QString> picsPaths = QList<QString>();
191 QDirIterator it(customPicsPath, QDirIterator::Subdirectories);
192
193 // Recursively check all subdirectories of the CUSTOM folder
194 while (it.hasNext()) {
195 QString thisPath(it.next());
196 QFileInfo thisFileInfo(thisPath);
197
198 if (thisFileInfo.isFile() && thisFileInfo.baseName() == correctedCardname)
199 picsPaths << thisPath; // Card found in the CUSTOM directory, somewhere
200 }
201
202 if (!setName.isEmpty()) {
203 picsPaths << picsPath + "/" + setName + "/" + correctedCardname
204 << picsPath + "/downloadedPics/" + setName + "/" + correctedCardname;
205 }
206
207 // Iterates through the list of paths, searching for images with the desired
208 // name with any QImageReader-supported
209 // extension
210 for (const auto &picsPath : picsPaths) {
211 imgReader.setFileName(picsPath);
212 if (imgReader.read(&image)) {
213 qDebug() << "PictureLoader: [card: " << correctedCardname << " set: " << setName
214 << "]: Picture found on disk.";
215 imageLoaded(cardBeingLoaded.getCard(), image);
216 return true;
217 }
218 imgReader.setFileName(picsPath + ".full");
219 if (imgReader.read(&image)) {
220 qDebug() << "PictureLoader: [card: " << correctedCardname << " set: " << setName
221 << "]: Picture.full found on disk.";
222 imageLoaded(cardBeingLoaded.getCard(), image);
223 return true;
224 }
225 imgReader.setFileName(picsPath + ".xlhq");
226 if (imgReader.read(&image)) {
227 qDebug() << "PictureLoader: [card: " << correctedCardname << " set: " << setName
228 << "]: Picture.xlhq found on disk.";
229 imageLoaded(cardBeingLoaded.getCard(), image);
230 return true;
231 }
232 }
233
234 return false;
235 }
236
transformUrl(const QString & urlTemplate) const237 QString PictureToLoad::transformUrl(const QString &urlTemplate) const
238 {
239 /* This function takes Url templates and substitutes actual card details
240 into the url. This is used for making Urls with follow a predictable format
241 for downloading images. If information is requested by the template that is
242 not populated for this specific card/set combination, an empty string is returned.*/
243
244 QString transformedUrl = urlTemplate;
245 CardSetPtr set = getCurrentSet();
246
247 QMap<QString, QString> transformMap = QMap<QString, QString>();
248 // name
249 transformMap["!name!"] = card->getName();
250 transformMap["!name_lower!"] = card->getName().toLower();
251 transformMap["!corrected_name!"] = card->getCorrectedName();
252 transformMap["!corrected_name_lower!"] = card->getCorrectedName().toLower();
253
254 // card properties
255 QRegExp rxCardProp("!prop:([^!]+)!");
256 int pos = 0;
257 while ((pos = rxCardProp.indexIn(transformedUrl, pos)) != -1) {
258 QString propertyName = rxCardProp.cap(1);
259 pos += rxCardProp.matchedLength();
260 QString propertyValue = card->getProperty(propertyName);
261 if (propertyValue.isEmpty()) {
262 qDebug() << "PictureLoader: [card: " << card->getName() << " set: " << getSetName()
263 << "]: Requested property (" << propertyName << ") for Url template (" << urlTemplate
264 << ") is not available";
265 return QString();
266 } else {
267 transformMap["!prop:" + propertyName + "!"] = propertyValue;
268 }
269 }
270
271 if (set) {
272 transformMap["!setcode!"] = set->getShortName();
273 transformMap["!setcode_lower!"] = set->getShortName().toLower();
274 transformMap["!setname!"] = set->getLongName();
275 transformMap["!setname_lower!"] = set->getLongName().toLower();
276
277 QRegExp rxSetProp("!set:([^!]+)!");
278 pos = 0; // Defined above
279 while ((pos = rxSetProp.indexIn(transformedUrl, pos)) != -1) {
280 QString propertyName = rxSetProp.cap(1);
281 pos += rxSetProp.matchedLength();
282 QString propertyValue = card->getSetProperty(set->getShortName(), propertyName);
283 if (propertyValue.isEmpty()) {
284 qDebug() << "PictureLoader: [card: " << card->getName() << " set: " << getSetName()
285 << "]: Requested set property (" << propertyName << ") for Url template (" << urlTemplate
286 << ") is not available";
287 return QString();
288 } else {
289 transformMap["!set:" + propertyName + "!"] = propertyValue;
290 }
291 }
292 }
293
294 // language setting
295 transformMap["!sflang!"] = QString(QCoreApplication::translate(
296 "PictureLoader", "en", "code for scryfall's language property, not available for all languages"));
297
298 for (const QString &prop : transformMap.keys()) {
299 if (transformedUrl.contains(prop)) {
300 if (!transformMap[prop].isEmpty()) {
301 transformedUrl.replace(prop, QUrl::toPercentEncoding(transformMap[prop]));
302 } else {
303 /* This means the template is requesting information that is not
304 * populated in this card, so it should return an empty string,
305 * indicating an invalid Url.
306 */
307 qDebug() << "PictureLoader: [card: " << card->getName() << " set: " << getSetName()
308 << "]: Requested information (" << prop << ") for Url template (" << urlTemplate
309 << ") is not available";
310 return QString();
311 }
312 }
313 }
314
315 return transformedUrl;
316 }
317
startNextPicDownload()318 void PictureLoaderWorker::startNextPicDownload()
319 {
320 if (cardsToDownload.isEmpty()) {
321 cardBeingDownloaded.clear();
322 downloadRunning = false;
323 return;
324 }
325
326 downloadRunning = true;
327
328 cardBeingDownloaded = cardsToDownload.takeFirst();
329
330 QString picUrl = cardBeingDownloaded.getCurrentUrl();
331
332 if (picUrl.isEmpty()) {
333 downloadRunning = false;
334 picDownloadFailed();
335 } else {
336 QUrl url(picUrl);
337 QNetworkRequest req(url);
338 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getCorrectedName()
339 << " set: " << cardBeingDownloaded.getSetName()
340 << "]: Trying to download picture from url:" << url.toDisplayString();
341 networkManager->get(req);
342 }
343 }
344
picDownloadFailed()345 void PictureLoaderWorker::picDownloadFailed()
346 {
347 /* Take advantage of short circuiting here to call the nextUrl until one
348 is not available. Only once nextUrl evaluates to false will this move
349 on to nextSet. If the Urls for a particular card are empty, this will
350 effectively go through the sets for that card. */
351 if (cardBeingDownloaded.nextUrl() || cardBeingDownloaded.nextSet()) {
352 mutex.lock();
353 loadQueue.prepend(cardBeingDownloaded);
354 mutex.unlock();
355 } else {
356 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getCorrectedName()
357 << " set: " << cardBeingDownloaded.getSetName()
358 << "]: Picture NOT found, download failed, no more url combinations "
359 "to try: BAILING OUT";
360 imageLoaded(cardBeingDownloaded.getCard(), QImage());
361 cardBeingDownloaded.clear();
362 }
363 emit startLoadQueue();
364 }
365
imageIsBlackListed(const QByteArray & picData)366 bool PictureLoaderWorker::imageIsBlackListed(const QByteArray &picData)
367 {
368 QString md5sum = QCryptographicHash::hash(picData, QCryptographicHash::Md5).toHex();
369 return md5Blacklist.contains(md5sum);
370 }
371
picDownloadFinished(QNetworkReply * reply)372 void PictureLoaderWorker::picDownloadFinished(QNetworkReply *reply)
373 {
374 if (reply->error()) {
375 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getName()
376 << " set: " << cardBeingDownloaded.getSetName() << "]: Download failed:" << reply->errorString();
377 }
378
379 int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
380 if (statusCode == 301 || statusCode == 302) {
381 QUrl redirectUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
382 QNetworkRequest req(redirectUrl);
383 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getName()
384 << " set: " << cardBeingDownloaded.getSetName() << "]: following redirect:" << req.url().toString();
385 networkManager->get(req);
386 return;
387 }
388
389 // peek is used to keep the data in the buffer for use by QImageReader
390 const QByteArray &picData = reply->peek(reply->size());
391
392 if (imageIsBlackListed(picData)) {
393 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getName()
394 << " set: " << cardBeingDownloaded.getSetName()
395 << "]:Picture downloaded, but blacklisted, will consider it as "
396 "not found";
397 picDownloadFailed();
398 reply->deleteLater();
399 startNextPicDownload();
400 return;
401 }
402
403 QImage testImage;
404
405 QImageReader imgReader;
406 imgReader.setDecideFormatFromContent(true);
407 imgReader.setDevice(reply);
408 QString extension = "." + imgReader.format(); // the format is determined
409 // prior to reading the
410 // QImageReader data
411 // into a QImage object, as that wipes the QImageReader buffer
412 if (extension == ".jpeg") {
413 extension = ".jpg";
414 }
415
416 if (imgReader.read(&testImage)) {
417 QString setName = cardBeingDownloaded.getSetName();
418 if (!setName.isEmpty()) {
419 if (!QDir().mkpath(picsPath + "/downloadedPics/" + setName)) {
420 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getName()
421 << " set: " << cardBeingDownloaded.getSetName()
422 << "]: " << picsPath + "/downloadedPics/" + setName + " could not be created.";
423 return;
424 }
425
426 QFile newPic(picsPath + "/downloadedPics/" + setName + "/" +
427 cardBeingDownloaded.getCard()->getCorrectedName() + extension);
428 if (!newPic.open(QIODevice::WriteOnly)) {
429 return;
430 }
431 newPic.write(picData);
432 newPic.close();
433 }
434
435 imageLoaded(cardBeingDownloaded.getCard(), testImage);
436 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getName()
437 << " set: " << cardBeingDownloaded.getSetName() << "]: Image successfully downloaded from "
438 << reply->request().url().toDisplayString();
439 } else {
440 qDebug() << "PictureLoader: [card: " << cardBeingDownloaded.getCard()->getName()
441 << " set: " << cardBeingDownloaded.getSetName() << "]: Possible picture at "
442 << reply->request().url().toDisplayString() << " could not be loaded";
443 picDownloadFailed();
444 }
445
446 reply->deleteLater();
447 startNextPicDownload();
448 }
449
enqueueImageLoad(CardInfoPtr card)450 void PictureLoaderWorker::enqueueImageLoad(CardInfoPtr card)
451 {
452 QMutexLocker locker(&mutex);
453
454 // avoid queueing the same card more than once
455 if (!card || card == cardBeingLoaded.getCard() || card == cardBeingDownloaded.getCard()) {
456 return;
457 }
458
459 for (const PictureToLoad &pic : loadQueue) {
460 if (pic.getCard() == card)
461 return;
462 }
463
464 for (const PictureToLoad &pic : cardsToDownload) {
465 if (pic.getCard() == card)
466 return;
467 }
468
469 loadQueue.append(PictureToLoad(card));
470 emit startLoadQueue();
471 }
472
picDownloadChanged()473 void PictureLoaderWorker::picDownloadChanged()
474 {
475 QMutexLocker locker(&mutex);
476 picDownload = SettingsCache::instance().getPicDownload();
477 }
478
picsPathChanged()479 void PictureLoaderWorker::picsPathChanged()
480 {
481 QMutexLocker locker(&mutex);
482 picsPath = SettingsCache::instance().getPicsPath();
483 customPicsPath = SettingsCache::instance().getCustomPicsPath();
484 }
485
PictureLoader()486 PictureLoader::PictureLoader() : QObject(nullptr)
487 {
488 worker = new PictureLoaderWorker;
489 connect(&SettingsCache::instance(), SIGNAL(picsPathChanged()), this, SLOT(picsPathChanged()));
490 connect(&SettingsCache::instance(), SIGNAL(picDownloadChanged()), this, SLOT(picDownloadChanged()));
491
492 connect(worker, SIGNAL(imageLoaded(CardInfoPtr, const QImage &)), this,
493 SLOT(imageLoaded(CardInfoPtr, const QImage &)));
494 }
495
~PictureLoader()496 PictureLoader::~PictureLoader()
497 {
498 worker->deleteLater();
499 }
500
getCardBackPixmap(QPixmap & pixmap,QSize size)501 void PictureLoader::getCardBackPixmap(QPixmap &pixmap, QSize size)
502 {
503 QString backCacheKey = "_trice_card_back_" + QString::number(size.width()) + QString::number(size.height());
504 if (!QPixmapCache::find(backCacheKey, &pixmap)) {
505 qDebug() << "PictureLoader: cache fail for" << backCacheKey;
506 pixmap = QPixmap("theme:cardback").scaled(size, Qt::KeepAspectRatio, Qt::SmoothTransformation);
507 QPixmapCache::insert(backCacheKey, pixmap);
508 }
509 }
510
getPixmap(QPixmap & pixmap,CardInfoPtr card,QSize size)511 void PictureLoader::getPixmap(QPixmap &pixmap, CardInfoPtr card, QSize size)
512 {
513 if (card == nullptr) {
514 return;
515 }
516
517 // search for an exact size copy of the picture in cache
518 QString key = card->getPixmapCacheKey();
519 QString sizeKey = key + QLatin1Char('_') + QString::number(size.width()) + QString::number(size.height());
520 if (QPixmapCache::find(sizeKey, &pixmap))
521 return;
522
523 // load the image and create a copy of the correct size
524 QPixmap bigPixmap;
525 if (QPixmapCache::find(key, &bigPixmap)) {
526 pixmap = bigPixmap.scaled(size, Qt::KeepAspectRatio, Qt::SmoothTransformation);
527 QPixmapCache::insert(sizeKey, pixmap);
528 return;
529 }
530
531 // add the card to the load queue
532 getInstance().worker->enqueueImageLoad(card);
533 }
534
imageLoaded(CardInfoPtr card,const QImage & image)535 void PictureLoader::imageLoaded(CardInfoPtr card, const QImage &image)
536 {
537 if (image.isNull()) {
538 QPixmapCache::insert(card->getPixmapCacheKey(), QPixmap());
539 } else {
540 if (card->getUpsideDownArt()) {
541 QImage mirrorImage = image.mirrored(true, true);
542 QPixmapCache::insert(card->getPixmapCacheKey(), QPixmap::fromImage(mirrorImage));
543 } else {
544 QPixmapCache::insert(card->getPixmapCacheKey(), QPixmap::fromImage(image));
545 }
546 }
547
548 card->emitPixmapUpdated();
549 }
550
clearPixmapCache(CardInfoPtr card)551 void PictureLoader::clearPixmapCache(CardInfoPtr card)
552 {
553 if (card) {
554 QPixmapCache::remove(card->getPixmapCacheKey());
555 }
556 }
557
clearPixmapCache()558 void PictureLoader::clearPixmapCache()
559 {
560 QPixmapCache::clear();
561 }
562
cacheCardPixmaps(QList<CardInfoPtr> cards)563 void PictureLoader::cacheCardPixmaps(QList<CardInfoPtr> cards)
564 {
565 QPixmap tmp;
566 int max = qMin(cards.size(), CACHED_CARD_PER_DECK_MAX);
567 for (int i = 0; i < max; ++i) {
568 const CardInfoPtr &card = cards.at(i);
569 if (!card) {
570 continue;
571 }
572
573 QString key = card->getPixmapCacheKey();
574 if (QPixmapCache::find(key, &tmp)) {
575 continue;
576 }
577
578 getInstance().worker->enqueueImageLoad(card);
579 }
580 }
581
picDownloadChanged()582 void PictureLoader::picDownloadChanged()
583 {
584 QPixmapCache::clear();
585 }
586
picsPathChanged()587 void PictureLoader::picsPathChanged()
588 {
589 QPixmapCache::clear();
590 }
591