1 /***************************************************************************
2 Copyright (C) 2021 Robby Stephenson <robby@periapsis.org>
3 ***************************************************************************/
4
5 /***************************************************************************
6 * *
7 * This program is free software; you can redistribute it and/or *
8 * modify it under the terms of the GNU General Public License as *
9 * published by the Free Software Foundation; either version 2 of *
10 * the License or (at your option) version 3 or any later version *
11 * accepted by the membership of KDE e.V. (or its successor approved *
12 * by the membership of KDE e.V.), which shall act as a proxy *
13 * defined in Section 14 of version 3 of the license. *
14 * *
15 * This program is distributed in the hope that it will be useful, *
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
18 * GNU General Public License for more details. *
19 * *
20 * You should have received a copy of the GNU General Public License *
21 * along with this program. If not, see <http://www.gnu.org/licenses/>. *
22 * *
23 ***************************************************************************/
24
25 #include "upcitemdbfetcher.h"
26 #include "../collectionfactory.h"
27 #include "../images/imagefactory.h"
28 #include "../gui/combobox.h"
29 #include "../core/filehandler.h"
30 #include "../utils/guiproxy.h"
31 #include "../utils/string_utils.h"
32 #include "../utils/isbnvalidator.h"
33 #include "../tellico_debug.h"
34
35 #include <KLocalizedString>
36 #include <KConfigGroup>
37 #include <KJob>
38 #include <KJobUiDelegate>
39 #include <KJobWidgets/KJobWidgets>
40 #include <KIO/StoredTransferJob>
41
42 #include <QLabel>
43 #include <QFile>
44 #include <QTextStream>
45 #include <QVBoxLayout>
46 #include <QTextCodec>
47 #include <QJsonDocument>
48 #include <QJsonObject>
49 #include <QJsonArray>
50 #include <QUrlQuery>
51
52 namespace {
53 static const int UPCITEMDB_MAX_RETURNS_TOTAL = 20;
54 static const char* UPCITEMDB_API_URL = "https://api.upcitemdb.com/prod/trial";
55 }
56
57 using namespace Tellico;
58 using Tellico::Fetch::UPCItemDbFetcher;
59
UPCItemDbFetcher(QObject * parent_)60 UPCItemDbFetcher::UPCItemDbFetcher(QObject* parent_)
61 : Fetcher(parent_)
62 , m_started(false) {
63 }
64
~UPCItemDbFetcher()65 UPCItemDbFetcher::~UPCItemDbFetcher() {
66 }
67
source() const68 QString UPCItemDbFetcher::source() const {
69 return m_name.isEmpty() ? defaultName() : m_name;
70 }
71
canSearch(Fetch::FetchKey k) const72 bool UPCItemDbFetcher::canSearch(Fetch::FetchKey k) const {
73 return k == UPC || k == ISBN;
74 }
75
canFetch(int type) const76 bool UPCItemDbFetcher::canFetch(int type) const {
77 return type == Data::Collection::Video ||
78 type == Data::Collection::Book ||
79 type == Data::Collection::Album ||
80 type == Data::Collection::Game ||
81 type == Data::Collection::BoardGame;
82 }
83
readConfigHook(const KConfigGroup & config_)84 void UPCItemDbFetcher::readConfigHook(const KConfigGroup& config_) {
85 Q_UNUSED(config_)
86 }
87
saveConfigHook(KConfigGroup & config_)88 void UPCItemDbFetcher::saveConfigHook(KConfigGroup& config_) {
89 Q_UNUSED(config_)
90 }
91
search()92 void UPCItemDbFetcher::search() {
93 continueSearch();
94 }
95
continueSearch()96 void UPCItemDbFetcher::continueSearch() {
97 m_started = true;
98
99
100 QUrl u(QString::fromLatin1(UPCITEMDB_API_URL));
101 u = u.adjusted(QUrl::StripTrailingSlash);
102 u.setPath(u.path() + QLatin1String("/lookup"));
103 QUrlQuery q;
104 switch(request().key()) {
105 case ISBN:
106 // do a upc search by 13-digit isbn
107 {
108 // only grab first value
109 QString isbn = request().value().section(QLatin1Char(';'), 0);
110 isbn = ISBNValidator::isbn13(isbn);
111 isbn.remove(QLatin1Char('-'));
112 q.addQueryItem(QStringLiteral("upc"), isbn);
113 }
114 break;
115
116 case UPC:
117 q.addQueryItem(QStringLiteral("upc"), request().value());
118 break;
119
120 default:
121 myWarning() << "key not recognized:" << request().key();
122 stop();
123 return;
124 }
125 u.setQuery(q);
126
127 // myDebug() << u;
128 m_job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo);
129 KJobWidgets::setWindow(m_job, GUI::Proxy::widget());
130 connect(m_job.data(), &KJob::result, this, &UPCItemDbFetcher::slotComplete);
131 }
132
stop()133 void UPCItemDbFetcher::stop() {
134 if(!m_started) {
135 return;
136 }
137 if(m_job) {
138 m_job->kill();
139 m_job = nullptr;
140 }
141 m_started = false;
142 emit signalDone(this);
143 }
144
updateRequest(Data::EntryPtr entry_)145 Tellico::Fetch::FetchRequest UPCItemDbFetcher::updateRequest(Data::EntryPtr entry_) {
146 const QString isbn = entry_->field(QStringLiteral("isbn"));
147 if(!isbn.isEmpty()) {
148 return FetchRequest(ISBN, isbn);
149 }
150
151 const QString upc = entry_->field(QStringLiteral("upc"));
152 if(!upc.isEmpty()) {
153 return FetchRequest(UPC, upc);
154 }
155
156 const QString barcode = entry_->field(QStringLiteral("barcode"));
157 if(!barcode.isEmpty()) {
158 return FetchRequest(UPC, barcode);
159 }
160
161 return FetchRequest();
162 }
163
slotComplete(KJob * job_)164 void UPCItemDbFetcher::slotComplete(KJob* job_) {
165 KIO::StoredTransferJob* job = static_cast<KIO::StoredTransferJob*>(job_);
166
167 if(job->error()) {
168 job->uiDelegate()->showErrorMessage();
169 stop();
170 return;
171 }
172
173 const QByteArray data = job->data();
174 if(data.isEmpty()) {
175 myDebug() << "UPCItemDb: no data";
176 stop();
177 return;
178 }
179 // see bug 319662. If fetcher is cancelled, job is killed
180 // if the pointer is retained, it gets double-deleted
181 m_job = nullptr;
182
183 #if 0
184 myWarning() << "Remove debug from upcitemdbfetcher.cpp";
185 QFile f(QStringLiteral("/tmp/test-upcitemdb.json"));
186 if(f.open(QIODevice::WriteOnly)) {
187 QTextStream t(&f);
188 t.setCodec("UTF-8");
189 t << data;
190 }
191 f.close();
192 #endif
193
194 QJsonDocument doc = QJsonDocument::fromJson(data);
195 if(doc.isNull()) {
196 myDebug() << "null JSON document";
197 stop();
198 return;
199 }
200
201 Data::CollPtr coll = CollectionFactory::collection(collectionType(), true);
202 if(!coll) {
203 stop();
204 return;
205 }
206
207 if(optionalFields().contains(QStringLiteral("barcode"))) {
208 Data::FieldPtr field(new Data::Field(QStringLiteral("barcode"), i18n("Barcode")));
209 field->setCategory(i18n("General"));
210 coll->addField(field);
211 }
212
213 QJsonArray results = doc.object().value(QLatin1String("items")).toArray();
214 if(results.isEmpty()) {
215 myDebug() << "UPCItemdb: no results";
216 stop();
217 return;
218 }
219
220 int count = 0;
221 foreach(const QJsonValue& result, results) {
222 // myDebug() << "found result:" << result;
223
224 Data::EntryPtr entry(new Data::Entry(coll));
225 populateEntry(entry, result.toObject().toVariantMap());
226
227 FetchResult* r = new FetchResult(this, entry);
228 m_entries.insert(r->uid, entry);
229 emit signalResultFound(r);
230 ++count;
231 if(count >= UPCITEMDB_MAX_RETURNS_TOTAL) {
232 break;
233 }
234 }
235
236 stop();
237 }
238
fetchEntryHook(uint uid_)239 Tellico::Data::EntryPtr UPCItemDbFetcher::fetchEntryHook(uint uid_) {
240 Data::EntryPtr entry = m_entries.value(uid_);
241 if(!entry) {
242 myWarning() << "no entry in dict";
243 return Data::EntryPtr();
244 }
245
246 // image might still be a URL
247 const QString image_id = entry->field(QStringLiteral("cover"));
248 if(image_id.contains(QLatin1Char('/'))) {
249 const QString id = ImageFactory::addImage(QUrl::fromUserInput(image_id), true /* quiet */);
250 if(id.isEmpty()) {
251 message(i18n("The cover image could not be loaded."), MessageHandler::Warning);
252 }
253 // empty image ID is ok
254 entry->setField(QStringLiteral("cover"), id);
255 }
256
257 return entry;
258 }
259
populateEntry(Data::EntryPtr entry_,const QVariantMap & resultMap_)260 void UPCItemDbFetcher::populateEntry(Data::EntryPtr entry_, const QVariantMap& resultMap_) {
261 entry_->setField(QStringLiteral("title"), mapValue(resultMap_, "title"));
262 parseTitle(entry_);
263 // entry_->setField(QStringLiteral("year"), mapValue(resultMap_, "premiered").left(4));
264 const QString barcode = QStringLiteral("barcode");
265 if(optionalFields().contains(barcode)) {
266 entry_->setField(barcode, mapValue(resultMap_, "upc"));
267 }
268
269 // take the first cover
270 const auto imageList = resultMap_.value(QLatin1String("images")).toList();
271 if(!imageList.isEmpty()) {
272 entry_->setField(QStringLiteral("cover"), imageList.first().toString());
273 }
274
275 switch(collectionType()) {
276 case Data::Collection::Video:
277 entry_->setField(QStringLiteral("studio"), mapValue(resultMap_, "brand"));
278 entry_->setField(QStringLiteral("plot"), mapValue(resultMap_, "description"));
279 break;
280
281 case Data::Collection::Book:
282 entry_->setField(QStringLiteral("publisher"), mapValue(resultMap_, "publisher"));
283 entry_->setField(QStringLiteral("isbn"), mapValue(resultMap_, "isbn"));
284 break;
285
286 case Data::Collection::Album:
287 entry_->setField(QStringLiteral("label"), mapValue(resultMap_, "brand"));
288 {
289 const QString cat = mapValue(resultMap_, "category");
290 if(cat.contains(QStringLiteral("Music CDs"))) {
291 entry_->setField(QStringLiteral("medium"), i18n("Compact Disc"));
292 }
293 }
294 break;
295
296 case Data::Collection::Game:
297 case Data::Collection::BoardGame:
298 entry_->setField(QStringLiteral("publisher"), mapValue(resultMap_, "brand"));
299 entry_->setField(QStringLiteral("description"), mapValue(resultMap_, "description"));
300 break;
301
302 default:
303 break;
304 }
305
306 // do this after all other parsing
307 parseTitle(entry_);
308 }
309
parseTitle(Tellico::Data::EntryPtr entry_)310 void UPCItemDbFetcher::parseTitle(Tellico::Data::EntryPtr entry_) {
311 // assume that everything in brackets or parentheses is extra
312 static const QRegularExpression rx(QLatin1String("[\\(\\[](.*?)[\\)\\]]"));
313 QString title = entry_->field(QStringLiteral("title"));
314 int pos = 0;
315 QRegularExpressionMatch match = rx.match(title, pos);
316 while(match.hasMatch()) {
317 pos = match.capturedStart();
318 if(parseTitleToken(entry_, match.captured(1))) {
319 title.remove(match.capturedStart(), match.capturedLength());
320 --pos; // search again there
321 }
322 match = rx.match(title, pos+1);
323 }
324 // look for "word1 - word2"
325 static const QRegularExpression dashWords(QLatin1String("(.+) - (.+)"));
326 QRegularExpressionMatch dashMatch = dashWords.match(title);
327 if(dashMatch.hasMatch()) {
328 switch(collectionType()) {
329 case Data::Collection::Book:
330 title = dashMatch.captured(1);
331 {
332 QRegularExpression byAuthor(QLatin1String("by (.+)"));
333 QRegularExpressionMatch authorMatch = byAuthor.match(dashMatch.captured(2));
334 if(authorMatch.hasMatch()) {
335 entry_->setField(QStringLiteral("author"), authorMatch.captured(1).simplified());
336 }
337 }
338 break;
339
340 case Data::Collection::Album:
341 entry_->setField(QStringLiteral("artist"), dashMatch.captured(1).simplified());
342 title = dashMatch.captured(2);
343 break;
344
345 case Data::Collection::Game:
346 title = dashMatch.captured(1);
347 {
348 const QString platform = QStringLiteral("platform");
349 const QString maybe = i18n(dashMatch.captured(2).simplified().toUtf8().constData());
350 Data::FieldPtr f = entry_->collection()->fieldByName(platform);
351 if(f && f->allowed().contains(maybe)) {
352 entry_->setField(platform, maybe);
353 }
354 }
355 break;
356 }
357 }
358 entry_->setField(QStringLiteral("title"), title.simplified());
359 }
360
361 // mostly taken from amazonfetcher
parseTitleToken(Tellico::Data::EntryPtr entry_,const QString & token_)362 bool UPCItemDbFetcher::parseTitleToken(Tellico::Data::EntryPtr entry_, const QString& token_) {
363 // myDebug() << "title token:" << token_;
364 // if res = true, then the token gets removed from the title
365 bool res = false;
366 static const QRegularExpression yearRx(QLatin1String("\\d{4}"));
367 QRegularExpressionMatch yearMatch = yearRx.match(token_);
368 if(yearMatch.hasMatch()) {
369 entry_->setField(QStringLiteral("year"), yearMatch.captured());
370 res = true;
371 }
372 if(token_.indexOf(QLatin1String("widescreen"), 0, Qt::CaseInsensitive) > -1 ||
373 token_.indexOf(i18n("Widescreen"), 0, Qt::CaseInsensitive) > -1) {
374 entry_->setField(QStringLiteral("widescreen"), QStringLiteral("true"));
375 // res = true; leave it in the title
376 } else if(token_.indexOf(QLatin1String("full screen"), 0, Qt::CaseInsensitive) > -1) {
377 // skip, but go ahead and remove from title
378 res = true;
379 } else if(token_.indexOf(QLatin1String("standard edition"), 0, Qt::CaseInsensitive) > -1) {
380 // skip, but go ahead and remove from title
381 res = true;
382 } else if(token_.indexOf(QLatin1String("import"), 0, Qt::CaseInsensitive) > -1) {
383 // skip, but go ahead and remove from title
384 res = true;
385 }
386 if(token_.indexOf(QLatin1String("blu-ray"), 0, Qt::CaseInsensitive) > -1) {
387 entry_->setField(QStringLiteral("medium"), i18n("Blu-ray"));
388 res = true;
389 } else if(token_.indexOf(QLatin1String("hd dvd"), 0, Qt::CaseInsensitive) > -1) {
390 entry_->setField(QStringLiteral("medium"), i18n("HD DVD"));
391 res = true;
392 } else if(token_.indexOf(QLatin1String("vhs"), 0, Qt::CaseInsensitive) > -1) {
393 entry_->setField(QStringLiteral("medium"), i18n("VHS"));
394 res = true;
395 }
396 if(token_.indexOf(QLatin1String("director's cut"), 0, Qt::CaseInsensitive) > -1 ||
397 token_.indexOf(i18n("Director's Cut"), 0, Qt::CaseInsensitive) > -1) {
398 entry_->setField(QStringLiteral("directors-cut"), QStringLiteral("true"));
399 // res = true; leave it in the title
400 }
401 const QString tokenLower = token_.toLower();
402 if(tokenLower == QLatin1String("ntsc")) {
403 entry_->setField(QStringLiteral("format"), i18n("NTSC"));
404 res = true;
405 }
406 if(tokenLower == QLatin1String("dvd")) {
407 entry_->setField(QStringLiteral("medium"), i18n("DVD"));
408 res = true;
409 }
410 if(tokenLower == QLatin1String("cd") && collectionType() == Data::Collection::Album) {
411 entry_->setField(QStringLiteral("medium"), i18n("Compact Disc"));
412 res = true;
413 }
414 if(tokenLower == QLatin1String("dvd")) {
415 entry_->setField(QStringLiteral("medium"), i18n("DVD"));
416 res = true;
417 }
418 if(token_.indexOf(QLatin1String("series"), 0, Qt::CaseInsensitive) > -1) {
419 entry_->setField(QStringLiteral("series"), token_);
420 res = true;
421 }
422 static const QRegularExpression regionRx(QLatin1String("Region [1-9]"));
423 QRegularExpressionMatch match = regionRx.match(token_);
424 if(match.hasMatch()) {
425 entry_->setField(QStringLiteral("region"), i18n(match.captured().toUtf8().constData()));
426 res = true;
427 }
428 if(collectionType() == Data::Collection::Game) {
429 Data::FieldPtr f = entry_->collection()->fieldByName(QStringLiteral("platform"));
430 if(f && f->allowed().contains(token_)) {
431 res = true;
432 }
433 } else if(collectionType() == Data::Collection::Book) {
434 const QString binding = QStringLiteral("binding");
435 Data::FieldPtr f = entry_->collection()->fieldByName(binding);
436 const QString maybe = i18n(token_.toUtf8().constData());
437 if(f && f->allowed().contains(maybe)) {
438 entry_->setField(binding, maybe);
439 res = true;
440 }
441 }
442 return res;
443 }
444
configWidget(QWidget * parent_) const445 Tellico::Fetch::ConfigWidget* UPCItemDbFetcher::configWidget(QWidget* parent_) const {
446 return new UPCItemDbFetcher::ConfigWidget(parent_, this);
447 }
448
defaultName()449 QString UPCItemDbFetcher::defaultName() {
450 return QStringLiteral("UPCitemdb"); // this is the capitalization they use on their site
451 }
452
defaultIcon()453 QString UPCItemDbFetcher::defaultIcon() {
454 return favIcon("https://www.upcitemdb.com");
455 }
456
allOptionalFields()457 Tellico::StringHash UPCItemDbFetcher::allOptionalFields() {
458 StringHash hash;
459 hash[QStringLiteral("barcode")] = i18n("Barcode");
460 return hash;
461 }
462
ConfigWidget(QWidget * parent_,const UPCItemDbFetcher * fetcher_)463 UPCItemDbFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const UPCItemDbFetcher* fetcher_)
464 : Fetch::ConfigWidget(parent_) {
465 QVBoxLayout* l = new QVBoxLayout(optionsWidget());
466 l->addWidget(new QLabel(i18n("This source has no options."), optionsWidget()));
467 l->addStretch();
468
469 // now add additional fields widget
470 addFieldsWidget(UPCItemDbFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList());
471 }
472
preferredName() const473 QString UPCItemDbFetcher::ConfigWidget::preferredName() const {
474 return UPCItemDbFetcher::defaultName();
475 }
476