src/fetch/upcitemdbfetcher.cpp

0001 /***************************************************************************
0002     Copyright (C) 2021 Robby Stephenson <robby@periapsis.org>
0003  ***************************************************************************/
0004
0005 /***************************************************************************
0006  *                                                                         *
0007  *   This program is free software; you can redistribute it and/or         *
0008  *   modify it under the terms of the GNU General Public License as        *
0009  *   published by the Free Software Foundation; either version 2 of        *
0010  *   the License or (at your option) version 3 or any later version        *
0011  *   accepted by the membership of KDE e.V. (or its successor approved     *
0012  *   by the membership of KDE e.V.), which shall act as a proxy            *
0013  *   defined in Section 14 of version 3 of the license.                    *
0014  *                                                                         *
0015  *   This program is distributed in the hope that it will be useful,       *
0016  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
0017  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
0018  *   GNU General Public License for more details.                          *
0019  *                                                                         *
0020  *   You should have received a copy of the GNU General Public License     *
0021  *   along with this program.  If not, see <http://www.gnu.org/licenses/>. *
0022  *                                                                         *
0023  ***************************************************************************/
0024
0025 #include "upcitemdbfetcher.h"
0026 #include "../collectionfactory.h"
0027 #include "../images/imagefactory.h"
0028 #include "../utils/guiproxy.h"
0029 #include "../utils/mapvalue.h"
0030 #include "../utils/isbnvalidator.h"
0031 #include "../tellico_debug.h"
0032
0033 #include <KLocalizedString>
0034 #include <KConfigGroup>
0035 #include <KJob>
0036 #include <KJobUiDelegate>
0037 #include <KJobWidgets/KJobWidgets>
0038 #include <KIO/StoredTransferJob>
0039
0040 #include <QLabel>
0041 #include <QFile>
0042 #include <QTextStream>
0043 #include <QVBoxLayout>
0044 #include <QTextCodec>
0045 #include <QJsonDocument>
0046 #include <QJsonObject>
0047 #include <QJsonArray>
0048 #include <QUrlQuery>
0049
0050 namespace {
0051   static const int UPCITEMDB_MAX_RETURNS_TOTAL = 20;
0052   static const char* UPCITEMDB_API_URL = "https://api.upcitemdb.com/prod/trial";
0053 }
0054
0055 using namespace Tellico;
0056 using Tellico::Fetch::UPCItemDbFetcher;
0057
0058 UPCItemDbFetcher::UPCItemDbFetcher(QObject* parent_)
0059     : Fetcher(parent_)
0060     , m_started(false) {
0061 }
0062
0063 UPCItemDbFetcher::~UPCItemDbFetcher() {
0064 }
0065
0066 QString UPCItemDbFetcher::source() const {
0067   return m_name.isEmpty() ? defaultName() : m_name;
0068 }
0069
0070 bool UPCItemDbFetcher::canSearch(Fetch::FetchKey k) const {
0071   return k == UPC || k == ISBN;
0072 }
0073
0074 bool UPCItemDbFetcher::canFetch(int type) const {
0075   return type == Data::Collection::Video ||
0076          type == Data::Collection::Book ||
0077          type == Data::Collection::Album ||
0078          type == Data::Collection::Game ||
0079          type == Data::Collection::BoardGame;
0080 }
0081
0082 void UPCItemDbFetcher::readConfigHook(const KConfigGroup& config_) {
0083   Q_UNUSED(config_)
0084 }
0085
0086 void UPCItemDbFetcher::saveConfigHook(KConfigGroup& config_) {
0087   Q_UNUSED(config_)
0088 }
0089
0090 void UPCItemDbFetcher::search() {
0091   continueSearch();
0092 }
0093
0094 void UPCItemDbFetcher::continueSearch() {
0095   m_started = true;
0096
0097   QUrl u(QString::fromLatin1(UPCITEMDB_API_URL));
0098   u = u.adjusted(QUrl::StripTrailingSlash);
0099   u.setPath(u.path() + QLatin1String("/lookup"));
0100   QUrlQuery q;
0101   switch(request().key()) {
0102     case ISBN:
0103       // do a upc search by 13-digit isbn
0104       {
0105         // only grab first value
0106         QString isbn = request().value().section(QLatin1Char(';'), 0);
0107         isbn = ISBNValidator::isbn13(isbn);
0108         isbn.remove(QLatin1Char('-'));
0109         q.addQueryItem(QStringLiteral("upc"), isbn);
0110       }
0111       break;
0112
0113     case UPC:
0114       q.addQueryItem(QStringLiteral("upc"), request().value());
0115       break;
0116
0117     default:
0118       myWarning() << source() << "- key not recognized:" << request().key();
0119       stop();
0120       return;
0121   }
0122   u.setQuery(q);
0123
0124   myLog() << "Reading" << u.toDisplayString();
0125   m_job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo);
0126   KJobWidgets::setWindow(m_job, GUI::Proxy::widget());
0127   connect(m_job.data(), &KJob::result, this, &UPCItemDbFetcher::slotComplete);
0128 }
0129
0130 void UPCItemDbFetcher::stop() {
0131   if(!m_started) {
0132     return;
0133   }
0134   if(m_job) {
0135     m_job->kill();
0136     m_job = nullptr;
0137   }
0138   m_started = false;
0139   emit signalDone(this);
0140 }
0141
0142 Tellico::Fetch::FetchRequest UPCItemDbFetcher::updateRequest(Data::EntryPtr entry_) {
0143   const QString isbn = entry_->field(QStringLiteral("isbn"));
0144   if(!isbn.isEmpty()) {
0145     return FetchRequest(ISBN, isbn);
0146   }
0147
0148   const QString upc = entry_->field(QStringLiteral("upc"));
0149   if(!upc.isEmpty()) {
0150     return FetchRequest(UPC, upc);
0151   }
0152
0153   const QString barcode = entry_->field(QStringLiteral("barcode"));
0154   if(!barcode.isEmpty()) {
0155     return FetchRequest(UPC, barcode);
0156   }
0157
0158   return FetchRequest();
0159 }
0160
0161 void UPCItemDbFetcher::slotComplete(KJob* job_) {
0162   KIO::StoredTransferJob* job = static_cast<KIO::StoredTransferJob*>(job_);
0163
0164   if(job->error()) {
0165     job->uiDelegate()->showErrorMessage();
0166     stop();
0167     return;
0168   }
0169
0170   const QByteArray data = job->data();
0171   if(data.isEmpty()) {
0172     myDebug() << "No data";
0173     stop();
0174     return;
0175   }
0176   // see bug 319662. If fetcher is cancelled, job is killed
0177   // if the pointer is retained, it gets double-deleted
0178   m_job = nullptr;
0179
0180 #if 0
0181   myWarning() << "Remove debug from upcitemdbfetcher.cpp";
0182   QFile f(QStringLiteral("/tmp/test-upcitemdb.json"));
0183   if(f.open(QIODevice::WriteOnly)) {
0184     QTextStream t(&f);
0185     t.setCodec("UTF-8");
0186     t << data;
0187   }
0188   f.close();
0189 #endif
0190
0191   QJsonDocument doc = QJsonDocument::fromJson(data);
0192   if(doc.isNull()) {
0193     myDebug() << "null JSON document";
0194     stop();
0195     return;
0196   }
0197   const auto obj = doc.object();
0198   // check for error
0199   if(obj.value(QStringLiteral("code")) == QLatin1String("TOO_FAST")) {
0200     const QString msg = obj.value(QStringLiteral("message")).toString();
0201     message(msg, MessageHandler::Error);
0202     myDebug() << "UPCItemDbFetcher -" << msg;
0203     stop();
0204     return;
0205   }
0206
0207   Data::CollPtr coll = CollectionFactory::collection(collectionType(), true);
0208   if(!coll) {
0209     stop();
0210     return;
0211   }
0212
0213   if(optionalFields().contains(QStringLiteral("barcode"))) {
0214     Data::FieldPtr field(new Data::Field(QStringLiteral("barcode"), i18n("Barcode")));
0215     field->setCategory(i18n("General"));
0216     coll->addField(field);
0217   }
0218
0219   QJsonArray results = obj.value(QLatin1String("items")).toArray();
0220   if(results.isEmpty()) {
0221     myLog() << "No results";
0222     stop();
0223     return;
0224   }
0225
0226   int count = 0;
0227   foreach(const QJsonValue& result, results) {
0228 //    myDebug() << "found result:" << result;
0229
0230     Data::EntryPtr entry(new Data::Entry(coll));
0231     populateEntry(entry, result.toObject().toVariantMap());
0232
0233     FetchResult* r = new FetchResult(this, entry);
0234     m_entries.insert(r->uid, entry);
0235     emit signalResultFound(r);
0236     ++count;
0237     if(count >= UPCITEMDB_MAX_RETURNS_TOTAL) {
0238       break;
0239     }
0240   }
0241
0242   stop();
0243 }
0244
0245 Tellico::Data::EntryPtr UPCItemDbFetcher::fetchEntryHook(uint uid_) {
0246   Data::EntryPtr entry = m_entries.value(uid_);
0247   if(!entry) {
0248     myWarning() << "no entry in dict";
0249     return Data::EntryPtr();
0250   }
0251
0252   // image might still be a URL
0253   const QString cover(QStringLiteral("cover"));
0254   const QString image_id = entry->field(cover);
0255   if(image_id.contains(QLatin1Char('/'))) {
0256     const QUrl imageUrl = QUrl::fromUserInput(image_id);
0257     const QString id = ImageFactory::addImage(imageUrl, false /* quiet */, imageUrl.adjusted(QUrl::RemovePath));
0258     if(id.isEmpty()) {
0259       myDebug() << "image id is empty";
0260       message(i18n("The cover image could not be loaded."), MessageHandler::Warning);
0261     }
0262     // empty image ID is ok
0263     entry->setField(cover, id);
0264   }
0265
0266   return entry;
0267 }
0268
0269 void UPCItemDbFetcher::populateEntry(Data::EntryPtr entry_, const QVariantMap& resultMap_) {
0270   entry_->setField(QStringLiteral("title"), mapValue(resultMap_, "title"));
0271   parseTitle(entry_);
0272 //  entry_->setField(QStringLiteral("year"),  mapValue(resultMap_, "premiered").left(4));
0273   const QString barcode = QStringLiteral("barcode");
0274   if(optionalFields().contains(barcode)) {
0275     entry_->setField(barcode, mapValue(resultMap_, "upc"));
0276   }
0277
0278   // take the first cover
0279   const auto imageList = resultMap_.value(QLatin1String("images")).toList();
0280   for(const auto& imageValue : imageList) {
0281     // skip booksamillion images
0282     const QString image = imageValue.toString();
0283     if(!image.isEmpty() && !image.contains(QLatin1String("booksamillion.com"))) {
0284       entry_->setField(QStringLiteral("cover"), image);
0285       break;
0286     }
0287   }
0288
0289   switch(collectionType()) {
0290     case Data::Collection::Video:
0291       entry_->setField(QStringLiteral("studio"), mapValue(resultMap_, "brand"));
0292       entry_->setField(QStringLiteral("plot"), mapValue(resultMap_, "description"));
0293       break;
0294
0295     case Data::Collection::Book:
0296       entry_->setField(QStringLiteral("publisher"), mapValue(resultMap_, "publisher"));
0297       entry_->setField(QStringLiteral("isbn"), mapValue(resultMap_, "isbn"));
0298       break;
0299
0300     case Data::Collection::Album:
0301       entry_->setField(QStringLiteral("label"), mapValue(resultMap_, "brand"));
0302       {
0303         const QString cat = mapValue(resultMap_, "category");
0304         if(cat.contains(QStringLiteral("Music CDs"))) {
0305           entry_->setField(QStringLiteral("medium"), i18n("Compact Disc"));
0306         }
0307       }
0308       break;
0309
0310     case Data::Collection::Game:
0311     case Data::Collection::BoardGame:
0312       entry_->setField(QStringLiteral("publisher"), mapValue(resultMap_, "brand"));
0313       entry_->setField(QStringLiteral("description"), mapValue(resultMap_, "description"));
0314       break;
0315
0316     default:
0317       break;
0318   }
0319
0320   // do this after all other parsing
0321   parseTitle(entry_);
0322 }
0323
0324 void UPCItemDbFetcher::parseTitle(Tellico::Data::EntryPtr entry_) {
0325   // assume that everything in brackets or parentheses is extra
0326   static const QRegularExpression rx(QLatin1String("[\\(\\[](.*?)[\\)\\]]"));
0327   QString title = entry_->field(QStringLiteral("title"));
0328   int pos = 0;
0329   QRegularExpressionMatch match = rx.match(title, pos);
0330   while(match.hasMatch()) {
0331     pos = match.capturedStart();
0332     if(parseTitleToken(entry_, match.captured(1))) {
0333       title.remove(match.capturedStart(), match.capturedLength());
0334       --pos; // search again there
0335     }
0336     match = rx.match(title, pos+1);
0337   }
0338   // look for "word1 - word2"
0339   static const QRegularExpression dashWords(QLatin1String("(.+) - (.+)"));
0340   QRegularExpressionMatch dashMatch = dashWords.match(title);
0341   if(dashMatch.hasMatch()) {
0342     switch(collectionType()) {
0343       case Data::Collection::Book:
0344         title = dashMatch.captured(1);
0345         {
0346           static const QRegularExpression byAuthor(QLatin1String("by (.+)"));
0347           QRegularExpressionMatch authorMatch = byAuthor.match(dashMatch.captured(2));
0348           if(authorMatch.hasMatch()) {
0349             entry_->setField(QStringLiteral("author"), authorMatch.captured(1).simplified());
0350           }
0351         }
0352         break;
0353
0354       case Data::Collection::Album:
0355         entry_->setField(QStringLiteral("artist"), dashMatch.captured(1).simplified());
0356         title = dashMatch.captured(2);
0357         break;
0358
0359       case Data::Collection::Game:
0360         title = dashMatch.captured(1);
0361         {
0362           const QString platform = QStringLiteral("platform");
0363           const QString maybe = i18n(dashMatch.captured(2).simplified().toUtf8().constData());
0364           Data::FieldPtr f = entry_->collection()->fieldByName(platform);
0365           if(f && f->allowed().contains(maybe)) {
0366             entry_->setField(platform, maybe);
0367           }
0368         }
0369         break;
0370     }
0371   }
0372   entry_->setField(QStringLiteral("title"), title.simplified());
0373 }
0374
0375 // mostly taken from amazonfetcher
0376 bool UPCItemDbFetcher::parseTitleToken(Tellico::Data::EntryPtr entry_, const QString& token_) {
0377 //  myDebug() << "title token:" << token_;
0378   // if res = true, then the token gets removed from the title
0379   bool res = false;
0380   static const QRegularExpression yearRx(QLatin1String("\\d{4}"));
0381   QRegularExpressionMatch yearMatch = yearRx.match(token_);
0382   if(yearMatch.hasMatch()) {
0383     entry_->setField(QStringLiteral("year"), yearMatch.captured());
0384     res = true;
0385   }
0386   if(token_.indexOf(QLatin1String("widescreen"), 0, Qt::CaseInsensitive) > -1 ||
0387      token_.indexOf(i18n("Widescreen"), 0, Qt::CaseInsensitive) > -1) {
0388     entry_->setField(QStringLiteral("widescreen"), QStringLiteral("true"));
0389     // res = true; leave it in the title
0390   } else if(token_.indexOf(QLatin1String("full screen"), 0, Qt::CaseInsensitive) > -1) {
0391     // skip, but go ahead and remove from title
0392     res = true;
0393   } else if(token_.indexOf(QLatin1String("standard edition"), 0, Qt::CaseInsensitive) > -1) {
0394     // skip, but go ahead and remove from title
0395     res = true;
0396   } else if(token_.indexOf(QLatin1String("import"), 0, Qt::CaseInsensitive) > -1) {
0397     // skip, but go ahead and remove from title
0398     res = true;
0399   }
0400   if(token_.indexOf(QLatin1String("blu-ray"), 0, Qt::CaseInsensitive) > -1) {
0401     entry_->setField(QStringLiteral("medium"), i18n("Blu-ray"));
0402     res = true;
0403   } else if(token_.indexOf(QLatin1String("hd dvd"), 0, Qt::CaseInsensitive) > -1) {
0404     entry_->setField(QStringLiteral("medium"), i18n("HD DVD"));
0405     res = true;
0406   } else if(token_.indexOf(QLatin1String("vhs"), 0, Qt::CaseInsensitive) > -1) {
0407     entry_->setField(QStringLiteral("medium"), i18n("VHS"));
0408     res = true;
0409   }
0410   if(token_.indexOf(QLatin1String("director's cut"), 0, Qt::CaseInsensitive) > -1 ||
0411      token_.indexOf(i18n("Director's Cut"), 0, Qt::CaseInsensitive) > -1) {
0412     entry_->setField(QStringLiteral("directors-cut"), QStringLiteral("true"));
0413     // res = true; leave it in the title
0414   }
0415   const QString tokenLower = token_.toLower();
0416   if(tokenLower == QLatin1String("ntsc")) {
0417     entry_->setField(QStringLiteral("format"), i18n("NTSC"));
0418     res = true;
0419   }
0420   if(tokenLower == QLatin1String("dvd")) {
0421     entry_->setField(QStringLiteral("medium"), i18n("DVD"));
0422     res = true;
0423   }
0424   if(tokenLower == QLatin1String("cd") && collectionType() == Data::Collection::Album) {
0425     entry_->setField(QStringLiteral("medium"), i18n("Compact Disc"));
0426     res = true;
0427   }
0428   if(tokenLower == QLatin1String("dvd")) {
0429     entry_->setField(QStringLiteral("medium"), i18n("DVD"));
0430     res = true;
0431   }
0432   if(token_.indexOf(QLatin1String("series"), 0, Qt::CaseInsensitive) > -1) {
0433     entry_->setField(QStringLiteral("series"), token_);
0434     res = true;
0435   }
0436   static const QRegularExpression regionRx(QLatin1String("Region [1-9]"));
0437   QRegularExpressionMatch match = regionRx.match(token_);
0438   if(match.hasMatch()) {
0439     entry_->setField(QStringLiteral("region"), i18n(match.captured().toUtf8().constData()));
0440     res = true;
0441   }
0442   if(collectionType() == Data::Collection::Game) {
0443     Data::FieldPtr f = entry_->collection()->fieldByName(QStringLiteral("platform"));
0444     if(f && f->allowed().contains(token_)) {
0445       res = true;
0446     }
0447   } else if(collectionType() == Data::Collection::Book) {
0448     const QString binding = QStringLiteral("binding");
0449     Data::FieldPtr f = entry_->collection()->fieldByName(binding);
0450     const QString maybe = i18n(token_.toUtf8().constData());
0451     if(f && f->allowed().contains(maybe)) {
0452       entry_->setField(binding, maybe);
0453       res = true;
0454     }
0455   }
0456   return res;
0457 }
0458
0459 Tellico::Fetch::ConfigWidget* UPCItemDbFetcher::configWidget(QWidget* parent_) const {
0460   return new UPCItemDbFetcher::ConfigWidget(parent_, this);
0461 }
0462
0463 QString UPCItemDbFetcher::defaultName() {
0464   return QStringLiteral("UPCitemdb"); // this is the capitalization they use on their site
0465 }
0466
0467 QString UPCItemDbFetcher::defaultIcon() {
0468   return favIcon("https://www.upcitemdb.com");
0469 }
0470
0471 Tellico::StringHash UPCItemDbFetcher::allOptionalFields() {
0472   StringHash hash;
0473   hash[QStringLiteral("barcode")] = i18n("Barcode");
0474   return hash;
0475 }
0476
0477 UPCItemDbFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const UPCItemDbFetcher* fetcher_)
0478     : Fetch::ConfigWidget(parent_) {
0479   QVBoxLayout* l = new QVBoxLayout(optionsWidget());
0480   l->addWidget(new QLabel(i18n("This source has no options."), optionsWidget()));
0481   l->addStretch();
0482
0483   // now add additional fields widget
0484   addFieldsWidget(UPCItemDbFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList());
0485 }
0486
0487 QString UPCItemDbFetcher::ConfigWidget::preferredName() const {
0488   return UPCItemDbFetcher::defaultName();
0489 }