File indexing completed on 2024-05-12 05:09:36 

0001 /***************************************************************************
0002     Copyright (C) 2006-2020 Robby Stephenson <robby@periapsis.org>
0003  ***************************************************************************/
0004 
0005 /***************************************************************************
0006  *                                                                         *
0007  *   This program is free software; you can redistribute it and/or         *
0008  *   modify it under the terms of the GNU General Public License as        *
0009  *   published by the Free Software Foundation; either version 2 of        *
0010  *   the License or (at your option) version 3 or any later version        *
0011  *   accepted by the membership of KDE e.V. (or its successor approved     *
0012  *   by the membership of KDE e.V.), which shall act as a proxy            *
0013  *   defined in Section 14 of version 3 of the license.                    *
0014  *                                                                         *
0015  *   This program is distributed in the hope that it will be useful,       *
0016  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
0017  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
0018  *   GNU General Public License for more details.                          *
0019  *                                                                         *
0020  *   You should have received a copy of the GNU General Public License     *
0021  *   along with this program.  If not, see <http://www.gnu.org/licenses/>. *
0022  *                                                                         *
0023  ***************************************************************************/
0024 
0025 #include "isbndbfetcher.h"
0026 #include "../collections/bookcollection.h"
0027 #include "../images/imagefactory.h"
0028 #include "../utils/guiproxy.h"
0029 #include "../utils/mapvalue.h"
0030 #include "../tellico_debug.h"
0031 
0032 #include <KLocalizedString>
0033 #include <KIO/Job>
0034 #include <KIO/JobUiDelegate>
0035 #include <KConfigGroup>
0036 #include <KJobWidgets/KJobWidgets>
0037 
0038 #include <QLineEdit>
0039 #include <QCheckBox>
0040 #include <QLabel>
0041 #include <QFile>
0042 #include <QTextStream>
0043 #include <QVBoxLayout>
0044 #include <QTextCodec>
0045 #include <QJsonDocument>
0046 #include <QJsonObject>
0047 #include <QUrlQuery>
0048 
0049 namespace {
0050   static const int ISBNDB_MAX_RETURNS_TOTAL = 25;
0051   static const char* ISBNDB_BASE_URL = "https://api2.isbndb.com";
0052 }
0053 
0054 using namespace Tellico;
0055 using Tellico::Fetch::ISBNdbFetcher;
0056 
0057 ISBNdbFetcher::ISBNdbFetcher(QObject* parent_)
0058     : Fetcher(parent_),
0059       m_limit(ISBNDB_MAX_RETURNS_TOTAL), m_total(-1), m_numResults(0),
0060       m_started(false),
0061       m_batchIsbn(false) {
0062 }
0063 
0064 ISBNdbFetcher::~ISBNdbFetcher() {
0065 }
0066 
0067 QString ISBNdbFetcher::source() const {
0068   return m_name.isEmpty() ? defaultName() : m_name;
0069 }
0070 
0071 bool ISBNdbFetcher::canFetch(int type) const {
0072   return type == Data::Collection::Book || type == Data::Collection::Bibtex;
0073 }
0074 
0075 bool ISBNdbFetcher::canSearch(Fetch::FetchKey k) const {
0076   return k == Title || k == Person || k == ISBN || k == Keyword;
0077 }
0078 
0079 void ISBNdbFetcher::readConfigHook(const KConfigGroup& config_) {
0080   QString k = config_.readEntry("API Key");
0081   if(!k.isEmpty()) {
0082     m_apiKey = k;
0083   }
0084   m_batchIsbn = config_.readEntry("Batch ISBN", false);
0085 }
0086 
0087 void ISBNdbFetcher::search() {
0088   m_started = true;
0089   m_total = -1;
0090   m_numResults = 0;
0091 
0092   // we only split ISBN when not doing batch searching
0093   QStringList searchTerms;
0094   if(request().key() == ISBN && !m_batchIsbn) {
0095     searchTerms = FieldFormat::splitValue(request().value());
0096   } else  {
0097     searchTerms += request().value();
0098   }
0099   foreach(const QString& searchTerm, searchTerms) {
0100     doSearch(searchTerm);
0101   }
0102   if(m_jobs.isEmpty()) {
0103     stop();
0104   }
0105 }
0106 
0107 void ISBNdbFetcher::continueSearch() {
0108   m_started = true;
0109 
0110   doSearch(request().value());
0111 }
0112 
0113 void ISBNdbFetcher::doSearch(const QString& term_) {
0114   const bool multipleIsbn = request().key() == ISBN && term_.contains(QLatin1Char(';'));
0115 
0116   QUrl u(QString::fromLatin1(ISBNDB_BASE_URL));
0117   switch(request().key()) {
0118     case Title:
0119       u.setPath(QStringLiteral("/books/") + term_);
0120       break;
0121 
0122     case Person:
0123       // the /books/query search endpoint seems to not work with the author column yet [2020-09-02]
0124       // so continue to user /author/query search (which may not return all the same info)
0125       u.setPath(QStringLiteral("/author/") + term_);
0126       break;
0127 
0128     case ISBN:
0129       if(multipleIsbn) {
0130         u.setPath(QStringLiteral("/books"));
0131       } else {
0132         u.setPath(QStringLiteral("/book/"));
0133         // can only grab first value
0134         QString v = term_.section(QLatin1Char(';'), 0);
0135         v.remove(QLatin1Char('-'));
0136         u.setPath(u.path() + v);
0137       }
0138       break;
0139 
0140     case Keyword:
0141       // the /books/query search endpoint seems to not work with the author column yet [2020-09-02]
0142       // so continue to user /author/query search (which may not return all the same info)
0143       u.setPath(QStringLiteral("/books/") + term_);
0144       {
0145          QUrlQuery q;
0146          q.addQueryItem(QStringLiteral("page"), QLatin1String("1"));
0147          q.addQueryItem(QStringLiteral("pageSize"), QString::number(ISBNDB_MAX_RETURNS_TOTAL));
0148          // disable beta searching
0149          q.addQueryItem(QStringLiteral("beta"), QLatin1String("0"));
0150          u.setQuery(q);
0151       }
0152       break;
0153 
0154     default:
0155       myWarning() << source() << "- key not recognized:" << request().key();
0156       stop();
0157       return;
0158   }
0159 
0160   if(m_apiKey.isEmpty()) {
0161     myDebug() << source() << "- empty API key";
0162     message(i18n("An access key is required to use this data source.")
0163             + QLatin1Char(' ') +
0164             i18n("Those values must be entered in the data source settings."), MessageHandler::Error);
0165     stop();
0166     return;
0167   }
0168 
0169 //  myDebug() << "url: " << u.url();
0170 
0171   QPointer<KIO::StoredTransferJob> job;
0172   if(multipleIsbn) {
0173     QString postData = request().value();
0174     postData = postData.replace(QLatin1Char(';'), QLatin1Char(','))
0175                        .remove(QLatin1Char('-'))
0176                        .remove(QLatin1Char(' '));
0177     postData.prepend(QStringLiteral("isbns="));
0178 //    myDebug() << "posting" << postData;
0179     job = KIO::storedHttpPost(postData.toUtf8(), u, KIO::HideProgressInfo);
0180   } else {
0181     job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo);
0182   }
0183 
0184   job->addMetaData(QStringLiteral("customHTTPHeader"), QStringLiteral("Authorization: ") + m_apiKey);
0185   job->addMetaData(QStringLiteral("content-type"), QStringLiteral("application/json"));
0186   KJobWidgets::setWindow(job, GUI::Proxy::widget());
0187   connect(job.data(), &KJob::result, this, &ISBNdbFetcher::slotComplete);
0188   m_jobs << job;
0189 }
0190 
0191 void ISBNdbFetcher::endJob(KIO::StoredTransferJob* job_) {
0192   m_jobs.removeAll(job_);
0193   if(m_jobs.isEmpty())  {
0194     stop();
0195   }
0196 }
0197 
0198 void ISBNdbFetcher::stop() {
0199   if(!m_started) {
0200     return;
0201   }
0202   foreach(QPointer<KIO::StoredTransferJob> job, m_jobs) {
0203     if(job) {
0204       job->kill();
0205     }
0206   }
0207   m_jobs.clear();
0208   m_started = false;
0209   emit signalDone(this);
0210 }
0211 
0212 void ISBNdbFetcher::slotComplete(KJob* job_) {
0213   KIO::StoredTransferJob* job = static_cast<KIO::StoredTransferJob*>(job_);
0214 
0215   if(job->error()) {
0216     job->uiDelegate()->showErrorMessage();
0217     endJob(job);
0218     return;
0219   }
0220 
0221   const QByteArray data = job->data();
0222   if(data.isEmpty()) {
0223     myDebug() << "no data";
0224     endJob(job);
0225     return;
0226   }
0227 
0228 #if 0
0229   myWarning() << "Remove debug from isbndbfetcher.cpp";
0230   QFile file(QString::fromLatin1("/tmp/test-isbndb.json"));
0231   if(file.open(QIODevice::WriteOnly)) {
0232     QTextStream t(&file);
0233     t.setCodec("UTF-8");
0234     t << data;
0235   }
0236   file.close();
0237 #endif
0238 
0239   QJsonDocument doc = QJsonDocument::fromJson(data);
0240   QVariantMap result = doc.object().toVariantMap();
0241   QVariantList resultList;
0242   if(result.contains(QStringLiteral("book"))) {
0243     resultList += result.value(QStringLiteral("book"));
0244     m_total = 1;
0245   } else if(result.contains(QStringLiteral("books"))) {
0246     m_total = result.value(QStringLiteral("total")).toInt();
0247     resultList = result.value(QStringLiteral("books")).toList();
0248   } else if(result.contains(QStringLiteral("data"))) {
0249     m_total = result.value(QStringLiteral("total")).toInt();
0250     resultList = result.value(QStringLiteral("data")).toList();
0251   } else {
0252     QString msg = result.value(QStringLiteral("message")).toString();
0253     if(msg.isEmpty()) msg = result.value(QStringLiteral("errorMessage")).toString();
0254     myDebug() << "no results from ISBNDBFetcher:" << msg;
0255     message(msg, MessageHandler::Error);
0256     endJob(job);
0257     return;
0258   }
0259 //  myDebug() << "Total:" << m_total;
0260 
0261   Data::CollPtr coll(new Data::BookCollection(true));
0262 
0263   int count = 0;
0264   foreach(const QVariant& result, resultList) {
0265 //    myDebug() << "found result:" << result;
0266 
0267     Data::EntryPtr entry(new Data::Entry(coll));
0268     populateEntry(entry, result.toMap());
0269 
0270     FetchResult* r = new FetchResult(this, entry);
0271     m_entries.insert(r->uid, entry);
0272     emit signalResultFound(r);
0273     ++count;
0274     ++m_numResults;
0275     if(count >= m_limit) {
0276       break;
0277     }
0278   }
0279 
0280   endJob(job);
0281 }
0282 
0283 Tellico::Data::EntryPtr ISBNdbFetcher::fetchEntryHook(uint uid_) {
0284   if(!m_entries.contains(uid_)) {
0285     myDebug() << "no entry ptr";
0286     return Data::EntryPtr();
0287   }
0288 
0289   Data::EntryPtr entry = m_entries.value(uid_);
0290 
0291   // image might still be a URL
0292   const QString image_id = entry->field(QStringLiteral("cover"));
0293   if(image_id.contains(QLatin1Char('/'))) {
0294     const QString id = ImageFactory::addImage(QUrl::fromUserInput(image_id), true /* quiet */);
0295     if(id.isEmpty()) {
0296       message(i18n("The cover image could not be loaded."), MessageHandler::Warning);
0297     }
0298     // empty image ID is ok
0299     entry->setField(QStringLiteral("cover"), id);
0300   }
0301 
0302   return entry;
0303 }
0304 
0305 Tellico::Fetch::FetchRequest ISBNdbFetcher::updateRequest(Data::EntryPtr entry_) {
0306   QString isbn = entry_->field(QStringLiteral("isbn"));
0307   if(!isbn.isEmpty()) {
0308     return FetchRequest(Fetch::ISBN, isbn);
0309   }
0310 
0311   // optimistically try searching for title and rely on Collection::sameEntry() to figure things out
0312   QString t = entry_->field(QStringLiteral("title"));
0313   if(!t.isEmpty()) {
0314     return FetchRequest(Fetch::Title, t);
0315   }
0316   return FetchRequest();
0317 }
0318 
0319 void ISBNdbFetcher::populateEntry(Data::EntryPtr entry_, const QVariantMap& resultMap_) {
0320   static const QRegularExpression nonDigits(QStringLiteral("[^\\d]"));
0321   entry_->setField(QStringLiteral("title"), mapValue(resultMap_, "title"));
0322   entry_->setField(QStringLiteral("isbn"), mapValue(resultMap_, "isbn"));
0323   // "date_published" can be "2008-12-13" or "July 2012"
0324   QString pubYear = mapValue(resultMap_, "date_published").remove(nonDigits).left(4);
0325   entry_->setField(QStringLiteral("pub_year"), pubYear);
0326   QStringList authors;
0327   foreach(const QVariant& author, resultMap_.value(QLatin1String("authors")).toList()) {
0328     authors += author.toString();
0329   }
0330   entry_->setField(QStringLiteral("author"), authors.join(FieldFormat::delimiterString()));
0331   entry_->setField(QStringLiteral("publisher"), mapValue(resultMap_, "publisher"));
0332   entry_->setField(QStringLiteral("edition"), mapValue(resultMap_, "edition"));
0333   QString binding = mapValue(resultMap_, "binding");
0334   if(binding.isEmpty()) {
0335     binding = mapValue(resultMap_, "format");
0336   }
0337   if(binding.startsWith(QStringLiteral("Hardcover"))) {
0338     binding = QStringLiteral("Hardback");
0339   } else if(binding.startsWith(QStringLiteral("Paperback"))) {
0340     binding = QStringLiteral("Paperback");
0341   }
0342   if(!binding.isEmpty()) {
0343     entry_->setField(QStringLiteral("binding"), i18n(binding.toUtf8().constData()));
0344   }
0345   QStringList subjects;
0346   foreach(const QVariant& subject, resultMap_.value(QLatin1String("subjects")).toList()) {
0347     subjects += subject.toString();
0348   }
0349   entry_->setField(QStringLiteral("genre"), subjects.join(FieldFormat::delimiterString()));
0350   entry_->setField(QStringLiteral("cover"), mapValue(resultMap_, "image"));
0351   entry_->setField(QStringLiteral("pages"), mapValue(resultMap_, "pages"));
0352   entry_->setField(QStringLiteral("language"), mapValue(resultMap_, "language"));
0353   entry_->setField(QStringLiteral("plot"), mapValue(resultMap_, "overview"));
0354   if(mapValue(resultMap_, "overview").isEmpty()) {
0355     entry_->setField(QStringLiteral("plot"), mapValue(resultMap_, "synopsis"));
0356   }
0357 
0358   const QString dewey = mapValue(resultMap_, "dewey_decimal");
0359   if(!dewey.isEmpty() && optionalFields().contains(QStringLiteral("dewey"))) {
0360     if(!entry_->collection()->hasField(QStringLiteral("dewey"))) {
0361       Data::FieldPtr field(new Data::Field(QStringLiteral("dewey"), i18n("Dewey Decimal"), Data::Field::Line));
0362       field->setCategory(i18n("Publishing"));
0363       entry_->collection()->addField(field);
0364     }
0365     entry_->setField(QStringLiteral("dewey"), dewey);
0366   }
0367 }
0368 
0369 Tellico::Fetch::ConfigWidget* ISBNdbFetcher::configWidget(QWidget* parent_) const {
0370   return new ISBNdbFetcher::ConfigWidget(parent_, this);
0371 }
0372 
0373 QString ISBNdbFetcher::defaultName() {
0374   return i18n("ISBNdb.com");
0375 }
0376 
0377 QString ISBNdbFetcher::defaultIcon() {
0378   return favIcon("https://isbndb.com/sites/default/files/favicon_0.ico");
0379 }
0380 
0381 Tellico::StringHash ISBNdbFetcher::allOptionalFields() {
0382   // same ones as z3950fetcher
0383   StringHash hash;
0384   hash[QStringLiteral("dewey")] = i18nc("Dewey Decimal classification system", "Dewey Decimal");
0385   return hash;
0386 }
0387 
0388 ISBNdbFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const ISBNdbFetcher* fetcher_)
0389     : Fetch::ConfigWidget(parent_) {
0390   QGridLayout* l = new QGridLayout(optionsWidget());
0391   l->setSpacing(4);
0392   l->setColumnStretch(1, 10);
0393 
0394   int row = -1;
0395   QLabel* al = new QLabel(i18n("Registration is required for accessing this data source. "
0396                                "If you agree to the terms and conditions, <a href='%1'>sign "
0397                                "up for an account</a>, and enter your information below.",
0398                                 QStringLiteral("https://isbndb.com/isbn-database")),
0399                           optionsWidget());
0400   al->setOpenExternalLinks(true);
0401   al->setWordWrap(true);
0402   ++row;
0403   l->addWidget(al, row, 0, 1, 2);
0404   // richtext gets weird with size
0405   al->setMinimumWidth(al->sizeHint().width());
0406 
0407   QLabel* label = new QLabel(i18n("Access key: "), optionsWidget());
0408   l->addWidget(label, ++row, 0);
0409 
0410   m_apiKeyEdit = new QLineEdit(optionsWidget());
0411   connect(m_apiKeyEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified);
0412   l->addWidget(m_apiKeyEdit, row, 1);
0413   label->setBuddy(m_apiKeyEdit);
0414 
0415   m_enableBatchIsbn = new QCheckBox(i18n("Enable batch ISBN searching (requires Premium or Pro plan)"), optionsWidget());
0416   connect(m_enableBatchIsbn, &QAbstractButton::clicked, this, &ConfigWidget::slotSetModified);
0417   ++row;
0418   l->addWidget(m_enableBatchIsbn, row, 0, 1, 2);
0419   QString w = i18n("Batch searching for ISBN values is faster but only available for Premium or Pro plans.");
0420   m_enableBatchIsbn->setWhatsThis(w);
0421 
0422   l->setRowStretch(++row, 10);
0423 
0424   if(fetcher_) {
0425     m_apiKeyEdit->setText(fetcher_->m_apiKey);
0426     m_enableBatchIsbn->setChecked(fetcher_->m_batchIsbn);
0427   } else { //defaults
0428     m_enableBatchIsbn->setChecked(false);
0429   }
0430 
0431   // now add additional fields widget
0432   addFieldsWidget(ISBNdbFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList());
0433 }
0434 
0435 void ISBNdbFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) {
0436   QString apiKey = m_apiKeyEdit->text().trimmed();
0437   if(!apiKey.isEmpty()) {
0438     config_.writeEntry("API Key", apiKey);
0439   }
0440   config_.writeEntry("Batch ISBN", m_enableBatchIsbn->isChecked());
0441 }
0442 
0443 QString ISBNdbFetcher::ConfigWidget::preferredName() const {
0444   return ISBNdbFetcher::defaultName();
0445 }