File indexing completed on 2024-05-12 16:45:58
0001 /*************************************************************************** 0002 Copyright (C) 2009 Robby Stephenson <robby@periapsis.org> 0003 ***************************************************************************/ 0004 0005 /*************************************************************************** 0006 * * 0007 * This program is free software; you can redistribute it and/or * 0008 * modify it under the terms of the GNU General Public License as * 0009 * published by the Free Software Foundation; either version 2 of * 0010 * the License or (at your option) version 3 or any later version * 0011 * accepted by the membership of KDE e.V. (or its successor approved * 0012 * by the membership of KDE e.V.), which shall act as a proxy * 0013 * defined in Section 14 of version 3 of the license. * 0014 * * 0015 * This program is distributed in the hope that it will be useful, * 0016 * but WITHOUT ANY WARRANTY; without even the implied warranty of * 0017 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 0018 * GNU General Public License for more details. * 0019 * * 0020 * You should have received a copy of the GNU General Public License * 0021 * along with this program. If not, see <http://www.gnu.org/licenses/>. * 0022 * * 0023 ***************************************************************************/ 0024 0025 #include "openlibraryfetcher.h" 0026 #include "../collections/bookcollection.h" 0027 #include "../images/imagefactory.h" 0028 #include "../utils/isbnvalidator.h" 0029 #include "../utils/guiproxy.h" 0030 #include "../utils/string_utils.h" 0031 #include "../entry.h" 0032 #include "../core/filehandler.h" 0033 #include "../tellico_debug.h" 0034 0035 #include <KLocalizedString> 0036 #include <KIO/Job> 0037 #include <KJobUiDelegate> 0038 #include <KJobWidgets/KJobWidgets> 0039 0040 #include <QLabel> 0041 #include <QFile> 0042 #include <QTextStream> 0043 #include <QGridLayout> 0044 #include <QTextCodec> 0045 #include <QJsonDocument> 0046 #include <QJsonObject> 0047 #include <QJsonArray> 0048 #include <QUrlQuery> 0049 0050 namespace { 0051 static const char* OPENLIBRARY_QUERY_URL = "http://openlibrary.org/query.json"; 0052 } 0053 0054 using namespace Tellico; 0055 using Tellico::Fetch::OpenLibraryFetcher; 0056 0057 OpenLibraryFetcher::OpenLibraryFetcher(QObject* parent_) 0058 : Fetcher(parent_), m_started(false) { 0059 } 0060 0061 OpenLibraryFetcher::~OpenLibraryFetcher() { 0062 } 0063 0064 QString OpenLibraryFetcher::source() const { 0065 return m_name.isEmpty() ? defaultName() : m_name; 0066 } 0067 0068 bool OpenLibraryFetcher::canSearch(Fetch::FetchKey k) const { 0069 return k == Title || k == Person || k == ISBN || k == LCCN || k == Keyword; 0070 } 0071 0072 bool OpenLibraryFetcher::canFetch(int type) const { 0073 return type == Data::Collection::Book || type == Data::Collection::Bibtex; 0074 } 0075 0076 void OpenLibraryFetcher::readConfigHook(const KConfigGroup&) { 0077 } 0078 0079 void OpenLibraryFetcher::search() { 0080 m_started = true; 0081 // we only split ISBN and LCCN values 0082 QStringList searchTerms; 0083 if(request().key() == ISBN || request().key() == LCCN) { 0084 searchTerms = FieldFormat::splitValue(request().value()); 0085 } else { 0086 searchTerms += request().value(); 0087 } 0088 foreach(const QString& searchTerm, searchTerms) { 0089 doSearch(searchTerm); 0090 } 0091 if(m_jobs.isEmpty()) { 0092 stop(); 0093 } 0094 } 0095 0096 void OpenLibraryFetcher::doSearch(const QString& term_) { 0097 QUrl u(QString::fromLatin1(OPENLIBRARY_QUERY_URL)); 0098 QUrlQuery q; 0099 // books are type/edition 0100 q.addQueryItem(QStringLiteral("type"), QStringLiteral("/type/edition")); 0101 0102 switch(request().key()) { 0103 case Title: 0104 q.addQueryItem(QStringLiteral("title"), term_); 0105 break; 0106 0107 case Person: 0108 { 0109 const QString author = getAuthorKeys(term_); 0110 if(author.isEmpty()) { 0111 myWarning() << "no authors found"; 0112 return; 0113 } 0114 q.addQueryItem(QStringLiteral("authors"), author); 0115 } 0116 break; 0117 0118 case ISBN: 0119 { 0120 const QString isbn = ISBNValidator::cleanValue(term_); 0121 if(isbn.size() > 10) { 0122 q.addQueryItem(QStringLiteral("isbn_13"), isbn); 0123 } else { 0124 q.addQueryItem(QStringLiteral("isbn_10"), isbn); 0125 } 0126 } 0127 break; 0128 0129 case LCCN: 0130 q.addQueryItem(QStringLiteral("lccn"), term_); 0131 break; 0132 0133 case Raw: 0134 { 0135 // raw query comes in as a query string, combine it 0136 QUrlQuery newQuery(term_); 0137 foreach(auto item, newQuery.queryItems()) { 0138 q.addQueryItem(item.first, item.second); 0139 } 0140 } 0141 break; 0142 0143 case Keyword: 0144 myWarning() << "not supported"; 0145 return; 0146 0147 default: 0148 myWarning() << "key not recognized:" << request().key(); 0149 return; 0150 } 0151 q.addQueryItem(QStringLiteral("*"), QString()); 0152 u.setQuery(q); 0153 // myDebug() << "url:" << u; 0154 0155 QPointer<KIO::StoredTransferJob> job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); 0156 KJobWidgets::setWindow(job, GUI::Proxy::widget()); 0157 connect(job.data(), &KJob::result, this, &OpenLibraryFetcher::slotComplete); 0158 m_jobs << job; 0159 } 0160 0161 void OpenLibraryFetcher::endJob(KIO::StoredTransferJob* job_) { 0162 m_jobs.removeAll(job_); 0163 if(m_jobs.isEmpty()) { 0164 stop(); 0165 } 0166 } 0167 0168 void OpenLibraryFetcher::stop() { 0169 if(!m_started) { 0170 return; 0171 } 0172 foreach(QPointer<KIO::StoredTransferJob> job, m_jobs) { 0173 if(job) { 0174 job->kill(); 0175 } 0176 } 0177 m_jobs.clear(); 0178 m_started = false; 0179 emit signalDone(this); 0180 } 0181 0182 Tellico::Data::EntryPtr OpenLibraryFetcher::fetchEntryHook(uint uid_) { 0183 Data::EntryPtr entry = m_entries.value(uid_); 0184 if(!entry) { 0185 myWarning() << "no entry in dict"; 0186 return Data::EntryPtr(); 0187 } 0188 0189 // if the entry is not set, go ahead and try to fetch it 0190 if(entry->field(QStringLiteral("cover")).isEmpty()) { 0191 const QString isbn = ISBNValidator::cleanValue(entry->field(QStringLiteral("isbn"))); 0192 if(!isbn.isEmpty()) { 0193 QUrl imageUrl(QStringLiteral("http://covers.openlibrary.org/b/isbn/%1-M.jpg?default=false").arg(isbn)); 0194 const QString id = ImageFactory::addImage(imageUrl, true); 0195 if(!id.isEmpty()) { 0196 entry->setField(QStringLiteral("cover"), id); 0197 } 0198 } 0199 } 0200 0201 return entry; 0202 } 0203 0204 Tellico::Fetch::FetchRequest OpenLibraryFetcher::updateRequest(Data::EntryPtr entry_) { 0205 const QString isbn = entry_->field(QStringLiteral("isbn")); 0206 if(!isbn.isEmpty()) { 0207 return FetchRequest(ISBN, isbn); 0208 } 0209 const QString lccn = entry_->field(QStringLiteral("lccn")); 0210 if(!lccn.isEmpty()) { 0211 return FetchRequest(LCCN, lccn); 0212 } 0213 const QString title = entry_->field(QStringLiteral("title")); 0214 if(title.isEmpty()) { 0215 return FetchRequest(); 0216 } 0217 0218 // can't search by authors, for now, since the author value is a key reference 0219 // can't search by pub year since many of the publish_date fields are a full month, day, year 0220 0221 const QString pub = entry_->field(QStringLiteral("publisher")); 0222 auto publishers = FieldFormat::splitValue(pub); 0223 if(!publishers.isEmpty()) { 0224 QUrlQuery q; 0225 q.addQueryItem(QStringLiteral("title"), title); 0226 q.addQueryItem(QStringLiteral("publishers"), publishers.first()); 0227 return FetchRequest(Raw, q.query()); 0228 } 0229 0230 // fallback to just title search 0231 return FetchRequest(Title, title); 0232 } 0233 0234 void OpenLibraryFetcher::slotComplete(KJob* job_) { 0235 KIO::StoredTransferJob* job = static_cast<KIO::StoredTransferJob*>(job_); 0236 0237 if(job->error()) { 0238 job->uiDelegate()->showErrorMessage(); 0239 endJob(job); 0240 return; 0241 } 0242 0243 QByteArray data = job->data(); 0244 if(data.isEmpty()) { 0245 myDebug() << "no data"; 0246 endJob(job); 0247 return; 0248 } 0249 0250 #if 0 0251 myWarning() << "Remove debug from openlibraryfetcher.cpp"; 0252 QFile f(QString::fromLatin1("/tmp/test.json")); 0253 if(f.open(QIODevice::WriteOnly)) { 0254 QTextStream t(&f); 0255 t.setCodec("UTF-8"); 0256 t << data; 0257 } 0258 f.close(); 0259 #endif 0260 0261 QJsonDocument doc = QJsonDocument::fromJson(data); 0262 QJsonArray array = doc.array(); 0263 if(array.isEmpty()) { 0264 // myDebug() << "no results"; 0265 endJob(job); 0266 return; 0267 } 0268 0269 Data::CollPtr coll(new Data::BookCollection(true)); 0270 if(!coll->hasField(QStringLiteral("openlibrary")) && optionalFields().contains(QStringLiteral("openlibrary"))) { 0271 Data::FieldPtr field(new Data::Field(QStringLiteral("openlibrary"), i18n("OpenLibrary Link"), Data::Field::URL)); 0272 field->setCategory(i18n("General")); 0273 coll->addField(field); 0274 } 0275 0276 for(int i = 0; i < array.count(); i++) { 0277 // be sure to check that the fetcher has not been stopped 0278 // crashes can occur if not 0279 if(!m_started) { 0280 break; 0281 } 0282 // myDebug() << "found result:" << result; 0283 QVariantMap resultMap = array.at(i).toObject().toVariantMap(); 0284 0285 // myDebug() << resultMap.value(QLatin1String("isbn_10")).toList().at(0); 0286 0287 Data::EntryPtr entry(new Data::Entry(coll)); 0288 0289 entry->setField(QStringLiteral("title"), mapValue(resultMap, "title")); 0290 entry->setField(QStringLiteral("subtitle"), mapValue(resultMap, "subtitle")); 0291 QRegularExpression yearRx(QStringLiteral("\\d{4}")); 0292 QRegularExpressionMatch yearMatch = yearRx.match(mapValue(resultMap, "publish_date")); 0293 if(yearMatch.hasMatch()) { 0294 entry->setField(QStringLiteral("pub_year"), yearMatch.captured()); 0295 } 0296 QString isbn = mapValue(resultMap, "isbn_10"); 0297 if(isbn.isEmpty()) { 0298 isbn = mapValue(resultMap, "isbn_13"); 0299 } 0300 if(!isbn.isEmpty()) { 0301 ISBNValidator val(this); 0302 val.fixup(isbn); 0303 entry->setField(QStringLiteral("isbn"), isbn); 0304 } 0305 entry->setField(QStringLiteral("lccn"), mapValue(resultMap, "lccn")); 0306 entry->setField(QStringLiteral("genre"), mapValue(resultMap, "genres")); 0307 entry->setField(QStringLiteral("keyword"), mapValue(resultMap, "subjects")); 0308 entry->setField(QStringLiteral("edition"), mapValue(resultMap, "edition_name")); 0309 QString binding = mapValue(resultMap, "physical_format"); 0310 if(binding.toLower() == QLatin1String("hardcover")) { 0311 binding = QStringLiteral("Hardback"); 0312 } else if(binding.contains(QStringLiteral("paperback"), Qt::CaseInsensitive)) { 0313 binding = QStringLiteral("Paperback"); 0314 } 0315 if(!binding.isEmpty()) { 0316 entry->setField(QStringLiteral("binding"), i18n(binding.toUtf8().constData())); 0317 } 0318 entry->setField(QStringLiteral("publisher"), mapValue(resultMap, "publishers")); 0319 entry->setField(QStringLiteral("series"), mapValue(resultMap, "series")); 0320 entry->setField(QStringLiteral("pages"), mapValue(resultMap, "number_of_pages")); 0321 entry->setField(QStringLiteral("comments"), mapValue(resultMap, "notes")); 0322 0323 if(optionalFields().contains(QStringLiteral("openlibrary"))) { 0324 entry->setField(QStringLiteral("openlibrary"), QLatin1String("https://openlibrary.org") + mapValue(resultMap, "key")); 0325 } 0326 0327 QStringList authors; 0328 foreach(const QVariant& authorMap, resultMap.value(QLatin1String("authors")).toList()) { 0329 const QString key = mapValue(authorMap.toMap(), "key"); 0330 if(!key.isEmpty()) { 0331 QUrl authorUrl(QString::fromLatin1(OPENLIBRARY_QUERY_URL)); 0332 QUrlQuery q; 0333 q.addQueryItem(QStringLiteral("type"), QStringLiteral("/type/author")); 0334 q.addQueryItem(QStringLiteral("key"), key); 0335 q.addQueryItem(QStringLiteral("name"), QString()); 0336 authorUrl.setQuery(q); 0337 0338 QString output = FileHandler::readTextFile(authorUrl, true /*quiet*/); 0339 QJsonDocument doc = QJsonDocument::fromJson(output.toUtf8()); 0340 QJsonArray array = doc.array(); 0341 QVariantMap authorResult = array.isEmpty() ? QVariantMap() : array.at(0).toObject().toVariantMap(); 0342 const QString name = mapValue(authorResult, "name"); 0343 if(!name.isEmpty()) { 0344 authors << name; 0345 } 0346 } 0347 } 0348 if(!authors.isEmpty()) { 0349 entry->setField(QStringLiteral("author"), authors.join(FieldFormat::delimiterString())); 0350 } 0351 0352 QStringList langs; 0353 foreach(const QVariant& langMap, resultMap.value(QLatin1String("languages")).toList()) { 0354 const QString key = mapValue(langMap.toMap(), "key"); 0355 if(!key.isEmpty()) { 0356 QUrl langUrl(QString::fromLatin1(OPENLIBRARY_QUERY_URL)); 0357 QUrlQuery q; 0358 q.addQueryItem(QStringLiteral("type"), QStringLiteral("/type/language")); 0359 q.addQueryItem(QStringLiteral("key"), key); 0360 q.addQueryItem(QStringLiteral("name"), QString()); 0361 langUrl.setQuery(q); 0362 0363 QString output = FileHandler::readTextFile(langUrl, true /*quiet*/, true /*utf8*/); 0364 QJsonDocument doc = QJsonDocument::fromJson(output.toUtf8()); 0365 QJsonArray array = doc.array(); 0366 QVariantMap langResult = array.isEmpty() ? QVariantMap() : array.at(0).toObject().toVariantMap(); 0367 const QString name = mapValue(langResult, "name"); 0368 if(!name.isEmpty()) { 0369 langs << i18n(name.toUtf8().constData()); 0370 } 0371 } 0372 } 0373 if(!langs.isEmpty()) { 0374 entry->setField(QStringLiteral("language"), langs.join(FieldFormat::delimiterString())); 0375 } 0376 0377 FetchResult* r = new FetchResult(this, entry); 0378 m_entries.insert(r->uid, entry); 0379 emit signalResultFound(r); 0380 } 0381 0382 // m_start = m_entries.count(); 0383 // m_hasMoreResults = m_start <= m_total; 0384 m_hasMoreResults = false; // for now, no continued searches 0385 endJob(job); 0386 } 0387 0388 QString OpenLibraryFetcher::getAuthorKeys(const QString& term_) { 0389 QUrl u(QString::fromLatin1(OPENLIBRARY_QUERY_URL)); 0390 QUrlQuery q; 0391 q.addQueryItem(QStringLiteral("type"), QStringLiteral("/type/author")); 0392 q.addQueryItem(QStringLiteral("name"), term_); 0393 u.setQuery(q); 0394 0395 QString output = FileHandler::readTextFile(u, true /*quiet*/, true /*utf8*/); 0396 QJsonDocument doc = QJsonDocument::fromJson(output.toUtf8()); 0397 QJsonArray array = doc.array(); 0398 // myDebug() << "found" << array.count() << "authors"; 0399 // right now, only use the first 0400 return array.isEmpty() ? QString() : mapValue(array.at(0).toObject().toVariantMap(), "key"); 0401 } 0402 0403 Tellico::Fetch::ConfigWidget* OpenLibraryFetcher::configWidget(QWidget* parent_) const { 0404 return new OpenLibraryFetcher::ConfigWidget(parent_, this); 0405 } 0406 0407 QString OpenLibraryFetcher::defaultName() { 0408 return QStringLiteral("Open Library"); // no translation 0409 } 0410 0411 QString OpenLibraryFetcher::defaultIcon() { 0412 return favIcon("http://openlibrary.org"); 0413 } 0414 0415 Tellico::StringHash OpenLibraryFetcher::allOptionalFields() { 0416 StringHash hash; 0417 hash[QStringLiteral("openlibrary")] = i18n("OpenLibrary Link"); 0418 return hash; 0419 } 0420 0421 OpenLibraryFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const OpenLibraryFetcher* fetcher_) 0422 : Fetch::ConfigWidget(parent_) { 0423 QVBoxLayout* l = new QVBoxLayout(optionsWidget()); 0424 l->addWidget(new QLabel(i18n("This source has no options."), optionsWidget())); 0425 l->addStretch(); 0426 0427 // now add additional fields widget 0428 addFieldsWidget(OpenLibraryFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); 0429 } 0430 0431 void OpenLibraryFetcher::ConfigWidget::saveConfigHook(KConfigGroup&) { 0432 } 0433 0434 QString OpenLibraryFetcher::ConfigWidget::preferredName() const { 0435 return OpenLibraryFetcher::defaultName(); 0436 }