File indexing completed on 2024-05-12 05:09:34
0001 /*************************************************************************** 0002 Copyright (C) 2011 Robby Stephenson <robby@periapsis.org> 0003 ***************************************************************************/ 0004 0005 /*************************************************************************** 0006 * * 0007 * This program is free software; you can redistribute it and/or * 0008 * modify it under the terms of the GNU General Public License as * 0009 * published by the Free Software Foundation; either version 2 of * 0010 * the License or (at your option) version 3 or any later version * 0011 * accepted by the membership of KDE e.V. (or its successor approved * 0012 * by the membership of KDE e.V.), which shall act as a proxy * 0013 * defined in Section 14 of version 3 of the license. * 0014 * * 0015 * This program is distributed in the hope that it will be useful, * 0016 * but WITHOUT ANY WARRANTY; without even the implied warranty of * 0017 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 0018 * GNU General Public License for more details. * 0019 * * 0020 * You should have received a copy of the GNU General Public License * 0021 * along with this program. If not, see <http://www.gnu.org/licenses/>. * 0022 * * 0023 ***************************************************************************/ 0024 0025 #include "googlebookfetcher.h" 0026 #include "../collections/bookcollection.h" 0027 #include "../entry.h" 0028 #include "../images/imagefactory.h" 0029 #include "../utils/isbnvalidator.h" 0030 #include "../utils/guiproxy.h" 0031 #include "../utils/string_utils.h" 0032 #include "../utils/mapvalue.h" 0033 #include "../core/filehandler.h" 0034 #include "../tellico_debug.h" 0035 0036 #include <KLocalizedString> 0037 #include <KIO/Job> 0038 #include <KIO/JobUiDelegate> 0039 #include <KJobWidgets/KJobWidgets> 0040 #include <KConfigGroup> 0041 0042 #include <QLineEdit> 0043 #include <QLabel> 0044 #include <QFile> 0045 #include <QTextStream> 0046 #include <QGridLayout> 0047 #include <QTextCodec> 0048 #include <QJsonDocument> 0049 #include <QJsonObject> 0050 #include <QUrlQuery> 0051 0052 namespace { 0053 static const int GOOGLEBOOK_MAX_RETURNS = 20; 0054 static const char* GOOGLEBOOK_API_URL = "https://www.googleapis.com/books/v1/volumes"; 0055 static const char* GOOGLEBOOK_API_KEY = "b0e1702513773b743b53b1c5566ea0f93e7c3b720351bad197f801491951d29afca54b32712ba6dc4e1e4c7a5b0ad99d9dedfdbab4f10642b7e821403340fc98692bcdb4dc8fd0b14339236ae4a5"; 0056 } 0057 0058 using namespace Tellico; 0059 using Tellico::Fetch::GoogleBookFetcher; 0060 0061 GoogleBookFetcher::GoogleBookFetcher(QObject* parent_) 0062 : Fetcher(parent_) 0063 , m_started(false) 0064 , m_start(0) 0065 , m_total(0) { 0066 m_apiKey = Tellico::reverseObfuscate(GOOGLEBOOK_API_KEY); 0067 } 0068 0069 GoogleBookFetcher::~GoogleBookFetcher() { 0070 } 0071 0072 QString GoogleBookFetcher::source() const { 0073 return m_name.isEmpty() ? defaultName() : m_name; 0074 } 0075 0076 bool GoogleBookFetcher::canSearch(Fetch::FetchKey k) const { 0077 return k == Title || k == Person || k == ISBN || k == Keyword; 0078 } 0079 0080 bool GoogleBookFetcher::canFetch(int type) const { 0081 return type == Data::Collection::Book || type == Data::Collection::Bibtex; 0082 } 0083 0084 void GoogleBookFetcher::readConfigHook(const KConfigGroup& config_) { 0085 // allow an empty key if the config key does exist 0086 m_apiKey = config_.readEntry("API Key", GOOGLEBOOK_API_KEY); 0087 } 0088 0089 void GoogleBookFetcher::search() { 0090 m_start = 0; 0091 m_total = -1; 0092 continueSearch(); 0093 } 0094 0095 void GoogleBookFetcher::continueSearch() { 0096 m_started = true; 0097 // we only split ISBN and LCCN values 0098 QStringList searchTerms; 0099 if(request().key() == ISBN) { 0100 searchTerms = FieldFormat::splitValue(request().value()); 0101 } else { 0102 searchTerms += request().value(); 0103 } 0104 foreach(const QString& searchTerm, searchTerms) { 0105 doSearch(searchTerm); 0106 } 0107 if(m_jobs.isEmpty()) { 0108 stop(); 0109 } 0110 } 0111 0112 void GoogleBookFetcher::doSearch(const QString& term_) { 0113 QUrl u(QString::fromLatin1(GOOGLEBOOK_API_URL)); 0114 QUrlQuery q; 0115 q.addQueryItem(QStringLiteral("maxResults"), QString::number(GOOGLEBOOK_MAX_RETURNS)); 0116 q.addQueryItem(QStringLiteral("startIndex"), QString::number(m_start)); 0117 q.addQueryItem(QStringLiteral("printType"), QStringLiteral("books")); 0118 // we don't require a key, cause it might work without it 0119 if(!m_apiKey.isEmpty()) { 0120 q.addQueryItem(QStringLiteral("key"), m_apiKey); 0121 } 0122 0123 switch(request().key()) { 0124 case Title: 0125 q.addQueryItem(QStringLiteral("q"), QLatin1String("intitle:") + term_); 0126 break; 0127 0128 case Person: 0129 // for people, go ahead and enclose in quotes 0130 // risk of missing middle initials, etc. balanced by google splitting front and last name 0131 q.addQueryItem(QStringLiteral("q"), QLatin1String("inauthor:\"") + term_ + QLatin1Char('"')); 0132 break; 0133 0134 case ISBN: 0135 { 0136 const QString isbn = ISBNValidator::cleanValue(term_); 0137 q.addQueryItem(QStringLiteral("q"), QLatin1String("isbn:") + isbn); 0138 } 0139 break; 0140 0141 case Keyword: 0142 q.addQueryItem(QStringLiteral("q"), term_); 0143 break; 0144 0145 default: 0146 myWarning() << source() << "- key not recognized:" << request().key(); 0147 return; 0148 } 0149 u.setQuery(q); 0150 // myDebug() << "url:" << u; 0151 0152 QPointer<KIO::StoredTransferJob> job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); 0153 KJobWidgets::setWindow(job, GUI::Proxy::widget()); 0154 connect(job.data(), &KJob::result, this, &GoogleBookFetcher::slotComplete); 0155 m_jobs << job; 0156 } 0157 0158 void GoogleBookFetcher::endJob(KIO::StoredTransferJob* job_) { 0159 m_jobs.removeOne(job_); 0160 if(m_jobs.isEmpty()) { 0161 stop(); 0162 } 0163 } 0164 0165 void GoogleBookFetcher::stop() { 0166 if(!m_started) { 0167 return; 0168 } 0169 foreach(QPointer<KIO::StoredTransferJob> job, m_jobs) { 0170 if(job) { 0171 job->kill(); 0172 } 0173 } 0174 m_jobs.clear(); 0175 m_started = false; 0176 emit signalDone(this); 0177 } 0178 0179 Tellico::Data::EntryPtr GoogleBookFetcher::fetchEntryHook(uint uid_) { 0180 Data::EntryPtr entry = m_entries.value(uid_); 0181 if(!entry) { 0182 myWarning() << "no entry in dict"; 0183 return Data::EntryPtr(); 0184 } 0185 0186 QString gbs = entry->field(QStringLiteral("gbs-link")); 0187 if(!gbs.isEmpty()) { 0188 // quiet 0189 QByteArray data = FileHandler::readDataFile(QUrl::fromUserInput(gbs), true); 0190 QJsonDocument doc = QJsonDocument::fromJson(data); 0191 populateEntry(entry, doc.object().toVariantMap()); 0192 } 0193 0194 const QString image_id = entry->field(QStringLiteral("cover")); 0195 // if it's still a url, we need to load it 0196 if(image_id.startsWith(QLatin1String("http"))) { 0197 const QString id = ImageFactory::addImage(QUrl::fromUserInput(image_id), true); 0198 if(id.isEmpty()) { 0199 message(i18n("The cover image could not be loaded."), MessageHandler::Warning); 0200 entry->setField(QStringLiteral("cover"), QString()); 0201 } else { 0202 entry->setField(QStringLiteral("cover"), id); 0203 } 0204 } 0205 0206 // don't want to include gbs json link 0207 entry->setField(QStringLiteral("gbs-link"), QString()); 0208 0209 return entry; 0210 } 0211 0212 Tellico::Fetch::FetchRequest GoogleBookFetcher::updateRequest(Data::EntryPtr entry_) { 0213 const QString isbn = entry_->field(QStringLiteral("isbn")); 0214 if(!isbn.isEmpty()) { 0215 return FetchRequest(ISBN, isbn); 0216 } 0217 QString title = entry_->field(QStringLiteral("title")); 0218 if(!title.isEmpty()) { 0219 return FetchRequest(Title, title); 0220 } 0221 return FetchRequest(); 0222 } 0223 0224 void GoogleBookFetcher::slotComplete(KJob* job_) { 0225 KIO::StoredTransferJob* job = static_cast<KIO::StoredTransferJob*>(job_); 0226 // myDebug(); 0227 0228 if(job->error()) { 0229 job->uiDelegate()->showErrorMessage(); 0230 endJob(job); 0231 return; 0232 } 0233 0234 QByteArray data = job->data(); 0235 if(data.isEmpty()) { 0236 myDebug() << "no data"; 0237 endJob(job); 0238 return; 0239 } 0240 0241 #if 0 0242 myWarning() << "Remove debug from googlebookfetcher.cpp"; 0243 QFile f(QString::fromLatin1("/tmp/test.json")); 0244 if(f.open(QIODevice::WriteOnly)) { 0245 QTextStream t(&f); 0246 t.setCodec("UTF-8"); 0247 t << data; 0248 } 0249 f.close(); 0250 #endif 0251 0252 Data::CollPtr coll(new Data::BookCollection(true)); 0253 // always add the gbs-link for fetchEntryHook 0254 Data::FieldPtr field(new Data::Field(QStringLiteral("gbs-link"), QStringLiteral("GBS Link"), Data::Field::URL)); 0255 field->setCategory(i18n("General")); 0256 coll->addField(field); 0257 0258 if(!coll->hasField(QStringLiteral("googlebook")) && optionalFields().contains(QStringLiteral("googlebook"))) { 0259 Data::FieldPtr field(new Data::Field(QStringLiteral("googlebook"), i18n("Google Book Link"), Data::Field::URL)); 0260 field->setCategory(i18n("General")); 0261 coll->addField(field); 0262 } 0263 0264 QJsonDocument doc = QJsonDocument::fromJson(data); 0265 QVariantMap result = doc.object().toVariantMap(); 0266 m_total = result.value(QStringLiteral("totalItems")).toInt(); 0267 // myDebug() << "total:" << m_total; 0268 0269 QVariantList resultList = result.value(QStringLiteral("items")).toList(); 0270 if(resultList.isEmpty()) { 0271 myDebug() << "no results"; 0272 endJob(job); 0273 return; 0274 } 0275 0276 foreach(const QVariant& result, resultList) { 0277 // myDebug() << "found result:" << result; 0278 0279 Data::EntryPtr entry(new Data::Entry(coll)); 0280 populateEntry(entry, result.toMap()); 0281 0282 FetchResult* r = new FetchResult(this, entry); 0283 m_entries.insert(r->uid, entry); 0284 emit signalResultFound(r); 0285 } 0286 0287 m_start = m_entries.count(); 0288 m_hasMoreResults = request().key() != ISBN && m_start <= m_total; 0289 endJob(job); 0290 } 0291 0292 void GoogleBookFetcher::populateEntry(Data::EntryPtr entry, const QVariantMap& resultMap) { 0293 if(entry->collection()->hasField(QStringLiteral("gbs-link"))) { 0294 entry->setField(QStringLiteral("gbs-link"), mapValue(resultMap, "selfLink")); 0295 } 0296 0297 const QVariantMap volumeMap = resultMap.value(QStringLiteral("volumeInfo")).toMap(); 0298 entry->setField(QStringLiteral("title"), mapValue(volumeMap, "title")); 0299 entry->setField(QStringLiteral("subtitle"), mapValue(volumeMap, "subtitle")); 0300 entry->setField(QStringLiteral("pub_year"), mapValue(volumeMap, "publishedDate").left(4)); 0301 entry->setField(QStringLiteral("author"), mapValue(volumeMap, "authors")); 0302 // workaround for bug, where publisher can be enclosed in quotes 0303 QString pub = mapValue(volumeMap, "publisher"); 0304 if(pub.startsWith(QLatin1Char('"')) && pub.endsWith(QLatin1Char('"'))) { 0305 pub.chop(1); 0306 pub = pub.remove(0, 1); 0307 } 0308 entry->setField(QStringLiteral("publisher"), pub); 0309 entry->setField(QStringLiteral("pages"), mapValue(volumeMap, "pageCount")); 0310 entry->setField(QStringLiteral("language"), mapValue(volumeMap, "language")); 0311 entry->setField(QStringLiteral("comments"), mapValue(volumeMap, "description")); 0312 0313 const QStringList catList = volumeMap.value(QStringLiteral("categories")).toStringList(); 0314 // google is going to give us a lot of categories 0315 static const QRegularExpression slash(QLatin1String("\\s*/\\s*")); 0316 QStringList cleanCategories; 0317 foreach(const QString& cat, catList) { 0318 // split them by the '/' character, too 0319 cleanCategories += cat.split(slash); 0320 } 0321 cleanCategories.sort(); 0322 cleanCategories.removeDuplicates(); 0323 // remove General since it's vague enough to not matter 0324 cleanCategories.removeOne(QStringLiteral("General")); 0325 entry->setField(QStringLiteral("keyword"), cleanCategories.join(FieldFormat::delimiterString())); 0326 0327 QString isbn; 0328 foreach(const QVariant& idVariant, volumeMap.value(QLatin1String("industryIdentifiers")).toList()) { 0329 const QVariantMap idMap = idVariant.toMap(); 0330 if(mapValue(idMap, "type") == QLatin1String("ISBN_10")) { 0331 isbn = mapValue(idMap, "identifier"); 0332 break; 0333 } else if(mapValue(idMap, "type") == QLatin1String("ISBN_13")) { 0334 isbn = mapValue(idMap, "identifier"); 0335 // allow isbn10 to override, so don't break here 0336 } 0337 } 0338 if(!isbn.isEmpty()) { 0339 ISBNValidator val(this); 0340 val.fixup(isbn); 0341 entry->setField(QStringLiteral("isbn"), isbn); 0342 } 0343 0344 const QVariantMap imageMap = volumeMap.value(QStringLiteral("imageLinks")).toMap(); 0345 if(imageMap.contains(QStringLiteral("small"))) { 0346 entry->setField(QStringLiteral("cover"), mapValue(imageMap, "small")); 0347 } else if(imageMap.contains(QStringLiteral("thumbnail"))) { 0348 entry->setField(QStringLiteral("cover"), mapValue(imageMap, "thumbnail")); 0349 } else if(imageMap.contains(QStringLiteral("smallThumbnail"))) { 0350 entry->setField(QStringLiteral("cover"), mapValue(imageMap, "smallThumbnail")); 0351 } 0352 0353 if(optionalFields().contains(QStringLiteral("googlebook"))) { 0354 entry->setField(QStringLiteral("googlebook"), mapValue(volumeMap, "infoLink")); 0355 } 0356 } 0357 0358 Tellico::Fetch::ConfigWidget* GoogleBookFetcher::configWidget(QWidget* parent_) const { 0359 return new GoogleBookFetcher::ConfigWidget(parent_, this); 0360 } 0361 0362 QString GoogleBookFetcher::defaultName() { 0363 return i18n("Google Book Search"); 0364 } 0365 0366 QString GoogleBookFetcher::defaultIcon() { 0367 return favIcon("http://books.google.com"); 0368 } 0369 0370 Tellico::StringHash GoogleBookFetcher::allOptionalFields() { 0371 StringHash hash; 0372 hash[QStringLiteral("googlebook")] = i18n("Google Book Link"); 0373 return hash; 0374 } 0375 0376 GoogleBookFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const GoogleBookFetcher* fetcher_) 0377 : Fetch::ConfigWidget(parent_) { 0378 QGridLayout* l = new QGridLayout(optionsWidget()); 0379 l->setSpacing(4); 0380 l->setColumnStretch(1, 10); 0381 0382 int row = -1; 0383 QLabel* al = new QLabel(i18n("Registration is required for accessing this data source. " 0384 "If you agree to the terms and conditions, <a href='%1'>sign " 0385 "up for an account</a>, and enter your information below.", 0386 QLatin1String("https://code.google.com/apis/console")), 0387 optionsWidget()); 0388 al->setOpenExternalLinks(true); 0389 al->setWordWrap(true); 0390 ++row; 0391 l->addWidget(al, row, 0, 1, 2); 0392 // richtext gets weird with size 0393 al->setMinimumWidth(al->sizeHint().width()); 0394 0395 QLabel* label = new QLabel(i18n("Access key: "), optionsWidget()); 0396 l->addWidget(label, ++row, 0); 0397 0398 m_apiKeyEdit = new QLineEdit(optionsWidget()); 0399 connect(m_apiKeyEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); 0400 l->addWidget(m_apiKeyEdit, row, 1); 0401 QString w = i18n("The default Tellico key may be used, but searching may fail due to reaching access limits."); 0402 label->setWhatsThis(w); 0403 m_apiKeyEdit->setWhatsThis(w); 0404 label->setBuddy(m_apiKeyEdit); 0405 0406 l->setRowStretch(++row, 10); 0407 0408 // now add additional fields widget 0409 addFieldsWidget(GoogleBookFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); 0410 0411 if(fetcher_ && fetcher_->m_apiKey != QLatin1String(GOOGLEBOOK_API_KEY)) { 0412 // only show the key if it is not the default Tellico one... 0413 // that way the user is prompted to apply for their own 0414 m_apiKeyEdit->setText(fetcher_->m_apiKey); 0415 } 0416 } 0417 0418 void GoogleBookFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { 0419 QString apiKey = m_apiKeyEdit->text().trimmed(); 0420 if(!apiKey.isEmpty()) { 0421 config_.writeEntry("API Key", apiKey); 0422 } 0423 } 0424 0425 QString GoogleBookFetcher::ConfigWidget::preferredName() const { 0426 return GoogleBookFetcher::defaultName(); 0427 }