src/fetch/kinopoiskfetcher.cpp

0001 /***************************************************************************
0002     Copyright (C) 2017-2021 Robby Stephenson <robby@periapsis.org>
0003  ***************************************************************************/
0004
0005 /***************************************************************************
0006  *                                                                         *
0007  *   This program is free software; you can redistribute it and/or         *
0008  *   modify it under the terms of the GNU General Public License as        *
0009  *   published by the Free Software Foundation; either version 2 of        *
0010  *   the License or (at your option) version 3 or any later version        *
0011  *   accepted by the membership of KDE e.V. (or its successor approved     *
0012  *   by the membership of KDE e.V.), which shall act as a proxy            *
0013  *   defined in Section 14 of version 3 of the license.                    *
0014  *                                                                         *
0015  *   This program is distributed in the hope that it will be useful,       *
0016  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
0017  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
0018  *   GNU General Public License for more details.                          *
0019  *                                                                         *
0020  *   You should have received a copy of the GNU General Public License     *
0021  *   along with this program.  If not, see <http://www.gnu.org/licenses/>. *
0022  *                                                                         *
0023  ***************************************************************************/
0024
0025 #include "kinopoiskfetcher.h"
0026 #include "../utils/guiproxy.h"
0027 #include "../collections/videocollection.h"
0028 #include "../entry.h"
0029 #include "../fieldformat.h"
0030 #include "../images/imagefactory.h"
0031 #include "../utils/string_utils.h"
0032 #include "../utils/mapvalue.h"
0033 #include "../tellico_debug.h"
0034
0035 #include <KLocalizedString>
0036 #include <KIO/Job>
0037 #include <KJobUiDelegate>
0038 #include <KJobWidgets/KJobWidgets>
0039
0040 #include <QRegularExpression>
0041 #include <QRegularExpressionMatch>
0042 #include <QLabel>
0043 #include <QFile>
0044 #include <QTextStream>
0045 #include <QGridLayout>
0046 #include <QSpinBox>
0047 #include <QUrlQuery>
0048 #include <QJsonDocument>
0049 #include <QJsonObject>
0050 #include <QJsonArray>
0051 #include <QJsonParseError>
0052
0053 namespace {
0054   static const char* KINOPOISK_SEARCH_URL = "https://www.kinopoisk.ru/index.php";
0055   static const char* KINOPOISK_IMAGE_SIZE = "300x450";
0056   static const char* KINOPOISK_API_FILM_URL  = "https://kinopoiskapiunofficial.tech/api/v2.2/films/";
0057   static const char* KINOPOISK_API_STAFF_URL = "https://kinopoiskapiunofficial.tech/api/v1/staff";
0058   static const char* KINOPOISK_API_KEY = "9ca8395a794fb28b82e01120a6968bbf03651271fd9ce5d5371a096d4f7dc7a3caa8361ba8914425a1c5c0f4f5d88dbd3d0fccaa781ca18cd4b2b587ebdeaac89cfa771622162a12";
0059   static const int KINOPOISK_DEFAULT_CAST_SIZE = 10;
0060 }
0061
0062 using namespace Tellico;
0063 using Tellico::Fetch::KinoPoiskFetcher;
0064
0065 KinoPoiskFetcher::KinoPoiskFetcher(QObject* parent_)
0066     : Fetcher(parent_), m_started(false), m_redirected(false), m_numCast(KINOPOISK_DEFAULT_CAST_SIZE) {
0067   m_apiKey = Tellico::reverseObfuscate(KINOPOISK_API_KEY);
0068 }
0069
0070 KinoPoiskFetcher::~KinoPoiskFetcher() {
0071 }
0072
0073 QString KinoPoiskFetcher::source() const {
0074   return m_name.isEmpty() ? defaultName() : m_name;
0075 }
0076
0077 bool KinoPoiskFetcher::canFetch(int type) const {
0078   return type == Data::Collection::Video;
0079 }
0080
0081 bool KinoPoiskFetcher::canSearch(Fetch::FetchKey k) const {
0082   return k == Title;
0083 }
0084
0085 void KinoPoiskFetcher::readConfigHook(const KConfigGroup& config_) {
0086   m_numCast = config_.readEntry("Max Cast", KINOPOISK_DEFAULT_CAST_SIZE);
0087 }
0088
0089 void KinoPoiskFetcher::search() {
0090   m_started = true;
0091   m_redirected = false;
0092   m_redirectUrl.clear();
0093   m_matches.clear();
0094
0095   QUrl u(QString::fromLatin1(KINOPOISK_SEARCH_URL));
0096   QUrlQuery q;
0097
0098   switch(request().key()) {
0099     case Title:
0100       // first means return first result only
0101       //q.addQueryItem(QStringLiteral("first"), QStringLiteral("yes"));
0102       q.addQueryItem(QStringLiteral("kp_query"), request().value());
0103       break;
0104
0105     default:
0106       myWarning() << source() << "- key not recognized:" << request().key();
0107       stop();
0108       return;
0109   }
0110   u.setQuery(q);
0111 //  myDebug() << "url: " << u.url();
0112
0113   m_job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo);
0114   KJobWidgets::setWindow(m_job, GUI::Proxy::widget());
0115   connect(m_job.data(), &KJob::result, this, &KinoPoiskFetcher::slotComplete);
0116   connect(m_job.data(), &KIO::TransferJob::redirection,
0117           this, &KinoPoiskFetcher::slotRedirection);
0118 }
0119
0120 void KinoPoiskFetcher::stop() {
0121   if(!m_started) {
0122     return;
0123   }
0124
0125   if(m_job) {
0126     m_job->kill();
0127     m_job = nullptr;
0128   }
0129   m_started = false;
0130   emit signalDone(this);
0131 }
0132
0133 void KinoPoiskFetcher::slotRedirection(KIO::Job*, const QUrl& toUrl_) {
0134   if(m_redirectUrl.isEmpty()) {
0135     myDebug() << "Redirected to" << toUrl_;
0136     m_redirectUrl = toUrl_;
0137   }
0138   m_redirected = true;
0139 }
0140
0141 void KinoPoiskFetcher::slotComplete(KJob*) {
0142   if(m_job->error()) {
0143     m_job->uiDelegate()->showErrorMessage();
0144     stop();
0145     return;
0146   }
0147
0148   QByteArray data = m_job->data();
0149   if(data.isEmpty()) {
0150     myDebug() << "no data";
0151     stop();
0152     return;
0153   }
0154
0155   const QString output = Tellico::decodeHTML(data);
0156 #if 0
0157   myWarning() << "Remove debug from kinopoiskfetcher.cpp";
0158   QFile f(QStringLiteral("/tmp/test1.html"));
0159   if(f.open(QIODevice::WriteOnly)) {
0160     QTextStream t(&f);
0161     t.setCodec("UTF-8");
0162     t << output;
0163   }
0164   f.close();
0165 #endif
0166
0167   if(m_started && m_redirected) {
0168     // don't pull the data here, just add it to a single response
0169     auto res = new FetchResult(this, request().value(), QString());
0170     m_matches.insert(res->uid, m_redirectUrl);
0171     emit signalResultFound(res);
0172   }
0173
0174   // look for a paragraph, class=",", with an internal /ink to "/level/1/film..."
0175   QRegularExpression resultRx(QStringLiteral("<p class=\"name\">\\s*"
0176                                              "<a href=\"/film[^\"]+\".*? data-url=\"([^\"]*)\".*?>(.*?)</a>\\s*"
0177                                              "<span class=\"year\">(.*?)</span"));
0178
0179   QString href, title, year;
0180   QRegularExpressionMatchIterator i = resultRx.globalMatch(output);
0181   while(m_started && !m_redirected && i.hasNext()) {
0182     QRegularExpressionMatch match = i.next();
0183     href = match.captured(1);
0184     title = match.captured(2);
0185     year = match.captured(3);
0186     if(!href.isEmpty()) {
0187       QUrl url(QString::fromLatin1(KINOPOISK_SEARCH_URL));
0188       url = url.resolved(QUrl(href));
0189 //      myDebug() << url << title << year;
0190       auto res = new FetchResult(this, title, year);
0191       m_matches.insert(res->uid, url);
0192       emit signalResultFound(res);
0193     }
0194   }
0195
0196   // since the fetch is done, don't worry about holding the job pointer
0197   m_job = nullptr;
0198   stop();
0199 }
0200
0201 Tellico::Data::EntryPtr KinoPoiskFetcher::fetchEntryHook(uint uid_) {
0202   // if we already grabbed this one, then just pull it out of the dict
0203   Data::EntryPtr entry = m_entries[uid_];
0204   if(entry) {
0205     return entry;
0206   }
0207
0208   QUrl url = m_matches[uid_];
0209   if(url.isEmpty()) {
0210     myWarning() << "no url in map";
0211     return Data::EntryPtr();
0212   }
0213
0214   QPointer<KIO::StoredTransferJob> getJob = KIO::storedGet(url, KIO::NoReload, KIO::HideProgressInfo);
0215   getJob->addMetaData(QStringLiteral("referrer"), QString::fromLatin1(KINOPOISK_SEARCH_URL));
0216   KJobWidgets::setWindow(getJob, GUI::Proxy::widget());
0217   if(!getJob->exec()) {
0218     myWarning() << "unable to read" << url;
0219     return Data::EntryPtr();
0220   }
0221
0222 // the HTML response has the character encoding after the first 1024 characters and Qt doesn't seem to detect that
0223 // and potentially falls back to iso-8859-1. Enforce UTF-8
0224 //  const QByteArray data = FileHandler::readDataFile(url, true);
0225   const QByteArray data = getJob->data();
0226   const QString results = Tellico::decodeHTML(Tellico::fromHtmlData(data, "UTF-8"));
0227   if(results.isEmpty()) {
0228     myDebug() << "KinoPoiskFetcher: no text results";
0229     return Data::EntryPtr();
0230   }
0231
0232 #if 0
0233   myDebug() << url.url();
0234   myWarning() << "Remove debug from kinopoiskfetcher.cpp";
0235   QFile f(QStringLiteral("/tmp/test2.html"));
0236   if(f.open(QIODevice::WriteOnly)) {
0237     QTextStream t(&f);
0238     t.setCodec("UTF-8");
0239     t << results;
0240   }
0241   f.close();
0242 #endif
0243
0244   if(results.contains(QStringLiteral("captcha")) || results.endsWith(QLatin1String("</script>"))) {
0245 //    myDebug() << "KinoPoiskFetcher: captcha triggered";
0246     static const QRegularExpression re(QLatin1String("/(\\d+)"));
0247     QRegularExpressionMatch match = re.match(url.url());
0248     if(match.hasMatch()) {
0249       entry = requestEntry(match.captured(1));
0250     }
0251   } else {
0252     entry = parseEntry(results);
0253     if(!entry) {
0254       // might want to check LD+JSON format
0255 //      myDebug() << "...trying Linked Data";
0256       entry = parseEntryLinkedData(results);
0257     }
0258   }
0259   if(!entry) {
0260 //    myDebug() << "No discernible entry data";
0261     return Data::EntryPtr();
0262   }
0263
0264   QString cover = entry->field(QStringLiteral("cover"));
0265   if(!cover.isEmpty()) {
0266     const QString id = ImageFactory::addImage(QUrl::fromUserInput(cover), true /* quiet */,
0267                                               QUrl(QString::fromLatin1(KINOPOISK_SEARCH_URL)) /* referer */);
0268     if(id.isEmpty()) {
0269       message(i18n("The cover image could not be loaded."), MessageHandler::Warning);
0270     }
0271     // empty image ID is ok
0272     entry->setField(QStringLiteral("cover"), id);
0273   }
0274
0275   if(optionalFields().contains(QStringLiteral("kinopoisk"))) {
0276     Data::FieldPtr field(new Data::Field(QStringLiteral("kinopoisk"), i18n("KinoPoisk Link"), Data::Field::URL));
0277     field->setCategory(i18n("General"));
0278     entry->collection()->addField(field);
0279     entry->setField(QStringLiteral("kinopoisk"), url.url());
0280   }
0281
0282   m_entries.insert(uid_, entry); // keep for later
0283   return entry;
0284 }
0285
0286 Tellico::Data::EntryPtr KinoPoiskFetcher::requestEntry(const QString& filmId_) {
0287   QUrl url(QLatin1String(KINOPOISK_API_FILM_URL) + filmId_);
0288
0289   QPointer<KIO::StoredTransferJob> getJob = KIO::storedGet(url, KIO::NoReload, KIO::HideProgressInfo);
0290   getJob->addMetaData(QStringLiteral("content-type"), QStringLiteral("application/json"));
0291   getJob->addMetaData(QStringLiteral("customHTTPHeader"), QStringLiteral("X-API-KEY: ") + m_apiKey);
0292   KJobWidgets::setWindow(getJob, GUI::Proxy::widget());
0293   if(!getJob->exec()) {
0294     myWarning() << "unable to read" << url;
0295     return Data::EntryPtr();
0296   }
0297
0298   QByteArray data = getJob->data();
0299 #if 0
0300   myDebug() << url;
0301   myWarning() << "Remove json debug from kinopoiskfetcher.cpp";
0302   QFile file(QString::fromLatin1("/tmp/test-kinopoisk.json"));
0303   if(file.open(QIODevice::WriteOnly)) {
0304     QTextStream t(&file);
0305     t.setCodec("UTF-8");
0306     t << data;
0307   }
0308   file.close();
0309 #endif
0310
0311   Data::CollPtr coll(new Data::VideoCollection(true));
0312   Data::EntryPtr entry(new Data::Entry(coll));
0313   coll->addEntries(entry);
0314
0315   QJsonDocument doc = QJsonDocument::fromJson(data);
0316   const QVariantMap resultMap = doc.object().toVariantMap();
0317
0318   entry->setField(QStringLiteral("title"), mapValue(resultMap, "nameRu"));
0319   entry->setField(QStringLiteral("year"), mapValue(resultMap, "year"));
0320   entry->setField(QStringLiteral("nationality"), mapValue(resultMap, "countries", "country"));
0321   entry->setField(QStringLiteral("genre"), mapValue(resultMap, "genres", "genre"));
0322   entry->setField(QStringLiteral("running-time"), mapValue(resultMap, "filmLength"));
0323   entry->setField(QStringLiteral("plot"), mapValue(resultMap, "description"));
0324   entry->setField(QStringLiteral("cover"), mapValue(resultMap, "posterUrl"));
0325
0326   const QString cert(QStringLiteral("certification"));
0327   auto certField = coll->fieldByName(cert);
0328   if(certField) {
0329     entry->setField(cert, mpaaRating(mapValue(resultMap, "ratingMpaa"), certField->allowed()));
0330   }
0331
0332   const QString imdb(QStringLiteral("imdb"));
0333   const QString imdbId = mapValue(resultMap, "imdbId");
0334   if(optionalFields().contains(imdb) && !imdbId.isEmpty()) {
0335     coll->addField(Data::Field::createDefaultField(Data::Field::ImdbField));
0336     entry->setField(imdb, QStringLiteral("https://www.imdb.com/title/") + imdbId);
0337   }
0338   const QString origTitle(QStringLiteral("origtitle"));
0339   if(optionalFields().contains(origTitle)) {
0340     if(!coll->hasField(origTitle)) {
0341       Data::FieldPtr f(new Data::Field(origTitle, i18n("Original Title")));
0342       f->setFormatType(FieldFormat::FormatTitle);
0343       coll->addField(f);
0344     }
0345     entry->setField(origTitle, mapValue(resultMap, "nameOriginal"));
0346   }
0347
0348   url = QUrl(QLatin1String(KINOPOISK_API_STAFF_URL));
0349   QUrlQuery q;
0350   q.addQueryItem(QStringLiteral("filmId"), filmId_);
0351   url.setQuery(q);
0352
0353   getJob = KIO::storedGet(url, KIO::NoReload, KIO::HideProgressInfo);
0354   getJob->addMetaData(QStringLiteral("content-type"), QStringLiteral("application/json"));
0355   getJob->addMetaData(QStringLiteral("customHTTPHeader"), QStringLiteral("X-API-KEY: ") + m_apiKey);
0356   KJobWidgets::setWindow(getJob, GUI::Proxy::widget());
0357   if(!getJob->exec()) {
0358     myWarning() << "unable to read" << url;
0359     return Data::EntryPtr();
0360   }
0361
0362   data = getJob->data();
0363 #if 0
0364   myDebug() << url;
0365   myWarning() << "Remove json2 debug from kinopoiskfetcher.cpp";
0366   QFile file2(QString::fromLatin1("/tmp/test-kinopoisk-staff.json"));
0367   if(file2.open(QIODevice::WriteOnly)) {
0368     QTextStream t(&file2);
0369     t.setCodec("UTF-8");
0370     t << data;
0371   }
0372   file2.close();
0373 #endif
0374
0375   QStringList directors, writers, actors, producers, composers;
0376
0377   const auto staffArray = QJsonDocument::fromJson(data).array();
0378   const int sz = staffArray.size();
0379   for(int i = 0; i < sz; ++i) {
0380     const auto obj = staffArray.at(i).toObject().toVariantMap();
0381     const QString key = mapValue(obj, "professionKey");
0382     QString name = mapValue(obj, "nameRu");
0383     if(name.isEmpty()) name = mapValue(obj, "nameEn");
0384     if(name.isEmpty()) continue;
0385     if(key == QLatin1String("DIRECTOR")) {
0386       directors += name;
0387     } else if(key == QLatin1String("ACTOR")) {
0388       if(actors.size() < m_numCast) {
0389         actors += (name + FieldFormat::columnDelimiterString() + mapValue(obj, "description"));
0390       }
0391     } else if(key == QLatin1String("WRITER")) {
0392       writers += name;
0393     } else if(key == QLatin1String("PRODUCER")) {
0394       producers += name;
0395     } else if(key == QLatin1String("COMPOSER")) {
0396       composers += name;
0397     } else {
0398 //      myDebug() << "...skipping" << key;
0399     }
0400   }
0401
0402   entry->setField(QStringLiteral("director"), directors.join(Tellico::FieldFormat::delimiterString()));
0403   entry->setField(QStringLiteral("writer"), writers.join(Tellico::FieldFormat::delimiterString()));
0404   entry->setField(QStringLiteral("producer"), producers.join(Tellico::FieldFormat::delimiterString()));
0405   entry->setField(QStringLiteral("composer"), composers.join(Tellico::FieldFormat::delimiterString()));
0406   entry->setField(QStringLiteral("cast"), actors.join(Tellico::FieldFormat::rowDelimiterString()));
0407
0408   return entry;
0409 }
0410
0411 Tellico::Data::EntryPtr KinoPoiskFetcher::parseEntry(const QString& str_) {
0412   static const QRegularExpression jsonRx(QStringLiteral("<script.*?type=\"application/json\".*?>(.+?)</script>"));
0413   QRegularExpressionMatch jsonMatch = jsonRx.match(str_);
0414   if(!jsonMatch.hasMatch()) {
0415     myDebug() << "No JSON data";
0416     return Data::EntryPtr();
0417   }
0418
0419   QJsonParseError parseError;
0420   QJsonDocument doc = QJsonDocument::fromJson(jsonMatch.captured(1).toUtf8(), &parseError);
0421   if(doc.isNull()) {
0422     myDebug() << "Bad json data:" << parseError.errorString();
0423     return Data::EntryPtr();
0424   }
0425
0426   const QString queryId = doc.object().value(QStringLiteral("query")).toObject()
0427                                       .value(QStringLiteral("id")).toString();
0428   // if there's no query ID, then this is not a film object to parse
0429   if(queryId.isEmpty()) {
0430 //    myDebug() << "No query ID...";
0431     return Data::EntryPtr();
0432   }
0433
0434   // otherwise, we're good to go
0435   Data::CollPtr coll(new Data::VideoCollection(true));
0436   Data::EntryPtr entry(new Data::Entry(coll));
0437   coll->addEntries(entry);
0438
0439   QJsonObject dataObject = doc.object().value(QStringLiteral("props")).toObject()
0440                                         .value(QStringLiteral("apolloState")).toObject()
0441                                         .value(QStringLiteral("data")).toObject();
0442   QJsonObject filmObject = dataObject.value(QStringLiteral("Film:") + queryId).toObject();
0443   if(filmObject.isEmpty()) {
0444     filmObject = dataObject.value(QStringLiteral("TvSeries:") + queryId).toObject();
0445   }
0446   if(filmObject.isEmpty()) {
0447     return Data::EntryPtr();
0448   }
0449   // iterate over the filmObject members to find the json keys in the dataObject
0450   QJsonObject::const_iterator i = filmObject.constBegin();
0451   for( ; i != filmObject.constEnd(); ++i) {
0452     const QString fieldName = fieldNameFromKey(i.key());
0453     if(fieldName.isEmpty()) {
0454       continue;
0455     }
0456     Data::FieldPtr field = entry->collection()->fieldByName(fieldName);
0457     Q_ASSERT(field);
0458     QString fieldValue = fieldValueFromObject(dataObject, fieldName, i.value(),
0459                                               field ? field->allowed() : QStringList());
0460     if(!fieldValue.isEmpty()) {
0461       entry->setField(fieldName, fieldValue);
0462     }
0463
0464     // also add original title
0465     if(fieldName == QLatin1String("title")) {
0466       const QString origTitle(QStringLiteral("origtitle"));
0467       if(optionalFields().contains(origTitle)) {
0468         if(!entry->collection()->hasField(origTitle)) {
0469           Data::FieldPtr f(new Data::Field(origTitle, i18n("Original Title")));
0470           f->setFormatType(FieldFormat::FormatTitle);
0471           entry->collection()->addField(f);
0472         }
0473         fieldValue = fieldValueFromObject(dataObject, origTitle, i.value(), QStringList());
0474         if(!fieldValue.isEmpty()) {
0475           entry->setField(origTitle, fieldValue);
0476         }
0477       }
0478     }
0479   }
0480
0481   return entry;
0482 }
0483
0484 Tellico::Data::EntryPtr KinoPoiskFetcher::parseEntryLinkedData(const QString& str_) {
0485   QRegularExpression jsonRx(QStringLiteral("<script.*?type=\"application/ld\\+json\".*?>(.+?)</script>"));
0486   QRegularExpressionMatch jsonMatch = jsonRx.match(str_);
0487   if(!jsonMatch.hasMatch()) {
0488     myDebug() << "No LD+JSON data";
0489     return Data::EntryPtr();
0490   }
0491
0492   QJsonParseError parseError;
0493   QJsonDocument doc = QJsonDocument::fromJson(jsonMatch.captured(1).toUtf8(), &parseError);
0494   if(doc.isNull()) {
0495     myDebug() << "Bad json data:" << parseError.errorString();
0496     return Data::EntryPtr();
0497   }
0498
0499   // otherwise, we're good to go
0500   Data::CollPtr coll(new Data::VideoCollection(true));
0501   Data::EntryPtr entry(new Data::Entry(coll));
0502   coll->addEntries(entry);
0503
0504   QVariantMap objectMap = doc.object().toVariantMap();
0505   entry->setField(QStringLiteral("title"), mapValue(objectMap, "name"));
0506   entry->setField(QStringLiteral("year"), mapValue(objectMap, "datePublished").left(4));
0507   entry->setField(QStringLiteral("nationality"), mapValue(objectMap, "countryOfOrigin"));
0508   entry->setField(QStringLiteral("cast"), mapValue(objectMap, "actor", "name"));
0509   entry->setField(QStringLiteral("director"), mapValue(objectMap, "director", "name"));
0510   entry->setField(QStringLiteral("producer"), mapValue(objectMap, "producer", "name"));
0511   entry->setField(QStringLiteral("genre"), mapValue(objectMap, "genre"));
0512   entry->setField(QStringLiteral("plot"), mapValue(objectMap, "description"));
0513   QString cover = mapValue(objectMap, "image");
0514   if(cover.startsWith(QLatin1Char('/'))) {
0515     cover.prepend(QLatin1String("https:"));
0516   }
0517   entry->setField(QStringLiteral("cover"), cover);
0518
0519   return entry;
0520 }
0521
0522 // static
0523 QString KinoPoiskFetcher::fieldNameFromKey(const QString& key_) {
0524   static QHash<QString, QString> fieldHash;
0525   if(fieldHash.isEmpty()) {
0526     fieldHash.insert(QStringLiteral("title"), QStringLiteral("title"));
0527     fieldHash.insert(QStringLiteral("productionYear"), QStringLiteral("year"));
0528     fieldHash.insert(QStringLiteral("duration"), QStringLiteral("running-time"));
0529     fieldHash.insert(QStringLiteral("countries"), QStringLiteral("nationality"));
0530     fieldHash.insert(QStringLiteral("genres"), QStringLiteral("genre"));
0531     fieldHash.insert(QStringLiteral("restriction"), QStringLiteral("certification"));
0532     fieldHash.insert(QStringLiteral("synopsis"), QStringLiteral("plot"));
0533     fieldHash.insert(QStringLiteral("poster"), QStringLiteral("cover"));
0534   }
0535   if(fieldHash.contains(key_)) {
0536     return fieldHash.value(key_);
0537   }
0538
0539   // otherwise some wonky key names
0540   if(key_.contains(QLatin1String("DIRECTOR"), Qt::CaseInsensitive)) {
0541     return QStringLiteral("director");
0542   }
0543   if(key_.contains(QLatin1String("WRITER"), Qt::CaseInsensitive)) {
0544     return QStringLiteral("writer");
0545   }
0546   if(key_.contains(QLatin1String("PRODUCER"), Qt::CaseInsensitive)) {
0547     return QStringLiteral("producer");
0548   }
0549   if(key_.contains(QLatin1String("COMPOSER"), Qt::CaseInsensitive)) {
0550     return QStringLiteral("composer");
0551   }
0552   if(key_.contains(QLatin1String("ACTOR"), Qt::CaseInsensitive)) {
0553     return QStringLiteral("cast");
0554   }
0555   return QString();
0556 }
0557
0558 QString KinoPoiskFetcher::fieldValueFromObject(const QJsonObject& obj_, const QString& field_,
0559                                                const QJsonValue& value_, const QStringList& allowed_) {
0560   // if it's an array, loop over and recurse
0561   if(value_.isArray()) {
0562     QJsonArray arr = value_.toArray();
0563     QStringList fieldValues;
0564     for(QJsonArray::const_iterator i = arr.constBegin(); i != arr.constEnd(); ++i) {
0565       const QString value = fieldValueFromObject(obj_, field_, *i, allowed_);
0566       if(!value.isEmpty()) {
0567         fieldValues << value;
0568       }
0569     }
0570     return fieldValues.isEmpty() ? QString() : fieldValues.join(field_ == QLatin1String("cast") ?
0571                                                                 Tellico::FieldFormat::rowDelimiterString() :
0572                                                                 Tellico::FieldFormat::delimiterString());
0573   }
0574
0575   if(field_ == QLatin1String("year") ||
0576      field_ == QLatin1String("running-time")) {
0577     const int n = value_.toInt();
0578     return n > 0 ? QString::number(n) : QString();
0579   }
0580   if(field_ == QLatin1String("plot")) {
0581     return value_.toString();
0582   }
0583
0584   QJsonObject valueObj = value_.toObject();
0585   // if there's a reference to another object, need to pull it from the higher level data object
0586   if(valueObj.contains(QStringLiteral("__ref"))) {
0587     valueObj = obj_.value(valueObj.value(QStringLiteral("__ref")).toString()).toObject();
0588   }
0589
0590   // if it has a 'person' field, gotta grab the person name
0591   if(valueObj.contains(QLatin1String("person"))) {
0592     return fieldValueFromObject(obj_, field_, valueObj.value(QLatin1String("person")), allowed_);
0593   }
0594
0595   if(field_ == QLatin1String("title")) {
0596     const QString title = valueObj.value(QStringLiteral("russian")).toString();
0597     // return original if russian is not available
0598     return title.isEmpty() ? valueObj.value(QStringLiteral("original")).toString() : title;
0599   } else if(field_ == QLatin1String("origtitle")) {
0600     return valueObj.value(QStringLiteral("original")).toString();
0601   } else if(field_ == QLatin1String("cover")) {
0602     QString url = valueObj.value(QStringLiteral("avatarsUrl")).toString();
0603     if(url.startsWith(QLatin1Char('/'))) {
0604       url.prepend(QLatin1String("https:"));
0605     }
0606     // also add size
0607     url.append(QLatin1Char('/') + QLatin1String(KINOPOISK_IMAGE_SIZE));
0608     return url;
0609   } else if(field_ == QLatin1String("certification")) {
0610     return mpaaRating(valueObj.value(QLatin1String("mpaa")).toString(), allowed_);
0611   // with an 'originalName' or 'name' field return that
0612   // and check this before comparing against field names for people, like 'director'
0613   } else if(valueObj.contains(QLatin1String("originalName")) || valueObj.contains(QLatin1String("name"))) {
0614     const QString name = valueObj.value(QStringLiteral("name")).toString();
0615     // prefer name to originalName
0616     return name.isEmpty() ? valueObj.value(QStringLiteral("originalName")).toString() : name;
0617   } else if(valueObj.contains(QLatin1String("items"))) {
0618     // some additional nesting apparently
0619     // key in film object points to director object, whose 'items' is an array where each 'is' points to
0620     // a director object which has a person.id pointing to a person object with a 'name' and 'original' value
0621     // valueObj is the director so we want the items array
0622     QJsonValue itemsValue = valueObj.value(QLatin1String("items"));
0623     if(!itemsValue.isArray()) {
0624       myDebug() << "items value is not an array";
0625       return QString();
0626     }
0627     return fieldValueFromObject(obj_, field_, itemsValue, allowed_);
0628   }
0629
0630   return QString();
0631 }
0632
0633 QString KinoPoiskFetcher::mpaaRating(const QString& value_, const QStringList& allowed_) {
0634   // default collection has 5 MPAA values
0635   if(allowed_.size() != 5) return value_;
0636   if(value_ == QLatin1String("g")) {
0637     return allowed_.at(0);
0638   } else if(value_ == QLatin1String("pg")) {
0639     return allowed_.at(1);
0640   } else if(value_ == QLatin1String("pg13")) {
0641     return allowed_.at(2);
0642   } else if(value_ == QLatin1String("r")) {
0643     return allowed_.at(3);
0644   } else {
0645     return allowed_.at(4);
0646   }
0647 }
0648
0649 Tellico::Fetch::FetchRequest KinoPoiskFetcher::updateRequest(Data::EntryPtr entry_) {
0650   QString t = entry_->field(QStringLiteral("title"));
0651   if(!t.isEmpty()) {
0652     return FetchRequest(Fetch::Title, t);
0653   }
0654   return FetchRequest();
0655 }
0656
0657 Tellico::Fetch::ConfigWidget* KinoPoiskFetcher::configWidget(QWidget* parent_) const {
0658   return new KinoPoiskFetcher::ConfigWidget(parent_);
0659 }
0660
0661 QString KinoPoiskFetcher::defaultName() {
0662   return QStringLiteral("КиноПоиск (KinoPoisk.ru)");
0663 }
0664
0665 QString KinoPoiskFetcher::defaultIcon() {
0666   return favIcon("http://www.kinopoisk.ru");
0667 }
0668
0669 Tellico::StringHash KinoPoiskFetcher::allOptionalFields() {
0670   StringHash hash;
0671   hash[QStringLiteral("kinopoisk")] = i18n("KinoPoisk Link");
0672   hash[QStringLiteral("imdb")] = i18n("IMDb Link");
0673   hash[QStringLiteral("origtitle")] = i18n("Original Title");
0674   return hash;
0675 }
0676
0677 KinoPoiskFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const KinoPoiskFetcher* fetcher_)
0678     : Fetch::ConfigWidget(parent_) {
0679   QGridLayout* l = new QGridLayout(optionsWidget());
0680   l->setSpacing(4);
0681   l->setColumnStretch(1, 10);
0682
0683   int row = -1;
0684
0685   QLabel* label = new QLabel(i18n("&Maximum cast: "), optionsWidget());
0686   l->addWidget(label, ++row, 0);
0687   m_numCast = new QSpinBox(optionsWidget());
0688   m_numCast->setMaximum(99);
0689   m_numCast->setMinimum(0);
0690   m_numCast->setValue(KINOPOISK_DEFAULT_CAST_SIZE);
0691 #if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0))
0692   void (QSpinBox::* textChanged)(const QString&) = &QSpinBox::valueChanged;
0693 #else
0694   void (QSpinBox::* textChanged)(const QString&) = &QSpinBox::textChanged;
0695 #endif
0696   connect(m_numCast, textChanged, this, &ConfigWidget::slotSetModified);
0697   l->addWidget(m_numCast, row, 1);
0698   QString w = i18n("The list of cast members may include many people. Set the maximum number returned from the search.");
0699   label->setWhatsThis(w);
0700   m_numCast->setWhatsThis(w);
0701   label->setBuddy(m_numCast);
0702
0703   l->setRowStretch(++row, 10);
0704
0705   addFieldsWidget(KinoPoiskFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList());
0706   if(fetcher_) {
0707     m_numCast->setValue(fetcher_->m_numCast);
0708   }
0709 }
0710
0711 QString KinoPoiskFetcher::ConfigWidget::preferredName() const {
0712   return KinoPoiskFetcher::defaultName();
0713 }
0714
0715 void KinoPoiskFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) {
0716   config_.writeEntry("Max Cast", m_numCast->value());
0717 }