File indexing completed on 2024-05-12 05:09:45
0001 /*************************************************************************** 0002 Copyright (C) 2013-2019 Robby Stephenson <robby@periapsis.org> 0003 ***************************************************************************/ 0004 0005 /*************************************************************************** 0006 * * 0007 * This program is free software; you can redistribute it and/or * 0008 * modify it under the terms of the GNU General Public License as * 0009 * published by the Free Software Foundation; either version 2 of * 0010 * the License or (at your option) version 3 or any later version * 0011 * accepted by the membership of KDE e.V. (or its successor approved * 0012 * by the membership of KDE e.V.), which shall act as a proxy * 0013 * defined in Section 14 of version 3 of the license. * 0014 * * 0015 * This program is distributed in the hope that it will be useful, * 0016 * but WITHOUT ANY WARRANTY; without even the implied warranty of * 0017 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 0018 * GNU General Public License for more details. * 0019 * * 0020 * You should have received a copy of the GNU General Public License * 0021 * along with this program. If not, see <http://www.gnu.org/licenses/>. * 0022 * * 0023 ***************************************************************************/ 0024 0025 #include <config.h> 0026 #include "vndbfetcher.h" 0027 #include "../collections/gamecollection.h" 0028 #include "../images/imagefactory.h" 0029 #include "../utils/mapvalue.h" 0030 #include "../entry.h" 0031 #include "../fieldformat.h" 0032 #include "../tellico_debug.h" 0033 0034 #include <KLocalizedString> 0035 0036 #include <QTcpSocket> 0037 #include <QLabel> 0038 #include <QFile> 0039 #include <QTextStream> 0040 #include <QGridLayout> 0041 #include <QTextCodec> 0042 #include <QJsonDocument> 0043 #include <QJsonObject> 0044 #include <QJsonValue> 0045 #include <QRegularExpression> 0046 #include <QTimer> 0047 0048 namespace { 0049 static const char* VNDB_HOSTNAME = "api.vndb.org"; 0050 static const int VNDB_PORT = 19534; 0051 } 0052 0053 using namespace Tellico; 0054 using Tellico::Fetch::VNDBFetcher; 0055 0056 VNDBFetcher::VNDBFetcher(QObject* parent_) 0057 : Fetcher(parent_), m_started(false), m_socket(nullptr), m_isConnected(false), m_state(PreLogin) { 0058 } 0059 0060 VNDBFetcher::~VNDBFetcher() { 0061 if(m_socket && m_isConnected) { 0062 m_socket->disconnectFromHost(); 0063 } 0064 delete m_socket; 0065 m_socket = nullptr; 0066 } 0067 0068 QString VNDBFetcher::source() const { 0069 return m_name.isEmpty() ? defaultName() : m_name; 0070 } 0071 0072 bool VNDBFetcher::canSearch(Fetch::FetchKey k) const { 0073 return k == Title; 0074 } 0075 0076 bool VNDBFetcher::canFetch(int type) const { 0077 return type == Data::Collection::Game; 0078 } 0079 0080 void VNDBFetcher::readConfigHook(const KConfigGroup&) { 0081 } 0082 0083 void VNDBFetcher::search() { 0084 m_started = true; 0085 m_data.clear(); 0086 0087 if(!m_socket) { 0088 m_socket = new QTcpSocket(this); 0089 QObject::connect(m_socket, &QTcpSocket::readyRead, this, &VNDBFetcher::slotRead); 0090 QObject::connect(m_socket, &QTcpSocket::stateChanged, this, &VNDBFetcher::slotState); 0091 #if (QT_VERSION < QT_VERSION_CHECK(5, 15, 0)) 0092 // see https://wiki.qt.io/New_Signal_Slot_Syntax#Overload for why this is necessary 0093 void (QTcpSocket::* errorSignal)(QAbstractSocket::SocketError) = &QAbstractSocket::error; 0094 QObject::connect(m_socket, errorSignal, this, &VNDBFetcher::slotError); 0095 #else 0096 QObject::connect(m_socket, &QTcpSocket::errorOccurred, this, &VNDBFetcher::slotError); 0097 #endif 0098 } 0099 if(!m_isConnected) { 0100 myLog() << "Connecting to" << VNDB_HOSTNAME << "at port" << VNDB_PORT; 0101 m_socket->connectToHost(QLatin1String(VNDB_HOSTNAME), VNDB_PORT); 0102 } 0103 0104 //the client ver only wants digits, I think? 0105 QString clientVersion(QStringLiteral(TELLICO_VERSION)); 0106 clientVersion.remove(QRegularExpression(QStringLiteral("[^0-9.]"))); 0107 0108 QByteArray login = "login {" 0109 "\"protocol\":1," 0110 "\"client\":\"Tellico\"," 0111 "\"clientver\": \""; 0112 login += clientVersion.toUtf8() + "\"}"; 0113 login.append(0x04); 0114 if(m_socket->waitForConnected()) { 0115 myLog() << "Sending login information:" << login; 0116 m_socket->write(login); 0117 m_socket->waitForReadyRead(5000); 0118 if(m_state == PreLogin) { 0119 // login did not work 0120 myLog() << "Login did not succeed"; 0121 stop(); 0122 return; 0123 } 0124 } 0125 0126 QByteArray get("get vn basic,details "); 0127 switch(request().key()) { 0128 case Title: 0129 get += "(search ~ \"" + request().value().toUtf8() + "\")"; 0130 break; 0131 0132 default: 0133 myWarning() << source() << "- key not recognized:" << request().key(); 0134 stop(); 0135 return; 0136 } 0137 0138 m_state = GetVN; 0139 myLog() << "Sending get request:" << get; 0140 get.append(0x04); 0141 m_socket->write(get); 0142 } 0143 0144 void VNDBFetcher::stop() { 0145 if(!m_started) { 0146 return; 0147 } 0148 m_started = false; 0149 emit signalDone(this); 0150 } 0151 0152 Tellico::Data::EntryPtr VNDBFetcher::fetchEntryHook(uint uid_) { 0153 Data::EntryPtr entry = m_entries.value(uid_); 0154 if(!entry) { 0155 myWarning() << "no entry in dict"; 0156 return Data::EntryPtr(); 0157 } 0158 0159 // clear past data 0160 m_data.clear(); 0161 // don't need anything for the "details" group 0162 QByteArray get("get release basic,producers "); 0163 get += "(vn = " + entry->field(QStringLiteral("vn-id")).toLatin1() 0164 + " and patch = false and type = \"complete\")"; 0165 0166 m_state = GetRelease; 0167 myLog() << "Sending get request:" << get; 0168 get.append(0x04); 0169 m_socket->write(get); 0170 // 5 sec was not long enough 0171 m_socket->waitForReadyRead(8000); 0172 0173 if(verifyData()) { 0174 parseReleaseResults(entry); 0175 } 0176 0177 // image might still be a URL 0178 const QString image_id = entry->field(QStringLiteral("cover")); 0179 if(image_id.contains(QLatin1Char('/'))) { 0180 const QString id = ImageFactory::addImage(QUrl::fromUserInput(image_id), true /* quiet */); 0181 if(id.isEmpty()) { 0182 message(i18n("The cover image could not be loaded."), MessageHandler::Warning); 0183 } 0184 // empty image ID is ok 0185 entry->setField(QStringLiteral("cover"), id); 0186 } 0187 // clear the placeholder fields 0188 entry->setField(QStringLiteral("vn-id"), QString()); 0189 0190 if(m_socket->isValid()) { 0191 myLog() << "Disconnecting from host"; 0192 m_socket->disconnectFromHost(); 0193 m_state = PreLogin; 0194 } 0195 0196 return entry; 0197 } 0198 0199 Tellico::Fetch::FetchRequest VNDBFetcher::updateRequest(Data::EntryPtr entry_) { 0200 const QString title = entry_->field(QStringLiteral("title")); 0201 if(!title.isEmpty()) { 0202 return FetchRequest(Title, title); 0203 } 0204 return FetchRequest(); 0205 } 0206 0207 bool VNDBFetcher::verifyData() { 0208 if(m_data.isEmpty()) { 0209 myDebug() << "no data"; 0210 return false; 0211 } 0212 0213 // remove the last hex character 0214 m_data.chop(1); 0215 0216 if(m_data.startsWith("error")) { //krazy:exclude=strings 0217 QJsonDocument doc = QJsonDocument::fromJson(m_data.mid(5)); 0218 QVariantMap result = doc.object().toVariantMap(); 0219 if(result.contains(QStringLiteral("msg"))) { 0220 const auto msg = result.value(QStringLiteral("msg")).toString(); 0221 myDebug() << "Data error:" << msg; 0222 message(msg, MessageHandler::Error); 0223 } 0224 return false; 0225 } 0226 0227 // myDebug() << m_data; 0228 if(m_state == PreLogin) { 0229 if(m_data.startsWith("ok")) { //krazy:exclude=strings 0230 m_state = PostLogin; 0231 m_data.clear(); // reset data buffer 0232 return true; 0233 } 0234 // some sort of error 0235 myDebug() << "Failed to login properly..."; 0236 return false; 0237 } 0238 0239 if(!m_data.startsWith("results")) { //krazy:exclude=strings 0240 myDebug() << "Expecting results and didn't get them!"; 0241 return false; 0242 } 0243 0244 // chop off "results" 0245 m_data = m_data.mid(7); 0246 return true; 0247 } 0248 0249 void VNDBFetcher::parseVNResults() { 0250 if(!verifyData()) { 0251 return; 0252 } 0253 #if 0 0254 myWarning() << "Remove debug from vndbfetcher.cpp"; 0255 QFile f(QString::fromLatin1("/tmp/vndbtest.json")); 0256 if(f.open(QIODevice::WriteOnly)) { 0257 QTextStream t(&f); 0258 t.setCodec("UTF-8"); 0259 t << m_data; 0260 } 0261 f.close(); 0262 #endif 0263 0264 QJsonParseError jsonError; 0265 QJsonDocument doc = QJsonDocument::fromJson(m_data, &jsonError); 0266 if(doc.isNull()) { 0267 myDebug() << "null JSON document:" << jsonError.errorString(); 0268 message(jsonError.errorString(), MessageHandler::Error); 0269 stop(); 0270 return; 0271 } 0272 QVariantMap topResultMap = doc.object().toVariantMap(); 0273 QVariantList resultList = topResultMap.value(QStringLiteral("items")).toList(); 0274 if(resultList.isEmpty()) { 0275 myLog() << "No results"; 0276 stop(); 0277 return; 0278 } 0279 0280 Data::CollPtr coll(new Data::GameCollection(true)); 0281 // placeholder for vn id, to be removed later 0282 Data::FieldPtr f1(new Data::Field(QStringLiteral("vn-id"), QString(), Data::Field::Number)); 0283 coll->addField(f1); 0284 // add new fields 0285 if(optionalFields().contains(QStringLiteral("origtitle"))) { 0286 Data::FieldPtr f(new Data::Field(QStringLiteral("origtitle"), i18n("Original Title"))); 0287 f->setFormatType(FieldFormat::FormatTitle); 0288 coll->addField(f); 0289 } 0290 if(optionalFields().contains(QStringLiteral("alias"))) { 0291 Data::FieldPtr f(new Data::Field(QStringLiteral("alias"), i18n("Alias"))); 0292 f->setFlags(Data::Field::AllowMultiple); 0293 f->setFormatType(FieldFormat::FormatTitle); 0294 coll->addField(f); 0295 } 0296 0297 QVariantMap resultMap; 0298 foreach(const QVariant& result, resultList) { 0299 // be sure to check that the fetcher has not been stopped 0300 // crashes can occur if not 0301 if(!m_started) { 0302 break; 0303 } 0304 resultMap = result.toMap(); 0305 0306 Data::EntryPtr entry(new Data::Entry(coll)); 0307 entry->setField(QStringLiteral("title"), mapValue(resultMap, "title")); 0308 entry->setField(QStringLiteral("vn-id"), mapValue(resultMap, "id")); 0309 entry->setField(QStringLiteral("year"), mapValue(resultMap, "released").left(4)); 0310 entry->setField(QStringLiteral("genre"), i18n("Visual Novel")); 0311 entry->setField(QStringLiteral("description"), mapValue(resultMap, "description")); 0312 entry->setField(QStringLiteral("cover"), mapValue(resultMap, "image")); 0313 if(optionalFields().contains(QStringLiteral("origtitle"))) { 0314 entry->setField(QStringLiteral("origtitle"), mapValue(resultMap, "original")); 0315 } 0316 if(optionalFields().contains(QStringLiteral("alias"))) { 0317 const QString aliases = mapValue(resultMap, "aliases"); 0318 entry->setField(QStringLiteral("alias"), aliases.split(QStringLiteral("\n")).join(FieldFormat::delimiterString())); 0319 } 0320 0321 FetchResult* r = new FetchResult(this, entry); 0322 m_entries.insert(r->uid, entry); 0323 emit signalResultFound(r); 0324 } 0325 0326 // m_start = m_entries.count(); 0327 // m_hasMoreResults = m_start <= m_total; 0328 m_hasMoreResults = false; // for now, no continued searches 0329 stop(); 0330 } 0331 0332 void VNDBFetcher::parseReleaseResults(Data::EntryPtr entry_) { 0333 #if 0 0334 myWarning() << "Remove debug2 from vndbfetcher.cpp"; 0335 QFile f(QString::fromLatin1("/tmp/vndb-release.json")); 0336 if(f.open(QIODevice::WriteOnly)) { 0337 QTextStream t(&f); 0338 t.setCodec("UTF-8"); 0339 t << m_data; 0340 } 0341 f.close(); 0342 #endif 0343 0344 QJsonParseError jsonError; 0345 QJsonDocument doc = QJsonDocument::fromJson(m_data, &jsonError); 0346 if(doc.isNull()) { 0347 myDebug() << "null JSON document:" << jsonError.errorString(); 0348 message(jsonError.errorString(), MessageHandler::Error); 0349 return; 0350 } 0351 QVariantMap topResultMap = doc.object().toVariantMap(); 0352 QVariantList resultList = topResultMap.value(QStringLiteral("items")).toList(); 0353 if(resultList.isEmpty()) { 0354 myLog() << "No results"; 0355 return; 0356 } 0357 0358 // only work on the first release item 0359 const QVariantMap resultMap = resultList.at(0).toMap(); 0360 0361 QStringList pubs, devs; 0362 QVariantList producerList = resultMap.value(QStringLiteral("producers")).toList(); 0363 foreach(const QVariant& producerObject, producerList) { 0364 const QVariantMap producerMap = producerObject.toMap(); 0365 if(producerMap.value(QStringLiteral("publisher")) == true) { 0366 pubs += mapValue(producerMap, "name"); 0367 } 0368 if(producerMap.value(QStringLiteral("developer")) == true) { 0369 devs += mapValue(producerMap, "name"); 0370 } 0371 } 0372 entry_->setField(QStringLiteral("publisher"), pubs.join(FieldFormat::delimiterString())); 0373 entry_->setField(QStringLiteral("developer"), devs.join(FieldFormat::delimiterString())); 0374 0375 // update release year 0376 entry_->setField(QStringLiteral("year"), mapValue(resultMap, "released").left(4)); 0377 } 0378 0379 void VNDBFetcher::slotState() { 0380 if(!m_socket) { 0381 return; 0382 } 0383 if(m_socket->state() == QAbstractSocket::ConnectedState) { 0384 m_isConnected = true; 0385 } else if(m_socket->state() == QAbstractSocket::UnconnectedState) { 0386 m_isConnected = false; 0387 } 0388 } 0389 0390 void VNDBFetcher::slotError() { 0391 if(!m_socket) { 0392 return; 0393 } 0394 myDebug() << "Socket error:" << m_socket->errorString(); 0395 } 0396 0397 void VNDBFetcher::slotRead() { 0398 m_data += m_socket->readAll(); 0399 0400 // check the last character, if it's not the hex character, continue waiting 0401 if(m_socket->atEnd() && m_data.endsWith(0x04)) { 0402 switch(m_state) { 0403 case PreLogin: 0404 verifyData(); 0405 break; 0406 case GetVN: 0407 // necessary for passing unit test 0408 QTimer::singleShot(0, this, &VNDBFetcher::parseVNResults); 0409 break; 0410 case GetRelease: 0411 // handled asynch 0412 break; 0413 case PostLogin: 0414 default: 0415 myDebug() << "Unexpected state in slotRead"; 0416 } 0417 } 0418 } 0419 0420 Tellico::Fetch::ConfigWidget* VNDBFetcher::configWidget(QWidget* parent_) const { 0421 return new VNDBFetcher::ConfigWidget(parent_, this); 0422 } 0423 0424 QString VNDBFetcher::defaultName() { 0425 return QStringLiteral("The Visual Novel Database"); // no translation 0426 } 0427 0428 QString VNDBFetcher::defaultIcon() { 0429 return favIcon("https://vndb.org"); 0430 } 0431 0432 Tellico::StringHash VNDBFetcher::allOptionalFields() { 0433 StringHash hash; 0434 hash[QStringLiteral("origtitle")] = i18n("Original Title"); 0435 hash[QStringLiteral("alias")] = i18n("Alias"); 0436 return hash; 0437 } 0438 0439 VNDBFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const VNDBFetcher* fetcher_) 0440 : Fetch::ConfigWidget(parent_) { 0441 QVBoxLayout* l = new QVBoxLayout(optionsWidget()); 0442 l->addWidget(new QLabel(i18n("This source has no options."), optionsWidget())); 0443 l->addStretch(); 0444 0445 // now add additional fields widget 0446 addFieldsWidget(VNDBFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); 0447 } 0448 0449 void VNDBFetcher::ConfigWidget::saveConfigHook(KConfigGroup&) { 0450 } 0451 0452 QString VNDBFetcher::ConfigWidget::preferredName() const { 0453 return VNDBFetcher::defaultName(); 0454 }