File indexing completed on 2025-07-13 05:02:14
0001 /* 0002 SPDX-FileCopyrightText: 2015 Ivan Cukic <ivan.cukic(at)kde.org> 0003 0004 SPDX-License-Identifier: GPL-2.0-or-later 0005 */ 0006 0007 #include "ResultSetQuickCheckTest.h" 0008 0009 #include <QCoreApplication> 0010 #include <QDBusConnection> 0011 #include <QDBusConnectionInterface> 0012 #include <QDebug> 0013 #include <QRandomGenerator> 0014 #include <QString> 0015 #include <QTemporaryDir> 0016 #include <QTest> 0017 #include <QUuid> 0018 0019 #include <boost/range/adaptor/filtered.hpp> 0020 #include <boost/range/algorithm.hpp> 0021 #include <boost/range/algorithm/sort.hpp> 0022 #include <boost/range/numeric.hpp> 0023 0024 #include <query.h> 0025 #include <resultset.h> 0026 0027 #include <iostream> 0028 0029 #include <common/database/Database.h> 0030 #include <common/database/schema/ResourcesDatabaseSchema.h> 0031 0032 #include <utils/qsqlquery_iterator.h> 0033 0034 #define NUMBER_ACTIVITIES 10 0035 #define NUMBER_AGENTS 10 0036 #define NUMBER_RESOURCES 50 0037 #define NUMBER_CACHES 200 0038 0039 namespace KAStats = KActivities::Stats; 0040 0041 static ResultSetQuickCheckTest *instance; 0042 0043 ResultSetQuickCheckTest::ResultSetQuickCheckTest(QObject *parent) 0044 : Test(parent) 0045 , activities(std::make_unique<KActivities::Consumer>()) 0046 { 0047 instance = this; 0048 } 0049 0050 namespace 0051 { 0052 QString resourceTitle(const QString &resource) 0053 { 0054 // We need to find the title 0055 ResourceInfo::Item key; 0056 key.targettedResource = resource; 0057 0058 auto &infos = instance->resourceInfos; 0059 0060 auto ri = infos.lower_bound(key); 0061 0062 return (ri != infos.cend() && ri->targettedResource == resource) ? ri->title : resource; 0063 } 0064 0065 QString toQString(const ResourceScoreCache::Item &item) 0066 { 0067 return item.targettedResource + QLatin1Char(':') // 0068 + resourceTitle(item.targettedResource) + QLatin1Char('(') + QString::number(item.cachedScore) + QLatin1Char(')'); 0069 } 0070 0071 QString toQString(const ResourceLink::Item &item) 0072 { 0073 return item.targettedResource + QLatin1Char(':') + resourceTitle(item.targettedResource); 0074 // + '(' + QString::number(0/* item.score */) + ')' 0075 } 0076 0077 QString toQString(const KAStats::ResultSet::Result &item) 0078 { 0079 return item.resource() + QLatin1Char(':') + item.title() + QLatin1Char('(') + QString::number(item.score()) + QLatin1Char(')'); 0080 } 0081 0082 bool operator==(const ResourceScoreCache::Item &left, const KAStats::ResultSet::Result &right) 0083 { 0084 return left.targettedResource == right.resource() && resourceTitle(left.targettedResource) == right.title() 0085 && qFuzzyCompare(left.cachedScore, right.score()); 0086 } 0087 0088 bool operator==(const ResourceLink::Item &left, const KAStats::ResultSet::Result &right) 0089 { 0090 return left.targettedResource == right.resource() // 0091 && resourceTitle(left.targettedResource) == right.title(); 0092 // && qFuzzyCompare(left.cachedScore, right.score); 0093 } 0094 0095 template<typename Left> 0096 void assert_range_equal(const Left &left, const KAStats::ResultSet &right, const char *file, int line) 0097 { 0098 auto leftIt = left.cbegin(); 0099 auto rightIt = right.cbegin(); 0100 auto leftEnd = left.cend(); 0101 auto rightEnd = right.cend(); 0102 0103 bool equal = true; 0104 0105 QString leftLine; 0106 QString rightLine; 0107 0108 for (; leftIt != leftEnd && rightIt != rightEnd; ++leftIt, ++rightIt) { 0109 auto leftString = toQString(*leftIt); 0110 auto rightString = toQString(*rightIt); 0111 0112 if (*leftIt == *rightIt) { 0113 rightString.fill(QLatin1Char('.')); 0114 0115 } else { 0116 equal = false; 0117 } 0118 0119 int longer = qMax(leftString.length(), rightString.length()); 0120 leftString = leftString.leftJustified(longer); 0121 rightString = rightString.leftJustified(longer, QLatin1Char('.')); 0122 0123 leftLine += QStringLiteral(" ") + leftString; 0124 rightLine += QStringLiteral(" ") + rightString; 0125 } 0126 0127 // So far, we are equal, but do we have the same number 0128 // of elements - did we reach the end of both ranges? 0129 if (leftIt != leftEnd) { 0130 for (; leftIt != leftEnd; ++leftIt) { 0131 auto item = toQString(*leftIt); 0132 leftLine += QStringLiteral(" ") + item; 0133 item.fill(QLatin1Char('X')); 0134 rightLine += QStringLiteral(" ") + item; 0135 } 0136 equal = false; 0137 0138 } else if (rightIt != rightEnd) { 0139 for (; rightIt != rightEnd; ++rightIt) { 0140 auto item = toQString(*leftIt); 0141 rightLine += QStringLiteral(" ") + item; 0142 item.fill(QLatin1Char('X')); 0143 leftLine += QStringLiteral(" ") + item; 0144 } 0145 equal = false; 0146 } 0147 0148 if (!equal) { 0149 qDebug() << "Ranges differ:\n" 0150 << "MEM: " << leftLine << '\n' 0151 << "SQL: " << rightLine; 0152 QTest::qFail("Results do not match", file, line); 0153 } 0154 } 0155 0156 #define ASSERT_RANGE_EQUAL(L, R) assert_range_equal(L, R, __FILE__, __LINE__) 0157 } 0158 0159 //_ Data init 0160 void ResultSetQuickCheckTest::initTestCase() 0161 { 0162 QString databaseFile; 0163 0164 int dbArgIndex = QCoreApplication::arguments().indexOf(QStringLiteral("--ResultSetQuickCheckDatabase")); 0165 if (dbArgIndex > 0) { 0166 databaseFile = QCoreApplication::arguments()[dbArgIndex + 1]; 0167 0168 qDebug() << "Using an existing database: " << databaseFile; 0169 Common::ResourcesDatabaseSchema::overridePath(databaseFile); 0170 0171 pullFromDatabase(); 0172 0173 } else { 0174 QTemporaryDir dir(QDir::tempPath() + QStringLiteral("/KActivitiesStatsTest_ResultSetQuickCheckTest_XXXXXX")); 0175 dir.setAutoRemove(false); 0176 0177 if (!dir.isValid()) { 0178 qFatal("Can not create a temporary directory"); 0179 } 0180 0181 databaseFile = dir.path() + QStringLiteral("/database"); 0182 0183 qDebug() << "Creating database in " << databaseFile; 0184 Common::ResourcesDatabaseSchema::overridePath(databaseFile); 0185 0186 while (activities->serviceStatus() == KActivities::Consumer::Unknown) { 0187 QCoreApplication::processEvents(); 0188 } 0189 0190 generateActivitiesList(); 0191 generateAgentsList(); 0192 generateTypesList(); 0193 generateResourcesList(); 0194 0195 generateResourceInfos(); 0196 generateResourceScoreCaches(); 0197 generateResourceLinks(); 0198 0199 pushToDatabase(); 0200 } 0201 0202 if (QCoreApplication::arguments().contains(QLatin1String("--show-data"))) { 0203 QString rscs; 0204 for (const auto &rsc : resourceScoreCaches) { 0205 rscs += QLatin1Char('(') + rsc.targettedResource // 0206 + QLatin1Char(',') + rsc.usedActivity // 0207 + QLatin1Char(',') + rsc.initiatingAgent // 0208 + QLatin1Char(',') + QString::number(rsc.cachedScore) + QLatin1Char(')'); 0209 } 0210 0211 QString ris; 0212 for (const auto &ri : resourceInfos) { 0213 ris += QLatin1Char('(') + ri.targettedResource + QLatin1Char(',') + ri.title + QLatin1Char(',') + ri.mimetype + QLatin1Char(')'); 0214 } 0215 0216 QString rls; 0217 for (const auto &rl : resourceLinks) { 0218 rls += QLatin1Char('(') + rl.targettedResource // 0219 + QLatin1Char(',') + rl.usedActivity // 0220 + QLatin1Char(',') + rl.initiatingAgent + QLatin1Char(')'); 0221 } 0222 0223 qDebug() << "\nUsed data: -----------------------------" 0224 << "\nActivities: " << activitiesList << "\nAgents: " << agentsList << "\nTypes: " << typesList << "\nResources: " << resourcesList 0225 << "\n----------------------------------------"; 0226 qDebug() << "\n RSCs: " << rscs; 0227 qDebug() << "\n RIs: " << ris; 0228 qDebug() << "\n RLs: " << rls << "\n----------------------------------------"; 0229 } 0230 } 0231 0232 void ResultSetQuickCheckTest::generateActivitiesList() 0233 { 0234 activitiesList = activities->activities(); 0235 0236 while (activitiesList.size() < NUMBER_ACTIVITIES) { 0237 activitiesList << QUuid::createUuid().toString().mid(1, 36); 0238 } 0239 } 0240 0241 void ResultSetQuickCheckTest::generateAgentsList() 0242 { 0243 for (int i = 0; i < NUMBER_AGENTS; ++i) { 0244 agentsList << QStringLiteral("Agent_") + QString::number(i); 0245 } 0246 } 0247 0248 void ResultSetQuickCheckTest::generateTypesList() 0249 { 0250 typesList = { 0251 QStringLiteral("application/postscript"), 0252 QStringLiteral("application/pdf"), 0253 QStringLiteral("image/x-psd"), 0254 QStringLiteral("image/x-sgi"), 0255 QStringLiteral("image/x-tga"), 0256 QStringLiteral("image/x-xbitmap"), 0257 QStringLiteral("image/x-xwindowdump"), 0258 QStringLiteral("image/x-xcf"), 0259 QStringLiteral("image/x-compressed-xcf"), 0260 QStringLiteral("image/tiff"), 0261 QStringLiteral("image/jpeg"), 0262 QStringLiteral("image/x-psp"), 0263 QStringLiteral("image/png"), 0264 QStringLiteral("image/x-icon"), 0265 QStringLiteral("image/x-xpixmap"), 0266 QStringLiteral("image/svg+xml"), 0267 QStringLiteral("application/pdf"), 0268 QStringLiteral("image/x-wmf"), 0269 QStringLiteral("image/jp2"), 0270 QStringLiteral("image/jpeg2000"), 0271 QStringLiteral("image/jpx"), 0272 QStringLiteral("image/x-xcursor"), 0273 }; 0274 } 0275 0276 void ResultSetQuickCheckTest::generateResourcesList() 0277 { 0278 for (int i = 0; i < NUMBER_RESOURCES; ++i) { 0279 resourcesList << (QStringLiteral("/r") // 0280 + (i < 10 ? QStringLiteral("0") : QString()) + QString::number(i)); 0281 } 0282 } 0283 0284 void ResultSetQuickCheckTest::generateResourceInfos() 0285 { 0286 auto *generator = QRandomGenerator::global(); 0287 for (const QString &resource : std::as_const(resourcesList)) { 0288 // We want every n-th or so to be without the title 0289 if (generator->bounded(3)) { 0290 continue; 0291 } 0292 0293 ResourceInfo::Item ri; 0294 ri.targettedResource = resource; 0295 ri.title = QStringLiteral("Title_") + QString::number(generator->bounded(100)); 0296 ri.mimetype = randItem(typesList); 0297 0298 resourceInfos.insert(ri); 0299 } 0300 } 0301 0302 void ResultSetQuickCheckTest::generateResourceScoreCaches() 0303 { 0304 auto *generator = QRandomGenerator::global(); 0305 for (int i = 0; i < NUMBER_CACHES; ++i) { 0306 ResourceScoreCache::Item rsc; 0307 0308 rsc.usedActivity = randItem(activitiesList); 0309 rsc.initiatingAgent = randItem(agentsList); 0310 rsc.targettedResource = randItem(resourcesList); 0311 0312 rsc.cachedScore = generator->bounded(1000); 0313 rsc.firstUpdate = generator->generate(); 0314 rsc.lastUpdate = generator->generate(); 0315 0316 resourceScoreCaches.insert(rsc); 0317 } 0318 } 0319 0320 void ResultSetQuickCheckTest::generateResourceLinks() 0321 { 0322 auto *generator = QRandomGenerator::global(); 0323 for (const QString &resource : std::as_const(resourcesList)) { 0324 // We don't want all the resources to be linked 0325 // to something 0326 if (generator->bounded(2)) { 0327 continue; 0328 } 0329 0330 ResourceLink::Item rl; 0331 0332 rl.targettedResource = resource; 0333 rl.usedActivity = randItem(activitiesList); 0334 rl.initiatingAgent = randItem(agentsList); 0335 0336 resourceLinks.insert(rl); 0337 } 0338 } 0339 0340 void ResultSetQuickCheckTest::pushToDatabase() 0341 { 0342 auto database = Common::Database::instance(Common::Database::ResourcesDatabase, Common::Database::ReadWrite); 0343 0344 Common::ResourcesDatabaseSchema::initSchema(*database); 0345 0346 // Inserting activities, so that a test can be replicated 0347 database->execQuery(QStringLiteral("CREATE TABLE Activity (activity TEXT)")); 0348 for (const auto &activity : activitiesList) { 0349 database->execQuery(QStringLiteral("INSERT INTO Activity VALUES ('%1')").arg(activity)); 0350 } 0351 0352 // Inserting agent, so that a test can be replicated 0353 database->execQuery(QStringLiteral("CREATE TABLE Agent (agent TEXT)")); 0354 for (const auto &agent : agentsList) { 0355 database->execQuery(QStringLiteral("INSERT INTO Agent VALUES ('%1')").arg(agent)); 0356 } 0357 0358 // Inserting types, so that a test can be replicated 0359 database->execQuery(QStringLiteral("CREATE TABLE Type (type TEXT)")); 0360 for (const auto &type : typesList) { 0361 database->execQuery(QStringLiteral("INSERT INTO Type VALUES ('%1')").arg(type)); 0362 } 0363 0364 // Inserting resources, so that a test can be replicated 0365 database->execQuery(QStringLiteral("CREATE TABLE Resource (resource TEXT)")); 0366 for (const auto &resource : resourcesList) { 0367 database->execQuery(QStringLiteral("INSERT INTO Resource VALUES ('%1')").arg(resource)); 0368 } 0369 0370 // Inserting resource score caches 0371 qDebug() << "Inserting" << resourceScoreCaches.size() << "items into ResourceScoreCache"; 0372 int i = 0; 0373 0374 for (const auto &rsc : std::as_const(resourceScoreCaches)) { 0375 std::cerr << '.'; 0376 0377 if (++i % 10 == 0) { 0378 std::cerr << i; 0379 } 0380 0381 database->execQuery( // 0382 QStringLiteral("INSERT INTO ResourceScoreCache (" 0383 " usedActivity" 0384 ", initiatingAgent" 0385 ", targettedResource" 0386 ", scoreType" 0387 ", cachedScore" 0388 ", firstUpdate" 0389 ", lastUpdate" 0390 ") VALUES (" 0391 " '%1'" // usedActivity 0392 ", '%2'" // initiatingAgent 0393 ", '%3'" // targettedResource 0394 ", 0 " // scoreType 0395 ", %4 " // cachedScore 0396 ", %5 " // firstUpdate 0397 ", %6 " // lastUpdate 0398 ")") 0399 .arg(rsc.usedActivity, rsc.initiatingAgent, rsc.targettedResource) 0400 .arg(rsc.cachedScore) 0401 .arg(rsc.firstUpdate) 0402 .arg(rsc.lastUpdate)); 0403 } 0404 std::cerr << std::endl; 0405 0406 // Inserting resource infos 0407 qDebug() << "Inserting" << resourceInfos.size() << "items into ResourceInfo"; 0408 i = 0; 0409 0410 for (const auto &ri : std::as_const(resourceInfos)) { 0411 std::cerr << '.'; 0412 0413 if (++i % 10 == 0) { 0414 std::cerr << i; 0415 } 0416 0417 database->execQuery( // 0418 QStringLiteral("INSERT INTO ResourceInfo (" 0419 " targettedResource" 0420 ", title" 0421 ", mimetype" 0422 ", autoTitle" 0423 ", autoMimetype" 0424 ") VALUES (" 0425 " '%1' " // targettedResource 0426 ", '%2' " // title 0427 ", '%3' " // mimetype 0428 ", 1 " // autoTitle 0429 ", 1 " // autoMimetype 0430 ")") 0431 .arg(ri.targettedResource, ri.title, ri.mimetype)); 0432 } 0433 std::cerr << std::endl; 0434 0435 // Inserting resource links 0436 qDebug() << "Inserting" << resourceLinks.size() << "items into ResourceLink"; 0437 i = 0; 0438 0439 for (const auto &rl : std::as_const(resourceLinks)) { 0440 std::cerr << '.'; 0441 0442 if (++i % 10 == 0) { 0443 std::cerr << i; 0444 } 0445 0446 database->execQuery( // 0447 QStringLiteral("INSERT INTO ResourceLink (" 0448 " targettedResource" 0449 ", usedActivity" 0450 ", initiatingAgent" 0451 ") VALUES (" 0452 " '%1' " // targettedResource 0453 ", '%2' " // usedActivity 0454 ", '%3' " // initiatingAgent 0455 ")") 0456 .arg(rl.targettedResource, rl.usedActivity, rl.initiatingAgent)); 0457 } 0458 std::cerr << std::endl; 0459 } 0460 0461 void ResultSetQuickCheckTest::pullFromDatabase() 0462 { 0463 auto database = Common::Database::instance(Common::Database::ResourcesDatabase, Common::Database::ReadWrite); 0464 0465 auto activityQuery = database->execQuery(QStringLiteral("SELECT * FROM Activity")); 0466 for (const auto &activity : activityQuery) { 0467 activitiesList << activity[0].toString(); 0468 } 0469 0470 auto agentQuery = database->execQuery(QStringLiteral("SELECT * FROM Agent")); 0471 for (const auto &agent : agentQuery) { 0472 agentsList << agent[0].toString(); 0473 } 0474 0475 auto typeQuery = database->execQuery(QStringLiteral("SELECT * FROM Type")); 0476 for (const auto &type : typeQuery) { 0477 typesList << type[0].toString(); 0478 } 0479 0480 auto resourceQuery = database->execQuery(QStringLiteral("SELECT * FROM Resource")); 0481 for (const auto &resource : resourceQuery) { 0482 resourcesList << resource[0].toString(); 0483 } 0484 0485 auto rscQuery = database->execQuery(QStringLiteral("SELECT * FROM ResourceScoreCache")); 0486 0487 for (const auto &rsc : rscQuery) { 0488 ResourceScoreCache::Item item; 0489 item.usedActivity = rsc[QStringLiteral("usedActivity")].toString(); 0490 item.initiatingAgent = rsc[QStringLiteral("initiatingAgent")].toString(); 0491 item.targettedResource = rsc[QStringLiteral("targettedResource")].toString(); 0492 item.cachedScore = rsc[QStringLiteral("cachedScore")].toDouble(); 0493 item.firstUpdate = rsc[QStringLiteral("firstUpdate")].toInt(); 0494 item.lastUpdate = rsc[QStringLiteral("lastUpdate")].toInt(); 0495 resourceScoreCaches.insert(item); 0496 } 0497 0498 auto riQuery = database->execQuery(QStringLiteral("SELECT * FROM ResourceInfo")); 0499 0500 for (const auto &ri : riQuery) { 0501 ResourceInfo::Item item; 0502 item.targettedResource = ri[QStringLiteral("targettedResource")].toString(); 0503 item.title = ri[QStringLiteral("title")].toString(); 0504 item.mimetype = ri[QStringLiteral("mimetype")].toString(); 0505 resourceInfos.insert(item); 0506 } 0507 0508 auto rlQuery = database->execQuery(QStringLiteral("SELECT * FROM ResourceLink")); 0509 0510 for (const auto &rl : rlQuery) { 0511 ResourceLink::Item item; 0512 item.targettedResource = rl[QStringLiteral("targettedResource")].toString(); 0513 item.usedActivity = rl[QStringLiteral("usedActivity")].toString(); 0514 item.initiatingAgent = rl[QStringLiteral("initiatingAgent")].toString(); 0515 resourceLinks.insert(item); 0516 } 0517 } 0518 0519 void ResultSetQuickCheckTest::cleanupTestCase() 0520 { 0521 Q_EMIT testFinished(); 0522 } 0523 0524 QString ResultSetQuickCheckTest::randItem(const QStringList &choices) const 0525 { 0526 return choices[QRandomGenerator::global()->bounded(choices.size())]; 0527 } 0528 //^ Data init 0529 0530 void ResultSetQuickCheckTest::testUsedResourcesForAgents() 0531 { 0532 using namespace KAStats; 0533 using namespace KAStats::Terms; 0534 using boost::sort; 0535 using boost::adaptors::filtered; 0536 0537 for (const auto &agent : std::as_const(agentsList)) { 0538 auto memItems = ResourceScoreCache::groupByResource(resourceScoreCaches | filtered(ResourceScoreCache::initiatingAgent() == agent)); 0539 0540 auto baseTerm = UsedResources | Agent{agent} | Activity::any(); 0541 0542 #define ORDERING_TEST(Column, Dir, OrderFlag) \ 0543 { \ 0544 sort(memItems, ResourceScoreCache::Column().Dir() | ResourceScoreCache::targettedResource().asc()); \ 0545 ResultSet dbItems = baseTerm | OrderFlag | Limit(100); \ 0546 ASSERT_RANGE_EQUAL(memItems, dbItems); \ 0547 } 0548 0549 ORDERING_TEST(targettedResource, asc, OrderByUrl) 0550 ORDERING_TEST(cachedScore, desc, HighScoredFirst); 0551 ORDERING_TEST(lastUpdate, desc, RecentlyUsedFirst); 0552 ORDERING_TEST(firstUpdate, desc, RecentlyCreatedFirst); 0553 0554 #undef ORDERING_TEST 0555 } 0556 } 0557 0558 void ResultSetQuickCheckTest::testUsedResourcesForActivities() 0559 { 0560 } 0561 0562 void ResultSetQuickCheckTest::testLinkedResourcesForAgents() 0563 { 0564 using namespace KAStats; 0565 using namespace KAStats::Terms; 0566 using boost::sort; 0567 using boost::adaptors::filtered; 0568 0569 for (const auto &agent : std::as_const(agentsList)) { 0570 auto memItems = ResourceLink::groupByResource(resourceLinks | filtered(ResourceLink::initiatingAgent() == agent)); 0571 0572 auto baseTerm = LinkedResources | Agent{agent} | Activity::any(); 0573 0574 #define ORDERING_TEST(Column, Dir, OrderFlag) \ 0575 { \ 0576 sort(memItems, ResourceLink::Column().Dir() | ResourceLink::targettedResource().asc()); \ 0577 ResultSet dbItems = baseTerm | OrderFlag; \ 0578 ASSERT_RANGE_EQUAL(memItems, dbItems); \ 0579 } 0580 0581 ORDERING_TEST(targettedResource, asc, OrderByUrl) 0582 // ORDERING_TEST(cachedScore, desc, HighScoredFirst); 0583 // ORDERING_TEST(lastUpdate, desc, RecentlyUsedFirst); 0584 // ORDERING_TEST(firstUpdate, desc, RecentlyCreatedFirst); 0585 0586 #undef ORDERING_TEST 0587 } 0588 } 0589 0590 #include "moc_ResultSetQuickCheckTest.cpp" 0591 0592 // vim: set foldmethod=marker: