Warning, file /pim/sink/tests/storagetest.cpp was not indexed or was modified since last indexation (in which case cross-reference links may be missing, inaccurate or erroneous).
0001 #include <QTest> 0002 0003 #include <iostream> 0004 0005 #include <QDebug> 0006 #include <QString> 0007 #include <QtConcurrent/QtConcurrentRun> 0008 0009 #include "common/storage.h" 0010 #include "storage/key.h" 0011 0012 /** 0013 * Test of the storage implementation to ensure it can do the low level operations as expected. 0014 */ 0015 class StorageTest : public QObject 0016 { 0017 Q_OBJECT 0018 private: 0019 QString testDataPath; 0020 QByteArray dbName; 0021 const char *keyPrefix = "key"; 0022 0023 void populate(int count) 0024 { 0025 Sink::Storage::DataStore storage(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite); 0026 auto transaction = storage.createTransaction(Sink::Storage::DataStore::ReadWrite); 0027 for (int i = 0; i < count; i++) { 0028 // This should perhaps become an implementation detail of the db? 0029 if (i % 10000 == 0) { 0030 if (i > 0) { 0031 transaction.commit(); 0032 transaction = storage.createTransaction(Sink::Storage::DataStore::ReadWrite); 0033 } 0034 } 0035 transaction.openDatabase().write(keyPrefix + QByteArray::number(i), keyPrefix + QByteArray::number(i)); 0036 } 0037 transaction.commit(); 0038 } 0039 0040 bool verify(Sink::Storage::DataStore &storage, int i) 0041 { 0042 bool success = true; 0043 bool keyMatch = true; 0044 const auto reference = keyPrefix + QByteArray::number(i); 0045 storage.createTransaction(Sink::Storage::DataStore::ReadOnly) 0046 .openDatabase() 0047 .scan(keyPrefix + QByteArray::number(i), 0048 [&keyMatch, &reference](const QByteArray &key, const QByteArray &value) -> bool { 0049 if (value != reference) { 0050 qDebug() << "Mismatch while reading"; 0051 keyMatch = false; 0052 } 0053 return keyMatch; 0054 }, 0055 [&success](const Sink::Storage::DataStore::Error &error) { 0056 qDebug() << error.message; 0057 success = false; 0058 }); 0059 return success && keyMatch; 0060 } 0061 0062 private slots: 0063 void initTestCase() 0064 { 0065 testDataPath = "./testdb"; 0066 dbName = "test"; 0067 Sink::Storage::DataStore{testDataPath, {dbName, {{"default", 0}}}}.removeFromDisk(); 0068 } 0069 0070 void cleanup() 0071 { 0072 Sink::Storage::DataStore{testDataPath, {dbName, {{"default", 0}}}}.removeFromDisk(); 0073 } 0074 0075 void testCleanup() 0076 { 0077 populate(1); 0078 Sink::Storage::DataStore{testDataPath, {dbName, {{"default", 0}}}}.removeFromDisk(); 0079 QFileInfo info(testDataPath + "/" + dbName); 0080 QVERIFY(!info.exists()); 0081 } 0082 0083 void testRead() 0084 { 0085 const int count = 100; 0086 0087 populate(count); 0088 0089 // ensure we can read everything back correctly 0090 { 0091 Sink::Storage::DataStore storage(testDataPath, dbName); 0092 for (int i = 0; i < count; i++) { 0093 QVERIFY(verify(storage, i)); 0094 } 0095 } 0096 } 0097 0098 void testScan() 0099 { 0100 const int count = 100; 0101 populate(count); 0102 0103 // ensure we can scan for values 0104 { 0105 int hit = 0; 0106 Sink::Storage::DataStore store(testDataPath, dbName); 0107 store.createTransaction(Sink::Storage::DataStore::ReadOnly) 0108 .openDatabase() 0109 .scan("", [&](const QByteArray &key, const QByteArray &value) -> bool { 0110 if (key == "key50") { 0111 hit++; 0112 } 0113 return true; 0114 }); 0115 QCOMPARE(hit, 1); 0116 } 0117 0118 // ensure we can read a single value 0119 { 0120 int hit = 0; 0121 bool foundInvalidValue = false; 0122 Sink::Storage::DataStore store(testDataPath, dbName); 0123 store.createTransaction(Sink::Storage::DataStore::ReadOnly) 0124 .openDatabase() 0125 .scan("key50", [&](const QByteArray &key, const QByteArray &value) -> bool { 0126 if (key != "key50") { 0127 foundInvalidValue = true; 0128 } 0129 hit++; 0130 return true; 0131 }); 0132 QVERIFY(!foundInvalidValue); 0133 QCOMPARE(hit, 1); 0134 } 0135 } 0136 0137 void testNestedOperations() 0138 { 0139 populate(3); 0140 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0141 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0142 transaction.openDatabase().scan("key1", [&](const QByteArray &key, const QByteArray &value) -> bool { 0143 transaction.openDatabase().remove(key, [](const Sink::Storage::DataStore::Error &) { QVERIFY(false); }); 0144 return false; 0145 }); 0146 } 0147 0148 void testNestedTransactions() 0149 { 0150 populate(3); 0151 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0152 store.createTransaction(Sink::Storage::DataStore::ReadOnly) 0153 .openDatabase() 0154 .scan("key1", [&](const QByteArray &key, const QByteArray &value) -> bool { 0155 store.createTransaction(Sink::Storage::DataStore::ReadWrite).openDatabase().remove(key, [](const Sink::Storage::DataStore::Error &) { QVERIFY(false); }); 0156 return false; 0157 }); 0158 } 0159 0160 void testReadEmptyDb() 0161 { 0162 bool gotResult = false; 0163 bool gotError = false; 0164 Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite); 0165 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0166 auto db = transaction.openDatabase("default", [&](const Sink::Storage::DataStore::Error &error) { 0167 qDebug() << error.message; 0168 gotError = true; 0169 }); 0170 int numValues = db.scan("", 0171 [&](const QByteArray &key, const QByteArray &value) -> bool { 0172 gotResult = true; 0173 return false; 0174 }, 0175 [&](const Sink::Storage::DataStore::Error &error) { 0176 qDebug() << error.message; 0177 gotError = true; 0178 }); 0179 QCOMPARE(numValues, 0); 0180 QVERIFY(!gotResult); 0181 QVERIFY(!gotError); 0182 } 0183 0184 void testConcurrentRead() 0185 { 0186 // With a count of 10000 this test is more likely to expose problems, but also takes some time to execute. 0187 const int count = 1000; 0188 0189 populate(count); 0190 // QTest::qWait(500); 0191 0192 // We repeat the test a bunch of times since failing is relatively random 0193 for (int tries = 0; tries < 10; tries++) { 0194 //clearEnv in combination with the bogus db layouts tests the dynamic named db opening as well. 0195 Sink::Storage::DataStore::clearEnv(); 0196 bool error = false; 0197 // Try to concurrently read 0198 QList<QFuture<void>> futures; 0199 const int concurrencyLevel = 20; 0200 for (int num = 0; num < concurrencyLevel; num++) { 0201 futures << QtConcurrent::run([this, &error]() { 0202 Sink::Storage::DataStore storage(testDataPath, {dbName, {{"bogus", 0}}}, Sink::Storage::DataStore::ReadOnly); 0203 Sink::Storage::DataStore storage2(testDataPath, {dbName+ "2", {{"bogus", 0}}}, Sink::Storage::DataStore::ReadOnly); 0204 for (int i = 0; i < count; i++) { 0205 if (!verify(storage, i)) { 0206 error = true; 0207 break; 0208 } 0209 } 0210 }); 0211 } 0212 for (auto future : futures) { 0213 future.waitForFinished(); 0214 } 0215 QVERIFY(!error); 0216 } 0217 0218 { 0219 Sink::Storage::DataStore(testDataPath, dbName).removeFromDisk(); 0220 Sink::Storage::DataStore(testDataPath, dbName + "2").removeFromDisk(); 0221 } 0222 } 0223 0224 void testNoDuplicates() 0225 { 0226 bool gotResult = false; 0227 bool gotError = false; 0228 Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite); 0229 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0230 auto db = transaction.openDatabase("default"); 0231 db.write("key", "value"); 0232 db.write("key", "value"); 0233 0234 int numValues = db.scan("", 0235 [&](const QByteArray &key, const QByteArray &value) -> bool { 0236 gotResult = true; 0237 return true; 0238 }, 0239 [&](const Sink::Storage::DataStore::Error &error) { 0240 qDebug() << error.message; 0241 gotError = true; 0242 }); 0243 0244 QCOMPARE(numValues, 1); 0245 QVERIFY(!gotError); 0246 QVERIFY(gotResult); 0247 } 0248 0249 void testDuplicates() 0250 { 0251 bool gotResult = false; 0252 bool gotError = false; 0253 const int flags = Sink::Storage::AllowDuplicates; 0254 Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", flags}}}, Sink::Storage::DataStore::ReadWrite); 0255 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0256 auto db = transaction.openDatabase("default", nullptr, flags); 0257 db.write("key", "value1"); 0258 db.write("key", "value2"); 0259 int numValues = db.scan("key", 0260 [&](const QByteArray &key, const QByteArray &value) -> bool { 0261 gotResult = true; 0262 return true; 0263 }, 0264 [&](const Sink::Storage::DataStore::Error &error) { 0265 qDebug() << error.message; 0266 gotError = true; 0267 }); 0268 0269 QCOMPARE(numValues, 2); 0270 QVERIFY(!gotError); 0271 } 0272 0273 void testNonexitingNamedDb() 0274 { 0275 bool gotResult = false; 0276 bool gotError = false; 0277 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadOnly); 0278 QVERIFY(!store.exists()); 0279 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0280 Sink::Storage::DataStore::getUids("test", transaction, [&](const auto &uid) {}); 0281 int numValues = transaction 0282 .openDatabase("test") 0283 .scan("", 0284 [&](const QByteArray &key, const QByteArray &value) -> bool { 0285 gotResult = true; 0286 return false; 0287 }, 0288 [&](const Sink::Storage::DataStore::Error &error) { 0289 qDebug() << error.message; 0290 gotError = true; 0291 }); 0292 QCOMPARE(numValues, 0); 0293 QVERIFY(!gotResult); 0294 QVERIFY(!gotError); 0295 } 0296 0297 /* 0298 * This scenario tests a very specific pattern that can appear with new named databases. 0299 * * A read-only transaction is opened 0300 * * A write-transaction creates a new named db. 0301 * * We try to access that named-db from the already open transaction. 0302 */ 0303 void testNewDbInOpenTransaction() 0304 { 0305 //Create env, otherwise we don't even get a transaction 0306 { 0307 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0308 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0309 } 0310 //Open a longlived transaction 0311 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly); 0312 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0313 0314 //Create the named database 0315 { 0316 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0317 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0318 transaction.openDatabase("test"); 0319 transaction.commit(); 0320 } 0321 0322 0323 //Try to access the named database in the existing transaction. Opening should fail. 0324 bool gotResult = false; 0325 bool gotError = false; 0326 int numValues = transaction 0327 .openDatabase("test") 0328 .scan("", 0329 [&](const QByteArray &key, const QByteArray &value) -> bool { 0330 gotResult = true; 0331 return false; 0332 }, 0333 [&](const Sink::Storage::DataStore::Error &error) { 0334 qDebug() << error.message; 0335 gotError = true; 0336 }); 0337 QCOMPARE(numValues, 0); 0338 QVERIFY(!gotResult); 0339 QVERIFY(!gotError); 0340 } 0341 0342 void testWriteToNamedDb() 0343 { 0344 bool gotError = false; 0345 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0346 store.createTransaction(Sink::Storage::DataStore::ReadWrite) 0347 .openDatabase("test") 0348 .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) { 0349 qDebug() << error.message; 0350 gotError = true; 0351 }); 0352 QVERIFY(!gotError); 0353 } 0354 0355 void testWriteDuplicatesToNamedDb() 0356 { 0357 bool gotError = false; 0358 0359 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0360 store.createTransaction(Sink::Storage::DataStore::ReadWrite) 0361 .openDatabase("test", nullptr, Sink::Storage::AllowDuplicates) 0362 .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) { 0363 qDebug() << error.message; 0364 gotError = true; 0365 }); 0366 QVERIFY(!gotError); 0367 } 0368 0369 // By default we want only exact matches 0370 void testSubstringKeys() 0371 { 0372 const int flags = Sink::Storage::AllowDuplicates; 0373 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", flags}}}, Sink::Storage::DataStore::ReadWrite); 0374 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0375 auto db = transaction.openDatabase("test", nullptr, flags); 0376 db.write("sub", "value1"); 0377 db.write("subsub", "value2"); 0378 int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); 0379 0380 QCOMPARE(numValues, 1); 0381 } 0382 0383 void testFindSubstringKeys() 0384 { 0385 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0386 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0387 auto db = transaction.openDatabase("test"); 0388 db.write("sub", "value1"); 0389 db.write("subsub", "value2"); 0390 db.write("wubsub", "value3"); 0391 int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }, nullptr, true); 0392 0393 QCOMPARE(numValues, 2); 0394 } 0395 0396 void testFindSubstringKeysWithDuplicatesEnabled() 0397 { 0398 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0399 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0400 auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); 0401 db.write("sub", "value1"); 0402 db.write("subsub", "value2"); 0403 db.write("wubsub", "value3"); 0404 int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }, nullptr, true); 0405 0406 QCOMPARE(numValues, 2); 0407 } 0408 0409 void testKeySorting() 0410 { 0411 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0412 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0413 auto db = transaction.openDatabase("test"); 0414 db.write("sub_2", "value2"); 0415 db.write("sub_1", "value1"); 0416 db.write("sub_3", "value3"); 0417 QList<QByteArray> results; 0418 int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { 0419 results << value; 0420 return true; 0421 }, nullptr, true); 0422 0423 QCOMPARE(numValues, 3); 0424 QCOMPARE(results.at(0), QByteArray("value1")); 0425 QCOMPARE(results.at(1), QByteArray("value2")); 0426 QCOMPARE(results.at(2), QByteArray("value3")); 0427 } 0428 0429 // Ensure we don't retrieve a key that is greater than the current key. We only want equal keys. 0430 void testKeyRange() 0431 { 0432 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0433 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0434 auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); 0435 db.write("sub1", "value1"); 0436 int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); 0437 0438 QCOMPARE(numValues, 0); 0439 } 0440 0441 void testFindLatest() 0442 { 0443 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0444 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0445 auto db = transaction.openDatabase("test"); 0446 db.write("sub1", "value1"); 0447 db.write("sub2", "value2"); 0448 db.write("wub3", "value3"); 0449 db.write("wub4", "value4"); 0450 QByteArray result; 0451 db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; }); 0452 0453 QCOMPARE(result, QByteArray("value2")); 0454 } 0455 0456 void testFindLatestInSingle() 0457 { 0458 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0459 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0460 auto db = transaction.openDatabase("test"); 0461 db.write("sub2", "value2"); 0462 QByteArray result; 0463 db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; }); 0464 0465 QCOMPARE(result, QByteArray("value2")); 0466 } 0467 0468 void testFindLast() 0469 { 0470 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0471 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0472 auto db = transaction.openDatabase("test"); 0473 db.write("sub2", "value2"); 0474 db.write("wub3", "value3"); 0475 QByteArray result; 0476 db.findLatest("wub", [&](const QByteArray &key, const QByteArray &value) { result = value; }); 0477 0478 QCOMPARE(result, QByteArray("value3")); 0479 } 0480 0481 void testRecordRevision() 0482 { 0483 Sink::Storage::DataStore store(testDataPath, {dbName, Sink::Storage::DataStore::baseDbs()}, Sink::Storage::DataStore::ReadWrite); 0484 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0485 auto id = Sink::Storage::Identifier::fromDisplayByteArray("{c5d06a9f-1534-4c52-b8ea-415db68bdadf}"); 0486 auto id2 = Sink::Storage::Identifier::fromDisplayByteArray("{c5d06a9f-1534-4c52-b8ea-415db68bdad2}"); 0487 auto id3 = Sink::Storage::Identifier::fromDisplayByteArray("{18a72a62-f8f7-4bc1-a087-ec25f143f60b}"); 0488 Sink::Storage::DataStore::recordRevision(transaction, 1, id, "type"); 0489 Sink::Storage::DataStore::recordRevision(transaction, 2, id2, "type"); 0490 Sink::Storage::DataStore::recordRevision(transaction, 3, id3, "type"); 0491 0492 QCOMPARE(Sink::Storage::DataStore::getTypeFromRevision(transaction, 1), QByteArray("type")); 0493 QCOMPARE(Sink::Storage::DataStore::getUidFromRevision(transaction, 1).toDisplayByteArray(), id.toDisplayByteArray()); 0494 QCOMPARE(Sink::Storage::DataStore::getLatestRevisionFromUid(transaction, id), 1); 0495 QCOMPARE(Sink::Storage::DataStore::getLatestRevisionFromUid(transaction, id2), 2); 0496 QCOMPARE(Sink::Storage::DataStore::getLatestRevisionFromUid(transaction, id3), 3); 0497 0498 Sink::Storage::DataStore::recordRevision(transaction, 10, id, "type"); 0499 QCOMPARE(Sink::Storage::DataStore::getLatestRevisionFromUid(transaction, id), 10); 0500 0501 QCOMPARE(Sink::Storage::DataStore::getRevisionsUntilFromUid(transaction, id, 10).size(), 1); 0502 QCOMPARE(Sink::Storage::DataStore::getRevisionsUntilFromUid(transaction, id, 11).size(), 2); 0503 QCOMPARE(Sink::Storage::DataStore::getRevisionsFromUid(transaction, id).size(), 2); 0504 } 0505 0506 void testRecordRevisionSorting() 0507 { 0508 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 0509 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0510 QByteArray result; 0511 auto db = transaction.openDatabase("test"); 0512 const auto uid = "{c5d06a9f-1534-4c52-b8ea-415db68bdadf}"; 0513 //Ensure we can sort 1 and 10 properly (by default string comparison 10 comes before 6) 0514 const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid); 0515 auto key = Sink::Storage::Key(id, 6); 0516 db.write(key.toInternalByteArray(), "value1"); 0517 key.setRevision(10); 0518 db.write(key.toInternalByteArray(), "value2"); 0519 db.findLatest(id.toInternalByteArray(), [&](const QByteArray &key, const QByteArray &value) { result = value; }); 0520 QCOMPARE(result, QByteArray("value2")); 0521 } 0522 0523 void testRecordRevisionRandom() 0524 { 0525 Sink::Storage::DataStore store(testDataPath, {dbName, Sink::Storage::DataStore::baseDbs()}, Sink::Storage::DataStore::ReadWrite); 0526 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0527 0528 for (auto i = 1; i <= 500; i++) { 0529 const auto uid = Sink::Storage::DataStore::generateUid(); 0530 const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid); 0531 Sink::Storage::DataStore::recordRevision(transaction, i, id, "type"); 0532 0533 QCOMPARE(Sink::Storage::DataStore::getTypeFromRevision(transaction, i), QByteArray("type")); 0534 QCOMPARE(Sink::Storage::DataStore::getUidFromRevision(transaction, i).toDisplayByteArray(), id.toDisplayByteArray()); 0535 QCOMPARE(Sink::Storage::DataStore::getLatestRevisionFromUid(transaction, id), i); 0536 } 0537 } 0538 0539 void setupTestFindRange(Sink::Storage::DataStore::NamedDatabase &db) 0540 { 0541 db.write("0002", "value1"); 0542 db.write("0003", "value2"); 0543 db.write("0004", "value3"); 0544 db.write("0005", "value4"); 0545 } 0546 0547 void testFindRangeOptimistic() 0548 { 0549 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0550 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0551 auto db = transaction.openDatabase("test"); 0552 setupTestFindRange(db); 0553 QByteArrayList results; 0554 db.findAllInRange("0002", "0004", [&](const QByteArray &key, const QByteArray &value) { results << value; }); 0555 0556 QCOMPARE(results, (QByteArrayList{"value1", "value2", "value3"})); 0557 } 0558 0559 void testFindRangeNothing() 0560 { 0561 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0562 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0563 auto db = transaction.openDatabase("test"); 0564 setupTestFindRange(db); 0565 0566 QByteArrayList results1; 0567 db.findAllInRange("0000", "0001", [&](const QByteArray &key, const QByteArray &value) { results1 << value; }); 0568 QCOMPARE(results1, QByteArrayList{}); 0569 0570 QByteArrayList results2; 0571 db.findAllInRange("0000", "0000", [&](const QByteArray &key, const QByteArray &value) { results2 << value; }); 0572 QCOMPARE(results2, QByteArrayList{}); 0573 0574 QByteArrayList results3; 0575 db.findAllInRange("0006", "0010", [&](const QByteArray &key, const QByteArray &value) { results3 << value; }); 0576 QCOMPARE(results3, QByteArrayList{}); 0577 0578 QByteArrayList results4; 0579 db.findAllInRange("0010", "0010", [&](const QByteArray &key, const QByteArray &value) { results4 << value; }); 0580 QCOMPARE(results4, QByteArrayList{}); 0581 } 0582 0583 void testFindRangeSingle() 0584 { 0585 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0586 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0587 auto db = transaction.openDatabase("test"); 0588 setupTestFindRange(db); 0589 0590 QByteArrayList results1; 0591 db.findAllInRange("0004", "0004", [&](const QByteArray &key, const QByteArray &value) { results1 << value; }); 0592 QCOMPARE(results1, QByteArrayList{"value3"}); 0593 } 0594 0595 void testFindRangeOutofBounds() 0596 { 0597 Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); 0598 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0599 auto db = transaction.openDatabase("test"); 0600 setupTestFindRange(db); 0601 0602 QByteArrayList results1; 0603 db.findAllInRange("0000", "0010", [&](const QByteArray &key, const QByteArray &value) { results1 << value; }); 0604 QCOMPARE(results1, (QByteArrayList{"value1", "value2", "value3", "value4"})); 0605 0606 QByteArrayList results2; 0607 db.findAllInRange("0003", "0010", [&](const QByteArray &key, const QByteArray &value) { results2 << value; }); 0608 QCOMPARE(results2, (QByteArrayList{"value2", "value3", "value4"})); 0609 0610 QByteArrayList results3; 0611 db.findAllInRange("0000", "0003", [&](const QByteArray &key, const QByteArray &value) { results3 << value; }); 0612 QCOMPARE(results3, (QByteArrayList{"value1", "value2"})); 0613 } 0614 0615 void testTransactionVisibility() 0616 { 0617 auto readValue = [](const Sink::Storage::DataStore::NamedDatabase &db, const QByteArray) { 0618 QByteArray result; 0619 db.scan("key1", [&](const QByteArray &, const QByteArray &value) { 0620 result = value; 0621 return true; 0622 }); 0623 return result; 0624 }; 0625 { 0626 Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); 0627 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0628 0629 auto db = transaction.openDatabase("testTransactionVisibility"); 0630 db.write("key1", "foo"); 0631 QCOMPARE(readValue(db, "key1"), QByteArray("foo")); 0632 0633 { 0634 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0635 auto db2 = transaction2 0636 .openDatabase("testTransactionVisibility"); 0637 QCOMPARE(readValue(db2, "key1"), QByteArray()); 0638 } 0639 transaction.commit(); 0640 { 0641 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0642 auto db2 = transaction2 0643 .openDatabase("testTransactionVisibility"); 0644 QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); 0645 } 0646 0647 } 0648 } 0649 0650 void testCopyTransaction() 0651 { 0652 Sink::Storage::DataStore store(testDataPath, {dbName, {{"a", 0}, {"b", 0}, {"c", 0}}}, Sink::Storage::DataStore::ReadWrite); 0653 { 0654 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0655 transaction.openDatabase("a"); 0656 transaction.openDatabase("b"); 0657 transaction.openDatabase("c"); 0658 transaction.commit(); 0659 } 0660 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0661 for (int i = 0; i < 1000; i++) { 0662 transaction.openDatabase("a"); 0663 transaction.openDatabase("b"); 0664 transaction.openDatabase("c"); 0665 transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0666 } 0667 } 0668 0669 /* 0670 * This test is meant to find problems with the multi-process architecture and initial database creation. 0671 * If we create named databases dynamically (not all up front), it is possilbe that we violate the rule 0672 * that mdb_open_dbi may only be used by a single thread at a time. 0673 * This test is meant to stress that condition. 0674 * 0675 * FIXME this test ends up locking up every now and then (don't know why). 0676 * All reader threads get stuck on the "QMutexLocker createDbiLocker(&sCreateDbiLock);" mutex in openDatabase, 0677 * and the writer probably crashed. The testfunction then times out. 0678 * I can't reliably reproduce it and thus fix it, so the test remains disabled for now. 0679 */ 0680 //void testReadDuringExternalProcessWrite() 0681 //{ 0682 0683 // QList<QFuture<void>> futures; 0684 // for (int i = 0; i < 5; i++) { 0685 // futures << QtConcurrent::run([&]() { 0686 // QTRY_VERIFY(Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly).exists()); 0687 // Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly); 0688 // auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0689 // for (int i = 0; i < 100000; i++) { 0690 // transaction.openDatabase("a"); 0691 // transaction.openDatabase("b"); 0692 // transaction.openDatabase("c"); 0693 // transaction.openDatabase("p"); 0694 // transaction.openDatabase("q"); 0695 // } 0696 // }); 0697 // } 0698 0699 // //Start writing to the db from a separate process 0700 // QVERIFY(QProcess::startDetached(QCoreApplication::applicationDirPath() + "/dbwriter", QStringList() << testDataPath << dbName << QString::number(100000))); 0701 0702 // for (auto future : futures) { 0703 // future.waitForFinished(); 0704 // } 0705 0706 //} 0707 0708 void testRecordUid() 0709 { 0710 0711 QMap<QByteArray, int> dbs = { 0712 {"revisionType", 0}, 0713 {"revisions", 0}, 0714 {"uids", 0}, 0715 {"default", 0}, 0716 {"__flagtable", 0}, 0717 {"typeuids", 0}, 0718 {"type2uids", 0} 0719 }; 0720 0721 Sink::Storage::DataStore store(testDataPath, {dbName, dbs}, Sink::Storage::DataStore::ReadWrite); 0722 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0723 auto id1 = Sink::Storage::Identifier::fromDisplayByteArray("{c5d06a9f-1534-4c52-b8ea-415db68bdad1}"); 0724 auto id2 = Sink::Storage::Identifier::fromDisplayByteArray("{c5d06a9f-1534-4c52-b8ea-415db68bdad2}"); 0725 auto id3 = Sink::Storage::Identifier::fromDisplayByteArray("{c5d06a9f-1534-4c52-b8ea-415db68bdad3}"); 0726 Sink::Storage::DataStore::recordUid(transaction, id1, "type"); 0727 Sink::Storage::DataStore::recordUid(transaction, id2, "type"); 0728 Sink::Storage::DataStore::recordUid(transaction, id3, "type2"); 0729 0730 { 0731 QVector<QByteArray> uids; 0732 Sink::Storage::DataStore::getUids("type", transaction, [&](const Sink::Storage::Identifier &r) { 0733 uids << r.toDisplayByteArray(); 0734 }); 0735 QVector<QByteArray> expected{id1.toDisplayByteArray(), id2.toDisplayByteArray()}; 0736 QCOMPARE(uids, expected); 0737 } 0738 0739 Sink::Storage::DataStore::removeUid(transaction, id2, "type"); 0740 0741 { 0742 QVector<QByteArray> uids; 0743 Sink::Storage::DataStore::getUids("type", transaction, [&](const Sink::Storage::Identifier &r) { 0744 uids << r.toDisplayByteArray(); 0745 }); 0746 QVector<QByteArray> expected{{id1.toDisplayByteArray()}}; 0747 QCOMPARE(uids, expected); 0748 } 0749 } 0750 0751 void testDbiVisibility() 0752 { 0753 auto readValue = [](const Sink::Storage::DataStore::NamedDatabase &db, const QByteArray) { 0754 QByteArray result; 0755 db.scan("key1", [&](const QByteArray &, const QByteArray &value) { 0756 result = value; 0757 return true; 0758 }); 0759 return result; 0760 }; 0761 { 0762 Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); 0763 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0764 0765 auto db = transaction.openDatabase("testTransactionVisibility"); 0766 db.write("key1", "foo"); 0767 QCOMPARE(readValue(db, "key1"), QByteArray("foo")); 0768 transaction.commit(); 0769 } 0770 Sink::Storage::DataStore::clearEnv(); 0771 0772 //Try to read-only dynamic opening of the db. 0773 //This is the case if we don't have all databases available upon initializatoin and we don't (e.g. because the db hasn't been created yet) 0774 { 0775 // Trick the db into not loading all dbs by passing in a bogus layout. 0776 Sink::Storage::DataStore store(testDataPath, {dbName, {{"bogus", 0}}}, Sink::Storage::DataStore::ReadOnly); 0777 0778 //This transaction should open the dbi 0779 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0780 auto db2 = transaction2.openDatabase("testTransactionVisibility"); 0781 QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); 0782 0783 //This transaction should have the dbi available 0784 auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0785 auto db3 = transaction3.openDatabase("testTransactionVisibility"); 0786 QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); 0787 } 0788 0789 Sink::Storage::DataStore::clearEnv(); 0790 //Try to read-write dynamic opening of the db. 0791 //This is the case if we don't have all databases available upon initialization and we don't (e.g. because the db hasn't been created yet) 0792 { 0793 // Trick the db into not loading all dbs by passing in a bogus layout. 0794 Sink::Storage::DataStore store(testDataPath, {dbName, {{"bogus", 0}}}, Sink::Storage::DataStore::ReadWrite); 0795 0796 //This transaction should open the dbi 0797 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0798 auto db2 = transaction2.openDatabase("testTransactionVisibility"); 0799 QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); 0800 0801 //This transaction should have the dbi available (creating two write transactions obviously doesn't work) 0802 //NOTE: we don't support this scenario. A write transaction must commit or abort before a read transaction opens the same database. 0803 // auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0804 // auto db3 = transaction3.openDatabase("testTransactionVisibility"); 0805 // QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); 0806 0807 //Ensure we can still open further dbis in the write transaction 0808 auto db4 = transaction2.openDatabase("anotherDb"); 0809 } 0810 0811 } 0812 0813 void testIntegerKeys() 0814 { 0815 const int flags = Sink::Storage::IntegerKeys; 0816 Sink::Storage::DataStore store(testDataPath, 0817 { dbName, { { "test", flags } } }, Sink::Storage::DataStore::ReadWrite); 0818 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0819 auto db = transaction.openDatabase("testIntegerKeys", {}, flags); 0820 db.write(0, "value1"); 0821 db.write(1, "value2"); 0822 0823 size_t resultKey; 0824 QByteArray result; 0825 int numValues = db.scan(0, [&](size_t key, const QByteArray &value) -> bool { 0826 resultKey = key; 0827 result = value; 0828 return true; 0829 }); 0830 0831 QCOMPARE(numValues, 1); 0832 QCOMPARE(resultKey, size_t{0}); 0833 QCOMPARE(result, QByteArray{"value1"}); 0834 0835 int numValues2 = db.scan(1, [&](size_t key, const QByteArray &value) -> bool { 0836 resultKey = key; 0837 result = value; 0838 return true; 0839 }); 0840 0841 QCOMPARE(numValues2, 1); 0842 QCOMPARE(resultKey, size_t{1}); 0843 QCOMPARE(result, QByteArray{"value2"}); 0844 } 0845 0846 void testDuplicateIntegerKeys() 0847 { 0848 const int flags = Sink::Storage::IntegerKeys | Sink::Storage::AllowDuplicates; 0849 Sink::Storage::DataStore store(testDataPath, 0850 { dbName, { { "testDuplicateIntegerKeys", flags} } }, 0851 Sink::Storage::DataStore::ReadWrite); 0852 { 0853 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0854 auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags); 0855 db.write(0, "value1"); 0856 db.write(1, "value2"); 0857 db.write(1, "value3"); 0858 QSet<QByteArray> results; 0859 int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { 0860 results << value; 0861 return true; 0862 }); 0863 0864 QCOMPARE(numValues, 2); 0865 QCOMPARE(results.size(), 2); 0866 QVERIFY(results.contains("value2")); 0867 QVERIFY(results.contains("value3")); 0868 } 0869 0870 //Test full scan over keys 0871 { 0872 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0873 auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags); 0874 bool success = true; 0875 QSet<QByteArray> results; 0876 db.scan({}, [&](const QByteArray &key, const QByteArray &value) { 0877 results << value; 0878 return true; 0879 }, 0880 [&success](const Sink::Storage::DataStore::Error &error) { 0881 qWarning() << error.message; 0882 success = false; 0883 }, true); 0884 QVERIFY(success); 0885 QCOMPARE(results.size(), 3); 0886 } 0887 0888 //Test find last 0889 { 0890 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0891 auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags); 0892 bool success = false; 0893 db.findLast(Sink::sizeTToByteArray(1), [&](const QByteArray &key, const QByteArray &value) { 0894 success = true; 0895 }, 0896 [&success](const Sink::Storage::DataStore::Error &error) { 0897 qDebug() << error.message; 0898 success = false; 0899 }); 0900 QVERIFY(success); 0901 } 0902 } 0903 0904 void testDuplicateWithIntegerValues() 0905 { 0906 const int flags = Sink::Storage::AllowDuplicates | Sink::Storage::IntegerValues; 0907 Sink::Storage::DataStore store(testDataPath, 0908 { dbName, { { "testDuplicateWithIntegerValues", flags} } }, 0909 Sink::Storage::DataStore::ReadWrite); 0910 0911 const size_t number1 = 1; 0912 const size_t number2 = 2; 0913 0914 const QByteArray number1BA = Sink::sizeTToByteArray(number1); 0915 const QByteArray number2BA = Sink::sizeTToByteArray(number2); 0916 { 0917 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0918 auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags); 0919 0920 db.write(0, number1BA); 0921 db.write(1, number2BA); 0922 db.write(1, number1BA); 0923 0924 QList<QByteArray> results; 0925 int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { 0926 results << value; 0927 return true; 0928 }); 0929 0930 QCOMPARE(numValues, 2); 0931 QCOMPARE(results.size(), 2); 0932 QCOMPARE(results[0], number1BA); 0933 QCOMPARE(results[1], number2BA); 0934 } 0935 0936 //Test find last 0937 { 0938 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 0939 auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags); 0940 bool success = false; 0941 QByteArray result; 0942 db.findLast(Sink::sizeTToByteArray(1), [&](const QByteArray &key, const QByteArray &value) { 0943 result = value; 0944 success = true; 0945 }, 0946 [&success](const Sink::Storage::DataStore::Error &error) { 0947 qDebug() << error.message; 0948 success = false; 0949 }); 0950 QVERIFY(success); 0951 QCOMPARE(result, number2BA); 0952 } 0953 } 0954 0955 void testIntegerKeyMultipleOf256() 0956 { 0957 const int flags = Sink::Storage::IntegerKeys; 0958 Sink::Storage::DataStore store(testDataPath, 0959 { dbName, { {"testIntegerKeyMultipleOf256", flags} } }, 0960 Sink::Storage::DataStore::ReadWrite); 0961 0962 { 0963 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0964 auto db = transaction.openDatabase("testIntegerKeyMultipleOf256", {}, flags); 0965 0966 db.write(0x100, "hello"); 0967 db.write(0x200, "hello2"); 0968 db.write(0x42, "hello3"); 0969 0970 transaction.commit(); 0971 } 0972 0973 { 0974 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0975 auto db = transaction2.openDatabase("testIntegerKeyMultipleOf256", {}, flags); 0976 0977 size_t resultKey; 0978 QByteArray resultValue; 0979 db.scan(0x100, [&] (size_t key, const QByteArray &value) { 0980 resultKey = key; 0981 resultValue = value; 0982 return false; 0983 }); 0984 0985 QCOMPARE(resultKey, size_t{0x100}); 0986 QCOMPARE(resultValue, QByteArray{"hello"}); 0987 } 0988 } 0989 0990 void testIntegerProperlySorted() 0991 { 0992 const int flags = Sink::Storage::IntegerKeys; 0993 Sink::Storage::DataStore store(testDataPath, 0994 { dbName, { {"testIntegerProperlySorted", flags} } }, 0995 Sink::Storage::DataStore::ReadWrite); 0996 0997 { 0998 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 0999 auto db = transaction.openDatabase("testIntegerProperlySorted", {}, flags); 1000 1001 for (size_t i = 0; i < 0x100; ++i) { 1002 db.write(i, "hello"); 1003 } 1004 1005 size_t previous = 0; 1006 bool success = true; 1007 db.scan("", [&] (const QByteArray &key, const QByteArray &value) { 1008 size_t current = Sink::byteArrayToSizeT(key); 1009 if (current < previous) { 1010 success = false; 1011 return false; 1012 } 1013 1014 previous = current; 1015 return true; 1016 }); 1017 1018 QVERIFY2(success, "Integer are not properly sorted before commit"); 1019 1020 transaction.commit(); 1021 } 1022 1023 { 1024 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 1025 auto db = transaction.openDatabase("testIntegerProperlySorted", {}, flags); 1026 1027 size_t previous = 0; 1028 bool success = true; 1029 db.scan("", [&] (const QByteArray &key, const QByteArray &value) { 1030 size_t current = Sink::byteArrayToSizeT(key); 1031 if (current < previous) { 1032 success = false; 1033 return false; 1034 } 1035 1036 previous = current; 1037 return true; 1038 }); 1039 1040 QVERIFY2(success, "Integer are not properly sorted after commit"); 1041 } 1042 } 1043 1044 /** 1045 * Demonstrate how long running transactions result in an accumulation of free-pages. 1046 */ 1047 void testFreePages() 1048 { 1049 Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); 1050 1051 // With any ro transaction ongoing we just accumulate endless free pages 1052 // auto rotransaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 1053 for (int i = 0; i < 5; i++) { 1054 { 1055 auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); 1056 transaction.openDatabase("test").write("sub" + QByteArray::number(i), "value1"); 1057 } 1058 // If we reset the rotransaction the accumulation is not a problem (because previous free pages can be reused at that point) 1059 // auto rotransaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); 1060 { 1061 auto stat = store.createTransaction(Sink::Storage::DataStore::ReadOnly).stat(false); 1062 QVERIFY(stat.freePages <= 6); 1063 } 1064 } 1065 } 1066 }; 1067 1068 QTEST_MAIN(StorageTest) 1069 #include "storagetest.moc"