File indexing completed on 2024-12-22 04:18:20
0001 /*************************************************************************** 0002 * * 0003 * Copyright : (C) 2012 Peter Kümmel * 0004 * email : syntheticpp@gmx.net * 0005 * * 0006 * This program is free software; you can redistribute it and/or modify * 0007 * it under the terms of the GNU General Public License as published by * 0008 * the Free Software Foundation; either version 2 of the License, or * 0009 * (at your option) any later version. * 0010 * * 0011 ***************************************************************************/ 0012 0013 #define KST_SMALL_PRREALLOC 0014 0015 #include "asciifilebuffer.h" 0016 0017 #include <QtTest> 0018 0019 namespace QTest 0020 { 0021 template<> 0022 bool qCompare<qint64, qint32>(qint64 const &t1, qint32 const &t2, const char *actual, const char *expected, const char *file, int line) 0023 { 0024 return qCompare<qint64>(t1, qint64(t2), actual, expected, file, line); 0025 } 0026 } 0027 0028 0029 class AsciiSourceTest: public QObject 0030 { 0031 Q_OBJECT 0032 0033 public: 0034 0035 AsciiSourceTest() 0036 { 0037 } 0038 0039 private slots: 0040 0041 void initTestcase() 0042 { 0043 buf.setFile(&file); 0044 } 0045 0046 0047 // int findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos) const 0048 0049 void find_wrong_parameters() 0050 { 0051 int rows = 3; 0052 int rowLength = 100; 0053 initRowIndex(rows, rowLength); 0054 QCOMPARE(buf.findRowOfPosition(idx, 0, -1), -1); 0055 QCOMPARE(buf.findRowOfPosition(AsciiFileBuffer::RowIndex(), 0, 1), -1); 0056 QCOMPARE(buf.findRowOfPosition(idx, 5, 1), -1); 0057 QCOMPARE(buf.findRowOfPosition(idx, 1, 99), -1); 0058 } 0059 0060 0061 void find_small() 0062 { 0063 int rows = 1; 0064 int rowLength = 1; 0065 initRowIndex(rows, rowLength); 0066 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0067 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), -1); 0068 QCOMPARE(buf.findRowOfPosition(idx, 0, 2), -1); 0069 0070 rows = 2; 0071 rowLength = 1; 0072 initRowIndex(rows, rowLength); 0073 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0074 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), 1); 0075 QCOMPARE(buf.findRowOfPosition(idx, 0, 2), -1); 0076 0077 rows = 3; 0078 rowLength = 1; 0079 initRowIndex(rows, rowLength); 0080 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0081 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), 1); 0082 QCOMPARE(buf.findRowOfPosition(idx, 0, 2), 2); 0083 QCOMPARE(buf.findRowOfPosition(idx, 0, 3), -1); 0084 0085 rows = 1; 0086 rowLength = 2; 0087 initRowIndex(rows, rowLength); 0088 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0089 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), 0); 0090 QCOMPARE(buf.findRowOfPosition(idx, 0, 2), -1); 0091 QCOMPARE(buf.findRowOfPosition(idx, 0, 3), -1); 0092 } 0093 0094 0095 void find_start_0() 0096 { 0097 int rows = 1; 0098 int rowLength = 100; 0099 initRowIndex(rows, rowLength); 0100 0101 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0102 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), 0); 0103 QCOMPARE(buf.findRowOfPosition(idx, 0, 99), 0); 0104 QCOMPARE(buf.findRowOfPosition(idx, 0, 100), -1); 0105 QCOMPARE(buf.findRowOfPosition(idx, 0, 101), -1); 0106 QCOMPARE(buf.findRowOfPosition(idx, 0, 199), -1); 0107 0108 rows = 2; 0109 rowLength = 100; 0110 initRowIndex(rows, rowLength); 0111 0112 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0113 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), 0); 0114 QCOMPARE(buf.findRowOfPosition(idx, 0, 99), 0); 0115 QCOMPARE(buf.findRowOfPosition(idx, 0, 100), 1); 0116 QCOMPARE(buf.findRowOfPosition(idx, 0, 101), 1); 0117 QCOMPARE(buf.findRowOfPosition(idx, 0, 199), 1); 0118 QCOMPARE(buf.findRowOfPosition(idx, 0, 200), -1); 0119 QCOMPARE(buf.findRowOfPosition(idx, 0, 201), -1); 0120 QCOMPARE(buf.findRowOfPosition(idx, 0, 299), -1); 0121 0122 rows = 3; 0123 rowLength = 100; 0124 initRowIndex(rows, rowLength); 0125 0126 QCOMPARE(buf.findRowOfPosition(idx, 0, 0), 0); 0127 QCOMPARE(buf.findRowOfPosition(idx, 0, 1), 0); 0128 QCOMPARE(buf.findRowOfPosition(idx, 0, 99), 0); 0129 QCOMPARE(buf.findRowOfPosition(idx, 0, 100), 1); 0130 QCOMPARE(buf.findRowOfPosition(idx, 0, 101), 1); 0131 QCOMPARE(buf.findRowOfPosition(idx, 0, 199), 1); 0132 QCOMPARE(buf.findRowOfPosition(idx, 0, 200), 2); 0133 QCOMPARE(buf.findRowOfPosition(idx, 0, 201), 2); 0134 QCOMPARE(buf.findRowOfPosition(idx, 0, 299), 2); 0135 QCOMPARE(buf.findRowOfPosition(idx, 0, 300), -1); 0136 QCOMPARE(buf.findRowOfPosition(idx, 0, 301), -1); 0137 QCOMPARE(buf.findRowOfPosition(idx, 0, 399), -1); 0138 } 0139 0140 0141 void find_start_n() 0142 { 0143 int rows = 3; 0144 int rowLength = 100; 0145 initRowIndex(rows, rowLength); 0146 0147 QCOMPARE(buf.findRowOfPosition(idx, 0, 100), 1); 0148 QCOMPARE(buf.findRowOfPosition(idx, 1, 100), 1); 0149 0150 QCOMPARE(buf.findRowOfPosition(idx, 0, 101), 1); 0151 QCOMPARE(buf.findRowOfPosition(idx, 1, 101), 1); 0152 0153 } 0154 0155 0156 // const QVector<AsciiFileData> splitFile(int chunkSize, const RowIndex& rowIndex, int start, int bytesToRead) const; 0157 0158 void split_wrong_parameters() 0159 { 0160 int rows = 1; 0161 int rowLength = 1; 0162 initRowIndex(rows, rowLength); 0163 QCOMPARE(buf.splitFile(1, AsciiFileBuffer::RowIndex(), 0, 1).size(), 0); 0164 QCOMPARE(buf.splitFile(-1, idx, 0, 1).size(), 0); 0165 QCOMPARE(buf.splitFile(1, idx, -1, 1).size(), 0); 0166 QCOMPARE(buf.splitFile(1, idx, -1, 1).size(), 0); 0167 QCOMPARE(buf.splitFile(1, idx, 10, 1).size(), 0); 0168 QCOMPARE(buf.splitFile(1, idx, 0, 2).size(), 0); 0169 } 0170 0171 0172 void split_small() 0173 { 0174 int rows = 1; 0175 int rowLength = 1; 0176 initRowIndex(rows, rowLength); 0177 QVector<AsciiFileData> c = buf.splitFile(1, idx, 0, 1); 0178 QCOMPARE(c.size(), 1); 0179 QCOMPARE(c[0].begin(), 0); 0180 QCOMPARE(c[0].bytesRead(), 1); 0181 QCOMPARE(c[0].rowBegin(), 0); 0182 QCOMPARE(c[0].rowsRead(), 1); 0183 0184 rows = 2; 0185 rowLength = 1; 0186 initRowIndex(rows, rowLength); 0187 c = buf.splitFile(1, idx, 0, 2); 0188 QCOMPARE(c.size(), 2); 0189 QCOMPARE(c[0].begin(), 0); 0190 QCOMPARE(c[0].bytesRead(), 1); 0191 QCOMPARE(c[0].rowBegin(), 0); 0192 QCOMPARE(c[0].rowsRead(), 1); 0193 QCOMPARE(c[1].begin(), 1); 0194 QCOMPARE(c[1].bytesRead(), 1); 0195 QCOMPARE(c[1].rowBegin(), 1); 0196 QCOMPARE(c[1].rowsRead(), 1); 0197 0198 rows = 3; 0199 rowLength = 1; 0200 initRowIndex(rows, rowLength); 0201 c = buf.splitFile(1, idx, 0, 3); 0202 QCOMPARE(c.size(), 3); 0203 QCOMPARE(c[2].begin(), 2); 0204 QCOMPARE(c[2].bytesRead(), 1); 0205 QCOMPARE(c[2].rowBegin(), 2); 0206 QCOMPARE(c[2].rowsRead(), 1); 0207 0208 0209 rows = 1; 0210 rowLength = 2; 0211 initRowIndex(rows, rowLength); 0212 c = buf.splitFile(2, idx, 0, 2); 0213 QCOMPARE(c.size(), 1); 0214 QCOMPARE(c[0].begin(), 0); 0215 QCOMPARE(c[0].bytesRead(), 2); 0216 QCOMPARE(c[0].rowBegin(), 0); 0217 QCOMPARE(c[0].rowsRead(), 1); 0218 0219 rows = 2; 0220 rowLength = 2; 0221 initRowIndex(rows, rowLength); 0222 c = buf.splitFile(2, idx, 0, 4); 0223 QCOMPARE(c.size(), 2); 0224 QCOMPARE(c[0].begin(), 0); 0225 QCOMPARE(c[0].bytesRead(), 2); 0226 QCOMPARE(c[0].rowBegin(), 0); 0227 QCOMPARE(c[0].rowsRead(), 1); 0228 QCOMPARE(c[1].begin(), 2); 0229 QCOMPARE(c[1].bytesRead(), 2); 0230 QCOMPARE(c[1].rowBegin(), 1); 0231 QCOMPARE(c[1].rowsRead(), 1); 0232 0233 // chunk ends in the middle of a row 0234 rows = 2; 0235 rowLength = 3; 0236 initRowIndex(rows, rowLength); 0237 c = buf.splitFile(5, idx, 0, 6); 0238 QCOMPARE(c.size(), 2); 0239 QCOMPARE(c[0].begin(), 0); 0240 QCOMPARE(c[0].bytesRead(), 3); 0241 QCOMPARE(c[0].rowBegin(), 0); 0242 QCOMPARE(c[0].rowsRead(), 1); 0243 QCOMPARE(c[1].begin(), 3); 0244 QCOMPARE(c[1].bytesRead(), 3); 0245 QCOMPARE(c[1].rowBegin(), 1); 0246 QCOMPARE(c[1].rowsRead(), 1); 0247 0248 rows = 1; 0249 rowLength = 2; 0250 initRowIndex(rows, rowLength); 0251 c = buf.splitFile(1, idx, 0, 2); 0252 QCOMPARE(c.size(), 0); // chunk size to small for one row 0253 } 0254 0255 0256 void split_into_1() 0257 { 0258 int rows = 3; 0259 int rowLength = 100; 0260 int bytes = rows * rowLength; 0261 initRowIndex(rows, rowLength); 0262 QVector<AsciiFileData> c = buf.splitFile(rows * rowLength, idx, 0, bytes); 0263 QCOMPARE(c[0].begin(), 0); 0264 QCOMPARE(c[0].bytesRead(), bytes); 0265 0266 idx[0] = 10; 0267 bytes -= 10; 0268 c = buf.splitFile(rows * rowLength, idx, 10, bytes); 0269 QCOMPARE(c[0].begin(), 10); 0270 QCOMPARE(c[0].bytesRead(), bytes); 0271 } 0272 0273 0274 void split_into_3() 0275 { 0276 int rows = 3; 0277 int rowLength = 100; 0278 int bytes = rows * rowLength; 0279 initRowIndex(rows, rowLength); 0280 QVector<AsciiFileData> c = buf.splitFile(rowLength, idx, 0, bytes); 0281 QCOMPARE(c.size(), 3); 0282 QCOMPARE(c[0].begin(), idx[0]); 0283 QCOMPARE(c[1].begin(), idx[1]); 0284 QCOMPARE(c[2].begin(), idx[2]); 0285 } 0286 0287 0288 void split() 0289 { 0290 int rows = 999; 0291 int rowLength = 100; 0292 int bytes = rows * rowLength; 0293 initRowIndex(rows, rowLength); 0294 QVector<AsciiFileData> c = buf.splitFile(rowLength * 100, idx, 0, bytes); 0295 QCOMPARE(c.size(), 10); 0296 QCOMPARE(c[0].begin(), idx[0]); 0297 QCOMPARE(c[1].rowsRead(), 100); 0298 QCOMPARE(c[1].begin(), idx[100]); 0299 QCOMPARE(c[1].rowsRead(), 100); 0300 QCOMPARE(c[9].begin(), idx[900]); 0301 QCOMPARE(c[9].rowsRead(), 99); 0302 0303 int rowsRead = 0; 0304 for (int i=0; i < 10; i++) { 0305 rowsRead += c[i].rowsRead(); 0306 } 0307 QCOMPARE(rowsRead, rows); 0308 } 0309 0310 0311 void split_equal_chunkSize_is_bytesToRead() 0312 { 0313 int rows = 20; 0314 int rowLength = 10; 0315 int offset = 5; 0316 int bytesToRead = (rows - offset) * rowLength; 0317 initRowIndex(rows, rowLength); 0318 int start = offset * rowLength; 0319 QVector<AsciiFileData> c = buf.splitFile(bytesToRead, idx, start, bytesToRead); 0320 QCOMPARE(c.size(), 1); 0321 QCOMPARE(c[0].bytesRead(), bytesToRead); 0322 QCOMPARE(c[0].begin(), idx[offset]); 0323 QCOMPARE(c[0].rowsRead(), rows - offset); 0324 } 0325 0326 //void useOneWindowWithChunks(const RowIndex& rowIndex, int start, int bytesToRead, int numChunks); 0327 0328 void useOneWindowWithChunks_small() 0329 { 0330 int rows = 1; 0331 int rowLength = 1; 0332 initRowIndex(rows, rowLength); 0333 buf.useOneWindowWithChunks(idx, 0, 1, 1); 0334 QVector<QVector<AsciiFileData> > d = buf.fileData(); 0335 QCOMPARE(d.size(), 1); 0336 QCOMPARE(d[0].size(), 1); 0337 0338 rows = 2; 0339 rowLength = 1; 0340 initRowIndex(rows, rowLength); 0341 buf.useOneWindowWithChunks(idx, 0, 2, 1); 0342 d = buf.fileData(); 0343 QCOMPARE(d.size(), 1); 0344 QCOMPARE(d[0].size(), 1); // only data for one chunk 0345 0346 rows = 2; 0347 rowLength = 1; 0348 initRowIndex(rows, rowLength); 0349 buf.useOneWindowWithChunks(idx, 0, 2, 2); 0350 d = buf.fileData(); 0351 QCOMPARE(d.size(), 1); 0352 QCOMPARE(d[0].size(), 2); 0353 0354 rows = 3; 0355 rowLength = 1; 0356 initRowIndex(rows, rowLength); 0357 buf.useOneWindowWithChunks(idx, 0, 3, 2); 0358 d = buf.fileData(); 0359 QCOMPARE(d.size(), 1); 0360 QCOMPARE(d[0].size(), 3); // more than 2 chunks needed 0361 } 0362 0363 void useOneWindowWithChunks() 0364 { 0365 int rows = 1000; 0366 int rowLength = 100; 0367 int bytes = rows * rowLength; 0368 initRowIndex(rows, rowLength); 0369 buf.useOneWindowWithChunks(idx, 0, bytes, 1); 0370 QVector<QVector<AsciiFileData> > d = buf.fileData(); 0371 QCOMPARE(d.size(), 1); 0372 QCOMPARE(d[0].size(), 1); 0373 0374 buf.useOneWindowWithChunks(idx, 0, bytes, 10); 0375 d = buf.fileData(); 0376 QCOMPARE(d.size(), 1); 0377 QCOMPARE(d[0].size(), 10); 0378 0379 buf.useOneWindowWithChunks(idx, 0, bytes, 1000); 0380 d = buf.fileData(); 0381 QCOMPARE(d.size(), 1); 0382 QCOMPARE(d[0].size(), 1000); 0383 0384 buf.useOneWindowWithChunks(idx, 0, bytes, 1001); // row too long for implizit chunk size 0385 d = buf.fileData(); 0386 QCOMPARE(d.size(), 0); 0387 } 0388 0389 0390 // void useSlidingWindow(const RowIndex& rowIndex, int start, int bytesToRead, int windowSize); 0391 0392 void useSlidingWindow_small() 0393 { 0394 int rows = 1; 0395 int rowLength = 1; 0396 initRowIndex(rows, rowLength); 0397 int windowSize = 1; 0398 buf.useSlidingWindow(idx, 0, 1, windowSize); 0399 QVector<QVector<AsciiFileData> > d = buf.fileData(); 0400 QCOMPARE(d.size(), 1); 0401 QCOMPARE(d[0].size(), 1); 0402 0403 rows = 2; 0404 rowLength = 1; 0405 initRowIndex(rows, rowLength); 0406 windowSize = 1; 0407 buf.useSlidingWindow(idx, 0, 2, windowSize); 0408 d = buf.fileData(); 0409 QCOMPARE(d.size(), 2); 0410 QCOMPARE(d[0].size(), 1); 0411 QCOMPARE(d[1].size(), 1); 0412 } 0413 0414 0415 void useSlidingWindow() 0416 { 0417 int rows = 1000; 0418 int rowLength = 100; 0419 int bytes = rows * rowLength; 0420 initRowIndex(rows, rowLength); 0421 int windowSize = rowLength; 0422 buf.useSlidingWindow(idx, 0, bytes, windowSize); 0423 QVector<QVector<AsciiFileData> > d = buf.fileData(); 0424 QCOMPARE(d.size(), 1000); 0425 QCOMPARE(d[0].size(), 1); 0426 0427 rows = 999; 0428 rowLength = 100; 0429 bytes = rows * rowLength; 0430 windowSize = rowLength * 100; 0431 buf.useSlidingWindow(idx, 0, bytes, windowSize); 0432 d = buf.fileData(); 0433 QCOMPARE(d.size(), 10); 0434 QCOMPARE(d[0].size(), 1); 0435 QCOMPARE(d[0][0].begin(), 0); 0436 QCOMPARE(d[0][0].bytesRead(), rowLength * 100); 0437 QCOMPARE(d[0][0].rowBegin(), 0); 0438 QCOMPARE(d[1].size(), 1); 0439 QCOMPARE(d[1][0].rowBegin(), 100); 0440 0441 int rowsRead = 0; 0442 for (int i=0; i < 10; i++) { 0443 rowsRead += d[i][0].rowsRead(); 0444 } 0445 QCOMPARE(rowsRead, rows); 0446 } 0447 0448 0449 // void useSlidingWindowWithChunks(const RowIndex& rowIndex, int start, int bytesToRead, int windowSize, int numWindowChunks) 0450 0451 void useSlidingWindowWithChunks_small() 0452 { 0453 int rows = 1; 0454 int rowLength = 1; 0455 int bytes = rows * rowLength; 0456 initRowIndex(rows, rowLength); 0457 int windowSize = 1; 0458 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 1); 0459 QVector<QVector<AsciiFileData> > d = buf.fileData(); 0460 QCOMPARE(d.size(), 1); 0461 QCOMPARE(d[0].size(), 1); 0462 0463 rows = 3; 0464 rowLength = 1; 0465 bytes = rows * rowLength; 0466 initRowIndex(rows, rowLength); 0467 windowSize = 3; 0468 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 3); 0469 d = buf.fileData(); 0470 QCOMPARE(d.size(), 1); 0471 QCOMPARE(d[0].size(), 3); 0472 0473 rows = 3; 0474 rowLength = 1; 0475 bytes = rows * rowLength; 0476 initRowIndex(rows, rowLength); 0477 windowSize = 1; 0478 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 3); // int chunkSize = windowSize / numWindowChunks, == 0 0479 d = buf.fileData(); 0480 QCOMPARE(d.size(), 0); 0481 0482 rows = 12; 0483 rowLength = 1; 0484 bytes = rows * rowLength; 0485 initRowIndex(rows, rowLength); 0486 windowSize = 4; 0487 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 1); 0488 d = buf.fileData(); 0489 QCOMPARE(d.size(), 3); 0490 QCOMPARE(d[0].size(), 1); 0491 QCOMPARE(d[2].size(), 1); 0492 0493 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 2); 0494 d = buf.fileData(); 0495 QCOMPARE(d.size(), 3); 0496 QCOMPARE(d[0].size(), 2); 0497 QCOMPARE(d[2].size(), 2); 0498 0499 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 3); 0500 d = buf.fileData(); 0501 QCOMPARE(d.size(), 4); // could not split 4 rows into 3 chunks 0502 QCOMPARE(d[0].size(), 3); 0503 QCOMPARE(d[1].size(), 3); 0504 QCOMPARE(d[2].size(), 3); 0505 QCOMPARE(d[3].size(), 3); 0506 } 0507 0508 0509 void useSlidingWindowWithChunks() 0510 { 0511 int rows = 1000; 0512 int rowLength = 100; 0513 int bytes = rows * rowLength; 0514 initRowIndex(rows, rowLength); 0515 int windowSize = rowLength * 100; 0516 buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 5); 0517 QVector<QVector<AsciiFileData> > d = buf.fileData(); 0518 QCOMPARE(d.size(), 10); 0519 QCOMPARE(d[0].size(), 5); 0520 QCOMPARE(d[9].size(), 5); 0521 } 0522 0523 0524 private: 0525 AsciiFileBuffer::RowIndex idx; 0526 AsciiFileBuffer buf; 0527 QFile file; 0528 0529 void initRowIndex(int rows, int rowLength) 0530 { 0531 idx.clear(); 0532 idx.resize(rows + 1); 0533 for (int i = 0; i < rows; i++) { 0534 idx[i] = i * rowLength; 0535 } 0536 idx[rows] = rows * rowLength; 0537 } 0538 }; 0539 0540 0541 0542 QTEST_MAIN(AsciiSourceTest) 0543 0544 0545 0546 #include "moc_asciifilebuffertest.cpp"