File indexing completed on 2024-12-22 04:18:20

0001 /***************************************************************************
0002  *                                                                         *
0003  *   Copyright : (C) 2012 Peter Kümmel                                     *
0004  *   email     : syntheticpp@gmx.net                                       *
0005  *                                                                         *
0006  *   This program is free software; you can redistribute it and/or modify  *
0007  *   it under the terms of the GNU General Public License as published by  *
0008  *   the Free Software Foundation; either version 2 of the License, or     *
0009  *   (at your option) any later version.                                   *
0010  *                                                                         *
0011  ***************************************************************************/
0012 
0013 #define KST_SMALL_PRREALLOC
0014 
0015 #include "asciifilebuffer.h"
0016 
0017 #include <QtTest>
0018 
0019 namespace QTest
0020 {
0021     template<>
0022     bool qCompare<qint64, qint32>(qint64 const &t1, qint32 const &t2, const char *actual, const char *expected, const char *file, int line)
0023     {
0024         return qCompare<qint64>(t1, qint64(t2), actual, expected, file, line);
0025     }
0026 }
0027 
0028 
0029 class AsciiSourceTest: public QObject
0030 {
0031     Q_OBJECT
0032     
0033 public:
0034     
0035     AsciiSourceTest()
0036     {
0037     }
0038    
0039 private slots:
0040 
0041     void initTestcase()
0042     {
0043       buf.setFile(&file);
0044     }
0045 
0046 
0047     // int findRowOfPosition(const AsciiFileBuffer::RowIndex& rowIndex, int searchStart, int pos) const
0048 
0049     void find_wrong_parameters()
0050     {
0051       int rows = 3;
0052       int rowLength = 100;
0053       initRowIndex(rows, rowLength);
0054       QCOMPARE(buf.findRowOfPosition(idx, 0, -1), -1);
0055       QCOMPARE(buf.findRowOfPosition(AsciiFileBuffer::RowIndex(), 0, 1), -1);
0056       QCOMPARE(buf.findRowOfPosition(idx, 5,  1), -1);
0057       QCOMPARE(buf.findRowOfPosition(idx, 1, 99), -1);
0058     }
0059 
0060 
0061     void find_small()
0062     {
0063       int rows = 1;
0064       int rowLength = 1;
0065       initRowIndex(rows, rowLength);
0066       QCOMPARE(buf.findRowOfPosition(idx, 0,  0), 0);
0067       QCOMPARE(buf.findRowOfPosition(idx, 0,  1), -1);
0068       QCOMPARE(buf.findRowOfPosition(idx, 0,  2), -1);
0069 
0070       rows = 2;
0071       rowLength = 1;
0072       initRowIndex(rows, rowLength);
0073       QCOMPARE(buf.findRowOfPosition(idx, 0,  0), 0);
0074       QCOMPARE(buf.findRowOfPosition(idx, 0,  1), 1);
0075       QCOMPARE(buf.findRowOfPosition(idx, 0,  2), -1);
0076 
0077       rows = 3;
0078       rowLength = 1;
0079       initRowIndex(rows, rowLength);
0080       QCOMPARE(buf.findRowOfPosition(idx, 0,  0), 0);
0081       QCOMPARE(buf.findRowOfPosition(idx, 0,  1), 1);
0082       QCOMPARE(buf.findRowOfPosition(idx, 0,  2), 2);
0083       QCOMPARE(buf.findRowOfPosition(idx, 0,  3), -1);
0084 
0085       rows = 1;
0086       rowLength = 2;
0087       initRowIndex(rows, rowLength);
0088       QCOMPARE(buf.findRowOfPosition(idx, 0,  0), 0);
0089       QCOMPARE(buf.findRowOfPosition(idx, 0,  1), 0);
0090       QCOMPARE(buf.findRowOfPosition(idx, 0,  2), -1);
0091       QCOMPARE(buf.findRowOfPosition(idx, 0,  3), -1);
0092     }
0093 
0094 
0095     void find_start_0()
0096     {
0097       int rows = 1;
0098       int rowLength = 100;
0099       initRowIndex(rows, rowLength);
0100 
0101       QCOMPARE(buf.findRowOfPosition(idx, 0,   0), 0);
0102       QCOMPARE(buf.findRowOfPosition(idx, 0,   1), 0);
0103       QCOMPARE(buf.findRowOfPosition(idx, 0,  99), 0);
0104       QCOMPARE(buf.findRowOfPosition(idx, 0, 100), -1);
0105       QCOMPARE(buf.findRowOfPosition(idx, 0, 101), -1);
0106       QCOMPARE(buf.findRowOfPosition(idx, 0, 199), -1);
0107 
0108       rows = 2;
0109       rowLength = 100;
0110       initRowIndex(rows, rowLength);
0111 
0112       QCOMPARE(buf.findRowOfPosition(idx, 0,   0), 0);
0113       QCOMPARE(buf.findRowOfPosition(idx, 0,   1), 0);
0114       QCOMPARE(buf.findRowOfPosition(idx, 0,  99), 0);
0115       QCOMPARE(buf.findRowOfPosition(idx, 0, 100), 1);
0116       QCOMPARE(buf.findRowOfPosition(idx, 0, 101), 1);
0117       QCOMPARE(buf.findRowOfPosition(idx, 0, 199), 1);
0118       QCOMPARE(buf.findRowOfPosition(idx, 0, 200), -1);
0119       QCOMPARE(buf.findRowOfPosition(idx, 0, 201), -1);
0120       QCOMPARE(buf.findRowOfPosition(idx, 0, 299), -1);
0121 
0122       rows = 3;
0123       rowLength = 100;
0124       initRowIndex(rows, rowLength);
0125 
0126       QCOMPARE(buf.findRowOfPosition(idx, 0,   0), 0);
0127       QCOMPARE(buf.findRowOfPosition(idx, 0,   1), 0);
0128       QCOMPARE(buf.findRowOfPosition(idx, 0,  99), 0);
0129       QCOMPARE(buf.findRowOfPosition(idx, 0, 100), 1);
0130       QCOMPARE(buf.findRowOfPosition(idx, 0, 101), 1);
0131       QCOMPARE(buf.findRowOfPosition(idx, 0, 199), 1);
0132       QCOMPARE(buf.findRowOfPosition(idx, 0, 200), 2);
0133       QCOMPARE(buf.findRowOfPosition(idx, 0, 201), 2);
0134       QCOMPARE(buf.findRowOfPosition(idx, 0, 299), 2);
0135       QCOMPARE(buf.findRowOfPosition(idx, 0, 300), -1);
0136       QCOMPARE(buf.findRowOfPosition(idx, 0, 301), -1);
0137       QCOMPARE(buf.findRowOfPosition(idx, 0, 399), -1);
0138     }
0139 
0140 
0141     void find_start_n()
0142     {
0143       int rows = 3;
0144       int rowLength = 100;
0145       initRowIndex(rows, rowLength);
0146 
0147       QCOMPARE(buf.findRowOfPosition(idx, 0, 100), 1);
0148       QCOMPARE(buf.findRowOfPosition(idx, 1, 100), 1);
0149 
0150       QCOMPARE(buf.findRowOfPosition(idx, 0, 101), 1);
0151       QCOMPARE(buf.findRowOfPosition(idx, 1, 101), 1);
0152 
0153     }
0154 
0155 
0156     // const QVector<AsciiFileData> splitFile(int chunkSize, const RowIndex& rowIndex, int start, int bytesToRead) const;
0157 
0158     void split_wrong_parameters()
0159     {
0160       int rows = 1;
0161       int rowLength = 1;
0162       initRowIndex(rows, rowLength);
0163       QCOMPARE(buf.splitFile(1, AsciiFileBuffer::RowIndex(), 0, 1).size(), 0);
0164       QCOMPARE(buf.splitFile(-1, idx, 0, 1).size(), 0);
0165       QCOMPARE(buf.splitFile(1, idx, -1, 1).size(), 0);
0166       QCOMPARE(buf.splitFile(1, idx, -1, 1).size(), 0);
0167       QCOMPARE(buf.splitFile(1, idx, 10, 1).size(), 0);
0168       QCOMPARE(buf.splitFile(1, idx,  0, 2).size(), 0);
0169     }
0170 
0171 
0172     void split_small()
0173     {
0174       int rows = 1;
0175       int rowLength = 1;
0176       initRowIndex(rows, rowLength);
0177       QVector<AsciiFileData> c = buf.splitFile(1, idx, 0, 1);
0178       QCOMPARE(c.size(), 1);
0179       QCOMPARE(c[0].begin(), 0);
0180       QCOMPARE(c[0].bytesRead(), 1);
0181       QCOMPARE(c[0].rowBegin(), 0);
0182       QCOMPARE(c[0].rowsRead(), 1);
0183 
0184       rows = 2;
0185       rowLength = 1;
0186       initRowIndex(rows, rowLength);
0187       c = buf.splitFile(1, idx, 0, 2);
0188       QCOMPARE(c.size(), 2);
0189       QCOMPARE(c[0].begin(), 0);
0190       QCOMPARE(c[0].bytesRead(), 1);
0191       QCOMPARE(c[0].rowBegin(), 0);
0192       QCOMPARE(c[0].rowsRead(), 1);
0193       QCOMPARE(c[1].begin(), 1);
0194       QCOMPARE(c[1].bytesRead(), 1);
0195       QCOMPARE(c[1].rowBegin(), 1);
0196       QCOMPARE(c[1].rowsRead(), 1);
0197 
0198       rows = 3;
0199       rowLength = 1;
0200       initRowIndex(rows, rowLength);
0201       c = buf.splitFile(1, idx, 0, 3);
0202       QCOMPARE(c.size(), 3);
0203       QCOMPARE(c[2].begin(), 2);
0204       QCOMPARE(c[2].bytesRead(), 1);
0205       QCOMPARE(c[2].rowBegin(), 2);
0206       QCOMPARE(c[2].rowsRead(), 1);
0207 
0208 
0209       rows = 1;
0210       rowLength = 2;
0211       initRowIndex(rows, rowLength);
0212       c = buf.splitFile(2, idx, 0, 2);
0213       QCOMPARE(c.size(), 1);
0214       QCOMPARE(c[0].begin(), 0);
0215       QCOMPARE(c[0].bytesRead(), 2);
0216       QCOMPARE(c[0].rowBegin(), 0);
0217       QCOMPARE(c[0].rowsRead(), 1);
0218 
0219       rows = 2;
0220       rowLength = 2;
0221       initRowIndex(rows, rowLength);
0222       c = buf.splitFile(2, idx, 0, 4);
0223       QCOMPARE(c.size(), 2);
0224       QCOMPARE(c[0].begin(), 0);
0225       QCOMPARE(c[0].bytesRead(), 2);
0226       QCOMPARE(c[0].rowBegin(), 0);
0227       QCOMPARE(c[0].rowsRead(), 1);
0228       QCOMPARE(c[1].begin(), 2);
0229       QCOMPARE(c[1].bytesRead(), 2);
0230       QCOMPARE(c[1].rowBegin(), 1);
0231       QCOMPARE(c[1].rowsRead(), 1);
0232 
0233       // chunk ends in the middle of a row
0234       rows = 2;
0235       rowLength = 3;
0236       initRowIndex(rows, rowLength);
0237       c = buf.splitFile(5, idx, 0, 6);
0238       QCOMPARE(c.size(), 2);
0239       QCOMPARE(c[0].begin(), 0);
0240       QCOMPARE(c[0].bytesRead(), 3);
0241       QCOMPARE(c[0].rowBegin(), 0);
0242       QCOMPARE(c[0].rowsRead(), 1);
0243       QCOMPARE(c[1].begin(), 3);
0244       QCOMPARE(c[1].bytesRead(), 3);
0245       QCOMPARE(c[1].rowBegin(), 1);
0246       QCOMPARE(c[1].rowsRead(), 1);
0247 
0248       rows = 1;
0249       rowLength = 2;
0250       initRowIndex(rows, rowLength);
0251       c = buf.splitFile(1, idx, 0, 2);
0252       QCOMPARE(c.size(), 0); // chunk size to small for one row
0253     }
0254 
0255 
0256     void split_into_1()
0257     {
0258       int rows = 3;
0259       int rowLength = 100;
0260       int bytes = rows * rowLength;
0261       initRowIndex(rows, rowLength);
0262       QVector<AsciiFileData> c = buf.splitFile(rows * rowLength, idx, 0, bytes);
0263       QCOMPARE(c[0].begin(), 0);
0264       QCOMPARE(c[0].bytesRead(), bytes);
0265 
0266       idx[0] = 10;
0267       bytes -= 10;
0268       c = buf.splitFile(rows * rowLength, idx, 10, bytes);
0269       QCOMPARE(c[0].begin(), 10);
0270       QCOMPARE(c[0].bytesRead(), bytes);
0271     }
0272 
0273 
0274     void split_into_3()
0275     {
0276       int rows = 3;
0277       int rowLength = 100;
0278       int bytes = rows * rowLength;
0279       initRowIndex(rows, rowLength);
0280       QVector<AsciiFileData> c = buf.splitFile(rowLength, idx, 0, bytes);
0281       QCOMPARE(c.size(), 3);
0282       QCOMPARE(c[0].begin(), idx[0]);
0283       QCOMPARE(c[1].begin(), idx[1]);
0284       QCOMPARE(c[2].begin(), idx[2]);
0285     }
0286 
0287 
0288     void split()
0289     {
0290       int rows = 999;
0291       int rowLength = 100;
0292       int bytes = rows * rowLength;
0293       initRowIndex(rows, rowLength);
0294       QVector<AsciiFileData> c = buf.splitFile(rowLength * 100, idx, 0, bytes);
0295       QCOMPARE(c.size(), 10);
0296       QCOMPARE(c[0].begin(), idx[0]);
0297       QCOMPARE(c[1].rowsRead(), 100);
0298       QCOMPARE(c[1].begin(), idx[100]);
0299       QCOMPARE(c[1].rowsRead(), 100);
0300       QCOMPARE(c[9].begin(), idx[900]);
0301       QCOMPARE(c[9].rowsRead(), 99);
0302 
0303       int rowsRead = 0;
0304       for (int i=0; i < 10; i++) {
0305         rowsRead += c[i].rowsRead();
0306       }
0307       QCOMPARE(rowsRead, rows);
0308     }
0309     
0310 
0311     void split_equal_chunkSize_is_bytesToRead()
0312     {
0313       int rows = 20;
0314       int rowLength = 10;
0315       int offset = 5;
0316       int bytesToRead = (rows - offset) * rowLength;
0317       initRowIndex(rows, rowLength);
0318       int start = offset * rowLength;
0319       QVector<AsciiFileData> c = buf.splitFile(bytesToRead, idx, start, bytesToRead);
0320       QCOMPARE(c.size(), 1);
0321       QCOMPARE(c[0].bytesRead(), bytesToRead);
0322       QCOMPARE(c[0].begin(), idx[offset]);
0323       QCOMPARE(c[0].rowsRead(), rows - offset);
0324     }
0325 
0326     //void useOneWindowWithChunks(const RowIndex& rowIndex, int start, int bytesToRead, int numChunks);
0327 
0328     void useOneWindowWithChunks_small()
0329     {
0330       int rows = 1;
0331       int rowLength = 1;
0332       initRowIndex(rows, rowLength);
0333       buf.useOneWindowWithChunks(idx, 0, 1, 1);
0334       QVector<QVector<AsciiFileData> > d = buf.fileData();
0335       QCOMPARE(d.size(), 1);
0336       QCOMPARE(d[0].size(), 1);
0337       
0338       rows = 2;
0339       rowLength = 1;
0340       initRowIndex(rows, rowLength);
0341       buf.useOneWindowWithChunks(idx, 0, 2, 1);
0342       d = buf.fileData();
0343       QCOMPARE(d.size(), 1);
0344       QCOMPARE(d[0].size(), 1); // only data for one chunk
0345 
0346       rows = 2;
0347       rowLength = 1;
0348       initRowIndex(rows, rowLength);
0349       buf.useOneWindowWithChunks(idx, 0, 2, 2);
0350       d = buf.fileData();
0351       QCOMPARE(d.size(), 1);
0352       QCOMPARE(d[0].size(), 2); 
0353 
0354       rows = 3;
0355       rowLength = 1;
0356       initRowIndex(rows, rowLength);
0357       buf.useOneWindowWithChunks(idx, 0, 3, 2);
0358       d = buf.fileData();
0359       QCOMPARE(d.size(), 1);
0360       QCOMPARE(d[0].size(), 3); // more than 2 chunks needed
0361     }
0362 
0363     void useOneWindowWithChunks()
0364     {
0365       int rows = 1000;
0366       int rowLength = 100;
0367       int bytes = rows * rowLength;
0368       initRowIndex(rows, rowLength);
0369       buf.useOneWindowWithChunks(idx, 0, bytes, 1);
0370       QVector<QVector<AsciiFileData> > d = buf.fileData();
0371       QCOMPARE(d.size(), 1);
0372       QCOMPARE(d[0].size(), 1);
0373 
0374       buf.useOneWindowWithChunks(idx, 0, bytes, 10);
0375       d = buf.fileData();
0376       QCOMPARE(d.size(), 1);
0377       QCOMPARE(d[0].size(), 10);
0378 
0379       buf.useOneWindowWithChunks(idx, 0, bytes, 1000);
0380       d = buf.fileData();
0381       QCOMPARE(d.size(), 1);
0382       QCOMPARE(d[0].size(), 1000);
0383 
0384       buf.useOneWindowWithChunks(idx, 0, bytes, 1001); // row too long for implizit chunk size
0385       d = buf.fileData();
0386       QCOMPARE(d.size(), 0);
0387     }
0388 
0389 
0390     // void useSlidingWindow(const RowIndex& rowIndex, int start, int bytesToRead, int windowSize);
0391 
0392     void useSlidingWindow_small()
0393     {
0394       int rows = 1;
0395       int rowLength = 1;
0396       initRowIndex(rows, rowLength);
0397       int windowSize = 1;
0398       buf.useSlidingWindow(idx, 0, 1, windowSize);
0399       QVector<QVector<AsciiFileData> > d = buf.fileData();
0400       QCOMPARE(d.size(), 1);
0401       QCOMPARE(d[0].size(), 1);
0402 
0403       rows = 2;
0404       rowLength = 1;
0405       initRowIndex(rows, rowLength);
0406       windowSize = 1;
0407       buf.useSlidingWindow(idx, 0, 2, windowSize);
0408       d = buf.fileData();
0409       QCOMPARE(d.size(), 2);
0410       QCOMPARE(d[0].size(), 1); 
0411       QCOMPARE(d[1].size(), 1); 
0412     }
0413 
0414 
0415     void useSlidingWindow()
0416     {
0417       int rows = 1000;
0418       int rowLength = 100;
0419       int bytes = rows * rowLength;
0420       initRowIndex(rows, rowLength);
0421       int windowSize = rowLength;
0422       buf.useSlidingWindow(idx, 0, bytes, windowSize);
0423       QVector<QVector<AsciiFileData> > d = buf.fileData();
0424       QCOMPARE(d.size(), 1000);
0425       QCOMPARE(d[0].size(), 1);
0426 
0427       rows = 999;
0428       rowLength = 100;
0429       bytes = rows * rowLength;
0430       windowSize = rowLength * 100;
0431       buf.useSlidingWindow(idx, 0, bytes, windowSize);
0432       d = buf.fileData();
0433       QCOMPARE(d.size(), 10);
0434       QCOMPARE(d[0].size(), 1);
0435       QCOMPARE(d[0][0].begin(), 0);
0436       QCOMPARE(d[0][0].bytesRead(), rowLength * 100);
0437       QCOMPARE(d[0][0].rowBegin(), 0);
0438       QCOMPARE(d[1].size(), 1);
0439       QCOMPARE(d[1][0].rowBegin(), 100);
0440 
0441       int rowsRead = 0;
0442       for (int i=0; i < 10; i++) {
0443         rowsRead += d[i][0].rowsRead();
0444       }
0445       QCOMPARE(rowsRead, rows);
0446     }
0447 
0448 
0449     // void useSlidingWindowWithChunks(const RowIndex& rowIndex, int start, int bytesToRead, int windowSize, int numWindowChunks)
0450     
0451     void useSlidingWindowWithChunks_small()
0452     {
0453       int rows = 1;
0454       int rowLength = 1;
0455       int bytes = rows * rowLength;
0456       initRowIndex(rows, rowLength);
0457       int windowSize = 1;
0458       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 1);
0459       QVector<QVector<AsciiFileData> > d = buf.fileData();
0460       QCOMPARE(d.size(), 1);
0461       QCOMPARE(d[0].size(), 1);
0462 
0463       rows = 3;
0464       rowLength = 1;
0465       bytes = rows * rowLength;
0466       initRowIndex(rows, rowLength);
0467       windowSize = 3;
0468       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 3);
0469       d = buf.fileData();
0470       QCOMPARE(d.size(), 1);
0471       QCOMPARE(d[0].size(), 3);
0472 
0473       rows = 3;
0474       rowLength = 1;
0475       bytes = rows * rowLength;
0476       initRowIndex(rows, rowLength);
0477       windowSize = 1;
0478       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 3); // int chunkSize = windowSize / numWindowChunks, == 0
0479       d = buf.fileData();
0480       QCOMPARE(d.size(), 0);
0481 
0482       rows = 12;
0483       rowLength = 1;
0484       bytes = rows * rowLength;
0485       initRowIndex(rows, rowLength);
0486       windowSize = 4;
0487       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 1);
0488       d = buf.fileData();
0489       QCOMPARE(d.size(), 3);
0490       QCOMPARE(d[0].size(), 1);
0491       QCOMPARE(d[2].size(), 1);
0492 
0493       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 2);
0494       d = buf.fileData();
0495       QCOMPARE(d.size(), 3);
0496       QCOMPARE(d[0].size(), 2);
0497       QCOMPARE(d[2].size(), 2);
0498 
0499       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 3);
0500       d = buf.fileData();
0501       QCOMPARE(d.size(), 4);  // could not split 4 rows into 3 chunks
0502       QCOMPARE(d[0].size(), 3);
0503       QCOMPARE(d[1].size(), 3);
0504       QCOMPARE(d[2].size(), 3);
0505       QCOMPARE(d[3].size(), 3);
0506     }
0507 
0508 
0509     void useSlidingWindowWithChunks()
0510     {
0511       int rows = 1000;
0512       int rowLength = 100;
0513       int bytes = rows * rowLength;
0514       initRowIndex(rows, rowLength);
0515       int windowSize = rowLength * 100;
0516       buf.useSlidingWindowWithChunks(idx, 0, bytes, windowSize, 5);
0517       QVector<QVector<AsciiFileData> > d = buf.fileData();
0518       QCOMPARE(d.size(), 10);
0519       QCOMPARE(d[0].size(), 5);
0520       QCOMPARE(d[9].size(), 5);
0521     }
0522 
0523 
0524 private:
0525     AsciiFileBuffer::RowIndex idx;
0526     AsciiFileBuffer buf;
0527     QFile file;
0528 
0529     void initRowIndex(int rows, int rowLength)
0530     {
0531       idx.clear();
0532       idx.resize(rows + 1);
0533       for (int i = 0; i < rows; i++) {
0534         idx[i] = i * rowLength;
0535       }
0536       idx[rows] = rows * rowLength;
0537     }
0538 };
0539 
0540 
0541 
0542 QTEST_MAIN(AsciiSourceTest)
0543 
0544 
0545 
0546 #include "moc_asciifilebuffertest.cpp"