import_export/Binary/BinaryFilterTest.cpp

0001 /*
0002     File                 : BinaryFilterTest.cpp
0003     Project              : LabPlot
0004     Description          : Tests for the binary filter
0005     --------------------------------------------------------------------
0006     SPDX-FileCopyrightText: 2022 Stefan Gerlach <stefan.gerlach@uni.kn>
0007
0008     SPDX-License-Identifier: GPL-2.0-or-later
0009 */
0010
0011 #include "BinaryFilterTest.h"
0012 #include "backend/datasources/filters/BinaryFilter.h"
0013 #include "backend/spreadsheet/Spreadsheet.h"
0014
0015 #include <gsl/gsl_randist.h>
0016 #include <gsl/gsl_rng.h>
0017
0018 void BinaryFilterTest::importInt8() {
0019     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0020     BinaryFilter filter;
0021     const QString& fileName = QFINDTESTDATA(QLatin1String("data/int8.bin"));
0022     filter.setDataType(BinaryFilter::DataType::INT8);
0023     // filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0024     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0025
0026     // spreadsheet size
0027     QCOMPARE(spreadsheet.columnCount(), 2);
0028     QCOMPARE(spreadsheet.rowCount(), 1000);
0029
0030     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer);
0031     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer);
0032
0033     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0);
0034     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0);
0035     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0);
0036     QCOMPARE(spreadsheet.column(1)->valueAt(1), 9);
0037     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0);
0038     QCOMPARE(spreadsheet.column(1)->valueAt(2), 19);
0039     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0);
0040     QCOMPARE(spreadsheet.column(1)->valueAt(3), 29);
0041     QCOMPARE(spreadsheet.column(0)->valueAt(4), 0);
0042     QCOMPARE(spreadsheet.column(1)->valueAt(4), 38);
0043     QCOMPARE(spreadsheet.column(0)->valueAt(998), 99);
0044     QCOMPARE(spreadsheet.column(1)->valueAt(998), -59);
0045     QCOMPARE(spreadsheet.column(0)->valueAt(999), 100);
0046     QCOMPARE(spreadsheet.column(1)->valueAt(999), -50);
0047 }
0048
0049 void BinaryFilterTest::importInt16BE() {
0050     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0051     BinaryFilter filter;
0052     const QString& fileName = QFINDTESTDATA(QLatin1String("data/int16.bin"));
0053     filter.setDataType(BinaryFilter::DataType::INT16);
0054     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0055     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0056
0057     // spreadsheet size
0058     QCOMPARE(spreadsheet.columnCount(), 2);
0059     QCOMPARE(spreadsheet.rowCount(), 1000);
0060
0061     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer);
0062     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer);
0063
0064     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0);
0065     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0);
0066     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0);
0067     QCOMPARE(spreadsheet.column(1)->valueAt(1), 9);
0068     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0);
0069     QCOMPARE(spreadsheet.column(1)->valueAt(2), 19);
0070     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0);
0071     QCOMPARE(spreadsheet.column(1)->valueAt(3), 29);
0072     QCOMPARE(spreadsheet.column(0)->valueAt(4), 0);
0073     QCOMPARE(spreadsheet.column(1)->valueAt(4), 38);
0074     QCOMPARE(spreadsheet.column(0)->valueAt(998), 99);
0075     QCOMPARE(spreadsheet.column(1)->valueAt(998), -59);
0076     QCOMPARE(spreadsheet.column(0)->valueAt(999), 100);
0077     QCOMPARE(spreadsheet.column(1)->valueAt(999), -50);
0078 }
0079
0080 void BinaryFilterTest::importInt32LE() {
0081     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0082     BinaryFilter filter;
0083     const QString& fileName = QFINDTESTDATA(QLatin1String("data/int32LE.bin"));
0084     filter.setDataType(BinaryFilter::DataType::INT32);
0085     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0086
0087     // spreadsheet size
0088     QCOMPARE(spreadsheet.columnCount(), 2);
0089     QCOMPARE(spreadsheet.rowCount(), 700);
0090
0091     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer);
0092     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer);
0093
0094     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0);
0095     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0);
0096     QCOMPARE(spreadsheet.column(0)->valueAt(1), 1);
0097     QCOMPARE(spreadsheet.column(1)->valueAt(1), 0);
0098     QCOMPARE(spreadsheet.column(0)->valueAt(2), 2);
0099     QCOMPARE(spreadsheet.column(1)->valueAt(2), 1);
0100     QCOMPARE(spreadsheet.column(0)->valueAt(3), 3);
0101     QCOMPARE(spreadsheet.column(1)->valueAt(3), 2);
0102     QCOMPARE(spreadsheet.column(0)->valueAt(4), 4);
0103     QCOMPARE(spreadsheet.column(1)->valueAt(4), 3);
0104     QCOMPARE(spreadsheet.column(0)->valueAt(698), 698);
0105     QCOMPARE(spreadsheet.column(1)->valueAt(698), 64);
0106     QCOMPARE(spreadsheet.column(0)->valueAt(699), 699);
0107     QCOMPARE(spreadsheet.column(1)->valueAt(699), 64);
0108 }
0109
0110 void BinaryFilterTest::importInt32BE() {
0111     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0112     BinaryFilter filter;
0113     const QString& fileName = QFINDTESTDATA(QLatin1String("data/int32.bin"));
0114     filter.setDataType(BinaryFilter::DataType::INT32);
0115     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0116     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0117
0118     // spreadsheet size
0119     QCOMPARE(spreadsheet.columnCount(), 2);
0120     QCOMPARE(spreadsheet.rowCount(), 1000);
0121
0122     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer);
0123     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer);
0124
0125     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0);
0126     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0);
0127     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0);
0128     QCOMPARE(spreadsheet.column(1)->valueAt(1), 9);
0129     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0);
0130     QCOMPARE(spreadsheet.column(1)->valueAt(2), 19);
0131     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0);
0132     QCOMPARE(spreadsheet.column(1)->valueAt(3), 29);
0133     QCOMPARE(spreadsheet.column(0)->valueAt(4), 0);
0134     QCOMPARE(spreadsheet.column(1)->valueAt(4), 38);
0135     QCOMPARE(spreadsheet.column(0)->valueAt(998), 99);
0136     QCOMPARE(spreadsheet.column(1)->valueAt(998), -59);
0137     QCOMPARE(spreadsheet.column(0)->valueAt(999), 100);
0138     QCOMPARE(spreadsheet.column(1)->valueAt(999), -50);
0139 }
0140
0141 void BinaryFilterTest::importInt64BE() {
0142     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0143     BinaryFilter filter;
0144     const QString& fileName = QFINDTESTDATA(QLatin1String("data/int64.bin"));
0145     filter.setDataType(BinaryFilter::DataType::INT64);
0146     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0147     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0148
0149     // spreadsheet size
0150     QCOMPARE(spreadsheet.columnCount(), 2);
0151     QCOMPARE(spreadsheet.rowCount(), 1000);
0152
0153     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::BigInt);
0154     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::BigInt);
0155
0156     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0);
0157     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0);
0158     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0);
0159     QCOMPARE(spreadsheet.column(1)->valueAt(1), 9);
0160     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0);
0161     QCOMPARE(spreadsheet.column(1)->valueAt(2), 19);
0162     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0);
0163     QCOMPARE(spreadsheet.column(1)->valueAt(3), 29);
0164     QCOMPARE(spreadsheet.column(0)->valueAt(4), 0);
0165     QCOMPARE(spreadsheet.column(1)->valueAt(4), 38);
0166     QCOMPARE(spreadsheet.column(0)->valueAt(998), 99);
0167     QCOMPARE(spreadsheet.column(1)->valueAt(998), -59);
0168     QCOMPARE(spreadsheet.column(0)->valueAt(999), 100);
0169     QCOMPARE(spreadsheet.column(1)->valueAt(999), -50);
0170 }
0171
0172 void BinaryFilterTest::importFloatBE() {
0173     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0174     BinaryFilter filter;
0175     const QString& fileName = QFINDTESTDATA(QLatin1String("data/float.bin"));
0176     filter.setDataType(BinaryFilter::DataType::REAL32);
0177     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0178     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0179
0180     // spreadsheet size
0181     QCOMPARE(spreadsheet.columnCount(), 2);
0182     QCOMPARE(spreadsheet.rowCount(), 1000);
0183
0184     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double);
0185     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double);
0186
0187     // DEBUG(Q_FUNC_INFO << ", value = " << spreadsheet.column(0)->valueAt(999))
0188     // DEBUG(Q_FUNC_INFO << ", value = " << spreadsheet.column(1)->valueAt(999))
0189     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.0);
0190     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.0);
0191     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0.100100100040436);
0192     QCOMPARE(spreadsheet.column(1)->valueAt(1), 9.9932918548584);
0193     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0.200200200080872);
0194     QCOMPARE(spreadsheet.column(1)->valueAt(2), 19.8865337371826);
0195     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0.300300300121307);
0196     QCOMPARE(spreadsheet.column(1)->valueAt(3), 29.5806789398193);
0197     QCOMPARE(spreadsheet.column(0)->valueAt(4), 0.400400400161743);
0198     QCOMPARE(spreadsheet.column(1)->valueAt(4), 38.9786720275879);
0199     QCOMPARE(spreadsheet.column(0)->valueAt(998), 99.89990234375);
0200     QCOMPARE(spreadsheet.column(1)->valueAt(998), -59.0004920959473);
0201     QCOMPARE(spreadsheet.column(0)->valueAt(999), 100);
0202     QCOMPARE(spreadsheet.column(1)->valueAt(999), -50.6365623474121);
0203 }
0204
0205 void BinaryFilterTest::importDoubleBE() {
0206     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0207     BinaryFilter filter;
0208     const QString& fileName = QFINDTESTDATA(QLatin1String("data/double.bin"));
0209     filter.setDataType(BinaryFilter::DataType::REAL64);
0210     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0211     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0212
0213     // spreadsheet size
0214     QCOMPARE(spreadsheet.columnCount(), 2);
0215     QCOMPARE(spreadsheet.rowCount(), 1000);
0216
0217     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double);
0218     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double);
0219
0220     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.0);
0221     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.0);
0222     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0.1001001001001);
0223     QCOMPARE(spreadsheet.column(1)->valueAt(1), 9.9932916564023);
0224     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0.2002002002002);
0225     QCOMPARE(spreadsheet.column(1)->valueAt(2), 19.8865340135936);
0226     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0.3003003003003);
0227     QCOMPARE(spreadsheet.column(1)->valueAt(3), 29.5806794305369);
0228     QCOMPARE(spreadsheet.column(0)->valueAt(4), 0.4004004004004);
0229     QCOMPARE(spreadsheet.column(1)->valueAt(4), 38.978673554296);
0230     QCOMPARE(spreadsheet.column(0)->valueAt(998), 99.8998998998999);
0231     QCOMPARE(spreadsheet.column(1)->valueAt(998), -59.000490423342);
0232     QCOMPARE(spreadsheet.column(0)->valueAt(999), 100);
0233     QCOMPARE(spreadsheet.column(1)->valueAt(999), -50.6365641109759);
0234 }
0235
0236 void BinaryFilterTest::importDoubleLE() {
0237     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0238     BinaryFilter filter;
0239     const QString& fileName = QFINDTESTDATA(QLatin1String("data/doubleLE.bin"));
0240     filter.setDataType(BinaryFilter::DataType::REAL64);
0241     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0242
0243     // spreadsheet size
0244     QCOMPARE(spreadsheet.columnCount(), 2);
0245     QCOMPARE(spreadsheet.rowCount(), 700);
0246
0247     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double);
0248     QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double);
0249
0250     for (int i = 0; i < 700; i++)
0251         QCOMPARE(spreadsheet.column(0)->valueAt(i), i * 0.01);
0252     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.0);
0253     QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.999983333416666);
0254     QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.99986666933331);
0255     QCOMPARE(spreadsheet.column(1)->valueAt(3), 2.99955002024957);
0256     QCOMPARE(spreadsheet.column(1)->valueAt(4), 3.99893341866342);
0257     QCOMPARE(spreadsheet.column(1)->valueAt(698), 64.1778165874934);
0258     QCOMPARE(spreadsheet.column(1)->valueAt(699), 64.9414852768911);
0259 }
0260
0261 void BinaryFilterTest::importDoubleMatrixBE() {
0262     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0263     BinaryFilter filter;
0264     const QString& fileName = QFINDTESTDATA(QLatin1String("data/matrix_double.bin"));
0265     filter.setDataType(BinaryFilter::DataType::REAL64);
0266     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0267     filter.setVectors(40);
0268     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0269
0270     // spreadsheet size
0271     QCOMPARE(spreadsheet.columnCount(), 40);
0272     QCOMPARE(spreadsheet.rowCount(), 40);
0273
0274     for (int i = 0; i < 40; i++)
0275         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Double);
0276
0277     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.0);
0278     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.0256381904333);
0279     QCOMPARE(spreadsheet.column(2)->valueAt(0), 0.0512595256828);
0280     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0.0256381904333);
0281     QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.0512595256828);
0282     QCOMPARE(spreadsheet.column(2)->valueAt(1), 0.0768471616458);
0283     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0.0512595256828);
0284     QCOMPARE(spreadsheet.column(1)->valueAt(2), 0.0768471616458);
0285     QCOMPARE(spreadsheet.column(2)->valueAt(2), 0.102384276374254);
0286     QCOMPARE(spreadsheet.column(38)->valueAt(38), 0.929433523316);
0287     QCOMPARE(spreadsheet.column(38)->valueAt(39), 0.919667781302);
0288     QCOMPARE(spreadsheet.column(39)->valueAt(38), 0.91966778130227);
0289     QCOMPARE(spreadsheet.column(39)->valueAt(39), 0.909297426825682);
0290 }
0291
0292 /////////////////////////////////////////////////////////////////
0293
0294 // INT data
0295
0296 void BinaryFilterTest::benchIntImport_data() {
0297     QTest::addColumn<size_t>("lineCount");
0298     // can't transfer file name since needed in clean up
0299
0300     // create data
0301     QTemporaryFile file;
0302     if (!file.open())
0303         return;
0304
0305     file.setAutoRemove(false);
0306     benchDataFileName = file.fileName();
0307
0308     QDataStream out(&file);
0309
0310     QTest::newRow("3 int cols");
0311     DEBUG("CREATE DATA FILE " << STDSTRING(benchDataFileName) << ", lines = " << lines)
0312     for (size_t i = 0; i < lines; i++)
0313         out << static_cast<int>(i) << static_cast<int>(100 * sin(i / 100.)) << static_cast<int>(100 * cos(i / 100));
0314     file.close();
0315 }
0316
0317 void BinaryFilterTest::benchIntImport() {
0318     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0319     BinaryFilter filter;
0320     filter.setDataType(BinaryFilter::DataType::INT32);
0321     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0322     filter.setVectors(3);
0323
0324     QBENCHMARK {
0325         filter.readDataFromFile(benchDataFileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0326
0327         QCOMPARE(spreadsheet.columnCount(), 3);
0328         QCOMPARE(spreadsheet.rowCount(), lines);
0329
0330         QCOMPARE(spreadsheet.column(0)->valueAt(0), 0);
0331         QCOMPARE(spreadsheet.column(1)->valueAt(0), 0);
0332         QCOMPARE(spreadsheet.column(2)->valueAt(0), 100);
0333     }
0334 }
0335
0336 void BinaryFilterTest::benchIntImport_cleanup() {
0337     DEBUG("REMOVE DATA FILE " << STDSTRING(benchDataFileName))
0338     QFile::remove(benchDataFileName);
0339 }
0340
0341 // DOUBLE data
0342
0343 void BinaryFilterTest::benchDoubleImport_data() {
0344     QTest::addColumn<size_t>("lineCount");
0345     // can't transfer file name since needed in clean up
0346
0347     // create data
0348     QTemporaryFile file;
0349     if (!file.open())
0350         return;
0351
0352     file.setAutoRemove(false);
0353     benchDataFileName = file.fileName();
0354
0355     gsl_rng_env_setup();
0356     gsl_rng* r = gsl_rng_alloc(gsl_rng_default);
0357     gsl_rng_set(r, 12345);
0358
0359     QDataStream out(&file);
0360
0361     QString testName(QString::number(paths) + QLatin1String(" random double paths"));
0362     QTest::newRow(qPrintable(testName)) << lines;
0363     DEBUG("CREATE DATA FILE " << STDSTRING(benchDataFileName) << ", lines = " << lines)
0364
0365     const double delta = 0.25;
0366     const int dt = 1;
0367     const double sigma = delta * delta * dt;
0368     double path[paths] = {0.0};
0369     for (size_t i = 0; i < lines; i++) {
0370         for (int p = 0; p < paths; p++) {
0371             path[p] += gsl_ran_gaussian_ziggurat(r, sigma);
0372             out << path[p];
0373         }
0374     }
0375     file.close();
0376 }
0377
0378 void BinaryFilterTest::benchDoubleImport() {
0379     QFETCH(size_t, lineCount);
0380
0381     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0382     BinaryFilter filter;
0383     filter.setDataType(BinaryFilter::DataType::REAL64);
0384     filter.setByteOrder(QDataStream::ByteOrder::BigEndian);
0385     filter.setVectors(paths);
0386
0387     const int p = paths; // need local variable
0388     QBENCHMARK {
0389         filter.readDataFromFile(benchDataFileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0390
0391         QCOMPARE(spreadsheet.columnCount(), p);
0392         QCOMPARE(spreadsheet.rowCount(), lineCount);
0393
0394         QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.120997813055);
0395         QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.119301077563219);
0396         QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.0209979608555485);
0397     }
0398 }
0399
0400 void BinaryFilterTest::benchDoubleImport_cleanup() {
0401     DEBUG("REMOVE DATA FILE " << STDSTRING(benchDataFileName))
0402     QFile::remove(benchDataFileName);
0403 }
0404
0405 ///////////////////////////////////////////////////////
0406
0407 QTEST_MAIN(BinaryFilterTest)