import_export/HDF5/HDF5FilterTest.cpp

0001 /*
0002     File                 : HDF5FilterTest.cpp
0003     Project              : LabPlot
0004     Description          : Tests for the HDF5 I/O-filter.
0005     --------------------------------------------------------------------
0006     SPDX-FileCopyrightText: 2021-2022 Stefan Gerlach <stefan.gerlach@uni.kn>
0007     SPDX-License-Identifier: GPL-2.0-or-later
0008 */
0009
0010 #include "HDF5FilterTest.h"
0011 #include "backend/datasources/filters/HDF5Filter.h"
0012 #include "backend/lib/macros.h"
0013 #include "backend/matrix/Matrix.h"
0014 #include "backend/spreadsheet/Spreadsheet.h"
0015
0016 #include <KLocalizedString>
0017
0018 #include <gsl/gsl_randist.h>
0019 #include <gsl/gsl_rng.h>
0020 extern "C" {
0021 #include <hdf5.h>
0022 }
0023
0024 void HDF5FilterTest::testImportDouble() {
0025     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0026     HDF5Filter filter;
0027
0028     const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5"));
0029     filter.setCurrentDataSetName(QLatin1String("/arrays/2D float array"));
0030     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0031
0032     QCOMPARE(spreadsheet.columnCount(), 50);
0033     QCOMPARE(spreadsheet.rowCount(), 100);
0034     for (int i = 0; i < spreadsheet.columnCount(); i++) {
0035         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Double);
0036         QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D float array_") + QString::number(i + 1));
0037     }
0038
0039     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0040     for (int i = 1; i < spreadsheet.columnCount(); i++)
0041         QCOMPARE(spreadsheet.column(i)->plotDesignation(), AbstractColumn::PlotDesignation::Y);
0042
0043     for (int i = 0; i < 4; i++)
0044         for (int j = 0; j < 4; j++)
0045             DEBUG(std::setprecision(15) << spreadsheet.column(j)->valueAt(i))
0046
0047     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.000123456804431044);
0048     QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.001234499970451);
0049     QCOMPARE(spreadsheet.column(2)->valueAt(0), 0);
0050     QCOMPARE(spreadsheet.column(3)->valueAt(0), 0);
0051     QCOMPARE(spreadsheet.column(0)->valueAt(1), 0);
0052     QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.899999976158142);
0053     QCOMPARE(spreadsheet.column(2)->valueAt(1), 1.70000004768372);
0054     QCOMPARE(spreadsheet.column(3)->valueAt(1), 2.59999990463257);
0055     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0);
0056     QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.70000004768372);
0057     QCOMPARE(spreadsheet.column(2)->valueAt(2), 3.5);
0058     QCOMPARE(spreadsheet.column(3)->valueAt(2), 5.19999980926514);
0059     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0.170000001788139);
0060     QCOMPARE(spreadsheet.column(1)->valueAt(3), 2.59999990463257);
0061     QCOMPARE(spreadsheet.column(2)->valueAt(3), 3.5);
0062     QCOMPARE(spreadsheet.column(3)->valueAt(3), 7.80000019073486);
0063 }
0064
0065 void HDF5FilterTest::testImportDoublePortion() {
0066     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0067     HDF5Filter filter;
0068
0069     const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5"));
0070     filter.setCurrentDataSetName(QLatin1String("/arrays/2D float array"));
0071     // set start/end row/col
0072     filter.setStartRow(2);
0073     filter.setEndRow(3);
0074     filter.setStartColumn(2);
0075     filter.setEndColumn(3);
0076     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0077
0078     QCOMPARE(spreadsheet.columnCount(), 2);
0079     QCOMPARE(spreadsheet.rowCount(), 2);
0080     for (int i = 0; i < 2; i++) {
0081         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Double);
0082         QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D float array_") + QString::number(i + 1));
0083     }
0084
0085     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0086     QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y);
0087
0088     for (int i = 0; i < 2; i++)
0089         for (int j = 0; j < 2; j++)
0090             DEBUG(std::setprecision(15) << spreadsheet.column(j)->valueAt(i))
0091
0092     QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.899999976158142);
0093     QCOMPARE(spreadsheet.column(0)->valueAt(1), 1.70000004768372);
0094     QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.70000004768372);
0095     QCOMPARE(spreadsheet.column(1)->valueAt(1), 3.5);
0096 }
0097
0098 void HDF5FilterTest::testImportInt() {
0099     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0100     HDF5Filter filter;
0101
0102     const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5"));
0103     filter.setCurrentDataSetName(QLatin1String("/arrays/2D int array"));
0104     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0105
0106     QCOMPARE(spreadsheet.columnCount(), 50);
0107     QCOMPARE(spreadsheet.rowCount(), 100);
0108     for (int i = 0; i < spreadsheet.columnCount(); i++) {
0109         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer);
0110         QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D int array_") + QString::number(i + 1));
0111     }
0112
0113     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0114     for (int i = 1; i < spreadsheet.columnCount(); i++)
0115         QCOMPARE(spreadsheet.column(i)->plotDesignation(), AbstractColumn::PlotDesignation::Y);
0116
0117     QCOMPARE(spreadsheet.column(0)->valueAt(0), 1000);
0118     QCOMPARE(spreadsheet.column(1)->valueAt(0), 1001);
0119     QCOMPARE(spreadsheet.column(2)->valueAt(0), 1002);
0120     QCOMPARE(spreadsheet.column(3)->valueAt(0), 1003);
0121     QCOMPARE(spreadsheet.column(0)->valueAt(1), 1100);
0122     QCOMPARE(spreadsheet.column(1)->valueAt(1), 1101);
0123     QCOMPARE(spreadsheet.column(2)->valueAt(1), 1102);
0124     QCOMPARE(spreadsheet.column(3)->valueAt(1), 1103);
0125     QCOMPARE(spreadsheet.column(0)->valueAt(2), 1200);
0126     QCOMPARE(spreadsheet.column(1)->valueAt(2), 1207);
0127     QCOMPARE(spreadsheet.column(2)->valueAt(2), 1202);
0128     QCOMPARE(spreadsheet.column(3)->valueAt(2), 1203);
0129     QCOMPARE(spreadsheet.column(0)->valueAt(3), 1300);
0130     QCOMPARE(spreadsheet.column(1)->valueAt(3), 1301);
0131     QCOMPARE(spreadsheet.column(2)->valueAt(3), 1302);
0132     QCOMPARE(spreadsheet.column(3)->valueAt(3), 1303);
0133 }
0134
0135 void HDF5FilterTest::testImportIntPortion() {
0136     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0137     HDF5Filter filter;
0138
0139     const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5"));
0140     filter.setCurrentDataSetName(QLatin1String("/arrays/2D int array"));
0141     // set start/end row/col
0142     filter.setStartRow(2);
0143     filter.setEndRow(3);
0144     filter.setStartColumn(2);
0145     filter.setEndColumn(3);
0146     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0147
0148     QCOMPARE(spreadsheet.columnCount(), 2);
0149     QCOMPARE(spreadsheet.rowCount(), 2);
0150     for (int i = 0; i < 2; i++) {
0151         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer);
0152         QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D int array_") + QString::number(i + 1));
0153     }
0154
0155     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0156     QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y);
0157
0158     QCOMPARE(spreadsheet.column(0)->valueAt(0), 1101);
0159     QCOMPARE(spreadsheet.column(1)->valueAt(0), 1102);
0160     QCOMPARE(spreadsheet.column(0)->valueAt(1), 1207);
0161     QCOMPARE(spreadsheet.column(1)->valueAt(1), 1202);
0162 }
0163
0164 void HDF5FilterTest::testImportVLEN() {
0165     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0166     HDF5Filter filter;
0167
0168     const QString& fileName = QFINDTESTDATA(QLatin1String("data/h5ex_t_vlen.h5"));
0169     filter.setCurrentDataSetName(QLatin1String("/DS1"));
0170     // set start/end row/col
0171     // filter.setStartRow(2);
0172     // filter.setEndRow(3);
0173     // filter.setStartColumn(2);
0174     // filter.setEndColumn(3);
0175     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0176
0177     QCOMPARE(spreadsheet.columnCount(), 2);
0178     QCOMPARE(spreadsheet.rowCount(), 12);
0179     for (int i = 0; i < 2; i++) {
0180         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer);
0181         QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("DS1_") + QString::number(i + 1));
0182     }
0183
0184     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0185     QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y);
0186
0187     QCOMPARE(spreadsheet.column(0)->valueAt(0), 3);
0188     QCOMPARE(spreadsheet.column(1)->valueAt(0), 1);
0189     QCOMPARE(spreadsheet.column(0)->valueAt(1), 2);
0190     QCOMPARE(spreadsheet.column(1)->valueAt(1), 1);
0191     QCOMPARE(spreadsheet.column(0)->valueAt(2), 1);
0192     QCOMPARE(spreadsheet.column(1)->valueAt(2), 2);
0193     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0);
0194     QCOMPARE(spreadsheet.column(1)->valueAt(3), 3);
0195     QCOMPARE(spreadsheet.column(0)->valueAt(11), 0);
0196     QCOMPARE(spreadsheet.column(1)->valueAt(11), 144);
0197 }
0198 void HDF5FilterTest::testImportVLENPortion() {
0199     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0200     HDF5Filter filter;
0201
0202     const QString& fileName = QFINDTESTDATA(QLatin1String("data/h5ex_t_vlen.h5"));
0203     filter.setCurrentDataSetName(QLatin1String("/DS1"));
0204     // set start/end row/col
0205     filter.setStartRow(2);
0206     filter.setEndRow(5);
0207     filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0208
0209     QCOMPARE(spreadsheet.columnCount(), 2);
0210     QCOMPARE(spreadsheet.rowCount(), 4);
0211     for (int i = 0; i < 2; i++) {
0212         QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer);
0213         QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("DS1_") + QString::number(i + 1));
0214     }
0215
0216     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0217     QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y);
0218
0219     QCOMPARE(spreadsheet.column(0)->valueAt(0), 2);
0220     QCOMPARE(spreadsheet.column(1)->valueAt(0), 1);
0221     QCOMPARE(spreadsheet.column(0)->valueAt(1), 1);
0222     QCOMPARE(spreadsheet.column(1)->valueAt(1), 2);
0223     QCOMPARE(spreadsheet.column(0)->valueAt(2), 0);
0224     QCOMPARE(spreadsheet.column(1)->valueAt(2), 3);
0225     QCOMPARE(spreadsheet.column(0)->valueAt(3), 0);
0226     QCOMPARE(spreadsheet.column(1)->valueAt(3), 5);
0227
0228     // first column
0229     HDF5Filter filter2;
0230     filter2.setCurrentDataSetName(QLatin1String("/DS1"));
0231     filter2.setStartRow(2);
0232     filter2.setEndRow(5);
0233     filter2.setEndColumn(1);
0234     filter2.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0235
0236     QCOMPARE(spreadsheet.columnCount(), 1);
0237     QCOMPARE(spreadsheet.rowCount(), 2);
0238     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer);
0239     QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("DS1_1"));
0240
0241     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0242
0243     QCOMPARE(spreadsheet.column(0)->valueAt(0), 2);
0244     QCOMPARE(spreadsheet.column(0)->valueAt(1), 1);
0245
0246     // second column
0247     HDF5Filter filter3;
0248     filter3.setCurrentDataSetName(QLatin1String("/DS1"));
0249     filter3.setStartRow(2);
0250     filter3.setEndRow(5);
0251     filter3.setStartColumn(2);
0252     filter3.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0253
0254     QCOMPARE(spreadsheet.columnCount(), 1);
0255     QCOMPARE(spreadsheet.rowCount(), 4);
0256     QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer);
0257     QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("DS1_2"));
0258
0259     QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X);
0260
0261     QCOMPARE(spreadsheet.column(0)->valueAt(0), 1);
0262     QCOMPARE(spreadsheet.column(0)->valueAt(1), 2);
0263     QCOMPARE(spreadsheet.column(0)->valueAt(2), 3);
0264     QCOMPARE(spreadsheet.column(0)->valueAt(3), 5);
0265 }
0266
0267 // BENCHMARKS
0268
0269 void HDF5FilterTest::benchDoubleImport_data() {
0270     QTest::addColumn<size_t>("lineCount");
0271     // can't transfer file name since needed in clean up
0272
0273     QTemporaryFile file;
0274     if (!file.open()) // needed to generate file name
0275         return;
0276     file.close(); // only file name is used
0277
0278     benchDataFileName = file.fileName();
0279     benchDataFileName.append(QStringLiteral(".h5"));
0280
0281     QString testName(QString::number(paths) + QLatin1String(" random double paths"));
0282
0283     QTest::newRow(qPrintable(testName)) << lines;
0284     DEBUG("CREATE DATA FILE " << STDSTRING(benchDataFileName) << ", lines = " << lines)
0285
0286     gsl_rng_env_setup();
0287     gsl_rng* r = gsl_rng_alloc(gsl_rng_default);
0288     gsl_rng_set(r, 12345);
0289
0290     // create file
0291     // see https://support.hdfgroup.org/HDF5/Tutor/introductory.html
0292     hid_t file_id = H5Fcreate(qPrintable(benchDataFileName), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
0293
0294     /* Create the data space for the dataset. */
0295     hsize_t dims[2];
0296     dims[0] = lines;
0297     dims[1] = paths;
0298     hid_t dataspace_id = H5Screate_simple(2, dims, nullptr);
0299
0300     /* Create the dataset. */
0301     hid_t dataset_id = H5Dcreate(file_id, "/data", H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
0302
0303     // create data
0304     double path[paths] = {0.0};
0305     double* data = new double[paths * lines];
0306
0307     const double delta = 0.25;
0308     const int dt = 1;
0309     const double sigma = delta * delta * dt;
0310     for (size_t i = 0; i < lines; ++i) {
0311         // std::cout << "line " << i+1 << std::endl;
0312
0313         for (int p = 0; p < paths; ++p) {
0314             path[p] += gsl_ran_gaussian_ziggurat(r, sigma);
0315             data[p + i * paths] = path[p];
0316         }
0317     }
0318
0319     herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
0320     if (status < 0)
0321         WARN(Q_FUNC_INFO << ", ERROR writing data")
0322     status = H5Dclose(dataset_id);
0323     status = H5Sclose(dataspace_id);
0324     status = H5Fclose(file_id);
0325
0326     delete[] data;
0327
0328     DEBUG(Q_FUNC_INFO << ", DONE")
0329 }
0330
0331 void HDF5FilterTest::benchDoubleImport() {
0332     Spreadsheet spreadsheet(QStringLiteral("test"), false);
0333     HDF5Filter filter;
0334     filter.setCurrentDataSetName(QLatin1String("/data"));
0335
0336     const int p = paths; // need local variable
0337     QBENCHMARK {
0338         filter.readDataFromFile(benchDataFileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace);
0339
0340         QCOMPARE(spreadsheet.columnCount(), p);
0341         QCOMPARE(spreadsheet.rowCount(), lines);
0342
0343         QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.120997813055);
0344         QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.119301077563219);
0345         QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.0209979608555485);
0346     }
0347 }
0348
0349 void HDF5FilterTest::benchDoubleImport_cleanup() {
0350     DEBUG("REMOVE DATA FILE " << STDSTRING(benchDataFileName))
0351     QFile::remove(benchDataFileName);
0352 }
0353
0354 QTEST_MAIN(HDF5FilterTest)