File indexing completed on 2024-04-28 03:48:15
0001 /* 0002 File : HDF5FilterTest.cpp 0003 Project : LabPlot 0004 Description : Tests for the HDF5 I/O-filter. 0005 -------------------------------------------------------------------- 0006 SPDX-FileCopyrightText: 2021-2022 Stefan Gerlach <stefan.gerlach@uni.kn> 0007 SPDX-License-Identifier: GPL-2.0-or-later 0008 */ 0009 0010 #include "HDF5FilterTest.h" 0011 #include "backend/datasources/filters/HDF5Filter.h" 0012 #include "backend/lib/macros.h" 0013 #include "backend/matrix/Matrix.h" 0014 #include "backend/spreadsheet/Spreadsheet.h" 0015 0016 #include <KLocalizedString> 0017 0018 #include <gsl/gsl_randist.h> 0019 #include <gsl/gsl_rng.h> 0020 extern "C" { 0021 #include <hdf5.h> 0022 } 0023 0024 void HDF5FilterTest::testImportDouble() { 0025 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0026 HDF5Filter filter; 0027 0028 const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5")); 0029 filter.setCurrentDataSetName(QLatin1String("/arrays/2D float array")); 0030 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0031 0032 QCOMPARE(spreadsheet.columnCount(), 50); 0033 QCOMPARE(spreadsheet.rowCount(), 100); 0034 for (int i = 0; i < spreadsheet.columnCount(); i++) { 0035 QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Double); 0036 QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D float array_") + QString::number(i + 1)); 0037 } 0038 0039 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0040 for (int i = 1; i < spreadsheet.columnCount(); i++) 0041 QCOMPARE(spreadsheet.column(i)->plotDesignation(), AbstractColumn::PlotDesignation::Y); 0042 0043 for (int i = 0; i < 4; i++) 0044 for (int j = 0; j < 4; j++) 0045 DEBUG(std::setprecision(15) << spreadsheet.column(j)->valueAt(i)) 0046 0047 QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.000123456804431044); 0048 QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.001234499970451); 0049 QCOMPARE(spreadsheet.column(2)->valueAt(0), 0); 0050 QCOMPARE(spreadsheet.column(3)->valueAt(0), 0); 0051 QCOMPARE(spreadsheet.column(0)->valueAt(1), 0); 0052 QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.899999976158142); 0053 QCOMPARE(spreadsheet.column(2)->valueAt(1), 1.70000004768372); 0054 QCOMPARE(spreadsheet.column(3)->valueAt(1), 2.59999990463257); 0055 QCOMPARE(spreadsheet.column(0)->valueAt(2), 0); 0056 QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.70000004768372); 0057 QCOMPARE(spreadsheet.column(2)->valueAt(2), 3.5); 0058 QCOMPARE(spreadsheet.column(3)->valueAt(2), 5.19999980926514); 0059 QCOMPARE(spreadsheet.column(0)->valueAt(3), 0.170000001788139); 0060 QCOMPARE(spreadsheet.column(1)->valueAt(3), 2.59999990463257); 0061 QCOMPARE(spreadsheet.column(2)->valueAt(3), 3.5); 0062 QCOMPARE(spreadsheet.column(3)->valueAt(3), 7.80000019073486); 0063 } 0064 0065 void HDF5FilterTest::testImportDoublePortion() { 0066 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0067 HDF5Filter filter; 0068 0069 const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5")); 0070 filter.setCurrentDataSetName(QLatin1String("/arrays/2D float array")); 0071 // set start/end row/col 0072 filter.setStartRow(2); 0073 filter.setEndRow(3); 0074 filter.setStartColumn(2); 0075 filter.setEndColumn(3); 0076 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0077 0078 QCOMPARE(spreadsheet.columnCount(), 2); 0079 QCOMPARE(spreadsheet.rowCount(), 2); 0080 for (int i = 0; i < 2; i++) { 0081 QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Double); 0082 QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D float array_") + QString::number(i + 1)); 0083 } 0084 0085 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0086 QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y); 0087 0088 for (int i = 0; i < 2; i++) 0089 for (int j = 0; j < 2; j++) 0090 DEBUG(std::setprecision(15) << spreadsheet.column(j)->valueAt(i)) 0091 0092 QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.899999976158142); 0093 QCOMPARE(spreadsheet.column(0)->valueAt(1), 1.70000004768372); 0094 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.70000004768372); 0095 QCOMPARE(spreadsheet.column(1)->valueAt(1), 3.5); 0096 } 0097 0098 void HDF5FilterTest::testImportInt() { 0099 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0100 HDF5Filter filter; 0101 0102 const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5")); 0103 filter.setCurrentDataSetName(QLatin1String("/arrays/2D int array")); 0104 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0105 0106 QCOMPARE(spreadsheet.columnCount(), 50); 0107 QCOMPARE(spreadsheet.rowCount(), 100); 0108 for (int i = 0; i < spreadsheet.columnCount(); i++) { 0109 QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer); 0110 QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D int array_") + QString::number(i + 1)); 0111 } 0112 0113 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0114 for (int i = 1; i < spreadsheet.columnCount(); i++) 0115 QCOMPARE(spreadsheet.column(i)->plotDesignation(), AbstractColumn::PlotDesignation::Y); 0116 0117 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1000); 0118 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1001); 0119 QCOMPARE(spreadsheet.column(2)->valueAt(0), 1002); 0120 QCOMPARE(spreadsheet.column(3)->valueAt(0), 1003); 0121 QCOMPARE(spreadsheet.column(0)->valueAt(1), 1100); 0122 QCOMPARE(spreadsheet.column(1)->valueAt(1), 1101); 0123 QCOMPARE(spreadsheet.column(2)->valueAt(1), 1102); 0124 QCOMPARE(spreadsheet.column(3)->valueAt(1), 1103); 0125 QCOMPARE(spreadsheet.column(0)->valueAt(2), 1200); 0126 QCOMPARE(spreadsheet.column(1)->valueAt(2), 1207); 0127 QCOMPARE(spreadsheet.column(2)->valueAt(2), 1202); 0128 QCOMPARE(spreadsheet.column(3)->valueAt(2), 1203); 0129 QCOMPARE(spreadsheet.column(0)->valueAt(3), 1300); 0130 QCOMPARE(spreadsheet.column(1)->valueAt(3), 1301); 0131 QCOMPARE(spreadsheet.column(2)->valueAt(3), 1302); 0132 QCOMPARE(spreadsheet.column(3)->valueAt(3), 1303); 0133 } 0134 0135 void HDF5FilterTest::testImportIntPortion() { 0136 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0137 HDF5Filter filter; 0138 0139 const QString& fileName = QFINDTESTDATA(QLatin1String("data/hdf5_test.h5")); 0140 filter.setCurrentDataSetName(QLatin1String("/arrays/2D int array")); 0141 // set start/end row/col 0142 filter.setStartRow(2); 0143 filter.setEndRow(3); 0144 filter.setStartColumn(2); 0145 filter.setEndColumn(3); 0146 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0147 0148 QCOMPARE(spreadsheet.columnCount(), 2); 0149 QCOMPARE(spreadsheet.rowCount(), 2); 0150 for (int i = 0; i < 2; i++) { 0151 QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer); 0152 QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("2D int array_") + QString::number(i + 1)); 0153 } 0154 0155 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0156 QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y); 0157 0158 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1101); 0159 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1102); 0160 QCOMPARE(spreadsheet.column(0)->valueAt(1), 1207); 0161 QCOMPARE(spreadsheet.column(1)->valueAt(1), 1202); 0162 } 0163 0164 void HDF5FilterTest::testImportVLEN() { 0165 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0166 HDF5Filter filter; 0167 0168 const QString& fileName = QFINDTESTDATA(QLatin1String("data/h5ex_t_vlen.h5")); 0169 filter.setCurrentDataSetName(QLatin1String("/DS1")); 0170 // set start/end row/col 0171 // filter.setStartRow(2); 0172 // filter.setEndRow(3); 0173 // filter.setStartColumn(2); 0174 // filter.setEndColumn(3); 0175 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0176 0177 QCOMPARE(spreadsheet.columnCount(), 2); 0178 QCOMPARE(spreadsheet.rowCount(), 12); 0179 for (int i = 0; i < 2; i++) { 0180 QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer); 0181 QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("DS1_") + QString::number(i + 1)); 0182 } 0183 0184 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0185 QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y); 0186 0187 QCOMPARE(spreadsheet.column(0)->valueAt(0), 3); 0188 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1); 0189 QCOMPARE(spreadsheet.column(0)->valueAt(1), 2); 0190 QCOMPARE(spreadsheet.column(1)->valueAt(1), 1); 0191 QCOMPARE(spreadsheet.column(0)->valueAt(2), 1); 0192 QCOMPARE(spreadsheet.column(1)->valueAt(2), 2); 0193 QCOMPARE(spreadsheet.column(0)->valueAt(3), 0); 0194 QCOMPARE(spreadsheet.column(1)->valueAt(3), 3); 0195 QCOMPARE(spreadsheet.column(0)->valueAt(11), 0); 0196 QCOMPARE(spreadsheet.column(1)->valueAt(11), 144); 0197 } 0198 void HDF5FilterTest::testImportVLENPortion() { 0199 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0200 HDF5Filter filter; 0201 0202 const QString& fileName = QFINDTESTDATA(QLatin1String("data/h5ex_t_vlen.h5")); 0203 filter.setCurrentDataSetName(QLatin1String("/DS1")); 0204 // set start/end row/col 0205 filter.setStartRow(2); 0206 filter.setEndRow(5); 0207 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0208 0209 QCOMPARE(spreadsheet.columnCount(), 2); 0210 QCOMPARE(spreadsheet.rowCount(), 4); 0211 for (int i = 0; i < 2; i++) { 0212 QCOMPARE(spreadsheet.column(i)->columnMode(), AbstractColumn::ColumnMode::Integer); 0213 QCOMPARE(spreadsheet.column(i)->name(), QLatin1String("DS1_") + QString::number(i + 1)); 0214 } 0215 0216 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0217 QCOMPARE(spreadsheet.column(1)->plotDesignation(), AbstractColumn::PlotDesignation::Y); 0218 0219 QCOMPARE(spreadsheet.column(0)->valueAt(0), 2); 0220 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1); 0221 QCOMPARE(spreadsheet.column(0)->valueAt(1), 1); 0222 QCOMPARE(spreadsheet.column(1)->valueAt(1), 2); 0223 QCOMPARE(spreadsheet.column(0)->valueAt(2), 0); 0224 QCOMPARE(spreadsheet.column(1)->valueAt(2), 3); 0225 QCOMPARE(spreadsheet.column(0)->valueAt(3), 0); 0226 QCOMPARE(spreadsheet.column(1)->valueAt(3), 5); 0227 0228 // first column 0229 HDF5Filter filter2; 0230 filter2.setCurrentDataSetName(QLatin1String("/DS1")); 0231 filter2.setStartRow(2); 0232 filter2.setEndRow(5); 0233 filter2.setEndColumn(1); 0234 filter2.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0235 0236 QCOMPARE(spreadsheet.columnCount(), 1); 0237 QCOMPARE(spreadsheet.rowCount(), 2); 0238 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0239 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("DS1_1")); 0240 0241 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0242 0243 QCOMPARE(spreadsheet.column(0)->valueAt(0), 2); 0244 QCOMPARE(spreadsheet.column(0)->valueAt(1), 1); 0245 0246 // second column 0247 HDF5Filter filter3; 0248 filter3.setCurrentDataSetName(QLatin1String("/DS1")); 0249 filter3.setStartRow(2); 0250 filter3.setEndRow(5); 0251 filter3.setStartColumn(2); 0252 filter3.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0253 0254 QCOMPARE(spreadsheet.columnCount(), 1); 0255 QCOMPARE(spreadsheet.rowCount(), 4); 0256 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0257 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("DS1_2")); 0258 0259 QCOMPARE(spreadsheet.column(0)->plotDesignation(), AbstractColumn::PlotDesignation::X); 0260 0261 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1); 0262 QCOMPARE(spreadsheet.column(0)->valueAt(1), 2); 0263 QCOMPARE(spreadsheet.column(0)->valueAt(2), 3); 0264 QCOMPARE(spreadsheet.column(0)->valueAt(3), 5); 0265 } 0266 0267 // BENCHMARKS 0268 0269 void HDF5FilterTest::benchDoubleImport_data() { 0270 QTest::addColumn<size_t>("lineCount"); 0271 // can't transfer file name since needed in clean up 0272 0273 QTemporaryFile file; 0274 if (!file.open()) // needed to generate file name 0275 return; 0276 file.close(); // only file name is used 0277 0278 benchDataFileName = file.fileName(); 0279 benchDataFileName.append(QStringLiteral(".h5")); 0280 0281 QString testName(QString::number(paths) + QLatin1String(" random double paths")); 0282 0283 QTest::newRow(qPrintable(testName)) << lines; 0284 DEBUG("CREATE DATA FILE " << STDSTRING(benchDataFileName) << ", lines = " << lines) 0285 0286 gsl_rng_env_setup(); 0287 gsl_rng* r = gsl_rng_alloc(gsl_rng_default); 0288 gsl_rng_set(r, 12345); 0289 0290 // create file 0291 // see https://support.hdfgroup.org/HDF5/Tutor/introductory.html 0292 hid_t file_id = H5Fcreate(qPrintable(benchDataFileName), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); 0293 0294 /* Create the data space for the dataset. */ 0295 hsize_t dims[2]; 0296 dims[0] = lines; 0297 dims[1] = paths; 0298 hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); 0299 0300 /* Create the dataset. */ 0301 hid_t dataset_id = H5Dcreate(file_id, "/data", H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 0302 0303 // create data 0304 double path[paths] = {0.0}; 0305 double* data = new double[paths * lines]; 0306 0307 const double delta = 0.25; 0308 const int dt = 1; 0309 const double sigma = delta * delta * dt; 0310 for (size_t i = 0; i < lines; ++i) { 0311 // std::cout << "line " << i+1 << std::endl; 0312 0313 for (int p = 0; p < paths; ++p) { 0314 path[p] += gsl_ran_gaussian_ziggurat(r, sigma); 0315 data[p + i * paths] = path[p]; 0316 } 0317 } 0318 0319 herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); 0320 if (status < 0) 0321 WARN(Q_FUNC_INFO << ", ERROR writing data") 0322 status = H5Dclose(dataset_id); 0323 status = H5Sclose(dataspace_id); 0324 status = H5Fclose(file_id); 0325 0326 delete[] data; 0327 0328 DEBUG(Q_FUNC_INFO << ", DONE") 0329 } 0330 0331 void HDF5FilterTest::benchDoubleImport() { 0332 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0333 HDF5Filter filter; 0334 filter.setCurrentDataSetName(QLatin1String("/data")); 0335 0336 const int p = paths; // need local variable 0337 QBENCHMARK { 0338 filter.readDataFromFile(benchDataFileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0339 0340 QCOMPARE(spreadsheet.columnCount(), p); 0341 QCOMPARE(spreadsheet.rowCount(), lines); 0342 0343 QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.120997813055); 0344 QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.119301077563219); 0345 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.0209979608555485); 0346 } 0347 } 0348 0349 void HDF5FilterTest::benchDoubleImport_cleanup() { 0350 DEBUG("REMOVE DATA FILE " << STDSTRING(benchDataFileName)) 0351 QFile::remove(benchDataFileName); 0352 } 0353 0354 QTEST_MAIN(HDF5FilterTest)