File indexing completed on 2024-05-05 03:48:41
0001 /* 0002 File : NetCDFFilterTest.cpp 0003 Project : LabPlot 0004 Description : Tests for the NetCDF filter 0005 -------------------------------------------------------------------- 0006 SPDX-FileCopyrightText: 2022 Stefan Gerlach <stefan.gerlach@uni.kn> 0007 0008 SPDX-License-Identifier: GPL-2.0-or-later 0009 */ 0010 0011 #include "NetCDFFilterTest.h" 0012 #include "backend/datasources/filters/NetCDFFilter.h" 0013 #include "backend/lib/macros.h" 0014 #include "backend/spreadsheet/Spreadsheet.h" 0015 0016 #include <gsl/gsl_randist.h> 0017 #include <gsl/gsl_rng.h> 0018 extern "C" { 0019 #include <netcdf.h> 0020 } 0021 0022 #define ERRCODE -1 0023 #define ERR(e) \ 0024 { \ 0025 printf("Error: %s\n", nc_strerror(e)); \ 0026 exit(ERRCODE); \ 0027 } 0028 0029 void NetCDFFilterTest::importFile1() { 0030 const QString& fileName = QFINDTESTDATA(QLatin1String("data/madis-hydro.nc")); 0031 0032 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0033 NetCDFFilter filter; 0034 filter.setCurrentVarName(QLatin1String("lastRecord")); 0035 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0036 0037 QCOMPARE(spreadsheet.columnCount(), 1); 0038 QCOMPARE(spreadsheet.rowCount(), 2000); 0039 0040 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1018); 0041 QCOMPARE(spreadsheet.column(0)->valueAt(1), 1132); 0042 QCOMPARE(spreadsheet.column(0)->valueAt(2), 980); 0043 QCOMPARE(spreadsheet.column(0)->valueAt(1998), -1); 0044 QCOMPARE(spreadsheet.column(0)->valueAt(1999), -1); 0045 0046 NetCDFFilter filter2; 0047 filter2.setCurrentVarName(QLatin1String("latitude")); 0048 filter2.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0049 0050 QCOMPARE(spreadsheet.columnCount(), 1); 0051 QCOMPARE(spreadsheet.rowCount(), 1176); 0052 0053 QCOMPARE(spreadsheet.column(0)->valueAt(0), 39.8050003052); 0054 QCOMPARE(spreadsheet.column(0)->valueAt(1), 39.8230018616); 0055 QCOMPARE(spreadsheet.column(0)->valueAt(2), 39.8199996948); 0056 QCOMPARE(spreadsheet.column(0)->valueAt(1174), 40.2159996033); 0057 QCOMPARE(spreadsheet.column(0)->valueAt(1175), 40.2480010986); 0058 } 0059 0060 void NetCDFFilterTest::importFile2() { 0061 const QString& fileName = QFINDTESTDATA(QLatin1String("data/OMI-Aura_L2-example.nc")); 0062 0063 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0064 NetCDFFilter filter; 0065 filter.setCurrentVarName(QLatin1String("CovarianceMatrix")); 0066 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0067 0068 QCOMPARE(spreadsheet.columnCount(), 465); 0069 QCOMPARE(spreadsheet.rowCount(), 2); 0070 0071 QCOMPARE(spreadsheet.column(0)->valueAt(0), 76); 0072 QCOMPARE(spreadsheet.column(0)->valueAt(1), 78); 0073 QCOMPARE(spreadsheet.column(1)->valueAt(0), -17); 0074 QCOMPARE(spreadsheet.column(1)->valueAt(1), -18); 0075 QCOMPARE(spreadsheet.column(2)->valueAt(0), 49); 0076 QCOMPARE(spreadsheet.column(2)->valueAt(1), 50); 0077 0078 NetCDFFilter filter2; 0079 filter2.setCurrentVarName(QLatin1String("O3.COLUMN.PARTIAL")); 0080 filter2.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0081 0082 QCOMPARE(spreadsheet.columnCount(), 18); 0083 QCOMPARE(spreadsheet.rowCount(), 2); 0084 0085 WARN(spreadsheet.column(0)->valueAt(1)) 0086 WARN(spreadsheet.column(1)->valueAt(0)) 0087 WARN(spreadsheet.column(1)->valueAt(1)) 0088 WARN(spreadsheet.column(17)->valueAt(0)) 0089 WARN(spreadsheet.column(17)->valueAt(1)) 0090 QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.323581278324); 0091 QCOMPARE(spreadsheet.column(0)->valueAt(1), 0.317059576511383); 0092 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.23419499397278); 0093 QCOMPARE(spreadsheet.column(1)->valueAt(1), 1.224156498909); 0094 QCOMPARE(spreadsheet.column(17)->valueAt(0), 8.46088886260986); 0095 QCOMPARE(spreadsheet.column(17)->valueAt(1), 8.35280704498291); 0096 } 0097 0098 void NetCDFFilterTest::importFile3() { 0099 const QString& fileName = QFINDTESTDATA(QLatin1String("data/testrh.nc")); 0100 0101 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0102 NetCDFFilter filter; 0103 filter.setCurrentVarName(QLatin1String("var1")); 0104 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0105 0106 QCOMPARE(spreadsheet.columnCount(), 1); 0107 QCOMPARE(spreadsheet.rowCount(), 10000); 0108 0109 QCOMPARE(spreadsheet.column(0)->valueAt(0), 420); 0110 QCOMPARE(spreadsheet.column(0)->valueAt(1), 197); 0111 QCOMPARE(spreadsheet.column(0)->valueAt(2), 391.5); 0112 QCOMPARE(spreadsheet.column(0)->valueAt(9998), 186.5); 0113 QCOMPARE(spreadsheet.column(0)->valueAt(9999), 444); 0114 } 0115 0116 // BENCHMARKS 0117 0118 void NetCDFFilterTest::benchDoubleImport_data() { 0119 QTest::addColumn<size_t>("lineCount"); 0120 // can't transfer file name since needed in clean up 0121 0122 QTemporaryFile file; 0123 if (!file.open()) // needed to generate file name 0124 return; 0125 file.close(); // only file name is used 0126 0127 benchDataFileName = file.fileName(); 0128 benchDataFileName.append(QStringLiteral(".nc4")); 0129 0130 QString testName(QString::number(paths) + QLatin1String(" random double paths")); 0131 0132 QTest::newRow(qPrintable(testName)) << lines; 0133 DEBUG("CREATE DATA FILE " << STDSTRING(benchDataFileName) << ", lines = " << lines) 0134 0135 gsl_rng_env_setup(); 0136 gsl_rng* r = gsl_rng_alloc(gsl_rng_default); 0137 gsl_rng_set(r, 12345); 0138 0139 // define parameter 0140 int status, ncid; 0141 if ((status = nc_create(qPrintable(benchDataFileName), NC_NETCDF4, &ncid))) 0142 ERR(status); 0143 0144 int xdimid, ydimid; 0145 if ((status = nc_def_dim(ncid, "line", lines, &xdimid))) 0146 ERR(status); 0147 if ((status = nc_def_dim(ncid, "path", paths, &ydimid))) 0148 ERR(status); 0149 0150 const int ndims = 2; 0151 int dimids[] = {xdimid, ydimid}; 0152 0153 int varid; 0154 if ((status = nc_def_var(ncid, "paths", NC_DOUBLE, ndims, dimids, &varid))) 0155 ERR(status); 0156 // compress data (~10-20% smaller, but 3x-5x slower) 0157 // (ncid, varid, shuffle, deflate, deflatelevel) 0158 // if ((status = nc_def_var_deflate(ncid, varid, 1, 1, 9))) 0159 // ERR(status); 0160 0161 if ((status = nc_enddef(ncid))) 0162 ERR(status); 0163 0164 // create data 0165 double path[paths] = {0.0}; 0166 double* data = new double[paths * lines]; 0167 0168 const double delta = 0.25; 0169 const int dt = 1; 0170 const double sigma = delta * delta * dt; 0171 for (size_t i = 0; i < lines; ++i) { 0172 // std::cout << "line " << i+1 << std::endl; 0173 0174 for (int p = 0; p < paths; ++p) { 0175 path[p] += gsl_ran_gaussian_ziggurat(r, sigma); 0176 data[p + i * paths] = path[p]; 0177 } 0178 } 0179 0180 nc_put_var_double(ncid, varid, data); 0181 0182 if ((status = nc_close(ncid))) 0183 ERR(status); 0184 0185 delete[] data; 0186 0187 DEBUG(Q_FUNC_INFO << ", DONE") 0188 } 0189 0190 void NetCDFFilterTest::benchDoubleImport() { 0191 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0192 NetCDFFilter filter; 0193 filter.setCurrentVarName(QLatin1String("paths")); 0194 0195 const int p = paths; // need local variable 0196 QBENCHMARK { 0197 filter.readDataFromFile(benchDataFileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0198 0199 QCOMPARE(spreadsheet.columnCount(), p); 0200 QCOMPARE(spreadsheet.rowCount(), lines); 0201 0202 QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.120997813055); 0203 QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.119301077563219); 0204 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.0209979608555485); 0205 } 0206 } 0207 0208 void NetCDFFilterTest::benchDoubleImport_cleanup() { 0209 DEBUG("REMOVE DATA FILE " << STDSTRING(benchDataFileName)) 0210 QFile::remove(benchDataFileName); 0211 } 0212 0213 QTEST_MAIN(NetCDFFilterTest)