File indexing completed on 2024-04-28 03:48:14
0001 /* 0002 File : AsciiFilterTest.cpp 0003 Project : LabPlot 0004 Description : Tests for the ascii filter 0005 -------------------------------------------------------------------- 0006 SPDX-FileCopyrightText: 2017-2023 Alexander Semke <alexander.semke@web.de> 0007 SPDX-FileCopyrightText: 2022 Stefan Gerlach <stefan.gerlach@uni.kn> 0008 0009 SPDX-License-Identifier: GPL-2.0-or-later 0010 */ 0011 0012 #include "AsciiFilterTest.h" 0013 #include "backend/core/Project.h" 0014 #include "backend/datasources/filters/AsciiFilter.h" 0015 #include "backend/lib/macros.h" 0016 #include "backend/matrix/Matrix.h" 0017 #include "backend/spreadsheet/Spreadsheet.h" 0018 #include "backend/worksheet/plots/cartesian/CartesianPlot.h" 0019 #include "backend/worksheet/plots/cartesian/XYCurve.h" 0020 0021 #include <gsl/gsl_randist.h> 0022 #include <gsl/gsl_rng.h> 0023 0024 void AsciiFilterTest::initTestCase() { 0025 // needed in order to have the signals triggered by SignallingUndoCommand, see LabPlot.cpp 0026 // TODO: redesign/remove this 0027 qRegisterMetaType<const AbstractAspect*>("const AbstractAspect*"); 0028 qRegisterMetaType<const AbstractColumn*>("const AbstractColumn*"); 0029 } 0030 0031 // ############################################################################## 0032 // ################# handling of empty and sparse files ######################## 0033 // ############################################################################## 0034 void AsciiFilterTest::testEmptyFileAppend() { 0035 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0036 AsciiFilter filter; 0037 0038 const int rowCount = spreadsheet.rowCount(); 0039 const int colCount = spreadsheet.columnCount(); 0040 const QString& fileName = QFINDTESTDATA(QLatin1String("data/empty_file.txt")); 0041 0042 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Append); 0043 0044 QCOMPARE(spreadsheet.rowCount(), rowCount); 0045 QCOMPARE(spreadsheet.columnCount(), colCount); 0046 } 0047 0048 void AsciiFilterTest::testEmptyFilePrepend() { 0049 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0050 AsciiFilter filter; 0051 0052 const int rowCount = spreadsheet.rowCount(); 0053 const int colCount = spreadsheet.columnCount(); 0054 const QString& fileName = QFINDTESTDATA(QLatin1String("data/empty_file.txt")); 0055 0056 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Prepend); 0057 0058 QCOMPARE(spreadsheet.rowCount(), rowCount); 0059 QCOMPARE(spreadsheet.columnCount(), colCount); 0060 } 0061 0062 void AsciiFilterTest::testEmptyFileReplace() { 0063 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0064 AsciiFilter filter; 0065 0066 const int rowCount = spreadsheet.rowCount(); 0067 const int colCount = spreadsheet.columnCount(); 0068 const QString& fileName = QFINDTESTDATA(QLatin1String("data/empty_file.txt")); 0069 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0070 0071 QCOMPARE(spreadsheet.rowCount(), rowCount); 0072 QCOMPARE(spreadsheet.columnCount(), colCount); 0073 } 0074 0075 void AsciiFilterTest::testEmptyLines01() { 0076 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0077 AsciiFilter filter; 0078 const QString& fileName = QFINDTESTDATA(QLatin1String("data/empty_lines_01.txt")); 0079 0080 filter.setSeparatingCharacter(QStringLiteral("auto")); 0081 filter.setHeaderEnabled(true); 0082 filter.setHeaderLine(1); 0083 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0084 0085 QCOMPARE(spreadsheet.rowCount(), 3); 0086 QCOMPARE(spreadsheet.columnCount(), 3); 0087 0088 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0089 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("y")); 0090 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("values")); 0091 0092 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0093 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0094 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 0095 0096 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0097 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 0098 QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); 0099 QCOMPARE(spreadsheet.column(1)->integerAt(0), 2); 0100 QCOMPARE(spreadsheet.column(1)->integerAt(1), 4); 0101 QCOMPARE(spreadsheet.column(1)->integerAt(2), 9); 0102 QCOMPARE(spreadsheet.column(2)->integerAt(0), 10); 0103 QCOMPARE(spreadsheet.column(2)->integerAt(1), 40); 0104 QCOMPARE(spreadsheet.column(2)->integerAt(2), 90); 0105 } 0106 0107 void AsciiFilterTest::testSparseFile01() { 0108 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0109 AsciiFilter filter; 0110 const QString& fileName = QFINDTESTDATA(QLatin1String("data/sparse_file_01.txt")); 0111 0112 filter.setSeparatingCharacter(QStringLiteral(",")); 0113 filter.setHeaderEnabled(true); 0114 filter.setHeaderLine(1); 0115 filter.setSimplifyWhitespacesEnabled(true); 0116 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0117 0118 QCOMPARE(spreadsheet.rowCount(), 3); 0119 QCOMPARE(spreadsheet.columnCount(), 3); 0120 0121 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("N")); 0122 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Col1")); 0123 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("Col2")); 0124 0125 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0126 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0127 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 0128 0129 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0130 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 0131 QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); 0132 0133 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1); 0134 QCOMPARE(spreadsheet.column(1)->integerAt(1), 0); 0135 QCOMPARE(spreadsheet.column(1)->integerAt(2), 1); 0136 0137 QCOMPARE(spreadsheet.column(2)->integerAt(0), 2); 0138 QCOMPARE(spreadsheet.column(2)->integerAt(1), 2); 0139 QCOMPARE(spreadsheet.column(2)->integerAt(2), 0); 0140 } 0141 0142 void AsciiFilterTest::testSparseFile02() { 0143 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0144 AsciiFilter filter; 0145 const QString& fileName = QFINDTESTDATA(QLatin1String("data/sparse_file_02.txt")); 0146 0147 filter.setSeparatingCharacter(QStringLiteral(",")); 0148 filter.setNaNValueToZero(false); 0149 filter.setSimplifyWhitespacesEnabled(true); 0150 filter.setSkipEmptyParts(false); 0151 filter.setHeaderEnabled(true); 0152 filter.setHeaderLine(1); 0153 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0154 0155 QCOMPARE(spreadsheet.rowCount(), 3); 0156 QCOMPARE(spreadsheet.columnCount(), 3); 0157 0158 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("N")); 0159 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Col1")); 0160 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("Col2")); 0161 0162 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0163 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0164 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0165 0166 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0167 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 0168 QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); 0169 0170 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.); 0171 QCOMPARE((bool)std::isnan(spreadsheet.column(1)->valueAt(1)), true); 0172 QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.); 0173 0174 QCOMPARE(spreadsheet.column(2)->valueAt(0), 2.); 0175 QCOMPARE(spreadsheet.column(2)->valueAt(1), 2.); 0176 QCOMPARE((bool)std::isnan(spreadsheet.column(2)->valueAt(2)), true); 0177 } 0178 0179 void AsciiFilterTest::testSparseFile03() { 0180 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0181 AsciiFilter filter; 0182 const QString& fileName = QFINDTESTDATA(QLatin1String("data/sparse_file_03.txt")); 0183 0184 filter.setSeparatingCharacter(QStringLiteral(",")); 0185 filter.setNaNValueToZero(true); 0186 filter.setSimplifyWhitespacesEnabled(true); 0187 filter.setSkipEmptyParts(false); 0188 filter.setHeaderEnabled(true); 0189 filter.setHeaderLine(1); 0190 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0191 0192 QCOMPARE(spreadsheet.rowCount(), 4); 0193 QCOMPARE(spreadsheet.columnCount(), 3); 0194 0195 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("N")); 0196 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Col1")); 0197 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("Col2")); 0198 0199 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0200 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0201 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0202 0203 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0204 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 0205 QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); 0206 QCOMPARE(spreadsheet.column(0)->integerAt(3), 0); 0207 0208 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.); 0209 QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.); 0210 QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.); 0211 QCOMPARE(spreadsheet.column(1)->valueAt(3), 0.); 0212 0213 QCOMPARE(spreadsheet.column(2)->valueAt(0), 2.); 0214 QCOMPARE(spreadsheet.column(2)->valueAt(1), 2.); 0215 QCOMPARE(spreadsheet.column(2)->valueAt(2), 0.); 0216 QCOMPARE(spreadsheet.column(2)->valueAt(3), 3.); 0217 } 0218 0219 // ############################################################################## 0220 // ################################ header handling ############################ 0221 // ############################################################################## 0222 void AsciiFilterTest::testHeader01() { 0223 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0224 AsciiFilter filter; 0225 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0226 0227 filter.setSeparatingCharacter(QStringLiteral(";")); 0228 filter.setHeaderEnabled(false); 0229 filter.setVectorNames(QString()); 0230 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0231 0232 QCOMPARE(spreadsheet.rowCount(), 3); 0233 QCOMPARE(spreadsheet.columnCount(), 2); 0234 } 0235 0236 void AsciiFilterTest::testHeader02() { 0237 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0238 AsciiFilter filter; 0239 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0240 0241 filter.setSeparatingCharacter(QStringLiteral(";")); 0242 filter.setHeaderEnabled(true); 0243 filter.setHeaderLine(1); 0244 filter.setVectorNames(QString()); 0245 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0246 0247 QCOMPARE(spreadsheet.rowCount(), 2); // out of 3 rows one row is used for the column names (header) 0248 QCOMPARE(spreadsheet.columnCount(), 2); 0249 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("1")); 0250 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("2")); 0251 } 0252 0253 void AsciiFilterTest::testHeader03() { 0254 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0255 AsciiFilter filter; 0256 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0257 0258 filter.setSeparatingCharacter(QStringLiteral(";")); 0259 filter.setHeaderEnabled(false); 0260 filter.setVectorNames(QStringLiteral("x")); 0261 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0262 0263 QCOMPARE(spreadsheet.rowCount(), 3); 0264 QCOMPARE(spreadsheet.columnCount(), 1); // one column name was specified, we import only one column 0265 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0266 } 0267 0268 void AsciiFilterTest::testHeader04() { 0269 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0270 AsciiFilter filter; 0271 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0272 0273 filter.setSeparatingCharacter(QStringLiteral(";")); 0274 filter.setHeaderEnabled(false); 0275 filter.setVectorNames(QStringLiteral("x")); 0276 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0277 0278 QCOMPARE(spreadsheet.rowCount(), 3); 0279 QCOMPARE(spreadsheet.columnCount(), 1); // one column name was specified -> we import only one column 0280 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0281 } 0282 0283 void AsciiFilterTest::testHeader05() { 0284 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0285 AsciiFilter filter; 0286 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0287 0288 filter.setSeparatingCharacter(QStringLiteral(";")); 0289 filter.setHeaderEnabled(false); 0290 filter.setVectorNames(QStringLiteral("x y")); 0291 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0292 0293 QCOMPARE(spreadsheet.rowCount(), 3); 0294 QCOMPARE(spreadsheet.columnCount(), 2); // two names were specified -> we import two columns 0295 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0296 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("y")); 0297 } 0298 0299 void AsciiFilterTest::testHeader06() { 0300 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0301 AsciiFilter filter; 0302 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0303 0304 filter.setSeparatingCharacter(QStringLiteral(";")); 0305 filter.setHeaderEnabled(false); 0306 filter.setVectorNames(QStringLiteral("x y z")); 0307 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0308 0309 QCOMPARE(spreadsheet.rowCount(), 3); 0310 QCOMPARE(spreadsheet.columnCount(), 2); // three names were specified, but there're only two columns in the file -> we import only two columns 0311 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0312 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("y")); 0313 } 0314 0315 /*! 0316 * test with a file containing the header in the second line 0317 * with a subsequent comment line without any comment character. 0318 * this line shouldn't disturb the detection of numeric column modes. 0319 */ 0320 void AsciiFilterTest::testHeader07() { 0321 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0322 AsciiFilter filter; 0323 const QString& fileName = QFINDTESTDATA(QLatin1String("data/comment_header_comment.txt")); 0324 0325 filter.setSeparatingCharacter(QStringLiteral("TAB")); 0326 filter.setHeaderEnabled(true); 0327 filter.setHeaderLine(2); 0328 filter.setDateTimeFormat(QLatin1String("yyyy-MM-dd hh:mm:ss.zzz")); 0329 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0330 0331 // spreadsheet size 0332 QCOMPARE(spreadsheet.columnCount(), 3); 0333 QCOMPARE(spreadsheet.rowCount(), 4); 0334 0335 // column names 0336 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("counter")); 0337 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("t[min]")); 0338 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("#1ch1")); 0339 0340 // data types 0341 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0342 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0343 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0344 0345 // values 0346 QCOMPARE(spreadsheet.column(0)->integerAt(0), 0); 0347 QCOMPARE((bool)std::isnan(spreadsheet.column(1)->valueAt(0)), true); 0348 QCOMPARE((bool)std::isnan(spreadsheet.column(2)->valueAt(0)), true); 0349 0350 QCOMPARE(spreadsheet.column(0)->integerAt(1), 1); 0351 QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.0513); 0352 QCOMPARE(spreadsheet.column(2)->valueAt(1), 0.3448); 0353 0354 QCOMPARE(spreadsheet.column(0)->integerAt(2), 2); 0355 QCOMPARE(spreadsheet.column(1)->valueAt(2), 0.1005); 0356 QCOMPARE(spreadsheet.column(2)->valueAt(2), 0.3418); 0357 0358 QCOMPARE(spreadsheet.column(0)->integerAt(3), 3); 0359 QCOMPARE(spreadsheet.column(1)->valueAt(3), 0.1516); 0360 QCOMPARE(spreadsheet.column(2)->valueAt(3), 0.3433); 0361 } 0362 0363 /*! 0364 * test with a file containing the header in the second line 0365 * with a subsequent comment line ignored by using startRow. 0366 */ 0367 void AsciiFilterTest::testHeader07a() { 0368 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0369 AsciiFilter filter; 0370 const QString& fileName = QFINDTESTDATA(QLatin1String("data/comment_header_comment.txt")); 0371 0372 filter.setSeparatingCharacter(QStringLiteral("TAB")); 0373 filter.setHeaderLine(2); 0374 filter.setHeaderEnabled(true); 0375 filter.setStartRow(4); 0376 filter.setDateTimeFormat(QLatin1String("yyyy-MM-dd hh:mm:ss.zzz")); 0377 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0378 0379 // spreadsheet size 0380 QCOMPARE(spreadsheet.columnCount(), 3); 0381 QCOMPARE(spreadsheet.rowCount(), 3); 0382 0383 // column names 0384 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("counter")); 0385 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("t[min]")); 0386 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("#1ch1")); 0387 0388 // data types 0389 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0390 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0391 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0392 0393 // values 0394 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0395 QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.0513); 0396 QCOMPARE(spreadsheet.column(2)->valueAt(0), 0.3448); 0397 0398 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 0399 QCOMPARE(spreadsheet.column(1)->valueAt(1), 0.1005); 0400 QCOMPARE(spreadsheet.column(2)->valueAt(1), 0.3418); 0401 0402 QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); 0403 QCOMPARE(spreadsheet.column(1)->valueAt(2), 0.1516); 0404 QCOMPARE(spreadsheet.column(2)->valueAt(2), 0.3433); 0405 } 0406 0407 /*! 0408 * the header contains spaces in the column names, values are tab separated. 0409 * when using "auto" for the separator characters, the tab character has to 0410 * be properly recognized and used. 0411 */ 0412 void AsciiFilterTest::testHeader08() { 0413 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0414 AsciiFilter filter; 0415 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_tab_with_header_with_spaces.txt")); 0416 0417 filter.setHeaderEnabled(true); 0418 filter.setHeaderLine(1); 0419 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0420 0421 // spreadsheet size 0422 QCOMPARE(spreadsheet.columnCount(), 2); 0423 QCOMPARE(spreadsheet.rowCount(), 2); 0424 0425 // column names 0426 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("first column")); 0427 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("second column")); 0428 0429 // data types 0430 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0431 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0432 0433 // values 0434 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0435 QCOMPARE(spreadsheet.column(1)->integerAt(0), 2); 0436 0437 QCOMPARE(spreadsheet.column(0)->integerAt(1), 3); 0438 QCOMPARE(spreadsheet.column(1)->integerAt(1), 4); 0439 } 0440 0441 /*! 0442 * test the handling of duplicated columns names provided by the user. 0443 */ 0444 void AsciiFilterTest::testHeader09() { 0445 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0446 AsciiFilter filter; 0447 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon.txt")); 0448 0449 filter.setSeparatingCharacter(QStringLiteral(";")); 0450 filter.setHeaderEnabled(false); 0451 filter.setVectorNames(QStringList{QStringLiteral("x"), QStringLiteral("x")}); 0452 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0453 0454 QCOMPARE(spreadsheet.rowCount(), 3); 0455 QCOMPARE(spreadsheet.columnCount(), 2); 0456 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0457 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("x 1")); // the duplicated name was renamed 0458 } 0459 0460 /*! 0461 * test the handling of duplicated columns in the file to be imported. 0462 */ 0463 void AsciiFilterTest::testHeader10() { 0464 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0465 AsciiFilter filter; 0466 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header_duplicated_names.txt")); 0467 0468 filter.setSeparatingCharacter(QStringLiteral(";")); 0469 filter.setHeaderEnabled(true); 0470 filter.setHeaderLine(1); 0471 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0472 0473 QCOMPARE(spreadsheet.rowCount(), 3); 0474 QCOMPARE(spreadsheet.columnCount(), 2); 0475 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); 0476 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("x 1")); // the duplicated name was renamed 0477 } 0478 0479 /*! 0480 * test the handling of duplicated columns in the file to be imported. 0481 */ 0482 void AsciiFilterTest::testHeader11() { 0483 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0484 AsciiFilter filter; 0485 const QString& fileName = QFINDTESTDATA(QLatin1String("data/column_names.txt")); 0486 0487 filter.setSeparatingCharacter(QStringLiteral(" ")); 0488 filter.setHeaderEnabled(true); 0489 filter.setHeaderLine(1); 0490 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0491 0492 QCOMPARE(spreadsheet.rowCount(), 1); 0493 QCOMPARE(spreadsheet.columnCount(), 2); 0494 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("A")); 0495 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("B")); 0496 0497 // import the second file with reversed column names into the same spreadsheet 0498 AsciiFilter filter2; // create a new filter so we go through the prepare logic from scratch for the 2nd file 0499 const QString& fileName2 = QFINDTESTDATA(QLatin1String("data/column_names_reversed.txt")); 0500 filter2.setSeparatingCharacter(QStringLiteral(" ")); 0501 filter.setHeaderEnabled(true); 0502 filter2.setHeaderLine(1); 0503 filter2.readDataFromFile(fileName2, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0504 0505 QCOMPARE(spreadsheet.rowCount(), 1); 0506 QCOMPARE(spreadsheet.columnCount(), 2); 0507 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("B")); 0508 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("A")); 0509 } 0510 0511 /*! 0512 * test the handling of column names with and without header 0513 */ 0514 void AsciiFilterTest::testHeader11a() { 0515 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0516 AsciiFilter filter; 0517 const QString& fileName = QFINDTESTDATA(QLatin1String("data/column_names.txt")); 0518 0519 filter.setSeparatingCharacter(QStringLiteral(" ")); 0520 filter.setHeaderEnabled(true); 0521 filter.setHeaderLine(1); 0522 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0523 0524 QCOMPARE(spreadsheet.rowCount(), 1); 0525 QCOMPARE(spreadsheet.columnCount(), 2); 0526 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("A")); 0527 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("B")); 0528 0529 AsciiFilter filter2; 0530 filter2.setHeaderEnabled(false); 0531 filter2.setSeparatingCharacter(QStringLiteral(" ")); 0532 filter2.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0533 0534 QCOMPARE(spreadsheet.rowCount(), 2); 0535 QCOMPARE(spreadsheet.columnCount(), 2); 0536 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("Column 1")); 0537 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Column 2")); 0538 } 0539 0540 // ############################################################################## 0541 // ##################### handling of different read ranges ##################### 0542 // ############################################################################## 0543 void AsciiFilterTest::testColumnRange00() { 0544 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0545 AsciiFilter filter; 0546 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0547 0548 filter.setSeparatingCharacter(QStringLiteral("auto")); 0549 filter.setHeaderEnabled(false); 0550 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0551 0552 // no ranges specified, all rows and columns have to be read 0553 QCOMPARE(spreadsheet.rowCount(), 5); 0554 QCOMPARE(spreadsheet.columnCount(), 3); 0555 0556 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double); 0557 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0558 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0559 0560 // check the values for the first line 0561 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1.716299); 0562 QCOMPARE(spreadsheet.column(1)->valueAt(0), -0.485527); 0563 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.288690); 0564 } 0565 0566 void AsciiFilterTest::testColumnRange01() { 0567 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0568 AsciiFilter filter; 0569 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0570 0571 filter.setSeparatingCharacter(QStringLiteral("auto")); 0572 filter.setHeaderEnabled(false); 0573 filter.setCreateIndexEnabled(true); 0574 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0575 0576 // no ranges specified, all rows and columns have to be read plus the additional column for the index 0577 QCOMPARE(spreadsheet.rowCount(), 5); 0578 QCOMPARE(spreadsheet.columnCount(), 4); 0579 0580 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0581 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0582 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0583 QCOMPARE(spreadsheet.column(3)->columnMode(), AbstractColumn::ColumnMode::Double); 0584 0585 // check the values for the first line 0586 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0587 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.716299); 0588 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.485527); 0589 QCOMPARE(spreadsheet.column(3)->valueAt(0), -0.288690); 0590 } 0591 0592 void AsciiFilterTest::testColumnRange02() { 0593 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0594 AsciiFilter filter; 0595 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0596 0597 filter.setSeparatingCharacter(QStringLiteral("auto")); 0598 filter.setHeaderEnabled(false); 0599 filter.setStartColumn(2); 0600 filter.setEndColumn(3); 0601 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0602 0603 // read all rows and the last two columns only 0604 QCOMPARE(spreadsheet.rowCount(), 5); 0605 QCOMPARE(spreadsheet.columnCount(), 2); 0606 0607 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double); 0608 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0609 0610 // check the values for the first line 0611 QCOMPARE(spreadsheet.column(0)->valueAt(0), -0.485527); 0612 QCOMPARE(spreadsheet.column(1)->valueAt(0), -0.288690); 0613 } 0614 0615 void AsciiFilterTest::testColumnRange03() { 0616 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0617 AsciiFilter filter; 0618 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0619 0620 filter.setSeparatingCharacter(QStringLiteral("auto")); 0621 filter.setHeaderEnabled(false); 0622 filter.setCreateIndexEnabled(true); 0623 filter.setStartColumn(2); 0624 filter.setEndColumn(3); 0625 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0626 0627 // read all rows and the last two columns only plus the additional column for the index 0628 QCOMPARE(spreadsheet.rowCount(), 5); 0629 QCOMPARE(spreadsheet.columnCount(), 3); 0630 0631 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0632 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0633 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0634 0635 // check the values for the first line 0636 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0637 QCOMPARE(spreadsheet.column(1)->valueAt(0), -0.485527); 0638 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.288690); 0639 } 0640 0641 void AsciiFilterTest::testColumnRange04() { 0642 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0643 AsciiFilter filter; 0644 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0645 0646 filter.setSeparatingCharacter(QStringLiteral("auto")); 0647 filter.setHeaderEnabled(false); 0648 filter.setStartColumn(3); 0649 filter.setEndColumn(3); 0650 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0651 0652 // read all rows and the last column only 0653 QCOMPARE(spreadsheet.rowCount(), 5); 0654 QCOMPARE(spreadsheet.columnCount(), 1); 0655 0656 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double); 0657 0658 // check the values for the first line 0659 QCOMPARE(spreadsheet.column(0)->valueAt(0), -0.288690); 0660 } 0661 0662 void AsciiFilterTest::testColumnRange05() { 0663 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0664 AsciiFilter filter; 0665 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0666 0667 filter.setSeparatingCharacter(QStringLiteral("auto")); 0668 filter.setHeaderEnabled(false); 0669 filter.setStartColumn(3); 0670 filter.setEndColumn(2); 0671 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0672 0673 // wrong column range specified (start>end), nothing to read, 0674 // empty spreadsheet because of the replace mode 0675 QCOMPARE(spreadsheet.rowCount(), 0); 0676 QCOMPARE(spreadsheet.columnCount(), 0); 0677 } 0678 0679 void AsciiFilterTest::testColumnRange06() { 0680 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0681 AsciiFilter filter; 0682 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0683 0684 filter.setSeparatingCharacter(QStringLiteral("auto")); 0685 filter.setHeaderEnabled(false); 0686 filter.setCreateIndexEnabled(true); 0687 filter.setStartColumn(3); 0688 filter.setEndColumn(2); 0689 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0690 0691 // wrong column range specified (start>end), only the index column is created 0692 QCOMPARE(spreadsheet.rowCount(), 5); 0693 QCOMPARE(spreadsheet.columnCount(), 1); 0694 0695 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0696 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0697 } 0698 0699 void AsciiFilterTest::testRowRange00() { 0700 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0701 AsciiFilter filter; 0702 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0703 0704 filter.setSeparatingCharacter(QStringLiteral("auto")); 0705 filter.setStartRow(3); 0706 filter.setEndRow(5); 0707 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0708 0709 // three rows to read 0710 QCOMPARE(spreadsheet.rowCount(), 3); 0711 QCOMPARE(spreadsheet.columnCount(), 3); 0712 0713 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double); 0714 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0715 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0716 0717 // check the values for the first and for the last lines 0718 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1.711721); 0719 QCOMPARE(spreadsheet.column(1)->valueAt(0), -0.485527); 0720 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.293267); 0721 0722 QCOMPARE(spreadsheet.column(0)->valueAt(2), 1.716299); 0723 QCOMPARE(spreadsheet.column(1)->valueAt(2), -0.494682); 0724 QCOMPARE(spreadsheet.column(2)->valueAt(2), -0.284112); 0725 } 0726 0727 void AsciiFilterTest::testRowRange01() { 0728 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0729 AsciiFilter filter; 0730 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0731 0732 filter.setSeparatingCharacter(QStringLiteral("auto")); 0733 filter.setHeaderEnabled(false); 0734 filter.setStartRow(3); 0735 filter.setEndRow(10); 0736 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0737 0738 // end row larger than the number of available rows, three rows to read 0739 QCOMPARE(spreadsheet.rowCount(), 3); 0740 QCOMPARE(spreadsheet.columnCount(), 3); 0741 0742 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double); 0743 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0744 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0745 0746 // check the values for the first and for the last lines 0747 QCOMPARE(spreadsheet.column(0)->valueAt(0), 1.711721); 0748 QCOMPARE(spreadsheet.column(1)->valueAt(0), -0.485527); 0749 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.293267); 0750 0751 QCOMPARE(spreadsheet.column(0)->valueAt(2), 1.716299); 0752 QCOMPARE(spreadsheet.column(1)->valueAt(2), -0.494682); 0753 QCOMPARE(spreadsheet.column(2)->valueAt(2), -0.284112); 0754 } 0755 0756 void AsciiFilterTest::testRowRange02() { 0757 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0758 AsciiFilter filter; 0759 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0760 0761 filter.setSeparatingCharacter(QStringLiteral("auto")); 0762 filter.setHeaderEnabled(false); 0763 filter.setStartRow(3); 0764 filter.setEndRow(1); 0765 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0766 0767 // start bigger than end, no rows to read 0768 // wrong row range specified (start>end), nothing to read, 0769 // spreadsheet is not touched, default number of rows and columns 0770 // TODO: this is inconsistent with the handling for columns, see testColumnRange05() 0771 QCOMPARE(spreadsheet.rowCount(), 100); 0772 QCOMPARE(spreadsheet.columnCount(), 2); 0773 } 0774 0775 void AsciiFilterTest::testRowColumnRange00() { 0776 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0777 AsciiFilter filter; 0778 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 0779 0780 filter.setSeparatingCharacter(QStringLiteral("auto")); 0781 filter.setHeaderEnabled(false); 0782 filter.setStartRow(3); 0783 filter.setEndRow(5); 0784 filter.setStartColumn(2); 0785 filter.setEndColumn(3); 0786 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0787 0788 // three rows and two columns to read 0789 QCOMPARE(spreadsheet.rowCount(), 3); 0790 QCOMPARE(spreadsheet.columnCount(), 2); 0791 0792 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Double); 0793 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 0794 0795 // check the values for the first and for the last lines 0796 QCOMPARE(spreadsheet.column(0)->valueAt(0), -0.485527); 0797 QCOMPARE(spreadsheet.column(1)->valueAt(0), -0.293267); 0798 0799 QCOMPARE(spreadsheet.column(0)->valueAt(2), -0.494682); 0800 QCOMPARE(spreadsheet.column(1)->valueAt(2), -0.284112); 0801 } 0802 0803 // ############################################################################## 0804 // ##################### handling of different separators ###################### 0805 // ############################################################################## 0806 0807 // ############################################################################## 0808 // ##################################### quoted strings ######################## 0809 // ############################################################################## 0810 void AsciiFilterTest::testQuotedStrings00() { 0811 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0812 AsciiFilter filter; 0813 const QString& fileName = QFINDTESTDATA(QLatin1String("data/quoted_strings.txt")); 0814 0815 filter.setSeparatingCharacter(QStringLiteral(",")); 0816 filter.setHeaderEnabled(false); 0817 filter.setRemoveQuotesEnabled(true); 0818 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0819 0820 // three rows and two columns to read 0821 QCOMPARE(spreadsheet.rowCount(), 3); 0822 QCOMPARE(spreadsheet.columnCount(), 4); 0823 0824 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Text); 0825 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0826 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 0827 QCOMPARE(spreadsheet.column(3)->columnMode(), AbstractColumn::ColumnMode::Double); 0828 0829 QCOMPARE(spreadsheet.column(0)->textAt(0), QLatin1String("a")); 0830 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1000); 0831 QCOMPARE(spreadsheet.column(2)->integerAt(0), 201811); 0832 QCOMPARE(spreadsheet.column(3)->valueAt(0), 1.1); 0833 0834 QCOMPARE(spreadsheet.column(0)->textAt(1), QLatin1String("ab")); 0835 QCOMPARE(spreadsheet.column(1)->integerAt(1), 2000); 0836 QCOMPARE(spreadsheet.column(2)->integerAt(1), 201812); 0837 QCOMPARE(spreadsheet.column(3)->valueAt(1), 1.2); 0838 0839 QCOMPARE(spreadsheet.column(0)->textAt(2), QLatin1String("abc")); 0840 QCOMPARE(spreadsheet.column(1)->integerAt(2), 3000); 0841 QCOMPARE(spreadsheet.column(2)->integerAt(2), 201901); 0842 QCOMPARE(spreadsheet.column(3)->valueAt(2), 1.3); 0843 } 0844 0845 void AsciiFilterTest::testQuotedStrings01() { 0846 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0847 AsciiFilter filter; 0848 const QString& fileName = QFINDTESTDATA(QLatin1String("data/quoted_strings_with_header.txt")); 0849 0850 filter.setSeparatingCharacter(QStringLiteral(",")); 0851 filter.setHeaderEnabled(true); 0852 filter.setHeaderLine(1); 0853 filter.setSimplifyWhitespacesEnabled(true); 0854 filter.setRemoveQuotesEnabled(true); 0855 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0856 0857 // three rows and two columns to read 0858 QCOMPARE(spreadsheet.rowCount(), 3); 0859 QCOMPARE(spreadsheet.columnCount(), 4); 0860 0861 // column names 0862 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("col1")); 0863 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("col2")); 0864 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("col3")); 0865 QCOMPARE(spreadsheet.column(3)->name(), QLatin1String("col4")); 0866 0867 // data types 0868 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Text); 0869 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0870 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 0871 QCOMPARE(spreadsheet.column(3)->columnMode(), AbstractColumn::ColumnMode::Double); 0872 0873 // values 0874 QCOMPARE(spreadsheet.column(0)->textAt(0), QLatin1String("a")); 0875 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1000); 0876 QCOMPARE(spreadsheet.column(2)->integerAt(0), 201811); 0877 QCOMPARE(spreadsheet.column(3)->valueAt(0), 1.1); 0878 0879 QCOMPARE(spreadsheet.column(0)->textAt(1), QLatin1String("ab")); 0880 QCOMPARE(spreadsheet.column(1)->integerAt(1), 2000); 0881 QCOMPARE(spreadsheet.column(2)->integerAt(1), 201812); 0882 QCOMPARE(spreadsheet.column(3)->valueAt(1), 1.2); 0883 0884 QCOMPARE(spreadsheet.column(0)->textAt(2), QLatin1String("abc")); 0885 QCOMPARE(spreadsheet.column(1)->integerAt(2), 3000); 0886 QCOMPARE(spreadsheet.column(2)->integerAt(2), 201901); 0887 QCOMPARE(spreadsheet.column(3)->valueAt(2), 1.3); 0888 } 0889 0890 void AsciiFilterTest::testQuotedStrings02() { 0891 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0892 AsciiFilter filter; 0893 const QString& fileName = QFINDTESTDATA(QLatin1String("data/quoted_strings_one_line.txt")); 0894 0895 QCOMPARE(QFile::exists(fileName), true); 0896 0897 filter.setSeparatingCharacter(QStringLiteral(",")); 0898 // filter.setHeaderEnabled(false); 0899 filter.setRemoveQuotesEnabled(true); 0900 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0901 0902 // three rows and two columns to read 0903 // QCOMPARE(spreadsheet.rowCount(), 1); 0904 // QCOMPARE(spreadsheet.columnCount(), 4); 0905 0906 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Text); 0907 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0908 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 0909 QCOMPARE(spreadsheet.column(3)->columnMode(), AbstractColumn::ColumnMode::Double); 0910 0911 QCOMPARE(spreadsheet.column(0)->textAt(0), QLatin1String("a")); 0912 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1000); 0913 QCOMPARE(spreadsheet.column(2)->integerAt(0), 201811); 0914 QCOMPARE(spreadsheet.column(3)->valueAt(0), 1.1); 0915 } 0916 0917 void AsciiFilterTest::testQuotedStrings03() { 0918 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0919 AsciiFilter filter; 0920 const QString& fileName = QFINDTESTDATA(QLatin1String("data/quoted_strings_one_line_with_header.txt")); 0921 0922 filter.setSeparatingCharacter(QStringLiteral(",")); 0923 filter.setHeaderEnabled(true); 0924 filter.setHeaderLine(1); 0925 filter.setSimplifyWhitespacesEnabled(true); 0926 filter.setRemoveQuotesEnabled(true); 0927 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0928 0929 // three rows and two columns to read 0930 // QCOMPARE(spreadsheet.rowCount(), 1); 0931 // QCOMPARE(spreadsheet.columnCount(), 4); 0932 0933 // column names 0934 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("col1")); 0935 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("col2")); 0936 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("col3")); 0937 QCOMPARE(spreadsheet.column(3)->name(), QLatin1String("col4")); 0938 0939 // data types 0940 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Text); 0941 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 0942 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 0943 QCOMPARE(spreadsheet.column(3)->columnMode(), AbstractColumn::ColumnMode::Double); 0944 0945 // values 0946 QCOMPARE(spreadsheet.column(0)->textAt(0), QLatin1String("a")); 0947 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1000); 0948 QCOMPARE(spreadsheet.column(2)->integerAt(0), 201811); 0949 QCOMPARE(spreadsheet.column(3)->valueAt(0), 1.1); 0950 } 0951 0952 /*! 0953 * test quoted text having separators inside - the text between quotes shouldn't be splitted into separate columns. 0954 */ 0955 void AsciiFilterTest::testQuotedStrings04() { 0956 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0957 AsciiFilter filter; 0958 const QString& fileName = QFINDTESTDATA(QLatin1String("data/quoted_strings_with_separator_inside.csv")); 0959 0960 filter.setHeaderLine(1); 0961 filter.setSimplifyWhitespacesEnabled(true); // TODO: this shouldn't be required, but QString::split() seems to introduce blanks... 0962 filter.setRemoveQuotesEnabled(true); 0963 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 0964 0965 // three rows and two columns to read 0966 // QCOMPARE(spreadsheet.rowCount(), 2); 0967 // QCOMPARE(spreadsheet.columnCount(), 3); 0968 0969 // column names 0970 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("id")); 0971 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("text")); 0972 QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("value")); 0973 0974 // data types 0975 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 0976 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Text); 0977 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Double); 0978 0979 // values 0980 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 0981 QCOMPARE(spreadsheet.column(1)->textAt(0), QLatin1String("some text, having a comma, and yet another comma")); 0982 QCOMPARE(spreadsheet.column(2)->valueAt(0), 1.0); 0983 0984 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 0985 QCOMPARE(spreadsheet.column(1)->textAt(1), QLatin1String("more text")); 0986 QCOMPARE(spreadsheet.column(2)->valueAt(1), 2.0); 0987 } 0988 0989 /*! 0990 * test quoted text having separators inside - a JSON file has a similar structure and we should't crash because of this "wrong" data. 0991 */ 0992 void AsciiFilterTest::testQuotedStrings05() { 0993 Spreadsheet spreadsheet(QStringLiteral("test"), false); 0994 AsciiFilter filter; 0995 const QString& fileName = QFINDTESTDATA(QLatin1String("data/object.json")); 0996 0997 filter.setSimplifyWhitespacesEnabled(true); // TODO: this shouldn't be required, but QString::split() seems to introduce blanks... 0998 filter.setRemoveQuotesEnabled(true); 0999 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1000 1001 // everything should be read into one single text column. 1002 // the actuall content is irrelevant, we just need to make sure we don't crash because of such wrong content 1003 QCOMPARE(spreadsheet.columnCount(), 1); 1004 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Text); 1005 } 1006 1007 // ############################################################################## 1008 // ################################## locales ################################### 1009 // ############################################################################## 1010 void AsciiFilterTest::testUtf8Cyrillic() { 1011 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1012 AsciiFilter filter; 1013 const QString& fileName = QFINDTESTDATA(QLatin1String("data/utf8_cyrillic.txt")); 1014 1015 filter.setSeparatingCharacter(QStringLiteral("auto")); 1016 filter.setHeaderLine(1); 1017 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1018 1019 // column names 1020 QCOMPARE(spreadsheet.column(0)->name(), QString::fromUtf8("перший_стовпець")); 1021 QCOMPARE(spreadsheet.column(1)->name(), QString::fromUtf8("другий_стовпець")); 1022 1023 // data types 1024 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Text); 1025 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 1026 1027 // values 1028 QCOMPARE(spreadsheet.column(0)->textAt(0), QString::fromUtf8("тест1")); 1029 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1); 1030 1031 QCOMPARE(spreadsheet.column(0)->textAt(1), QString::fromUtf8("тест2")); 1032 QCOMPARE(spreadsheet.column(1)->integerAt(1), 2); 1033 } 1034 1035 // ############################################################################## 1036 // ############################### skip comments ############################### 1037 // ############################################################################## 1038 void AsciiFilterTest::testComments00() { 1039 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1040 AsciiFilter filter; 1041 const QString& fileName = QFINDTESTDATA(QLatin1String("data/multi_line_comment.txt")); 1042 1043 filter.setHeaderEnabled(false); 1044 filter.setSeparatingCharacter(QStringLiteral(",")); 1045 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1046 1047 QCOMPARE(spreadsheet.columnCount(), 2); 1048 1049 // data types 1050 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 1051 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 1052 1053 // values 1054 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 1055 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.1); 1056 1057 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 1058 QCOMPARE(spreadsheet.column(1)->valueAt(1), 2.2); 1059 } 1060 1061 void AsciiFilterTest::testComments01() { 1062 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1063 AsciiFilter filter; 1064 const QString& fileName = QFINDTESTDATA(QLatin1String("data/multi_line_comment_with_empty_lines.txt")); 1065 1066 filter.setHeaderEnabled(false); 1067 filter.setSeparatingCharacter(QStringLiteral(",")); 1068 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1069 1070 QCOMPARE(spreadsheet.columnCount(), 2); 1071 1072 // data types 1073 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 1074 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 1075 1076 // values 1077 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 1078 QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.1); 1079 1080 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 1081 QCOMPARE(spreadsheet.column(1)->valueAt(1), 2.2); 1082 } 1083 1084 /*! 1085 * test with an empty comment character 1086 */ 1087 void AsciiFilterTest::testComments02() { 1088 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1089 AsciiFilter filter; 1090 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1091 1092 filter.setCommentCharacter(QString()); 1093 filter.setSeparatingCharacter(QStringLiteral(";")); 1094 filter.setHeaderEnabled(true); 1095 filter.setHeaderLine(1); 1096 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1097 1098 // spreadsheet size 1099 QCOMPARE(spreadsheet.columnCount(), 2); 1100 QCOMPARE(spreadsheet.rowCount(), 3); 1101 1102 // column names 1103 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("c1")); 1104 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("c2")); 1105 1106 // data types 1107 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::Integer); 1108 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Integer); 1109 1110 // values 1111 QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); 1112 QCOMPARE(spreadsheet.column(1)->integerAt(0), 1); 1113 1114 QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); 1115 QCOMPARE(spreadsheet.column(1)->integerAt(1), 2); 1116 1117 QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); 1118 QCOMPARE(spreadsheet.column(1)->integerAt(2), 3); 1119 } 1120 1121 // ############################################################################## 1122 // ######################### handling of datetime data ######################### 1123 // ############################################################################## 1124 /*! 1125 * read data containing only two characters for the year - 'yy'. The default year in 1126 * QDateTime is 1900 . When reading such two-characters DateTime values we want 1127 * to have the current centure after the import. 1128 */ 1129 void AsciiFilterTest::testDateTime00() { 1130 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1131 AsciiFilter filter; 1132 const QString& fileName = QFINDTESTDATA(QLatin1String("data/datetime_01.csv")); 1133 1134 filter.setSeparatingCharacter(QStringLiteral(",")); 1135 filter.setHeaderEnabled(true); 1136 filter.setHeaderLine(1); 1137 filter.setDateTimeFormat(QLatin1String("dd/MM/yy hh:mm:ss")); 1138 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1139 1140 // spreadsheet size 1141 QCOMPARE(spreadsheet.columnCount(), 2); 1142 QCOMPARE(spreadsheet.rowCount(), 2); 1143 1144 // column names 1145 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("Date")); 1146 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Water Pressure")); 1147 1148 // data types 1149 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::DateTime); 1150 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 1151 1152 // values 1153 auto value = QDateTime::fromString(QLatin1String("01/01/2019 00:00:00"), QLatin1String("dd/MM/yyyy hh:mm:ss")); 1154 value.setTimeSpec(Qt::UTC); 1155 QCOMPARE(spreadsheet.column(0)->dateTimeAt(0), value); 1156 QCOMPARE(spreadsheet.column(1)->valueAt(0), 14.7982); 1157 1158 value = QDateTime::fromString(QLatin1String("01/01/2019 00:30:00"), QLatin1String("dd/MM/yyyy hh:mm:ss")); 1159 value.setTimeSpec(Qt::UTC); 1160 QCOMPARE(spreadsheet.column(0)->dateTimeAt(1), value); 1161 QCOMPARE(spreadsheet.column(1)->valueAt(1), 14.8026); 1162 } 1163 1164 /*! 1165 * same as in the previous test, but with the auto-detection of the datetime format 1166 */ 1167 void AsciiFilterTest::testDateTime01() { 1168 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1169 AsciiFilter filter; 1170 const QString& fileName = QFINDTESTDATA(QLatin1String("data/datetime_01.csv")); 1171 1172 filter.setSeparatingCharacter(QStringLiteral(",")); 1173 filter.setHeaderEnabled(true); 1174 filter.setHeaderLine(1); 1175 filter.setDateTimeFormat(QLatin1String()); // auto detect the format 1176 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1177 1178 // spreadsheet size 1179 QCOMPARE(spreadsheet.columnCount(), 2); 1180 QCOMPARE(spreadsheet.rowCount(), 2); 1181 1182 // column names 1183 QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("Date")); 1184 QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Water Pressure")); 1185 1186 // data types 1187 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::DateTime); 1188 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Double); 1189 1190 // values 1191 auto value = QDateTime::fromString(QLatin1String("01/01/2019 00:00:00"), QLatin1String("dd/MM/yyyy hh:mm:ss")); 1192 value.setTimeSpec(Qt::UTC); 1193 QCOMPARE(spreadsheet.column(0)->dateTimeAt(0), value); 1194 QCOMPARE(spreadsheet.column(1)->valueAt(0), 14.7982); 1195 1196 value = QDateTime::fromString(QLatin1String("01/01/2019 00:30:00"), QLatin1String("dd/MM/yyyy hh:mm:ss")); 1197 value.setTimeSpec(Qt::UTC); 1198 QCOMPARE(spreadsheet.column(0)->dateTimeAt(1), value); 1199 QCOMPARE(spreadsheet.column(1)->valueAt(1), 14.8026); 1200 } 1201 1202 /* read datetime data before big int 1203 * TODO: handle hex value 1204 */ 1205 void AsciiFilterTest::testDateTimeHex() { 1206 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1207 AsciiFilter filter; 1208 const QString& fileName = QFINDTESTDATA(QLatin1String("data/datetime-hex.dat")); 1209 1210 filter.setHeaderEnabled(false); 1211 filter.setSeparatingCharacter(QStringLiteral("|")); 1212 filter.setDateTimeFormat(QLatin1String("yyyyMMddhhmmss")); 1213 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1214 1215 // spreadsheet size 1216 QCOMPARE(spreadsheet.columnCount(), 17); 1217 QCOMPARE(spreadsheet.rowCount(), 1); 1218 1219 // data types 1220 QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::ColumnMode::DateTime); 1221 QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::ColumnMode::Text); 1222 QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::ColumnMode::Integer); 1223 QCOMPARE(spreadsheet.column(3)->columnMode(), AbstractColumn::ColumnMode::Integer); 1224 QCOMPARE(spreadsheet.column(4)->columnMode(), AbstractColumn::ColumnMode::Integer); 1225 QCOMPARE(spreadsheet.column(5)->columnMode(), AbstractColumn::ColumnMode::Integer); 1226 QCOMPARE(spreadsheet.column(6)->columnMode(), AbstractColumn::ColumnMode::Integer); 1227 QCOMPARE(spreadsheet.column(7)->columnMode(), AbstractColumn::ColumnMode::Integer); 1228 QCOMPARE(spreadsheet.column(8)->columnMode(), AbstractColumn::ColumnMode::Integer); 1229 QCOMPARE(spreadsheet.column(9)->columnMode(), AbstractColumn::ColumnMode::Integer); 1230 QCOMPARE(spreadsheet.column(10)->columnMode(), AbstractColumn::ColumnMode::Double); 1231 QCOMPARE(spreadsheet.column(11)->columnMode(), AbstractColumn::ColumnMode::Integer); 1232 QCOMPARE(spreadsheet.column(12)->columnMode(), AbstractColumn::ColumnMode::Integer); 1233 QCOMPARE(spreadsheet.column(13)->columnMode(), AbstractColumn::ColumnMode::Text); 1234 QCOMPARE(spreadsheet.column(14)->columnMode(), AbstractColumn::ColumnMode::Double); 1235 QCOMPARE(spreadsheet.column(15)->columnMode(), AbstractColumn::ColumnMode::Integer); 1236 QCOMPARE(spreadsheet.column(16)->columnMode(), AbstractColumn::ColumnMode::Text); 1237 1238 auto value = QDateTime::fromString(QLatin1String("18/12/2019 02:36:08"), QLatin1String("dd/MM/yyyy hh:mm:ss")); 1239 value.setTimeSpec(Qt::UTC); 1240 QCOMPARE(spreadsheet.column(0)->dateTimeAt(0), value); 1241 QCOMPARE(spreadsheet.column(1)->textAt(0), QLatin1String("F")); 1242 QCOMPARE(spreadsheet.column(2)->integerAt(0), 1000); 1243 QCOMPARE(spreadsheet.column(3)->integerAt(0), 0); 1244 QCOMPARE(spreadsheet.column(4)->integerAt(0), 0); 1245 QCOMPARE(spreadsheet.column(5)->integerAt(0), 3190); 1246 QCOMPARE(spreadsheet.column(6)->integerAt(0), 528); 1247 QCOMPARE(spreadsheet.column(7)->integerAt(0), 3269); 1248 QCOMPARE(spreadsheet.column(8)->integerAt(0), 15); 1249 QCOMPARE(spreadsheet.column(9)->integerAt(0), 9); 1250 QCOMPARE(spreadsheet.column(10)->valueAt(0), 1.29); 1251 QCOMPARE(spreadsheet.column(11)->integerAt(0), 934); 1252 QCOMPARE(spreadsheet.column(12)->integerAt(0), -105); 1253 QCOMPARE(spreadsheet.column(13)->textAt(0), QLatin1String("G 03935")); 1254 QCOMPARE(spreadsheet.column(14)->valueAt(0), 94.09); 1255 QCOMPARE(spreadsheet.column(15)->integerAt(0), 9680); 1256 QCOMPARE(spreadsheet.column(16)->textAt(0), QLatin1String("5AD17")); 1257 } 1258 1259 void AsciiFilterTest::testMatrixHeader() { 1260 Matrix matrix(QStringLiteral("test"), false); 1261 AsciiFilter filter; 1262 const QString& fileName = QFINDTESTDATA(QLatin1String("data/numeric_data.txt")); 1263 1264 filter.setSeparatingCharacter(QStringLiteral("auto")); 1265 filter.readDataFromFile(fileName, &matrix, AbstractFileFilter::ImportMode::Replace); 1266 1267 QCOMPARE(matrix.rowCount(), 5); 1268 QCOMPARE(matrix.columnCount(), 3); 1269 1270 QCOMPARE(matrix.mode(), AbstractColumn::ColumnMode::Double); 1271 1272 // check all values 1273 QCOMPARE(matrix.cell<double>(0, 0), 1.716299); 1274 QCOMPARE(matrix.cell<double>(0, 1), -0.485527); 1275 QCOMPARE(matrix.cell<double>(0, 2), -0.288690); 1276 QCOMPARE(matrix.cell<double>(1, 0), 1.716299); 1277 QCOMPARE(matrix.cell<double>(1, 1), -0.476371); 1278 QCOMPARE(matrix.cell<double>(1, 2), -0.274957); 1279 QCOMPARE(matrix.cell<double>(2, 0), 1.711721); 1280 QCOMPARE(matrix.cell<double>(2, 1), -0.485527); 1281 QCOMPARE(matrix.cell<double>(2, 2), -0.293267); 1282 QCOMPARE(matrix.cell<double>(3, 0), 1.711721); 1283 QCOMPARE(matrix.cell<double>(3, 1), -0.480949); 1284 QCOMPARE(matrix.cell<double>(3, 2), -0.293267); 1285 QCOMPARE(matrix.cell<double>(4, 0), 1.716299); 1286 QCOMPARE(matrix.cell<double>(4, 1), -0.494682); 1287 QCOMPARE(matrix.cell<double>(4, 2), -0.284112); 1288 } 1289 1290 // ############################################################################## 1291 // ############# updates in the dependent objects after the import ############## 1292 // ############################################################################## 1293 /*! 1294 * test the update of the column values calculated via a formula after the values 1295 * in the source spreadsheet were modified by the import. 1296 */ 1297 void AsciiFilterTest::spreadsheetFormulaUpdateAfterImport() { 1298 // create the first spreadsheet with the source data 1299 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1300 spreadsheet.setColumnCount(2); 1301 spreadsheet.setRowCount(3); 1302 1303 auto* col = spreadsheet.column(0); 1304 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1305 col->setName(QStringLiteral("c1")); 1306 col->setValueAt(0, 10.); 1307 col->setValueAt(1, 20.); 1308 col->setValueAt(2, 30.); 1309 1310 col = spreadsheet.column(1); 1311 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1312 col->setName(QStringLiteral("c2")); 1313 col->setValueAt(0, 10.); 1314 col->setValueAt(1, 20.); 1315 col->setValueAt(2, 30.); 1316 1317 // create the second spreadsheet with one single column calculated via a formula from the first spreadsheet 1318 Spreadsheet spreadsheetFormula(QStringLiteral("formula"), false); 1319 spreadsheetFormula.setColumnCount(1); 1320 spreadsheetFormula.setRowCount(3); 1321 col = spreadsheetFormula.column(0); 1322 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1323 1324 QStringList variableNames{QLatin1String("x"), QLatin1String("y")}; 1325 QVector<Column*> variableColumns{spreadsheet.column(0), spreadsheet.column(1)}; 1326 col->setFormula(QLatin1String("x+y"), variableNames, variableColumns, true); 1327 col->updateFormula(); 1328 1329 // check the results of the calculation first 1330 QCOMPARE(spreadsheetFormula.columnCount(), 1); 1331 QCOMPARE(spreadsheetFormula.rowCount(), 3); 1332 QCOMPARE(col->columnMode(), AbstractColumn::ColumnMode::Double); 1333 QCOMPARE(col->valueAt(0), 20.); 1334 QCOMPARE(col->valueAt(1), 40.); 1335 QCOMPARE(col->valueAt(2), 60.); 1336 1337 // import the data into the source spreadsheet 1338 AsciiFilter filter; 1339 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1340 filter.setCommentCharacter(QString()); 1341 filter.setSeparatingCharacter(QStringLiteral(";")); 1342 filter.setHeaderEnabled(true); 1343 filter.setHeaderLine(1); 1344 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1345 1346 // re-check the results of the calculation 1347 QCOMPARE(spreadsheetFormula.columnCount(), 1); 1348 QCOMPARE(spreadsheetFormula.rowCount(), 3); 1349 QCOMPARE(col->columnMode(), AbstractColumn::ColumnMode::Double); 1350 QCOMPARE(col->valueAt(0), 2.); 1351 QCOMPARE(col->valueAt(1), 4.); 1352 QCOMPARE(col->valueAt(2), 6.); 1353 } 1354 1355 /*! 1356 * test the update of the column values calculated via a formula after one of the source columns 1357 * was deleted first and was restored and the source values were modified by the import. 1358 */ 1359 void AsciiFilterTest::spreadsheetFormulaUpdateAfterImportWithColumnRestore() { 1360 Project project; // need a project object since the column restore logic is in project 1361 1362 // create the first spreadsheet with the source data 1363 auto* spreadsheet = new Spreadsheet(QStringLiteral("test"), false); 1364 project.addChild(spreadsheet); 1365 spreadsheet->setColumnCount(2); 1366 spreadsheet->setRowCount(3); 1367 1368 auto* col = spreadsheet->column(0); 1369 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1370 col->setName(QStringLiteral("c1")); 1371 col->setValueAt(0, 10.); 1372 col->setValueAt(1, 20.); 1373 col->setValueAt(2, 30.); 1374 1375 col = spreadsheet->column(1); 1376 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1377 col->setName(QStringLiteral("c2")); 1378 col->setValueAt(0, 10.); 1379 col->setValueAt(1, 20.); 1380 col->setValueAt(2, 30.); 1381 1382 // create the second spreadsheet with one single column calculated via a formula from the first spreadsheet 1383 auto* spreadsheetFormula = new Spreadsheet(QStringLiteral("formula"), false); 1384 project.addChild(spreadsheetFormula); 1385 spreadsheetFormula->setColumnCount(1); 1386 spreadsheetFormula->setRowCount(3); 1387 col = spreadsheetFormula->column(0); 1388 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1389 1390 QStringList variableNames{QLatin1String("x"), QLatin1String("y")}; 1391 QVector<Column*> variableColumns{spreadsheet->column(0), spreadsheet->column(1)}; 1392 col->setFormula(QLatin1String("x+y"), variableNames, variableColumns, true); 1393 col->updateFormula(); 1394 1395 // delete the first column in the source spreadsheet and check the results of the calculation first, 1396 // the cells should be empty 1397 spreadsheet->removeChild(spreadsheet->column(0)); 1398 QCOMPARE(spreadsheetFormula->columnCount(), 1); 1399 QCOMPARE(spreadsheetFormula->rowCount(), 3); 1400 QCOMPARE(col->columnMode(), AbstractColumn::ColumnMode::Double); 1401 QCOMPARE((bool)std::isnan(col->valueAt(0)), true); 1402 QCOMPARE((bool)std::isnan(col->valueAt(1)), true); 1403 QCOMPARE((bool)std::isnan(col->valueAt(2)), true); 1404 1405 // import the data into the source spreadsheet, the deleted column with the name "c1" is re-created again 1406 AsciiFilter filter; 1407 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1408 filter.setCommentCharacter(QString()); 1409 filter.setSeparatingCharacter(QStringLiteral(";")); 1410 filter.setHeaderEnabled(true); 1411 filter.setHeaderLine(1); 1412 filter.readDataFromFile(fileName, spreadsheet, AbstractFileFilter::ImportMode::Replace); 1413 1414 // re-check the results of the calculation after one of the source columns was re-created and the values were changed 1415 QCOMPARE(spreadsheetFormula->columnCount(), 1); 1416 QCOMPARE(spreadsheetFormula->rowCount(), 3); 1417 QCOMPARE(col->columnMode(), AbstractColumn::ColumnMode::Double); 1418 QCOMPARE(col->valueAt(0), 2.); 1419 QCOMPARE(col->valueAt(1), 4.); 1420 QCOMPARE(col->valueAt(2), 6.); 1421 } 1422 1423 /*! 1424 * test the update of the xycurve and plot ranges after the values 1425 * in the source columns were modified by the import. 1426 */ 1427 void AsciiFilterTest::plotUpdateAfterImport() { 1428 // create the spreadsheet with the source data 1429 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1430 spreadsheet.setColumnCount(2); 1431 spreadsheet.setRowCount(3); 1432 1433 auto* col = spreadsheet.column(0); 1434 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1435 col->setName(QStringLiteral("c1")); 1436 col->setValueAt(0, 10.); 1437 col->setValueAt(1, 20.); 1438 col->setValueAt(2, 30.); 1439 1440 col = spreadsheet.column(1); 1441 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1442 col->setName(QStringLiteral("c2")); 1443 col->setValueAt(0, 10.); 1444 col->setValueAt(1, 20.); 1445 col->setValueAt(2, 30.); 1446 1447 // create a xy-curve with the both columns in the source spreadsheet and check the ranges 1448 CartesianPlot p(QStringLiteral("plot")); 1449 auto* curve = new XYCurve(QStringLiteral("curve")); 1450 p.addChild(curve); 1451 curve->setXColumn(spreadsheet.column(0)); 1452 curve->setYColumn(spreadsheet.column(1)); 1453 1454 auto rangeX = p.range(Dimension::X); 1455 QCOMPARE(rangeX.start(), 10); 1456 QCOMPARE(rangeX.end(), 30); 1457 1458 auto rangeY = p.range(Dimension::Y); 1459 QCOMPARE(rangeY.start(), 10); 1460 QCOMPARE(rangeY.end(), 30); 1461 1462 // import the data into the source spreadsheet 1463 AsciiFilter filter; 1464 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1465 filter.setCommentCharacter(QString()); 1466 filter.setSeparatingCharacter(QStringLiteral(";")); 1467 filter.setHeaderEnabled(true); 1468 filter.setHeaderLine(1); 1469 filter.readDataFromFile(fileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1470 1471 // re-check the plot ranges with the new data 1472 rangeX = p.range(Dimension::X); 1473 QCOMPARE(rangeX.start(), 1); 1474 QCOMPARE(rangeX.end(), 3); 1475 1476 rangeY = p.range(Dimension::Y); 1477 QCOMPARE(rangeY.start(), 1); 1478 QCOMPARE(rangeY.end(), 3); 1479 } 1480 1481 /*! 1482 * test the update of the xycurve and plot ranges after one of the source columns 1483 * was deleted first and was restored and the source values were modified by the import. 1484 */ 1485 void AsciiFilterTest::plotUpdateAfterImportWithColumnRestore() { 1486 Project project; // need a project object since the column restore logic is in project 1487 1488 // create the spreadsheet with the source data 1489 auto* spreadsheet = new Spreadsheet(QStringLiteral("test"), false); 1490 project.addChild(spreadsheet); 1491 spreadsheet->setColumnCount(2); 1492 spreadsheet->setRowCount(3); 1493 1494 auto* col = spreadsheet->column(0); 1495 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1496 col->setName(QStringLiteral("c1")); 1497 col->setValueAt(0, 10.); 1498 col->setValueAt(1, 20.); 1499 col->setValueAt(2, 30.); 1500 1501 col = spreadsheet->column(1); 1502 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1503 col->setName(QStringLiteral("c2")); 1504 col->setValueAt(0, 10.); 1505 col->setValueAt(1, 20.); 1506 col->setValueAt(2, 30.); 1507 1508 // create a xy-curve with the both columns in the source spreadsheet and check the ranges 1509 auto* p = new CartesianPlot(QStringLiteral("plot")); 1510 project.addChild(p); 1511 auto* curve = new XYCurve(QStringLiteral("curve")); 1512 p->addChild(curve); 1513 curve->setXColumn(spreadsheet->column(0)); 1514 curve->setYColumn(spreadsheet->column(1)); 1515 1516 auto rangeX = p->range(Dimension::X); 1517 QCOMPARE(rangeX.start(), 10); 1518 QCOMPARE(rangeX.end(), 30); 1519 1520 auto rangeY = p->range(Dimension::Y); 1521 QCOMPARE(rangeY.start(), 10); 1522 QCOMPARE(rangeY.end(), 30); 1523 1524 // delete the first source column 1525 spreadsheet->removeChild(spreadsheet->column(0)); 1526 1527 // import the data into the source spreadsheet, the deleted column with the name "c1" is re-created again 1528 AsciiFilter filter; 1529 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1530 filter.setCommentCharacter(QString()); 1531 filter.setSeparatingCharacter(QStringLiteral(";")); 1532 filter.setHeaderEnabled(true); 1533 filter.setHeaderLine(1); 1534 filter.readDataFromFile(fileName, spreadsheet, AbstractFileFilter::ImportMode::Replace); 1535 1536 // re-check the plot ranges with the new data 1537 rangeX = p->range(Dimension::X); 1538 QCOMPARE(rangeX.start(), 1); 1539 QCOMPARE(rangeX.end(), 3); 1540 1541 rangeY = p->range(Dimension::Y); 1542 QCOMPARE(rangeY.start(), 1); 1543 QCOMPARE(rangeY.end(), 3); 1544 } 1545 1546 /*! 1547 * test the update of the xycurve and plot ranges after the order or columns (their names) 1548 * was changed during the import. 1549 */ 1550 void AsciiFilterTest::plotUpdateAfterImportWithColumnRenaming() { 1551 Project project; // need a project object since the column restore logic is in project 1552 1553 // create the spreadsheet with the source data 1554 auto* spreadsheet = new Spreadsheet(QStringLiteral("test"), false); 1555 project.addChild(spreadsheet); 1556 spreadsheet->setColumnCount(3); 1557 spreadsheet->setRowCount(3); 1558 1559 auto* col = spreadsheet->column(0); 1560 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1561 col->setName(QStringLiteral("c0")); // dummy column, values not required 1562 1563 col = spreadsheet->column(1); 1564 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1565 col->setName(QStringLiteral("c1")); 1566 col->setValueAt(0, 10.); 1567 col->setValueAt(1, 20.); 1568 col->setValueAt(2, 30.); 1569 1570 col = spreadsheet->column(2); 1571 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1572 col->setName(QStringLiteral("c2")); 1573 col->setValueAt(0, 10.); 1574 col->setValueAt(1, 20.); 1575 col->setValueAt(2, 30.); 1576 1577 // create a xy-curve with the both columns in the source spreadsheet and check the ranges 1578 auto* p = new CartesianPlot(QStringLiteral("plot")); 1579 project.addChild(p); 1580 auto* curve = new XYCurve(QStringLiteral("curve")); 1581 p->addChild(curve); 1582 curve->setXColumn(spreadsheet->column(1)); // use "c1" for x 1583 curve->setYColumn(spreadsheet->column(2)); // use "c2" for y 1584 1585 auto rangeX = p->range(Dimension::X); 1586 QCOMPARE(rangeX.start(), 10); 1587 QCOMPARE(rangeX.end(), 30); 1588 1589 auto rangeY = p->range(Dimension::Y); 1590 QCOMPARE(rangeY.start(), 10); 1591 QCOMPARE(rangeY.end(), 30); 1592 1593 // import the data into the source spreadsheet: 1594 // the columns "c0", "c1" and "c2" are renamed to "c1", "c2" and "c3" and xy-curve should still be using "c1" and "c2" 1595 // event hough their positions in the spreadsheet have changed 1596 AsciiFilter filter; 1597 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1598 filter.setCommentCharacter(QString()); 1599 filter.setSeparatingCharacter(QStringLiteral(";")); 1600 filter.setHeaderEnabled(true); 1601 filter.setHeaderLine(1); 1602 filter.readDataFromFile(fileName, spreadsheet, AbstractFileFilter::ImportMode::Replace); 1603 1604 // check the connection to the new columns 1605 QCOMPARE(curve->xColumn(), spreadsheet->column(0)); // "c1" for x 1606 QCOMPARE(curve->yColumn(), spreadsheet->column(1)); // "c2" for y 1607 1608 // re-check the plot ranges with the new data 1609 rangeX = p->range(Dimension::X); 1610 QCOMPARE(rangeX.start(), 1); 1611 QCOMPARE(rangeX.end(), 3); 1612 1613 rangeY = p->range(Dimension::Y); 1614 QCOMPARE(rangeY.start(), 1); 1615 QCOMPARE(rangeY.end(), 3); 1616 } 1617 1618 /*! 1619 * test the update of the xycurve after the source columns were renamed 1620 * during the import, the curve becomes invalid after this. 1621 */ 1622 void AsciiFilterTest::plotUpdateAfterImportWithColumnRemove() { 1623 Project project; // need a project object since the column restore logic is in project 1624 1625 // create the spreadsheet with the source data 1626 auto* spreadsheet = new Spreadsheet(QStringLiteral("test"), false); 1627 project.addChild(spreadsheet); 1628 spreadsheet->setColumnCount(2); 1629 spreadsheet->setRowCount(3); 1630 1631 auto* col = spreadsheet->column(0); 1632 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1633 col->setName(QStringLiteral("1")); 1634 col->setValueAt(0, 10.); 1635 col->setValueAt(1, 20.); 1636 col->setValueAt(2, 30.); 1637 1638 col = spreadsheet->column(1); 1639 col->setColumnMode(AbstractColumn::ColumnMode::Double); 1640 col->setName(QStringLiteral("2")); 1641 col->setValueAt(0, 10.); 1642 col->setValueAt(1, 20.); 1643 col->setValueAt(2, 30.); 1644 1645 // create a xy-curve with the both columns in the source spreadsheet and check the ranges 1646 auto* p = new CartesianPlot(QStringLiteral("plot")); 1647 project.addChild(p); 1648 auto* curve = new XYCurve(QStringLiteral("curve")); 1649 p->addChild(curve); 1650 curve->setXColumn(spreadsheet->column(0)); // use "1" for x 1651 curve->setYColumn(spreadsheet->column(1)); // use "2" for y 1652 1653 auto rangeX = p->range(Dimension::X); 1654 QCOMPARE(rangeX.start(), 10); 1655 QCOMPARE(rangeX.end(), 30); 1656 1657 auto rangeY = p->range(Dimension::Y); 1658 QCOMPARE(rangeY.start(), 10); 1659 QCOMPARE(rangeY.end(), 30); 1660 1661 // import the data into the source spreadsheet, the columns are renamed to "c1" and "c2" 1662 AsciiFilter filter; 1663 const QString& fileName = QFINDTESTDATA(QLatin1String("data/separator_semicolon_with_header.txt")); 1664 filter.setCommentCharacter(QString()); 1665 filter.setSeparatingCharacter(QStringLiteral(";")); 1666 filter.setHeaderEnabled(true); 1667 filter.setHeaderLine(1); 1668 filter.readDataFromFile(fileName, spreadsheet, AbstractFileFilter::ImportMode::Replace); 1669 1670 // the assignment to the data columns got lost since the columns were renamed 1671 QCOMPARE(curve->xColumn(), nullptr); 1672 QCOMPARE(curve->yColumn(), nullptr); 1673 1674 // TODO: further checks to make sure the curve was really and properly invalidated 1675 } 1676 1677 // ############################################################################## 1678 // ################################# Benchmarks ################################# 1679 // ############################################################################## 1680 void AsciiFilterTest::benchDoubleImport_data() { 1681 QTest::addColumn<size_t>("lineCount"); 1682 // can't transfer file name since needed in clean up 1683 1684 QTemporaryFile file; 1685 if (!file.open()) // needed to generate file name 1686 return; 1687 1688 file.setAutoRemove(false); 1689 benchDataFileName = file.fileName(); 1690 1691 QString testName(QString::number(paths) + QLatin1String(" random double paths")); 1692 1693 QTest::newRow(qPrintable(testName)) << lines; 1694 DEBUG("CREATE DATA FILE " << STDSTRING(benchDataFileName) << ", lines = " << lines) 1695 1696 gsl_rng_env_setup(); 1697 gsl_rng* r = gsl_rng_alloc(gsl_rng_default); 1698 gsl_rng_set(r, 12345); 1699 1700 // create file 1701 QTextStream out(&file); 1702 // for higher precision 1703 // out.setRealNumberPrecision(13); 1704 1705 // create data 1706 double path[paths] = {0.0}; 1707 1708 const double delta = 0.25; 1709 const int dt = 1; 1710 const double sigma = delta * delta * dt; 1711 for (size_t i = 0; i < lines; ++i) { 1712 // std::cout << "line " << i+1 << std::endl; 1713 1714 for (int p = 0; p < paths; ++p) { 1715 path[p] += gsl_ran_gaussian_ziggurat(r, sigma); 1716 out << path[p]; 1717 if (p < paths - 1) 1718 out << ' '; 1719 } 1720 out << QStringLiteral("\n"); 1721 } 1722 1723 DEBUG(Q_FUNC_INFO << ", DONE") 1724 } 1725 1726 void AsciiFilterTest::benchDoubleImport() { 1727 Spreadsheet spreadsheet(QStringLiteral("test"), false); 1728 AsciiFilter filter; 1729 filter.setHeaderEnabled(false); 1730 1731 const int p = paths; // need local variable 1732 QBENCHMARK { 1733 filter.readDataFromFile(benchDataFileName, &spreadsheet, AbstractFileFilter::ImportMode::Replace); 1734 1735 QCOMPARE(spreadsheet.columnCount(), p); 1736 QCOMPARE(spreadsheet.rowCount(), lines); 1737 1738 QCOMPARE(spreadsheet.column(0)->valueAt(0), 0.120998); 1739 QCOMPARE(spreadsheet.column(1)->valueAt(0), 0.119301); 1740 QCOMPARE(spreadsheet.column(2)->valueAt(0), -0.0209980); 1741 } 1742 } 1743 1744 void AsciiFilterTest::benchDoubleImport_cleanup() { 1745 DEBUG("REMOVE DATA FILE " << STDSTRING(benchDataFileName)) 1746 QFile::remove(benchDataFileName); 1747 } 1748 1749 QTEST_MAIN(AsciiFilterTest)