core/column/ColumnPrivate.cpp

0001 /*
0002     File                 : ColumnPrivate.cpp
0003     Project              : AbstractColumn
0004     Description          : Private data class of Column
0005     --------------------------------------------------------------------
0006     SPDX-FileCopyrightText: 2007-2008 Tilman Benkert <thzs@gmx.net>
0007     SPDX-FileCopyrightText: 2012-2022 Alexander Semke <alexander.semke@web.de>
0008     SPDX-FileCopyrightText: 2017-2022 Stefan Gerlach <stefan.gerlach@uni.kn>
0009
0010     SPDX-License-Identifier: GPL-2.0-or-later
0011 */
0012
0013 #include "ColumnPrivate.h"
0014 #include "Column.h"
0015 #include "ColumnStringIO.h"
0016 #include "backend/core/datatypes/filter.h"
0017 #include "backend/gsl/ExpressionParser.h"
0018 #include "backend/lib/trace.h"
0019 #include "backend/spreadsheet/Spreadsheet.h"
0020
0021 #include "backend/nsl/nsl_stats.h"
0022 #include <gsl/gsl_math.h>
0023 #include <gsl/gsl_statistics.h>
0024
0025 #include "functions.h"
0026
0027 #include <array>
0028 #include <unordered_map>
0029
0030 namespace {
0031 template<typename T>
0032 int indexForValueCommon(const T* obj,
0033                         double x,
0034                         const std::function<AbstractColumn::ColumnMode(const T*)> columnMode,
0035                         const std::function<int(const T*)> rowCount,
0036                         const std::function<double(const T*, int)> valueAt,
0037                         const std::function<QDateTime(const T*, int)> dateTimeAt,
0038                         const std::function<AbstractColumn::Properties(const T*)> properties,
0039                         const std::function<bool(const T*, int)> isValid,
0040                         const std::function<bool(const T*, int)> isMasked) {
0041     int rc = rowCount(obj);
0042     double prevValue = 0;
0043     qint64 prevValueDateTime = 0;
0044     auto mode = columnMode(obj);
0045     auto property = properties(obj);
0046     if (property == Column::Properties::MonotonicIncreasing || property == Column::Properties::MonotonicDecreasing) {
0047         // bisects the index every time, so it is possible to find the value in log_2(rowCount) steps
0048         bool increase = (property != Column::Properties::MonotonicDecreasing);
0049
0050         int lowerIndex = 0;
0051         int higherIndex = rc - 1;
0052
0053         unsigned int maxSteps = Column::calculateMaxSteps(static_cast<unsigned int>(rc)) + 1;
0054
0055         switch (mode) {
0056         case Column::ColumnMode::Double:
0057         case Column::ColumnMode::Integer:
0058         case Column::ColumnMode::BigInt:
0059             for (unsigned int i = 0; i < maxSteps; i++) { // so no log_2(rowCount) needed
0060                 int index = lowerIndex + round(static_cast<double>(higherIndex - lowerIndex) / 2);
0061                 double value = valueAt(obj, index);
0062
0063                 if (higherIndex - lowerIndex < 2) {
0064                     if (std::abs(valueAt(obj, lowerIndex) - x) < std::abs(valueAt(obj, higherIndex) - x))
0065                         index = lowerIndex;
0066                     else
0067                         index = higherIndex;
0068
0069                     return index;
0070                 }
0071
0072                 if (value > x && increase)
0073                     higherIndex = index;
0074                 else if (value >= x && !increase)
0075                     lowerIndex = index;
0076                 else if (value <= x && increase)
0077                     lowerIndex = index;
0078                 else if (value < x && !increase)
0079                     higherIndex = index;
0080             }
0081             break;
0082         case Column::ColumnMode::Text:
0083             break;
0084         case Column::ColumnMode::DateTime:
0085         case Column::ColumnMode::Month:
0086         case Column::ColumnMode::Day: {
0087             qint64 xInt64 = static_cast<qint64>(x);
0088             for (unsigned int i = 0; i < maxSteps; i++) { // so no log_2(rowCount) needed
0089                 int index = lowerIndex + round(static_cast<double>(higherIndex - lowerIndex) / 2);
0090                 qint64 value = dateTimeAt(obj, index).toMSecsSinceEpoch();
0091
0092                 if (higherIndex - lowerIndex < 2) {
0093                     if (std::abs(dateTimeAt(obj, lowerIndex).toMSecsSinceEpoch() - xInt64)
0094                         < std::abs(dateTimeAt(obj, higherIndex).toMSecsSinceEpoch() - xInt64))
0095                         index = lowerIndex;
0096                     else
0097                         index = higherIndex;
0098
0099                     return index;
0100                 }
0101
0102                 if (value > xInt64 && increase)
0103                     higherIndex = index;
0104                 else if (value >= xInt64 && !increase)
0105                     lowerIndex = index;
0106                 else if (value <= xInt64 && increase)
0107                     lowerIndex = index;
0108                 else if (value < xInt64 && !increase)
0109                     higherIndex = index;
0110             }
0111         }
0112         }
0113
0114     } else if (property == Column::Properties::Constant) {
0115         if (rc > 0)
0116             return 0;
0117         else
0118             return -1;
0119     } else {
0120         // naiv way
0121         int index = 0;
0122         switch (mode) {
0123         case Column::ColumnMode::Double:
0124         case Column::ColumnMode::Integer:
0125         case Column::ColumnMode::BigInt:
0126             for (int row = 0; row < rc; row++) {
0127                 if (!isValid(obj, row) || isMasked(obj, row))
0128                     continue;
0129                 if (row == 0)
0130                     prevValue = valueAt(obj, row);
0131
0132                 double value = valueAt(obj, row);
0133                 if (std::abs(value - x) <= std::abs(prevValue - x)) { // <= prevents also that row - 1 become < 0
0134                     prevValue = value;
0135                     index = row;
0136                 }
0137             }
0138             return index;
0139         case Column::ColumnMode::Text:
0140             break;
0141         case Column::ColumnMode::DateTime:
0142         case Column::ColumnMode::Month:
0143         case Column::ColumnMode::Day: {
0144             qint64 xInt64 = static_cast<qint64>(x);
0145             for (int row = 0; row < rc; row++) {
0146                 if (!isValid(obj, row) || isMasked(obj, row))
0147                     continue;
0148
0149                 if (row == 0)
0150                     prevValueDateTime = dateTimeAt(obj, row).toMSecsSinceEpoch();
0151
0152                 qint64 value = dateTimeAt(obj, row).toMSecsSinceEpoch();
0153                 if (std::abs(value - xInt64) <= std::abs(prevValueDateTime - xInt64)) { // "<=" prevents also that row - 1 become < 0
0154                     prevValueDateTime = value;
0155                     index = row;
0156                 }
0157             }
0158             return index;
0159         }
0160         }
0161     }
0162     return -1;
0163 }
0164 } // anonymous namespace
0165
0166 void ColumnPrivate::ValueLabels::setMode(AbstractColumn::ColumnMode mode) {
0167     if (!initialized())
0168         m_mode = mode;
0169     else
0170         migrateLabels(mode);
0171 }
0172
0173 bool ColumnPrivate::ValueLabels::init(AbstractColumn::ColumnMode mode) {
0174     if (initialized())
0175         return false;
0176
0177     invalidateStatistics();
0178
0179     m_mode = mode;
0180     switch (m_mode) {
0181     case AbstractColumn::ColumnMode::Double:
0182         m_labels = new QVector<Column::ValueLabel<double>>();
0183         break;
0184     case AbstractColumn::ColumnMode::Integer:
0185         m_labels = new QVector<Column::ValueLabel<int>>();
0186         break;
0187     case AbstractColumn::ColumnMode::BigInt:
0188         m_labels = new QVector<Column::ValueLabel<qint64>>();
0189         break;
0190     case AbstractColumn::ColumnMode::Text:
0191         m_labels = new QVector<Column::ValueLabel<QString>>();
0192         break;
0193     case AbstractColumn::ColumnMode::DateTime:
0194     case AbstractColumn::ColumnMode::Month:
0195     case AbstractColumn::ColumnMode::Day:
0196         m_labels = new QVector<Column::ValueLabel<QDateTime>>();
0197         break;
0198     }
0199     return true;
0200 }
0201
0202 void ColumnPrivate::ValueLabels::deinit() {
0203     invalidateStatistics();
0204     if (m_labels) {
0205         switch (m_mode) {
0206         case AbstractColumn::ColumnMode::Double:
0207             delete cast_vector<double>();
0208             break;
0209         case AbstractColumn::ColumnMode::Integer:
0210             delete cast_vector<int>();
0211             break;
0212         case AbstractColumn::ColumnMode::BigInt:
0213             delete cast_vector<qint64>();
0214             break;
0215         case AbstractColumn::ColumnMode::Text:
0216             delete cast_vector<QString>();
0217             break;
0218         case AbstractColumn::ColumnMode::DateTime:
0219         case AbstractColumn::ColumnMode::Month:
0220         case AbstractColumn::ColumnMode::Day:
0221             delete cast_vector<QDateTime>();
0222             break;
0223         }
0224
0225         m_labels = nullptr;
0226     }
0227 }
0228
0229 AbstractColumn::ColumnMode ColumnPrivate::ValueLabels::mode() const {
0230     return m_mode;
0231 }
0232 AbstractColumn::Properties ColumnPrivate::ValueLabels::properties() const {
0233     return AbstractColumn::Properties::No; // Performance improvements not yet implemented
0234 }
0235
0236 double ColumnPrivate::ValueLabels::valueAt(int index) const {
0237     if (!initialized())
0238         return 0;
0239
0240     switch (m_mode) {
0241     case AbstractColumn::ColumnMode::Double:
0242         return cast_vector<double>()->at(index).value;
0243     case AbstractColumn::ColumnMode::Integer:
0244         return cast_vector<int>()->at(index).value;
0245     case AbstractColumn::ColumnMode::BigInt:
0246         return cast_vector<qint64>()->at(index).value;
0247     case AbstractColumn::ColumnMode::Text:
0248         return std::nan("0");
0249     case AbstractColumn::ColumnMode::DateTime:
0250     case AbstractColumn::ColumnMode::Month:
0251     case AbstractColumn::ColumnMode::Day:
0252         return cast_vector<QDateTime>()->at(index).value.toMSecsSinceEpoch();
0253     }
0254     Q_ASSERT(false);
0255     return std::nan("0");
0256 }
0257
0258 QDateTime ColumnPrivate::ValueLabels::dateTimeAt(int index) const {
0259     if (!initialized())
0260         return QDateTime();
0261
0262     switch (m_mode) {
0263     case AbstractColumn::ColumnMode::DateTime:
0264     case AbstractColumn::ColumnMode::Month:
0265     case AbstractColumn::ColumnMode::Day:
0266         return cast_vector<QDateTime>()->at(index).value;
0267     case AbstractColumn::ColumnMode::Double:
0268     case AbstractColumn::ColumnMode::Integer:
0269     case AbstractColumn::ColumnMode::BigInt:
0270     case AbstractColumn::ColumnMode::Text:
0271         return QDateTime();
0272     }
0273     Q_ASSERT(false);
0274     return QDateTime();
0275 }
0276
0277 void ColumnPrivate::ValueLabels::migrateLabels(AbstractColumn::ColumnMode newMode) {
0278     switch (mode()) {
0279     case AbstractColumn::ColumnMode::Double:
0280         migrateDoubleTo(newMode);
0281         break;
0282     case AbstractColumn::ColumnMode::Integer:
0283         migrateIntTo(newMode);
0284         break;
0285     case AbstractColumn::ColumnMode::BigInt:
0286         migrateBigIntTo(newMode);
0287         break;
0288     case AbstractColumn::ColumnMode::Text:
0289         migrateTextTo(newMode);
0290         break;
0291     case AbstractColumn::ColumnMode::DateTime:
0292     case AbstractColumn::ColumnMode::Month:
0293     case AbstractColumn::ColumnMode::Day:
0294         migrateDateTimeTo(newMode);
0295         break;
0296     }
0297 }
0298
0299 void ColumnPrivate::ValueLabels::migrateDoubleTo(AbstractColumn::ColumnMode newMode) {
0300     if (newMode == AbstractColumn::ColumnMode::Double)
0301         return;
0302
0303     auto vector = *cast_vector<double>();
0304     deinit();
0305     init(newMode);
0306     switch (newMode) {
0307     case AbstractColumn::ColumnMode::Double:
0308         break; // Nothing to do
0309     case AbstractColumn::ColumnMode::Integer:
0310         for (const auto& value : vector)
0311             add((int)value.value, value.label);
0312         break;
0313     case AbstractColumn::ColumnMode::BigInt:
0314         for (const auto& value : vector)
0315             add((qint64)value.value, value.label);
0316         break;
0317     case AbstractColumn::ColumnMode::Text:
0318         for (const auto& value : vector)
0319             add(QString::number(value.value), value.label);
0320         break;
0321     case AbstractColumn::ColumnMode::DateTime:
0322     case AbstractColumn::ColumnMode::Month:
0323     case AbstractColumn::ColumnMode::Day:
0324         // Not possible
0325         // All value labels deleted
0326         break;
0327     }
0328 }
0329
0330 void ColumnPrivate::ValueLabels::migrateIntTo(AbstractColumn::ColumnMode newMode) {
0331     if (newMode == AbstractColumn::ColumnMode::Integer)
0332         return;
0333
0334     auto vector = *cast_vector<int>();
0335     deinit();
0336     init(newMode);
0337     switch (newMode) {
0338     case AbstractColumn::ColumnMode::Double:
0339         for (const auto& value : vector)
0340             add((double)value.value, value.label);
0341         break;
0342     case AbstractColumn::ColumnMode::Integer:
0343         // nothing to do
0344         break;
0345     case AbstractColumn::ColumnMode::BigInt:
0346         for (const auto& value : vector)
0347             add((qint64)value.value, value.label);
0348         break;
0349     case AbstractColumn::ColumnMode::Text:
0350         for (const auto& value : vector)
0351             add(QString::number(value.value), value.label);
0352         break;
0353     case AbstractColumn::ColumnMode::DateTime:
0354     case AbstractColumn::ColumnMode::Month:
0355     case AbstractColumn::ColumnMode::Day:
0356         // Not possible
0357         // All value labels deleted
0358         break;
0359     }
0360 }
0361
0362 void ColumnPrivate::ValueLabels::migrateBigIntTo(AbstractColumn::ColumnMode newMode) {
0363     if (newMode == AbstractColumn::ColumnMode::BigInt)
0364         return;
0365
0366     auto vector = *cast_vector<qint64>();
0367     deinit();
0368     init(newMode);
0369     switch (newMode) {
0370     case AbstractColumn::ColumnMode::Double:
0371         for (const auto& value : vector)
0372             add((double)value.value, value.label);
0373         break;
0374     case AbstractColumn::ColumnMode::Integer:
0375         for (const auto& value : vector)
0376             add((int)value.value, value.label);
0377         break;
0378     case AbstractColumn::ColumnMode::BigInt:
0379         // Nothing to do
0380         break;
0381     case AbstractColumn::ColumnMode::Text:
0382         for (const auto& value : vector)
0383             add(QString::number(value.value), value.label);
0384         break;
0385     case AbstractColumn::ColumnMode::DateTime:
0386     case AbstractColumn::ColumnMode::Month:
0387     case AbstractColumn::ColumnMode::Day:
0388         // Not possible
0389         // All value labels deleted
0390         break;
0391     }
0392 }
0393
0394 void ColumnPrivate::ValueLabels::migrateTextTo(AbstractColumn::ColumnMode newMode) {
0395     if (newMode == AbstractColumn::ColumnMode::Text)
0396         return;
0397
0398     auto vector = *cast_vector<QString>();
0399     deinit();
0400     init(newMode);
0401     switch (newMode) {
0402     case AbstractColumn::ColumnMode::Double: {
0403         for (const auto& value : vector) {
0404             bool ok;
0405             double v = value.value.toDouble(&ok);
0406             if (ok)
0407                 add(v, value.label);
0408         }
0409         break;
0410     }
0411     case AbstractColumn::ColumnMode::Integer: {
0412         for (const auto& value : vector) {
0413             bool ok;
0414             int v = value.value.toInt(&ok);
0415             if (ok)
0416                 add(v, value.label);
0417         }
0418         break;
0419     }
0420     case AbstractColumn::ColumnMode::BigInt: {
0421         for (const auto& value : vector) {
0422             bool ok;
0423             qint64 v = value.value.toLongLong(&ok);
0424             if (ok)
0425                 add(v, value.label);
0426         }
0427         break;
0428     }
0429     case AbstractColumn::ColumnMode::Text:
0430         // Nothing to do
0431         break;
0432     case AbstractColumn::ColumnMode::DateTime:
0433     case AbstractColumn::ColumnMode::Month:
0434     case AbstractColumn::ColumnMode::Day:
0435         // Not supported
0436         break;
0437     }
0438 }
0439
0440 int ColumnPrivate::ValueLabels::indexForValue(double value) const {
0441     return indexForValueCommon<ValueLabels>(this,
0442                                             value,
0443                                             std::mem_fn(&ValueLabels::mode),
0444                                             std::mem_fn<int() const>(&ValueLabels::count),
0445                                             std::mem_fn(&ValueLabels::valueAt),
0446                                             std::mem_fn(&ValueLabels::dateTimeAt),
0447                                             std::mem_fn(&ValueLabels::properties),
0448                                             std::mem_fn<bool(int) const>(&ValueLabels::isValid),
0449                                             std::mem_fn<bool(int) const>(&ValueLabels::isMasked));
0450 }
0451
0452 bool ColumnPrivate::ValueLabels::isValid(int) const {
0453     return true;
0454 }
0455
0456 bool ColumnPrivate::ValueLabels::isMasked(int) const {
0457     return false;
0458 }
0459
0460 QString ColumnPrivate::ValueLabels::labelAt(int index) const {
0461     if (!initialized())
0462         return QStringLiteral();
0463
0464     switch (m_mode) {
0465     case AbstractColumn::ColumnMode::Double:
0466         return cast_vector<double>()->at(index).label;
0467     case AbstractColumn::ColumnMode::Integer:
0468         return cast_vector<int>()->at(index).label;
0469     case AbstractColumn::ColumnMode::BigInt:
0470         return cast_vector<qint64>()->at(index).label;
0471     case AbstractColumn::ColumnMode::Text:
0472         return cast_vector<QString>()->at(index).label;
0473     case AbstractColumn::ColumnMode::DateTime:
0474     case AbstractColumn::ColumnMode::Month:
0475     case AbstractColumn::ColumnMode::Day:
0476         return cast_vector<QDateTime>()->at(index).label;
0477     }
0478     Q_ASSERT(false);
0479     return QStringLiteral();
0480 }
0481
0482 double ColumnPrivate::ValueLabels::minimum() {
0483     if (!m_statistics.available)
0484         recalculateStatistics();
0485     return m_statistics.minimum;
0486 }
0487
0488 double ColumnPrivate::ValueLabels::maximum() {
0489     if (!m_statistics.available)
0490         recalculateStatistics();
0491     return m_statistics.maximum;
0492 }
0493
0494 void ColumnPrivate::ValueLabels::recalculateStatistics() {
0495     m_statistics.available = false;
0496     m_statistics.minimum = INFINITY;
0497     m_statistics.maximum = -INFINITY;
0498
0499     const int rowValuesSize = count();
0500     for (int row = 0; row < rowValuesSize; ++row) {
0501         const double val = valueAt(row);
0502         if (val < m_statistics.minimum)
0503             m_statistics.minimum = val;
0504         if (val > m_statistics.maximum)
0505             m_statistics.maximum = val;
0506     }
0507
0508     m_statistics.available = true;
0509 }
0510
0511 void ColumnPrivate::ValueLabels::invalidateStatistics() {
0512     m_statistics.available = false;
0513 }
0514
0515 void ColumnPrivate::ValueLabels::migrateDateTimeTo(AbstractColumn::ColumnMode newMode) {
0516     if (newMode == AbstractColumn::ColumnMode::DateTime || newMode == AbstractColumn::ColumnMode::Day || newMode == AbstractColumn::ColumnMode::Month)
0517         return;
0518
0519     // auto vector = *cast_vector<QDateTime>();
0520     deinit();
0521     init(newMode);
0522     // switch (newMode) {
0523     // case AbstractColumn::ColumnMode::Double: {
0524     //     // Not possible
0525     //     break;
0526     // }
0527     // case AbstractColumn::ColumnMode::Integer: {
0528     //     // Not possible
0529     //     break;
0530     // }
0531     // case AbstractColumn::ColumnMode::BigInt: {
0532     //     // Not possible
0533     //     break;
0534     // }
0535     // case AbstractColumn::ColumnMode::Text:
0536     //     // Not supported
0537     //     break;
0538     // case AbstractColumn::ColumnMode::DateTime:
0539     // case AbstractColumn::ColumnMode::Month:
0540     // case AbstractColumn::ColumnMode::Day:
0541     //     // Nothing to do
0542     //     break;
0543     // }
0544 }
0545
0546 int ColumnPrivate::ValueLabels::count() const {
0547     if (!initialized())
0548         return 0;
0549
0550     switch (m_mode) {
0551     case AbstractColumn::ColumnMode::Double:
0552         return cast_vector<double>()->count();
0553     case AbstractColumn::ColumnMode::Integer:
0554         return cast_vector<int>()->count();
0555     case AbstractColumn::ColumnMode::BigInt:
0556         return cast_vector<qint64>()->count();
0557     case AbstractColumn::ColumnMode::Text:
0558         return cast_vector<QString>()->count();
0559     case AbstractColumn::ColumnMode::DateTime:
0560     case AbstractColumn::ColumnMode::Month:
0561     case AbstractColumn::ColumnMode::Day:
0562         return cast_vector<QDateTime>()->count();
0563     }
0564     return 0;
0565 }
0566
0567 int ColumnPrivate::ValueLabels::count(double min, double max) const {
0568     if (!initialized())
0569         return 0;
0570
0571     min = qMin(min, max);
0572     max = qMax(min, max);
0573
0574     int counter = 0;
0575     switch (m_mode) {
0576     case AbstractColumn::ColumnMode::Double: {
0577         const auto* data = cast_vector<double>();
0578         for (const auto& d : *data) {
0579             if (d.value >= min && d.value <= max)
0580                 counter++;
0581         }
0582         break;
0583     }
0584     case AbstractColumn::ColumnMode::Integer: {
0585         const auto* data = cast_vector<int>();
0586         for (const auto& d : *data) {
0587             if (d.value >= min && d.value <= max)
0588                 counter++;
0589         }
0590         break;
0591     }
0592     case AbstractColumn::ColumnMode::BigInt: {
0593         const auto* data = cast_vector<qint64>();
0594         for (const auto& d : *data) {
0595             if (d.value >= min && d.value <= max)
0596                 counter++;
0597         }
0598         break;
0599     }
0600     case AbstractColumn::ColumnMode::Text:
0601         return 0;
0602     case AbstractColumn::ColumnMode::DateTime:
0603     case AbstractColumn::ColumnMode::Month:
0604     case AbstractColumn::ColumnMode::Day: {
0605         const auto* data = cast_vector<QDateTime>();
0606         for (const auto& d : *data) {
0607             const auto value = d.value.toMSecsSinceEpoch();
0608             if (value >= min && value <= max)
0609                 counter++;
0610         }
0611         break;
0612     }
0613     }
0614     return counter;
0615 }
0616
0617 void ColumnPrivate::ValueLabels::add(const QString& value, const QString& label) {
0618     if (initialized() && m_mode != AbstractColumn::ColumnMode::Text)
0619         return;
0620
0621     init(AbstractColumn::ColumnMode::Text);
0622     invalidateStatistics();
0623     cast_vector<QString>()->append({value, label});
0624 }
0625
0626 void ColumnPrivate::ValueLabels::add(const QDateTime& value, const QString& label) {
0627     if (initialized() && m_mode != AbstractColumn::ColumnMode::DateTime && m_mode != AbstractColumn::ColumnMode::Day
0628         && m_mode != AbstractColumn::ColumnMode::Month)
0629         return;
0630
0631     init(AbstractColumn::ColumnMode::Month);
0632     invalidateStatistics();
0633     cast_vector<QDateTime>()->append({value, label});
0634 }
0635
0636 void ColumnPrivate::ValueLabels::add(double value, const QString& label) {
0637     if (initialized() && m_mode != AbstractColumn::ColumnMode::Double)
0638         return;
0639
0640     init(AbstractColumn::ColumnMode::Double);
0641     invalidateStatistics();
0642     cast_vector<double>()->append({value, label});
0643 }
0644
0645 void ColumnPrivate::ValueLabels::add(int value, const QString& label) {
0646     if (initialized() && m_mode != AbstractColumn::ColumnMode::Integer)
0647         return;
0648
0649     init(AbstractColumn::ColumnMode::Integer);
0650     invalidateStatistics();
0651     cast_vector<int>()->append({value, label});
0652 }
0653
0654 void ColumnPrivate::ValueLabels::add(qint64 value, const QString& label) {
0655     if (initialized() && m_mode != AbstractColumn::ColumnMode::BigInt)
0656         return;
0657
0658     init(AbstractColumn::ColumnMode::BigInt);
0659     invalidateStatistics();
0660     cast_vector<qint64>()->append({value, label});
0661 }
0662
0663 void ColumnPrivate::ValueLabels::removeAll() {
0664     if (!initialized())
0665         return;
0666
0667     deinit();
0668     init(m_mode);
0669 }
0670
0671 void ColumnPrivate::ValueLabels::remove(const QString& key) {
0672     if (!initialized())
0673         return;
0674
0675     invalidateStatistics();
0676     bool ok;
0677     switch (m_mode) {
0678     case AbstractColumn::ColumnMode::Double: {
0679         double value = QLocale().toDouble(key, &ok);
0680         if (!ok)
0681             return;
0682         remove<double>(value);
0683         break;
0684     }
0685     case AbstractColumn::ColumnMode::Integer: {
0686         int value = QLocale().toInt(key, &ok);
0687         if (!ok)
0688             return;
0689         remove<int>(value);
0690         break;
0691     }
0692     case AbstractColumn::ColumnMode::BigInt: {
0693         qint64 value = QLocale().toLongLong(key, &ok);
0694         if (!ok)
0695             return;
0696         remove<qint64>(value);
0697         break;
0698     }
0699     case AbstractColumn::ColumnMode::Text: {
0700         remove<QString>(key);
0701         break;
0702     }
0703     case AbstractColumn::ColumnMode::Month:
0704     case AbstractColumn::ColumnMode::Day:
0705     case AbstractColumn::ColumnMode::DateTime: {
0706         DateTime2StringFilter f;
0707         if (m_mode == AbstractColumn::ColumnMode::Month) {
0708             f.setFormat(QStringLiteral("MMMM"));
0709         } else {
0710             f.setFormat(QStringLiteral("dddd"));
0711         }
0712         const auto ref = QDateTime::fromString(key, f.format());
0713         remove<QDateTime>(ref);
0714         break;
0715     }
0716     }
0717 }
0718
0719 const QVector<Column::ValueLabel<QString>>* ColumnPrivate::ValueLabels::textValueLabels() const {
0720     if (!initialized() || m_mode != AbstractColumn::ColumnMode::Text)
0721         return nullptr;
0722     return cast_vector<QString>();
0723 }
0724
0725 const QVector<Column::ValueLabel<QDateTime>>* ColumnPrivate::ValueLabels::dateTimeValueLabels() const {
0726     if (!initialized()
0727         || (m_mode != AbstractColumn::ColumnMode::DateTime && m_mode != AbstractColumn::ColumnMode::Day && m_mode != AbstractColumn::ColumnMode::Month))
0728         return nullptr;
0729     return cast_vector<QDateTime>();
0730 }
0731
0732 const QVector<Column::ValueLabel<double>>* ColumnPrivate::ValueLabels::valueLabels() const {
0733     if (!initialized() || m_mode != AbstractColumn::ColumnMode::Double)
0734         return nullptr;
0735     return cast_vector<double>();
0736 }
0737
0738 const QVector<Column::ValueLabel<int>>* ColumnPrivate::ValueLabels::intValueLabels() const {
0739     if (!initialized() || m_mode != AbstractColumn::ColumnMode::Integer)
0740         return nullptr;
0741     return cast_vector<int>();
0742 }
0743
0744 const QVector<Column::ValueLabel<qint64>>* ColumnPrivate::ValueLabels::bigIntValueLabels() const {
0745     if (!initialized() || m_mode != AbstractColumn::ColumnMode::BigInt)
0746         return nullptr;
0747     return cast_vector<qint64>();
0748 }
0749
0750 // ######################################################################################################
0751 // ######################################################################################################
0752 // ######################################################################################################
0753
0754 ColumnPrivate::ColumnPrivate(Column* owner, AbstractColumn::ColumnMode mode)
0755     : m_columnMode(mode)
0756     , m_owner(owner) {
0757     initIOFilters();
0758 }
0759
0760 /**
0761  * \brief Special ctor (to be called from Column only!)
0762  */
0763 ColumnPrivate::ColumnPrivate(Column* owner, AbstractColumn::ColumnMode mode, void* data)
0764     : m_columnMode(mode)
0765     , m_data(data)
0766     , m_owner(owner) {
0767     initIOFilters();
0768 }
0769
0770 /*!
0771  * initializes the interal vector for data. This is where the actual allocation on the heap is happening.
0772  * If \c resize is set to \false, the vector is not resized after its creation. This should be used
0773  * if there is already a vector created somewhere and the content of the column is going to be replaced
0774  * with the existing content where the memory was already allocated.
0775  */
0776 bool ColumnPrivate::initDataContainer(bool resize) {
0777     switch (m_columnMode) {
0778     case AbstractColumn::ColumnMode::Double: {
0779         auto* vec = new QVector<double>();
0780         try {
0781             if (resize)
0782                 vec->resize(m_rowCount);
0783         } catch (std::bad_alloc&) { return false; }
0784         vec->fill(std::numeric_limits<double>::quiet_NaN());
0785         m_data = vec;
0786         break;
0787     }
0788     case AbstractColumn::ColumnMode::Integer: {
0789         auto* vec = new QVector<int>();
0790         try {
0791             if (resize)
0792                 vec->resize(m_rowCount);
0793         } catch (std::bad_alloc&) { return false; }
0794         m_data = vec;
0795         break;
0796     }
0797     case AbstractColumn::ColumnMode::BigInt: {
0798         auto* vec = new QVector<qint64>();
0799         try {
0800             if (resize)
0801                 vec->resize(m_rowCount);
0802         } catch (std::bad_alloc&) { return false; }
0803         m_data = vec;
0804         break;
0805     }
0806     case AbstractColumn::ColumnMode::Text: {
0807         auto* vec = new QVector<QString>();
0808         try {
0809             if (resize)
0810                 vec->resize(m_rowCount);
0811         } catch (std::bad_alloc&) { return false; }
0812         m_data = vec;
0813         break;
0814     }
0815     case AbstractColumn::ColumnMode::DateTime:
0816     case AbstractColumn::ColumnMode::Month:
0817     case AbstractColumn::ColumnMode::Day: {
0818         auto* vec = new QVector<QDateTime>();
0819         try {
0820             if (resize)
0821                 vec->resize(m_rowCount);
0822         } catch (std::bad_alloc&) { return false; }
0823         m_data = vec;
0824         break;
0825     }
0826     }
0827
0828     return true;
0829 }
0830
0831 void ColumnPrivate::initIOFilters() {
0832     const auto numberLocale = QLocale();
0833     switch (m_columnMode) {
0834     case AbstractColumn::ColumnMode::Double:
0835         m_inputFilter = new String2DoubleFilter();
0836         m_inputFilter->setNumberLocale(numberLocale);
0837         m_outputFilter = new Double2StringFilter();
0838         m_outputFilter->setNumberLocale(numberLocale);
0839         break;
0840     case AbstractColumn::ColumnMode::Integer:
0841         m_inputFilter = new String2IntegerFilter();
0842         m_inputFilter->setNumberLocale(numberLocale);
0843         m_outputFilter = new Integer2StringFilter();
0844         m_outputFilter->setNumberLocale(numberLocale);
0845         break;
0846     case AbstractColumn::ColumnMode::BigInt:
0847         m_inputFilter = new String2BigIntFilter();
0848         m_inputFilter->setNumberLocale(numberLocale);
0849         m_outputFilter = new BigInt2StringFilter();
0850         m_outputFilter->setNumberLocale(numberLocale);
0851         break;
0852     case AbstractColumn::ColumnMode::Text:
0853         m_inputFilter = new SimpleCopyThroughFilter();
0854         m_outputFilter = new SimpleCopyThroughFilter();
0855         break;
0856     case AbstractColumn::ColumnMode::DateTime:
0857         m_inputFilter = new String2DateTimeFilter();
0858         m_outputFilter = new DateTime2StringFilter();
0859         break;
0860     case AbstractColumn::ColumnMode::Month:
0861         m_inputFilter = new String2MonthFilter();
0862         m_outputFilter = new DateTime2StringFilter();
0863         static_cast<DateTime2StringFilter*>(m_outputFilter)->setFormat(QStringLiteral("MMMM"));
0864         break;
0865     case AbstractColumn::ColumnMode::Day:
0866         m_inputFilter = new String2DayOfWeekFilter();
0867         m_outputFilter = new DateTime2StringFilter();
0868         static_cast<DateTime2StringFilter*>(m_outputFilter)->setFormat(QStringLiteral("dddd"));
0869         break;
0870     }
0871
0872     connect(m_outputFilter, &AbstractSimpleFilter::formatChanged, m_owner, &Column::handleFormatChange);
0873 }
0874
0875 ColumnPrivate::~ColumnPrivate() {
0876     deleteData();
0877 }
0878
0879 void ColumnPrivate::deleteData() {
0880     if (!m_data)
0881         return;
0882
0883     switch (m_columnMode) {
0884     case AbstractColumn::ColumnMode::Double:
0885         delete static_cast<QVector<double>*>(m_data);
0886         break;
0887     case AbstractColumn::ColumnMode::Integer:
0888         delete static_cast<QVector<int>*>(m_data);
0889         break;
0890     case AbstractColumn::ColumnMode::BigInt:
0891         delete static_cast<QVector<qint64>*>(m_data);
0892         break;
0893     case AbstractColumn::ColumnMode::Text:
0894         delete static_cast<QVector<QString>*>(m_data);
0895         break;
0896     case AbstractColumn::ColumnMode::DateTime:
0897     case AbstractColumn::ColumnMode::Month:
0898     case AbstractColumn::ColumnMode::Day:
0899         delete static_cast<QVector<QDateTime>*>(m_data);
0900         break;
0901     }
0902     m_data = nullptr;
0903 }
0904
0905 AbstractColumn::ColumnMode ColumnPrivate::columnMode() const {
0906     return m_columnMode;
0907 }
0908
0909 /**
0910  * \brief Set the column mode
0911  *
0912  * This sets the column mode and, if
0913  * necessary, converts it to another datatype.
0914  * Remark: setting the mode back to undefined (the
0915  * initial value) is not supported.
0916  */
0917 void ColumnPrivate::setColumnMode(AbstractColumn::ColumnMode mode) {
0918     DEBUG(Q_FUNC_INFO << ", " << ENUM_TO_STRING(AbstractColumn, ColumnMode, m_columnMode) << " -> " << ENUM_TO_STRING(AbstractColumn, ColumnMode, mode))
0919     if (mode == m_columnMode)
0920         return;
0921
0922     void* old_data = m_data;
0923     // remark: the deletion of the old data will be done in the dtor of a command
0924
0925     AbstractSimpleFilter *filter{nullptr}, *new_in_filter{nullptr}, *new_out_filter{nullptr};
0926     bool filter_is_temporary = false; // it can also become outputFilter(), which we may not delete here
0927     Column* temp_col = nullptr;
0928
0929     Q_EMIT m_owner->modeAboutToChange(m_owner);
0930
0931     // determine the conversion filter and allocate the new data vector
0932     switch (m_columnMode) { // old mode
0933     case AbstractColumn::ColumnMode::Double: {
0934         disconnect(static_cast<Double2StringFilter*>(m_outputFilter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
0935         switch (mode) {
0936         case AbstractColumn::ColumnMode::Double:
0937             break;
0938         case AbstractColumn::ColumnMode::Integer:
0939             filter = new Double2IntegerFilter();
0940             filter_is_temporary = true;
0941             if (m_data) {
0942                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<double>*>(old_data)));
0943                 m_data = new QVector<int>();
0944             }
0945             break;
0946         case AbstractColumn::ColumnMode::BigInt:
0947             filter = new Double2BigIntFilter();
0948             filter_is_temporary = true;
0949             if (m_data) {
0950                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<double>*>(old_data)));
0951                 m_data = new QVector<qint64>();
0952             }
0953             break;
0954         case AbstractColumn::ColumnMode::Text:
0955             filter = outputFilter();
0956             filter_is_temporary = false;
0957             if (m_data) {
0958                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<double>*>(old_data)));
0959                 m_data = new QVector<QString>();
0960             }
0961             break;
0962         case AbstractColumn::ColumnMode::DateTime:
0963             filter = new Double2DateTimeFilter();
0964             filter_is_temporary = true;
0965             if (m_data) {
0966                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<double>*>(old_data)));
0967                 m_data = new QVector<QDateTime>();
0968             }
0969             break;
0970         case AbstractColumn::ColumnMode::Month:
0971             filter = new Double2MonthFilter();
0972             filter_is_temporary = true;
0973             if (m_data) {
0974                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<double>*>(old_data)));
0975                 m_data = new QVector<QDateTime>();
0976             }
0977             break;
0978         case AbstractColumn::ColumnMode::Day:
0979             filter = new Double2DayOfWeekFilter();
0980             filter_is_temporary = true;
0981             if (m_data) {
0982                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<double>*>(old_data)));
0983                 m_data = new QVector<QDateTime>();
0984             }
0985             break;
0986         } // switch(mode)
0987
0988         break;
0989     }
0990     case AbstractColumn::ColumnMode::Integer: {
0991         disconnect(static_cast<Integer2StringFilter*>(m_outputFilter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
0992         switch (mode) {
0993         case AbstractColumn::ColumnMode::Integer:
0994             break;
0995         case AbstractColumn::ColumnMode::BigInt:
0996             filter = new Integer2BigIntFilter();
0997             filter_is_temporary = true;
0998             if (m_data) {
0999                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<int>*>(old_data)));
1000                 m_data = new QVector<qint64>();
1001             }
1002             break;
1003         case AbstractColumn::ColumnMode::Double:
1004             filter = new Integer2DoubleFilter();
1005             filter_is_temporary = true;
1006             if (m_data) {
1007                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<int>*>(old_data)));
1008                 m_data = new QVector<double>();
1009             }
1010             break;
1011         case AbstractColumn::ColumnMode::Text:
1012             filter = outputFilter();
1013             filter_is_temporary = false;
1014             if (m_data) {
1015                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<int>*>(old_data)));
1016                 m_data = new QVector<QString>();
1017             }
1018             break;
1019         case AbstractColumn::ColumnMode::DateTime:
1020             DEBUG(Q_FUNC_INFO << ", int -> datetime")
1021             filter = new Integer2DateTimeFilter();
1022             filter_is_temporary = true;
1023             if (m_data) {
1024                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<int>*>(old_data)));
1025                 m_data = new QVector<QDateTime>();
1026             }
1027             DEBUG(Q_FUNC_INFO << ", int -> datetime done")
1028             break;
1029         case AbstractColumn::ColumnMode::Month:
1030             filter = new Integer2MonthFilter();
1031             filter_is_temporary = true;
1032             if (m_data) {
1033                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<int>*>(old_data)));
1034                 m_data = new QVector<QDateTime>();
1035             }
1036             break;
1037         case AbstractColumn::ColumnMode::Day:
1038             filter = new Integer2DayOfWeekFilter();
1039             filter_is_temporary = true;
1040             if (m_data) {
1041                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<int>*>(old_data)));
1042                 m_data = new QVector<QDateTime>();
1043             }
1044             break;
1045         } // switch(mode)
1046
1047         break;
1048     }
1049     case AbstractColumn::ColumnMode::BigInt: {
1050         disconnect(static_cast<BigInt2StringFilter*>(m_outputFilter), &BigInt2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1051         switch (mode) {
1052         case AbstractColumn::ColumnMode::BigInt:
1053             break;
1054         case AbstractColumn::ColumnMode::Integer: // TODO: timeUnit
1055             filter = new BigInt2IntegerFilter();
1056             filter_is_temporary = true;
1057             if (m_data) {
1058                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<qint64>*>(old_data)));
1059                 m_data = new QVector<int>();
1060             }
1061             break;
1062         case AbstractColumn::ColumnMode::Double: // TODO: timeUnit
1063             filter = new BigInt2DoubleFilter();
1064             filter_is_temporary = true;
1065             if (m_data) {
1066                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<qint64>*>(old_data)));
1067                 m_data = new QVector<double>();
1068             }
1069             break;
1070         case AbstractColumn::ColumnMode::Text:
1071             filter = outputFilter();
1072             filter_is_temporary = false;
1073             if (m_data) {
1074                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<qint64>*>(old_data)));
1075                 m_data = new QVector<QString>();
1076             }
1077             break;
1078         case AbstractColumn::ColumnMode::DateTime: // TODO: timeUnit
1079             filter = new BigInt2DateTimeFilter();
1080             filter_is_temporary = true;
1081             if (m_data) {
1082                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<qint64>*>(old_data)));
1083                 m_data = new QVector<QDateTime>();
1084             }
1085             break;
1086         case AbstractColumn::ColumnMode::Month:
1087             filter = new BigInt2MonthFilter();
1088             filter_is_temporary = true;
1089             if (m_data) {
1090                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<qint64>*>(old_data)));
1091                 m_data = new QVector<QDateTime>();
1092             }
1093             break;
1094         case AbstractColumn::ColumnMode::Day:
1095             filter = new BigInt2DayOfWeekFilter();
1096             filter_is_temporary = true;
1097             if (m_data) {
1098                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<qint64>*>(old_data)));
1099                 m_data = new QVector<QDateTime>();
1100             }
1101             break;
1102         } // switch(mode)
1103
1104         break;
1105     }
1106     case AbstractColumn::ColumnMode::Text: {
1107         switch (mode) {
1108         case AbstractColumn::ColumnMode::Text:
1109             break;
1110         case AbstractColumn::ColumnMode::Double:
1111             filter = new String2DoubleFilter();
1112             filter->setNumberLocale(QLocale());
1113             filter_is_temporary = true;
1114             if (m_data) {
1115                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QString>*>(old_data)));
1116                 m_data = new QVector<double>();
1117             }
1118             break;
1119         case AbstractColumn::ColumnMode::Integer:
1120             filter = new String2IntegerFilter();
1121             filter->setNumberLocale(QLocale());
1122             filter_is_temporary = true;
1123             if (m_data) {
1124                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QString>*>(old_data)));
1125                 m_data = new QVector<int>();
1126             }
1127             break;
1128         case AbstractColumn::ColumnMode::BigInt:
1129             filter = new String2BigIntFilter();
1130             filter->setNumberLocale(QLocale());
1131             filter_is_temporary = true;
1132             if (m_data) {
1133                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QString>*>(old_data)));
1134                 m_data = new QVector<qint64>();
1135             }
1136             break;
1137         case AbstractColumn::ColumnMode::DateTime:
1138             filter = new String2DateTimeFilter();
1139             filter_is_temporary = true;
1140             if (m_data) {
1141                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QString>*>(old_data)));
1142                 m_data = new QVector<QDateTime>();
1143             }
1144             break;
1145         case AbstractColumn::ColumnMode::Month:
1146             filter = new String2MonthFilter();
1147             filter_is_temporary = true;
1148             if (m_data) {
1149                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QString>*>(old_data)));
1150                 m_data = new QVector<QDateTime>();
1151             }
1152             break;
1153         case AbstractColumn::ColumnMode::Day:
1154             filter = new String2DayOfWeekFilter();
1155             filter_is_temporary = true;
1156             if (m_data) {
1157                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QString>*>(old_data)));
1158                 m_data = new QVector<QDateTime>();
1159             }
1160             break;
1161         } // switch(mode)
1162
1163         break;
1164     }
1165     case AbstractColumn::ColumnMode::DateTime:
1166     case AbstractColumn::ColumnMode::Month:
1167     case AbstractColumn::ColumnMode::Day: {
1168         disconnect(static_cast<DateTime2StringFilter*>(m_outputFilter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1169         switch (mode) {
1170         case AbstractColumn::ColumnMode::DateTime:
1171         case AbstractColumn::ColumnMode::Month:
1172         case AbstractColumn::ColumnMode::Day:
1173             break;
1174         case AbstractColumn::ColumnMode::Text:
1175             filter = outputFilter();
1176             filter_is_temporary = false;
1177             if (m_data) {
1178                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QDateTime>*>(old_data)), m_columnMode);
1179                 m_data = new QStringList();
1180             }
1181             break;
1182         case AbstractColumn::ColumnMode::Double:
1183             if (m_columnMode == AbstractColumn::ColumnMode::Month)
1184                 filter = new Month2DoubleFilter();
1185             else if (m_columnMode == AbstractColumn::ColumnMode::Day)
1186                 filter = new DayOfWeek2DoubleFilter();
1187             else // TODO: timeUnit
1188                 filter = new DateTime2DoubleFilter();
1189             filter_is_temporary = true;
1190             if (m_data) {
1191                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QDateTime>*>(old_data)), m_columnMode);
1192                 m_data = new QVector<double>();
1193             }
1194             break;
1195         case AbstractColumn::ColumnMode::Integer:
1196             if (m_columnMode == AbstractColumn::ColumnMode::Month)
1197                 filter = new Month2IntegerFilter();
1198             else if (m_columnMode == AbstractColumn::ColumnMode::Day)
1199                 filter = new DayOfWeek2IntegerFilter();
1200             else // TODO: timeUnit
1201                 filter = new DateTime2IntegerFilter();
1202             filter_is_temporary = true;
1203             if (m_data) {
1204                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QDateTime>*>(old_data)), m_columnMode);
1205                 m_data = new QVector<int>();
1206             }
1207             break;
1208         case AbstractColumn::ColumnMode::BigInt:
1209             if (m_columnMode == AbstractColumn::ColumnMode::Month)
1210                 filter = new Month2BigIntFilter();
1211             else if (m_columnMode == AbstractColumn::ColumnMode::Day)
1212                 filter = new DayOfWeek2BigIntFilter();
1213             else // TODO: timeUnit
1214                 filter = new DateTime2BigIntFilter();
1215             filter_is_temporary = true;
1216             if (m_data) {
1217                 temp_col = new Column(QStringLiteral("temp_col"), *(static_cast<QVector<QDateTime>*>(old_data)), m_columnMode);
1218                 m_data = new QVector<qint64>();
1219             }
1220             break;
1221         } // switch(mode)
1222
1223         break;
1224     }
1225     }
1226
1227     // determine the new input and output filters
1228     switch (mode) { // new mode
1229     case AbstractColumn::ColumnMode::Double:
1230         new_in_filter = new String2DoubleFilter();
1231         new_in_filter->setNumberLocale(QLocale());
1232         new_out_filter = new Double2StringFilter();
1233         new_out_filter->setNumberLocale(QLocale());
1234         connect(static_cast<Double2StringFilter*>(new_out_filter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1235         break;
1236     case AbstractColumn::ColumnMode::Integer:
1237         new_in_filter = new String2IntegerFilter();
1238         new_in_filter->setNumberLocale(QLocale());
1239         new_out_filter = new Integer2StringFilter();
1240         new_out_filter->setNumberLocale(QLocale());
1241         connect(static_cast<Integer2StringFilter*>(new_out_filter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1242         break;
1243     case AbstractColumn::ColumnMode::BigInt:
1244         new_in_filter = new String2BigIntFilter();
1245         new_in_filter->setNumberLocale(QLocale());
1246         new_out_filter = new BigInt2StringFilter();
1247         new_out_filter->setNumberLocale(QLocale());
1248         connect(static_cast<BigInt2StringFilter*>(new_out_filter), &BigInt2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1249         break;
1250     case AbstractColumn::ColumnMode::Text:
1251         new_in_filter = new SimpleCopyThroughFilter();
1252         new_out_filter = new SimpleCopyThroughFilter();
1253         break;
1254     case AbstractColumn::ColumnMode::DateTime:
1255         new_in_filter = new String2DateTimeFilter();
1256         new_out_filter = new DateTime2StringFilter();
1257         connect(static_cast<DateTime2StringFilter*>(new_out_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1258         break;
1259     case AbstractColumn::ColumnMode::Month:
1260         new_in_filter = new String2MonthFilter();
1261         new_out_filter = new DateTime2StringFilter();
1262         static_cast<DateTime2StringFilter*>(new_out_filter)->setFormat(QStringLiteral("MMMM"));
1263         // DEBUG("  Month out_filter format: " << STDSTRING(static_cast<DateTime2StringFilter*>(new_out_filter)->format()));
1264         connect(static_cast<DateTime2StringFilter*>(new_out_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1265         break;
1266     case AbstractColumn::ColumnMode::Day:
1267         new_in_filter = new String2DayOfWeekFilter();
1268         new_out_filter = new DateTime2StringFilter();
1269         static_cast<DateTime2StringFilter*>(new_out_filter)->setFormat(QStringLiteral("dddd"));
1270         connect(static_cast<DateTime2StringFilter*>(new_out_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1271         break;
1272     } // switch(mode)
1273
1274     m_columnMode = mode;
1275
1276     m_inputFilter = new_in_filter;
1277     m_outputFilter = new_out_filter;
1278     m_inputFilter->input(0, m_owner->m_string_io);
1279     m_outputFilter->input(0, m_owner);
1280     m_inputFilter->setHidden(true);
1281     m_outputFilter->setHidden(true);
1282
1283     if (temp_col) { // if temp_col == 0, only the input/output filters need to be changed
1284         // copy the filtered, i.e. converted, column (mode is orig mode)
1285         DEBUG(" temp_col column mode = " << ENUM_TO_STRING(AbstractColumn, ColumnMode, temp_col->columnMode()));
1286         filter->input(0, temp_col);
1287         DEBUG(" filter->output size = " << filter->output(0)->rowCount());
1288         copy(filter->output(0));
1289         DEBUG(" DONE")
1290         delete temp_col;
1291     }
1292
1293     if (filter_is_temporary)
1294         delete filter;
1295
1296     Q_EMIT m_owner->modeChanged(m_owner);
1297 }
1298
1299 /**
1300  * \brief Replace all mode related members
1301  *
1302  * Replace column mode, data type, data pointer and filters directly
1303  */
1304 void ColumnPrivate::replaceModeData(AbstractColumn::ColumnMode mode, void* data, AbstractSimpleFilter* in_filter, AbstractSimpleFilter* out_filter) {
1305     Q_EMIT m_owner->modeAboutToChange(m_owner);
1306     // disconnect formatChanged()
1307     switch (m_columnMode) {
1308     case AbstractColumn::ColumnMode::Double:
1309         disconnect(static_cast<Double2StringFilter*>(m_outputFilter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1310         break;
1311     case AbstractColumn::ColumnMode::Integer:
1312         disconnect(static_cast<Integer2StringFilter*>(m_outputFilter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1313         break;
1314     case AbstractColumn::ColumnMode::BigInt:
1315         disconnect(static_cast<BigInt2StringFilter*>(m_outputFilter), &BigInt2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1316         break;
1317     case AbstractColumn::ColumnMode::Text:
1318         break;
1319     case AbstractColumn::ColumnMode::DateTime:
1320     case AbstractColumn::ColumnMode::Month:
1321     case AbstractColumn::ColumnMode::Day:
1322         disconnect(static_cast<DateTime2StringFilter*>(m_outputFilter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1323         break;
1324     }
1325
1326     m_columnMode = mode;
1327     setLabelsMode(mode);
1328     m_data = data;
1329
1330     m_inputFilter = in_filter;
1331     m_outputFilter = out_filter;
1332     m_inputFilter->input(0, m_owner->m_string_io);
1333     m_outputFilter->input(0, m_owner);
1334
1335     // connect formatChanged()
1336     switch (m_columnMode) {
1337     case AbstractColumn::ColumnMode::Double:
1338         connect(static_cast<Double2StringFilter*>(m_outputFilter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1339         break;
1340     case AbstractColumn::ColumnMode::Integer:
1341         connect(static_cast<Integer2StringFilter*>(m_outputFilter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1342         break;
1343     case AbstractColumn::ColumnMode::BigInt:
1344         connect(static_cast<BigInt2StringFilter*>(m_outputFilter), &BigInt2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1345         break;
1346     case AbstractColumn::ColumnMode::Text:
1347         break;
1348     case AbstractColumn::ColumnMode::DateTime:
1349     case AbstractColumn::ColumnMode::Month:
1350     case AbstractColumn::ColumnMode::Day:
1351         connect(static_cast<DateTime2StringFilter*>(m_outputFilter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange);
1352         break;
1353     }
1354
1355     Q_EMIT m_owner->modeChanged(m_owner);
1356 }
1357
1358 /**
1359  * \brief Replace data pointer
1360  */
1361 void ColumnPrivate::replaceData(void* data) {
1362     Q_EMIT m_owner->dataAboutToChange(m_owner);
1363
1364     m_data = data;
1365     invalidate();
1366     if (!m_owner->m_suppressDataChangedSignal)
1367         Q_EMIT m_owner->dataChanged(m_owner);
1368 }
1369
1370 /**
1371  * \brief Copy another column of the same type
1372  *
1373  * This function will return false if the data type
1374  * of 'other' is not the same as the type of 'this'.
1375  * Use a filter to convert a column to another type.
1376  */
1377 bool ColumnPrivate::copy(const AbstractColumn* other) {
1378     DEBUG(Q_FUNC_INFO)
1379     if (other->columnMode() != columnMode())
1380         return false;
1381     //  DEBUG(Q_FUNC_INFO << ", mode = " << ENUM_TO_STRING(AbstractColumn, ColumnMode, columnMode()));
1382     int num_rows = other->rowCount();
1383     //  DEBUG(Q_FUNC_INFO << ", rows " << num_rows);
1384
1385     Q_EMIT m_owner->dataAboutToChange(m_owner);
1386     resizeTo(num_rows);
1387
1388     if (!m_data) {
1389         if (!initDataContainer())
1390             return false; // failed to allocate memory
1391     }
1392
1393     // copy the data
1394     switch (m_columnMode) {
1395     case AbstractColumn::ColumnMode::Double: {
1396         double* ptr = static_cast<QVector<double>*>(m_data)->data();
1397         for (int i = 0; i < num_rows; ++i)
1398             ptr[i] = other->valueAt(i);
1399         break;
1400     }
1401     case AbstractColumn::ColumnMode::Integer: {
1402         int* ptr = static_cast<QVector<int>*>(m_data)->data();
1403         for (int i = 0; i < num_rows; ++i)
1404             ptr[i] = other->integerAt(i);
1405         break;
1406     }
1407     case AbstractColumn::ColumnMode::BigInt: {
1408         qint64* ptr = static_cast<QVector<qint64>*>(m_data)->data();
1409         for (int i = 0; i < num_rows; ++i)
1410             ptr[i] = other->bigIntAt(i);
1411         break;
1412     }
1413     case AbstractColumn::ColumnMode::Text: {
1414         auto* vec = static_cast<QVector<QString>*>(m_data);
1415         for (int i = 0; i < num_rows; ++i)
1416             vec->replace(i, other->textAt(i));
1417         break;
1418     }
1419     case AbstractColumn::ColumnMode::DateTime:
1420     case AbstractColumn::ColumnMode::Month:
1421     case AbstractColumn::ColumnMode::Day: {
1422         auto* vec = static_cast<QVector<QDateTime>*>(m_data);
1423         for (int i = 0; i < num_rows; ++i)
1424             vec->replace(i, other->dateTimeAt(i));
1425         break;
1426     }
1427     }
1428
1429     invalidate();
1430
1431     if (!m_owner->m_suppressDataChangedSignal)
1432         Q_EMIT m_owner->dataChanged(m_owner);
1433
1434     DEBUG(Q_FUNC_INFO << ", done")
1435     return true;
1436 }
1437
1438 /**
1439  * \brief Copies a part of another column of the same type
1440  *
1441  * This function will return false if the data type
1442  * of 'other' is not the same as the type of 'this'.
1443  * \param source pointer to the column to copy
1444  * \param source_start first row to copy in the column to copy
1445  * \param dest_start first row to copy in
1446  * \param num_rows the number of rows to copy
1447  */
1448 bool ColumnPrivate::copy(const AbstractColumn* source, int source_start, int dest_start, int num_rows) {
1449     if (source->columnMode() != m_columnMode)
1450         return false;
1451     if (num_rows == 0)
1452         return true;
1453
1454     Q_EMIT m_owner->dataAboutToChange(m_owner);
1455     if (dest_start + num_rows > rowCount())
1456         resizeTo(dest_start + num_rows);
1457
1458     if (!m_data) {
1459         if (!initDataContainer())
1460             return false; // failed to allocate memory
1461     }
1462
1463     // copy the data
1464     switch (m_columnMode) {
1465     case AbstractColumn::ColumnMode::Double: {
1466         double* ptr = static_cast<QVector<double>*>(m_data)->data();
1467         for (int i = 0; i < num_rows; i++)
1468             ptr[dest_start + i] = source->valueAt(source_start + i);
1469         break;
1470     }
1471     case AbstractColumn::ColumnMode::Integer: {
1472         int* ptr = static_cast<QVector<int>*>(m_data)->data();
1473         for (int i = 0; i < num_rows; i++)
1474             ptr[dest_start + i] = source->integerAt(source_start + i);
1475         break;
1476     }
1477     case AbstractColumn::ColumnMode::BigInt: {
1478         qint64* ptr = static_cast<QVector<qint64>*>(m_data)->data();
1479         for (int i = 0; i < num_rows; i++)
1480             ptr[dest_start + i] = source->bigIntAt(source_start + i);
1481         break;
1482     }
1483     case AbstractColumn::ColumnMode::Text:
1484         for (int i = 0; i < num_rows; i++)
1485             static_cast<QVector<QString>*>(m_data)->replace(dest_start + i, source->textAt(source_start + i));
1486         break;
1487     case AbstractColumn::ColumnMode::DateTime:
1488     case AbstractColumn::ColumnMode::Month:
1489     case AbstractColumn::ColumnMode::Day:
1490         for (int i = 0; i < num_rows; i++)
1491             static_cast<QVector<QDateTime>*>(m_data)->replace(dest_start + i, source->dateTimeAt(source_start + i));
1492         break;
1493     }
1494
1495     invalidate();
1496
1497     if (!m_owner->m_suppressDataChangedSignal)
1498         Q_EMIT m_owner->dataChanged(m_owner);
1499
1500     return true;
1501 }
1502
1503 /**
1504  * \brief Copy another column of the same type
1505  *
1506  * This function will return false if the data type
1507  * of 'other' is not the same as the type of 'this'.
1508  * Use a filter to convert a column to another type.
1509  */
1510 bool ColumnPrivate::copy(const ColumnPrivate* other) {
1511     if (other->columnMode() != m_columnMode)
1512         return false;
1513     int num_rows = other->rowCount();
1514
1515     Q_EMIT m_owner->dataAboutToChange(m_owner);
1516     resizeTo(num_rows);
1517
1518     if (!m_data) {
1519         if (!initDataContainer())
1520             return false; // failed to allocate memory
1521     }
1522
1523     // copy the data
1524     switch (m_columnMode) {
1525     case AbstractColumn::ColumnMode::Double: {
1526         double* ptr = static_cast<QVector<double>*>(m_data)->data();
1527         for (int i = 0; i < num_rows; ++i)
1528             ptr[i] = other->valueAt(i);
1529         break;
1530     }
1531     case AbstractColumn::ColumnMode::Integer: {
1532         int* ptr = static_cast<QVector<int>*>(m_data)->data();
1533         for (int i = 0; i < num_rows; ++i)
1534             ptr[i] = other->integerAt(i);
1535         break;
1536     }
1537     case AbstractColumn::ColumnMode::BigInt: {
1538         qint64* ptr = static_cast<QVector<qint64>*>(m_data)->data();
1539         for (int i = 0; i < num_rows; ++i)
1540             ptr[i] = other->bigIntAt(i);
1541         break;
1542     }
1543     case AbstractColumn::ColumnMode::Text:
1544         for (int i = 0; i < num_rows; ++i)
1545             static_cast<QVector<QString>*>(m_data)->replace(i, other->textAt(i));
1546         break;
1547     case AbstractColumn::ColumnMode::DateTime:
1548     case AbstractColumn::ColumnMode::Month:
1549     case AbstractColumn::ColumnMode::Day:
1550         for (int i = 0; i < num_rows; ++i)
1551             static_cast<QVector<QDateTime>*>(m_data)->replace(i, other->dateTimeAt(i));
1552         break;
1553     }
1554
1555     invalidate();
1556
1557     if (!m_owner->m_suppressDataChangedSignal)
1558         Q_EMIT m_owner->dataChanged(m_owner);
1559
1560     return true;
1561 }
1562
1563 /**
1564  * \brief Copies a part of another column of the same type
1565  *
1566  * This function will return false if the data type
1567  * of 'other' is not the same as the type of 'this'.
1568  * \param source pointer to the column to copy
1569  * \param source_start first row to copy in the column to copy
1570  * \param dest_start first row to copy in
1571  * \param num_rows the number of rows to copy
1572  */
1573 bool ColumnPrivate::copy(const ColumnPrivate* source, int source_start, int dest_start, int num_rows) {
1574     if (source->columnMode() != m_columnMode)
1575         return false;
1576     if (num_rows == 0)
1577         return true;
1578
1579     Q_EMIT m_owner->dataAboutToChange(m_owner);
1580     if (dest_start + num_rows > rowCount())
1581         resizeTo(dest_start + num_rows);
1582
1583     if (!m_data) {
1584         if (!initDataContainer())
1585             return false; // failed to allocate memory
1586     }
1587
1588     // copy the data
1589     switch (m_columnMode) {
1590     case AbstractColumn::ColumnMode::Double: {
1591         double* ptr = static_cast<QVector<double>*>(m_data)->data();
1592         for (int i = 0; i < num_rows; ++i)
1593             ptr[dest_start + i] = source->valueAt(source_start + i);
1594         break;
1595     }
1596     case AbstractColumn::ColumnMode::Integer: {
1597         int* ptr = static_cast<QVector<int>*>(m_data)->data();
1598         for (int i = 0; i < num_rows; ++i)
1599             ptr[dest_start + i] = source->integerAt(source_start + i);
1600         break;
1601     }
1602     case AbstractColumn::ColumnMode::BigInt: {
1603         qint64* ptr = static_cast<QVector<qint64>*>(m_data)->data();
1604         for (int i = 0; i < num_rows; ++i)
1605             ptr[dest_start + i] = source->bigIntAt(source_start + i);
1606         break;
1607     }
1608     case AbstractColumn::ColumnMode::Text:
1609         for (int i = 0; i < num_rows; ++i)
1610             static_cast<QVector<QString>*>(m_data)->replace(dest_start + i, source->textAt(source_start + i));
1611         break;
1612     case AbstractColumn::ColumnMode::DateTime:
1613     case AbstractColumn::ColumnMode::Month:
1614     case AbstractColumn::ColumnMode::Day:
1615         for (int i = 0; i < num_rows; ++i)
1616             static_cast<QVector<QDateTime>*>(m_data)->replace(dest_start + i, source->dateTimeAt(source_start + i));
1617         break;
1618     }
1619
1620     invalidate();
1621
1622     if (!m_owner->m_suppressDataChangedSignal)
1623         Q_EMIT m_owner->dataChanged(m_owner);
1624
1625     return true;
1626 }
1627
1628 /**
1629  * \brief Return the data vector size
1630  *
1631  * This returns the size of the column container
1632  */
1633 int ColumnPrivate::rowCount() const {
1634     if (!m_data)
1635         return m_rowCount;
1636
1637     switch (m_columnMode) {
1638     case AbstractColumn::ColumnMode::Double:
1639         return static_cast<QVector<double>*>(m_data)->size();
1640     case AbstractColumn::ColumnMode::Integer:
1641         return static_cast<QVector<int>*>(m_data)->size();
1642     case AbstractColumn::ColumnMode::BigInt:
1643         return static_cast<QVector<qint64>*>(m_data)->size();
1644     case AbstractColumn::ColumnMode::DateTime:
1645     case AbstractColumn::ColumnMode::Month:
1646     case AbstractColumn::ColumnMode::Day:
1647         return static_cast<QVector<QDateTime>*>(m_data)->size();
1648     case AbstractColumn::ColumnMode::Text:
1649         return static_cast<QVector<QString>*>(m_data)->size();
1650     }
1651
1652     return 0;
1653 }
1654
1655 int ColumnPrivate::rowCount(double min, double max) const {
1656     if (!m_data)
1657         return m_rowCount;
1658
1659     int counter = 0;
1660     switch (m_columnMode) {
1661     case AbstractColumn::ColumnMode::Double: {
1662         const auto* data = static_cast<QVector<double>*>(m_data);
1663         for (const auto& d : *data) {
1664             if (d >= min && d <= max)
1665                 counter++;
1666         }
1667         break;
1668     }
1669     case AbstractColumn::ColumnMode::Integer: {
1670         const auto* data = static_cast<QVector<int>*>(m_data);
1671         for (const auto& d : *data) {
1672             if (d >= min && d <= max)
1673                 counter++;
1674         }
1675         break;
1676     }
1677     case AbstractColumn::ColumnMode::BigInt: {
1678         const auto* data = static_cast<QVector<qint64>*>(m_data);
1679         for (const auto& d : *data) {
1680             if (d >= min && d <= max)
1681                 counter++;
1682         }
1683         break;
1684     }
1685     case AbstractColumn::ColumnMode::DateTime:
1686     case AbstractColumn::ColumnMode::Month:
1687     case AbstractColumn::ColumnMode::Day: {
1688         const auto* data = static_cast<QVector<QDateTime>*>(m_data);
1689         for (const auto& d : *data) {
1690             const auto value = d.toMSecsSinceEpoch();
1691             if (value >= min && value <= max)
1692                 counter++;
1693         }
1694         break;
1695     }
1696     case AbstractColumn::ColumnMode::Text:
1697         break;
1698     }
1699     return counter;
1700 }
1701
1702 /**
1703  * \brief Return the number of available rows
1704  *
1705  * This returns the number of rows that actually contain data.
1706  * Rows beyond this can be masked etc. but should be ignored by filters,
1707  * plots etc.
1708  */
1709 int ColumnPrivate::availableRowCount(int max) const {
1710     int count = 0;
1711     for (int row = 0; row < rowCount(); row++) {
1712         if (m_owner->isValid(row) && !m_owner->isMasked(row)) {
1713             count++;
1714             if (count == max)
1715                 return max;
1716         }
1717     }
1718
1719     return count;
1720 }
1721
1722 /**
1723  * \brief Resize the vector to the specified number of rows
1724  *
1725  * Since selecting and masking rows higher than the
1726  * real internal number of rows is supported, this
1727  * does not change the interval attributes. Also
1728  * no signal is emitted. If the new rows are filled
1729  * with values AbstractColumn::dataChanged()
1730  * must be emitted.
1731  */
1732 void ColumnPrivate::resizeTo(int new_size) {
1733     int old_size = rowCount();
1734     if (new_size == old_size)
1735         return;
1736
1737     //  DEBUG("ColumnPrivate::resizeTo() " << old_size << " -> " << new_size);
1738     const int new_rows = new_size - old_size;
1739
1740     if (!m_data) {
1741         m_rowCount += new_rows;
1742         return;
1743     }
1744
1745     switch (m_columnMode) {
1746     case AbstractColumn::ColumnMode::Double: {
1747         auto* data = static_cast<QVector<double>*>(m_data);
1748         if (new_rows > 0)
1749             data->insert(data->end(), new_rows, NAN);
1750         else
1751             data->remove(old_size - 1 + new_rows, -new_rows);
1752         break;
1753     }
1754     case AbstractColumn::ColumnMode::Integer: {
1755         auto* data = static_cast<QVector<int>*>(m_data);
1756         if (new_rows > 0)
1757             data->insert(data->end(), new_rows, 0);
1758         else
1759             data->remove(old_size - 1 + new_rows, -new_rows);
1760         break;
1761     }
1762     case AbstractColumn::ColumnMode::BigInt: {
1763         auto* data = static_cast<QVector<qint64>*>(m_data);
1764         if (new_rows > 0)
1765             data->insert(data->end(), new_rows, 0);
1766         else
1767             data->remove(old_size - 1 + new_rows, -new_rows);
1768         break;
1769     }
1770     case AbstractColumn::ColumnMode::Text: {
1771         auto* data = static_cast<QVector<QString>*>(m_data);
1772         if (new_rows > 0)
1773             data->insert(data->end(), new_rows, QString());
1774         else
1775             data->remove(old_size - 1 + new_rows, -new_rows);
1776         break;
1777     }
1778     case AbstractColumn::ColumnMode::DateTime:
1779     case AbstractColumn::ColumnMode::Month:
1780     case AbstractColumn::ColumnMode::Day: {
1781         auto* data = static_cast<QVector<QDateTime>*>(m_data);
1782         if (new_rows > 0)
1783             data->insert(data->end(), new_rows, QDateTime());
1784         else
1785             data->remove(old_size - 1 + new_rows, -new_rows);
1786         break;
1787     }
1788     }
1789
1790     invalidate();
1791 }
1792
1793 /**
1794  * \brief Insert some empty (or initialized with zero) rows
1795  */
1796 void ColumnPrivate::insertRows(int before, int count) {
1797     if (count == 0)
1798         return;
1799
1800     m_formulas.insertRows(before, count);
1801
1802     if (!m_data) {
1803         m_rowCount += count;
1804         return;
1805     }
1806
1807     if (before <= rowCount()) {
1808         switch (m_columnMode) {
1809         case AbstractColumn::ColumnMode::Double:
1810             static_cast<QVector<double>*>(m_data)->insert(before, count, NAN);
1811             break;
1812         case AbstractColumn::ColumnMode::Integer:
1813             static_cast<QVector<int>*>(m_data)->insert(before, count, 0);
1814             break;
1815         case AbstractColumn::ColumnMode::BigInt:
1816             static_cast<QVector<qint64>*>(m_data)->insert(before, count, 0);
1817             break;
1818         case AbstractColumn::ColumnMode::DateTime:
1819         case AbstractColumn::ColumnMode::Month:
1820         case AbstractColumn::ColumnMode::Day:
1821             for (int i = 0; i < count; ++i)
1822                 static_cast<QVector<QDateTime>*>(m_data)->insert(before, QDateTime());
1823             break;
1824         case AbstractColumn::ColumnMode::Text:
1825             for (int i = 0; i < count; ++i)
1826                 static_cast<QVector<QString>*>(m_data)->insert(before, QString());
1827             break;
1828         }
1829     }
1830
1831     invalidate();
1832 }
1833
1834 /**
1835  * \brief Remove 'count' rows starting from row 'first'
1836  */
1837 void ColumnPrivate::removeRows(int first, int count) {
1838     if (count == 0)
1839         return;
1840
1841     m_formulas.removeRows(first, count);
1842
1843     if (first < rowCount()) {
1844         int corrected_count = count;
1845         if (first + count > rowCount())
1846             corrected_count = rowCount() - first;
1847
1848         if (!m_data) {
1849             m_rowCount -= corrected_count;
1850             return;
1851         }
1852
1853         switch (m_columnMode) {
1854         case AbstractColumn::ColumnMode::Double:
1855             static_cast<QVector<double>*>(m_data)->remove(first, corrected_count);
1856             break;
1857         case AbstractColumn::ColumnMode::Integer:
1858             static_cast<QVector<int>*>(m_data)->remove(first, corrected_count);
1859             break;
1860         case AbstractColumn::ColumnMode::BigInt:
1861             static_cast<QVector<qint64>*>(m_data)->remove(first, corrected_count);
1862             break;
1863         case AbstractColumn::ColumnMode::DateTime:
1864         case AbstractColumn::ColumnMode::Month:
1865         case AbstractColumn::ColumnMode::Day:
1866             for (int i = 0; i < corrected_count; ++i)
1867                 static_cast<QVector<QDateTime>*>(m_data)->removeAt(first);
1868             break;
1869         case AbstractColumn::ColumnMode::Text:
1870             for (int i = 0; i < corrected_count; ++i)
1871                 static_cast<QVector<QString>*>(m_data)->removeAt(first);
1872             break;
1873         }
1874     }
1875
1876     invalidate();
1877 }
1878
1879 int ColumnPrivate::indexForValue(double x) const {
1880     return indexForValueCommon<Column>(m_owner,
1881                                        x,
1882                                        std::mem_fn(&Column::columnMode),
1883                                        std::mem_fn<int() const>(&Column::rowCount),
1884                                        std::mem_fn(&Column::valueAt),
1885                                        std::mem_fn(&Column::dateTimeAt),
1886                                        std::mem_fn(&Column::properties),
1887                                        std::mem_fn<bool(int) const>(&Column::isValid),
1888                                        std::mem_fn<bool(int) const>(&Column::isMasked));
1889 }
1890
1891 //! Return the column name
1892 QString ColumnPrivate::name() const {
1893     return m_owner->name();
1894 }
1895
1896 /**
1897  * \brief Return the column plot designation
1898  */
1899 AbstractColumn::PlotDesignation ColumnPrivate::plotDesignation() const {
1900     return m_plotDesignation;
1901 }
1902
1903 /**
1904  * \brief Set the column plot designation
1905  */
1906 void ColumnPrivate::setPlotDesignation(AbstractColumn::PlotDesignation pd) {
1907     Q_EMIT m_owner->plotDesignationAboutToChange(m_owner);
1908     m_plotDesignation = pd;
1909     Q_EMIT m_owner->plotDesignationChanged(m_owner);
1910 }
1911
1912 /**
1913  * \brief Get width
1914  */
1915 int ColumnPrivate::width() const {
1916     return m_width;
1917 }
1918
1919 /**
1920  * \brief Set width
1921  */
1922 void ColumnPrivate::setWidth(int value) {
1923     m_width = value;
1924 }
1925
1926 /**
1927  * @brief ColumnPrivate::setData
1928  * Set new column data
1929  */
1930 void ColumnPrivate::setData(void* data) {
1931     deleteData();
1932     m_data = data;
1933     invalidate();
1934 }
1935
1936 /**
1937  * \brief Return the data pointer
1938  */
1939 void* ColumnPrivate::data() const {
1940     if (!m_data)
1941         const_cast<ColumnPrivate*>(this)->initDataContainer();
1942
1943     return m_data;
1944 }
1945
1946 /**
1947  * \brief Return the input filter (for string -> data type conversion)
1948  */
1949 AbstractSimpleFilter* ColumnPrivate::inputFilter() const {
1950     return m_inputFilter;
1951 }
1952
1953 /**
1954  * \brief Return the output filter (for data type -> string  conversion)
1955  */
1956 AbstractSimpleFilter* ColumnPrivate::outputFilter() const {
1957     return m_outputFilter;
1958 }
1959
1960 //! \name Labels related functions
1961 //@{
1962 void ColumnPrivate::setLabelsMode(Column::ColumnMode mode) {
1963     m_labels.setMode(mode);
1964 }
1965
1966 void ColumnPrivate::valueLabelsRemoveAll() {
1967     m_labels.removeAll();
1968 }
1969
1970 bool ColumnPrivate::valueLabelsInitialized() const {
1971     return m_labels.initialized();
1972 }
1973
1974 void ColumnPrivate::removeValueLabel(const QString& key) {
1975     m_labels.remove(key);
1976 }
1977
1978 double ColumnPrivate::valueLabelsMinimum() {
1979     return m_labels.minimum();
1980 }
1981
1982 double ColumnPrivate::valueLabelsMaximum() {
1983     return m_labels.maximum();
1984 }
1985
1986 const QVector<Column::ValueLabel<QString>>* ColumnPrivate::textValueLabels() const {
1987     return m_labels.textValueLabels();
1988 }
1989
1990 const QVector<Column::ValueLabel<QDateTime>>* ColumnPrivate::dateTimeValueLabels() const {
1991     return m_labels.dateTimeValueLabels();
1992 }
1993
1994 int ColumnPrivate::valueLabelsCount() const {
1995     return m_labels.count();
1996 }
1997
1998 int ColumnPrivate::valueLabelsCount(double min, double max) const {
1999     return m_labels.count(min, max);
2000 }
2001
2002 int ColumnPrivate::valueLabelsIndexForValue(double value) const {
2003     return m_labels.indexForValue(value);
2004 }
2005
2006 double ColumnPrivate::valueLabelsValueAt(int index) const {
2007     return m_labels.valueAt(index);
2008 }
2009
2010 QString ColumnPrivate::valueLabelAt(int index) const {
2011     return m_labels.labelAt(index);
2012 }
2013
2014 const QVector<Column::ValueLabel<double>>* ColumnPrivate::valueLabels() const {
2015     return m_labels.valueLabels();
2016 }
2017
2018 const QVector<Column::ValueLabel<int>>* ColumnPrivate::intValueLabels() const {
2019     return m_labels.intValueLabels();
2020 }
2021
2022 const QVector<Column::ValueLabel<qint64>>* ColumnPrivate::bigIntValueLabels() const {
2023     return m_labels.bigIntValueLabels();
2024 }
2025 //@}
2026
2027 //! \name Formula related functions
2028 //@{
2029 /**
2030  * \brief Return the formula last used to generate data for the column
2031  */
2032 QString ColumnPrivate::formula() const {
2033     return m_formula;
2034 }
2035
2036 const QVector<Column::FormulaData>& ColumnPrivate::formulaData() const {
2037     return m_formulaData;
2038 }
2039
2040 bool ColumnPrivate::formulaAutoUpdate() const {
2041     return m_formulaAutoUpdate;
2042 }
2043
2044 bool ColumnPrivate::formulaAutoResize() const {
2045     return m_formulaAutoResize;
2046 }
2047
2048 /**
2049  * \brief Sets the formula used to generate column values
2050  */
2051 void ColumnPrivate::setFormula(const QString& formula, const QVector<Column::FormulaData>& formulaData, bool autoUpdate, bool autoResize) {
2052     m_formula = formula;
2053     m_formulaData = formulaData; // TODO: disconnecting everything?
2054     m_formulaAutoUpdate = autoUpdate;
2055     m_formulaAutoResize = autoResize;
2056
2057     for (auto& connection : m_connectionsUpdateFormula)
2058         if (static_cast<bool>(connection))
2059             disconnect(connection);
2060
2061     for (const auto& data : m_formulaData) {
2062         const auto* column = data.column();
2063         assert(column);
2064         if (autoUpdate)
2065             connectFormulaColumn(column);
2066     }
2067
2068     Q_EMIT m_owner->formulaChanged(m_owner);
2069 }
2070
2071 /*!
2072  * called after the import of the project was done and all columns were loaded in \sa Project::load()
2073  * to establish the required slot-signal connections for the formula update
2074  */
2075 void ColumnPrivate::finalizeLoad() {
2076     if (m_formulaAutoUpdate) {
2077         for (const auto& formulaData : m_formulaData) {
2078             const auto* column = formulaData.column();
2079             if (column)
2080                 connectFormulaColumn(column);
2081         }
2082     }
2083 }
2084
2085 /*!
2086  * \brief ColumnPrivate::connectFormulaColumn
2087  * This function is used to connect the columns to the needed slots for updating formulas
2088  * \param column
2089  */
2090 void ColumnPrivate::connectFormulaColumn(const AbstractColumn* column) {
2091     if (!column)
2092         return;
2093
2094     // avoid circular dependencies - the current column cannot be part of the variable columns.
2095     // this should't actually happen because of the checks done when the formula is defined,
2096     // but in case we have bugs somewhere or somebody manipulated the project xml file we add
2097     // a sanity check to avoid recursive calls here and crash because of the stack overflow.
2098     if (column == m_owner)
2099         return;
2100
2101     DEBUG(Q_FUNC_INFO)
2102     m_connectionsUpdateFormula << connect(column, &AbstractColumn::dataChanged, m_owner, &Column::updateFormula);
2103     connect(column->parentAspect(),
2104             QOverload<const AbstractAspect*>::of(&AbstractAspect::childAspectAboutToBeRemoved),
2105             this,
2106             &ColumnPrivate::formulaVariableColumnRemoved);
2107     connect(column, &AbstractColumn::aboutToReset, this, &ColumnPrivate::formulaVariableColumnRemoved);
2108     connect(column->parentAspect(), &AbstractAspect::childAspectAdded, this, &ColumnPrivate::formulaVariableColumnAdded);
2109 }
2110
2111 /*!
2112  * helper function used in \c Column::load() to set parameters read from the xml file.
2113  * \param variableColumnPaths is used to restore the pointers to columns from pathes
2114  * after the project was loaded in Project::load().
2115  */
2116 void ColumnPrivate::setFormula(const QString& formula,
2117                                const QStringList& variableNames,
2118                                const QStringList& variableColumnPaths,
2119                                bool autoUpdate,
2120                                bool autoResize) {
2121     m_formula = formula;
2122     m_formulaData.clear();
2123     for (int i = 0; i < variableNames.count(); i++)
2124         m_formulaData.append(Column::FormulaData(variableNames.at(i), variableColumnPaths.at(i)));
2125
2126     m_formulaAutoUpdate = autoUpdate;
2127     m_formulaAutoResize = autoResize;
2128 }
2129
2130 void ColumnPrivate::setFormulVariableColumnsPath(int index, const QString& path) {
2131     if (!m_formulaData[index].setColumnPath(path))
2132         DEBUG(Q_FUNC_INFO << ": For some reason, there was already a column assigned");
2133 }
2134
2135 void ColumnPrivate::setFormulVariableColumn(int index, Column* column) {
2136     if (m_formulaData.at(index).column()) // if there exists already a valid column, disconnect it first
2137         disconnect(m_formulaData.at(index).column(), nullptr, this, nullptr);
2138     m_formulaData[index].setColumn(column);
2139     connectFormulaColumn(column);
2140 }
2141
2142 void ColumnPrivate::setFormulVariableColumn(Column* c) {
2143     for (auto& d : m_formulaData) {
2144         if (d.columnName() == c->path()) {
2145             d.setColumn(c);
2146             break;
2147         }
2148     }
2149 }
2150
2151 struct PayloadColumn : public Payload {
2152     PayloadColumn(const QVector<Column::FormulaData>& data)
2153         : Payload(true)
2154         , formulaData(data) {
2155     }
2156     const QVector<Column::FormulaData>& formulaData;
2157 };
2158
2159 #define COLUMN_FUNCTION(function_name, evaluation_function)                                                                                                    \
2160     double column##function_name(const char* variable, const std::weak_ptr<Payload> payload) {                                                                 \
2161         const auto p = std::dynamic_pointer_cast<PayloadColumn>(payload.lock());                                                                               \
2162         if (!p) {                                                                                                                                              \
2163             assert(p); /* Debug build */                                                                                                                       \
2164             return NAN;                                                                                                                                        \
2165         }                                                                                                                                                      \
2166         for (const auto& formulaData : p->formulaData) {                                                                                                       \
2167             if (formulaData.variableName().compare(QLatin1String(variable)) == 0)                                                                              \
2168                 return formulaData.column()->evaluation_function;                                                                                              \
2169         }                                                                                                                                                      \
2170         return NAN;                                                                                                                                            \
2171     }
2172
2173 // Constant functions, which return always the same value independet of the row index
2174 COLUMN_FUNCTION(Size, statistics().size)
2175 COLUMN_FUNCTION(Min, minimum())
2176 COLUMN_FUNCTION(Max, maximum())
2177 COLUMN_FUNCTION(Mean, statistics().arithmeticMean)
2178 COLUMN_FUNCTION(Median, statistics().median)
2179 COLUMN_FUNCTION(Stdev, statistics().standardDeviation)
2180 COLUMN_FUNCTION(Var, statistics().variance)
2181 COLUMN_FUNCTION(Gm, statistics().geometricMean)
2182 COLUMN_FUNCTION(Hm, statistics().harmonicMean)
2183 COLUMN_FUNCTION(Chm, statistics().contraharmonicMean)
2184 COLUMN_FUNCTION(StatisticsMode, statistics().mode)
2185 COLUMN_FUNCTION(Quartile1, statistics().firstQuartile)
2186 COLUMN_FUNCTION(Quartile3, statistics().thirdQuartile)
2187 COLUMN_FUNCTION(Iqr, statistics().iqr)
2188 COLUMN_FUNCTION(Percentile1, statistics().percentile_1)
2189 COLUMN_FUNCTION(Percentile5, statistics().percentile_5)
2190 COLUMN_FUNCTION(Percentile10, statistics().percentile_10)
2191 COLUMN_FUNCTION(Percentile90, statistics().percentile_90)
2192 COLUMN_FUNCTION(Percentile95, statistics().percentile_95)
2193 COLUMN_FUNCTION(Percentile99, statistics().percentile_99)
2194 COLUMN_FUNCTION(Trimean, statistics().trimean)
2195 COLUMN_FUNCTION(Meandev, statistics().meanDeviation)
2196 COLUMN_FUNCTION(Meandevmedian, statistics().meanDeviationAroundMedian)
2197 COLUMN_FUNCTION(Mediandev, statistics().medianDeviation)
2198 COLUMN_FUNCTION(Skew, statistics().skewness)
2199 COLUMN_FUNCTION(Kurt, statistics().kurtosis)
2200 COLUMN_FUNCTION(Entropy, statistics().entropy)
2201
2202 double columnQuantile(double p, const char* variable, const std::weak_ptr<Payload> payload) {
2203     const auto pd = std::dynamic_pointer_cast<PayloadColumn>(payload.lock());
2204     if (!pd) {
2205         assert(pd); // Debug build
2206         return NAN;
2207     }
2208
2209     if (p < 0)
2210         return NAN;
2211
2212     const Column* column = nullptr;
2213     for (const auto& formulaData : pd->formulaData) {
2214         if (formulaData.variableName().compare(QLatin1String(variable)) == 0) {
2215             column = formulaData.column();
2216             break;
2217         }
2218     }
2219     if (!column)
2220         return NAN;
2221
2222     double value = 0.0;
2223     switch (column->columnMode()) { // all types
2224     case AbstractColumn::ColumnMode::Double: {
2225         auto data = reinterpret_cast<QVector<double>*>(column->data());
2226         value = nsl_stats_quantile(data->data(), 1, column->statistics().size, p, nsl_stats_quantile_type7);
2227         break;
2228     }
2229     case AbstractColumn::ColumnMode::Integer: {
2230         auto* intData = reinterpret_cast<QVector<int>*>(column->data());
2231
2232         QVector<double> data = QVector<double>(); // copy data to double
2233         data.reserve(column->rowCount());
2234         for (auto v : *intData)
2235             data << static_cast<double>(v);
2236         value = nsl_stats_quantile(data.data(), 1, column->statistics().size, p, nsl_stats_quantile_type7);
2237         break;
2238     }
2239     case AbstractColumn::ColumnMode::BigInt: {
2240         auto* bigIntData = reinterpret_cast<QVector<qint64>*>(column->data());
2241
2242         QVector<double> data = QVector<double>(); // copy data to double
2243         data.reserve(column->rowCount());
2244         for (auto v : *bigIntData)
2245             data << static_cast<double>(v);
2246         value = nsl_stats_quantile(data.data(), 1, column->statistics().size, p, nsl_stats_quantile_type7);
2247         break;
2248     }
2249     case AbstractColumn::ColumnMode::DateTime: // not supported yet
2250     case AbstractColumn::ColumnMode::Day:
2251     case AbstractColumn::ColumnMode::Month:
2252     case AbstractColumn::ColumnMode::Text:
2253         break;
2254     }
2255     return value;
2256 }
2257
2258 double columnPercentile(double p, const char* variable, const std::weak_ptr<Payload> payload) {
2259     return columnQuantile(p / 100., variable, payload);
2260 }
2261
2262 /*!
2263  * \sa FunctionValuesDialog::generate()
2264  */
2265 void ColumnPrivate::updateFormula() {
2266     if (m_formula.isEmpty())
2267         return;
2268     DEBUG(Q_FUNC_INFO)
2269     // determine variable names and the data vectors of the specified columns
2270     QVector<QVector<double>*> xVectors;
2271
2272     bool valid = true;
2273     QStringList formulaVariableNames;
2274     int maxRowCount = 0;
2275
2276     auto numberLocale = QLocale();
2277     // need to disable group separator since parser can't handle it
2278     numberLocale.setNumberOptions(QLocale::OmitGroupSeparator);
2279
2280     for (const auto& formulaData : m_formulaData) {
2281         auto* column = formulaData.column();
2282         if (!column) {
2283             valid = false;
2284             break;
2285         }
2286         auto varName = formulaData.variableName();
2287         formulaVariableNames << varName;
2288
2289         if (column->columnMode() == AbstractColumn::ColumnMode::Integer || column->columnMode() == AbstractColumn::ColumnMode::BigInt) {
2290             // convert integers to doubles first
2291             auto* xVector = new QVector<double>(column->rowCount());
2292             for (int i = 0; i < column->rowCount(); ++i)
2293                 (*xVector)[i] = column->valueAt(i);
2294
2295             xVectors << xVector;
2296         } else
2297             xVectors << static_cast<QVector<double>*>(column->data());
2298
2299         if (column->rowCount() > maxRowCount)
2300             maxRowCount = column->rowCount();
2301     }
2302
2303     if (valid) {
2304         // resize the spreadsheet if one of the data vectors from
2305         // other spreadsheet(s) has more elements than the parent spreadsheet
2306         // and if the option "auto resize" is activated
2307         if (m_formulaAutoResize && rowCount() < maxRowCount) {
2308             auto* spreadsheet = static_cast<Spreadsheet*>(m_owner->parentAspect());
2309             // In the tests spreadsheet might not exist, because directly the column was created
2310             if (spreadsheet)
2311                 spreadsheet->setRowCount(maxRowCount);
2312         }
2313
2314         // create new vector for storing the calculated values
2315         // the vectors with the variable data can be smaller then the result vector. So, not all values in the result vector might get initialized.
2316         //->"clean" the result vector first
2317         QVector<double> new_data(rowCount(), NAN);
2318
2319         const auto payload = std::make_shared<PayloadColumn>(m_formulaData);
2320
2321         // evaluate the expression for f(x_1, x_2, ...) and write the calculated values into a new vector.
2322         auto* parser = ExpressionParser::getInstance();
2323         parser->setSpecialFunction1(colfun_size, columnSize, payload);
2324         parser->setSpecialFunction1(colfun_min, columnMin, payload);
2325         parser->setSpecialFunction1(colfun_max, columnMax, payload);
2326         parser->setSpecialFunction1(colfun_mean, columnMean, payload);
2327         parser->setSpecialFunction1(colfun_median, columnMedian, payload);
2328         parser->setSpecialFunction1(colfun_stdev, columnStdev, payload);
2329         parser->setSpecialFunction1(colfun_var, columnVar, payload);
2330         parser->setSpecialFunction1(colfun_gm, columnGm, payload);
2331         parser->setSpecialFunction1(colfun_hm, columnHm, payload);
2332         parser->setSpecialFunction1(colfun_chm, columnChm, payload);
2333         parser->setSpecialFunction1(colfun_mode, columnStatisticsMode, payload);
2334         parser->setSpecialFunction1(colfun_quartile1, columnQuartile1, payload);
2335         parser->setSpecialFunction1(colfun_quartile3, columnQuartile3, payload);
2336         parser->setSpecialFunction1(colfun_iqr, columnIqr, payload);
2337         parser->setSpecialFunction1(colfun_percentile1, columnPercentile1, payload);
2338         parser->setSpecialFunction1(colfun_percentile5, columnPercentile5, payload);
2339         parser->setSpecialFunction1(colfun_percentile10, columnPercentile10, payload);
2340         parser->setSpecialFunction1(colfun_percentile90, columnPercentile90, payload);
2341         parser->setSpecialFunction1(colfun_percentile95, columnPercentile95, payload);
2342         parser->setSpecialFunction1(colfun_percentile99, columnPercentile99, payload);
2343         parser->setSpecialFunction1(colfun_trimean, columnTrimean, payload);
2344         parser->setSpecialFunction1(colfun_meandev, columnMeandev, payload);
2345         parser->setSpecialFunction1(colfun_meandevmedian, columnMeandevmedian, payload);
2346         parser->setSpecialFunction1(colfun_mediandev, columnMediandev, payload);
2347         parser->setSpecialFunction1(colfun_skew, columnSkew, payload);
2348         parser->setSpecialFunction1(colfun_kurt, columnKurt, payload);
2349         parser->setSpecialFunction1(colfun_entropy, columnEntropy, payload);
2350         parser->setSpecialFunction2(colfun_percentile, columnPercentile, payload);
2351         parser->setSpecialFunction2(colfun_quantile, columnQuantile, payload);
2352
2353         QDEBUG(Q_FUNC_INFO << ", Calling evaluateCartesian(). formula: " << m_formula << ", var names: " << formulaVariableNames)
2354         parser->evaluateCartesian(m_formula, formulaVariableNames, xVectors, &new_data);
2355         DEBUG(Q_FUNC_INFO << ", Calling replaceValues()")
2356         replaceValues(-1, new_data);
2357
2358         // initialize remaining rows with NAN
2359         // This will be done already in evaluateCartesian()
2360         // int remainingRows = rowCount() - maxRowCount;
2361         // if (remainingRows > 0) {
2362         //  QVector<double> emptyRows(remainingRows, NAN);
2363         //  replaceValues(maxRowCount, emptyRows);
2364         //}
2365     } else { // not valid
2366         QVector<double> new_data(rowCount(), NAN);
2367         replaceValues(-1, new_data);
2368     }
2369
2370     DEBUG(Q_FUNC_INFO << " DONE")
2371 }
2372
2373 void ColumnPrivate::formulaVariableColumnRemoved(const AbstractAspect* aspect) {
2374     const Column* column = dynamic_cast<const Column*>(aspect);
2375     disconnect(column, nullptr, this, nullptr);
2376     int index = -1;
2377     for (int i = 0; i < formulaData().count(); i++) {
2378         auto& d = formulaData().at(i);
2379         if (d.column() == column) {
2380             index = i;
2381             break;
2382         }
2383     }
2384     if (index != -1) {
2385         m_formulaData[index].setColumn(nullptr);
2386         DEBUG(Q_FUNC_INFO << ", calling updateFormula()")
2387         updateFormula();
2388     }
2389 }
2390
2391 void ColumnPrivate::formulaVariableColumnAdded(const AbstractAspect* aspect) {
2392     PERFTRACE(QLatin1String(Q_FUNC_INFO));
2393     const auto& path = aspect->path();
2394     int index = -1;
2395     for (int i = 0; i < formulaData().count(); i++) {
2396         if (formulaData().at(i).columnName() == path) {
2397             index = i;
2398             break;
2399         }
2400     }
2401     if (index != -1) {
2402         const Column* column = dynamic_cast<const Column*>(aspect);
2403         m_formulaData[index].setColumn(const_cast<Column*>(column));
2404         DEBUG(Q_FUNC_INFO << ", calling updateFormula()")
2405         updateFormula();
2406     }
2407 }
2408
2409 /**
2410  * \brief Return the formula associated with row 'row'
2411  */
2412 QString ColumnPrivate::formula(int row) const {
2413     return m_formulas.value(row);
2414 }
2415
2416 /**
2417  * \brief Return the intervals that have associated formulas
2418  *
2419  * This can be used to make a list of formulas with their intervals.
2420  * Here is some example code:
2421  *
2422  * \code
2423  * QStringList list;
2424  * QVector< Interval<int> > intervals = my_column.formulaIntervals();
2425  * foreach(Interval<int> interval, intervals)
2426  *  list << QString(interval.toString() + ": " + my_column.formula(interval.start()));
2427  * \endcode
2428  */
2429 QVector<Interval<int>> ColumnPrivate::formulaIntervals() const {
2430     return m_formulas.intervals();
2431 }
2432
2433 /**
2434  * \brief Set a formula string for an interval of rows
2435  */
2436 void ColumnPrivate::setFormula(const Interval<int>& i, const QString& formula) {
2437     m_formulas.setValue(i, formula);
2438 }
2439
2440 /**
2441  * \brief Overloaded function for convenience
2442  */
2443 void ColumnPrivate::setFormula(int row, const QString& formula) {
2444     setFormula(Interval<int>(row, row), formula);
2445 }
2446
2447 /**
2448  * \brief Clear all formulas
2449  */
2450 void ColumnPrivate::clearFormulas() {
2451     m_formulas.clear();
2452 }
2453 //@}
2454
2455 ////////////////////////////////////////////////////////////////////////////////
2456 //! \name type specific functions
2457 //@{
2458 ////////////////////////////////////////////////////////////////////////////////
2459 void ColumnPrivate::setValueAt(int row, int new_value) {
2460     setIntegerAt(row, new_value);
2461 }
2462
2463 void ColumnPrivate::setValueAt(int row, qint64 new_value) {
2464     setBigIntAt(row, new_value);
2465 }
2466
2467 void ColumnPrivate::setValueAt(int row, QDateTime new_value) {
2468     setDateTimeAt(row, new_value);
2469 }
2470
2471 void ColumnPrivate::setValueAt(int row, QString new_value) {
2472     setTextAt(row, new_value);
2473 }
2474
2475 void ColumnPrivate::replaceValues(int first, const QVector<int>& new_values) {
2476     replaceInteger(first, new_values);
2477 }
2478
2479 void ColumnPrivate::replaceValues(int first, const QVector<qint64>& new_values) {
2480     replaceBigInt(first, new_values);
2481 }
2482
2483 void ColumnPrivate::replaceValues(int first, const QVector<QDateTime>& new_values) {
2484     replaceDateTimes(first, new_values);
2485 }
2486
2487 void ColumnPrivate::replaceValues(int first, const QVector<QString>& new_values) {
2488     replaceTexts(first, new_values);
2489 }
2490
2491 /**
2492  * \brief Return the content of row 'row'.
2493  *
2494  * Use this only when columnMode() is Text
2495  */
2496 QString ColumnPrivate::textAt(int row) const {
2497     if (!m_data || m_columnMode != AbstractColumn::ColumnMode::Text)
2498         return {};
2499     return static_cast<QVector<QString>*>(m_data)->value(row);
2500 }
2501
2502 /**
2503  * \brief Return the date part of row 'row'
2504  *
2505  * Use this only when columnMode() is DateTime, Month or Day
2506  */
2507 QDate ColumnPrivate::dateAt(int row) const {
2508     if (!m_data
2509         || (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2510             && m_columnMode != AbstractColumn::ColumnMode::Day))
2511         return QDate{};
2512     return dateTimeAt(row).date();
2513 }
2514
2515 /**
2516  * \brief Return the time part of row 'row'
2517  *
2518  * Use this only when columnMode() is DateTime, Month or Day
2519  */
2520 QTime ColumnPrivate::timeAt(int row) const {
2521     if (!m_data
2522         || (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2523             && m_columnMode != AbstractColumn::ColumnMode::Day))
2524         return QTime{};
2525     return dateTimeAt(row).time();
2526 }
2527
2528 /**
2529  * \brief Return the QDateTime in row 'row'
2530  *
2531  * Use this only when columnMode() is DateTime, Month or Day
2532  */
2533 QDateTime ColumnPrivate::dateTimeAt(int row) const {
2534     if (!m_data
2535         || (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2536             && m_columnMode != AbstractColumn::ColumnMode::Day))
2537         return QDateTime();
2538     return static_cast<QVector<QDateTime>*>(m_data)->value(row);
2539 }
2540
2541 double ColumnPrivate::doubleAt(int index) const {
2542     if (!m_data)
2543         return NAN;
2544
2545     return static_cast<QVector<double>*>(m_data)->value(index, NAN);
2546 }
2547
2548 /**
2549  * \brief Return the double value at index 'index' for columns with type Numeric, Integer or BigInt.
2550  * This function has to be used everywhere where the exact type (double, int or qint64) is not relevant for numerical calculations.
2551  * For cases where the integer value is needed without any implicit conversions, \sa integerAt() has to be used.
2552  */
2553 double ColumnPrivate::valueAt(int index) const {
2554     if (!m_data)
2555         return NAN;
2556
2557     switch (m_columnMode) {
2558     case AbstractColumn::ColumnMode::Double:
2559         return static_cast<QVector<double>*>(m_data)->value(index, NAN);
2560     case AbstractColumn::ColumnMode::Integer:
2561         return static_cast<QVector<int>*>(m_data)->value(index, 0);
2562     case AbstractColumn::ColumnMode::BigInt:
2563         return static_cast<QVector<qint64>*>(m_data)->value(index, 0);
2564     case AbstractColumn::ColumnMode::DateTime:
2565         return static_cast<QVector<QDateTime>*>(m_data)->value(index).toMSecsSinceEpoch();
2566     case AbstractColumn::ColumnMode::Month: // Fall through
2567     case AbstractColumn::ColumnMode::Day: // Fall through
2568     case AbstractColumn::ColumnMode::Text: // Fall through
2569         break;
2570     }
2571     return NAN;
2572 }
2573
2574 /**
2575  * \brief Return the int value in row 'row'
2576  */
2577 int ColumnPrivate::integerAt(int row) const {
2578     if (!m_data || m_columnMode != AbstractColumn::ColumnMode::Integer)
2579         return 0;
2580     return static_cast<QVector<int>*>(m_data)->value(row, 0);
2581 }
2582
2583 /**
2584  * \brief Return the bigint value in row 'row'
2585  */
2586 qint64 ColumnPrivate::bigIntAt(int row) const {
2587     if (!m_data || m_columnMode != AbstractColumn::ColumnMode::BigInt)
2588         return 0;
2589     return static_cast<QVector<qint64>*>(m_data)->value(row, 0);
2590 }
2591
2592 void ColumnPrivate::invalidate() {
2593     available.setUnavailable();
2594 }
2595
2596 /**
2597  * \brief Set the content of row 'row'
2598  *
2599  * Use this only when columnMode() is Text
2600  */
2601 void ColumnPrivate::setTextAt(int row, const QString& new_value) {
2602     if (m_columnMode != AbstractColumn::ColumnMode::Text)
2603         return;
2604
2605     setValueAtPrivate<QString>(row, new_value);
2606 }
2607
2608 /**
2609  * \brief Replace a range of values
2610  *
2611  * Use this only when columnMode() is Text
2612  */
2613 void ColumnPrivate::replaceTexts(int first, const QVector<QString>& new_values) {
2614     if (m_columnMode != AbstractColumn::ColumnMode::Text)
2615         return;
2616
2617     replaceValuePrivate<QString>(first, new_values);
2618 }
2619
2620 int ColumnPrivate::dictionaryIndex(int row) const {
2621     if (!available.dictionary)
2622         const_cast<ColumnPrivate*>(this)->initDictionary();
2623
2624     const auto& value = textAt(row);
2625     int index = 0;
2626     auto it = m_dictionary.constBegin();
2627     while (it != m_dictionary.constEnd()) {
2628         if (*it == value)
2629             break;
2630         ++index;
2631         ++it;
2632     }
2633
2634     return index;
2635 }
2636
2637 const QMap<QString, int>& ColumnPrivate::frequencies() const {
2638     if (!available.dictionary)
2639         const_cast<ColumnPrivate*>(this)->initDictionary();
2640
2641     return m_dictionaryFrequencies;
2642 }
2643
2644 void ColumnPrivate::initDictionary() {
2645     m_dictionary.clear();
2646     m_dictionaryFrequencies.clear();
2647     if (!m_data || columnMode() != AbstractColumn::ColumnMode::Text)
2648         return;
2649
2650     auto data = static_cast<QVector<QString>*>(m_data);
2651     for (auto& value : *data) {
2652         if (value.isEmpty())
2653             continue;
2654
2655         if (!m_dictionary.contains(value))
2656             m_dictionary << value;
2657
2658         if (m_dictionaryFrequencies.constFind(value) == m_dictionaryFrequencies.constEnd())
2659             m_dictionaryFrequencies[value] = 1;
2660         else
2661             m_dictionaryFrequencies[value]++;
2662     }
2663
2664     available.dictionary = true;
2665 }
2666
2667 /**
2668  * \brief Set the content of row 'row'
2669  *
2670  * Use this only when columnMode() is DateTime, Month or Day
2671  */
2672 void ColumnPrivate::setDateAt(int row, QDate new_value) {
2673     if (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2674         && m_columnMode != AbstractColumn::ColumnMode::Day)
2675         return;
2676
2677     if (!m_data)
2678         initDataContainer();
2679
2680     if (!m_data) // failed to allocate memory
2681         return;
2682
2683     setDateTimeAt(row, QDateTime(new_value, timeAt(row)));
2684 }
2685
2686 /**
2687  * \brief Set the content of row 'row'
2688  *
2689  * Use this only when columnMode() is DateTime, Month or Day
2690  */
2691 void ColumnPrivate::setTimeAt(int row, QTime new_value) {
2692     if (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2693         && m_columnMode != AbstractColumn::ColumnMode::Day)
2694         return;
2695
2696     if (!m_data)
2697         initDataContainer();
2698
2699     if (!m_data) // failed to allocate memory
2700         return;
2701
2702     setDateTimeAt(row, QDateTime(dateAt(row), new_value));
2703 }
2704
2705 /**
2706  * \brief Set the content of row 'row'
2707  *
2708  * Use this only when columnMode() is DateTime, Month or Day
2709  */
2710 void ColumnPrivate::setDateTimeAt(int row, const QDateTime& new_value) {
2711     if (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2712         && m_columnMode != AbstractColumn::ColumnMode::Day)
2713         return;
2714
2715     setValueAtPrivate<QDateTime>(row, new_value);
2716 }
2717
2718 /**
2719  * \brief Replace a range of values
2720  * \param first first index which should be replaced. If first < 0, the complete vector
2721  * will be replaced
2722  * \param new_values
2723  * Use this only when columnMode() is DateTime, Month or Day
2724  */
2725 void ColumnPrivate::replaceDateTimes(int first, const QVector<QDateTime>& new_values) {
2726     if (m_columnMode != AbstractColumn::ColumnMode::DateTime && m_columnMode != AbstractColumn::ColumnMode::Month
2727         && m_columnMode != AbstractColumn::ColumnMode::Day)
2728         return;
2729
2730     replaceValuePrivate<QDateTime>(first, new_values);
2731 }
2732
2733 /**
2734  * \brief Set the content of row 'row'
2735  *
2736  * Use this only when columnMode() is Numeric
2737  */
2738 void ColumnPrivate::setValueAt(int row, double new_value) {
2739     // DEBUG(Q_FUNC_INFO);
2740     if (m_columnMode != AbstractColumn::ColumnMode::Double)
2741         return;
2742
2743     setValueAtPrivate<double>(row, new_value);
2744 }
2745
2746 /**
2747  * \brief Replace a range of values
2748  *
2749  * Use this only when columnMode() is Numeric
2750  */
2751 void ColumnPrivate::replaceValues(int first, const QVector<double>& new_values) {
2752     // DEBUG(Q_FUNC_INFO);
2753     if (m_columnMode != AbstractColumn::ColumnMode::Double)
2754         return;
2755
2756     if (!m_data) {
2757         const bool resize = (first >= 0);
2758         if (!initDataContainer(resize))
2759             return; // failed to allocate memory
2760     }
2761
2762     invalidate();
2763
2764     Q_EMIT m_owner->dataAboutToChange(m_owner);
2765     if (first < 0)
2766         *static_cast<QVector<double>*>(m_data) = new_values;
2767     else {
2768         const int num_rows = new_values.size();
2769         resizeTo(first + num_rows);
2770
2771         double* ptr = static_cast<QVector<double>*>(m_data)->data();
2772         for (int i = 0; i < num_rows; ++i)
2773             ptr[first + i] = new_values.at(i);
2774     }
2775
2776     if (!m_owner->m_suppressDataChangedSignal)
2777         Q_EMIT m_owner->dataChanged(m_owner);
2778 }
2779
2780 void ColumnPrivate::addValueLabel(const QString& value, const QString& label) {
2781     m_labels.add(value, label);
2782 }
2783
2784 void ColumnPrivate::addValueLabel(const QDateTime& value, const QString& label) {
2785     m_labels.add(value, label);
2786 }
2787
2788 void ColumnPrivate::addValueLabel(double value, const QString& label) {
2789     m_labels.add(value, label);
2790 }
2791
2792 void ColumnPrivate::addValueLabel(int value, const QString& label) {
2793     m_labels.add(value, label);
2794 }
2795
2796 void ColumnPrivate::addValueLabel(qint64 value, const QString& label) {
2797     m_labels.add(value, label);
2798 }
2799
2800 /**
2801  * \brief Set the content of row 'row'
2802  *
2803  * Use this only when columnMode() is Integer
2804  */
2805 void ColumnPrivate::setIntegerAt(int row, int new_value) {
2806     // DEBUG(Q_FUNC_INFO);
2807     if (m_columnMode != AbstractColumn::ColumnMode::Integer)
2808         return;
2809
2810     setValueAtPrivate<int>(row, new_value);
2811 }
2812
2813 /**
2814  * \brief Replace a range of values
2815  *
2816  * Use this only when columnMode() is Integer
2817  */
2818 void ColumnPrivate::replaceInteger(int first, const QVector<int>& new_values) {
2819     // DEBUG(Q_FUNC_INFO);
2820     if (m_columnMode != AbstractColumn::ColumnMode::Integer)
2821         return;
2822
2823     replaceValuePrivate<int>(first, new_values);
2824 }
2825
2826 /**
2827  * \brief Set the content of row 'row'
2828  *
2829  * Use this only when columnMode() is BigInt
2830  */
2831 void ColumnPrivate::setBigIntAt(int row, qint64 new_value) {
2832     // DEBUG(Q_FUNC_INFO);
2833     if (m_columnMode != AbstractColumn::ColumnMode::BigInt)
2834         return;
2835
2836     setValueAtPrivate<qint64>(row, new_value);
2837 }
2838
2839 /**
2840  * \brief Replace a range of values
2841  *
2842  * Use this only when columnMode() is BigInt
2843  */
2844 void ColumnPrivate::replaceBigInt(int first, const QVector<qint64>& new_values) {
2845     // DEBUG(Q_FUNC_INFO);
2846     if (m_columnMode != AbstractColumn::ColumnMode::BigInt)
2847         return;
2848
2849     replaceValuePrivate<qint64>(first, new_values);
2850 }
2851
2852 /*!
2853  * Updates the properties. Will be called, when data in the column changed.
2854  * The properties will be used to speed up some algorithms.
2855  * See where variable properties will be used.
2856  */
2857 void ColumnPrivate::updateProperties() {
2858     // DEBUG(Q_FUNC_INFO);
2859
2860     // TODO: for double Properties::Constant will never be used. Use an epsilon (difference smaller than epsilon is zero)
2861     int rows = rowCount();
2862     if (rowCount() == 0) {
2863         properties = AbstractColumn::Properties::No;
2864         available.properties = true;
2865         return;
2866     }
2867
2868     double prevValue = NAN;
2869     int prevValueInt = 0;
2870     qint64 prevValueBigInt = 0;
2871     qint64 prevValueDatetime = 0;
2872
2873     if (m_columnMode == AbstractColumn::ColumnMode::Integer)
2874         prevValueInt = integerAt(0);
2875     else if (m_columnMode == AbstractColumn::ColumnMode::BigInt)
2876         prevValueBigInt = bigIntAt(0);
2877     else if (m_columnMode == AbstractColumn::ColumnMode::Double)
2878         prevValue = valueAt(0);
2879     else if (m_columnMode == AbstractColumn::ColumnMode::DateTime || m_columnMode == AbstractColumn::ColumnMode::Month
2880              || m_columnMode == AbstractColumn::ColumnMode::Day)
2881         prevValueDatetime = dateTimeAt(0).toMSecsSinceEpoch();
2882     else {
2883         properties = AbstractColumn::Properties::No;
2884         available.properties = true;
2885         return;
2886     }
2887
2888     int monotonic_decreasing = -1;
2889     int monotonic_increasing = -1;
2890
2891     double value;
2892     int valueInt;
2893     qint64 valueBigInt;
2894     qint64 valueDateTime;
2895     for (int row = 1; row < rows; row++) {
2896         if (!m_owner->isValid(row) || m_owner->isMasked(row)) {
2897             // if there is one invalid or masked value, the property is No, because
2898             // otherwise it's difficult to find the correct index in indexForValue().
2899             // You don't know if you should increase the index or decrease it when
2900             // you hit an invalid value
2901             properties = AbstractColumn::Properties::No;
2902             available.properties = true;
2903             return;
2904         }
2905
2906         if (m_columnMode == AbstractColumn::ColumnMode::Integer) {
2907             valueInt = integerAt(row);
2908
2909             if (valueInt > prevValueInt) {
2910                 monotonic_decreasing = 0;
2911                 if (monotonic_increasing < 0)
2912                     monotonic_increasing = 1;
2913                 else if (monotonic_increasing == 0)
2914                     break; // when nor increasing, nor decreasing, break
2915
2916             } else if (valueInt < prevValueInt) {
2917                 monotonic_increasing = 0;
2918                 if (monotonic_decreasing < 0)
2919                     monotonic_decreasing = 1;
2920                 else if (monotonic_decreasing == 0)
2921                     break; // when nor increasing, nor decreasing, break
2922
2923             } else {
2924                 if (monotonic_increasing < 0 && monotonic_decreasing < 0) {
2925                     monotonic_decreasing = 1;
2926                     monotonic_increasing = 1;
2927                 }
2928             }
2929
2930             prevValueInt = valueInt;
2931         } else if (m_columnMode == AbstractColumn::ColumnMode::BigInt) {
2932             valueBigInt = bigIntAt(row);
2933
2934             if (valueBigInt > prevValueBigInt) {
2935                 monotonic_decreasing = 0;
2936                 if (monotonic_increasing < 0)
2937                     monotonic_increasing = 1;
2938                 else if (monotonic_increasing == 0)
2939                     break; // when nor increasing, nor decreasing, break
2940
2941             } else if (valueBigInt < prevValueBigInt) {
2942                 monotonic_increasing = 0;
2943                 if (monotonic_decreasing < 0)
2944                     monotonic_decreasing = 1;
2945                 else if (monotonic_decreasing == 0)
2946                     break; // when nor increasing, nor decreasing, break
2947
2948             } else {
2949                 if (monotonic_increasing < 0 && monotonic_decreasing < 0) {
2950                     monotonic_decreasing = 1;
2951                     monotonic_increasing = 1;
2952                 }
2953             }
2954
2955             prevValueBigInt = valueBigInt;
2956         } else if (m_columnMode == AbstractColumn::ColumnMode::Double) {
2957             value = valueAt(row);
2958
2959             if (std::isnan(value)) {
2960                 monotonic_increasing = 0;
2961                 monotonic_decreasing = 0;
2962                 break;
2963             }
2964
2965             if (value > prevValue) {
2966                 monotonic_decreasing = 0;
2967                 if (monotonic_increasing < 0)
2968                     monotonic_increasing = 1;
2969                 else if (monotonic_increasing == 0)
2970                     break; // when nor increasing, nor decreasing, break
2971
2972             } else if (value < prevValue) {
2973                 monotonic_increasing = 0;
2974                 if (monotonic_decreasing < 0)
2975                     monotonic_decreasing = 1;
2976                 else if (monotonic_decreasing == 0)
2977                     break; // when nor increasing, nor decreasing, break
2978
2979             } else {
2980                 if (monotonic_increasing < 0 && monotonic_decreasing < 0) {
2981                     monotonic_decreasing = 1;
2982                     monotonic_increasing = 1;
2983                 }
2984             }
2985
2986             prevValue = value;
2987         } else if (m_columnMode == AbstractColumn::ColumnMode::DateTime || m_columnMode == AbstractColumn::ColumnMode::Month
2988                    || m_columnMode == AbstractColumn::ColumnMode::Day) {
2989             valueDateTime = dateTimeAt(row).toMSecsSinceEpoch();
2990
2991             if (valueDateTime > prevValueDatetime) {
2992                 monotonic_decreasing = 0;
2993                 if (monotonic_increasing < 0)
2994                     monotonic_increasing = 1;
2995                 else if (monotonic_increasing == 0)
2996                     break; // when nor increasing, nor decreasing, break
2997
2998             } else if (valueDateTime < prevValueDatetime) {
2999                 monotonic_increasing = 0;
3000                 if (monotonic_decreasing < 0)
3001                     monotonic_decreasing = 1;
3002                 else if (monotonic_decreasing == 0)
3003                     break; // when nor increasing, nor decreasing, break
3004
3005             } else {
3006                 if (monotonic_increasing < 0 && monotonic_decreasing < 0) {
3007                     monotonic_decreasing = 1;
3008                     monotonic_increasing = 1;
3009                 }
3010             }
3011
3012             prevValueDatetime = valueDateTime;
3013         }
3014     }
3015
3016     properties = AbstractColumn::Properties::NonMonotonic;
3017     if (monotonic_increasing > 0 && monotonic_decreasing > 0) {
3018         properties = AbstractColumn::Properties::Constant;
3019         DEBUG(" setting column CONSTANT")
3020     } else if (monotonic_decreasing > 0) {
3021         properties = AbstractColumn::Properties::MonotonicDecreasing;
3022         DEBUG(" setting column MONTONIC DECREASING")
3023     } else if (monotonic_increasing > 0) {
3024         properties = AbstractColumn::Properties::MonotonicIncreasing;
3025         DEBUG(" setting column MONTONIC INCREASING")
3026     }
3027
3028     available.properties = true;
3029 }
3030
3031 ////////////////////////////////////////////////////////////////////////////////
3032 //@}
3033 ////////////////////////////////////////////////////////////////////////////////
3034
3035 /**
3036  * \brief Return the interval attribute representing the formula strings
3037  */
3038 IntervalAttribute<QString> ColumnPrivate::formulaAttribute() const {
3039     return m_formulas;
3040 }
3041
3042 /**
3043  * \brief Replace the interval attribute for the formula strings
3044  */
3045 void ColumnPrivate::replaceFormulas(const IntervalAttribute<QString>& formulas) {
3046     m_formulas = formulas;
3047 }
3048
3049 void ColumnPrivate::calculateStatistics() {
3050     PERFTRACE(QStringLiteral("calculate column statistics"));
3051     statistics = AbstractColumn::ColumnStatistics();
3052
3053     if (m_owner->columnMode() == AbstractColumn::ColumnMode::Text) {
3054         calculateTextStatistics();
3055         return;
3056     }
3057
3058     if (!m_owner->isNumeric()) {
3059         calculateDateTimeStatistics();
3060         return;
3061     }
3062
3063     // ######  location measures  #######
3064     int rowValuesSize = rowCount();
3065     double columnSum = 0.0;
3066     double columnProduct = 1.0;
3067     double columnSumNeg = 0.0;
3068     double columnSumSquare = 0.0;
3069     statistics.minimum = INFINITY;
3070     statistics.maximum = -INFINITY;
3071     std::unordered_map<double, int> frequencyOfValues;
3072     QVector<double> rowData;
3073     rowData.reserve(rowValuesSize);
3074
3075     for (int row = 0; row < rowValuesSize; ++row) {
3076         double val = valueAt(row);
3077         if (std::isnan(val) || m_owner->isMasked(row))
3078             continue;
3079
3080         if (val < statistics.minimum)
3081             statistics.minimum = val;
3082         if (val > statistics.maximum)
3083             statistics.maximum = val;
3084         columnSum += val;
3085         columnSumNeg += (1.0 / val); // will be Inf when val == 0
3086         columnSumSquare += val * val;
3087         columnProduct *= val;
3088         if (frequencyOfValues.find(val) != frequencyOfValues.end())
3089             frequencyOfValues.operator[](val)++;
3090         else
3091             frequencyOfValues.insert(std::make_pair(val, 1));
3092         rowData.push_back(val);
3093     }
3094
3095     const size_t notNanCount = rowData.size();
3096
3097     if (notNanCount == 0) {
3098         available.statistics = true;
3099         available.min = true;
3100         available.max = true;
3101         return;
3102     }
3103
3104     if (rowData.size() < rowValuesSize)
3105         rowData.squeeze();
3106
3107     statistics.size = notNanCount;
3108     statistics.arithmeticMean = columnSum / notNanCount;
3109
3110     // geometric mean
3111     if (statistics.minimum <= -100.) // invalid
3112         statistics.geometricMean = NAN;
3113     else if (statistics.minimum < 0) { // interpret as percentage (/100) and add 1
3114         columnProduct = 1.; // recalculate
3115         for (auto val : rowData)
3116             columnProduct *= val / 100. + 1.;
3117         // n-th root and convert back to percentage changes
3118         statistics.geometricMean = 100. * (std::pow(columnProduct, 1.0 / notNanCount) - 1.);
3119     } else if (statistics.minimum == 0) { // replace zero values with 1
3120         columnProduct = 1.; // recalculate
3121         for (auto val : rowData)
3122             columnProduct *= (val == 0.) ? 1. : val;
3123         statistics.geometricMean = std::pow(columnProduct, 1.0 / notNanCount);
3124     } else
3125         statistics.geometricMean = std::pow(columnProduct, 1.0 / notNanCount);
3126
3127     statistics.harmonicMean = notNanCount / columnSumNeg;
3128     statistics.contraharmonicMean = columnSumSquare / columnSum;
3129
3130     // calculate the mode, the most frequent value in the data set
3131     int maxFreq = 0;
3132     double mode = NAN;
3133     for (const auto& it : frequencyOfValues) {
3134         if (it.second > maxFreq) {
3135             maxFreq = it.second;
3136             mode = it.first;
3137         }
3138     }
3139     // check how many times the max frequency occurs in the data set.
3140     // if more than once, we have a multi-modal distribution and don't show any mode
3141     int maxFreqOccurance = 0;
3142     for (const auto& it : frequencyOfValues) {
3143         if (it.second == maxFreq)
3144             ++maxFreqOccurance;
3145
3146         if (maxFreqOccurance > 1) {
3147             mode = NAN;
3148             break;
3149         }
3150     }
3151     statistics.mode = mode;
3152
3153     // sort the data to calculate the percentiles
3154     std::sort(rowData.begin(), rowData.end());
3155     statistics.firstQuartile = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.25);
3156     statistics.median = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.50);
3157     statistics.thirdQuartile = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.75);
3158     statistics.percentile_1 = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.01);
3159     statistics.percentile_5 = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.05);
3160     statistics.percentile_10 = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.1);
3161     statistics.percentile_90 = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.9);
3162     statistics.percentile_95 = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.95);
3163     statistics.percentile_99 = gsl_stats_quantile_from_sorted_data(rowData.constData(), 1, notNanCount, 0.99);
3164     statistics.iqr = statistics.thirdQuartile - statistics.firstQuartile;
3165     statistics.trimean = (statistics.firstQuartile + 2. * statistics.median + statistics.thirdQuartile) / 4.;
3166
3167     // ######  dispersion and shape measures  #######
3168     statistics.variance = 0.;
3169     statistics.meanDeviation = 0.;
3170     statistics.meanDeviationAroundMedian = 0.;
3171     double centralMoment_r3 = 0.;
3172     double centralMoment_r4 = 0.;
3173     QVector<double> absoluteMedianList;
3174     absoluteMedianList.reserve(notNanCount);
3175     absoluteMedianList.resize(notNanCount);
3176
3177     for (size_t row = 0; row < notNanCount; ++row) {
3178         double val = rowData.value(row);
3179         statistics.variance += gsl_pow_2(val - statistics.arithmeticMean);
3180         statistics.meanDeviation += std::abs(val - statistics.arithmeticMean);
3181
3182         absoluteMedianList[row] = std::abs(val - statistics.median);
3183         statistics.meanDeviationAroundMedian += absoluteMedianList[row];
3184
3185         centralMoment_r3 += gsl_pow_3(val - statistics.arithmeticMean);
3186         centralMoment_r4 += gsl_pow_4(val - statistics.arithmeticMean);
3187     }
3188
3189     double centralMoment_r2 = statistics.variance / notNanCount;
3190
3191     // normalize
3192     statistics.variance = (notNanCount != 1) ? statistics.variance / (notNanCount - 1) : NAN;
3193     statistics.meanDeviationAroundMedian = statistics.meanDeviationAroundMedian / notNanCount;
3194     statistics.meanDeviation = statistics.meanDeviation / notNanCount;
3195
3196     // standard deviation
3197     statistics.standardDeviation = std::sqrt(statistics.variance);
3198
3199     //"median absolute deviation" - the median of the absolute deviations from the data's median.
3200     std::sort(absoluteMedianList.begin(), absoluteMedianList.end());
3201     statistics.medianDeviation = gsl_stats_quantile_from_sorted_data(absoluteMedianList.data(), 1, notNanCount, 0.50);
3202
3203     // skewness and kurtosis
3204     centralMoment_r3 /= notNanCount;
3205     centralMoment_r4 /= notNanCount;
3206     statistics.skewness = centralMoment_r3 / gsl_pow_3(std::sqrt(centralMoment_r2));
3207     statistics.kurtosis = centralMoment_r4 / gsl_pow_2(centralMoment_r2);
3208
3209     // entropy
3210     double entropy = 0.;
3211     for (const auto& v : frequencyOfValues) {
3212         const double frequencyNorm = static_cast<double>(v.second) / notNanCount;
3213         entropy += (frequencyNorm * std::log2(frequencyNorm));
3214     }
3215
3216     statistics.entropy = -entropy;
3217
3218     available.statistics = true;
3219     available.min = true;
3220     available.max = true;
3221 }
3222
3223 void ColumnPrivate::calculateTextStatistics() {
3224     if (!available.dictionary)
3225         initDictionary();
3226
3227     int valid = 0;
3228     for (int row = 0; row < rowCount(); ++row) {
3229         if (m_owner->isMasked(row))
3230             continue;
3231
3232         ++valid;
3233     }
3234
3235     statistics.size = valid;
3236     statistics.unique = m_dictionary.count();
3237     available.statistics = true;
3238 }
3239
3240 void ColumnPrivate::calculateDateTimeStatistics() {
3241     statistics.minimum = INFINITY;
3242     statistics.maximum = -INFINITY;
3243
3244     int valid = 0;
3245     for (int row = 0; row < rowCount(); ++row) {
3246         if (m_owner->isMasked(row))
3247             continue;
3248
3249         const auto& value = dateTimeAt(row);
3250         if (!value.isValid())
3251             continue;
3252
3253         quint64 val = value.toMSecsSinceEpoch();
3254         if (val < statistics.minimum)
3255             statistics.minimum = val;
3256         if (val > statistics.maximum)
3257             statistics.maximum = val;
3258
3259         ++valid;
3260     }
3261
3262     statistics.size = valid;
3263     available.statistics = true;
3264 }