import/skrooge_import_backend/skgimportpluginbackend.cpp

0001 /***************************************************************************
0002  * SPDX-FileCopyrightText: 2022 S. MANKOWSKI stephane@mankowski.fr
0003  * SPDX-FileCopyrightText: 2022 G. DE BURE support@mankowski.fr
0004  * SPDX-License-Identifier: GPL-3.0-or-later
0005  ***************************************************************************/
0006 /** @file
0007  * This file is Skrooge plugin for BACKEND import / export.
0008  *
0009  * @author Stephane MANKOWSKI / Guillaume DE BURE
0010  */
0011 #include "skgimportpluginbackend.h"
0012
0013 #include <qapplication.h>
0014 #include <qdir.h>
0015 #include <qdiriterator.h>
0016 #include <qfile.h>
0017 #include <qfileinfo.h>
0018 #include <qprocess.h>
0019 #include <qstandardpaths.h>
0020 #include <qtconcurrentmap.h>
0021 #include <qregularexpression.h>
0022
0023 #include <kaboutdata.h>
0024 #include <klocalizedstring.h>
0025 #include <kpluginfactory.h>
0026 #include <qvariant.h>
0027
0028 #include "skgbankincludes.h"
0029 #include "skgimportexportmanager.h"
0030 #include "skgservices.h"
0031 #include "skgtraces.h"
0032
0033 /**
0034  * This plugin factory.
0035  */
0036 K_PLUGIN_CLASS_WITH_JSON(SKGImportPluginBackend, "metadata.json")
0037
0038 SKGImportPluginBackend::SKGImportPluginBackend(QObject* iImporter, const QVariantList& iArg)
0039     : SKGImportPlugin(iImporter)
0040 {
0041     SKGTRACEINFUNC(10)
0042     Q_UNUSED(iArg)
0043
0044     m_listBackends = KServiceTypeTrader::self()->query(QStringLiteral("Skrooge/Import/Backend"));
0045     SKGTRACEL(10) << m_listBackends.count() << " plugins found" << SKGENDL;
0046 }
0047
0048 SKGImportPluginBackend::~SKGImportPluginBackend()
0049     = default;
0050
0051 QExplicitlySharedDataPointer<KService> SKGImportPluginBackend::getService() const
0052 {
0053     for (const auto& service : m_listBackends) {
0054 #ifdef SKG_KF_5102
0055         if (service->property(QStringLiteral("X-Krunner-ID"), QMetaType::QString).toString().toUpper() == m_importer->getFileNameExtension()) {
0056             return service;
0057         }
0058 #else
0059         if (service->property(QStringLiteral("X-Krunner-ID"), QVariant::String).toString().toUpper() == m_importer->getFileNameExtension()) {
0060             return service;
0061         }
0062 #endif
0063     }
0064     return QExplicitlySharedDataPointer<KService>(nullptr);
0065 }
0066
0067 QString SKGImportPluginBackend::getParameter(const QString& iAttribute)
0068 {
0069     auto service = getService();
0070 #ifdef SKG_KF_5102
0071     auto output = service->property(iAttribute, QMetaType::QString).toString();
0072 #else
0073     auto output = service->property(iAttribute, QVariant::String).toString();
0074 #endif
0075     QMap<QString, QString> parameters = this->getImportParameters();
0076
0077     for (int i = 1; i <= 10; ++i) {
0078         QString param = "parameter" + SKGServices::intToString(i);
0079         if (output.contains(QStringLiteral("%") % param)) {
0080             output = output.replace(QStringLiteral("%") % param, parameters.value(param));
0081         }
0082     }
0083
0084     return output;
0085 }
0086
0087 bool SKGImportPluginBackend::isImportPossible()
0088 {
0089     SKGTRACEINFUNC(10)
0090     return (m_importer->getDocument() == nullptr ? true : getService().data() != nullptr);
0091 }
0092
0093 struct download {
0094     download(int iNbToDownload, QString  iDate, QString  iCmd, QString  iPwd, QString  iPath)
0095         : m_nbToDownload(iNbToDownload), m_date(std::move(iDate)), m_cmd(std::move(iCmd)), m_pwd(std::move(iPwd)), m_path(std::move(iPath))
0096     {
0097     }
0098
0099     using result_type = QString;
0100
0101     QString operator()(const QString& iAccountId)
0102     {
0103         QString file = m_path % "/" % iAccountId % ".csv";
0104         // Build cmd
0105         QString cmd = m_cmd;
0106         cmd = cmd.replace(QStringLiteral("%2"), SKGServices::intToString(m_nbToDownload)).replace(QStringLiteral("%1"), iAccountId).replace(QStringLiteral("%3"), m_pwd).replace(QStringLiteral("%4"), m_date);
0107
0108         // Execute
0109         QProcess p;
0110         cmd = SKGServices::getFullPathCommandLine(cmd);
0111         SKGTRACEL(10) << "Execute: " << cmd << SKGENDL;
0112         p.setStandardOutputFile(file);
0113
0114         int retry = 0;
0115         do {
0116             p.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd);
0117             if (p.waitForFinished(1000 * 60 * 2)) {
0118                 if (p.exitCode() == 0) {
0119                     return iAccountId;
0120                 }
0121                 SKGTRACE << i18nc("A warning message", "WARNING: The command %1 failed with code %2 (Retry %3)", cmd, p.exitCode(), retry + 1) << SKGENDL;
0122
0123             } else {
0124                 SKGTRACE << i18nc("A warning message", "WARNING: The command %1 failed due to a time out (Retry %2)", cmd, retry + 1) << SKGENDL;
0125                 p.terminate();
0126                 p.kill();
0127             }
0128             ++retry;
0129         } while (retry < 6);
0130
0131         QString errorMsg = i18nc("Error message",  "The following command line failed with code %2:\n'%1'", cmd, p.exitCode());
0132         SKGTRACE << errorMsg << SKGENDL;
0133
0134         return QStringLiteral("ERROR:") + errorMsg;
0135     }
0136
0137     int m_nbToDownload;
0138     QString m_date;
0139     QString m_cmd;
0140     QString m_pwd;
0141     QString m_path;
0142 };
0143
0144 SKGError SKGImportPluginBackend::importFile()
0145 {
0146     if (m_importer->getDocument() == nullptr) {
0147         return SKGError(ERR_ABORT, i18nc("Error message", "Invalid parameters"));
0148     }
0149     SKGError err;
0150     SKGTRACEINFUNCRC(2, err)
0151
0152     SKGBEGINPROGRESSTRANSACTION(*m_importer->getDocument(), i18nc("Noun, name of the user action", "Import with %1", "Backend"), err, 3)
0153     QString bankendName = m_importer->getFileNameExtension().toLower();
0154
0155     // Get parameters
0156     QMap<QString, QString> parameters = this->getImportParameters();
0157     QString pwd = parameters[QStringLiteral("password")];
0158
0159     // Get list of accounts
0160     QStringList backendAccounts;
0161     QMap<QString, QString> backendAccountsBalance;
0162     QMap<QString, QString> backendAccountsName;
0163     QString csvfile = m_tempDir.path() % "/skrooge_backend.csv";
0164     QString cmd = getParameter(QStringLiteral("X-SKROOGE-getaccounts")).replace(QStringLiteral("%3"), pwd);
0165     QProcess p;
0166     cmd = SKGServices::getFullPathCommandLine(cmd);
0167     SKGTRACEL(10) << "Execute: " << cmd << SKGENDL;
0168     p.setStandardOutputFile(csvfile);
0169     p.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd);
0170     if (!p.waitForFinished(1000 * 60 * 2) || p.exitCode() != 0) {
0171         err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message",  "The following command line failed with code %2:\n'%1'", cmd, p.exitCode()));
0172     } else {
0173         QFile file(csvfile);
0174         if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
0175             err.setReturnCode(ERR_INVALIDARG).setMessage(i18nc("Error message",  "Open file '%1' failed", csvfile));
0176         } else {
0177             QRegularExpression reggetaccounts(getParameter(QStringLiteral("X-SKROOGE-getaccountid")));
0178             QRegularExpression reggetaccountbalance(getParameter(QStringLiteral("X-SKROOGE-getaccountbalance")));
0179             QRegularExpression reggetaccountname(getParameter(QStringLiteral("X-SKROOGE-getaccountname")));
0180
0181             QTextStream stream(&file);
0182             stream.readLine();  // To avoid header
0183             QStringList backendAccountsUniqueId;
0184             while (!stream.atEnd()) {
0185                 // Read line
0186                 QString line = stream.readLine().trimmed();
0187                 SKGTRACEL(10) << "Read line: " << line << SKGENDL;
0188
0189                 // Get account id
0190                 auto match = reggetaccounts.match(line);
0191                 if (match.hasMatch()) {
0192                     QString accountid = match.captured(1);
0193                     QString uniqueid = SKGServices::splitCSVLine(accountid, QLatin1Char('@')).at(0);
0194
0195                     if (!backendAccounts.contains(accountid) && !backendAccountsUniqueId.contains(uniqueid)) {
0196                         backendAccounts.push_back(accountid);
0197                         backendAccountsUniqueId.push_back(uniqueid);
0198
0199                         // Get account balance
0200                         match = reggetaccountbalance.match(line);
0201                         if (match.hasMatch()) {
0202                             backendAccountsBalance[accountid] = match.captured(1);
0203                         } else {
0204                             backendAccountsBalance[accountid] = '0';
0205                         }
0206
0207                         // Get account name
0208                         match = reggetaccountname.match(line);
0209                         if (match.hasMatch()) {
0210                             backendAccountsName[accountid] = match.captured(1);
0211                         } else {
0212                             backendAccountsName[accountid] = QLatin1String("");
0213                         }
0214                     }
0215                 } else {
0216                     // This is an error
0217                     err.setReturnCode(ERR_FAIL).setMessage(line).addError(ERR_FAIL, i18nc("Error message",  "Impossible to find the account id with the regular expression '%1' in line '%2'", getParameter(QStringLiteral("X-SKROOGE-getaccountid")), line));
0218                     break;
0219                 }
0220             }
0221
0222             // close file
0223             file.close();
0224             file.remove();
0225         }
0226     }
0227
0228     // Download transactions
0229     IFOKDO(err, m_importer->getDocument()->stepForward(1, i18nc("Progress message", "Download transactions")))
0230     IFOK(err) {
0231         if (backendAccounts.isEmpty()) {
0232             err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message",  "Your backend '%1' seems to be not well configure because no account has been found.", bankendName));
0233         } else {
0234             // Compute the begin date for download
0235             QDate lastDownload = SKGServices::stringToTime(m_importer->getDocument()->getParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_DATE")).date();
0236             QString lastList = m_importer->getDocument()->getParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_LIST");
0237             QString currentList = backendAccounts.join(QStringLiteral(";"));
0238
0239             int nbToDownload = 0;
0240             QString fromDate;
0241             if (currentList != lastList || !lastDownload.isValid()) {
0242                 nbToDownload = 99999;
0243                 fromDate = QStringLiteral("2000-01-01");
0244             } else {
0245                 nbToDownload = qMax(lastDownload.daysTo(QDate::currentDate()) * 10, qint64(20));
0246                 fromDate = SKGServices::dateToSqlString(lastDownload.addDays(-4));
0247             }
0248
0249             // Download
0250             QStringList listDownloadedId;
0251             QString bulk = getParameter(QStringLiteral("X-SKROOGE-getbulk"));
0252             QString cmddownload;
0253             if (!bulk.isEmpty()) {
0254                 // mode bulk
0255                 SKGTRACEL(10) << "Mode getbulk" << SKGENDL;
0256                 QProcess pbulk;
0257                 QString cmd = bulk.replace(QStringLiteral("%1"), m_tempDir.path());
0258                 cmd = SKGServices::getFullPathCommandLine(cmd);
0259                 cmddownload = cmd;
0260                 SKGTRACEL(10) << "Execute: " << cmd << SKGENDL;
0261                 pbulk.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd);
0262                 if (!pbulk.waitForFinished(1000 * 60 * 2) || pbulk.exitCode() != 0) {
0263                     err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message",  "The following command line failed with code %2:\n'%1'", cmd, pbulk.exitCode()));
0264                 } else {
0265                     SKGTRACEL(10) << "Searching csv files " << SKGENDL;
0266                     QDirIterator it(m_tempDir.path(), QStringList() << QStringLiteral("*.csv"));
0267                     while (it.hasNext()) {
0268                         auto id = QFileInfo(it.next()).baseName();
0269                         listDownloadedId.push_back(id);
0270
0271                         SKGTRACEL(10) << "Find id: " << id << SKGENDL;
0272                     }
0273                 }
0274             } else {
0275                 // mode getoperations
0276                 SKGTRACEL(10) << "Mode getoperations" << SKGENDL;
0277                 cmddownload = getParameter(QStringLiteral("X-SKROOGE-getoperations"));
0278                 QFuture<QString> f = QtConcurrent::mapped(backendAccounts, download(nbToDownload, fromDate, cmddownload, pwd, m_tempDir.path()));
0279                 f.waitForFinished();
0280                 listDownloadedId = f.results();
0281             }
0282             listDownloadedId.removeAll(QLatin1String(""));
0283             // Build list of errors
0284             QStringList errors;
0285             int nb = listDownloadedId.count();
0286             errors.reserve(nb);
0287             for (int i = nb - 1; i >= 0; --i) {
0288                 auto item = listDownloadedId.value(i);
0289                 if (item.startsWith(QLatin1String("ERROR:"))) {
0290                     listDownloadedId.removeAt(i);
0291                     errors.push_back(item.right(item.length() - 6));
0292                 }
0293             }
0294
0295             // Check
0296             IFOK(err) {
0297                 bool checkOK = true;
0298                 int nb = listDownloadedId.count();
0299                 if (errors.count() != 0) {
0300                     // Some accounts have not been downloaded
0301                     if (nb == 0) {
0302                         err = SKGError(ERR_FAIL, i18nc("Error message", "No accounts downloaded with the following command:\n%1\nCheck your backend installation.", cmddownload));
0303                     } else {
0304                         // Warning
0305                         m_importer->getDocument()->sendMessage(i18nc("Warning message", "Some accounts have not been downloaded. %1", errors.join(QStringLiteral(". "))), SKGDocument::Warning);
0306                     }
0307                     SKGTRACEL(10) << errors.count() << " accounts not imported => checkOK=false" << SKGENDL;
0308                     checkOK = false;
0309                 }
0310
0311                 // import
0312                 IFOKDO(err, m_importer->getDocument()->stepForward(2, i18nc("Progress message", "Import")))
0313                 if (!err && (nb != 0)) {
0314                     // import
0315                     SKGBEGINPROGRESSTRANSACTION(*m_importer->getDocument(), "#INTERNAL#" % i18nc("Noun, name of the user action", "Import one account with %1", "Backend"), err, nb)
0316
0317                     // Get all messages
0318                     SKGDocument::SKGMessageList messages;
0319                     IFOKDO(err, m_importer->getDocument()->getMessages(m_importer->getDocument()->getCurrentTransaction(), messages, true))
0320
0321                     auto extension = getParameter(QStringLiteral("X-SKROOGE-extension")).toLower();
0322                     if (extension.length() == 0) {
0323                         extension = "csv";
0324                     }
0325
0326                     // Import all files
0327                     for (int i = 0; !err && i < nb; ++i) {
0328                         // Rename the imported name
0329                         QString file = m_tempDir.path() % "/" % listDownloadedId.at(i) % ".csv";
0330                         if (!listDownloadedId.at(i).contains(QStringLiteral("-")) && !backendAccountsName[listDownloadedId.at(i)].isEmpty()) {
0331                             QString newFileName = m_tempDir.path() % "/" % backendAccountsName[listDownloadedId.at(i)] % '-' % listDownloadedId.at(i) % "." % extension;
0332                             if (QFile::rename(file, newFileName)) {
0333                                 file = newFileName;
0334                             }
0335                         }
0336
0337                         // Import
0338                         SKGImportExportManager imp1(m_importer->getDocument(), QUrl::fromLocalFile(file));
0339                         imp1.setAutomaticValidation(m_importer->automaticValidation());
0340                         imp1.setAutomaticApplyRules(m_importer->automaticApplyRules());
0341                         // This option is not used with backend import
0342                         imp1.setSinceLastImportDate(false);
0343                         imp1.setCodec(m_importer->getCodec());
0344
0345                         if (extension == "csv") {
0346                             QMap<QString, QString> newParameters = imp1.getImportParameters();
0347                             newParameters[QStringLiteral("automatic_search_header")] = 'N';
0348                             newParameters[QStringLiteral("header_position")] = '1';
0349                             newParameters[QStringLiteral("automatic_search_columns")] = 'N';
0350                             newParameters[QStringLiteral("columns_positions")] = getParameter(QStringLiteral("X-SKROOGE-csvcolumns"));
0351                             newParameters[QStringLiteral("mode_csv_unit")] = 'N';
0352                             newParameters[QStringLiteral("mode_csv_rule")] = 'N';
0353                             newParameters[QStringLiteral("balance")] = backendAccountsBalance[listDownloadedId.at(i)];
0354                             newParameters[QStringLiteral("donotfinalize")] = 'Y';
0355                             imp1.setImportParameters(newParameters);
0356                         }
0357                         IFOKDO(err, imp1.importFile())
0358
0359                         if (!backendAccountsBalance[listDownloadedId.at(i)].isEmpty()) {
0360                             SKGAccountObject act;
0361                             IFOKDO(err, imp1.getDefaultAccount(act))
0362                             m_importer->addAccountToCheck(act, SKGServices::stringToDouble(backendAccountsBalance[listDownloadedId.at(i)]));
0363                         }
0364                         IFOKDO(err, m_importer->getDocument()->stepForward(i + 1))
0365                     }
0366
0367                     // Remove all temporary files
0368                     for (int i = 0; i < nb; ++i) {
0369                         QString file = m_tempDir.path() % "/" % listDownloadedId.at(i) % ".csv";
0370                         QFile::remove(file);
0371                     }
0372
0373                     // Reset message
0374                     IFOKDO(err, m_importer->getDocument()->removeMessages(m_importer->getDocument()->getCurrentTransaction()))
0375                     int nbm = messages.count();
0376                     for (int j = 0; j < nbm; ++j) {
0377                         SKGDocument::SKGMessage msg = messages.at(j);
0378                         m_importer->getDocument()->sendMessage(msg.Text, msg.Type, msg.Action);
0379                     }
0380
0381                     // Finalize import
0382                     IFOKDO(err, m_importer->finalizeImportation())
0383
0384                     // Disable std finalisation
0385                     QMap<QString, QString> parameters = m_importer->getImportParameters();
0386                     parameters[QStringLiteral("donotfinalize")] = 'Y';
0387                     m_importer->setImportParameters(parameters);
0388
0389                     // Check balances of accounts
0390                     auto accountsToCheck = m_importer->getAccountsToCheck();
0391                     int nb = accountsToCheck.count();
0392                     for (int i = 0; !err && i < nb; ++i) {
0393                         // Get the account to check
0394                         auto act = accountsToCheck[i].first;
0395                         auto targetBalance = accountsToCheck[i].second;
0396                         auto soluces = act.getPossibleReconciliations(targetBalance, false);
0397                         if (soluces.isEmpty()) {
0398                             SKGTRACEL(10) << "Account " << listDownloadedId.at(i) << " not reconciliable => checkOK=false" << SKGENDL;
0399                             checkOK = false;
0400                         }
0401                     }
0402
0403                     if (checkOK) {
0404                         // Last import is memorized only in case of 100% success
0405                         IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_DATE", SKGServices::dateToSqlString(QDateTime::currentDateTime())))
0406                         IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_LIST", currentList))
0407                     } else  {
0408                         // Remove last import for next import
0409                         IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_DATE", QLatin1String("")))
0410                         IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_LIST", QLatin1String("")))
0411                     }
0412                 }
0413                 IFOKDO(err, m_importer->getDocument()->stepForward(3))
0414             }
0415         }
0416     }
0417
0418     return err;
0419 }
0420
0421 QString SKGImportPluginBackend::getMimeTypeFilter() const
0422 {
0423     return QLatin1String("");
0424 }
0425
0426 #include <skgimportpluginbackend.moc>