File indexing completed on 2025-02-02 04:57:01
0001 /*************************************************************************** 0002 * SPDX-FileCopyrightText: 2022 S. MANKOWSKI stephane@mankowski.fr 0003 * SPDX-FileCopyrightText: 2022 G. DE BURE support@mankowski.fr 0004 * SPDX-License-Identifier: GPL-3.0-or-later 0005 ***************************************************************************/ 0006 /** @file 0007 * This file is Skrooge plugin for KMY import / export. 0008 * 0009 * @author Stephane MANKOWSKI / Guillaume DE BURE 0010 */ 0011 #include "skgimportpluginmny.h" 0012 0013 #include <klocalizedstring.h> 0014 #include <kpluginfactory.h> 0015 0016 #include <qdir.h> 0017 #include <qjsondocument.h> 0018 #include <qprocess.h> 0019 #include <qstandardpaths.h> 0020 #include <quuid.h> 0021 0022 #include "skgbankincludes.h" 0023 #include "skgimportexportmanager.h" 0024 #include "skgobjectbase.h" 0025 #include "skgpayeeobject.h" 0026 #include "skgservices.h" 0027 #include "skgtraces.h" 0028 0029 QMap<QString, SKGUnitObject> SKGImportPluginMny::m_mapIdSecurity; 0030 QMap<QString, SKGAccountObject> SKGImportPluginMny::m_mapIdAccount; 0031 QMap<QString, SKGCategoryObject> SKGImportPluginMny::m_mapIdCategory; 0032 QMap<QString, SKGPayeeObject> SKGImportPluginMny::m_mapIdPayee; 0033 0034 /** 0035 * This plugin factory. 0036 */ 0037 K_PLUGIN_CLASS_WITH_JSON(SKGImportPluginMny, "metadata.json") 0038 0039 SKGImportPluginMny::SKGImportPluginMny(QObject* iImporter, const QVariantList& iArg) 0040 : SKGImportPlugin(iImporter) 0041 { 0042 SKGTRACEINFUNC(10) 0043 Q_UNUSED(iArg) 0044 0045 m_importParameters[QStringLiteral("password")] = QString(); 0046 m_importParameters[QStringLiteral("install_sunriise")] = 'N'; 0047 } 0048 0049 SKGImportPluginMny::~SKGImportPluginMny() 0050 = default; 0051 0052 bool SKGImportPluginMny::removeDir(const QString& dirName) 0053 { 0054 bool result = false; 0055 QDir dir(dirName); 0056 0057 if (dir.exists(dirName)) { 0058 const auto list = dir.entryInfoList(QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::AllDirs | QDir::Files, QDir::DirsFirst); 0059 for (const auto& info : list) { 0060 if (info.isDir()) { 0061 result = SKGImportPluginMny::removeDir(info.absoluteFilePath()); 0062 } else { 0063 result = QFile::remove(info.absoluteFilePath()); 0064 } 0065 0066 if (!result) { 0067 return result; 0068 } 0069 } 0070 result = dir.rmdir(dirName); 0071 } 0072 return result; 0073 } 0074 0075 bool SKGImportPluginMny::isImportPossible() 0076 { 0077 SKGTRACEINFUNC(10) 0078 if (m_importer->getDocument() == nullptr) { 0079 return true; 0080 } 0081 QString extension = m_importer->getFileNameExtension(); 0082 return (extension == QStringLiteral("MNY")); 0083 } 0084 0085 SKGError SKGImportPluginMny::readJsonFile(const QString& iFileName, QVariant& oVariant) 0086 { 0087 SKGError err; 0088 SKGTRACEINFUNCRC(2, err) 0089 QFile file(iFileName); 0090 if (Q_UNLIKELY(!file.open(QIODevice::ReadOnly))) { 0091 err.setReturnCode(ERR_INVALIDARG).setMessage(i18nc("Error message", "Open file '%1' failed", iFileName)); 0092 } else { 0093 QByteArray json = file.readAll(); 0094 file.close(); 0095 QJsonParseError ok{}; 0096 oVariant = QJsonDocument::fromJson(json, &ok).toVariant(); 0097 if (ok.error != QJsonParseError::NoError || json.isEmpty()) { 0098 err.setReturnCode(ERR_FAIL).setMessage(ok.errorString()).addError(ERR_FAIL, i18nc("Error message", "Error during parsing of '%1'", file.fileName())); 0099 } 0100 } 0101 return err; 0102 } 0103 0104 SKGError SKGImportPluginMny::importFile() 0105 { 0106 if (m_importer->getDocument() == nullptr) { 0107 return SKGError(ERR_ABORT, i18nc("Error message", "Invalid parameters")); 0108 } 0109 0110 SKGError err; 0111 SKGTRACEINFUNCRC(2, err) 0112 0113 // Check read access file 0114 if (!QFileInfo(m_importer->getLocalFileName()).isReadable()) { 0115 err.setReturnCode(ERR_READACCESS).setMessage(i18nc("Error message", "The file %1 does not have read access rights", m_importer->getLocalFileName())); 0116 return err; 0117 } 0118 0119 // Check install 0120 if (QStandardPaths::findExecutable(QStringLiteral("java")).isEmpty()) { 0121 err.setReturnCode(ERR_ABORT).setMessage(i18nc("Error message", "The java application is not installed. You must manually install it.")); 0122 return err; 0123 } 0124 0125 QString sunriise_jar = QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation) % "/sunriise.jar"; 0126 if (!QFile(sunriise_jar).exists()) { 0127 if (m_importParameters.value(QStringLiteral("install_sunriise")) == QStringLiteral("Y")) { 0128 QDir().mkpath(QFileInfo(sunriise_jar).dir().path()); 0129 err = SKGServices::upload(QUrl(QStringLiteral("https://skrooge.org/files/sunriise.jar")), QUrl::fromLocalFile(sunriise_jar)); 0130 IFKO(err) return err; 0131 } else { 0132 err.setReturnCode(ERR_INSTALL).setMessage(i18nc("Error message", "The sunriise application is needed for Microsoft Money import but is not installed in '%1'", sunriise_jar)); 0133 err.setProperty(QStringLiteral("sunriise")); 0134 return err; 0135 } 0136 } 0137 0138 // Initialisation 0139 m_mapIdSecurity.clear(); 0140 m_mapIdAccount.clear(); 0141 m_mapIdCategory.clear(); 0142 m_mapIdPayee.clear(); 0143 0144 // Execute sunriise 0145 QString uniqueId = QUuid::createUuid().toString(); 0146 QString temporaryPath = QDir::tempPath() % "/" % uniqueId; 0147 removeDir(temporaryPath); 0148 QDir::temp().mkdir(uniqueId); 0149 QDir temporaryDir(temporaryPath); 0150 0151 QString cmd = "java -cp \"" % sunriise_jar % "\" com/le/sunriise/export/ExportToJSON \"" % m_importer->getLocalFileName() % "\" \"" % m_importParameters.value(QStringLiteral("password")) % "\" " % temporaryPath; 0152 SKGTRACEL(10) << "Execution of :" << cmd << SKGENDL; 0153 QProcess p; 0154 p.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd); 0155 if (p.waitForFinished(1000 * 60 * 5) && p.exitCode() == 0) { 0156 // Import 0157 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import %1 file", "MNY"), 6); 0158 // Step 1-Import categories 0159 IFOK(err) { 0160 /*[ { 0161 "id" : 137, 0162 "parentId" : 130, 0163 "name" : "Other Income", 0164 "classificationId" : 0, 0165 "level" : 1 0166 },*/ 0167 QVariant var; 0168 err = readJsonFile(temporaryPath % "/categories.json", var); 0169 QVariantList list = var.toList(); 0170 IFOK(err) { 0171 SKGTRACEINRC(10, "SKGImportPluginMny::importFile-categories", err) 0172 int nb = list.count(); 0173 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import categories"), nb); 0174 // Create categories 0175 int index = 0; 0176 for (int t = 0; !err && t < 20 && index < nb; ++t) { 0177 for (int i = 0; !err && i < nb && index < nb; ++i) { 0178 QVariantMap category = list.at(i).toMap(); 0179 0180 QString parentId = category[QStringLiteral("parentId")].toString(); 0181 QString id = category[QStringLiteral("id")].toString(); 0182 QString name = category[QStringLiteral("name")].toString(); 0183 int level = category[QStringLiteral("level")].toInt(); 0184 0185 if (level == t) { 0186 SKGCategoryObject catObject(m_importer->getDocument()); 0187 err = catObject.setName(name); 0188 if (!err && !parentId.isEmpty()) { 0189 err = catObject.setParentCategory(m_mapIdCategory[parentId]); 0190 } 0191 IFOKDO(err, catObject.save()) 0192 0193 m_mapIdCategory[id] = catObject; 0194 0195 ++index; 0196 IFOKDO(err, m_importer->getDocument()->stepForward(index)) 0197 } 0198 } 0199 } 0200 0201 SKGENDTRANSACTION(m_importer->getDocument(), err) 0202 } 0203 IFOKDO(err, m_importer->getDocument()->stepForward(1)) 0204 } 0205 0206 // Step 2-Import payees 0207 IFOK(err) { 0208 QVariant var; 0209 err = readJsonFile(temporaryPath % "/payees.json", var); 0210 QVariantList list = var.toList(); 0211 IFOK(err) { 0212 SKGTRACEINRC(10, "SKGImportPluginMny::importFile-payees", err) 0213 int nb = list.count(); 0214 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import payees"), nb); 0215 // Create categories 0216 for (int i = 0; !err && i < nb; ++i) { 0217 QVariantMap payee = list.at(i).toMap(); 0218 0219 QString id = payee[QStringLiteral("id")].toString(); 0220 QString name = payee[QStringLiteral("name")].toString(); 0221 0222 // Is already created? 0223 SKGPayeeObject payeeObject(m_importer->getDocument()); 0224 err = payeeObject.setName(name); 0225 IFOKDO(err, payeeObject.save()) 0226 0227 m_mapIdPayee[id] = payeeObject; 0228 0229 IFOKDO(err, m_importer->getDocument()->stepForward(i + 1)) 0230 } 0231 0232 SKGENDTRANSACTION(m_importer->getDocument(), err) 0233 } 0234 IFOKDO(err, m_importer->getDocument()->stepForward(2)) 0235 } 0236 0237 // Step 3-Import securities 0238 IFOK(err) { 0239 QVariant var; 0240 err = readJsonFile(temporaryPath % "/securities.json", var); 0241 QVariantList list = var.toList(); 0242 IFOK(err) { 0243 SKGTRACEINRC(10, "SKGImportPluginMny::importFile-securities", err) 0244 int nb = list.count(); 0245 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import units"), nb); 0246 // Create categories 0247 for (int i = 0; !err && i < nb; ++i) { 0248 QVariantMap unit = list.at(i).toMap(); 0249 0250 QString id = unit[QStringLiteral("id")].toString(); 0251 QString name = unit[QStringLiteral("name")].toString(); 0252 QString symbol = unit[QStringLiteral("symbol")].toString(); 0253 if (symbol.isEmpty()) { 0254 symbol = name; 0255 } 0256 0257 // Is already created? 0258 SKGUnitObject unitobject(m_importer->getDocument()); 0259 err = unitobject.setName(name); 0260 IFOKDO(err, unitobject.setSymbol(symbol)) 0261 IFOKDO(err, unitobject.setType(SKGUnitObject::SHARE)) 0262 // The unit is not saved because it will be saved when used 0263 0264 m_mapIdSecurity[id] = unitobject; 0265 0266 IFOKDO(err, m_importer->getDocument()->stepForward(i + 1)) 0267 } 0268 0269 SKGENDTRANSACTION(m_importer->getDocument(), err) 0270 } 0271 IFOKDO(err, m_importer->getDocument()->stepForward(3)) 0272 } 0273 0274 // Step 4-Import accounts 0275 IFOK(err) { 0276 /* 0277 { 0278 "id" : 1, 0279 "name" : "Investments to Watch", 0280 "relatedToAccountId" : null, 0281 "relatedToAccount" : null, 0282 "type" : 5, 0283 "accountType" : "INVESTMENT", 0284 "closed" : false, 0285 "startingBalance" : 0.0000, 0286 "currentBalance" : 0, 0287 "currencyId" : 18, 0288 "currencyCode" : "GBP", 0289 "retirement" : false, 0290 "investmentSubType" : -1, 0291 "securityHoldings" : [ ], 0292 "amountLimit" : null, 0293 "creditCard" : false, 0294 "401k403b" : false 0295 }*/ 0296 // List directories 0297 QStringList list = temporaryDir.entryList(QStringList() << QStringLiteral("*.d"), QDir::Dirs); 0298 SKGTRACEINFUNCRC(10, err) 0299 int nb = list.count(); 0300 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import accounts"), nb); 0301 0302 SKGBankObject bank(m_importer->getDocument()); 0303 IFOKDO(err, bank.setName(QStringLiteral("Microsof Money"))) 0304 IFOKDO(err, bank.save()) 0305 0306 // Create accounts 0307 for (int i = 0; !err && i < nb; ++i) { 0308 const QString& accountDir = list.at(i); 0309 0310 QVariant var; 0311 err = readJsonFile(temporaryPath % "/" % accountDir % "/account.json", var); 0312 QVariantMap account = var.toMap(); 0313 IFOK(err) { 0314 // Create currency 0315 SKGUnitObject unitObj; 0316 err = SKGUnitObject::createCurrencyUnit(m_importer->getDocument(), account[QStringLiteral("currencyCode")].toString(), unitObj); 0317 0318 // Create account 0319 SKGAccountObject accountObj; 0320 IFOKDO(err, bank.addAccount(accountObj)) 0321 IFOKDO(err, accountObj.setName(account[QStringLiteral("name")].toString())) 0322 int type = account[QStringLiteral("type")].toInt(); 0323 IFOKDO(err, accountObj.setType(type == 0 ? SKGAccountObject::CURRENT : 0324 type == 1 ? SKGAccountObject::CREDITCARD : 0325 type == 3 ? SKGAccountObject::ASSETS : 0326 type == 5 ? SKGAccountObject::INVESTMENT : 0327 type == 6 ? SKGAccountObject::LOAN : 0328 SKGAccountObject::OTHER)); // TODO(Stephane MANKOWSKI) 0329 IFOKDO(err, accountObj.save()) 0330 0331 // Update initial balance 0332 IFOKDO(err, accountObj.setInitialBalance(account[QStringLiteral("startingBalance")].toDouble(), unitObj)) 0333 0334 IFOKDO(err, accountObj.setClosed(account[QStringLiteral("closed")].toBool())) 0335 IFOKDO(err, accountObj.save()) 0336 0337 m_mapIdAccount[account[QStringLiteral("id")].toString()] = accountObj; 0338 } 0339 0340 IFOKDO(err, m_importer->getDocument()->stepForward(i + 1)) 0341 } 0342 0343 SKGENDTRANSACTION(m_importer->getDocument(), err) 0344 } 0345 IFOKDO(err, m_importer->getDocument()->stepForward(4)) 0346 0347 // Step 5-Import operation 0348 IFOK(err) { 0349 // List directories 0350 QStringList list = temporaryDir.entryList(QStringList() << QStringLiteral("*.d"), QDir::Dirs); 0351 SKGTRACEINFUNCRC(10, err) 0352 int nb = list.count(); 0353 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import transactions"), nb); 0354 0355 // Create accounts 0356 int index = 0; 0357 for (int i = 0; !err && i < nb; ++i) { 0358 const QString& accountDir = list.at(i); 0359 0360 QVariant var; 0361 err = readJsonFile(temporaryPath % "/" % accountDir % "/transactions.json", var); 0362 QVariantList transactions = var.toList(); 0363 IFOK(err) { 0364 int nbo = transactions.count(); 0365 err = m_importer->getDocument()->beginTransaction("#INTERNAL#" % i18nc("Import step", "Import transactions"), nbo); 0366 for (int j = 0; !err && j < nbo; ++j) { 0367 QVariantMap operation = transactions.at(j).toMap(); 0368 if (!operation[QStringLiteral("void")].toBool()) { 0369 SKGAccountObject accountObj = m_mapIdAccount.value(operation[QStringLiteral("accountId")].toString()); 0370 QString securityId = operation[QStringLiteral("securityId")].toString(); 0371 0372 SKGUnitObject unitObj; 0373 if (securityId.isEmpty()) { 0374 IFOKDO(err, accountObj.getUnit(unitObj)) 0375 } else { 0376 unitObj = m_mapIdSecurity[securityId]; 0377 if (unitObj.getID() == 0) { 0378 err = unitObj.save(); 0379 m_mapIdSecurity[securityId] = unitObj; 0380 } 0381 } 0382 0383 SKGOperationObject operationObj; 0384 IFOKDO(err, accountObj.addOperation(operationObj, true)) 0385 IFOKDO(err, operationObj.setUnit(unitObj)) 0386 IFOKDO(err, operationObj.setDate(QDateTime::fromMSecsSinceEpoch(operation[QStringLiteral("date")].toULongLong()).date())) 0387 IFOKDO(err, operationObj.setComment(operation[QStringLiteral("memo")].toString())) 0388 IFOK(err) { 0389 // The number is something like "0 4220725" or "1string" 0390 QString number = operation[QStringLiteral("number")].toString(); 0391 if (!number.isEmpty()) { 0392 if (number.startsWith(QLatin1Char('1'))) { 0393 err = operationObj.setMode(number.right(number.count() - 1)); 0394 } else { 0395 QStringList ln = SKGServices::splitCSVLine(number, ' '); 0396 err = operationObj.setNumber(ln.at(ln.count() - 1)); 0397 } 0398 } 0399 } 0400 IFOKDO(err, operationObj.setAttribute(QStringLiteral("t_imported"), QStringLiteral("T"))) 0401 IFOKDO(err, operationObj.setImportID("MNY-" % operation[QStringLiteral("id")].toString())) 0402 QString payId = operation[QStringLiteral("payeeId")].toString(); 0403 if (!payId.isEmpty() && !err) { 0404 err = operationObj.setPayee(m_mapIdPayee[payId]); 0405 } 0406 IFOKDO(err, operationObj.setStatus(operation[QStringLiteral("reconciled")].toBool() ? SKGOperationObject::CHECKED : 0407 operation[QStringLiteral("cleared")].toBool() ? SKGOperationObject::MARKED : 0408 SKGOperationObject::NONE)); 0409 if (operation[QStringLiteral("transfer")].toBool()) { 0410 IFOKDO(err, operationObj.setComment("#MNYTRANSFER#" % operationObj.getComment())) 0411 } 0412 IFOKDO(err, operationObj.save(false)) 0413 0414 double amount = operation[QStringLiteral("amount")].toDouble(); 0415 0416 // Is it a split? 0417 QVariantList splits = operation[QStringLiteral("splits")].toList(); 0418 int nbs = splits.count(); 0419 if (nbs != 0) { 0420 // Yes 0421 for (int k = 0; !err && k < nbs; ++k) { 0422 QVariantMap split = splits[k].toMap(); 0423 QVariantMap transaction = split[QStringLiteral("transaction")].toMap(); 0424 0425 SKGSubOperationObject subOperationObj; 0426 IFOKDO(err, operationObj.addSubOperation(subOperationObj)) 0427 QString catId = transaction[QStringLiteral("categoryId")].toString(); 0428 if (!catId.isEmpty() && !err) { 0429 err = subOperationObj.setCategory(m_mapIdCategory[catId]); 0430 } 0431 double splitAmount = transaction[QStringLiteral("amount")].toDouble(); 0432 IFOKDO(err, subOperationObj.setQuantity(splitAmount)) 0433 IFOKDO(err, subOperationObj.setComment(operation[QStringLiteral("memo")].toString())) 0434 IFOKDO(err, subOperationObj.save(false, false)) 0435 amount -= splitAmount; 0436 } 0437 0438 // Is the amount equal to the sum of split? 0439 if (qAbs(amount) > 0.00001) { 0440 // Create one more sub transaction to align amounts 0441 SKGSubOperationObject subOperationObj; 0442 IFOKDO(err, operationObj.addSubOperation(subOperationObj)) 0443 IFOKDO(err, subOperationObj.setQuantity(amount)) 0444 IFOKDO(err, subOperationObj.setComment(operation[QStringLiteral("memo")].toString())) 0445 IFOKDO(err, subOperationObj.save(false, false)) 0446 0447 IFOKDO(err, m_importer->getDocument()->sendMessage(i18nc("Warning message", "The transaction '%1' has been repaired because its amount was not equal to the sum of the amounts of its splits", operationObj.getDisplayName()), SKGDocument::Warning)) 0448 } 0449 } else { 0450 // No 0451 SKGSubOperationObject subOperationObj; 0452 IFOKDO(err, operationObj.addSubOperation(subOperationObj)) 0453 QString catId = operation[QStringLiteral("categoryId")].toString(); 0454 if (!catId.isEmpty() && !err) { 0455 err = subOperationObj.setCategory(m_mapIdCategory[catId]); 0456 } 0457 IFOKDO(err, subOperationObj.setQuantity(amount)) 0458 IFOKDO(err, subOperationObj.setComment(operation[QStringLiteral("memo")].toString())) 0459 IFOKDO(err, subOperationObj.save(false, false)) 0460 } 0461 } 0462 0463 if (!err && index % 500 == 0) { 0464 err = m_importer->getDocument()->executeSqliteOrder(QStringLiteral("ANALYZE")); 0465 } 0466 ++index; 0467 0468 IFOKDO(err, m_importer->getDocument()->stepForward(j + 1)) 0469 } 0470 0471 SKGENDTRANSACTION(m_importer->getDocument(), err) 0472 } 0473 0474 IFOKDO(err, m_importer->getDocument()->stepForward(i + 1)) 0475 } 0476 0477 SKGENDTRANSACTION(m_importer->getDocument(), err) 0478 } 0479 0480 IFOKDO(err, m_importer->getDocument()->stepForward(5)) 0481 0482 // Step 6-Group operation 0483 int nbg = 0; 0484 IFOKDO(err, m_importer->findAndGroupTransfers(nbg, QStringLiteral("A.t_comment LIKE '#MNYTRANSFER#%' AND B.t_comment LIKE '#MNYTRANSFER#%'"))) 0485 IFOKDO(err, m_importer->getDocument()->executeSqliteOrder(QStringLiteral("UPDATE operation SET t_comment=SUBSTR(t_comment, 14) WHERE t_comment LIKE '#MNYTRANSFER#%'"))) 0486 0487 IFOKDO(err, m_importer->getDocument()->stepForward(6)) 0488 0489 SKGENDTRANSACTION(m_importer->getDocument(), err) 0490 } else { 0491 if (p.exitCode() == 1) { 0492 err.setReturnCode(ERR_ENCRYPTION).setMessage(i18nc("Error message", "Invalid password")); 0493 } else { 0494 err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message", "The execution of '%1' failed", cmd)).addError(ERR_FAIL, i18nc("Error message", "The extraction from the Microsoft Money document '%1' failed", m_importer->getFileName().toDisplayString())); 0495 } 0496 } 0497 0498 removeDir(temporaryPath); 0499 0500 // Clean 0501 m_mapIdSecurity.clear(); 0502 m_mapIdAccount.clear(); 0503 m_mapIdCategory.clear(); 0504 m_mapIdPayee.clear(); 0505 0506 return err; 0507 } 0508 0509 QString SKGImportPluginMny::getMimeTypeFilter() const 0510 { 0511 return "*.mny|" % i18nc("A file format", "Microsoft Money document"); 0512 } 0513 0514 #include <skgimportpluginmny.moc>