1 /***************************************************************************
2 * SPDX-FileCopyrightText: 2021 S. MANKOWSKI stephane@mankowski.fr
3 * SPDX-FileCopyrightText: 2021 G. DE BURE support@mankowski.fr
4 * SPDX-License-Identifier: GPL-3.0-or-later
5 ***************************************************************************/
6 /** @file
7 * This file is Skrooge plugin for BACKEND import / export.
8 *
9 * @author Stephane MANKOWSKI / Guillaume DE BURE
10 */
11 #include "skgimportpluginbackend.h"
12
13 #include <qapplication.h>
14 #include <qdir.h>
15 #include <qdiriterator.h>
16 #include <qfile.h>
17 #include <qfileinfo.h>
18 #include <qprocess.h>
19 #include <qstandardpaths.h>
20 #include <qtconcurrentmap.h>
21 #include <qregularexpression.h>
22
23 #include <kaboutdata.h>
24 #include <klocalizedstring.h>
25 #include <kpluginfactory.h>
26
27 #include "skgbankincludes.h"
28 #include "skgimportexportmanager.h"
29 #include "skgservices.h"
30 #include "skgtraces.h"
31
32 /**
33 * This plugin factory.
34 */
K_PLUGIN_FACTORY(SKGImportPluginBackendFactory,registerPlugin<SKGImportPluginBackend> ();)35 K_PLUGIN_FACTORY(SKGImportPluginBackendFactory, registerPlugin<SKGImportPluginBackend>();)
36
37 SKGImportPluginBackend::SKGImportPluginBackend(QObject* iImporter, const QVariantList& iArg)
38 : SKGImportPlugin(iImporter)
39 {
40 SKGTRACEINFUNC(10)
41 Q_UNUSED(iArg)
42
43 m_listBackends = KServiceTypeTrader::self()->query(QStringLiteral("Skrooge/Import/Backend"));
44 }
45
46 SKGImportPluginBackend::~SKGImportPluginBackend()
47 = default;
48
getService() const49 QExplicitlySharedDataPointer<KService> SKGImportPluginBackend::getService() const
50 {
51 for (const auto& service : m_listBackends) {
52 if (service->property(QStringLiteral("X-Krunner-ID"), QVariant::String).toString().toUpper() == m_importer->getFileNameExtension()) {
53 return service;
54 }
55 }
56 return QExplicitlySharedDataPointer<KService>(nullptr);
57 }
58
getParameter(const QString & iAttribute)59 QString SKGImportPluginBackend::getParameter(const QString& iAttribute)
60 {
61 auto service = getService();
62 QString output = service->property(iAttribute, QVariant::String).toString();
63 QMap<QString, QString> parameters = this->getImportParameters();
64
65 for (int i = 1; i <= 10; ++i) {
66 QString param = "parameter" + SKGServices::intToString(i);
67 if (output.contains(QStringLiteral("%") % param)) {
68 output = output.replace(QStringLiteral("%") % param, parameters.value(param));
69 }
70 }
71
72 return output;
73 }
74
isImportPossible()75 bool SKGImportPluginBackend::isImportPossible()
76 {
77 SKGTRACEINFUNC(10)
78 return (m_importer == nullptr ? true : getService().data() != nullptr);
79 }
80
81 struct download {
downloaddownload82 download(int iNbToDownload, QString iDate, QString iCmd, QString iPwd, QString iPath)
83 : m_nbToDownload(iNbToDownload), m_date(std::move(iDate)), m_cmd(std::move(iCmd)), m_pwd(std::move(iPwd)), m_path(std::move(iPath))
84 {
85 }
86
87 using result_type = QString;
88
operator ()download89 QString operator()(const QString& iAccountId)
90 {
91 QString file = m_path % "/" % iAccountId % ".csv";
92 // Build cmd
93 QString cmd = m_cmd;
94 cmd = cmd.replace(QStringLiteral("%2"), SKGServices::intToString(m_nbToDownload)).replace(QStringLiteral("%1"), iAccountId).replace(QStringLiteral("%3"), m_pwd).replace(QStringLiteral("%4"), m_date);
95
96 // Execute
97 QProcess p;
98 cmd = SKGServices::getFullPathCommandLine(cmd);
99 SKGTRACEL(10) << "Execute: " << cmd << SKGENDL;
100 p.setStandardOutputFile(file);
101
102 int retry = 0;
103 do {
104 p.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd);
105 if (p.waitForFinished(1000 * 60 * 2)) {
106 if (p.exitCode() == 0) {
107 return iAccountId;
108 }
109 SKGTRACE << i18nc("A warning message", "WARNING: The command %1 failed with code %2 (Retry %3)", cmd, p.exitCode(), retry + 1) << SKGENDL;
110
111 } else {
112 SKGTRACE << i18nc("A warning message", "WARNING: The command %1 failed due to a time out (Retry %2)", cmd, retry + 1) << SKGENDL;
113 p.terminate();
114 p.kill();
115 }
116 ++retry;
117 } while (retry < 6);
118
119 QString errorMsg = i18nc("Error message", "The following command line failed with code %2:\n'%1'", cmd, p.exitCode());
120 SKGTRACE << errorMsg << SKGENDL;
121
122 return QStringLiteral("ERROR:") + errorMsg;
123 }
124
125 int m_nbToDownload;
126 QString m_date;
127 QString m_cmd;
128 QString m_pwd;
129 QString m_path;
130 };
131
importFile()132 SKGError SKGImportPluginBackend::importFile()
133 {
134 if (m_importer == nullptr) {
135 return SKGError(ERR_ABORT, i18nc("Error message", "Invalid parameters"));
136 }
137 SKGError err;
138 SKGTRACEINFUNCRC(2, err)
139
140 SKGBEGINPROGRESSTRANSACTION(*m_importer->getDocument(), i18nc("Noun, name of the user action", "Import with %1", "Backend"), err, 3)
141 QString bankendName = m_importer->getFileNameExtension().toLower();
142
143 // Get parameters
144 QMap<QString, QString> parameters = this->getImportParameters();
145 QString pwd = parameters[QStringLiteral("password")];
146
147 // Get list of accounts
148 QStringList backendAccounts;
149 QMap<QString, QString> backendAccountsBalance;
150 QMap<QString, QString> backendAccountsName;
151 QString csvfile = m_tempDir.path() % "/skrooge_backend.csv";
152 QString cmd = getParameter(QStringLiteral("X-SKROOGE-getaccounts")).replace(QStringLiteral("%3"), pwd);
153 QProcess p;
154 cmd = SKGServices::getFullPathCommandLine(cmd);
155 SKGTRACEL(10) << "Execute: " << cmd << SKGENDL;
156 p.setStandardOutputFile(csvfile);
157 p.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd);
158 if (!p.waitForFinished(1000 * 60 * 2) || p.exitCode() != 0) {
159 err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message", "The following command line failed with code %2:\n'%1'", cmd, p.exitCode()));
160 } else {
161 QFile file(csvfile);
162 if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
163 err.setReturnCode(ERR_INVALIDARG).setMessage(i18nc("Error message", "Open file '%1' failed", csvfile));
164 } else {
165 QRegularExpression reggetaccounts(getParameter(QStringLiteral("X-SKROOGE-getaccountid")));
166 QRegularExpression reggetaccountbalance(getParameter(QStringLiteral("X-SKROOGE-getaccountbalance")));
167 QRegularExpression reggetaccountname(getParameter(QStringLiteral("X-SKROOGE-getaccountname")));
168
169 QTextStream stream(&file);
170 stream.readLine(); // To avoid header
171 QStringList backendAccountsUniqueId;
172 while (!stream.atEnd()) {
173 // Read line
174 QString line = stream.readLine().trimmed();
175 SKGTRACEL(10) << "Read line: " << line << SKGENDL;
176
177 // Get account id
178 auto match = reggetaccounts.match(line);
179 if (match.hasMatch()) {
180 QString accountid = match.captured(1);
181 QString uniqueid = SKGServices::splitCSVLine(accountid, QLatin1Char('@')).at(0);
182
183 if (!backendAccounts.contains(accountid) && !backendAccountsUniqueId.contains(uniqueid)) {
184 backendAccounts.push_back(accountid);
185 backendAccountsUniqueId.push_back(uniqueid);
186
187 // Get account balance
188 match = reggetaccountbalance.match(line);
189 if (match.hasMatch()) {
190 backendAccountsBalance[accountid] = match.captured(1);
191 } else {
192 backendAccountsBalance[accountid] = '0';
193 }
194
195 // Get account name
196 match = reggetaccountname.match(line);
197 if (match.hasMatch()) {
198 backendAccountsName[accountid] = match.captured(1);
199 } else {
200 backendAccountsName[accountid] = QLatin1String("");
201 }
202 }
203 } else {
204 // This is an error
205 err.setReturnCode(ERR_FAIL).setMessage(line).addError(ERR_FAIL, i18nc("Error message", "Impossible to find the account id with the regular expression '%1' in line '%2'", getParameter(QStringLiteral("X-SKROOGE-getaccountid")), line));
206 break;
207 }
208 }
209
210 // close file
211 file.close();
212 file.remove();
213 }
214 }
215
216 // Download operations
217 IFOKDO(err, m_importer->getDocument()->stepForward(1, i18nc("Progress message", "Download operations")))
218 IFOK(err) {
219 if (backendAccounts.isEmpty()) {
220 err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message", "Your backend '%1' seems to be not well configure because no account has been found.", bankendName));
221 } else {
222 // Compute the begin date for download
223 QDate lastDownload = SKGServices::stringToTime(m_importer->getDocument()->getParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_DATE")).date();
224 QString lastList = m_importer->getDocument()->getParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_LIST");
225 QString currentList = backendAccounts.join(QStringLiteral(";"));
226
227 int nbToDownload = 0;
228 QString fromDate;
229 if (currentList != lastList || !lastDownload.isValid()) {
230 nbToDownload = 99999;
231 fromDate = QStringLiteral("2000-01-01");
232 } else {
233 nbToDownload = qMax(lastDownload.daysTo(QDate::currentDate()) * 10, qint64(20));
234 fromDate = SKGServices::dateToSqlString(lastDownload.addDays(-4));
235 }
236
237 // Download
238 QStringList listDownloadedId;
239 QString bulk = getParameter(QStringLiteral("X-SKROOGE-getbulk"));
240 QString cmddownload;
241 if (!bulk.isEmpty()) {
242 // mode bulk
243 SKGTRACEL(10) << "Mode getbulk" << SKGENDL;
244 QProcess pbulk;
245 QString cmd = bulk.replace(QStringLiteral("%1"), m_tempDir.path());
246 cmd = SKGServices::getFullPathCommandLine(cmd);
247 cmddownload = cmd;
248 SKGTRACEL(10) << "Execute: " << cmd << SKGENDL;
249 pbulk.start(QStringLiteral("/bin/bash"), QStringList() << QStringLiteral("-c") << cmd);
250 if (!pbulk.waitForFinished(1000 * 60 * 2) || pbulk.exitCode() != 0) {
251 err.setReturnCode(ERR_FAIL).setMessage(i18nc("Error message", "The following command line failed with code %2:\n'%1'", cmd, pbulk.exitCode()));
252 } else {
253 SKGTRACEL(10) << "Searching csv files " << SKGENDL;
254 QDirIterator it(m_tempDir.path(), QStringList() << QStringLiteral("*.csv"));
255 while (it.hasNext()) {
256 auto id = QFileInfo(it.next()).baseName();
257 listDownloadedId.push_back(id);
258
259 SKGTRACEL(10) << "Find id: " << id << SKGENDL;
260 }
261 }
262 } else {
263 // mode getoperations
264 SKGTRACEL(10) << "Mode getoperations" << SKGENDL;
265 cmddownload = getParameter(QStringLiteral("X-SKROOGE-getoperations"));
266 QFuture<QString> f = QtConcurrent::mapped(backendAccounts, download(nbToDownload, fromDate, cmddownload, pwd, m_tempDir.path()));
267 f.waitForFinished();
268 listDownloadedId = f.results();
269 }
270 listDownloadedId.removeAll(QLatin1String(""));
271 // Build list of errors
272 QStringList errors;
273 int nb = listDownloadedId.count();
274 errors.reserve(nb);
275 for (int i = nb - 1; i >= 0; --i) {
276 auto item = listDownloadedId.value(i);
277 if (item.startsWith(QLatin1String("ERROR:"))) {
278 listDownloadedId.removeAt(i);
279 errors.push_back(item.right(item.length() - 6));
280 }
281 }
282
283 // Check
284 IFOK(err) {
285 bool checkOK = true;
286 int nb = listDownloadedId.count();
287 if (errors.count() != 0) {
288 // Some accounts have not been downloaded
289 if (nb == 0) {
290 err = SKGError(ERR_FAIL, i18nc("Error message", "No accounts downloaded with the following command:\n%1\nCheck your backend installation.", cmddownload));
291 } else {
292 // Warning
293 m_importer->getDocument()->sendMessage(i18nc("Warning message", "Some accounts have not been downloaded. %1", errors.join(QStringLiteral(". "))), SKGDocument::Warning);
294 }
295 SKGTRACEL(10) << errors.count() << " accounts not imported => checkOK=false" << SKGENDL;
296 checkOK = false;
297 }
298
299 // import
300 IFOKDO(err, m_importer->getDocument()->stepForward(2, i18nc("Progress message", "Import")))
301 if (!err && (nb != 0)) {
302 // import
303 SKGBEGINPROGRESSTRANSACTION(*m_importer->getDocument(), "#INTERNAL#" % i18nc("Noun, name of the user action", "Import one account with %1", "Backend"), err, nb)
304
305 // Get all messages
306 SKGDocument::SKGMessageList messages;
307 IFOKDO(err, m_importer->getDocument()->getMessages(m_importer->getDocument()->getCurrentTransaction(), messages, true))
308
309 // Import all files
310 for (int i = 0; !err && i < nb; ++i) {
311 // Rename the imported name
312 QString file = m_tempDir.path() % "/" % listDownloadedId.at(i) % ".csv";
313 if (!listDownloadedId.at(i).contains(QStringLiteral("-")) && !backendAccountsName[listDownloadedId.at(i)].isEmpty()) {
314 QString newFileName = m_tempDir.path() % "/" % backendAccountsName[listDownloadedId.at(i)] % '-' % listDownloadedId.at(i) % ".csv";
315 if (QFile::rename(file, newFileName)) {
316 file = newFileName;
317 }
318 }
319
320 // Import
321 SKGImportExportManager imp1(m_importer->getDocument(), QUrl::fromLocalFile(file));
322 imp1.setAutomaticValidation(m_importer->automaticValidation());
323 imp1.setAutomaticApplyRules(m_importer->automaticApplyRules());
324 // This option is not used with backend import
325 imp1.setSinceLastImportDate(false);
326 imp1.setCodec(m_importer->getCodec());
327
328 QMap<QString, QString> newParameters = imp1.getImportParameters();
329 newParameters[QStringLiteral("automatic_search_header")] = 'N';
330 newParameters[QStringLiteral("header_position")] = '1';
331 newParameters[QStringLiteral("automatic_search_columns")] = 'N';
332 newParameters[QStringLiteral("columns_positions")] = getParameter(QStringLiteral("X-SKROOGE-csvcolumns"));
333 newParameters[QStringLiteral("mode_csv_unit")] = 'N';
334 newParameters[QStringLiteral("mode_csv_rule")] = 'N';
335 newParameters[QStringLiteral("balance")] = backendAccountsBalance[listDownloadedId.at(i)];
336 newParameters[QStringLiteral("donotfinalize")] = 'Y';
337 imp1.setImportParameters(newParameters);
338 IFOKDO(err, imp1.importFile())
339
340 if (!backendAccountsBalance[listDownloadedId.at(i)].isEmpty()) {
341 SKGAccountObject act;
342 IFOKDO(err, imp1.getDefaultAccount(act))
343 m_importer->addAccountToCheck(act, SKGServices::stringToDouble(backendAccountsBalance[listDownloadedId.at(i)]));
344 }
345 IFOKDO(err, m_importer->getDocument()->stepForward(i + 1))
346 }
347
348 // Remove all temporary files
349 for (int i = 0; i < nb; ++i) {
350 QString file = m_tempDir.path() % "/" % listDownloadedId.at(i) % ".csv";
351 QFile::remove(file);
352 }
353
354 // Reset message
355 IFOKDO(err, m_importer->getDocument()->removeMessages(m_importer->getDocument()->getCurrentTransaction()))
356 int nbm = messages.count();
357 for (int j = 0; j < nbm; ++j) {
358 SKGDocument::SKGMessage msg = messages.at(j);
359 m_importer->getDocument()->sendMessage(msg.Text, msg.Type, msg.Action);
360 }
361
362 // Finalize import
363 IFOKDO(err, m_importer->finalizeImportation())
364
365 // Disable std finalisation
366 QMap<QString, QString> parameters = m_importer->getImportParameters();
367 parameters[QStringLiteral("donotfinalize")] = 'Y';
368 m_importer->setImportParameters(parameters);
369
370 // Check balances of accounts
371 auto accountsToCheck = m_importer->getAccountsToCheck();
372 int nb = accountsToCheck.count();
373 for (int i = 0; !err && i < nb; ++i) {
374 // Get the account to check
375 auto act = accountsToCheck[i].first;
376 auto targetBalance = accountsToCheck[i].second;
377 auto soluces = act.getPossibleReconciliations(targetBalance, false);
378 if (soluces.isEmpty()) {
379 SKGTRACEL(10) << "Account " << listDownloadedId.at(i) << " not reconciliable => checkOK=false" << SKGENDL;
380 checkOK = false;
381 }
382 }
383
384 if (checkOK) {
385 // Last import is memorized only in case of 100% success
386 IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_DATE", SKGServices::dateToSqlString(QDateTime::currentDateTime())))
387 IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_LIST", currentList))
388 } else {
389 // Remove last import for next import
390 IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_DATE", QLatin1String("")))
391 IFOKDO(err, m_importer->getDocument()->setParameter("SKG_LAST_" % bankendName.toUpper() % "_IMPORT_LIST", QLatin1String("")))
392 }
393 }
394 IFOKDO(err, m_importer->getDocument()->stepForward(3))
395 }
396 }
397 }
398
399 return err;
400 }
401
getMimeTypeFilter() const402 QString SKGImportPluginBackend::getMimeTypeFilter() const
403 {
404 return QLatin1String("");
405 }
406
407 #include <skgimportpluginbackend.moc>
408