1 /*
2 This file is part of Lokalize
3
4 SPDX-FileCopyrightText: 2007-2014 Nick Shaforostoff <shafff@ukr.net>
5 SPDX-FileCopyrightText: 2018-2019 Simon Depiets <sdepiets@gmail.com>
6
7 SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
8 */
9
10 #include "jobs.h"
11
12 #include "lokalize_debug.h"
13
14 #include "catalog.h"
15 #include "project.h"
16 #include "diff.h"
17 #include "prefs_lokalize.h"
18 #include "version.h"
19
20 #include "stemming.h"
21
22 #include <QSqlDatabase>
23 #include <QSqlQuery>
24 #include <QSqlError>
25 #include <QStringBuilder>
26 #include <QRegExp>
27 #include <QMap>
28 #include <QStandardPaths>
29 #include <QFile>
30 #include <QElapsedTimer>
31 #include <QDir>
32
33 #include <iostream>
34
35 #include <math.h>
36 using namespace TM;
37
38
threadPool()39 QThreadPool* TM::threadPool()
40 {
41 static QThreadPool* inst = new QThreadPool;
42 return inst;
43 }
44
45 #ifdef Q_OS_WIN
46 #define U QLatin1String
47 #else
48 #define U QStringLiteral
49 #endif
50
51 #define TM_DELIMITER '\v'
52 #define TM_SEPARATOR '\b'
53 #define TM_NOTAPPROVED 0x04
54
55
56 static bool stop = false;
cancelAllJobs()57 void TM::cancelAllJobs()
58 {
59 stop = true;
60 }
getConnectionName(const QString dbName)61 static const QString getConnectionName(const QString dbName)
62 {
63 return dbName + QString::number((long)QThread::currentThreadId());
64 }
65
66 static qlonglong newTMSourceEntryCount = 0;
67 static qlonglong reusedTMSourceEntryCount = 0;
68
69 /**
70 * splits string into words, removing any markup
71 *
72 * TODO segmentation by sentences...
73 **/
doSplit(QString & cleanEn,QStringList & words,QRegExp & rxClean1,const QString & accel)74 static void doSplit(QString& cleanEn,
75 QStringList& words,
76 QRegExp& rxClean1,
77 const QString& accel
78 )
79 {
80 static QRegExp rxSplit(QStringLiteral("\\W+|\\d+"));
81
82 if (!rxClean1.pattern().isEmpty())
83 cleanEn.replace(rxClean1, QStringLiteral(" "));
84 cleanEn.remove(accel);
85
86 words = cleanEn.toLower().split(rxSplit, Qt::SkipEmptyParts);
87 if (words.size() > 4) {
88 int i = 0;
89 for (; i < words.size(); ++i) {
90 if (words.at(i).size() < 4)
91 words.removeAt(i--);
92 else if (words.at(i).startsWith('t') && words.at(i).size() == 4) {
93 if (words.at(i) == QLatin1String("then")
94 || words.at(i) == QLatin1String("than")
95 || words.at(i) == QLatin1String("that")
96 || words.at(i) == QLatin1String("this")
97 )
98 words.removeAt(i--);
99 }
100 }
101 }
102
103 }
104
getFileId(const QString & path,QSqlDatabase & db)105 static qlonglong getFileId(const QString& path,
106 QSqlDatabase& db)
107 {
108 QSqlQuery query1(db);
109 QString escapedPath = path;
110 escapedPath.replace(QLatin1Char('\''), QLatin1String("''"));
111
112 QString pathExpr = QStringLiteral("path='") + escapedPath + '\'';
113 if (path.isEmpty())
114 pathExpr = QStringLiteral("path ISNULL");
115 if (Q_UNLIKELY(!query1.exec(U("SELECT id FROM files WHERE "
116 "path='") + escapedPath + '\'')))
117 qCWarning(LOKALIZE_LOG) << "select db error: " << query1.lastError().text();
118
119 if (Q_LIKELY(query1.next())) {
120 //this is translation of en string that is already present in db
121
122 qlonglong id = query1.value(0).toLongLong();
123 query1.clear();
124 return id;
125 }
126 query1.clear();
127
128 //nope, this is new file
129 bool qpsql = (db.driverName() == QLatin1String("QPSQL"));
130 QString sql = QStringLiteral("INSERT INTO files (path) VALUES (?)");
131 if (qpsql)
132 sql += QLatin1String(" RETURNING id");
133 query1.prepare(sql);
134
135 query1.bindValue(0, path);
136 if (Q_LIKELY(query1.exec()))
137 return qpsql ? (query1.next(), query1.value(0).toLongLong()) : query1.lastInsertId().toLongLong();
138 else
139 qCWarning(LOKALIZE_LOG) << "insert db error: " << query1.lastError().text();
140
141 return -1;
142 }
143
144
145
146
addToIndex(qlonglong sourceId,QString sourceString,QRegExp & rxClean1,const QString & accel,QSqlDatabase & db)147 static void addToIndex(qlonglong sourceId, QString sourceString,
148 QRegExp& rxClean1, const QString& accel, QSqlDatabase& db)
149 {
150 QStringList words;
151 doSplit(sourceString, words, rxClean1, accel);
152
153 if (Q_UNLIKELY(words.isEmpty()))
154 return;
155
156 QSqlQuery query1(db);
157
158 QByteArray sourceIdStr = QByteArray::number(sourceId, 36);
159
160 bool isShort = words.size() < 20;
161 int j = words.size();
162 while (--j >= 0) {
163 // insert word (if we do not have it)
164 if (Q_UNLIKELY(!query1.exec(U("SELECT word, ids_short, ids_long FROM words WHERE "
165 "word='") + words.at(j) + '\'')))
166 qCWarning(LOKALIZE_LOG) << "select error 3: " << query1.lastError().text();
167
168 //we _have_ it
169 bool weHaveIt = query1.next();
170
171 if (weHaveIt) {
172 //just add new id
173 QByteArray arr;
174 QString field;
175 if (isShort) {
176 arr = query1.value(1).toByteArray();
177 field = QStringLiteral("ids_short");
178 } else {
179 arr = query1.value(2).toByteArray();
180 field = QStringLiteral("ids_long");
181 }
182 query1.clear();
183
184 if (arr.contains(' ' + sourceIdStr + ' ')
185 || arr.startsWith(sourceIdStr + ' ')
186 || arr.endsWith(' ' + sourceIdStr)
187 || arr == sourceIdStr)
188 return;//this string is already indexed
189
190 query1.prepare(QStringLiteral("UPDATE words SET ") + field + QStringLiteral("=? WHERE word='") + words.at(j) + '\'');
191
192 if (!arr.isEmpty())
193 arr += ' ';
194 arr += sourceIdStr;
195 query1.bindValue(0, arr);
196
197 if (Q_UNLIKELY(!query1.exec()))
198 qCWarning(LOKALIZE_LOG) << "update error 4: " << query1.lastError().text();
199
200 } else {
201 query1.clear();
202 query1.prepare(QStringLiteral("INSERT INTO words (word, ids_short, ids_long) VALUES (?, ?, ?)"));
203 QByteArray idsShort;
204 QByteArray idsLong;
205 if (isShort)
206 idsShort = sourceIdStr;
207 else
208 idsLong = sourceIdStr;
209 query1.bindValue(0, words.at(j));
210 query1.bindValue(1, idsShort);
211 query1.bindValue(2, idsLong);
212 if (Q_UNLIKELY(!query1.exec()))
213 qCWarning(LOKALIZE_LOG) << "insert error 2: " << query1.lastError().text() ;
214
215 }
216 }
217 }
218
219 /**
220 * remove source string from index if there are no other
221 * 'good' entries using it but the entry specified with mainId
222 */
removeFromIndex(qlonglong mainId,qlonglong sourceId,QString sourceString,QRegExp & rxClean1,const QString & accel,QSqlDatabase & db)223 static void removeFromIndex(qlonglong mainId, qlonglong sourceId, QString sourceString,
224 QRegExp& rxClean1, const QString& accel, QSqlDatabase& db)
225 {
226 QStringList words;
227 doSplit(sourceString, words, rxClean1, accel);
228
229 if (Q_UNLIKELY(words.isEmpty()))
230 return;
231
232 QSqlQuery query1(db);
233 QByteArray sourceIdStr = QByteArray::number(sourceId, 36);
234
235 //BEGIN check
236 //TM_NOTAPPROVED=4
237 if (Q_UNLIKELY(!query1.exec(U("SELECT count(*) FROM main, target_strings WHERE "
238 "main.source=") + QString::number(sourceId) + U(" AND "
239 "main.target=target_strings.id AND "
240 "target_strings.target NOTNULL AND "
241 "main.id!=") + QString::number(mainId) + U(" AND "
242 "(main.bits&4)!=4")))) {
243 qCWarning(LOKALIZE_LOG) << "select error 500: " << query1.lastError().text();
244 return;
245 }
246
247 bool exit = query1.next() && (query1.value(0).toLongLong() > 0);
248 query1.clear();
249 if (exit)
250 return;
251 //END check
252
253 bool isShort = words.size() < 20;
254 int j = words.size();
255 while (--j >= 0) {
256 // remove from record for the word (if we do not have it)
257 if (Q_UNLIKELY(!query1.exec(U("SELECT word, ids_short, ids_long FROM words WHERE "
258 "word='") + words.at(j) + '\''))) {
259 qCWarning(LOKALIZE_LOG) << "select error 3: " << query1.lastError().text();
260 return;
261 }
262 if (!query1.next()) {
263 qCWarning(LOKALIZE_LOG) << "exit here 1";
264 //we don't have record for the word, so nothing to remove
265 query1.clear();
266 return;
267 }
268
269 QByteArray arr;
270 QString field;
271 if (isShort) {
272 arr = query1.value(1).toByteArray();
273 field = QStringLiteral("ids_short");
274 } else {
275 arr = query1.value(2).toByteArray();
276 field = QStringLiteral("ids_long");
277 }
278 query1.clear();
279
280 if (arr.contains(' ' + sourceIdStr + ' '))
281 arr.replace(' ' + sourceIdStr + ' ', " ");
282 else if (arr.startsWith(sourceIdStr + ' '))
283 arr.remove(0, sourceIdStr.size() + 1);
284 else if (arr.endsWith(' ' + sourceIdStr))
285 arr.chop(sourceIdStr.size() + 1);
286 else if (arr == sourceIdStr)
287 arr.clear();
288
289
290 query1.prepare(U("UPDATE words "
291 "SET ") + field + U("=? "
292 "WHERE word='") + words.at(j) + '\'');
293
294 query1.bindValue(0, arr);
295
296 if (Q_UNLIKELY(!query1.exec()))
297 qCWarning(LOKALIZE_LOG) << "update error 504: " << query1.lastError().text();
298
299 }
300 }
301
doRemoveEntry(qlonglong mainId,QRegExp & rxClean1,const QString & accel,QSqlDatabase & db)302 static bool doRemoveEntry(qlonglong mainId, QRegExp& rxClean1, const QString& accel, QSqlDatabase& db)
303 {
304 QSqlQuery query1(db);
305
306 if (Q_UNLIKELY(!query1.exec(U("SELECT source_strings.id, source_strings.source FROM source_strings, main WHERE "
307 "source_strings.id=main.source AND main.id=") + QString::number(mainId))))
308 return false;
309
310 if (!query1.next())
311 return false;
312
313 const qlonglong sourceId = query1.value(0).toLongLong();
314 const QString sourceString = query1.value(1).toString();
315
316 query1.clear();
317
318 if (Q_UNLIKELY(!query1.exec(U("SELECT target_strings.id FROM target_strings, main WHERE target_strings.id=main.target AND main.id=") + QString::number(mainId))))
319 return false;
320
321 if (!query1.next())
322 return false;
323
324 const qlonglong targetId = query1.value(0).toLongLong();
325 query1.clear();
326
327 query1.exec(QStringLiteral("DELETE FROM main WHERE source=") + QString::number(sourceId) + QStringLiteral(" AND target=") + QString::number(targetId));
328
329 if (!query1.exec(QStringLiteral("SELECT count(*) FROM main WHERE source=") + QString::number(sourceId))
330 || !query1.next())
331 return false;
332
333 const bool noSourceLeft = query1.value(0).toInt() == 0;
334 query1.clear();
335 if (noSourceLeft) {
336 removeFromIndex(mainId, sourceId, sourceString, rxClean1, accel, db);
337 query1.exec(QStringLiteral("DELETE FROM source_strings WHERE id=") + QString::number(sourceId));
338 }
339
340 if (!query1.exec(QStringLiteral("SELECT count(*) FROM main WHERE target=") + QString::number(targetId))
341 || ! query1.next())
342 return false;
343 const bool noTargetLeft = query1.value(0).toInt() == 0;
344 query1.clear();
345 if (noTargetLeft)
346 query1.exec(QStringLiteral("DELETE FROM target_strings WHERE id=") + QString::number(targetId));
347 return true;
348 }
349
350
doRemoveFile(const QString & filePath,QSqlDatabase & db)351 static bool doRemoveFile(const QString& filePath, QSqlDatabase& db)
352 {
353 qlonglong fileId = getFileId(filePath, db);
354 QSqlQuery query1(db);
355
356 if (Q_UNLIKELY(!query1.exec(U("SELECT id FROM files WHERE "
357 "id=") + QString::number(fileId))))
358 return false;
359
360 if (!query1.next())
361 return false;
362
363 query1.clear();
364
365 query1.exec(QStringLiteral("DELETE source_strings FROM source_strings, main WHERE source_strings.id = main.source AND main.file =") + QString::number(fileId));
366 query1.exec(QStringLiteral("DELETE target_strings FROM target_strings, main WHERE target_strings.id = main.target AND main.file =") + QString::number(fileId));
367 query1.exec(QStringLiteral("DELETE FROM main WHERE file = ") + QString::number(fileId));
368 return query1.exec(QStringLiteral("DELETE FROM files WHERE id=") + QString::number(fileId));
369 }
370
doRemoveMissingFiles(QSqlDatabase & db,const QString & dbName,QObject * job)371 static int doRemoveMissingFiles(QSqlDatabase& db, const QString& dbName, QObject *job)
372 {
373 int deletedFiles = 0;
374 QSqlQuery query1(db);
375
376 if (Q_UNLIKELY(!query1.exec(U("SELECT files.path FROM files"))))
377 return false;
378
379 if (!query1.next())
380 return false;
381
382 do {
383 QString filePath = query1.value(0).toString();
384 if (Project::instance()->isFileMissing(filePath)) {
385 qCWarning(LOKALIZE_LOG) << "Removing file " << filePath << " from translation memory";
386 RemoveFileJob* job_removefile = new RemoveFileJob(filePath, dbName, job);
387 TM::threadPool()->start(job_removefile, REMOVEFILE);
388 deletedFiles++;
389 }
390 } while (query1.next());
391
392 return deletedFiles;
393 }
394
escape(QString str)395 static QString escape(QString str)
396 {
397 return str.replace(QLatin1Char('\''), QStringLiteral("''"));
398 }
399
doInsertEntry(CatalogString source,CatalogString target,const QString & ctxt,bool approved,qlonglong fileId,QSqlDatabase & db,QRegExp & rxClean1,const QString & accel,qlonglong priorId,qlonglong & mainId)400 static bool doInsertEntry(CatalogString source,
401 CatalogString target,
402 const QString& ctxt, //TODO QStringList -- after XLIFF
403 bool approved,
404 qlonglong fileId,
405 QSqlDatabase& db,
406 QRegExp& rxClean1,//cleaning regexps for word index update
407 const QString& accel,
408 qlonglong priorId,
409 qlonglong& mainId
410 )
411 {
412 // QTime a; a.start();
413
414 mainId = -1;
415
416 if (Q_UNLIKELY(source.isEmpty())) {
417 qCWarning(LOKALIZE_LOG) << "doInsertEntry: source empty";
418 return false;
419 }
420
421 bool qpsql = (db.driverName() == QLatin1String("QPSQL"));
422
423 //we store non-entranslaed entries to make search over all source parts possible
424 bool untranslated = target.isEmpty();
425 bool shouldBeInIndex = !untranslated && approved;
426
427 //remove first occurrence of accel character so that search returns words containing accel mark
428 int sourceAccelPos = source.string.indexOf(accel);
429 if (sourceAccelPos != -1)
430 source.string.remove(sourceAccelPos, accel.size());
431 int targetAccelPos = target.string.indexOf(accel);
432 if (targetAccelPos != -1)
433 target.string.remove(targetAccelPos, accel.size());
434
435 //check if we already have record with the same en string
436 QSqlQuery query1(db);
437 QString escapedCtxt = escape(ctxt);
438
439 QByteArray sourceTags = source.tagsAsByteArray();
440 QByteArray targetTags = target.tagsAsByteArray();
441
442 //BEGIN get sourceId
443 query1.prepare(QString(U("SELECT id FROM source_strings WHERE "
444 "source=? AND (source_accel%1) AND source_markup%2")).arg
445 (sourceAccelPos != -1 ? QStringLiteral("=?") : QStringLiteral("=-1 OR source_accel ISNULL"),
446 sourceTags.isEmpty() ? QStringLiteral(" ISNULL") : QStringLiteral("=?")));
447 int paranum = 0;
448 query1.bindValue(paranum++, source.string);
449 if (sourceAccelPos != -1)
450 query1.bindValue(paranum++, sourceAccelPos);
451 if (!sourceTags.isEmpty())
452 query1.bindValue(paranum++, sourceTags);
453 if (Q_UNLIKELY(!query1.exec())) {
454 qCWarning(LOKALIZE_LOG) << "doInsertEntry: select db source_strings error: " << query1.lastError().text();
455 return false;
456 }
457 qlonglong sourceId;
458 if (!query1.next()) {
459 //BEGIN insert source anew
460 //qCDebug(LOKALIZE_LOG) <<"insert source anew";;
461 ++newTMSourceEntryCount;
462
463 QString sql = QStringLiteral("INSERT INTO source_strings (source, source_markup, source_accel) VALUES (?, ?, ?)");
464 if (qpsql)
465 sql += QLatin1String(" RETURNING id");
466
467 query1.clear();
468 query1.prepare(sql);
469
470 query1.bindValue(0, source.string);
471 query1.bindValue(1, sourceTags);
472 query1.bindValue(2, sourceAccelPos != -1 ? QVariant(sourceAccelPos) : QVariant());
473 if (Q_UNLIKELY(!query1.exec())) {
474 qCWarning(LOKALIZE_LOG) << "doInsertEntry: select db source_strings error: " << query1.lastError().text();
475 return false;
476 }
477 sourceId = qpsql ? (query1.next(), query1.value(0).toLongLong()) : query1.lastInsertId().toLongLong();
478 query1.clear();
479
480 //update index
481 if (shouldBeInIndex)
482 addToIndex(sourceId, source.string, rxClean1, accel, db);
483 //END insert source anew
484 } else {
485 sourceId = query1.value(0).toLongLong();
486 ++reusedTMSourceEntryCount;
487 //qCDebug(LOKALIZE_LOG)<<"SOURCE ALREADY PRESENT"<<source.string<<sourceId;
488 }
489 query1.clear();
490 //END get sourceId
491
492
493 if (Q_UNLIKELY(!query1.exec(QString(U("SELECT id, target, bits FROM main WHERE "
494 "source=%1 AND file=%2 AND ctxt%3")).arg(sourceId).arg(fileId).arg
495 (escapedCtxt.isEmpty() ? QStringLiteral(" ISNULL") : QString("='" + escapedCtxt + '\''))))) {
496 qCWarning(LOKALIZE_LOG) << "doInsertEntry: select db main error: " << query1.lastError().text();
497 return false;
498 }
499
500 //case:
501 // aaa-bbb
502 // aaa-""
503 // aaa-ccc
504 //bbb shouldn't be present in db
505
506
507 //update instead of adding record to main?
508 qlonglong bits = 0;
509 //BEGIN target update
510 if (query1.next()) {
511 //qCDebug(LOKALIZE_LOG)<<target.string<<": update instead of adding record to main";
512 mainId = query1.value(0).toLongLong();
513 bits = query1.value(2).toLongLong();
514 bits = bits & (0xff - 1); //clear obsolete bit
515 qlonglong targetId = query1.value(1).toLongLong();
516 query1.clear();
517
518 //qCWarning(LOKALIZE_LOG)<<"8... "<<a.elapsed();
519
520 bool dbApproved = !(bits & TM_NOTAPPROVED);
521 bool approvalChanged = dbApproved != approved;
522 if (approvalChanged) {
523 query1.prepare(U("UPDATE main "
524 "SET bits=?, change_date=CURRENT_DATE "
525 "WHERE id=") + QString::number(mainId));
526
527 query1.bindValue(0, bits ^ TM_NOTAPPROVED);
528 if (Q_UNLIKELY(!query1.exec())) {
529 qCWarning(LOKALIZE_LOG) << "doInsertEntry: fail #9" << query1.lastError().text();
530 return false;
531 }
532 }
533 query1.clear();
534
535 //check if target in TM matches
536 if (Q_UNLIKELY(!query1.exec(U("SELECT target, target_markup, target_accel FROM target_strings WHERE "
537 "id=") + QString::number(targetId)))) {
538 qCWarning(LOKALIZE_LOG) << "doInsertEntry: select db target_strings error: " << query1.lastError().text();
539 return false;
540 }
541
542 if (Q_UNLIKELY(!query1.next())) {
543 qCWarning(LOKALIZE_LOG) << "doInsertEntry: linking to non-existing target should never happen";
544 return false;
545 }
546 QString dbTarget = query1.value(0).toString();
547 int accelPos = query1.value(2).isNull() ? -1 : query1.value(2).toInt();
548 bool matches = dbTarget == target.string && accelPos == targetAccelPos;
549 query1.clear();
550
551 bool untransStatusChanged = ((target.isEmpty() || dbTarget.isEmpty()) && !matches);
552 if (approvalChanged || untransStatusChanged) { //only modify index if there were changes (this is not rescan)
553 if (shouldBeInIndex)
554 //this adds source to index if it's not already there
555 addToIndex(sourceId, source.string, rxClean1, accel, db);
556 else
557 //entry changed from indexable to non-indexable:
558 //remove source string from index if there are no other
559 //'good' entries using it
560 removeFromIndex(mainId, sourceId, source.string, rxClean1, accel, db);
561 }
562
563 if (matches) { //TODO XLIFF target_markup
564 return false;
565 }
566 // no, translation has changed: just update old target if it isn't used elsewhere
567 if (Q_UNLIKELY(!query1.exec(U("SELECT count(*) FROM main WHERE "
568 "target=") + QString::number(targetId))))
569 qCWarning(LOKALIZE_LOG) << "doInsertEntry: select db target_strings error: " << query1.lastError().text();
570
571 if (query1.next() && query1.value(0).toLongLong() == 1) {
572 //TODO tnis may create duplicates, while no strings should be lost
573 query1.clear();
574
575 query1.prepare(U("UPDATE target_strings "
576 "SET target=?, target_accel=?, target_markup=? "
577 "WHERE id=") + QString::number(targetId));
578
579 query1.bindValue(0, target.string.isEmpty() ? QVariant() : target.string);
580 query1.bindValue(1, targetAccelPos != -1 ? QVariant(targetAccelPos) : QVariant());
581 query1.bindValue(2, target.tagsAsByteArray());
582 bool ok = query1.exec(); //note the RETURN!!!!
583 if (!ok)
584 qCWarning(LOKALIZE_LOG) << "doInsertEntry: target update failed" << query1.lastError().text();
585 else {
586 ok = query1.exec(QStringLiteral("UPDATE main SET change_date=CURRENT_DATE WHERE target=") + QString::number(targetId));
587 if (!ok)
588 qCWarning(LOKALIZE_LOG) << "doInsertEntry: main update failed" << query1.lastError().text();
589 }
590 return ok;
591 }
592 //else -> there will be new record insertion and main table update below
593 }
594 //qCDebug(LOKALIZE_LOG)<<target.string<<": update instead of adding record to main NOT"<<query1.executedQuery();
595 query1.clear();
596 //END target update
597
598 //BEGIN get targetId
599 query1.prepare(QString(U("SELECT id FROM target_strings WHERE "
600 "target=? AND (target_accel%1) AND target_markup%2")).arg
601 (targetAccelPos != -1 ? QStringLiteral("=?") : QStringLiteral("=-1 OR target_accel ISNULL"),
602 targetTags.isEmpty() ? QStringLiteral(" ISNULL") : QStringLiteral("=?")));
603 paranum = 0;
604 query1.bindValue(paranum++, target.string);
605 if (targetAccelPos != -1)
606 query1.bindValue(paranum++, targetAccelPos);
607 if (!targetTags.isEmpty())
608 query1.bindValue(paranum++, targetTags);
609 if (Q_UNLIKELY(!query1.exec())) {
610 qCWarning(LOKALIZE_LOG) << "doInsertEntry: select db target_strings error: " << query1.lastError().text();
611 return false;
612 }
613 qlonglong targetId;
614 if (!query1.next()) {
615 QString sql = QStringLiteral("INSERT INTO target_strings (target, target_markup, target_accel) VALUES (?, ?, ?)");
616 if (qpsql)
617 sql += QLatin1String(" RETURNING id");
618
619 query1.clear();
620 query1.prepare(sql);
621
622 query1.bindValue(0, target.string);
623 query1.bindValue(1, target.tagsAsByteArray());
624 query1.bindValue(2, targetAccelPos != -1 ? QVariant(targetAccelPos) : QVariant());
625 if (Q_UNLIKELY(!query1.exec())) {
626 qCWarning(LOKALIZE_LOG) << "doInsertEntry: error inserting";
627 return false;
628 }
629 targetId = qpsql ? (query1.next(), query1.value(0).toLongLong()) : query1.lastInsertId().toLongLong();
630 } else {
631 //very unlikely, except for empty string case
632 targetId = query1.value(0).toLongLong();
633 }
634 query1.clear();
635 //END get targetId
636
637 bool dbApproved = !(bits & TM_NOTAPPROVED);
638 if (dbApproved != approved)
639 bits ^= TM_NOTAPPROVED;
640
641 if (mainId != -1) {
642 //just update main with new targetId
643 //(this is the case when target changed, but there were other users of the old one)
644
645 query1.prepare(U("UPDATE main "
646 "SET target=?, bits=?, change_date=CURRENT_DATE "
647 "WHERE id=") + QString::number(mainId));
648
649 query1.bindValue(0, targetId);
650 query1.bindValue(1, bits);
651 bool ok = query1.exec();
652 //qCDebug(LOKALIZE_LOG)<<"ok?"<<ok;
653 if (!ok)
654 qCWarning(LOKALIZE_LOG) << "doInsertEntry: main update failed" << query1.lastError().text();
655 return ok;
656 }
657
658 //for case when previous source additions were
659 //for entries that didn't deserve indexing
660 if (shouldBeInIndex)
661 //this adds source to index if it's not already there
662 addToIndex(sourceId, source.string, rxClean1, accel, db);
663
664 QString sql = U("INSERT INTO main (source, target, file, ctxt, bits, prior) "
665 "VALUES (?, ?, ?, ?, ?, ?)");
666 if (qpsql)
667 sql += QLatin1String(" RETURNING id");
668
669 query1.prepare(sql);
670
671 // query1.prepare(QString("INSERT INTO main (source, target, file, ctxt, bits%1) "
672 // "VALUES (?, ?, ?, ?, ?%2)").arg((priorId!=-1)?", prior":"").arg((priorId!=-1)?", ?":""));
673
674 query1.bindValue(0, sourceId);
675 query1.bindValue(1, targetId);
676 query1.bindValue(2, fileId);
677 query1.bindValue(3, ctxt.isEmpty() ? QVariant() : ctxt);
678 query1.bindValue(4, bits);
679 query1.bindValue(5, priorId != -1 ? QVariant(priorId) : QVariant());
680 bool ok = query1.exec();
681 mainId = qpsql ? (query1.next(), query1.value(0).toLongLong()) : query1.lastInsertId().toLongLong();
682 if (!ok)
683 qCWarning(LOKALIZE_LOG) << "doInsertEntry: main insert failed" << query1.lastError().text();
684 //qCDebug(LOKALIZE_LOG)<<"ok?"<<ok;
685 return ok;
686 }
687
688 //TODO smth with its usage in places except opendbjob
initSqliteDb(QSqlDatabase & db)689 static bool initSqliteDb(QSqlDatabase& db)
690 {
691 QSqlQuery queryMain(db);
692 //NOTE do this only if no japanese, chinese etc?
693 queryMain.exec(QStringLiteral("PRAGMA encoding = \"UTF-8\""));
694 queryMain.exec(U(
695 "CREATE TABLE IF NOT EXISTS source_strings ("
696 "id INTEGER PRIMARY KEY ON CONFLICT REPLACE, "// AUTOINCREMENT,"
697 "source TEXT, "
698 "source_markup BLOB, "//XLIFF markup info, see catalog/catalogstring.h catalog/xliff/*
699 "source_accel INTEGER "
700 ")"));
701
702 queryMain.exec(U(
703 "CREATE TABLE IF NOT EXISTS target_strings ("
704 "id INTEGER PRIMARY KEY ON CONFLICT REPLACE, "// AUTOINCREMENT,"
705 "target TEXT, "
706 "target_markup BLOB, "//XLIFF markup info, see catalog/catalogstring.h catalog/xliff/*
707 "target_accel INTEGER "
708 ")"));
709
710 queryMain.exec(U(
711 "CREATE TABLE IF NOT EXISTS main ("
712 "id INTEGER PRIMARY KEY ON CONFLICT REPLACE, "// AUTOINCREMENT,"
713 "source INTEGER, "
714 "target INTEGER, "
715 "file INTEGER, "// AUTOINCREMENT,"
716 "ctxt TEXT, "//context, after \v may be a plural form
717 "date DEFAULT CURRENT_DATE, "//creation date
718 "change_date DEFAULT CURRENT_DATE, "//last update date
719 //change_author
720 "bits NUMERIC DEFAULT 0, "
721 //bits&0x01 means entry obsolete (not present in file)
722 //bits&0x02 means entry is NOT equiv-trans (see XLIFF spec)
723 //bits&0x04 TM_NOTAPPROVED entry is NOT approved?
724
725 //ALTER TABLE main ADD COLUMN prior INTEGER;
726 "prior INTEGER"// helps restoring full context!
727 //"reusability NUMERIC DEFAULT 0" //e.g. whether the translation is context-free, see XLIFF spec (equiv-trans)
728 //"hits NUMERIC DEFAULT 0"
729 ")"));
730
731 queryMain.exec(U("CREATE INDEX IF NOT EXISTS source_index ON source_strings ("
732 "source"
733 ")"));
734
735 queryMain.exec(U("CREATE INDEX IF NOT EXISTS target_index ON target_strings ("
736 "target"
737 ")"));
738
739 queryMain.exec(U("CREATE INDEX IF NOT EXISTS main_index ON main ("
740 "source, target, file"
741 ")"));
742
743 queryMain.exec(U(
744 "CREATE TABLE IF NOT EXISTS files ("
745 "id INTEGER PRIMARY KEY ON CONFLICT REPLACE, "
746 "path TEXT UNIQUE ON CONFLICT REPLACE, "
747 "date DEFAULT CURRENT_DATE " //last edit date when last scanned
748 ")"));
749
750 /* NOTE //"don't implement it till i'm sure it is actually useful"
751 //this is used to prevent readding translations that were removed by user
752 queryMain.exec("CREATE TABLE IF NOT EXISTS tm_removed ("
753 "id INTEGER PRIMARY KEY ON CONFLICT REPLACE, "
754 "english BLOB, "//qChecksum
755 "target BLOB, "
756 "ctxt TEXT, "//context; delimiter is \b
757 "date DEFAULT CURRENT_DATE, "
758 "hits NUMERIC DEFAULT 0"
759 ")");*/
760
761
762 //we create indexes manually, in a customized way
763 //OR: SELECT (tm_links.id) FROM tm_links,words WHERE tm_links.wordid==words.wordid AND (words.word) IN ("africa","abidjan");
764 queryMain.exec(U(
765 "CREATE TABLE IF NOT EXISTS words ("
766 "word TEXT UNIQUE ON CONFLICT REPLACE, "
767 "ids_short BLOB, " // actually, it's text,
768 "ids_long BLOB " // but it will never contain non-latin chars
769 ")"));
770
771 queryMain.exec(U(
772 "CREATE TABLE IF NOT EXISTS tm_config ("
773 "key INTEGER PRIMARY KEY ON CONFLICT REPLACE, "// AUTOINCREMENT,"
774 "value TEXT "
775 ")"));
776
777
778 bool ok = queryMain.exec(QStringLiteral("select * from main limit 1"));
779 return ok || !queryMain.lastError().text().contains(QLatin1String("database disk image is malformed"));
780
781 //queryMain.exec("CREATE TEMP TRIGGER set_user_id_trigger AFTER UPDATE ON main FOR EACH ROW BEGIN UPDATE main SET change_author = 0 WHERE main.id=NEW.id; END;");
782 //CREATE TEMP TRIGGER set_user_id_trigger INSTEAD OF UPDATE ON main FOR EACH ROW BEGIN UPDATE main SET ctxt = 'test', source=NEW.source, target=NEW.target, WHERE main.id=NEW.id; END;
783 //config:
784 //accel
785 //markup
786 //(see a little below)
787 }
788
789 //special SQL for PostgreSQL
initPgDb(QSqlDatabase & db)790 static void initPgDb(QSqlDatabase& db)
791 {
792 QSqlQuery queryMain(db);
793 queryMain.exec(QStringLiteral("CREATE SEQUENCE source_id_serial"));
794 queryMain.exec(U(
795 "CREATE TABLE source_strings ("
796 "id INTEGER PRIMARY KEY DEFAULT nextval('source_id_serial'), "
797 "source TEXT, "
798 "source_markup TEXT, "//XLIFF markup info, see catalog/catalogstring.h catalog/xliff/*
799 "source_accel INTEGER "
800 ")"));
801
802 queryMain.exec(QStringLiteral("CREATE SEQUENCE target_id_serial"));
803 queryMain.exec(U(
804 "CREATE TABLE target_strings ("
805 "id INTEGER PRIMARY KEY DEFAULT nextval('target_id_serial'), "
806 "target TEXT, "
807 "target_markup TEXT, "//XLIFF markup info, see catalog/catalogstring.h catalog/xliff/*
808 "target_accel INTEGER "
809 ")"));
810
811 queryMain.exec(QStringLiteral("CREATE SEQUENCE main_id_serial"));
812 queryMain.exec(U(
813 "CREATE TABLE main ("
814 "id INTEGER PRIMARY KEY DEFAULT nextval('main_id_serial'), "
815 "source INTEGER, "
816 "target INTEGER, "
817 "file INTEGER, "// AUTOINCREMENT,"
818 "ctxt TEXT, "//context, after \v may be a plural form
819 "date DATE DEFAULT CURRENT_DATE, "//last update date
820 "change_date DATE DEFAULT CURRENT_DATE, "//last update date
821 "change_author OID, "//last update date
822 "bits INTEGER DEFAULT 0, "
823 "prior INTEGER"// helps restoring full context!
824 ")"));
825
826 queryMain.exec(U("CREATE INDEX source_index ON source_strings ("
827 "source"
828 ")"));
829
830 queryMain.exec(U("CREATE INDEX target_index ON target_strings ("
831 "target"
832 ")"));
833
834 queryMain.exec(U("CREATE INDEX main_index ON main ("
835 "source, target, file"
836 ")"));
837
838 queryMain.exec(QStringLiteral("CREATE SEQUENCE file_id_serial"));
839 queryMain.exec(U("CREATE TABLE files ("
840 "id INTEGER PRIMARY KEY DEFAULT nextval('file_id_serial'), "
841 "path TEXT UNIQUE, "
842 "date DATE DEFAULT CURRENT_DATE " //last edit date when last scanned
843 ")"));
844
845 //we create indexes manually, in a customized way
846 //OR: SELECT (tm_links.id) FROM tm_links,words WHERE tm_links.wordid==words.wordid AND (words.word) IN ("africa","abidjan");
847 queryMain.exec(U("CREATE TABLE words ("
848 "word TEXT UNIQUE, "
849 "ids_short BYTEA, " // actually, it's text,
850 "ids_long BYTEA " //` but it will never contain non-latin chars
851 ")"));
852
853 queryMain.exec(U("CREATE TABLE tm_config ("
854 "key INTEGER PRIMARY KEY, "// AUTOINCREMENT,"
855 "value TEXT "
856 ")"));
857 //config:
858 //accel
859 //markup
860 //(see a little below)
861
862 queryMain.exec(U(
863 "CREATE OR REPLACE FUNCTION set_user_id() RETURNS trigger AS $$"
864 "BEGIN"
865 " NEW.change_author = (SELECT usesysid FROM pg_user WHERE usename = CURRENT_USER);"
866 " RETURN NEW;"
867 "END"
868 "$$ LANGUAGE plpgsql;"));
869
870 //DROP TRIGGER set_user_id_trigger ON main;
871 queryMain.exec(QStringLiteral("CREATE TRIGGER set_user_id_trigger BEFORE INSERT OR UPDATE ON main FOR EACH ROW EXECUTE PROCEDURE set_user_id();"));
872 }
873
874 QMap<QString, TMConfig> tmConfigCache;
875
setConfig(QSqlDatabase & db,const TMConfig & c)876 static void setConfig(QSqlDatabase& db, const TMConfig& c)
877 {
878 QSqlQuery query(db);
879 query.prepare(QStringLiteral("INSERT INTO tm_config (key, value) VALUES (?, ?)"));
880
881 query.addBindValue(0);
882 query.addBindValue(c.markup);
883 //qCDebug(LOKALIZE_LOG)<<"setting tm db config:"<<query.exec();
884 bool ok = query.exec();
885 if (!ok)
886 qCWarning(LOKALIZE_LOG) << "setting tm db config failed";
887
888 query.addBindValue(1);
889 query.addBindValue(c.accel);
890 query.exec();
891
892 query.addBindValue(2);
893 query.addBindValue(c.sourceLangCode);
894 query.exec();
895
896 query.addBindValue(3);
897 query.addBindValue(c.targetLangCode);
898 query.exec();
899
900 tmConfigCache[db.databaseName()] = c;
901 }
902
getConfig(QSqlDatabase & db,bool useCache=true)903 static TMConfig getConfig(QSqlDatabase& db, bool useCache = true) //int& emptyTargetId
904 {
905 if (useCache && tmConfigCache.contains(db.databaseName())) {
906 //qCDebug(LOKALIZE_LOG)<<"using config cache for"<<db.databaseName();
907 return tmConfigCache.value(db.databaseName());
908 }
909
910 QSqlQuery query(db);
911 bool ok = query.exec(QStringLiteral("SELECT key, value FROM tm_config ORDER BY key ASC"));
912 Project& p = *(Project::instance());
913 bool f = query.next();
914 TMConfig c;
915 c.markup = f ? query.value(1).toString() : p.markup();
916 c.accel = query.next() ? query.value(1).toString() : p.accel();
917 c.sourceLangCode = query.next() ? query.value(1).toString() : p.sourceLangCode();
918 c.targetLangCode = query.next() ? query.value(1).toString() : p.targetLangCode();
919 query.clear();
920
921 if (Q_UNLIKELY(!f)) //tmConfigCache[db.databaseName()]=c;
922 setConfig(db, c);
923
924 if (!ok)
925 qCWarning(LOKALIZE_LOG) << "accessing tm db config" << ok << "use cache:" << useCache << "lang:" << c.sourceLangCode << c.targetLangCode;
926
927 tmConfigCache.insert(db.databaseName(), c);
928 return c;
929 }
930
931
getStats(const QSqlDatabase & db,int & pairsCount,int & uniqueSourcesCount,int & uniqueTranslationsCount)932 static void getStats(const QSqlDatabase& db,
933 int& pairsCount,
934 int& uniqueSourcesCount,
935 int& uniqueTranslationsCount
936 )
937
938 {
939 QSqlQuery query(db);
940 if (!query.exec(QStringLiteral("SELECT count(id) FROM main"))
941 || !query.next()) {
942 qCWarning(LOKALIZE_LOG) << "getStats fail" << db.databaseName();
943 return;
944 }
945 pairsCount = query.value(0).toInt();
946 query.clear();
947
948 if (!query.exec(QStringLiteral("SELECT count(*) FROM source_strings"))
949 || !query.next()) {
950 qCWarning(LOKALIZE_LOG) << "getStats fail" << db.databaseName();
951 return;
952 }
953 uniqueSourcesCount = query.value(0).toInt();
954 query.clear();
955
956 if (!query.exec(QStringLiteral("SELECT count(*) FROM target_strings"))
957 || !query.next()) {
958 qCWarning(LOKALIZE_LOG) << "getStats fail" << db.databaseName();
959 return;
960 }
961 uniqueTranslationsCount = query.value(0).toInt();
962
963 query.clear();
964 }
965
OpenDBJob(const QString & name,DbType type,bool reconnect,const ConnectionParams & connParams)966 OpenDBJob::OpenDBJob(const QString& name, DbType type, bool reconnect, const ConnectionParams& connParams)
967 : QObject(), QRunnable()
968 , m_dbName(name)
969 , m_type(type)
970 , m_setParams(false)
971 , m_connectionSuccessful(false)
972 , m_reconnect(reconnect)
973 , m_connParams(connParams)
974 {
975 setAutoDelete(false);
976 //qCWarning(LOKALIZE_LOG)<<"OpenDBJob ctor"<<m_dbName;
977 }
978
run()979 void OpenDBJob::run()
980 {
981 QElapsedTimer a; a.start();
982 const QString connectionName = getConnectionName(m_dbName);
983 if (!QSqlDatabase::contains(connectionName) || m_reconnect) {
984 QThread::currentThread()->setPriority(QThread::IdlePriority);
985
986 if (m_type == TM::Local) {
987 QSqlDatabase db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connectionName);
988 QString dbFolder = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
989 QFileInfo fileInfo(dbFolder);
990 if (!fileInfo.exists(dbFolder)) fileInfo.absoluteDir().mkpath(fileInfo.fileName());
991 db.setDatabaseName(dbFolder + QLatin1Char('/') + m_dbName + TM_DATABASE_EXTENSION);
992 m_connectionSuccessful = db.open();
993 if (Q_UNLIKELY(!m_connectionSuccessful)) {
994 qCWarning(LOKALIZE_LOG) << "failed to open db" << db.databaseName() << db.lastError().text();
995 QSqlDatabase::removeDatabase(connectionName);
996 Q_EMIT done(this);
997 return;
998 }
999 if (!initSqliteDb(db)) { //need to recreate db ;(
1000 QString filename = db.databaseName();
1001 db.close();
1002 QSqlDatabase::removeDatabase(connectionName);
1003 qCWarning(LOKALIZE_LOG) << "We need to recreate the database " << filename;
1004 QFile::remove(filename);
1005
1006 db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connectionName);
1007 db.setDatabaseName(filename);
1008 m_connectionSuccessful = db.open() && initSqliteDb(db);
1009 if (!m_connectionSuccessful) {
1010 QSqlDatabase::removeDatabase(connectionName);
1011 Q_EMIT done(this);
1012 return;
1013 }
1014 }
1015 } else {
1016 if (QSqlDatabase::contains(connectionName)) { //reconnect is true
1017 QSqlDatabase::database(connectionName).close();
1018 QSqlDatabase::removeDatabase(connectionName);
1019 }
1020
1021 if (!m_connParams.isFilled()) {
1022 QFile rdb(QStandardPaths::writableLocation(QStandardPaths::DataLocation) + QLatin1Char('/') + m_dbName + REMOTETM_DATABASE_EXTENSION);
1023 if (!rdb.open(QIODevice::ReadOnly | QIODevice::Text)) {
1024 Q_EMIT done(this);
1025 return;
1026 }
1027
1028 QTextStream rdbParams(&rdb);
1029
1030 m_connParams.driver = rdbParams.readLine();
1031 m_connParams.host = rdbParams.readLine();
1032 m_connParams.db = rdbParams.readLine();
1033 m_connParams.user = rdbParams.readLine();
1034 m_connParams.passwd = rdbParams.readLine();
1035 }
1036
1037 QSqlDatabase db = QSqlDatabase::addDatabase(m_connParams.driver, connectionName);
1038 db.setHostName(m_connParams.host);
1039 db.setDatabaseName(m_connParams.db);
1040 db.setUserName(m_connParams.user);
1041 db.setPassword(m_connParams.passwd);
1042 m_connectionSuccessful = db.open();
1043 if (Q_UNLIKELY(!m_connectionSuccessful)) {
1044 QSqlDatabase::removeDatabase(connectionName);
1045 Q_EMIT done(this);
1046 return;
1047 }
1048 m_connParams.user = db.userName();
1049 initPgDb(db);
1050 }
1051
1052 }
1053 QSqlDatabase db = QSqlDatabase::database(connectionName);
1054 //if (!m_markup.isEmpty()||!m_accel.isEmpty())
1055 if (m_setParams)
1056 setConfig(db, m_tmConfig);
1057 else
1058 m_tmConfig = getConfig(db);
1059 qCWarning(LOKALIZE_LOG) << "db" << connectionName << "opened" << a.elapsed() << m_tmConfig.targetLangCode;
1060
1061 getStats(db, m_stat.pairsCount, m_stat.uniqueSourcesCount, m_stat.uniqueTranslationsCount);
1062
1063 if (m_type == TM::Local) {
1064 db.close();
1065 db.open();
1066 }
1067 Q_EMIT done(this);
1068 }
1069
1070
CloseDBJob(const QString & name)1071 CloseDBJob::CloseDBJob(const QString& name)
1072 : QObject(), QRunnable()
1073 , m_dbName(name)
1074 {
1075 setAutoDelete(false);
1076 }
1077
~CloseDBJob()1078 CloseDBJob::~CloseDBJob()
1079 {
1080 qCDebug(LOKALIZE_LOG) << "closedb dtor" << m_dbName;
1081 }
1082
run()1083 void CloseDBJob::run()
1084 {
1085 const QString connectionName = getConnectionName(m_dbName);
1086 if (connectionName.length())
1087 QSqlDatabase::removeDatabase(connectionName);
1088 qCDebug(LOKALIZE_LOG) << "closedb " << connectionName;
1089 Q_EMIT done(this);
1090 }
1091
1092
makeAcceledString(QString source,const QString & accel,const QVariant & accelPos)1093 static QString makeAcceledString(QString source, const QString& accel, const QVariant& accelPos)
1094 {
1095 if (accelPos.isNull())
1096 return source;
1097 int accelPosInt = accelPos.toInt();
1098 if (accelPosInt != -1)
1099 source.insert(accelPosInt, accel);
1100 return source;
1101 }
1102
1103
initSelectJob(Catalog * catalog,DocPosition pos,QString db,int opt)1104 SelectJob* TM::initSelectJob(Catalog* catalog, DocPosition pos, QString db, int opt)
1105 {
1106 SelectJob* job = new SelectJob(catalog->sourceWithTags(pos),
1107 catalog->context(pos.entry).first(),
1108 catalog->url(),
1109 pos,
1110 db.isEmpty() ? Project::instance()->projectID() : db);
1111 if (opt & Enqueue) {
1112 //deletion should be done by receiver, e.g. slotSuggestionsCame()
1113 threadPool()->start(job, SELECT);
1114 }
1115 return job;
1116 }
1117
SelectJob(const CatalogString & source,const QString & ctxt,const QString & file,const DocPosition & pos,const QString & dbName)1118 SelectJob::SelectJob(const CatalogString& source,
1119 const QString& ctxt,
1120 const QString& file,
1121 const DocPosition& pos,
1122 const QString& dbName)
1123 : QObject(), QRunnable()
1124 , m_source(source)
1125 , m_ctxt(ctxt)
1126 , m_file(file)
1127 , m_dequeued(false)
1128 , m_pos(pos)
1129 , m_dbName(dbName)
1130 {
1131 setAutoDelete(false);
1132 //qCDebug(LOKALIZE_LOG)<<"selectjob"<<dbName<<m_source.string;
1133 }
1134
~SelectJob()1135 SelectJob::~SelectJob()
1136 {
1137 //qCDebug(LOKALIZE_LOG)<<m_source.string;
1138 }
1139
invertMap(const QMap<qlonglong,uint> & source)1140 inline QMap<uint, qlonglong> invertMap(const QMap<qlonglong, uint>& source)
1141 {
1142 //uses the fact that map has its keys always sorted
1143 QMap<uint, qlonglong> sortingMap;
1144 for (QMap<qlonglong, uint>::const_iterator i = source.constBegin(); i != source.constEnd(); ++i) {
1145 sortingMap.insertMulti(i.value(), i.key());
1146 }
1147 return sortingMap;
1148 }
1149
1150 //returns true if seen translation with >85%
doSelect(QSqlDatabase & db,QStringList & words,bool isShort)1151 bool SelectJob::doSelect(QSqlDatabase& db,
1152 QStringList& words,
1153 //QList<TMEntry>& entries,
1154 bool isShort)
1155 {
1156 bool qpsql = (db.driverName() == QLatin1String("QPSQL"));
1157 QMap<qlonglong, uint> occurencies;
1158 QVector<qlonglong> idsForWord;
1159
1160 QSqlQuery queryWords(db);
1161 //TODO ??? not sure. make another loop before to create QList< QList<qlonglong> > then reorder it by size
1162 static const QString queryC[] = {U("SELECT ids_long FROM words WHERE word='%1'"),
1163 U("SELECT ids_short FROM words WHERE word='%1'")
1164 };
1165 QString queryString = queryC[isShort];
1166
1167 //for each word...
1168 int o = words.size();
1169 while (--o >= 0) {
1170 //if this is not the first word occurrence, just readd ids for it
1171 if (!(!idsForWord.isEmpty() && words.at(o) == words.at(o + 1))) {
1172 idsForWord.clear();
1173 queryWords.exec(queryString.arg(words.at(o)));
1174 if (Q_UNLIKELY(!queryWords.exec(queryString.arg(words.at(o)))))
1175 qCWarning(LOKALIZE_LOG) << "select error: " << queryWords.lastError().text() << Qt::endl;
1176
1177 if (queryWords.next()) {
1178 QByteArray arr(queryWords.value(0).toByteArray());
1179 queryWords.clear();
1180
1181 QList<QByteArray> ids(arr.split(' '));
1182 int p = ids.size();
1183 idsForWord.reserve(p);
1184 while (--p >= 0)
1185 idsForWord.append(ids.at(p).toLongLong(/*bool ok*/nullptr, 36));
1186 } else {
1187 queryWords.clear();
1188 continue;
1189 }
1190 }
1191
1192 //qCWarning(LOKALIZE_LOG) <<"SelectJob: idsForWord.size() "<<idsForWord.size()<<endl;
1193
1194 //iterate over ids: this computes hit count for each id
1195 for (QVector<qlonglong>::const_iterator i = idsForWord.constBegin(); i != idsForWord.constEnd(); i++)
1196 occurencies[*i]++; //0 is default value
1197 }
1198
1199 //accels are removed
1200 TMConfig c = getConfig(db);
1201 QString tmp = c.markup;
1202 if (!c.markup.isEmpty())
1203 tmp += '|';
1204 QRegExp rxSplit(QLatin1Char('(') + tmp + QStringLiteral("\\W+|\\d+)+"));
1205
1206 QString sourceClean(m_source.string);
1207 sourceClean.remove(c.accel);
1208 //split m_english for use in wordDiff later--all words are needed so we cant use list we already have
1209 QStringList englishList(sourceClean.toLower().split(rxSplit, Qt::SkipEmptyParts));
1210 static QRegExp delPart(QStringLiteral("<KBABELDEL>*</KBABELDEL>"), Qt::CaseSensitive, QRegExp::Wildcard);
1211 static QRegExp addPart(QStringLiteral("<KBABELADD>*</KBABELADD>"), Qt::CaseSensitive, QRegExp::Wildcard);
1212 delPart.setMinimal(true);
1213 addPart.setMinimal(true);
1214
1215 //QList<uint> concordanceLevels=sortedUniqueValues(occurencies); //we start from entries with higher word-concordance level
1216 QMap<uint, qlonglong> concordanceLevelToIds = invertMap(occurencies);
1217 if (concordanceLevelToIds.isEmpty())
1218 return false;
1219 bool seen85 = false;
1220 int limit = 200;
1221 auto clit = concordanceLevelToIds.constEnd();
1222 if (concordanceLevelToIds.size()) --clit;
1223 if (concordanceLevelToIds.size()) while (--limit >= 0) {
1224 if (Q_UNLIKELY(m_dequeued))
1225 break;
1226
1227 //for every concordance level
1228 qlonglong level = clit.key();
1229 QString joined;
1230 while (level == clit.key()) {
1231 joined += QString::number(clit.value()) + ',';
1232 if (clit == concordanceLevelToIds.constBegin() || --limit < 0)
1233 break;
1234 --clit;
1235 }
1236 joined.chop(1);
1237
1238 //get records containing current word
1239 QSqlQuery queryFetch(U(
1240 "SELECT id, source, source_accel, source_markup FROM source_strings WHERE "
1241 "source_strings.id IN (") + joined + ')', db);
1242 TMEntry e;
1243 while (queryFetch.next()) {
1244 e.id = queryFetch.value(0).toLongLong();
1245 if (queryFetch.value(3).toByteArray().size())
1246 qCDebug(LOKALIZE_LOG) << "BA" << queryFetch.value(3).toByteArray();
1247 e.source = CatalogString(makeAcceledString(queryFetch.value(1).toString(), c.accel, queryFetch.value(2)),
1248 queryFetch.value(3).toByteArray());
1249 if (e.source.string.contains(TAGRANGE_IMAGE_SYMBOL)) {
1250 if (!e.source.tags.size())
1251 qCWarning(LOKALIZE_LOG) << "problem:" << queryFetch.value(3).toByteArray().size() << queryFetch.value(3).toByteArray();
1252 }
1253 //e.target=queryFetch.value(2).toString();
1254 //QStringList e_ctxt=queryFetch.value(3).toString().split('\b',Qt::SkipEmptyParts);
1255 //e.date=queryFetch.value(4).toString();
1256 e.markupExpr = c.markup;
1257 e.accelExpr = c.accel;
1258 e.dbName = db.connectionName();
1259
1260
1261 //BEGIN calc score
1262 QString str = e.source.string;
1263 str.remove(c.accel);
1264
1265 QStringList englishSuggList(str.toLower().split(rxSplit, Qt::SkipEmptyParts));
1266 if (englishSuggList.size() > 10 * englishList.size())
1267 continue;
1268 //sugg is 'old' --translator has to adapt its translation to 'new'--current
1269 QString result = wordDiff(englishSuggList, englishList);
1270 //qCWarning(LOKALIZE_LOG) <<"SelectJob: doin "<<j<<" "<<result;
1271
1272 int pos = 0;
1273 int delSubStrCount = 0;
1274 int delLen = 0;
1275 while ((pos = delPart.indexIn(result, pos)) != -1) {
1276 //qCWarning(LOKALIZE_LOG) <<"SelectJob: match del "<<delPart.cap(0);
1277 delLen += delPart.matchedLength() - 23;
1278 ++delSubStrCount;
1279 pos += delPart.matchedLength();
1280 }
1281 pos = 0;
1282 int addSubStrCount = 0;
1283 int addLen = 0;
1284 while ((pos = addPart.indexIn(result, pos)) != -1) {
1285 addLen += addPart.matchedLength() - 23;
1286 ++addSubStrCount;
1287 pos += addPart.matchedLength();
1288 }
1289
1290 //allLen - length of suggestion
1291 int allLen = result.size() - 23 * addSubStrCount - 23 * delSubStrCount;
1292 int commonLen = allLen - delLen - addLen;
1293 //now, allLen is the length of the string being translated
1294 allLen = m_source.string.size();
1295 bool possibleExactMatch = !(delLen + addLen);
1296 if (!possibleExactMatch) {
1297 //del is better than add
1298 if (addLen) {
1299 //qCWarning(LOKALIZE_LOG) <<"SelectJob: addLen:"<<addLen<<" "<<9500*(pow(float(commonLen)/float(allLen),0.20))<<" / "
1300 //<<pow(float(addLen*addSubStrCount),0.2)<<" "
1301 //<<endl;
1302
1303 float score = 9500 * (pow(float(commonLen) / float(allLen), 0.12f)) //this was < 1 so we have increased it
1304 //this was > 1 so we have decreased it, and increased result:
1305 / exp(0.014 * float(addLen) * log10(3.0f + addSubStrCount));
1306
1307 if (delLen) {
1308 //qCWarning(LOKALIZE_LOG) <<"SelectJob: delLen:"<<delLen<<" / "
1309 //<<pow(float(delLen*delSubStrCount),0.1)<<" "
1310 //<<endl;
1311
1312 float a = exp(0.008 * float(delLen) * log10(3.0f + delSubStrCount));
1313
1314 if (a != 0.0)
1315 score /= a;
1316 }
1317 e.score = (int)score;
1318
1319 } else { //==to adapt, only deletion is needed
1320 //qCWarning(LOKALIZE_LOG) <<"SelectJob: b "<<int(pow(float(delLen*delSubStrCount),0.10));
1321 float score = 9900 * (pow(float(commonLen) / float(allLen), 0.15f))
1322 / exp(0.008 * float(delLen) * log10(3.0f + delSubStrCount));
1323 e.score = (int)score;
1324 }
1325 } else
1326 e.score = 10000;
1327
1328 //END calc score
1329 if (e.score < 3500)
1330 continue;
1331 seen85 = seen85 || e.score > 8500;
1332 if (seen85 && e.score < 6000)
1333 continue;
1334
1335 if (e.score < Settings::suggScore() * 100)
1336 continue;
1337 //BEGIN fetch rest of the data
1338 QString change_author_str;
1339 QString authors_table_str;
1340 if (qpsql) {
1341 //change_author_str=", main.change_author ";
1342 change_author_str = QStringLiteral(", pg_user.usename ");
1343 authors_table_str = QStringLiteral(" JOIN pg_user ON (pg_user.usesysid=main.change_author) ");
1344 }
1345
1346 QSqlQuery queryRest(U(
1347 "SELECT main.id, main.date, main.ctxt, main.bits, "
1348 "target_strings.target, target_strings.target_accel, target_strings.target_markup, "
1349 "files.path, main.change_date ") + change_author_str + U(
1350 "FROM main JOIN target_strings ON (target_strings.id=main.target) JOIN files ON (files.id=main.file) ")
1351 + authors_table_str + U("WHERE "
1352 "main.source=") + QString::number(e.id) + U(" AND "
1353 "(main.bits&4)!=4 AND "
1354 "target_strings.target NOTNULL")
1355 , db); //ORDER BY tm_main.id ?
1356 queryRest.exec();
1357 //qCDebug(LOKALIZE_LOG)<<"main select error"<<queryRest.lastError().text();
1358 QMap<TMEntry, bool> sortedEntryList; //to eliminate same targets from different files
1359 while (queryRest.next()) {
1360 e.id = queryRest.value(0).toLongLong();
1361 e.date = queryRest.value(1).toDate();
1362 e.ctxt = queryRest.value(2).toString();
1363 e.target = CatalogString(makeAcceledString(queryRest.value(4).toString(), c.accel, queryRest.value(5)),
1364 queryRest.value(6).toByteArray());
1365
1366 QStringList matchData = queryRest.value(2).toString().split(TM_DELIMITER, Qt::KeepEmptyParts); //context|plural
1367 e.file = queryRest.value(7).toString();
1368 if (e.target.isEmpty())
1369 continue;
1370
1371 e.obsolete = queryRest.value(3).toInt() & 1;
1372
1373 e.changeDate = queryRest.value(8).toDate();
1374 if (qpsql)
1375 e.changeAuthor = queryRest.value(9).toString();
1376
1377 //BEGIN exact match score++
1378 if (possibleExactMatch) { //"exact" match (case insensitive+w/o non-word characters!)
1379 if (m_source.string == e.source.string)
1380 e.score = 10000;
1381 else
1382 e.score = 9900;
1383 }
1384 if (!m_ctxt.isEmpty() && matchData.size() > 0) { //check not needed?
1385 if (matchData.at(0) == m_ctxt)
1386 e.score += 33;
1387 }
1388 //qCWarning(LOKALIZE_LOG)<<"m_pos"<<QString::number(m_pos.form);
1389 // bool pluralMatches=false;
1390 if (matchData.size() > 1) {
1391 int form = matchData.at(1).toInt();
1392
1393 //pluralMatches=(form&&form==m_pos.form);
1394 if (form && form == (int)m_pos.form) {
1395 //qCWarning(LOKALIZE_LOG)<<"this"<<matchData.at(1);
1396 e.score += 33;
1397 }
1398 }
1399 if (e.file == m_file)
1400 e.score += 33;
1401 //END exact match score++
1402 //qCWarning(LOKALIZE_LOG)<<"appending"<<e.target;
1403 sortedEntryList.insertMulti(e, false);
1404 }
1405 queryRest.clear();
1406 //eliminate same targets from different files
1407 QHash<QString, int> hash;
1408 int oldCount = m_entries.size();
1409 QMap<TMEntry, bool>::const_iterator it = sortedEntryList.constEnd();
1410 if (sortedEntryList.size()) while (true) {
1411 --it;
1412 const TMEntry& e = it.key();
1413 int& hits = hash[e.target.string];
1414 if (!hits) //0 was default value
1415 m_entries.append(e);
1416 hits++;
1417 if (it == sortedEntryList.constBegin())
1418 break;
1419 }
1420 for (int i = oldCount; i < m_entries.size(); ++i)
1421 m_entries[i].hits = hash.value(m_entries.at(i).target.string);
1422 //END fetch rest of the data
1423 }
1424 queryFetch.clear();
1425 if (clit == concordanceLevelToIds.constBegin())
1426 break;
1427 if (seen85) limit = qMin(limit, 100); //be more restrictive for the next concordance levels
1428 }
1429 return seen85;
1430 }
1431
run()1432 void SelectJob::run()
1433 {
1434 const QString connectionName = getConnectionName(m_dbName);
1435 //qCDebug(LOKALIZE_LOG)<<"select started"<<m_dbName<<m_source.string;
1436 if (m_source.isEmpty() || stop) { //sanity check
1437 Q_EMIT done(this);
1438 return;
1439 }
1440 //thread()->setPriority(QThread::IdlePriority);
1441 // QTime a; a.start();
1442
1443 if (Q_UNLIKELY(!QSqlDatabase::contains(connectionName))) {
1444 Q_EMIT done(this);
1445 return;
1446 }
1447 QSqlDatabase db = QSqlDatabase::database(connectionName);
1448 if (Q_UNLIKELY(!db.isValid() || !db.isOpen())) {
1449 Q_EMIT done(this);
1450 return;
1451 }
1452 //qCDebug(LOKALIZE_LOG)<<"select started 2"<<m_dbName<<m_source.string;
1453
1454 TMConfig c = getConfig(db);
1455 QRegExp rxClean1(c.markup); rxClean1.setMinimal(true);
1456
1457 QString cleanSource = m_source.string;
1458 QStringList words;
1459 doSplit(cleanSource, words, rxClean1, c.accel);
1460 if (Q_UNLIKELY(words.isEmpty())) {
1461 Q_EMIT done(this);
1462 return;
1463 }
1464 std::sort(words.begin(), words.end());//to speed up if some words occur multiple times
1465
1466 bool isShort = words.size() < 20;
1467
1468 if (!doSelect(db, words, isShort))
1469 doSelect(db, words, !isShort);
1470
1471 //qCWarning(LOKALIZE_LOG) <<"SelectJob: done "<<a.elapsed()<<m_entries.size();
1472 std::sort(m_entries.begin(), m_entries.end(), std::greater<TMEntry>());
1473 const int limit = qMin(Settings::suggCount(), m_entries.size());
1474 const int minScore = Settings::suggScore() * 100;
1475 int i = m_entries.size() - 1;
1476 while (i >= 0 && (i >= limit || m_entries.last().score < minScore)) {
1477 m_entries.removeLast();
1478 i--;
1479 }
1480
1481 if (Q_UNLIKELY(m_dequeued)) {
1482 Q_EMIT done(this);
1483 return;
1484 }
1485
1486 ++i;
1487 while (--i >= 0) {
1488 m_entries[i].accelExpr = c.accel;
1489 m_entries[i].markupExpr = c.markup;
1490 m_entries[i].diff = userVisibleWordDiff(m_entries.at(i).source.string,
1491 m_source.string,
1492 m_entries.at(i).accelExpr,
1493 m_entries.at(i).markupExpr);
1494 }
1495 Q_EMIT done(this);
1496 }
1497
1498
1499
1500
1501
1502
ScanJob(const QString & filePath,const QString & dbName)1503 ScanJob::ScanJob(const QString& filePath, const QString& dbName)
1504 : QRunnable()
1505 , m_filePath(filePath)
1506 , m_time(0)
1507 , m_added(0)
1508 , m_newVersions(0)
1509 , m_size(0)
1510 , m_dbName(dbName)
1511 {
1512 qCDebug(LOKALIZE_LOG) << m_dbName << m_filePath;
1513 }
1514
run()1515 void ScanJob::run()
1516 {
1517 const QString connectionName = getConnectionName(m_dbName);
1518 if (stop || !QSqlDatabase::contains(connectionName)) {
1519 return;
1520 }
1521 qCDebug(LOKALIZE_LOG) << "scan job started for" << m_filePath << m_dbName << stop << m_dbName;
1522 //QThread::currentThread()->setPriority(QThread::IdlePriority);
1523 QElapsedTimer a; a.start();
1524
1525 QSqlDatabase db = QSqlDatabase::database(connectionName);
1526 if (!db.isOpen())
1527 return;
1528 //initSqliteDb(db);
1529 TMConfig c = getConfig(db, true);
1530 QRegExp rxClean1(c.markup); rxClean1.setMinimal(true);
1531
1532 Catalog catalog(nullptr);
1533 if (Q_LIKELY(catalog.loadFromUrl(m_filePath, QString(), &m_size, /*no auto save*/true) == 0)) {
1534 if (c.targetLangCode != catalog.targetLangCode()) {
1535 qCWarning(LOKALIZE_LOG) << "not indexing file because target languages don't match:" << c.targetLangCode << "in TM vs" << catalog.targetLangCode() << "in file";
1536 return;
1537 }
1538 qlonglong priorId = -1;
1539
1540 QSqlQuery queryBegin(QStringLiteral("BEGIN"), db);
1541 //qCWarning(LOKALIZE_LOG) <<"queryBegin error: " <<queryBegin.lastError().text();
1542
1543 qlonglong fileId = getFileId(m_filePath, db);
1544 //mark everything as obsolete
1545 queryBegin.exec(QStringLiteral("UPDATE main SET bits=(bits|1) WHERE file=") + QString::number(fileId));
1546 //qCWarning(LOKALIZE_LOG) <<"UPDATE error: " <<queryBegin.lastError().text();
1547
1548 int numberOfEntries = catalog.numberOfEntries();
1549 DocPosition pos(0);
1550 for (; pos.entry < numberOfEntries; pos.entry++) {
1551 bool ok = true;
1552 if (catalog.isPlural(pos.entry)) {
1553 DocPosition ppos = pos;
1554 for (ppos.form = 0; ppos.form < catalog.numberOfPluralForms(); ++ppos.form) {
1555 /*
1556 QString target;
1557 if ( catalog.isApproved(i) && !catalog.isUntranslated(pos))
1558 target=catalog.target(pos);
1559 */
1560 ok = ok && doInsertEntry(catalog.sourceWithTags(ppos),
1561 catalog.targetWithTags(ppos),
1562 catalog.context(ppos).first() + TM_DELIMITER + QString::number(ppos.form),
1563 catalog.isApproved(ppos),
1564 fileId, db, rxClean1, c.accel, priorId, priorId);
1565 }
1566 } else {
1567 /*
1568 QString target;
1569 if ( catalog.isApproved(i) && !catalog.isUntranslated(i))
1570 target=catalog.target(i);
1571 */
1572 ok = doInsertEntry(catalog.sourceWithTags(pos),
1573 catalog.targetWithTags(pos),
1574 catalog.context(pos).first(),
1575 catalog.isApproved(pos),
1576 fileId, db, rxClean1, c.accel, priorId, priorId);
1577 }
1578 if (Q_LIKELY(ok))
1579 ++m_added;
1580 }
1581 QSqlQuery queryEnd(QStringLiteral("END"), db);
1582 qCDebug(LOKALIZE_LOG) << "ScanJob: done " << a.elapsed() << "new source entries:" << newTMSourceEntryCount << "reused:" << reusedTMSourceEntryCount;
1583 }
1584 //qCWarning(LOKALIZE_LOG) <<"Done scanning "<<m_url.prettyUrl();
1585 m_time = a.elapsed();
1586 }
1587
RemoveMissingFilesJob(const QString & dbName)1588 RemoveMissingFilesJob::RemoveMissingFilesJob(const QString& dbName)
1589 : QObject(), QRunnable()
1590 , m_dbName(dbName)
1591 {
1592 qCDebug(LOKALIZE_LOG) << "removingmissingfiles" << m_dbName;
1593 }
1594
1595
~RemoveMissingFilesJob()1596 RemoveMissingFilesJob::~RemoveMissingFilesJob()
1597 {
1598 qCDebug(LOKALIZE_LOG) << "removingmissingfilesjob dtor" << m_dbName;
1599 }
1600
1601
1602
run()1603 void RemoveMissingFilesJob::run()
1604 {
1605 const QString connectionName = getConnectionName(m_dbName);
1606 // qCDebug(LOKALIZE_LOG)<<m_dbName;
1607 QSqlDatabase db = QSqlDatabase::database(connectionName);
1608
1609 doRemoveMissingFiles(db, m_dbName, this);
1610
1611 Q_EMIT done();
1612 }
1613
RemoveFileJob(const QString & filePath,const QString & dbName,QObject * parent)1614 RemoveFileJob::RemoveFileJob(const QString& filePath, const QString& dbName, QObject *parent)
1615 : QObject(), QRunnable()
1616 , m_filePath(filePath)
1617 , m_dbName(dbName)
1618 , m_parent(parent)
1619 {
1620 qCDebug(LOKALIZE_LOG) << "removingfile" << m_dbName << m_filePath;
1621 }
1622
1623
~RemoveFileJob()1624 RemoveFileJob::~RemoveFileJob()
1625 {
1626 qCDebug(LOKALIZE_LOG) << "removingfilejob dtor" << m_dbName << m_filePath;
1627 }
1628
1629
1630
run()1631 void RemoveFileJob::run()
1632 {
1633 const QString connectionName = getConnectionName(m_dbName);
1634 // qCDebug(LOKALIZE_LOG)<<m_dbName;
1635 QSqlDatabase db = QSqlDatabase::database(connectionName);
1636
1637 if (!doRemoveFile(m_filePath, db)) {
1638 qCWarning(LOKALIZE_LOG) << "error while removing file" << m_dbName << m_filePath;
1639 }
1640
1641 Q_EMIT done();
1642 }
1643
1644
RemoveJob(const TMEntry & entry)1645 RemoveJob::RemoveJob(const TMEntry& entry)
1646 : QObject(), QRunnable()
1647 , m_entry(entry)
1648 {
1649 //RemoveJob instances are deleted automatically because their signal does not contain pointer to the job
1650 qCDebug(LOKALIZE_LOG) << "removing" << m_entry.dbName << m_entry.source.string << m_entry.target.string;
1651 }
1652
~RemoveJob()1653 RemoveJob::~RemoveJob()
1654 {
1655 qCDebug(LOKALIZE_LOG) << "removejob dtor" << m_entry.dbName << m_entry.source.string << m_entry.target.string;
1656 }
1657
1658
run()1659 void RemoveJob::run()
1660 {
1661 // qCDebug(LOKALIZE_LOG)<<m_entry.dbName;
1662 QSqlDatabase db = QSqlDatabase::database(m_entry.dbName);
1663
1664 //cleaning regexps for word index update
1665 TMConfig c = getConfig(db);
1666 QRegExp rxClean1(c.markup); rxClean1.setMinimal(true);
1667
1668 if (!doRemoveEntry(m_entry.id, rxClean1, c.accel, db))
1669 qCWarning(LOKALIZE_LOG) << "error removing entry" << m_entry.dbName << m_entry.source.string << m_entry.target.string;
1670
1671 Q_EMIT done();
1672 }
1673
1674
UpdateJob(const QString & filePath,const QString & ctxt,const CatalogString & english,const CatalogString & newTarget,int form,bool approved,const QString & dbName)1675 UpdateJob::UpdateJob(const QString& filePath,
1676 const QString& ctxt,
1677 const CatalogString& english,
1678 const CatalogString& newTarget,
1679 //const DocPosition&,//for back tracking
1680 int form,
1681 bool approved,
1682 const QString& dbName)
1683 : QRunnable()
1684 , m_filePath(filePath)
1685 , m_ctxt(ctxt)
1686 , m_english(english)
1687 , m_newTarget(newTarget)
1688 , m_form(form)
1689 , m_approved(approved)
1690 , m_dbName(dbName)
1691 {
1692 qCDebug(LOKALIZE_LOG) << m_english.string << m_newTarget.string;
1693 }
1694
run()1695 void UpdateJob::run()
1696 {
1697 const QString connectionName = getConnectionName(m_dbName);
1698 qCDebug(LOKALIZE_LOG) << "UpdateJob run" << m_english.string << m_newTarget.string;
1699 QSqlDatabase db = QSqlDatabase::database(connectionName);
1700
1701 //cleaning regexps for word index update
1702 TMConfig c = getConfig(db);
1703 QRegExp rxClean1(c.markup); rxClean1.setMinimal(true);
1704
1705
1706 qlonglong fileId = getFileId(m_filePath, db);
1707
1708 if (m_form != -1)
1709 m_ctxt += TM_DELIMITER + QString::number(m_form);
1710
1711 QSqlQuery queryBegin(QStringLiteral("BEGIN"), db);
1712 qlonglong priorId = -1;
1713 doInsertEntry(m_english, m_newTarget,
1714 m_ctxt, //TODO QStringList -- after XLIFF
1715 m_approved, fileId, db, rxClean1, c.accel, priorId, priorId);
1716 QSqlQuery queryEnd(QStringLiteral("END"), db);
1717 }
1718
1719
1720
1721 //BEGIN TMX
1722
1723 #include <QXmlDefaultHandler>
1724 #include <QXmlSimpleReader>
1725
1726 /**
1727 @author Nick Shaforostoff <shafff@ukr.net>
1728 */
1729 class TmxParser : public QXmlDefaultHandler
1730 {
1731 enum State { //localstate for getting chars into right place
1732 null = 0,
1733 seg,
1734 propContext,
1735 propFile,
1736 propPluralForm,
1737 propApproved
1738 };
1739
1740 enum Lang {
1741 Source,
1742 Target,
1743 Null
1744 };
1745
1746 public:
1747 TmxParser(const QString& dbName);
1748 ~TmxParser();
1749
1750 private:
1751 bool startDocument() override;
1752 bool startElement(const QString&, const QString&, const QString&, const QXmlAttributes&) override;
1753 bool endElement(const QString&, const QString&, const QString&) override;
1754 bool characters(const QString&) override;
1755
1756 private:
1757 QSqlDatabase db;
1758 QRegExp rxClean1;
1759 QString accel;
1760
1761 int m_hits;
1762 CatalogString m_segment[3]; //Lang enum
1763 QList<InlineTag> m_inlineTags;
1764 QString m_context;
1765 QString m_pluralForm;
1766 QString m_filePath;
1767 QString m_approvedString;
1768
1769 State m_state: 8;
1770 Lang m_lang: 8;
1771
1772 ushort m_added;
1773
1774
1775 QMap<QString, qlonglong> m_fileIds;
1776 QString m_dbLangCode;
1777 };
1778
1779
TmxParser(const QString & dbName)1780 TmxParser::TmxParser(const QString& dbName)
1781 : m_hits(0)
1782 , m_state(null)
1783 , m_lang(Null)
1784 , m_added(0)
1785 , m_dbLangCode(Project::instance()->langCode().toLower())
1786 {
1787 db = QSqlDatabase::database(dbName);
1788
1789 TMConfig c = getConfig(db);
1790 rxClean1.setPattern(c.markup); rxClean1.setMinimal(true);
1791 accel = c.accel;
1792 }
1793
startDocument()1794 bool TmxParser::startDocument()
1795 {
1796 //initSqliteDb(db);
1797 m_fileIds.clear();
1798
1799 QSqlQuery queryBegin(QLatin1String("BEGIN"), db);
1800
1801 m_state = null;
1802 m_lang = Null;
1803 return true;
1804 }
1805
1806
~TmxParser()1807 TmxParser::~TmxParser()
1808 {
1809 QSqlQuery queryEnd(QLatin1String("END"), db);
1810 }
1811
1812
startElement(const QString &,const QString &,const QString & qName,const QXmlAttributes & attr)1813 bool TmxParser::startElement(const QString&, const QString&,
1814 const QString& qName,
1815 const QXmlAttributes& attr)
1816 {
1817 if (qName == QLatin1String("tu")) {
1818 bool ok;
1819 m_hits = attr.value(QLatin1String("usagecount")).toInt(&ok);
1820 if (!ok)
1821 m_hits = -1;
1822
1823 m_segment[Source].clear();
1824 m_segment[Target].clear();
1825 m_context.clear();
1826 m_pluralForm.clear();
1827 m_filePath.clear();
1828 m_approvedString.clear();
1829
1830 } else if (qName == QLatin1String("tuv")) {
1831 QString attrLang = attr.value(QStringLiteral("xml:lang")).toLower();
1832 if (attrLang == QLatin1String("en")) //TODO startsWith?
1833 m_lang = Source;
1834 else if (attrLang == m_dbLangCode)
1835 m_lang = Target;
1836 else {
1837 qCWarning(LOKALIZE_LOG) << "skipping lang" << attr.value("xml:lang");
1838 m_lang = Null;
1839 }
1840 } else if (qName == QLatin1String("prop")) {
1841 QString attrType = attr.value(QStringLiteral("type")).toLower();
1842 if (attrType == QLatin1String("x-context"))
1843 m_state = propContext;
1844 else if (attrType == QLatin1String("x-file"))
1845 m_state = propFile;
1846 else if (attrType == QLatin1String("x-pluralform"))
1847 m_state = propPluralForm;
1848 else if (attrType == QLatin1String("x-approved"))
1849 m_state = propApproved;
1850 else
1851 m_state = null;
1852 } else if (qName == QLatin1String("seg")) {
1853 m_state = seg;
1854 } else if (m_state == seg && m_lang != Null) {
1855 InlineTag::InlineElement t = InlineTag::getElementType(qName.toLatin1());
1856 if (t != InlineTag::_unknown) {
1857 m_segment[m_lang].string += QChar(TAGRANGE_IMAGE_SYMBOL);
1858 int pos = m_segment[m_lang].string.size();
1859 m_inlineTags.append(InlineTag(pos, pos, t, attr.value(QStringLiteral("id"))));
1860 }
1861 }
1862 return true;
1863 }
1864
endElement(const QString &,const QString &,const QString & qName)1865 bool TmxParser::endElement(const QString&, const QString&, const QString& qName)
1866 {
1867 if (qName == QLatin1String("tu")) {
1868 if (m_filePath.isEmpty())
1869 m_filePath = QLatin1String("tmx-import");
1870 if (!m_fileIds.contains(m_filePath))
1871 m_fileIds.insert(m_filePath, getFileId(m_filePath, db));
1872 qlonglong fileId = m_fileIds.value(m_filePath);
1873
1874 if (!m_pluralForm.isEmpty())
1875 m_context += TM_DELIMITER + m_pluralForm;
1876
1877 qlonglong priorId = -1;
1878 bool ok = doInsertEntry(m_segment[Source],
1879 m_segment[Target],
1880 m_context,
1881 m_approvedString != QLatin1String("no"),
1882 fileId, db, rxClean1, accel, priorId, priorId);
1883 if (Q_LIKELY(ok))
1884 ++m_added;
1885 } else if (m_state == seg && m_lang != Null) {
1886 InlineTag::InlineElement t = InlineTag::getElementType(qName.toLatin1());
1887 if (t != InlineTag::_unknown) {
1888 InlineTag tag = m_inlineTags.takeLast();
1889 qCWarning(LOKALIZE_LOG) << qName << tag.getElementName();
1890
1891 if (tag.isPaired()) {
1892 tag.end = m_segment[m_lang].string.size();
1893 m_segment[m_lang].string += QChar(TAGRANGE_IMAGE_SYMBOL);
1894 }
1895 m_segment[m_lang].tags.append(tag);
1896 }
1897 }
1898 m_state = null;
1899 return true;
1900 }
1901
1902
1903
characters(const QString & ch)1904 bool TmxParser::characters(const QString& ch)
1905 {
1906 if (m_state == seg && m_lang != Null)
1907 m_segment[m_lang].string += ch;
1908 else if (m_state == propFile)
1909 m_filePath += ch;
1910 else if (m_state == propContext)
1911 m_context += ch;
1912 else if (m_state == propPluralForm)
1913 m_pluralForm += ch;
1914 else if (m_state == propApproved)
1915 m_approvedString += ch;
1916
1917 return true;
1918 }
1919
1920
1921
1922
1923
ImportTmxJob(const QString & filename,const QString & dbName)1924 ImportTmxJob::ImportTmxJob(const QString& filename, const QString& dbName)
1925 : QRunnable()
1926 , m_filename(filename)
1927 , m_time(0)
1928 , m_dbName(dbName)
1929 {
1930 }
1931
~ImportTmxJob()1932 ImportTmxJob::~ImportTmxJob()
1933 {
1934 qCDebug(LOKALIZE_LOG) << "ImportTmxJob dtor";
1935 }
1936
run()1937 void ImportTmxJob::run()
1938 {
1939 QElapsedTimer a; a.start();
1940
1941 QFile file(m_filename);
1942 if (!file.open(QFile::ReadOnly | QFile::Text))
1943 return;
1944
1945 TmxParser parser(m_dbName);
1946 QXmlSimpleReader reader;
1947 reader.setContentHandler(&parser);
1948
1949 QXmlInputSource xmlInputSource(&file);
1950 if (!reader.parse(xmlInputSource))
1951 qCWarning(LOKALIZE_LOG) << "failed to load" << m_filename;
1952
1953 //qCWarning(LOKALIZE_LOG) <<"Done scanning "<<m_url.prettyUrl();
1954 m_time = a.elapsed();
1955 }
1956
1957
1958
1959
1960 #include <QXmlStreamWriter>
1961
ExportTmxJob(const QString & filename,const QString & dbName)1962 ExportTmxJob::ExportTmxJob(const QString& filename, const QString& dbName)
1963 : QRunnable()
1964 , m_filename(filename)
1965 , m_time(0)
1966 , m_dbName(dbName)
1967 {
1968 }
1969
~ExportTmxJob()1970 ExportTmxJob::~ExportTmxJob()
1971 {
1972 qCDebug(LOKALIZE_LOG) << "ExportTmxJob dtor";
1973 }
1974
run()1975 void ExportTmxJob::run()
1976 {
1977 const QString connectionName = getConnectionName(m_dbName);
1978 QElapsedTimer a; a.start();
1979
1980 QFile out(m_filename);
1981 if (!out.open(QFile::WriteOnly | QFile::Text))
1982 return;
1983
1984 QXmlStreamWriter xmlOut(&out);
1985 xmlOut.setAutoFormatting(true);
1986 xmlOut.writeStartDocument(QStringLiteral("1.0"));
1987
1988
1989
1990 xmlOut.writeStartElement(QStringLiteral("tmx"));
1991 xmlOut.writeAttribute(QStringLiteral("version"), QStringLiteral("2.0"));
1992
1993 xmlOut.writeStartElement(QStringLiteral("header"));
1994 xmlOut.writeAttribute(QStringLiteral("creationtool"), QStringLiteral("lokalize"));
1995 xmlOut.writeAttribute(QStringLiteral("creationtoolversion"), QStringLiteral(LOKALIZE_VERSION));
1996 xmlOut.writeAttribute(QStringLiteral("segtype"), QStringLiteral("paragraph"));
1997 xmlOut.writeAttribute(QStringLiteral("o-encoding"), QStringLiteral("UTF-8"));
1998 xmlOut.writeEndElement();
1999
2000 xmlOut.writeStartElement(QStringLiteral("body"));
2001
2002
2003
2004 QString dbLangCode = Project::instance()->langCode();
2005
2006 QSqlDatabase db = QSqlDatabase::database(connectionName);
2007 QSqlQuery query1(db);
2008
2009 if (Q_UNLIKELY(!query1.exec(U(
2010 "SELECT main.id, main.ctxt, main.date, main.bits, "
2011 "source_strings.source, source_strings.source_accel, "
2012 "target_strings.target, target_strings.target_accel, "
2013 "files.path, main.change_date "
2014 "FROM main, source_strings, target_strings, files "
2015 "WHERE source_strings.id=main.source AND "
2016 "target_strings.id=main.target AND "
2017 "files.id=main.file"))))
2018 qCWarning(LOKALIZE_LOG) << "select error: " << query1.lastError().text();
2019
2020 TMConfig c = getConfig(db);
2021
2022 const QString DATE_FORMAT = QStringLiteral("yyyyMMdd");
2023 const QString PROP = QStringLiteral("prop");
2024 const QString TYPE = QStringLiteral("type");
2025 while (query1.next()) {
2026 QString source = makeAcceledString(query1.value(4).toString(), c.accel, query1.value(5));
2027 QString target = makeAcceledString(query1.value(6).toString(), c.accel, query1.value(7));
2028
2029 xmlOut.writeStartElement(QStringLiteral("tu"));
2030 xmlOut.writeAttribute(QStringLiteral("tuid"), QString::number(query1.value(0).toLongLong()));
2031
2032 xmlOut.writeStartElement(QStringLiteral("tuv"));
2033 xmlOut.writeAttribute(QStringLiteral("xml:lang"), QStringLiteral("en"));
2034 xmlOut.writeStartElement(QStringLiteral("seg"));
2035 xmlOut.writeCharacters(source);
2036 xmlOut.writeEndElement();
2037 xmlOut.writeEndElement();
2038
2039 xmlOut.writeStartElement(QStringLiteral("tuv"));
2040 xmlOut.writeAttribute(QStringLiteral("xml:lang"), dbLangCode);
2041 xmlOut.writeAttribute(QStringLiteral("creationdate"), QDate::fromString(query1.value(2).toString(), Qt::ISODate).toString(DATE_FORMAT));
2042 xmlOut.writeAttribute(QStringLiteral("changedate"), QDate::fromString(query1.value(9).toString(), Qt::ISODate).toString(DATE_FORMAT));
2043 QString ctxt = query1.value(1).toString();
2044 if (!ctxt.isEmpty()) {
2045 int pos = ctxt.indexOf(TM_DELIMITER);
2046 if (pos != -1) {
2047 QString plural = ctxt;
2048 plural.remove(0, pos + 1);
2049 ctxt.remove(pos, plural.size());
2050 xmlOut.writeStartElement(PROP);
2051 xmlOut.writeAttribute(TYPE, "x-pluralform");
2052 xmlOut.writeCharacters(plural);
2053 xmlOut.writeEndElement();
2054 }
2055 if (!ctxt.isEmpty()) {
2056 xmlOut.writeStartElement(PROP);
2057 xmlOut.writeAttribute(TYPE, "x-context");
2058 xmlOut.writeCharacters(ctxt);
2059 xmlOut.writeEndElement();
2060 }
2061 }
2062 QString filePath = query1.value(8).toString();
2063 if (!filePath.isEmpty()) {
2064 xmlOut.writeStartElement(PROP);
2065 xmlOut.writeAttribute(TYPE, "x-file");
2066 xmlOut.writeCharacters(filePath);
2067 xmlOut.writeEndElement();
2068 }
2069 qlonglong bits = query1.value(8).toLongLong();
2070 if (bits & TM_NOTAPPROVED)
2071 if (!filePath.isEmpty()) {
2072 xmlOut.writeStartElement(PROP);
2073 xmlOut.writeAttribute(TYPE, "x-approved");
2074 xmlOut.writeCharacters("no");
2075 xmlOut.writeEndElement();
2076 }
2077 xmlOut.writeStartElement(QStringLiteral("seg"));
2078 xmlOut.writeCharacters(target);
2079 xmlOut.writeEndElement();
2080 xmlOut.writeEndElement();
2081 xmlOut.writeEndElement();
2082
2083
2084
2085 }
2086 query1.clear();
2087
2088
2089 xmlOut.writeEndDocument();
2090 out.close();
2091
2092 qCWarning(LOKALIZE_LOG) << "ExportTmxJob done exporting:" << a.elapsed();
2093 m_time = a.elapsed();
2094 }
2095
2096
2097 //END TMX
2098
2099
ExecQueryJob(const QString & queryString,const QString & dbName,QMutex * dbOperation)2100 ExecQueryJob::ExecQueryJob(const QString& queryString, const QString& dbName, QMutex *dbOperation)
2101 : QObject(), QRunnable()
2102 , query(nullptr)
2103 , m_dbName(dbName)
2104 , m_query(queryString)
2105 , m_dbOperationMutex(dbOperation)
2106 {
2107 setAutoDelete(false);
2108 //qCDebug(LOKALIZE_LOG)<<"ExecQueryJob"<<dbName<<queryString;
2109 }
2110
~ExecQueryJob()2111 ExecQueryJob::~ExecQueryJob()
2112 {
2113 m_dbOperationMutex->lock();
2114 delete query;
2115 m_dbOperationMutex->unlock();
2116 qCDebug(LOKALIZE_LOG) << "ExecQueryJob dtor";
2117 }
2118
run()2119 void ExecQueryJob::run()
2120 {
2121 const QString connectionName = getConnectionName(m_dbName);
2122 m_dbOperationMutex->lock();
2123 QSqlDatabase db = QSqlDatabase::database(connectionName);
2124 qCDebug(LOKALIZE_LOG) << "ExecQueryJob " << m_dbName << " " << connectionName <<" db.isOpen() =" << db.isOpen();
2125 //temporarily:
2126 if (!db.isOpen())
2127 qCWarning(LOKALIZE_LOG) << "ExecQueryJob db.open()=" << db.open();
2128 query = new QSqlQuery(m_query, db);
2129 query->exec();
2130 qCDebug(LOKALIZE_LOG) << "ExecQueryJob done" << query->lastError().text();
2131 m_dbOperationMutex->unlock();
2132 Q_EMIT done(this);
2133 }
2134
2135
2136