1 /****************************************************************************
2 **
3 ** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/legal
5 **
6 ** This file is part of the tools applications of the Qt Toolkit.
7 **
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and Digia. For licensing terms and
14 ** conditions see http://qt.digia.com/licensing. For further information
15 ** use the contact form at http://qt.digia.com/contact-us.
16 **
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 2.1 as published by the Free Software
20 ** Foundation and appearing in the file LICENSE.LGPL included in the
21 ** packaging of this file. Please review the following information to
22 ** ensure the GNU Lesser General Public License version 2.1 requirements
23 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
24 **
25 ** In addition, as a special exception, Digia gives you certain additional
26 ** rights. These rights are described in the Digia Qt LGPL Exception
27 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
28 **
29 ** GNU General Public License Usage
30 ** Alternatively, this file may be used under the terms of the GNU
31 ** General Public License version 3.0 as published by the Free Software
32 ** Foundation and appearing in the file LICENSE.GPL included in the
33 ** packaging of this file. Please review the following information to
34 ** ensure the GNU General Public License version 3.0 requirements will be
35 ** met: http://www.gnu.org/copyleft/gpl.html.
36 **
37 **
38 ** $QT_END_LICENSE$
39 **
40 ****************************************************************************/
41
42 #include <stdio.h>
43
44 #include "pylupdate.h"
45 #include "simtexth.h"
46
47 // defined in numberh.cpp
48 extern int applyNumberHeuristic( MetaTranslator *tor );
49 // defined in sametexth.cpp
50 extern int applySameTextHeuristic( MetaTranslator *tor );
51
52 typedef QList<MetaTranslatorMessage> TML;
53
54 /*
55 Merges two MetaTranslator objects into the first one. The first one
56 is a set of source texts and translations for a previous version of
57 the internationalized program; the second one is a set of fresh
58 source texts newly extracted from the source code, without any
59 translation yet.
60 */
61
merge(const MetaTranslator * tor,const MetaTranslator * virginTor,MetaTranslator * outTor,bool noObsolete,bool verbose,const QString & filename)62 void merge( const MetaTranslator *tor, const MetaTranslator *virginTor, MetaTranslator *outTor, bool noObsolete, bool verbose, const QString &filename )
63 {
64 if (verbose)
65 fprintf(stderr, "Updating '%s'...\n", filename.toLocal8Bit().constData());
66
67 int known = 0;
68 int neww = 0;
69 int obsoleted = 0;
70 int UntranslatedObsoleted = 0;
71 int similarTextHeuristicCount = 0;
72 TML all = tor->messages();
73 TML::Iterator it;
74 outTor->setLanguageCode(tor->languageCode());
75 outTor->setSourceLanguageCode(tor->sourceLanguageCode());
76
77 /*
78 The types of all the messages from the vernacular translator
79 are updated according to the virgin translator.
80 */
81 for ( it = all.begin(); it != all.end(); ++it ) {
82 MetaTranslatorMessage::Type newType = MetaTranslatorMessage::Finished;
83 MetaTranslatorMessage m = *it;
84
85 // skip context comment
86 if ( !QByteArray(m.sourceText()).isEmpty() ) {
87 MetaTranslatorMessage mv = virginTor->find(m.context(), m.sourceText(), m.comment());
88 if ( mv.isNull() ) {
89 mv = virginTor->find(m.context(), m.comment(), m.fileName(), m.lineNumber());
90 if ( mv.isNull() ) {
91 // did not find it in the virgin, mark it as obsolete
92 newType = MetaTranslatorMessage::Obsolete;
93 if ( m.type() != MetaTranslatorMessage::Obsolete )
94 obsoleted++;
95 } else {
96 // Do not just accept it if its on the same line number, but different source text.
97 // Also check if the texts are more or less similar before we consider them to represent the same message...
98 // ### The QString() cast is evil
99 if (getSimilarityScore(QString(m.sourceText()), mv.sourceText()) >= textSimilarityThreshold) {
100 // It is just slightly modified, assume that it is the same string
101 m = MetaTranslatorMessage(m.context(), mv.sourceText(), m.comment(), m.fileName(), m.lineNumber(), m.translations());
102 m.setPlural(mv.isPlural());
103
104 // Mark it as unfinished. (Since the source text was changed it might require re-translating...)
105 newType = MetaTranslatorMessage::Unfinished;
106 ++similarTextHeuristicCount;
107 } else {
108 // The virgin and vernacular sourceTexts are so different that we could not find it.
109 newType = MetaTranslatorMessage::Obsolete;
110 if ( m.type() != MetaTranslatorMessage::Obsolete )
111 obsoleted++;
112 }
113 neww++;
114 }
115 } else {
116 switch ( m.type() ) {
117 case MetaTranslatorMessage::Finished:
118 default:
119 if (m.isPlural() == mv.isPlural()) {
120 newType = MetaTranslatorMessage::Finished;
121 } else {
122 newType = MetaTranslatorMessage::Unfinished;
123 }
124 known++;
125 break;
126 case MetaTranslatorMessage::Unfinished:
127 newType = MetaTranslatorMessage::Unfinished;
128 known++;
129 break;
130 case MetaTranslatorMessage::Obsolete:
131 newType = MetaTranslatorMessage::Unfinished;
132 neww++;
133 }
134
135 // Always get the filename and linenumber info from the virgin Translator, in case it has changed location.
136 // This should also enable us to read a file that does not have the <location> element.
137 m.setFileName(mv.fileName());
138 m.setLineNumber(mv.lineNumber());
139 m.setPlural(mv.isPlural()); // ### why not use operator=?
140 }
141
142 if (newType == MetaTranslatorMessage::Obsolete && !m.isTranslated()) {
143 ++UntranslatedObsoleted;
144 }
145
146 m.setType(newType);
147 outTor->insert(m);
148 }
149 }
150
151 /*
152 Messages found only in the virgin translator are added to the
153 vernacular translator. Among these are all the context comments.
154 */
155 all = virginTor->messages();
156
157 for ( it = all.begin(); it != all.end(); ++it ) {
158 MetaTranslatorMessage mv = *it;
159 bool found = tor->contains(mv.context(), mv.sourceText(), mv.comment());
160 if (!found) {
161 MetaTranslatorMessage m = tor->find(mv.context(), mv.comment(), mv.fileName(), mv.lineNumber());
162 if (!m.isNull()) {
163 if (getSimilarityScore(QString(m.sourceText()), mv.sourceText()) >= textSimilarityThreshold) {
164 found = true;
165 }
166 } else {
167 found = false;
168 }
169 }
170 if ( !found ) {
171 outTor->insert( mv );
172 if ( !QByteArray(mv.sourceText()).isEmpty() )
173 neww++;
174 }
175 }
176
177 /*
178 The same-text heuristic handles cases where a message has an
179 obsolete counterpart with a different context or comment.
180 */
181 int sameTextHeuristicCount = applySameTextHeuristic( outTor );
182
183 /*
184 The number heuristic handles cases where a message has an
185 obsolete counterpart with mostly numbers differing in the
186 source text.
187 */
188 int sameNumberHeuristicCount = applyNumberHeuristic( outTor );
189
190 if ( verbose ) {
191 int totalFound = neww + known;
192 fprintf( stderr, " Found %d source text%s (%d new and %d already existing)\n",
193 totalFound, totalFound == 1 ? "" : "s", neww, known);
194
195 if (obsoleted) {
196 if (noObsolete) {
197 fprintf( stderr, " Removed %d obsolete entr%s\n",
198 obsoleted, obsoleted == 1 ? "y" : "ies" );
199 } else {
200 int total = obsoleted - UntranslatedObsoleted;
201 fprintf( stderr, " Kept %d obsolete translation%s\n",
202 total, total == 1 ? "" : "s" );
203
204 fprintf( stderr, " Removed %d obsolete untranslated entr%s\n",
205 UntranslatedObsoleted, UntranslatedObsoleted == 1 ? "y" : "ies" );
206
207 }
208 }
209
210 if (sameNumberHeuristicCount)
211 fprintf( stderr, " Number heuristic provided %d translation%s\n",
212 sameNumberHeuristicCount, sameNumberHeuristicCount == 1 ? "" : "s" );
213 if (sameTextHeuristicCount)
214 fprintf( stderr, " Same-text heuristic provided %d translation%s\n",
215 sameTextHeuristicCount, sameTextHeuristicCount == 1 ? "" : "s" );
216 if (similarTextHeuristicCount)
217 fprintf( stderr, " Similar-text heuristic provided %d translation%s\n",
218 similarTextHeuristicCount, similarTextHeuristicCount == 1 ? "" : "s" );
219 }
220 }
221