1import warnings
2from io import BytesIO
3
4from pytest import mark
5
6from translate.convert import test_convert
7from translate.storage import po, xliff
8from translate.tools import pretranslate
9
10
11class TestPretranslate:
12    xliff_skeleton = """<?xml version="1.0" encoding="utf-8"?>
13<xliff version="1.1" xmlns="urn:oasis:names:tc:xliff:document:1.1">
14        <file original="doc.txt" source-language="en-US">
15                <body>
16                        %s
17                </body>
18        </file>
19</xliff>"""
20
21    def setup_method(self, method):
22        warnings.resetwarnings()
23
24    def teardown_method(self, method):
25        warnings.resetwarnings()
26
27    def pretranslatepo(self, input_source, template_source=None):
28        """helper that converts strings to po source without requiring files"""
29        input_file = BytesIO(input_source.encode())
30        if template_source:
31            template_file = BytesIO(template_source.encode())
32        else:
33            template_file = None
34        output_file = BytesIO()
35
36        pretranslate.pretranslate_file(input_file, output_file, template_file)
37        output_file.seek(0)
38        return po.pofile(output_file.read())
39
40    def pretranslatexliff(self, input_source, template_source=None):
41        """helper that converts strings to po source without requiring files"""
42        input_file = BytesIO(input_source)
43        if template_source:
44            template_file = BytesIO(template_source)
45        else:
46            template_file = None
47        output_file = BytesIO()
48
49        pretranslate.pretranslate_file(input_file, output_file, template_file)
50        output_file.seek(0)
51        return xliff.xlifffile(output_file.read())
52
53    def singleunit(self, pofile):
54        """
55        checks that the pofile contains a single non-header unit, and
56        returns it
57        """
58        if len(pofile.units) == 2 and pofile.units[0].isheader():
59            print(pofile.units[1])
60            return pofile.units[1]
61        else:
62            print(pofile.units[0])
63            return pofile.units[0]
64
65    def test_pretranslatepo_blank(self):
66        """
67        checks that the pretranslatepo function is working for a simple file
68        initialisation
69        """
70        input_source = (
71            """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n"""
72            % po.lsep
73        )
74        newpo = self.pretranslatepo(input_source)
75        assert str(self.singleunit(newpo)) == input_source
76
77    def test_merging_simple(self):
78        """checks that the pretranslatepo function is working for a simple merge"""
79        input_source = (
80            """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n"""
81            % po.lsep
82        )
83        template_source = (
84            """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n"""
85            % po.lsep
86        )
87        newpo = self.pretranslatepo(input_source, template_source)
88        assert str(self.singleunit(newpo)) == template_source
89
90    def test_merging_messages_marked_fuzzy(self):
91        """test that when we merge PO files with a fuzzy message that it remains fuzzy"""
92        input_source = (
93            """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n"""
94            % po.lsep
95        )
96        template_source = (
97            """#: simple.label%ssimple.accesskey\n#, fuzzy\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n"""
98            % po.lsep
99        )
100        newpo = self.pretranslatepo(input_source, template_source)
101        assert str(self.singleunit(newpo)) == template_source
102
103    def test_merging_plurals_with_fuzzy_matching(self):
104        """test that when we merge PO files with a fuzzy message that it remains fuzzy"""
105        input_source = r"""#: file.cpp:2
106msgid "%d manual"
107msgid_plural "%d manuals"
108msgstr[0] ""
109msgstr[1] ""
110"""
111        template_source = r"""#: file.cpp:3
112#, fuzzy
113msgid "%d manual"
114msgid_plural "%d manuals"
115msgstr[0] "%d handleiding."
116msgstr[1] "%d handleidings."
117"""
118        # The #: comment and msgid's are different between the pot and the po
119        poexpected = r"""#: file.cpp:2
120#, fuzzy
121msgid "%d manual"
122msgid_plural "%d manuals"
123msgstr[0] "%d handleiding."
124msgstr[1] "%d handleidings."
125"""
126        newpo = self.pretranslatepo(input_source, template_source)
127        assert str(self.singleunit(newpo)) == poexpected
128
129    @mark.xfail(reason="Not Implemented")
130    def test_merging_msgid_change(self):
131        """
132        tests that if the msgid changes but the location stays the same that
133        we merge
134        """
135        input_source = """#: simple.label\n#: simple.accesskey\nmsgid "Its &hard coding a newline.\\n"\nmsgstr ""\n"""
136        template_source = """#: simple.label\n#: simple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n"""
137        poexpected = """#: simple.label\n#: simple.accesskey\n#, fuzzy\nmsgid "Its &hard coding a newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n"""
138        newpo = self.pretranslatepo(input_source, template_source)
139        print(bytes(newpo))
140        assert bytes(newpo).decode("utf-8") == poexpected
141
142    def test_merging_location_change(self):
143        """
144        tests that if the location changes but the msgid stays the same that
145        we merge
146        """
147        input_source = (
148            """#: new_simple.label%snew_simple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n"""
149            % po.lsep
150        )
151        template_source = (
152            """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n"""
153            % po.lsep
154        )
155        poexpected = (
156            """#: new_simple.label%snew_simple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n"""
157            % po.lsep
158        )
159        newpo = self.pretranslatepo(input_source, template_source)
160        print(bytes(newpo))
161        assert bytes(newpo).decode("utf-8") == poexpected
162
163    def test_merging_location_and_whitespace_change(self):
164        """
165        test that even if the location changes that if the msgid only has
166        whitespace changes we can still merge
167        """
168        input_source = (
169            """#: singlespace.label%ssinglespace.accesskey\nmsgid "&We have spaces"\nmsgstr ""\n"""
170            % po.lsep
171        )
172        template_source = (
173            """#: doublespace.label%sdoublespace.accesskey\nmsgid "&We  have  spaces"\nmsgstr "&One  het  spasies"\n"""
174            % po.lsep
175        )
176        poexpected = (
177            """#: singlespace.label%ssinglespace.accesskey\n#, fuzzy\nmsgid "&We have spaces"\nmsgstr "&One  het  spasies"\n"""
178            % po.lsep
179        )
180        newpo = self.pretranslatepo(input_source, template_source)
181        print(bytes(newpo))
182        assert bytes(newpo).decode("utf-8") == poexpected
183
184    @mark.xfail(reason="Not Implemented")
185    def test_merging_accelerator_changes(self):
186        """
187        test that a change in the accelerator localtion still allows
188        merging
189        """
190        input_source = """#: someline.c\nmsgid "A&bout"\nmsgstr ""\n"""
191        template_source = """#: someline.c\nmsgid "&About"\nmsgstr "&Info"\n"""
192        poexpected = """#: someline.c\nmsgid "A&bout"\nmsgstr "&Info"\n"""
193        newpo = self.pretranslatepo(input_source, template_source)
194        print(bytes(newpo))
195        assert bytes(newpo).decode("utf-8") == poexpected
196
197    @mark.xfail(reason="Not Implemented")
198    def test_lines_cut_differently(self):
199        """
200        Checks that the correct formatting is preserved when pot an po lines
201        differ.
202        """
203        input_source = (
204            """#: simple.label\nmsgid "Line split "\n"differently"\nmsgstr ""\n"""
205        )
206        template_source = """#: simple.label\nmsgid "Line"\n" split differently"\nmsgstr "Lyne verskillend gesny"\n"""
207        newpo = self.pretranslatepo(input_source, template_source)
208        newpounit = self.singleunit(newpo)
209        assert str(newpounit) == template_source
210
211    def test_merging_automatic_comments_dont_duplicate(self):
212        """ensure that we can merge #. comments correctly"""
213        input_source = """#. Row 35\nmsgid "&About"\nmsgstr ""\n"""
214        template_source = """#. Row 35\nmsgid "&About"\nmsgstr "&Info"\n"""
215        newpo = self.pretranslatepo(input_source, template_source)
216        newpounit = self.singleunit(newpo)
217        assert str(newpounit) == template_source
218
219    def test_merging_automatic_comments_new_overides_old(self):
220        """ensure that new #. comments override the old comments"""
221        input_source = """#. new comment\n#: someline.c\nmsgid "&About"\nmsgstr ""\n"""
222        template_source = (
223            """#. old comment\n#: someline.c\nmsgid "&About"\nmsgstr "&Info"\n"""
224        )
225        poexpected = (
226            """#. new comment\n#: someline.c\nmsgid "&About"\nmsgstr "&Info"\n"""
227        )
228        newpo = self.pretranslatepo(input_source, template_source)
229        newpounit = self.singleunit(newpo)
230        assert str(newpounit) == poexpected
231
232    def test_merging_comments_with_blank_comment_lines(self):
233        """
234        test that when we merge a comment that has a blank line we keep the
235        blank line
236        """
237        input_source = """#: someline.c\nmsgid "About"\nmsgstr ""\n"""
238        template_source = """# comment1\n#\n# comment2\n#: someline.c\nmsgid "About"\nmsgstr "Omtrent"\n"""
239        poexpected = template_source
240        newpo = self.pretranslatepo(input_source, template_source)
241        newpounit = self.singleunit(newpo)
242        assert str(newpounit) == poexpected
243
244    def test_empty_commentlines(self):
245        input_source = """#: paneSecurity.title
246msgid "Security"
247msgstr ""
248"""
249        template_source = """# - Contributor(s):
250# -
251# - Alternatively, the
252# -
253#: paneSecurity.title
254msgid "Security"
255msgstr "Sekuriteit"
256"""
257        poexpected = template_source
258        newpo = self.pretranslatepo(input_source, template_source)
259        newpounit = self.singleunit(newpo)
260        print("expected")
261        print(poexpected)
262        print("got:")
263        print(str(newpounit))
264        assert str(newpounit) == poexpected
265
266    def test_merging_msgidcomments(self):
267        """ensure that we can merge msgidcomments messages"""
268        input_source = r"""#: window.width
269msgid ""
270"_: Do not translate this.\n"
271"36em"
272msgstr ""
273"""
274        template_source = r"""#: window.width
275msgid ""
276"_: Do not translate this.\n"
277"36em"
278msgstr "36em"
279"""
280        newpo = self.pretranslatepo(input_source, template_source)
281        newpounit = self.singleunit(newpo)
282        assert str(newpounit) == template_source
283
284    def test_merging_plurals(self):
285        """ensure that we can merge plural messages"""
286        input_source = (
287            """msgid "One"\nmsgid_plural "Two"\nmsgstr[0] ""\nmsgstr[1] ""\n"""
288        )
289        template_source = """msgid "One"\nmsgid_plural "Two"\nmsgstr[0] "Een"\nmsgstr[1] "Twee"\nmsgstr[2] "Drie"\n"""
290        newpo = self.pretranslatepo(input_source, template_source)
291        print(newpo)
292        newpounit = self.singleunit(newpo)
293        assert str(newpounit) == template_source
294
295    def test_merging_resurect_obsolete_messages(self):
296        """
297        check that we can reuse old obsolete messages if the message comes
298        back
299        """
300        input_source = """#: resurect.c\nmsgid "&About"\nmsgstr ""\n"""
301        template_source = """#~ msgid "&About"\n#~ msgstr "&Omtrent"\n"""
302        expected = """#: resurect.c\nmsgid "&About"\nmsgstr "&Omtrent"\n"""
303        newpo = self.pretranslatepo(input_source, template_source)
304        print(bytes(newpo))
305        assert bytes(newpo).decode("utf-8") == expected
306
307    def test_merging_comments(self):
308        """Test that we can merge comments correctly"""
309        input_source = """#. Don't do it!\n#: file.py:1\nmsgid "One"\nmsgstr ""\n"""
310        template_source = (
311            """#. Don't do it!\n#: file.py:2\nmsgid "One"\nmsgstr "Een"\n"""
312        )
313        poexpected = """#. Don't do it!\n#: file.py:1\nmsgid "One"\nmsgstr "Een"\n"""
314        newpo = self.pretranslatepo(input_source, template_source)
315        print(newpo)
316        newpounit = self.singleunit(newpo)
317        assert str(newpounit) == poexpected
318
319    def test_merging_typecomments(self):
320        """Test that we can merge with typecomments"""
321        input_source = """#: file.c:1\n#, c-format\nmsgid "%d pipes"\nmsgstr ""\n"""
322        template_source = """#: file.c:2\nmsgid "%d pipes"\nmsgstr "%d pype"\n"""
323        poexpected = (
324            """#: file.c:1\n#, c-format\nmsgid "%d pipes"\nmsgstr "%d pype"\n"""
325        )
326        newpo = self.pretranslatepo(input_source, template_source)
327        newpounit = self.singleunit(newpo)
328        print(newpounit)
329        assert str(newpounit) == poexpected
330
331        input_source = """#: file.c:1\n#, c-format\nmsgid "%d computers"\nmsgstr ""\n"""
332        template_source = """#: file.c:2\n#, c-format\nmsgid "%s computers "\nmsgstr "%s-rekenaars"\n"""
333        poexpected = """#: file.c:1\n#, fuzzy, c-format\nmsgid "%d computers"\nmsgstr "%s-rekenaars"\n"""
334        newpo = self.pretranslatepo(input_source, template_source)
335        newpounit = self.singleunit(newpo)
336        assert newpounit.isfuzzy()
337        assert newpounit.hastypecomment("c-format")
338
339    def test_xliff_states(self):
340        """Test correct maintenance of XLIFF states."""
341        xlf_template = self.xliff_skeleton % (
342            """<trans-unit id="1" xml:space="preserve">
343                   <source> File  1 </source>
344               </trans-unit>"""
345        )
346        xlf_old = self.xliff_skeleton % (
347            """<trans-unit id="1" xml:space="preserve" approved="yes">
348                   <source> File  1 </source>
349                   <target> Lêer 1 </target>
350               </trans-unit>"""
351        )
352
353        template = xliff.xlifffile.parsestring(xlf_template)
354        old = xliff.xlifffile.parsestring(xlf_old)
355        # Serialize files
356        new = self.pretranslatexliff(bytes(template), bytes(old))
357        print(bytes(old))
358        print("---")
359        print(bytes(new))
360        assert new.units[0].isapproved()
361        # Layout might have changed, so we won't compare the serialised
362        # versions
363
364
365class TestPretranslateCommand(test_convert.TestConvertCommand, TestPretranslate):
366    """Tests running actual pretranslate commands on files"""
367
368    convertmodule = pretranslate
369
370    def test_help(self, capsys):
371        """tests getting help"""
372        options = super().test_help(capsys)
373        options = self.help_check(options, "-t TEMPLATE, --template=TEMPLATE")
374        options = self.help_check(options, "--tm")
375        options = self.help_check(
376            options, "-s MIN_SIMILARITY, --similarity=MIN_SIMILARITY"
377        )
378        options = self.help_check(options, "--nofuzzymatching", last=True)
379