1from io import BytesIO
2
3from pytest import mark
4
5from translate.convert import dtd2po, test_convert
6from translate.storage import dtd, po
7
8
9class TestDTD2PO:
10    def dtd2po(self, dtdsource, dtdtemplate=None):
11        """helper that converts dtd source to po source without requiring files"""
12        inputfile = BytesIO(dtdsource.encode())
13        inputdtd = dtd.dtdfile(inputfile)
14        convertor = dtd2po.dtd2po()
15        if dtdtemplate is None:
16            outputpo = convertor.convertstore(inputdtd)
17        else:
18            templatefile = BytesIO(dtdtemplate.encode())
19            templatedtd = dtd.dtdfile(templatefile)
20            outputpo = convertor.mergestore(templatedtd, inputdtd)
21        return outputpo
22
23    def convertdtd(self, dtdsource):
24        """call the convertdtd, return the outputfile"""
25        inputfile = BytesIO(dtdsource.encode())
26        outputfile = BytesIO()
27        templatefile = None
28        assert dtd2po.convertdtd(inputfile, outputfile, templatefile)
29        return outputfile.getvalue()
30
31    def singleelement(self, pofile):
32        """checks that the pofile contains a single non-header element, and returns it"""
33        assert len(pofile.units) == 2
34        assert pofile.units[0].isheader()
35        print(pofile.units[1])
36        return pofile.units[1]
37
38    def countelements(self, pofile):
39        """returns the number of non-header items"""
40        if pofile.units[0].isheader():
41            return len(pofile.units) - 1
42        else:
43            return len(pofile.units)
44
45    def test_simpleentity(self):
46        """checks that a simple dtd entity definition converts properly to a po entry"""
47        dtdsource = '<!ENTITY test.me "bananas for sale">\n'
48        pofile = self.dtd2po(dtdsource)
49        pounit = self.singleelement(pofile)
50        assert pounit.source == "bananas for sale"
51        assert pounit.target == ""
52        # Now with a template language
53        dtdtemplate = '<!ENTITY test.me "bananas for sale">\n'
54        dtdtranslated = '<!ENTITY test.me "piesangs te koop">\n'
55        pofile = self.dtd2po(dtdtranslated, dtdtemplate)
56        pounit = self.singleelement(pofile)
57        assert pounit.source == "bananas for sale"
58        assert pounit.target == "piesangs te koop"
59
60    def test_convertdtd(self):
61        """checks that the convertdtd function is working"""
62        dtdsource = '<!ENTITY saveas.label "Save As...">\n'
63        posource = self.convertdtd(dtdsource)
64        pofile = po.pofile(BytesIO(posource))
65        unit = self.singleelement(pofile)
66        assert unit.source == "Save As..."
67        assert unit.target == ""
68
69    def test_apos(self):
70        """apostrophe should not break a single-quoted entity definition, bug 69"""
71        dtdsource = "<!ENTITY test.me 'bananas &apos; for sale'>\n"
72        pofile = self.dtd2po(dtdsource)
73        pounit = self.singleelement(pofile)
74        assert pounit.source == "bananas ' for sale"
75
76    def test_quotes(self):
77        """quotes should be handled in a single-quoted entity definition"""
78        dtdsource = """<!ENTITY test.metoo '"Bananas" for sale'>\n"""
79        pofile = self.dtd2po(dtdsource)
80        pounit = self.singleelement(pofile)
81        print(str(pounit))
82        assert pounit.source == '"Bananas" for sale'
83
84    def test_emptyentity(self):
85        """checks that empty entity definitions survive into po file, bug 15"""
86        dtdsource = '<!ENTITY credit.translation "">\n'
87        pofile = self.dtd2po(dtdsource)
88        pounit = self.singleelement(pofile)
89        assert "credit.translation" in str(pounit)
90        assert 'msgctxt "credit.translation"' in str(pounit)
91
92    def test_two_empty_entities(self):
93        """checks that two empty entitu definitions have correct context (bug 2190)."""
94        dtdsource = '<!ENTITY community.exp.start "">\n<!ENTITY contribute.end "">\n'
95        pofile = self.dtd2po(dtdsource)
96        assert pofile.units[-2].getcontext() == "community.exp.start"
97        assert pofile.units[-1].getcontext() == "contribute.end"
98
99    def test_emptyentity_translated(self):
100        """checks that if we translate an empty entity it makes it into the PO, bug 101"""
101        dtdtemplate = '<!ENTITY credit.translation "">\n'
102        dtdsource = '<!ENTITY credit.translation "Translators Names">\n'
103        pofile = self.dtd2po(dtdsource, dtdtemplate)
104        unit = self.singleelement(pofile)
105        print(unit)
106        assert "credit.translation" in str(unit)
107        # We don't want this to simply be seen as a header:
108        assert len(unit.getid()) != 0
109        assert unit.target == "Translators Names"
110
111    def test_localisaton_note_simple(self):
112        """test the simple localisation more becomes a #. comment"""
113        dtdsource = """<!-- LOCALIZATION NOTE (alwaysCheckDefault.height):
114  There's some sort of bug which makes wrapping checkboxes not properly reflow,
115  causing the bottom border of the groupbox to be cut off; set this
116  appropriately if your localization causes this checkbox to wrap.
117-->
118<!ENTITY alwaysCheckDefault.height  "3em">
119"""
120        pofile = self.dtd2po(dtdsource)
121        posource = bytes(pofile).decode("utf-8")
122        print(posource)
123        assert (
124            posource.count("#.") == 5
125        )  # 1 Header extracted from, 3 comment lines, 1 autoinserted comment
126
127    def test_localisation_note_merge(self):
128        """test that LOCALIZATION NOTES are added properly as #. comments and disambiguated with msgctxt entries"""
129        dtdtemplate = (
130            "<!--LOCALIZATION NOTE (%s): Some note -->\n"
131            + '<!ENTITY %s "Source text">\n'
132        )
133        dtdsource = dtdtemplate % ("note1.label", "note1.label") + dtdtemplate % (
134            "note2.label",
135            "note2.label",
136        )
137        pofile = self.dtd2po(dtdsource)
138        posource = str(pofile.units[1]) + str(pofile.units[2])
139        print(posource)
140        assert posource.count("#.") == 2
141        assert posource.count("msgctxt") == 2
142
143    def test_donttranslate_simple(self):
144        """check that we handle DONT_TRANSLATE messages properly"""
145        dtdsource = """<!-- LOCALIZATION NOTE (region.Altitude): DONT_TRANSLATE -->
146<!ENTITY region.Altitude "Very High">"""
147        pofile = self.dtd2po(dtdsource)
148        assert self.countelements(pofile) == 0
149        dtdsource = """<!-- LOCALIZATION NOTE (exampleOpenTag.label): DONT_TRANSLATE: they are text for HTML tagnames: "<i>" and "</i>" -->
150<!ENTITY exampleOpenTag.label "&lt;i&gt;">"""
151        pofile = self.dtd2po(dtdsource)
152        assert self.countelements(pofile) == 0
153        dtdsource = """<!-- LOCALIZATION NOTE (imapAdvanced.label): Do not translate "IMAP" -->
154<!ENTITY imapAdvanced.label "Advanced IMAP Server Settings">"""
155        pofile = self.dtd2po(dtdsource)
156        assert self.countelements(pofile) == 1
157
158    def test_donttranslate_label(self):
159        """test strangeness when label entity is marked DONT_TRANSLATE and accesskey is not, bug 30"""
160        dtdsource = (
161            "<!--LOCALIZATION NOTE (editorCheck.label): DONT_TRANSLATE -->\n"
162            + '<!ENTITY editorCheck.label "Composer">\n<!ENTITY editorCheck.accesskey "c">\n'
163        )
164        pofile = self.dtd2po(dtdsource)
165        posource = bytes(pofile).decode("utf-8")
166        # we need to decided what we're going to do here - see the comments in bug 30
167        # this tests the current implementation which is that the DONT_TRANSLATE string is removed, but the other remains
168        assert "editorCheck.label" not in posource
169        assert "editorCheck.accesskey" in posource
170
171    def test_donttranslate_onlyentity(self):
172        """if the entity is itself just another entity then it shouldn't appear in the output PO file"""
173        dtdsource = """<!-- LOCALIZATION NOTE (mainWindow.title): DONT_TRANSLATE -->
174<!ENTITY mainWindow.title "&brandFullName;">"""
175        pofile = self.dtd2po(dtdsource)
176        assert self.countelements(pofile) == 0
177
178    def test_donttranslate_commentedout(self):
179        """check that we don't process messages in <!-- comments -->: bug 102"""
180        dtdsource = """<!-- commenting out until bug 38906 is fixed
181<!ENTITY messagesHeader.label         "Messages"> -->"""
182        pofile = self.dtd2po(dtdsource)
183        assert self.countelements(pofile) == 0
184
185    def test_spaces_at_start_of_dtd_lines(self):
186        """test that pretty print spaces at the start of subsequent DTD element lines are removed from the PO file, bug 79"""
187        # Space at the end of the line
188        dtdsource = (
189            '<!ENTITY  noupdatesfound.intro "First line then \n'
190            '                                next lines.">\n'
191        )
192        pofile = self.dtd2po(dtdsource)
193        pounit = self.singleelement(pofile)
194        # We still need to decide how we handle line line breaks in the DTD entities.  It seems that we should actually
195        # drop the line break but this has not been implemented yet.
196        assert pounit.source == "First line then next lines."
197        # No space at the end of the line
198        dtdsource = (
199            '<!ENTITY  noupdatesfound.intro "First line then\n'
200            '                                next lines.">\n'
201        )
202        pofile = self.dtd2po(dtdsource)
203        pounit = self.singleelement(pofile)
204        assert pounit.source == "First line then next lines."
205
206    def test_accesskeys_folding(self):
207        """test that we fold accesskeys into message strings"""
208        dtdsource_template = (
209            '<!ENTITY  fileSaveAs.%s "Save As...">\n<!ENTITY  fileSaveAs.%s "S">\n'
210        )
211        lang_template = (
212            '<!ENTITY  fileSaveAs.%s "Gcina ka...">\n<!ENTITY  fileSaveAs.%s "G">\n'
213        )
214        for label in ("label", "title"):
215            for accesskey in ("accesskey", "accessKey", "akey"):
216                pofile = self.dtd2po(dtdsource_template % (label, accesskey))
217                pounit = self.singleelement(pofile)
218                assert pounit.source == "&Save As..."
219                # Test with template (bug 155)
220                pofile = self.dtd2po(
221                    lang_template % (label, accesskey),
222                    dtdsource_template % (label, accesskey),
223                )
224                pounit = self.singleelement(pofile)
225                assert pounit.source == "&Save As..."
226                assert pounit.target == "&Gcina ka..."
227
228    def test_accesskeys_mismatch(self):
229        """check that we can handle accesskeys that don't match and thus can't be folded into the .label entry"""
230        dtdsource = (
231            '<!ENTITY  fileSave.label "Save">\n' '<!ENTITY  fileSave.accesskey "z">\n'
232        )
233        pofile = self.dtd2po(dtdsource)
234        assert self.countelements(pofile) == 2
235
236    def test_carriage_return_in_multiline_dtd(self):
237        r"""test that we create nice PO files when we find a \r\n in a multiline DTD element"""
238        dtdsource = (
239            '<!ENTITY  noupdatesfound.intro "First line then \r\n'
240            '                                next lines.">\n'
241        )
242        pofile = self.dtd2po(dtdsource)
243        unit = self.singleelement(pofile)
244        assert unit.source == "First line then next lines."
245
246    def test_multiline_with_blankline(self):
247        """test that we can process a multiline entity that has a blank line in it, bug 331"""
248        dtdsource = """
249<!ENTITY multiline.text "
250Some text
251
252Some other text
253">"""
254        pofile = self.dtd2po(dtdsource)
255        unit = self.singleelement(pofile)
256        assert unit.source == "Some text  Some other text"
257
258    def test_multiline_closing_quotes(self):
259        """test that we support various styles and spaces after closing quotes on multiline entities"""
260        dtdsource = """
261<!ENTITY pref.plural '<span>opsies</span><span
262                      class="noWin">preferences</span>' >
263"""
264        pofile = self.dtd2po(dtdsource)
265        unit = self.singleelement(pofile)
266        assert (
267            unit.source == '<span>opsies</span><span class="noWin">preferences</span>'
268        )
269
270    def test_preserving_spaces(self):
271        """test that we preserve space that appear at the start of the first line of a DTD entity"""
272        # Space before first character
273        dtdsource = '<!ENTITY mainWindow.titlemodifiermenuseparator " - ">'
274        pofile = self.dtd2po(dtdsource)
275        unit = self.singleelement(pofile)
276        assert unit.source == " - "
277        # Double line and spaces
278        dtdsource = '<!ENTITY mainWindow.titlemodifiermenuseparator " - with a newline\n    and more text">'
279        pofile = self.dtd2po(dtdsource)
280        unit = self.singleelement(pofile)
281        print(repr(unit.source))
282        assert unit.source == " - with a newline and more text"
283
284    def test_escaping_newline_tabs(self):
285        """test that we handle all kinds of newline permutations"""
286        dtdsource = '<!ENTITY  noupdatesfound.intro "A hard coded newline.\\nAnd tab\\t and a \\r carriage return.">\n'
287        converter = dtd2po.dtd2po()
288        thedtd = dtd.dtdunit()
289        thedtd.parse(dtdsource)
290        thepo = po.pounit()
291        converter.convertstrings(thedtd, thepo)
292        print(thedtd)
293        print(thepo.source)
294        # \n in a dtd should also appear as \n in the PO file
295        assert (
296            thepo.source
297            == r"A hard coded newline.\nAnd tab\t and a \r carriage return."
298        )
299
300    def test_abandoned_accelerator(self):
301        """test that when a language DTD has an accelerator but the template DTD does not that we abandon the accelerator"""
302        dtdtemplate = '<!ENTITY test.label "Test">\n'
303        dtdlanguage = '<!ENTITY test.label "Toets">\n<!ENTITY test.accesskey "T">\n'
304        pofile = self.dtd2po(dtdlanguage, dtdtemplate)
305        unit = self.singleelement(pofile)
306        assert unit.source == "Test"
307        assert unit.target == "Toets"
308
309    def test_unassociable_accelerator(self):
310        """test to see that we can handle accelerator keys that cannot be associated correctly"""
311        dtdsource = '<!ENTITY  managecerts.button "Manage Certificates...">\n<!ENTITY  managecerts.accesskey "M">'
312        pofile = self.dtd2po(dtdsource)
313        assert pofile.units[1].source == "Manage Certificates..."
314        assert pofile.units[2].source == "M"
315        pofile = self.dtd2po(dtdsource, dtdsource)
316        assert pofile.units[1].target == "Manage Certificates..."
317        assert pofile.units[2].target == "M"
318
319    def test_changed_labels_and_accelerators(self):
320        """test to ensure that when the template changes an entity name we can still manage the accelerators"""
321        dtdtemplate = """<!ENTITY  managecerts.caption      "Manage Certificates">
322<!ENTITY  managecerts.text         "Use the Certificate Manager to manage your personal certificates, as well as those of other people and certificate authorities.">
323<!ENTITY  managecerts.button       "Manage Certificates...">
324<!ENTITY  managecerts.accesskey    "M">"""
325        dtdlanguage = """<!ENTITY managecerts.label "ﺇﺩﺍﺭﺓ ﺎﻠﺸﻫﺍﺩﺎﺗ">
326<!ENTITY managecerts.text "ﺎﺴﺘﺧﺪﻣ ﻡﺪﻳﺭ ﺎﻠﺸﻫﺍﺩﺎﺗ ﻹﺩﺍﺭﺓ ﺶﻫﺍﺩﺎﺘﻛ ﺎﻠﺸﺨﺼﻳﺓ، ﺏﺍﻺﺿﺎﻓﺓ ﻞﺘﻠﻛ ﺎﻠﺧﺎﺻﺓ ﺏﺍﻶﺧﺮﻴﻧ ﻭ ﺲﻠﻃﺎﺗ ﺎﻠﺸﻫﺍﺩﺎﺗ.">
327<!ENTITY managecerts.button "ﺇﺩﺍﺭﺓ ﺎﻠﺸﻫﺍﺩﺎﺗ...">
328<!ENTITY managecerts.accesskey "ﺩ">"""
329        pofile = self.dtd2po(dtdlanguage, dtdtemplate)
330        print(pofile)
331        assert pofile.units[3].source == "Manage Certificates..."
332        assert pofile.units[3].target == "ﺇﺩﺍﺭﺓ ﺎﻠﺸﻫﺍﺩﺎﺗ..."
333        assert pofile.units[4].source == "M"
334        assert pofile.units[4].target == "ﺩ"
335
336    @mark.xfail(reason="Not Implemented")
337    def test_accelerator_keys_not_in_sentence(self):
338        """tests to ensure that we can manage accelerator keys that are not part of the transated sentence eg in Chinese"""
339        dtdtemplate = """<!ENTITY useAutoScroll.label             "Use autoscrolling">
340<!ENTITY useAutoScroll.accesskey         "a">"""
341        dtdlanguage = """<!ENTITY useAutoScroll.label             "使用自動捲動(Autoscrolling)">
342<!ENTITY useAutoScroll.accesskey         "a">"""
343        pofile = self.dtd2po(dtdlanguage, dtdtemplate)
344        print(pofile)
345        expected_target = "使用自動捲動(&Autoscrolling)".decode("utf-8")
346        assert pofile.units[1].target == expected_target
347        # We assume that accesskeys with no associated key should be done as follows "XXXX (&A)"
348        # TODO - check that we can unfold this from PO -> DTD
349        dtdlanguage = """<!ENTITY useAutoScroll.label             "使用自動捲動">
350<!ENTITY useAutoScroll.accesskey         "a">"""
351        pofile = self.dtd2po(dtdlanguage, dtdtemplate)
352        print(pofile)
353        assert pofile.units[1].target == "使用自動捲動 (&A)".decode("utf-8")
354
355    def test_exclude_entity_includes(self):
356        """test that we don't turn an include into a translatable string"""
357        dtdsource = '<!ENTITY % brandDTD SYSTEM "chrome://branding/locale/brand.dtd">'
358        pofile = self.dtd2po(dtdsource)
359        assert self.countelements(pofile) == 0
360
361    def test_linewraps(self):
362        """check that redundant line wraps are removed from the po file"""
363        dtdsource = """<!ENTITY generic.longDesc "
364<p>Test me.</p>
365">"""
366        pofile = self.dtd2po(dtdsource)
367        pounit = self.singleelement(pofile)
368        assert pounit.source == "<p>Test me.</p>"
369
370    def test_merging_with_new_untranslated(self):
371        """test that when we merge in new untranslated strings with existing translations we manage the encodings properly"""
372        # This should probably be in test_po.py but was easier to do here
373        dtdtemplate = """<!ENTITY unreadFolders.label "Unread">\n<!ENTITY viewPickerUnread.label "Unread">\n<!ENTITY unreadColumn.label "Unread">"""
374        dtdlanguage = """<!ENTITY viewPickerUnread.label "Непрочетени">\n<!ENTITY unreadFolders.label "Непрочетени">"""
375        pofile = self.dtd2po(dtdlanguage, dtdtemplate)
376        print(pofile)
377        assert pofile.units[1].source == "Unread"
378
379    def test_merge_without_template(self):
380        """test that we we manage the case where we merge and their is no template file"""
381        # If we supply a template file we should fail if the template file does not exist or is blank.  We should
382        # not put the translation in as the source.
383        # TODO: this test fails, since line 16 checks for "not dtdtemplate"
384        #   instead of checking for "dtdtemplate is None". What is correct?
385        dtdtemplate = ""
386        dtdsource = '<!ENTITY no.template "Target">'
387        pofile = self.dtd2po(dtdsource, dtdtemplate)
388        print(pofile)
389        assert self.countelements(pofile) == 0
390
391
392class TestDTD2POCommand(test_convert.TestConvertCommand, TestDTD2PO):
393    """Tests running actual dtd2po commands on files"""
394
395    convertmodule = dtd2po
396    defaultoptions = {"progress": "none"}
397
398    def test_help(self, capsys):
399        """tests getting help"""
400        options = super().test_help(capsys)
401        options = self.help_check(options, "-P, --pot")
402        options = self.help_check(options, "-t TEMPLATE, --template=TEMPLATE")
403        options = self.help_check(options, "--duplicates=DUPLICATESTYLE", last=True)
404