1import warnings 2from io import BytesIO 3 4from pytest import mark 5 6from translate.convert import test_convert 7from translate.storage import po, xliff 8from translate.tools import pretranslate 9 10 11class TestPretranslate: 12 xliff_skeleton = """<?xml version="1.0" encoding="utf-8"?> 13<xliff version="1.1" xmlns="urn:oasis:names:tc:xliff:document:1.1"> 14 <file original="doc.txt" source-language="en-US"> 15 <body> 16 %s 17 </body> 18 </file> 19</xliff>""" 20 21 def setup_method(self, method): 22 warnings.resetwarnings() 23 24 def teardown_method(self, method): 25 warnings.resetwarnings() 26 27 def pretranslatepo(self, input_source, template_source=None): 28 """helper that converts strings to po source without requiring files""" 29 input_file = BytesIO(input_source.encode()) 30 if template_source: 31 template_file = BytesIO(template_source.encode()) 32 else: 33 template_file = None 34 output_file = BytesIO() 35 36 pretranslate.pretranslate_file(input_file, output_file, template_file) 37 output_file.seek(0) 38 return po.pofile(output_file.read()) 39 40 def pretranslatexliff(self, input_source, template_source=None): 41 """helper that converts strings to po source without requiring files""" 42 input_file = BytesIO(input_source) 43 if template_source: 44 template_file = BytesIO(template_source) 45 else: 46 template_file = None 47 output_file = BytesIO() 48 49 pretranslate.pretranslate_file(input_file, output_file, template_file) 50 output_file.seek(0) 51 return xliff.xlifffile(output_file.read()) 52 53 def singleunit(self, pofile): 54 """ 55 checks that the pofile contains a single non-header unit, and 56 returns it 57 """ 58 if len(pofile.units) == 2 and pofile.units[0].isheader(): 59 print(pofile.units[1]) 60 return pofile.units[1] 61 else: 62 print(pofile.units[0]) 63 return pofile.units[0] 64 65 def test_pretranslatepo_blank(self): 66 """ 67 checks that the pretranslatepo function is working for a simple file 68 initialisation 69 """ 70 input_source = ( 71 """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n""" 72 % po.lsep 73 ) 74 newpo = self.pretranslatepo(input_source) 75 assert str(self.singleunit(newpo)) == input_source 76 77 def test_merging_simple(self): 78 """checks that the pretranslatepo function is working for a simple merge""" 79 input_source = ( 80 """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n""" 81 % po.lsep 82 ) 83 template_source = ( 84 """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n""" 85 % po.lsep 86 ) 87 newpo = self.pretranslatepo(input_source, template_source) 88 assert str(self.singleunit(newpo)) == template_source 89 90 def test_merging_messages_marked_fuzzy(self): 91 """test that when we merge PO files with a fuzzy message that it remains fuzzy""" 92 input_source = ( 93 """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n""" 94 % po.lsep 95 ) 96 template_source = ( 97 """#: simple.label%ssimple.accesskey\n#, fuzzy\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n""" 98 % po.lsep 99 ) 100 newpo = self.pretranslatepo(input_source, template_source) 101 assert str(self.singleunit(newpo)) == template_source 102 103 def test_merging_plurals_with_fuzzy_matching(self): 104 """test that when we merge PO files with a fuzzy message that it remains fuzzy""" 105 input_source = r"""#: file.cpp:2 106msgid "%d manual" 107msgid_plural "%d manuals" 108msgstr[0] "" 109msgstr[1] "" 110""" 111 template_source = r"""#: file.cpp:3 112#, fuzzy 113msgid "%d manual" 114msgid_plural "%d manuals" 115msgstr[0] "%d handleiding." 116msgstr[1] "%d handleidings." 117""" 118 # The #: comment and msgid's are different between the pot and the po 119 poexpected = r"""#: file.cpp:2 120#, fuzzy 121msgid "%d manual" 122msgid_plural "%d manuals" 123msgstr[0] "%d handleiding." 124msgstr[1] "%d handleidings." 125""" 126 newpo = self.pretranslatepo(input_source, template_source) 127 assert str(self.singleunit(newpo)) == poexpected 128 129 @mark.xfail(reason="Not Implemented") 130 def test_merging_msgid_change(self): 131 """ 132 tests that if the msgid changes but the location stays the same that 133 we merge 134 """ 135 input_source = """#: simple.label\n#: simple.accesskey\nmsgid "Its &hard coding a newline.\\n"\nmsgstr ""\n""" 136 template_source = """#: simple.label\n#: simple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n""" 137 poexpected = """#: simple.label\n#: simple.accesskey\n#, fuzzy\nmsgid "Its &hard coding a newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n""" 138 newpo = self.pretranslatepo(input_source, template_source) 139 print(bytes(newpo)) 140 assert bytes(newpo).decode("utf-8") == poexpected 141 142 def test_merging_location_change(self): 143 """ 144 tests that if the location changes but the msgid stays the same that 145 we merge 146 """ 147 input_source = ( 148 """#: new_simple.label%snew_simple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr ""\n""" 149 % po.lsep 150 ) 151 template_source = ( 152 """#: simple.label%ssimple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n""" 153 % po.lsep 154 ) 155 poexpected = ( 156 """#: new_simple.label%snew_simple.accesskey\nmsgid "A &hard coded newline.\\n"\nmsgstr "&Hart gekoeerde nuwe lyne\\n"\n""" 157 % po.lsep 158 ) 159 newpo = self.pretranslatepo(input_source, template_source) 160 print(bytes(newpo)) 161 assert bytes(newpo).decode("utf-8") == poexpected 162 163 def test_merging_location_and_whitespace_change(self): 164 """ 165 test that even if the location changes that if the msgid only has 166 whitespace changes we can still merge 167 """ 168 input_source = ( 169 """#: singlespace.label%ssinglespace.accesskey\nmsgid "&We have spaces"\nmsgstr ""\n""" 170 % po.lsep 171 ) 172 template_source = ( 173 """#: doublespace.label%sdoublespace.accesskey\nmsgid "&We have spaces"\nmsgstr "&One het spasies"\n""" 174 % po.lsep 175 ) 176 poexpected = ( 177 """#: singlespace.label%ssinglespace.accesskey\n#, fuzzy\nmsgid "&We have spaces"\nmsgstr "&One het spasies"\n""" 178 % po.lsep 179 ) 180 newpo = self.pretranslatepo(input_source, template_source) 181 print(bytes(newpo)) 182 assert bytes(newpo).decode("utf-8") == poexpected 183 184 @mark.xfail(reason="Not Implemented") 185 def test_merging_accelerator_changes(self): 186 """ 187 test that a change in the accelerator localtion still allows 188 merging 189 """ 190 input_source = """#: someline.c\nmsgid "A&bout"\nmsgstr ""\n""" 191 template_source = """#: someline.c\nmsgid "&About"\nmsgstr "&Info"\n""" 192 poexpected = """#: someline.c\nmsgid "A&bout"\nmsgstr "&Info"\n""" 193 newpo = self.pretranslatepo(input_source, template_source) 194 print(bytes(newpo)) 195 assert bytes(newpo).decode("utf-8") == poexpected 196 197 @mark.xfail(reason="Not Implemented") 198 def test_lines_cut_differently(self): 199 """ 200 Checks that the correct formatting is preserved when pot an po lines 201 differ. 202 """ 203 input_source = ( 204 """#: simple.label\nmsgid "Line split "\n"differently"\nmsgstr ""\n""" 205 ) 206 template_source = """#: simple.label\nmsgid "Line"\n" split differently"\nmsgstr "Lyne verskillend gesny"\n""" 207 newpo = self.pretranslatepo(input_source, template_source) 208 newpounit = self.singleunit(newpo) 209 assert str(newpounit) == template_source 210 211 def test_merging_automatic_comments_dont_duplicate(self): 212 """ensure that we can merge #. comments correctly""" 213 input_source = """#. Row 35\nmsgid "&About"\nmsgstr ""\n""" 214 template_source = """#. Row 35\nmsgid "&About"\nmsgstr "&Info"\n""" 215 newpo = self.pretranslatepo(input_source, template_source) 216 newpounit = self.singleunit(newpo) 217 assert str(newpounit) == template_source 218 219 def test_merging_automatic_comments_new_overides_old(self): 220 """ensure that new #. comments override the old comments""" 221 input_source = """#. new comment\n#: someline.c\nmsgid "&About"\nmsgstr ""\n""" 222 template_source = ( 223 """#. old comment\n#: someline.c\nmsgid "&About"\nmsgstr "&Info"\n""" 224 ) 225 poexpected = ( 226 """#. new comment\n#: someline.c\nmsgid "&About"\nmsgstr "&Info"\n""" 227 ) 228 newpo = self.pretranslatepo(input_source, template_source) 229 newpounit = self.singleunit(newpo) 230 assert str(newpounit) == poexpected 231 232 def test_merging_comments_with_blank_comment_lines(self): 233 """ 234 test that when we merge a comment that has a blank line we keep the 235 blank line 236 """ 237 input_source = """#: someline.c\nmsgid "About"\nmsgstr ""\n""" 238 template_source = """# comment1\n#\n# comment2\n#: someline.c\nmsgid "About"\nmsgstr "Omtrent"\n""" 239 poexpected = template_source 240 newpo = self.pretranslatepo(input_source, template_source) 241 newpounit = self.singleunit(newpo) 242 assert str(newpounit) == poexpected 243 244 def test_empty_commentlines(self): 245 input_source = """#: paneSecurity.title 246msgid "Security" 247msgstr "" 248""" 249 template_source = """# - Contributor(s): 250# - 251# - Alternatively, the 252# - 253#: paneSecurity.title 254msgid "Security" 255msgstr "Sekuriteit" 256""" 257 poexpected = template_source 258 newpo = self.pretranslatepo(input_source, template_source) 259 newpounit = self.singleunit(newpo) 260 print("expected") 261 print(poexpected) 262 print("got:") 263 print(str(newpounit)) 264 assert str(newpounit) == poexpected 265 266 def test_merging_msgidcomments(self): 267 """ensure that we can merge msgidcomments messages""" 268 input_source = r"""#: window.width 269msgid "" 270"_: Do not translate this.\n" 271"36em" 272msgstr "" 273""" 274 template_source = r"""#: window.width 275msgid "" 276"_: Do not translate this.\n" 277"36em" 278msgstr "36em" 279""" 280 newpo = self.pretranslatepo(input_source, template_source) 281 newpounit = self.singleunit(newpo) 282 assert str(newpounit) == template_source 283 284 def test_merging_plurals(self): 285 """ensure that we can merge plural messages""" 286 input_source = ( 287 """msgid "One"\nmsgid_plural "Two"\nmsgstr[0] ""\nmsgstr[1] ""\n""" 288 ) 289 template_source = """msgid "One"\nmsgid_plural "Two"\nmsgstr[0] "Een"\nmsgstr[1] "Twee"\nmsgstr[2] "Drie"\n""" 290 newpo = self.pretranslatepo(input_source, template_source) 291 print(newpo) 292 newpounit = self.singleunit(newpo) 293 assert str(newpounit) == template_source 294 295 def test_merging_resurect_obsolete_messages(self): 296 """ 297 check that we can reuse old obsolete messages if the message comes 298 back 299 """ 300 input_source = """#: resurect.c\nmsgid "&About"\nmsgstr ""\n""" 301 template_source = """#~ msgid "&About"\n#~ msgstr "&Omtrent"\n""" 302 expected = """#: resurect.c\nmsgid "&About"\nmsgstr "&Omtrent"\n""" 303 newpo = self.pretranslatepo(input_source, template_source) 304 print(bytes(newpo)) 305 assert bytes(newpo).decode("utf-8") == expected 306 307 def test_merging_comments(self): 308 """Test that we can merge comments correctly""" 309 input_source = """#. Don't do it!\n#: file.py:1\nmsgid "One"\nmsgstr ""\n""" 310 template_source = ( 311 """#. Don't do it!\n#: file.py:2\nmsgid "One"\nmsgstr "Een"\n""" 312 ) 313 poexpected = """#. Don't do it!\n#: file.py:1\nmsgid "One"\nmsgstr "Een"\n""" 314 newpo = self.pretranslatepo(input_source, template_source) 315 print(newpo) 316 newpounit = self.singleunit(newpo) 317 assert str(newpounit) == poexpected 318 319 def test_merging_typecomments(self): 320 """Test that we can merge with typecomments""" 321 input_source = """#: file.c:1\n#, c-format\nmsgid "%d pipes"\nmsgstr ""\n""" 322 template_source = """#: file.c:2\nmsgid "%d pipes"\nmsgstr "%d pype"\n""" 323 poexpected = ( 324 """#: file.c:1\n#, c-format\nmsgid "%d pipes"\nmsgstr "%d pype"\n""" 325 ) 326 newpo = self.pretranslatepo(input_source, template_source) 327 newpounit = self.singleunit(newpo) 328 print(newpounit) 329 assert str(newpounit) == poexpected 330 331 input_source = """#: file.c:1\n#, c-format\nmsgid "%d computers"\nmsgstr ""\n""" 332 template_source = """#: file.c:2\n#, c-format\nmsgid "%s computers "\nmsgstr "%s-rekenaars"\n""" 333 poexpected = """#: file.c:1\n#, fuzzy, c-format\nmsgid "%d computers"\nmsgstr "%s-rekenaars"\n""" 334 newpo = self.pretranslatepo(input_source, template_source) 335 newpounit = self.singleunit(newpo) 336 assert newpounit.isfuzzy() 337 assert newpounit.hastypecomment("c-format") 338 339 def test_xliff_states(self): 340 """Test correct maintenance of XLIFF states.""" 341 xlf_template = self.xliff_skeleton % ( 342 """<trans-unit id="1" xml:space="preserve"> 343 <source> File 1 </source> 344 </trans-unit>""" 345 ) 346 xlf_old = self.xliff_skeleton % ( 347 """<trans-unit id="1" xml:space="preserve" approved="yes"> 348 <source> File 1 </source> 349 <target> Lêer 1 </target> 350 </trans-unit>""" 351 ) 352 353 template = xliff.xlifffile.parsestring(xlf_template) 354 old = xliff.xlifffile.parsestring(xlf_old) 355 # Serialize files 356 new = self.pretranslatexliff(bytes(template), bytes(old)) 357 print(bytes(old)) 358 print("---") 359 print(bytes(new)) 360 assert new.units[0].isapproved() 361 # Layout might have changed, so we won't compare the serialised 362 # versions 363 364 365class TestPretranslateCommand(test_convert.TestConvertCommand, TestPretranslate): 366 """Tests running actual pretranslate commands on files""" 367 368 convertmodule = pretranslate 369 370 def test_help(self, capsys): 371 """tests getting help""" 372 options = super().test_help(capsys) 373 options = self.help_check(options, "-t TEMPLATE, --template=TEMPLATE") 374 options = self.help_check(options, "--tm") 375 options = self.help_check( 376 options, "-s MIN_SIMILARITY, --similarity=MIN_SIMILARITY" 377 ) 378 options = self.help_check(options, "--nofuzzymatching", last=True) 379