1# -*- coding: utf-8 -*- 2# This Source Code Form is subject to the terms of the Mozilla Public 3# License, v. 2.0. If a copy of the MPL was not distributed with this 4# file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6import unittest 7 8from compare_locales.checks import getChecker 9from compare_locales.parser import getParser, Entity 10from compare_locales.paths import File 11 12 13class BaseHelper(unittest.TestCase): 14 file = None 15 refContent = None 16 17 def setUp(self): 18 p = getParser(self.file.file) 19 p.readContents(self.refContent) 20 self.refList, self.refMap = p.parse() 21 22 def _test(self, content, refWarnOrErrors, with_ref_file=False): 23 p = getParser(self.file.file) 24 p.readContents(content) 25 l10n = [e for e in p] 26 assert len(l10n) == 1 27 l10n = l10n[0] 28 if with_ref_file: 29 kwargs = { 30 'reference': self.refList 31 } 32 else: 33 kwargs = {} 34 checker = getChecker(self.file, **kwargs) 35 ref = self.refList[self.refMap[l10n.key]] 36 found = tuple(checker.check(ref, l10n)) 37 self.assertEqual(found, refWarnOrErrors) 38 39 40class TestProperties(BaseHelper): 41 file = File('foo.properties', 'foo.properties') 42 refContent = '''some = value 43''' 44 45 def testGood(self): 46 self._test('''some = localized''', 47 tuple()) 48 49 def testMissedEscape(self): 50 self._test(r'''some = \u67ood escape, bad \escape''', 51 (('warning', 20, r'unknown escape sequence, \e', 52 'escape'),)) 53 54 55class TestPlurals(BaseHelper): 56 file = File('foo.properties', 'foo.properties') 57 refContent = '''\ 58# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. 59# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals 60# #1 number of files 61# example: 111 files - Downloads 62downloadsTitleFiles=#1 file - Downloads;#1 files - #2 63''' 64 65 def testGood(self): 66 self._test('''\ 67# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. 68# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals 69# #1 number of files 70# example: 111 files - Downloads 71downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 filers 72''', 73 tuple()) 74 75 def testNotUsed(self): 76 self._test('''\ 77# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. 78# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals 79# #1 number of files 80# example: 111 files - Downloads 81downloadsTitleFiles=#1 file - Downloads;#1 files - Downloads;#1 filers 82''', 83 (('warning', 0, 'not all variables used in l10n', 84 'plural'),)) 85 86 def testNotDefined(self): 87 self._test('''\ 88# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms. 89# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals 90# #1 number of files 91# example: 111 files - Downloads 92downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 #3 93''', 94 (('error', 0, 'unreplaced variables in l10n', 'plural'),)) 95 96 97class TestDTDs(BaseHelper): 98 file = File('foo.dtd', 'foo.dtd') 99 refContent = '''<!ENTITY foo "This is 'good'"> 100<!ENTITY width "10ch"> 101<!ENTITY style "width: 20ch; height: 280px;"> 102<!ENTITY minStyle "min-height: 50em;"> 103<!ENTITY ftd "0"> 104<!ENTITY formatPercent "This is 100% correct"> 105<!ENTITY some.key "K"> 106''' 107 108 def testWarning(self): 109 self._test('''<!ENTITY foo "This is ¬ good"> 110''', 111 (('warning', (0, 0), 'Referencing unknown entity `not`', 112 'xmlparse'),)) 113 # make sure we only handle translated entity references 114 self._test(u'''<!ENTITY foo "This is &ƞǿŧ; good"> 115'''.encode('utf-8'), 116 (('warning', (0, 0), u'Referencing unknown entity `ƞǿŧ`', 117 'xmlparse'),)) 118 119 def testErrorFirstLine(self): 120 self._test('''<!ENTITY foo "This is </bad> stuff"> 121''', 122 (('error', (1, 10), 'mismatched tag', 'xmlparse'),)) 123 124 def testErrorSecondLine(self): 125 self._test('''<!ENTITY foo "This is 126 </bad> 127stuff"> 128''', 129 (('error', (2, 4), 'mismatched tag', 'xmlparse'),)) 130 131 def testKeyErrorSingleAmpersand(self): 132 self._test('''<!ENTITY some.key "&"> 133''', 134 (('error', (1, 1), 'not well-formed (invalid token)', 135 'xmlparse'),)) 136 137 def testXMLEntity(self): 138 self._test('''<!ENTITY foo "This is "good""> 139''', 140 tuple()) 141 142 def testPercentEntity(self): 143 self._test('''<!ENTITY formatPercent "Another 100%"> 144''', 145 tuple()) 146 self._test('''<!ENTITY formatPercent "Bad 100% should fail"> 147''', 148 (('error', (0, 32), 'not well-formed (invalid token)', 149 'xmlparse'),)) 150 151 def testNoNumber(self): 152 self._test('''<!ENTITY ftd "foo">''', 153 (('warning', 0, 'reference is a number', 'number'),)) 154 155 def testNoLength(self): 156 self._test('''<!ENTITY width "15miles">''', 157 (('error', 0, 'reference is a CSS length', 'css'),)) 158 159 def testNoStyle(self): 160 self._test('''<!ENTITY style "15ch">''', 161 (('error', 0, 'reference is a CSS spec', 'css'),)) 162 self._test('''<!ENTITY style "junk">''', 163 (('error', 0, 'reference is a CSS spec', 'css'),)) 164 165 def testStyleWarnings(self): 166 self._test('''<!ENTITY style "width:15ch">''', 167 (('warning', 0, 'height only in reference', 'css'),)) 168 self._test('''<!ENTITY style "width:15em;height:200px;">''', 169 (('warning', 0, "units for width don't match (em != ch)", 170 'css'),)) 171 172 def testNoWarning(self): 173 self._test('''<!ENTITY width "12em">''', tuple()) 174 self._test('''<!ENTITY style "width:12ch;height:200px;">''', tuple()) 175 self._test('''<!ENTITY ftd "0">''', tuple()) 176 177 178class TestEntitiesInDTDs(BaseHelper): 179 file = File('foo.dtd', 'foo.dtd') 180 refContent = '''<!ENTITY short "This is &brandShortName;"> 181<!ENTITY shorter "This is &brandShorterName;"> 182<!ENTITY ent.start "Using &brandShorterName; start to"> 183<!ENTITY ent.end " end"> 184''' 185 186 def testOK(self): 187 self._test('''<!ENTITY ent.start "Mit &brandShorterName;">''', tuple(), 188 with_ref_file=True) 189 190 def testMismatch(self): 191 self._test('''<!ENTITY ent.start "Mit &brandShortName;">''', 192 (('warning', (0, 0), 193 'Entity brandShortName referenced, ' 194 'but brandShorterName used in context', 195 'xmlparse'),), 196 with_ref_file=True) 197 198 def testAcross(self): 199 self._test('''<!ENTITY ent.end "Mit &brandShorterName;">''', 200 tuple(), 201 with_ref_file=True) 202 203 def testAcrossWithMismatch(self): 204 '''If we could tell that ent.start and ent.end are one string, 205 we should warn. Sadly, we can't, so this goes without warning.''' 206 self._test('''<!ENTITY ent.end "Mit &brandShortName;">''', 207 tuple(), 208 with_ref_file=True) 209 210 def testUnknownWithRef(self): 211 self._test('''<!ENTITY ent.start "Mit &foopy;">''', 212 (('warning', 213 (0, 0), 214 'Referencing unknown entity `foopy` ' 215 '(brandShorterName used in context, ' 216 'brandShortName known)', 217 'xmlparse'),), 218 with_ref_file=True) 219 220 def testUnknown(self): 221 self._test('''<!ENTITY ent.end "Mit &foopy;">''', 222 (('warning', 223 (0, 0), 224 'Referencing unknown entity `foopy`' 225 ' (brandShortName, brandShorterName known)', 226 'xmlparse'),), 227 with_ref_file=True) 228 229 230class TestAndroid(unittest.TestCase): 231 """Test Android checker 232 233 Make sure we're hitting our extra rules only if 234 we're passing in a DTD file in the embedding/android module. 235 """ 236 apos_msg = u"Apostrophes in Android DTDs need escaping with \\' or " + \ 237 u"\\u0027, or use \u2019, or put string in quotes." 238 quot_msg = u"Quotes in Android DTDs need escaping with \\\" or " + \ 239 u"\\u0022, or put string in apostrophes." 240 241 def getEntity(self, v): 242 return Entity(v, lambda s: s, (0, len(v)), (), (0, 0), (), (), 243 (0, len(v)), ()) 244 245 def getDTDEntity(self, v): 246 v = v.replace('"', '"') 247 return Entity('<!ENTITY foo "%s">' % v, 248 lambda s: s, 249 (0, len(v) + 16), (), (0, 0), (), (9, 12), 250 (14, len(v) + 14), ()) 251 252 def test_android_dtd(self): 253 """Testing the actual android checks. The logic is involved, 254 so this is a lot of nitty gritty detail tests. 255 """ 256 f = File("embedding/android/strings.dtd", "strings.dtd", 257 "embedding/android") 258 checker = getChecker(f) 259 # good string 260 ref = self.getDTDEntity("plain string") 261 l10n = self.getDTDEntity("plain localized string") 262 self.assertEqual(tuple(checker.check(ref, l10n)), 263 ()) 264 # dtd warning 265 l10n = self.getDTDEntity("plain localized string &ref;") 266 self.assertEqual(tuple(checker.check(ref, l10n)), 267 (('warning', (0, 0), 268 'Referencing unknown entity `ref`', 'xmlparse'),)) 269 # no report on stray ampersand or quote, if not completely quoted 270 for i in xrange(3): 271 # make sure we're catching unescaped apostrophes, 272 # try 0..5 backticks 273 l10n = self.getDTDEntity("\\"*(2*i) + "'") 274 self.assertEqual(tuple(checker.check(ref, l10n)), 275 (('error', 2*i, self.apos_msg, 'android'),)) 276 l10n = self.getDTDEntity("\\"*(2*i + 1) + "'") 277 self.assertEqual(tuple(checker.check(ref, l10n)), 278 ()) 279 # make sure we don't report if apos string is quoted 280 l10n = self.getDTDEntity('"' + "\\"*(2*i) + "'\"") 281 tpl = tuple(checker.check(ref, l10n)) 282 self.assertEqual(tpl, (), 283 "`%s` shouldn't fail but got %s" 284 % (l10n.val, str(tpl))) 285 l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"") 286 tpl = tuple(checker.check(ref, l10n)) 287 self.assertEqual(tpl, (), 288 "`%s` shouldn't fail but got %s" 289 % (l10n.val, str(tpl))) 290 # make sure we're catching unescaped quotes, try 0..5 backticks 291 l10n = self.getDTDEntity("\\"*(2*i) + "\"") 292 self.assertEqual(tuple(checker.check(ref, l10n)), 293 (('error', 2*i, self.quot_msg, 'android'),)) 294 l10n = self.getDTDEntity("\\"*(2*i + 1) + "'") 295 self.assertEqual(tuple(checker.check(ref, l10n)), 296 ()) 297 # make sure we don't report if quote string is single quoted 298 l10n = self.getDTDEntity("'" + "\\"*(2*i) + "\"'") 299 tpl = tuple(checker.check(ref, l10n)) 300 self.assertEqual(tpl, (), 301 "`%s` shouldn't fail but got %s" % 302 (l10n.val, str(tpl))) 303 l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"") 304 tpl = tuple(checker.check(ref, l10n)) 305 self.assertEqual(tpl, (), 306 "`%s` shouldn't fail but got %s" % 307 (l10n.val, str(tpl))) 308 # check for mixed quotes and ampersands 309 l10n = self.getDTDEntity("'\"") 310 self.assertEqual(tuple(checker.check(ref, l10n)), 311 (('error', 0, self.apos_msg, 'android'), 312 ('error', 1, self.quot_msg, 'android'))) 313 l10n = self.getDTDEntity("''\"'") 314 self.assertEqual(tuple(checker.check(ref, l10n)), 315 (('error', 1, self.apos_msg, 'android'),)) 316 l10n = self.getDTDEntity('"\'""') 317 self.assertEqual(tuple(checker.check(ref, l10n)), 318 (('error', 2, self.quot_msg, 'android'),)) 319 320 # broken unicode escape 321 l10n = self.getDTDEntity("Some broken \u098 unicode") 322 self.assertEqual(tuple(checker.check(ref, l10n)), 323 (('error', 12, 'truncated \\uXXXX escape', 324 'android'),)) 325 # broken unicode escape, try to set the error off 326 l10n = self.getDTDEntity(u"\u9690"*14+"\u006"+" "+"\u0064") 327 self.assertEqual(tuple(checker.check(ref, l10n)), 328 (('error', 14, 'truncated \\uXXXX escape', 329 'android'),)) 330 331 def test_android_prop(self): 332 f = File("embedding/android/strings.properties", "strings.properties", 333 "embedding/android") 334 checker = getChecker(f) 335 # good plain string 336 ref = self.getEntity("plain string") 337 l10n = self.getEntity("plain localized string") 338 self.assertEqual(tuple(checker.check(ref, l10n)), 339 ()) 340 # no dtd warning 341 ref = self.getEntity("plain string") 342 l10n = self.getEntity("plain localized string &ref;") 343 self.assertEqual(tuple(checker.check(ref, l10n)), 344 ()) 345 # no report on stray ampersand 346 ref = self.getEntity("plain string") 347 l10n = self.getEntity("plain localized string with apos: '") 348 self.assertEqual(tuple(checker.check(ref, l10n)), 349 ()) 350 # report on bad printf 351 ref = self.getEntity("string with %s") 352 l10n = self.getEntity("string with %S") 353 self.assertEqual(tuple(checker.check(ref, l10n)), 354 (('error', 0, 'argument 1 `S` should be `s`', 355 'printf'),)) 356 357 def test_non_android_dtd(self): 358 f = File("browser/strings.dtd", "strings.dtd", "browser") 359 checker = getChecker(f) 360 # good string 361 ref = self.getDTDEntity("plain string") 362 l10n = self.getDTDEntity("plain localized string") 363 self.assertEqual(tuple(checker.check(ref, l10n)), 364 ()) 365 # dtd warning 366 ref = self.getDTDEntity("plain string") 367 l10n = self.getDTDEntity("plain localized string &ref;") 368 self.assertEqual(tuple(checker.check(ref, l10n)), 369 (('warning', (0, 0), 370 'Referencing unknown entity `ref`', 'xmlparse'),)) 371 # no report on stray ampersand 372 ref = self.getDTDEntity("plain string") 373 l10n = self.getDTDEntity("plain localized string with apos: '") 374 self.assertEqual(tuple(checker.check(ref, l10n)), 375 ()) 376 377 def test_entities_across_dtd(self): 378 f = File("browser/strings.dtd", "strings.dtd", "browser") 379 p = getParser(f.file) 380 p.readContents('<!ENTITY other "some &good.ref;">') 381 ref = p.parse() 382 checker = getChecker(f, reference=ref[0]) 383 # good string 384 ref = self.getDTDEntity("plain string") 385 l10n = self.getDTDEntity("plain localized string") 386 self.assertEqual(tuple(checker.check(ref, l10n)), 387 ()) 388 # dtd warning 389 ref = self.getDTDEntity("plain string") 390 l10n = self.getDTDEntity("plain localized string &ref;") 391 self.assertEqual(tuple(checker.check(ref, l10n)), 392 (('warning', (0, 0), 393 'Referencing unknown entity `ref` (good.ref known)', 394 'xmlparse'),)) 395 # no report on stray ampersand 396 ref = self.getDTDEntity("plain string") 397 l10n = self.getDTDEntity("plain localized string with &good.ref;") 398 self.assertEqual(tuple(checker.check(ref, l10n)), 399 ()) 400 401 402if __name__ == '__main__': 403 unittest.main() 404