1# -*- coding: utf-8 -*-
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this
4# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6import unittest
7
8from compare_locales.checks import getChecker
9from compare_locales.parser import getParser, Entity
10from compare_locales.paths import File
11
12
13class BaseHelper(unittest.TestCase):
14    file = None
15    refContent = None
16
17    def setUp(self):
18        p = getParser(self.file.file)
19        p.readContents(self.refContent)
20        self.refList, self.refMap = p.parse()
21
22    def _test(self, content, refWarnOrErrors, with_ref_file=False):
23        p = getParser(self.file.file)
24        p.readContents(content)
25        l10n = [e for e in p]
26        assert len(l10n) == 1
27        l10n = l10n[0]
28        if with_ref_file:
29            kwargs = {
30                'reference': self.refList
31            }
32        else:
33            kwargs = {}
34        checker = getChecker(self.file, **kwargs)
35        ref = self.refList[self.refMap[l10n.key]]
36        found = tuple(checker.check(ref, l10n))
37        self.assertEqual(found, refWarnOrErrors)
38
39
40class TestProperties(BaseHelper):
41    file = File('foo.properties', 'foo.properties')
42    refContent = '''some = value
43'''
44
45    def testGood(self):
46        self._test('''some = localized''',
47                   tuple())
48
49    def testMissedEscape(self):
50        self._test(r'''some = \u67ood escape, bad \escape''',
51                   (('warning', 20, r'unknown escape sequence, \e',
52                     'escape'),))
53
54
55class TestPlurals(BaseHelper):
56    file = File('foo.properties', 'foo.properties')
57    refContent = '''\
58# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
59# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
60# #1 number of files
61# example: 111 files - Downloads
62downloadsTitleFiles=#1 file - Downloads;#1 files - #2
63'''
64
65    def testGood(self):
66        self._test('''\
67# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
68# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
69# #1 number of files
70# example: 111 files - Downloads
71downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 filers
72''',
73                   tuple())
74
75    def testNotUsed(self):
76        self._test('''\
77# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
78# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
79# #1 number of files
80# example: 111 files - Downloads
81downloadsTitleFiles=#1 file - Downloads;#1 files - Downloads;#1 filers
82''',
83                   (('warning', 0, 'not all variables used in l10n',
84                     'plural'),))
85
86    def testNotDefined(self):
87        self._test('''\
88# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
89# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
90# #1 number of files
91# example: 111 files - Downloads
92downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 #3
93''',
94                   (('error', 0, 'unreplaced variables in l10n', 'plural'),))
95
96
97class TestDTDs(BaseHelper):
98    file = File('foo.dtd', 'foo.dtd')
99    refContent = '''<!ENTITY foo "This is &apos;good&apos;">
100<!ENTITY width "10ch">
101<!ENTITY style "width: 20ch; height: 280px;">
102<!ENTITY minStyle "min-height: 50em;">
103<!ENTITY ftd "0">
104<!ENTITY formatPercent "This is 100&#037; correct">
105<!ENTITY some.key "K">
106'''
107
108    def testWarning(self):
109        self._test('''<!ENTITY foo "This is &not; good">
110''',
111                   (('warning', (0, 0), 'Referencing unknown entity `not`',
112                     'xmlparse'),))
113        # make sure we only handle translated entity references
114        self._test(u'''<!ENTITY foo "This is &ƞǿŧ; good">
115'''.encode('utf-8'),
116            (('warning', (0, 0), u'Referencing unknown entity `ƞǿŧ`',
117              'xmlparse'),))
118
119    def testErrorFirstLine(self):
120        self._test('''<!ENTITY foo "This is </bad> stuff">
121''',
122                   (('error', (1, 10), 'mismatched tag', 'xmlparse'),))
123
124    def testErrorSecondLine(self):
125        self._test('''<!ENTITY foo "This is
126  </bad>
127stuff">
128''',
129                   (('error', (2, 4), 'mismatched tag', 'xmlparse'),))
130
131    def testKeyErrorSingleAmpersand(self):
132        self._test('''<!ENTITY some.key "&">
133''',
134                   (('error', (1, 1), 'not well-formed (invalid token)',
135                     'xmlparse'),))
136
137    def testXMLEntity(self):
138        self._test('''<!ENTITY foo "This is &quot;good&quot;">
139''',
140                   tuple())
141
142    def testPercentEntity(self):
143        self._test('''<!ENTITY formatPercent "Another 100&#037;">
144''',
145                   tuple())
146        self._test('''<!ENTITY formatPercent "Bad 100% should fail">
147''',
148                   (('error', (0, 32), 'not well-formed (invalid token)',
149                     'xmlparse'),))
150
151    def testNoNumber(self):
152        self._test('''<!ENTITY ftd "foo">''',
153                   (('warning', 0, 'reference is a number', 'number'),))
154
155    def testNoLength(self):
156        self._test('''<!ENTITY width "15miles">''',
157                   (('error', 0, 'reference is a CSS length', 'css'),))
158
159    def testNoStyle(self):
160        self._test('''<!ENTITY style "15ch">''',
161                   (('error', 0, 'reference is a CSS spec', 'css'),))
162        self._test('''<!ENTITY style "junk">''',
163                   (('error', 0, 'reference is a CSS spec', 'css'),))
164
165    def testStyleWarnings(self):
166        self._test('''<!ENTITY style "width:15ch">''',
167                   (('warning', 0, 'height only in reference', 'css'),))
168        self._test('''<!ENTITY style "width:15em;height:200px;">''',
169                   (('warning', 0, "units for width don't match (em != ch)",
170                     'css'),))
171
172    def testNoWarning(self):
173        self._test('''<!ENTITY width "12em">''', tuple())
174        self._test('''<!ENTITY style "width:12ch;height:200px;">''', tuple())
175        self._test('''<!ENTITY ftd "0">''', tuple())
176
177
178class TestEntitiesInDTDs(BaseHelper):
179    file = File('foo.dtd', 'foo.dtd')
180    refContent = '''<!ENTITY short "This is &brandShortName;">
181<!ENTITY shorter "This is &brandShorterName;">
182<!ENTITY ent.start "Using &brandShorterName; start to">
183<!ENTITY ent.end " end">
184'''
185
186    def testOK(self):
187        self._test('''<!ENTITY ent.start "Mit &brandShorterName;">''', tuple(),
188                   with_ref_file=True)
189
190    def testMismatch(self):
191        self._test('''<!ENTITY ent.start "Mit &brandShortName;">''',
192                   (('warning', (0, 0),
193                     'Entity brandShortName referenced, '
194                     'but brandShorterName used in context',
195                     'xmlparse'),),
196                   with_ref_file=True)
197
198    def testAcross(self):
199        self._test('''<!ENTITY ent.end "Mit &brandShorterName;">''',
200                   tuple(),
201                   with_ref_file=True)
202
203    def testAcrossWithMismatch(self):
204        '''If we could tell that ent.start and ent.end are one string,
205        we should warn. Sadly, we can't, so this goes without warning.'''
206        self._test('''<!ENTITY ent.end "Mit &brandShortName;">''',
207                   tuple(),
208                   with_ref_file=True)
209
210    def testUnknownWithRef(self):
211        self._test('''<!ENTITY ent.start "Mit &foopy;">''',
212                   (('warning',
213                     (0, 0),
214                     'Referencing unknown entity `foopy` '
215                     '(brandShorterName used in context, '
216                     'brandShortName known)',
217                     'xmlparse'),),
218                   with_ref_file=True)
219
220    def testUnknown(self):
221        self._test('''<!ENTITY ent.end "Mit &foopy;">''',
222                   (('warning',
223                     (0, 0),
224                     'Referencing unknown entity `foopy`'
225                     ' (brandShortName, brandShorterName known)',
226                     'xmlparse'),),
227                   with_ref_file=True)
228
229
230class TestAndroid(unittest.TestCase):
231    """Test Android checker
232
233    Make sure we're hitting our extra rules only if
234    we're passing in a DTD file in the embedding/android module.
235    """
236    apos_msg = u"Apostrophes in Android DTDs need escaping with \\' or " + \
237               u"\\u0027, or use \u2019, or put string in quotes."
238    quot_msg = u"Quotes in Android DTDs need escaping with \\\" or " + \
239               u"\\u0022, or put string in apostrophes."
240
241    def getEntity(self, v):
242        return Entity(v, lambda s: s, (0, len(v)), (), (0, 0), (), (),
243                      (0, len(v)), ())
244
245    def getDTDEntity(self, v):
246        v = v.replace('"', '&quot;')
247        return Entity('<!ENTITY foo "%s">' % v,
248                      lambda s: s,
249                      (0, len(v) + 16), (), (0, 0), (), (9, 12),
250                      (14, len(v) + 14), ())
251
252    def test_android_dtd(self):
253        """Testing the actual android checks. The logic is involved,
254        so this is a lot of nitty gritty detail tests.
255        """
256        f = File("embedding/android/strings.dtd", "strings.dtd",
257                 "embedding/android")
258        checker = getChecker(f)
259        # good string
260        ref = self.getDTDEntity("plain string")
261        l10n = self.getDTDEntity("plain localized string")
262        self.assertEqual(tuple(checker.check(ref, l10n)),
263                         ())
264        # dtd warning
265        l10n = self.getDTDEntity("plain localized string &ref;")
266        self.assertEqual(tuple(checker.check(ref, l10n)),
267                         (('warning', (0, 0),
268                           'Referencing unknown entity `ref`', 'xmlparse'),))
269        # no report on stray ampersand or quote, if not completely quoted
270        for i in xrange(3):
271            # make sure we're catching unescaped apostrophes,
272            # try 0..5 backticks
273            l10n = self.getDTDEntity("\\"*(2*i) + "'")
274            self.assertEqual(tuple(checker.check(ref, l10n)),
275                             (('error', 2*i, self.apos_msg, 'android'),))
276            l10n = self.getDTDEntity("\\"*(2*i + 1) + "'")
277            self.assertEqual(tuple(checker.check(ref, l10n)),
278                             ())
279            # make sure we don't report if apos string is quoted
280            l10n = self.getDTDEntity('"' + "\\"*(2*i) + "'\"")
281            tpl = tuple(checker.check(ref, l10n))
282            self.assertEqual(tpl, (),
283                             "`%s` shouldn't fail but got %s"
284                             % (l10n.val, str(tpl)))
285            l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"")
286            tpl = tuple(checker.check(ref, l10n))
287            self.assertEqual(tpl, (),
288                             "`%s` shouldn't fail but got %s"
289                             % (l10n.val, str(tpl)))
290            # make sure we're catching unescaped quotes, try 0..5 backticks
291            l10n = self.getDTDEntity("\\"*(2*i) + "\"")
292            self.assertEqual(tuple(checker.check(ref, l10n)),
293                             (('error', 2*i, self.quot_msg, 'android'),))
294            l10n = self.getDTDEntity("\\"*(2*i + 1) + "'")
295            self.assertEqual(tuple(checker.check(ref, l10n)),
296                             ())
297            # make sure we don't report if quote string is single quoted
298            l10n = self.getDTDEntity("'" + "\\"*(2*i) + "\"'")
299            tpl = tuple(checker.check(ref, l10n))
300            self.assertEqual(tpl, (),
301                             "`%s` shouldn't fail but got %s" %
302                             (l10n.val, str(tpl)))
303            l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"")
304            tpl = tuple(checker.check(ref, l10n))
305            self.assertEqual(tpl, (),
306                             "`%s` shouldn't fail but got %s" %
307                             (l10n.val, str(tpl)))
308        # check for mixed quotes and ampersands
309        l10n = self.getDTDEntity("'\"")
310        self.assertEqual(tuple(checker.check(ref, l10n)),
311                         (('error', 0, self.apos_msg, 'android'),
312                          ('error', 1, self.quot_msg, 'android')))
313        l10n = self.getDTDEntity("''\"'")
314        self.assertEqual(tuple(checker.check(ref, l10n)),
315                         (('error', 1, self.apos_msg, 'android'),))
316        l10n = self.getDTDEntity('"\'""')
317        self.assertEqual(tuple(checker.check(ref, l10n)),
318                         (('error', 2, self.quot_msg, 'android'),))
319
320        # broken unicode escape
321        l10n = self.getDTDEntity("Some broken \u098 unicode")
322        self.assertEqual(tuple(checker.check(ref, l10n)),
323                         (('error', 12, 'truncated \\uXXXX escape',
324                           'android'),))
325        # broken unicode escape, try to set the error off
326        l10n = self.getDTDEntity(u"\u9690"*14+"\u006"+"  "+"\u0064")
327        self.assertEqual(tuple(checker.check(ref, l10n)),
328                         (('error', 14, 'truncated \\uXXXX escape',
329                           'android'),))
330
331    def test_android_prop(self):
332        f = File("embedding/android/strings.properties", "strings.properties",
333                 "embedding/android")
334        checker = getChecker(f)
335        # good plain string
336        ref = self.getEntity("plain string")
337        l10n = self.getEntity("plain localized string")
338        self.assertEqual(tuple(checker.check(ref, l10n)),
339                         ())
340        # no dtd warning
341        ref = self.getEntity("plain string")
342        l10n = self.getEntity("plain localized string &ref;")
343        self.assertEqual(tuple(checker.check(ref, l10n)),
344                         ())
345        # no report on stray ampersand
346        ref = self.getEntity("plain string")
347        l10n = self.getEntity("plain localized string with apos: '")
348        self.assertEqual(tuple(checker.check(ref, l10n)),
349                         ())
350        # report on bad printf
351        ref = self.getEntity("string with %s")
352        l10n = self.getEntity("string with %S")
353        self.assertEqual(tuple(checker.check(ref, l10n)),
354                         (('error', 0, 'argument 1 `S` should be `s`',
355                           'printf'),))
356
357    def test_non_android_dtd(self):
358        f = File("browser/strings.dtd", "strings.dtd", "browser")
359        checker = getChecker(f)
360        # good string
361        ref = self.getDTDEntity("plain string")
362        l10n = self.getDTDEntity("plain localized string")
363        self.assertEqual(tuple(checker.check(ref, l10n)),
364                         ())
365        # dtd warning
366        ref = self.getDTDEntity("plain string")
367        l10n = self.getDTDEntity("plain localized string &ref;")
368        self.assertEqual(tuple(checker.check(ref, l10n)),
369                         (('warning', (0, 0),
370                          'Referencing unknown entity `ref`', 'xmlparse'),))
371        # no report on stray ampersand
372        ref = self.getDTDEntity("plain string")
373        l10n = self.getDTDEntity("plain localized string with apos: '")
374        self.assertEqual(tuple(checker.check(ref, l10n)),
375                         ())
376
377    def test_entities_across_dtd(self):
378        f = File("browser/strings.dtd", "strings.dtd", "browser")
379        p = getParser(f.file)
380        p.readContents('<!ENTITY other "some &good.ref;">')
381        ref = p.parse()
382        checker = getChecker(f, reference=ref[0])
383        # good string
384        ref = self.getDTDEntity("plain string")
385        l10n = self.getDTDEntity("plain localized string")
386        self.assertEqual(tuple(checker.check(ref, l10n)),
387                         ())
388        # dtd warning
389        ref = self.getDTDEntity("plain string")
390        l10n = self.getDTDEntity("plain localized string &ref;")
391        self.assertEqual(tuple(checker.check(ref, l10n)),
392                         (('warning', (0, 0),
393                           'Referencing unknown entity `ref` (good.ref known)',
394                           'xmlparse'),))
395        # no report on stray ampersand
396        ref = self.getDTDEntity("plain string")
397        l10n = self.getDTDEntity("plain localized string with &good.ref;")
398        self.assertEqual(tuple(checker.check(ref, l10n)),
399                         ())
400
401
402if __name__ == '__main__':
403    unittest.main()
404