1# Test the Unicode versions of normal file functions
2# open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir
3import sys, os, unittest
4from unicodedata import normalize
5from test import test_support
6
7filenames = [
8    '1_abc',
9    u'2_ascii',
10    u'3_Gr\xfc\xdf-Gott',
11    u'4_\u0393\u03b5\u03b9\u03ac-\u03c3\u03b1\u03c2',
12    u'5_\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435',
13    u'6_\u306b\u307d\u3093',
14    u'7_\u05d4\u05e9\u05e7\u05e6\u05e5\u05e1',
15    u'8_\u66e8\u66e9\u66eb',
16    u'9_\u66e8\u05e9\u3093\u0434\u0393\xdf',
17    # Specific code points: fn, NFC(fn) and NFKC(fn) all differents
18    u'10_\u1fee\u1ffd',
19    ]
20
21# Mac OS X decomposes Unicode names, using Normal Form D.
22# http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
23# "However, most volume formats do not follow the exact specification for
24# these normal forms.  For example, HFS Plus uses a variant of Normal Form D
25# in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through
26# U+2FAFF are not decomposed."
27if sys.platform != 'darwin':
28    filenames.extend([
29        # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all differents
30        u'11_\u0385\u03d3\u03d4',
31        u'12_\u00a8\u0301\u03d2\u0301\u03d2\u0308',    # == NFD(u'\u0385\u03d3\u03d4')
32        u'13_\u0020\u0308\u0301\u038e\u03ab',          # == NFKC(u'\u0385\u03d3\u03d4')
33        u'14_\u1e9b\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed',
34
35        # Specific code points: fn, NFC(fn) and NFKC(fn) all differents
36        u'15_\u1fee\u1ffd\ufad1',
37        u'16_\u2000\u2000\u2000A',
38        u'17_\u2001\u2001\u2001A',
39        u'18_\u2003\u2003\u2003A', # == NFC(u'\u2001\u2001\u2001A')
40        u'19_\u0020\u0020\u0020A', # u'\u0020' == u' ' == NFKC(u'\u2000') ==
41                                   #   NFKC(u'\u2001') == NFKC(u'\u2003')
42])
43
44
45# Is it Unicode-friendly?
46if not os.path.supports_unicode_filenames:
47    fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
48    try:
49        for name in filenames:
50            name.encode(fsencoding)
51    except UnicodeEncodeError:
52        raise unittest.SkipTest("only NT+ and systems with "
53                                "Unicode-friendly filesystem encoding")
54
55
56# Destroy directory dirname and all files under it, to one level.
57def deltree(dirname):
58    # Don't hide legitimate errors:  if one of these suckers exists, it's
59    # an error if we can't remove it.
60    if os.path.exists(dirname):
61        # must pass unicode to os.listdir() so we get back unicode results.
62        for fname in os.listdir(unicode(dirname)):
63            os.unlink(os.path.join(dirname, fname))
64        os.rmdir(dirname)
65
66
67class UnicodeFileTests(unittest.TestCase):
68    files = set(filenames)
69    normal_form = None
70
71    def setUp(self):
72        try:
73            os.mkdir(test_support.TESTFN)
74        except OSError:
75            pass
76        files = set()
77        for name in self.files:
78            name = os.path.join(test_support.TESTFN, self.norm(name))
79            with open(name, 'w') as f:
80                f.write((name+'\n').encode("utf-8"))
81            os.stat(name)
82            files.add(name)
83        self.files = files
84
85    def tearDown(self):
86        deltree(test_support.TESTFN)
87
88    def norm(self, s):
89        if self.normal_form and isinstance(s, unicode):
90            return normalize(self.normal_form, s)
91        return s
92
93    def _apply_failure(self, fn, filename, expected_exception,
94                       check_fn_in_exception = True):
95        with self.assertRaises(expected_exception) as c:
96            fn(filename)
97        exc_filename = c.exception.filename
98        # the "filename" exception attribute may be encoded
99        if isinstance(exc_filename, str):
100            filename = filename.encode(sys.getfilesystemencoding())
101        if check_fn_in_exception:
102            self.assertEqual(exc_filename, filename, "Function '%s(%r) failed "
103                             "with bad filename in the exception: %r" %
104                             (fn.__name__, filename, exc_filename))
105
106    def test_failures(self):
107        # Pass non-existing Unicode filenames all over the place.
108        for name in self.files:
109            name = "not_" + name
110            self._apply_failure(open, name, IOError)
111            self._apply_failure(os.stat, name, OSError)
112            self._apply_failure(os.chdir, name, OSError)
113            self._apply_failure(os.rmdir, name, OSError)
114            self._apply_failure(os.remove, name, OSError)
115            # listdir may append a wildcard to the filename, so dont check
116            self._apply_failure(os.listdir, name, OSError, False)
117
118    def test_open(self):
119        for name in self.files:
120            f = open(name, 'w')
121            f.write((name+'\n').encode("utf-8"))
122            f.close()
123            os.stat(name)
124
125    # Skip the test on darwin, because darwin does normalize the filename to
126    # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC,
127    # NFKD in Python is useless, because darwin will normalize it later and so
128    # open(), os.stat(), etc. don't raise any exception.
129    @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
130    def test_normalize(self):
131        files = set(f for f in self.files if isinstance(f, unicode))
132        others = set()
133        for nf in set(['NFC', 'NFD', 'NFKC', 'NFKD']):
134            others |= set(normalize(nf, file) for file in files)
135        others -= files
136        for name in others:
137            self._apply_failure(open, name, IOError)
138            self._apply_failure(os.stat, name, OSError)
139            self._apply_failure(os.chdir, name, OSError)
140            self._apply_failure(os.rmdir, name, OSError)
141            self._apply_failure(os.remove, name, OSError)
142            # listdir may append a wildcard to the filename, so dont check
143            self._apply_failure(os.listdir, name, OSError, False)
144
145    # Skip the test on darwin, because darwin uses a normalization different
146    # than Python NFD normalization: filenames are different even if we use
147    # Python NFD normalization.
148    @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
149    def test_listdir(self):
150        sf0 = set(self.files)
151        f1 = os.listdir(test_support.TESTFN)
152        f2 = os.listdir(unicode(test_support.TESTFN,
153                                sys.getfilesystemencoding()))
154        sf2 = set(os.path.join(unicode(test_support.TESTFN), f) for f in f2)
155        self.assertEqual(sf0, sf2)
156        self.assertEqual(len(f1), len(f2))
157
158    def test_rename(self):
159        for name in self.files:
160            os.rename(name, "tmp")
161            os.rename("tmp", name)
162
163    def test_directory(self):
164        dirname = os.path.join(test_support.TESTFN,
165                               u'Gr\xfc\xdf-\u66e8\u66e9\u66eb')
166        filename = u'\xdf-\u66e8\u66e9\u66eb'
167        with test_support.temp_cwd(dirname):
168            with open(filename, 'w') as f:
169                f.write((filename + '\n').encode("utf-8"))
170            os.access(filename,os.R_OK)
171            os.remove(filename)
172
173
174class UnicodeNFCFileTests(UnicodeFileTests):
175    normal_form = 'NFC'
176
177
178class UnicodeNFDFileTests(UnicodeFileTests):
179    normal_form = 'NFD'
180
181
182class UnicodeNFKCFileTests(UnicodeFileTests):
183    normal_form = 'NFKC'
184
185
186class UnicodeNFKDFileTests(UnicodeFileTests):
187    normal_form = 'NFKD'
188
189
190def test_main():
191    try:
192        test_support.run_unittest(
193            UnicodeFileTests,
194            UnicodeNFCFileTests,
195            UnicodeNFDFileTests,
196            UnicodeNFKCFileTests,
197            UnicodeNFKDFileTests,
198        )
199    finally:
200        deltree(test_support.TESTFN)
201
202
203if __name__ == "__main__":
204    test_main()
205