1"""Tests for distutils.command.sdist."""
2import os
3import tarfile
4import unittest
5import warnings
6import zipfile
7from os.path import join
8from textwrap import dedent
9from test.support import captured_stdout, check_warnings, run_unittest
10
11try:
12    import zlib
13    ZLIB_SUPPORT = True
14except ImportError:
15    ZLIB_SUPPORT = False
16
17try:
18    import grp
19    import pwd
20    UID_GID_SUPPORT = True
21except ImportError:
22    UID_GID_SUPPORT = False
23
24from distutils.command.sdist import sdist, show_formats
25from distutils.core import Distribution
26from distutils.tests.test_config import BasePyPIRCCommandTestCase
27from distutils.errors import DistutilsOptionError
28from distutils.spawn import find_executable
29from distutils.log import WARN
30from distutils.filelist import FileList
31from distutils.archive_util import ARCHIVE_FORMATS
32
33SETUP_PY = """
34from distutils.core import setup
35import somecode
36
37setup(name='fake')
38"""
39
40MANIFEST = """\
41# file GENERATED by distutils, do NOT edit
42README
43buildout.cfg
44inroot.txt
45setup.py
46data%(sep)sdata.dt
47scripts%(sep)sscript.py
48some%(sep)sfile.txt
49some%(sep)sother_file.txt
50somecode%(sep)s__init__.py
51somecode%(sep)sdoc.dat
52somecode%(sep)sdoc.txt
53"""
54
55class SDistTestCase(BasePyPIRCCommandTestCase):
56
57    def setUp(self):
58        # PyPIRCCommandTestCase creates a temp dir already
59        # and put it in self.tmp_dir
60        super(SDistTestCase, self).setUp()
61        # setting up an environment
62        self.old_path = os.getcwd()
63        os.mkdir(join(self.tmp_dir, 'somecode'))
64        os.mkdir(join(self.tmp_dir, 'dist'))
65        # a package, and a README
66        self.write_file((self.tmp_dir, 'README'), 'xxx')
67        self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#')
68        self.write_file((self.tmp_dir, 'setup.py'), SETUP_PY)
69        os.chdir(self.tmp_dir)
70
71    def tearDown(self):
72        # back to normal
73        os.chdir(self.old_path)
74        super(SDistTestCase, self).tearDown()
75
76    def get_cmd(self, metadata=None):
77        """Returns a cmd"""
78        if metadata is None:
79            metadata = {'name': 'fake', 'version': '1.0',
80                        'url': 'xxx', 'author': 'xxx',
81                        'author_email': 'xxx'}
82        dist = Distribution(metadata)
83        dist.script_name = 'setup.py'
84        dist.packages = ['somecode']
85        dist.include_package_data = True
86        cmd = sdist(dist)
87        cmd.dist_dir = 'dist'
88        return dist, cmd
89
90    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
91    def test_prune_file_list(self):
92        # this test creates a project with some VCS dirs and an NFS rename
93        # file, then launches sdist to check they get pruned on all systems
94
95        # creating VCS directories with some files in them
96        os.mkdir(join(self.tmp_dir, 'somecode', '.svn'))
97        self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx')
98
99        os.mkdir(join(self.tmp_dir, 'somecode', '.hg'))
100        self.write_file((self.tmp_dir, 'somecode', '.hg',
101                         'ok'), 'xxx')
102
103        os.mkdir(join(self.tmp_dir, 'somecode', '.git'))
104        self.write_file((self.tmp_dir, 'somecode', '.git',
105                         'ok'), 'xxx')
106
107        self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx')
108
109        # now building a sdist
110        dist, cmd = self.get_cmd()
111
112        # zip is available universally
113        # (tar might not be installed under win32)
114        cmd.formats = ['zip']
115
116        cmd.ensure_finalized()
117        cmd.run()
118
119        # now let's check what we have
120        dist_folder = join(self.tmp_dir, 'dist')
121        files = os.listdir(dist_folder)
122        self.assertEqual(files, ['fake-1.0.zip'])
123
124        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
125        try:
126            content = zip_file.namelist()
127        finally:
128            zip_file.close()
129
130        # making sure everything has been pruned correctly
131        expected = ['', 'PKG-INFO', 'README', 'setup.py',
132                    'somecode/', 'somecode/__init__.py']
133        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
134
135    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
136    @unittest.skipIf(find_executable('tar') is None,
137                     "The tar command is not found")
138    @unittest.skipIf(find_executable('gzip') is None,
139                     "The gzip command is not found")
140    def test_make_distribution(self):
141        # now building a sdist
142        dist, cmd = self.get_cmd()
143
144        # creating a gztar then a tar
145        cmd.formats = ['gztar', 'tar']
146        cmd.ensure_finalized()
147        cmd.run()
148
149        # making sure we have two files
150        dist_folder = join(self.tmp_dir, 'dist')
151        result = os.listdir(dist_folder)
152        result.sort()
153        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
154
155        os.remove(join(dist_folder, 'fake-1.0.tar'))
156        os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
157
158        # now trying a tar then a gztar
159        cmd.formats = ['tar', 'gztar']
160
161        cmd.ensure_finalized()
162        cmd.run()
163
164        result = os.listdir(dist_folder)
165        result.sort()
166        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
167
168    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
169    def test_add_defaults(self):
170
171        # http://bugs.python.org/issue2279
172
173        # add_default should also include
174        # data_files and package_data
175        dist, cmd = self.get_cmd()
176
177        # filling data_files by pointing files
178        # in package_data
179        dist.package_data = {'': ['*.cfg', '*.dat'],
180                             'somecode': ['*.txt']}
181        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
182        self.write_file((self.tmp_dir, 'somecode', 'doc.dat'), '#')
183
184        # adding some data in data_files
185        data_dir = join(self.tmp_dir, 'data')
186        os.mkdir(data_dir)
187        self.write_file((data_dir, 'data.dt'), '#')
188        some_dir = join(self.tmp_dir, 'some')
189        os.mkdir(some_dir)
190        # make sure VCS directories are pruned (#14004)
191        hg_dir = join(self.tmp_dir, '.hg')
192        os.mkdir(hg_dir)
193        self.write_file((hg_dir, 'last-message.txt'), '#')
194        # a buggy regex used to prevent this from working on windows (#6884)
195        self.write_file((self.tmp_dir, 'buildout.cfg'), '#')
196        self.write_file((self.tmp_dir, 'inroot.txt'), '#')
197        self.write_file((some_dir, 'file.txt'), '#')
198        self.write_file((some_dir, 'other_file.txt'), '#')
199
200        dist.data_files = [('data', ['data/data.dt',
201                                     'buildout.cfg',
202                                     'inroot.txt',
203                                     'notexisting']),
204                           'some/file.txt',
205                           'some/other_file.txt']
206
207        # adding a script
208        script_dir = join(self.tmp_dir, 'scripts')
209        os.mkdir(script_dir)
210        self.write_file((script_dir, 'script.py'), '#')
211        dist.scripts = [join('scripts', 'script.py')]
212
213        cmd.formats = ['zip']
214        cmd.use_defaults = True
215
216        cmd.ensure_finalized()
217        cmd.run()
218
219        # now let's check what we have
220        dist_folder = join(self.tmp_dir, 'dist')
221        files = os.listdir(dist_folder)
222        self.assertEqual(files, ['fake-1.0.zip'])
223
224        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
225        try:
226            content = zip_file.namelist()
227        finally:
228            zip_file.close()
229
230        # making sure everything was added
231        expected = ['', 'PKG-INFO', 'README', 'buildout.cfg',
232                    'data/', 'data/data.dt', 'inroot.txt',
233                    'scripts/', 'scripts/script.py', 'setup.py',
234                    'some/', 'some/file.txt', 'some/other_file.txt',
235                    'somecode/', 'somecode/__init__.py', 'somecode/doc.dat',
236                    'somecode/doc.txt']
237        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
238
239        # checking the MANIFEST
240        f = open(join(self.tmp_dir, 'MANIFEST'))
241        try:
242            manifest = f.read()
243        finally:
244            f.close()
245        self.assertEqual(manifest, MANIFEST % {'sep': os.sep})
246
247    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
248    def test_metadata_check_option(self):
249        # testing the `medata-check` option
250        dist, cmd = self.get_cmd(metadata={})
251
252        # this should raise some warnings !
253        # with the `check` subcommand
254        cmd.ensure_finalized()
255        cmd.run()
256        warnings = [msg for msg in self.get_logs(WARN) if
257                    msg.startswith('warning: check:')]
258        self.assertEqual(len(warnings), 2)
259
260        # trying with a complete set of metadata
261        self.clear_logs()
262        dist, cmd = self.get_cmd()
263        cmd.ensure_finalized()
264        cmd.metadata_check = 0
265        cmd.run()
266        warnings = [msg for msg in self.get_logs(WARN) if
267                    msg.startswith('warning: check:')]
268        self.assertEqual(len(warnings), 0)
269
270    def test_check_metadata_deprecated(self):
271        # makes sure make_metadata is deprecated
272        dist, cmd = self.get_cmd()
273        with check_warnings() as w:
274            warnings.simplefilter("always")
275            cmd.check_metadata()
276            self.assertEqual(len(w.warnings), 1)
277
278    def test_show_formats(self):
279        with captured_stdout() as stdout:
280            show_formats()
281
282        # the output should be a header line + one line per format
283        num_formats = len(ARCHIVE_FORMATS.keys())
284        output = [line for line in stdout.getvalue().split('\n')
285                  if line.strip().startswith('--formats=')]
286        self.assertEqual(len(output), num_formats)
287
288    def test_finalize_options(self):
289        dist, cmd = self.get_cmd()
290        cmd.finalize_options()
291
292        # default options set by finalize
293        self.assertEqual(cmd.manifest, 'MANIFEST')
294        self.assertEqual(cmd.template, 'MANIFEST.in')
295        self.assertEqual(cmd.dist_dir, 'dist')
296
297        # formats has to be a string splitable on (' ', ',') or
298        # a stringlist
299        cmd.formats = 1
300        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
301        cmd.formats = ['zip']
302        cmd.finalize_options()
303
304        # formats has to be known
305        cmd.formats = 'supazipa'
306        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
307
308    # the following tests make sure there is a nice error message instead
309    # of a traceback when parsing an invalid manifest template
310
311    def _check_template(self, content):
312        dist, cmd = self.get_cmd()
313        os.chdir(self.tmp_dir)
314        self.write_file('MANIFEST.in', content)
315        cmd.ensure_finalized()
316        cmd.filelist = FileList()
317        cmd.read_template()
318        warnings = self.get_logs(WARN)
319        self.assertEqual(len(warnings), 1)
320
321    def test_invalid_template_unknown_command(self):
322        self._check_template('taunt knights *')
323
324    def test_invalid_template_wrong_arguments(self):
325        # this manifest command takes one argument
326        self._check_template('prune')
327
328    @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only')
329    def test_invalid_template_wrong_path(self):
330        # on Windows, trailing slashes are not allowed
331        # this used to crash instead of raising a warning: #8286
332        self._check_template('include examples/')
333
334    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
335    def test_get_file_list(self):
336        # make sure MANIFEST is recalculated
337        dist, cmd = self.get_cmd()
338
339        # filling data_files by pointing files in package_data
340        dist.package_data = {'somecode': ['*.txt']}
341        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
342        cmd.formats = ['gztar']
343        cmd.ensure_finalized()
344        cmd.run()
345
346        f = open(cmd.manifest)
347        try:
348            manifest = [line.strip() for line in f.read().split('\n')
349                        if line.strip() != '']
350        finally:
351            f.close()
352
353        self.assertEqual(len(manifest), 5)
354
355        # adding a file
356        self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#')
357
358        # make sure build_py is reinitialized, like a fresh run
359        build_py = dist.get_command_obj('build_py')
360        build_py.finalized = False
361        build_py.ensure_finalized()
362
363        cmd.run()
364
365        f = open(cmd.manifest)
366        try:
367            manifest2 = [line.strip() for line in f.read().split('\n')
368                         if line.strip() != '']
369        finally:
370            f.close()
371
372        # do we have the new file in MANIFEST ?
373        self.assertEqual(len(manifest2), 6)
374        self.assertIn('doc2.txt', manifest2[-1])
375
376    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
377    def test_manifest_marker(self):
378        # check that autogenerated MANIFESTs have a marker
379        dist, cmd = self.get_cmd()
380        cmd.ensure_finalized()
381        cmd.run()
382
383        f = open(cmd.manifest)
384        try:
385            manifest = [line.strip() for line in f.read().split('\n')
386                        if line.strip() != '']
387        finally:
388            f.close()
389
390        self.assertEqual(manifest[0],
391                         '# file GENERATED by distutils, do NOT edit')
392
393    @unittest.skipUnless(ZLIB_SUPPORT, "Need zlib support to run")
394    def test_manifest_comments(self):
395        # make sure comments don't cause exceptions or wrong includes
396        contents = dedent("""\
397            # bad.py
398            #bad.py
399            good.py
400            """)
401        dist, cmd = self.get_cmd()
402        cmd.ensure_finalized()
403        self.write_file((self.tmp_dir, cmd.manifest), contents)
404        self.write_file((self.tmp_dir, 'good.py'), '# pick me!')
405        self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!")
406        self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!")
407        cmd.run()
408        self.assertEqual(cmd.filelist.files, ['good.py'])
409
410    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
411    def test_manual_manifest(self):
412        # check that a MANIFEST without a marker is left alone
413        dist, cmd = self.get_cmd()
414        cmd.formats = ['gztar']
415        cmd.ensure_finalized()
416        self.write_file((self.tmp_dir, cmd.manifest), 'README.manual')
417        self.write_file((self.tmp_dir, 'README.manual'),
418                         'This project maintains its MANIFEST file itself.')
419        cmd.run()
420        self.assertEqual(cmd.filelist.files, ['README.manual'])
421
422        f = open(cmd.manifest)
423        try:
424            manifest = [line.strip() for line in f.read().split('\n')
425                        if line.strip() != '']
426        finally:
427            f.close()
428
429        self.assertEqual(manifest, ['README.manual'])
430
431        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
432        archive = tarfile.open(archive_name)
433        try:
434            filenames = [tarinfo.name for tarinfo in archive]
435        finally:
436            archive.close()
437        self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO',
438                                             'fake-1.0/README.manual'])
439
440    @unittest.skipUnless(ZLIB_SUPPORT, "requires zlib")
441    @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
442    @unittest.skipIf(find_executable('tar') is None,
443                     "The tar command is not found")
444    @unittest.skipIf(find_executable('gzip') is None,
445                     "The gzip command is not found")
446    def test_make_distribution_owner_group(self):
447        # now building a sdist
448        dist, cmd = self.get_cmd()
449
450        # creating a gztar and specifying the owner+group
451        cmd.formats = ['gztar']
452        cmd.owner = pwd.getpwuid(0)[0]
453        cmd.group = grp.getgrgid(0)[0]
454        cmd.ensure_finalized()
455        cmd.run()
456
457        # making sure we have the good rights
458        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
459        archive = tarfile.open(archive_name)
460        try:
461            for member in archive.getmembers():
462                self.assertEqual(member.uid, 0)
463                self.assertEqual(member.gid, 0)
464        finally:
465            archive.close()
466
467        # building a sdist again
468        dist, cmd = self.get_cmd()
469
470        # creating a gztar
471        cmd.formats = ['gztar']
472        cmd.ensure_finalized()
473        cmd.run()
474
475        # making sure we have the good rights
476        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
477        archive = tarfile.open(archive_name)
478
479        # note that we are not testing the group ownership here
480        # because, depending on the platforms and the container
481        # rights (see #7408)
482        try:
483            for member in archive.getmembers():
484                self.assertEqual(member.uid, os.getuid())
485        finally:
486            archive.close()
487
488def test_suite():
489    return unittest.makeSuite(SDistTestCase)
490
491if __name__ == "__main__":
492    run_unittest(test_suite())
493