1"""Tests for distutils.command.sdist."""
2import os
3import tarfile
4import unittest
5import warnings
6import zipfile
7from os.path import join
8from textwrap import dedent
9from test.support import captured_stdout, run_unittest
10from test.support.warnings_helper import check_warnings
11
12try:
13    import zlib
14    ZLIB_SUPPORT = True
15except ImportError:
16    ZLIB_SUPPORT = False
17
18try:
19    import grp
20    import pwd
21    UID_GID_SUPPORT = True
22except ImportError:
23    UID_GID_SUPPORT = False
24
25from distutils.command.sdist import sdist, show_formats
26from distutils.core import Distribution
27from distutils.tests.test_config import BasePyPIRCCommandTestCase
28from distutils.errors import DistutilsOptionError
29from distutils.spawn import find_executable
30from distutils.log import WARN
31from distutils.filelist import FileList
32from distutils.archive_util import ARCHIVE_FORMATS
33
34SETUP_PY = """
35from distutils.core import setup
36import somecode
37
38setup(name='fake')
39"""
40
41MANIFEST = """\
42# file GENERATED by distutils, do NOT edit
43README
44buildout.cfg
45inroot.txt
46setup.py
47data%(sep)sdata.dt
48scripts%(sep)sscript.py
49some%(sep)sfile.txt
50some%(sep)sother_file.txt
51somecode%(sep)s__init__.py
52somecode%(sep)sdoc.dat
53somecode%(sep)sdoc.txt
54"""
55
56class SDistTestCase(BasePyPIRCCommandTestCase):
57
58    def setUp(self):
59        # PyPIRCCommandTestCase creates a temp dir already
60        # and put it in self.tmp_dir
61        super(SDistTestCase, self).setUp()
62        # setting up an environment
63        self.old_path = os.getcwd()
64        os.mkdir(join(self.tmp_dir, 'somecode'))
65        os.mkdir(join(self.tmp_dir, 'dist'))
66        # a package, and a README
67        self.write_file((self.tmp_dir, 'README'), 'xxx')
68        self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#')
69        self.write_file((self.tmp_dir, 'setup.py'), SETUP_PY)
70        os.chdir(self.tmp_dir)
71
72    def tearDown(self):
73        # back to normal
74        os.chdir(self.old_path)
75        super(SDistTestCase, self).tearDown()
76
77    def get_cmd(self, metadata=None):
78        """Returns a cmd"""
79        if metadata is None:
80            metadata = {'name': 'fake', 'version': '1.0',
81                        'url': 'xxx', 'author': 'xxx',
82                        'author_email': 'xxx'}
83        dist = Distribution(metadata)
84        dist.script_name = 'setup.py'
85        dist.packages = ['somecode']
86        dist.include_package_data = True
87        cmd = sdist(dist)
88        cmd.dist_dir = 'dist'
89        return dist, cmd
90
91    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
92    def test_prune_file_list(self):
93        # this test creates a project with some VCS dirs and an NFS rename
94        # file, then launches sdist to check they get pruned on all systems
95
96        # creating VCS directories with some files in them
97        os.mkdir(join(self.tmp_dir, 'somecode', '.svn'))
98        self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx')
99
100        os.mkdir(join(self.tmp_dir, 'somecode', '.hg'))
101        self.write_file((self.tmp_dir, 'somecode', '.hg',
102                         'ok'), 'xxx')
103
104        os.mkdir(join(self.tmp_dir, 'somecode', '.git'))
105        self.write_file((self.tmp_dir, 'somecode', '.git',
106                         'ok'), 'xxx')
107
108        self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx')
109
110        # now building a sdist
111        dist, cmd = self.get_cmd()
112
113        # zip is available universally
114        # (tar might not be installed under win32)
115        cmd.formats = ['zip']
116
117        cmd.ensure_finalized()
118        cmd.run()
119
120        # now let's check what we have
121        dist_folder = join(self.tmp_dir, 'dist')
122        files = os.listdir(dist_folder)
123        self.assertEqual(files, ['fake-1.0.zip'])
124
125        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
126        try:
127            content = zip_file.namelist()
128        finally:
129            zip_file.close()
130
131        # making sure everything has been pruned correctly
132        expected = ['', 'PKG-INFO', 'README', 'setup.py',
133                    'somecode/', 'somecode/__init__.py']
134        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
135
136    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
137    @unittest.skipIf(find_executable('tar') is None,
138                     "The tar command is not found")
139    @unittest.skipIf(find_executable('gzip') is None,
140                     "The gzip command is not found")
141    def test_make_distribution(self):
142        # now building a sdist
143        dist, cmd = self.get_cmd()
144
145        # creating a gztar then a tar
146        cmd.formats = ['gztar', 'tar']
147        cmd.ensure_finalized()
148        cmd.run()
149
150        # making sure we have two files
151        dist_folder = join(self.tmp_dir, 'dist')
152        result = os.listdir(dist_folder)
153        result.sort()
154        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
155
156        os.remove(join(dist_folder, 'fake-1.0.tar'))
157        os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
158
159        # now trying a tar then a gztar
160        cmd.formats = ['tar', 'gztar']
161
162        cmd.ensure_finalized()
163        cmd.run()
164
165        result = os.listdir(dist_folder)
166        result.sort()
167        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
168
169    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
170    def test_add_defaults(self):
171
172        # http://bugs.python.org/issue2279
173
174        # add_default should also include
175        # data_files and package_data
176        dist, cmd = self.get_cmd()
177
178        # filling data_files by pointing files
179        # in package_data
180        dist.package_data = {'': ['*.cfg', '*.dat'],
181                             'somecode': ['*.txt']}
182        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
183        self.write_file((self.tmp_dir, 'somecode', 'doc.dat'), '#')
184
185        # adding some data in data_files
186        data_dir = join(self.tmp_dir, 'data')
187        os.mkdir(data_dir)
188        self.write_file((data_dir, 'data.dt'), '#')
189        some_dir = join(self.tmp_dir, 'some')
190        os.mkdir(some_dir)
191        # make sure VCS directories are pruned (#14004)
192        hg_dir = join(self.tmp_dir, '.hg')
193        os.mkdir(hg_dir)
194        self.write_file((hg_dir, 'last-message.txt'), '#')
195        # a buggy regex used to prevent this from working on windows (#6884)
196        self.write_file((self.tmp_dir, 'buildout.cfg'), '#')
197        self.write_file((self.tmp_dir, 'inroot.txt'), '#')
198        self.write_file((some_dir, 'file.txt'), '#')
199        self.write_file((some_dir, 'other_file.txt'), '#')
200
201        dist.data_files = [('data', ['data/data.dt',
202                                     'buildout.cfg',
203                                     'inroot.txt',
204                                     'notexisting']),
205                           'some/file.txt',
206                           'some/other_file.txt']
207
208        # adding a script
209        script_dir = join(self.tmp_dir, 'scripts')
210        os.mkdir(script_dir)
211        self.write_file((script_dir, 'script.py'), '#')
212        dist.scripts = [join('scripts', 'script.py')]
213
214        cmd.formats = ['zip']
215        cmd.use_defaults = True
216
217        cmd.ensure_finalized()
218        cmd.run()
219
220        # now let's check what we have
221        dist_folder = join(self.tmp_dir, 'dist')
222        files = os.listdir(dist_folder)
223        self.assertEqual(files, ['fake-1.0.zip'])
224
225        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
226        try:
227            content = zip_file.namelist()
228        finally:
229            zip_file.close()
230
231        # making sure everything was added
232        expected = ['', 'PKG-INFO', 'README', 'buildout.cfg',
233                    'data/', 'data/data.dt', 'inroot.txt',
234                    'scripts/', 'scripts/script.py', 'setup.py',
235                    'some/', 'some/file.txt', 'some/other_file.txt',
236                    'somecode/', 'somecode/__init__.py', 'somecode/doc.dat',
237                    'somecode/doc.txt']
238        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
239
240        # checking the MANIFEST
241        f = open(join(self.tmp_dir, 'MANIFEST'))
242        try:
243            manifest = f.read()
244        finally:
245            f.close()
246        self.assertEqual(manifest, MANIFEST % {'sep': os.sep})
247
248    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
249    def test_metadata_check_option(self):
250        # testing the `medata-check` option
251        dist, cmd = self.get_cmd(metadata={})
252
253        # this should raise some warnings !
254        # with the `check` subcommand
255        cmd.ensure_finalized()
256        cmd.run()
257        warnings = [msg for msg in self.get_logs(WARN) if
258                    msg.startswith('warning: check:')]
259        self.assertEqual(len(warnings), 2)
260
261        # trying with a complete set of metadata
262        self.clear_logs()
263        dist, cmd = self.get_cmd()
264        cmd.ensure_finalized()
265        cmd.metadata_check = 0
266        cmd.run()
267        warnings = [msg for msg in self.get_logs(WARN) if
268                    msg.startswith('warning: check:')]
269        self.assertEqual(len(warnings), 0)
270
271    def test_check_metadata_deprecated(self):
272        # makes sure make_metadata is deprecated
273        dist, cmd = self.get_cmd()
274        with check_warnings() as w:
275            warnings.simplefilter("always")
276            cmd.check_metadata()
277            self.assertEqual(len(w.warnings), 1)
278
279    def test_show_formats(self):
280        with captured_stdout() as stdout:
281            show_formats()
282
283        # the output should be a header line + one line per format
284        num_formats = len(ARCHIVE_FORMATS.keys())
285        output = [line for line in stdout.getvalue().split('\n')
286                  if line.strip().startswith('--formats=')]
287        self.assertEqual(len(output), num_formats)
288
289    def test_finalize_options(self):
290        dist, cmd = self.get_cmd()
291        cmd.finalize_options()
292
293        # default options set by finalize
294        self.assertEqual(cmd.manifest, 'MANIFEST')
295        self.assertEqual(cmd.template, 'MANIFEST.in')
296        self.assertEqual(cmd.dist_dir, 'dist')
297
298        # formats has to be a string splitable on (' ', ',') or
299        # a stringlist
300        cmd.formats = 1
301        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
302        cmd.formats = ['zip']
303        cmd.finalize_options()
304
305        # formats has to be known
306        cmd.formats = 'supazipa'
307        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
308
309    # the following tests make sure there is a nice error message instead
310    # of a traceback when parsing an invalid manifest template
311
312    def _check_template(self, content):
313        dist, cmd = self.get_cmd()
314        os.chdir(self.tmp_dir)
315        self.write_file('MANIFEST.in', content)
316        cmd.ensure_finalized()
317        cmd.filelist = FileList()
318        cmd.read_template()
319        warnings = self.get_logs(WARN)
320        self.assertEqual(len(warnings), 1)
321
322    def test_invalid_template_unknown_command(self):
323        self._check_template('taunt knights *')
324
325    def test_invalid_template_wrong_arguments(self):
326        # this manifest command takes one argument
327        self._check_template('prune')
328
329    @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only')
330    def test_invalid_template_wrong_path(self):
331        # on Windows, trailing slashes are not allowed
332        # this used to crash instead of raising a warning: #8286
333        self._check_template('include examples/')
334
335    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
336    def test_get_file_list(self):
337        # make sure MANIFEST is recalculated
338        dist, cmd = self.get_cmd()
339
340        # filling data_files by pointing files in package_data
341        dist.package_data = {'somecode': ['*.txt']}
342        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
343        cmd.formats = ['gztar']
344        cmd.ensure_finalized()
345        cmd.run()
346
347        f = open(cmd.manifest)
348        try:
349            manifest = [line.strip() for line in f.read().split('\n')
350                        if line.strip() != '']
351        finally:
352            f.close()
353
354        self.assertEqual(len(manifest), 5)
355
356        # adding a file
357        self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#')
358
359        # make sure build_py is reinitialized, like a fresh run
360        build_py = dist.get_command_obj('build_py')
361        build_py.finalized = False
362        build_py.ensure_finalized()
363
364        cmd.run()
365
366        f = open(cmd.manifest)
367        try:
368            manifest2 = [line.strip() for line in f.read().split('\n')
369                         if line.strip() != '']
370        finally:
371            f.close()
372
373        # do we have the new file in MANIFEST ?
374        self.assertEqual(len(manifest2), 6)
375        self.assertIn('doc2.txt', manifest2[-1])
376
377    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
378    def test_manifest_marker(self):
379        # check that autogenerated MANIFESTs have a marker
380        dist, cmd = self.get_cmd()
381        cmd.ensure_finalized()
382        cmd.run()
383
384        f = open(cmd.manifest)
385        try:
386            manifest = [line.strip() for line in f.read().split('\n')
387                        if line.strip() != '']
388        finally:
389            f.close()
390
391        self.assertEqual(manifest[0],
392                         '# file GENERATED by distutils, do NOT edit')
393
394    @unittest.skipUnless(ZLIB_SUPPORT, "Need zlib support to run")
395    def test_manifest_comments(self):
396        # make sure comments don't cause exceptions or wrong includes
397        contents = dedent("""\
398            # bad.py
399            #bad.py
400            good.py
401            """)
402        dist, cmd = self.get_cmd()
403        cmd.ensure_finalized()
404        self.write_file((self.tmp_dir, cmd.manifest), contents)
405        self.write_file((self.tmp_dir, 'good.py'), '# pick me!')
406        self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!")
407        self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!")
408        cmd.run()
409        self.assertEqual(cmd.filelist.files, ['good.py'])
410
411    @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
412    def test_manual_manifest(self):
413        # check that a MANIFEST without a marker is left alone
414        dist, cmd = self.get_cmd()
415        cmd.formats = ['gztar']
416        cmd.ensure_finalized()
417        self.write_file((self.tmp_dir, cmd.manifest), 'README.manual')
418        self.write_file((self.tmp_dir, 'README.manual'),
419                         'This project maintains its MANIFEST file itself.')
420        cmd.run()
421        self.assertEqual(cmd.filelist.files, ['README.manual'])
422
423        f = open(cmd.manifest)
424        try:
425            manifest = [line.strip() for line in f.read().split('\n')
426                        if line.strip() != '']
427        finally:
428            f.close()
429
430        self.assertEqual(manifest, ['README.manual'])
431
432        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
433        archive = tarfile.open(archive_name)
434        try:
435            filenames = [tarinfo.name for tarinfo in archive]
436        finally:
437            archive.close()
438        self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO',
439                                             'fake-1.0/README.manual'])
440
441    @unittest.skipUnless(ZLIB_SUPPORT, "requires zlib")
442    @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
443    @unittest.skipIf(find_executable('tar') is None,
444                     "The tar command is not found")
445    @unittest.skipIf(find_executable('gzip') is None,
446                     "The gzip command is not found")
447    def test_make_distribution_owner_group(self):
448        # now building a sdist
449        dist, cmd = self.get_cmd()
450
451        # creating a gztar and specifying the owner+group
452        cmd.formats = ['gztar']
453        cmd.owner = pwd.getpwuid(0)[0]
454        cmd.group = grp.getgrgid(0)[0]
455        cmd.ensure_finalized()
456        cmd.run()
457
458        # making sure we have the good rights
459        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
460        archive = tarfile.open(archive_name)
461        try:
462            for member in archive.getmembers():
463                self.assertEqual(member.uid, 0)
464                self.assertEqual(member.gid, 0)
465        finally:
466            archive.close()
467
468        # building a sdist again
469        dist, cmd = self.get_cmd()
470
471        # creating a gztar
472        cmd.formats = ['gztar']
473        cmd.ensure_finalized()
474        cmd.run()
475
476        # making sure we have the good rights
477        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
478        archive = tarfile.open(archive_name)
479
480        # note that we are not testing the group ownership here
481        # because, depending on the platforms and the container
482        # rights (see #7408)
483        try:
484            for member in archive.getmembers():
485                self.assertEqual(member.uid, os.getuid())
486        finally:
487            archive.close()
488
489def test_suite():
490    return unittest.TestLoader().loadTestsFromTestCase(SDistTestCase)
491
492if __name__ == "__main__":
493    run_unittest(test_suite())
494