1"""Tests for distutils.command.sdist."""
2import os
3import tarfile
4import unittest
5import warnings
6import zipfile
7from os.path import join
8from textwrap import dedent
9from test.test_support import captured_stdout, check_warnings, run_unittest
10
11# zlib is not used here, but if it's not available
12# the tests that use zipfile may fail
13try:
14    import zlib
15except ImportError:
16    zlib = None
17
18try:
19    import grp
20    import pwd
21    UID_GID_SUPPORT = True
22except ImportError:
23    UID_GID_SUPPORT = False
24
25
26from distutils.command.sdist import sdist, show_formats
27from distutils.core import Distribution
28from distutils.tests.test_config import PyPIRCCommandTestCase
29from distutils.errors import DistutilsOptionError
30from distutils.spawn import find_executable
31from distutils.log import WARN
32from distutils.filelist import FileList
33from distutils.archive_util import ARCHIVE_FORMATS
34
35SETUP_PY = """
36from distutils.core import setup
37import somecode
38
39setup(name='fake')
40"""
41
42MANIFEST = """\
43# file GENERATED by distutils, do NOT edit
44README
45buildout.cfg
46inroot.txt
47setup.py
48data%(sep)sdata.dt
49scripts%(sep)sscript.py
50some%(sep)sfile.txt
51some%(sep)sother_file.txt
52somecode%(sep)s__init__.py
53somecode%(sep)sdoc.dat
54somecode%(sep)sdoc.txt
55"""
56
57class SDistTestCase(PyPIRCCommandTestCase):
58
59    def setUp(self):
60        # PyPIRCCommandTestCase creates a temp dir already
61        # and put it in self.tmp_dir
62        super(SDistTestCase, self).setUp()
63        # setting up an environment
64        self.old_path = os.getcwd()
65        os.mkdir(join(self.tmp_dir, 'somecode'))
66        os.mkdir(join(self.tmp_dir, 'dist'))
67        # a package, and a README
68        self.write_file((self.tmp_dir, 'README'), 'xxx')
69        self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#')
70        self.write_file((self.tmp_dir, 'setup.py'), SETUP_PY)
71        os.chdir(self.tmp_dir)
72
73    def tearDown(self):
74        # back to normal
75        os.chdir(self.old_path)
76        super(SDistTestCase, self).tearDown()
77
78    def get_cmd(self, metadata=None):
79        """Returns a cmd"""
80        if metadata is None:
81            metadata = {'name': 'fake', 'version': '1.0',
82                        'url': 'xxx', 'author': 'xxx',
83                        'author_email': 'xxx'}
84        dist = Distribution(metadata)
85        dist.script_name = 'setup.py'
86        dist.packages = ['somecode']
87        dist.include_package_data = True
88        cmd = sdist(dist)
89        cmd.dist_dir = 'dist'
90        return dist, cmd
91
92    @unittest.skipUnless(zlib, "requires zlib")
93    def test_prune_file_list(self):
94        # this test creates a project with some VCS dirs and an NFS rename
95        # file, then launches sdist to check they get pruned on all systems
96
97        # creating VCS directories with some files in them
98        os.mkdir(join(self.tmp_dir, 'somecode', '.svn'))
99        self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx')
100
101        os.mkdir(join(self.tmp_dir, 'somecode', '.hg'))
102        self.write_file((self.tmp_dir, 'somecode', '.hg',
103                         'ok'), 'xxx')
104
105        os.mkdir(join(self.tmp_dir, 'somecode', '.git'))
106        self.write_file((self.tmp_dir, 'somecode', '.git',
107                         'ok'), 'xxx')
108
109        self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx')
110
111        # now building a sdist
112        dist, cmd = self.get_cmd()
113
114        # zip is available universally
115        # (tar might not be installed under win32)
116        cmd.formats = ['zip']
117
118        cmd.ensure_finalized()
119        cmd.run()
120
121        # now let's check what we have
122        dist_folder = join(self.tmp_dir, 'dist')
123        files = os.listdir(dist_folder)
124        self.assertEqual(files, ['fake-1.0.zip'])
125
126        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
127        try:
128            content = zip_file.namelist()
129        finally:
130            zip_file.close()
131
132        # making sure everything has been pruned correctly
133        expected = ['', 'PKG-INFO', 'README', 'setup.py',
134                    'somecode/', 'somecode/__init__.py']
135        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
136
137    @unittest.skipUnless(zlib, "requires zlib")
138    def test_make_distribution(self):
139        # now building a sdist
140        dist, cmd = self.get_cmd()
141
142        # creating a gztar then a tar
143        cmd.formats = ['gztar', 'tar']
144        cmd.ensure_finalized()
145        cmd.run()
146
147        # making sure we have two files
148        dist_folder = join(self.tmp_dir, 'dist')
149        result = os.listdir(dist_folder)
150        result.sort()
151        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
152
153        os.remove(join(dist_folder, 'fake-1.0.tar'))
154        os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
155
156        # now trying a tar then a gztar
157        cmd.formats = ['tar', 'gztar']
158
159        cmd.ensure_finalized()
160        cmd.run()
161
162        result = os.listdir(dist_folder)
163        result.sort()
164        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
165
166    @unittest.skipUnless(zlib, "requires zlib")
167    def test_unicode_metadata_tgz(self):
168        """
169        Unicode name or version should not break building to tar.gz format.
170        Reference issue #11638.
171        """
172
173        # create the sdist command with unicode parameters
174        dist, cmd = self.get_cmd({'name': u'fake', 'version': u'1.0'})
175
176        # create the sdist as gztar and run the command
177        cmd.formats = ['gztar']
178        cmd.ensure_finalized()
179        cmd.run()
180
181        # The command should have created the .tar.gz file
182        dist_folder = join(self.tmp_dir, 'dist')
183        result = os.listdir(dist_folder)
184        self.assertEqual(result, ['fake-1.0.tar.gz'])
185
186        os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
187
188    @unittest.skipUnless(zlib, "requires zlib")
189    def test_add_defaults(self):
190
191        # http://bugs.python.org/issue2279
192
193        # add_default should also include
194        # data_files and package_data
195        dist, cmd = self.get_cmd()
196
197        # filling data_files by pointing files
198        # in package_data
199        dist.package_data = {'': ['*.cfg', '*.dat'],
200                             'somecode': ['*.txt']}
201        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
202        self.write_file((self.tmp_dir, 'somecode', 'doc.dat'), '#')
203
204        # adding some data in data_files
205        data_dir = join(self.tmp_dir, 'data')
206        os.mkdir(data_dir)
207        self.write_file((data_dir, 'data.dt'), '#')
208        some_dir = join(self.tmp_dir, 'some')
209        os.mkdir(some_dir)
210        # make sure VCS directories are pruned (#14004)
211        hg_dir = join(self.tmp_dir, '.hg')
212        os.mkdir(hg_dir)
213        self.write_file((hg_dir, 'last-message.txt'), '#')
214        # a buggy regex used to prevent this from working on windows (#6884)
215        self.write_file((self.tmp_dir, 'buildout.cfg'), '#')
216        self.write_file((self.tmp_dir, 'inroot.txt'), '#')
217        self.write_file((some_dir, 'file.txt'), '#')
218        self.write_file((some_dir, 'other_file.txt'), '#')
219
220        dist.data_files = [('data', ['data/data.dt',
221                                     'buildout.cfg',
222                                     'inroot.txt',
223                                     'notexisting']),
224                           'some/file.txt',
225                           'some/other_file.txt']
226
227        # adding a script
228        script_dir = join(self.tmp_dir, 'scripts')
229        os.mkdir(script_dir)
230        self.write_file((script_dir, 'script.py'), '#')
231        dist.scripts = [join('scripts', 'script.py')]
232
233        cmd.formats = ['zip']
234        cmd.use_defaults = True
235
236        cmd.ensure_finalized()
237        cmd.run()
238
239        # now let's check what we have
240        dist_folder = join(self.tmp_dir, 'dist')
241        files = os.listdir(dist_folder)
242        self.assertEqual(files, ['fake-1.0.zip'])
243
244        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
245        try:
246            content = zip_file.namelist()
247        finally:
248            zip_file.close()
249
250        # making sure everything was added
251        expected = ['', 'PKG-INFO', 'README', 'buildout.cfg',
252                    'data/', 'data/data.dt', 'inroot.txt',
253                    'scripts/', 'scripts/script.py', 'setup.py',
254                    'some/', 'some/file.txt', 'some/other_file.txt',
255                    'somecode/', 'somecode/__init__.py', 'somecode/doc.dat',
256                    'somecode/doc.txt']
257        self.assertEqual(sorted(content), ['fake-1.0/' + x for x in expected])
258
259        # checking the MANIFEST
260        f = open(join(self.tmp_dir, 'MANIFEST'))
261        try:
262            manifest = f.read()
263        finally:
264            f.close()
265        self.assertEqual(manifest, MANIFEST % {'sep': os.sep})
266
267    @unittest.skipUnless(zlib, "requires zlib")
268    def test_metadata_check_option(self):
269        # testing the `medata-check` option
270        dist, cmd = self.get_cmd(metadata={})
271
272        # this should raise some warnings !
273        # with the `check` subcommand
274        cmd.ensure_finalized()
275        cmd.run()
276        warnings = [msg for msg in self.get_logs(WARN) if
277                    msg.startswith('warning: check:')]
278        self.assertEqual(len(warnings), 2)
279
280        # trying with a complete set of metadata
281        self.clear_logs()
282        dist, cmd = self.get_cmd()
283        cmd.ensure_finalized()
284        cmd.metadata_check = 0
285        cmd.run()
286        warnings = [msg for msg in self.get_logs(WARN) if
287                    msg.startswith('warning: check:')]
288        self.assertEqual(len(warnings), 0)
289
290    def test_check_metadata_deprecated(self):
291        # makes sure make_metadata is deprecated
292        dist, cmd = self.get_cmd()
293        with check_warnings() as w:
294            warnings.simplefilter("always")
295            cmd.check_metadata()
296            self.assertEqual(len(w.warnings), 1)
297
298    def test_show_formats(self):
299        with captured_stdout() as stdout:
300            show_formats()
301
302        # the output should be a header line + one line per format
303        num_formats = len(ARCHIVE_FORMATS.keys())
304        output = [line for line in stdout.getvalue().split('\n')
305                  if line.strip().startswith('--formats=')]
306        self.assertEqual(len(output), num_formats)
307
308    def test_finalize_options(self):
309        dist, cmd = self.get_cmd()
310        cmd.finalize_options()
311
312        # default options set by finalize
313        self.assertEqual(cmd.manifest, 'MANIFEST')
314        self.assertEqual(cmd.template, 'MANIFEST.in')
315        self.assertEqual(cmd.dist_dir, 'dist')
316
317        # formats has to be a string splitable on (' ', ',') or
318        # a stringlist
319        cmd.formats = 1
320        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
321        cmd.formats = ['zip']
322        cmd.finalize_options()
323
324        # formats has to be known
325        cmd.formats = 'supazipa'
326        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
327
328    @unittest.skipUnless(zlib, "requires zlib")
329    @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
330    @unittest.skipIf(find_executable('tar') is None,
331                     "The tar command is not found")
332    @unittest.skipIf(find_executable('gzip') is None,
333                     "The gzip command is not found")
334    def test_make_distribution_owner_group(self):
335        # now building a sdist
336        dist, cmd = self.get_cmd()
337
338        # creating a gztar and specifying the owner+group
339        cmd.formats = ['gztar']
340        cmd.owner = pwd.getpwuid(0)[0]
341        cmd.group = grp.getgrgid(0)[0]
342        cmd.ensure_finalized()
343        cmd.run()
344
345        # making sure we have the good rights
346        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
347        archive = tarfile.open(archive_name)
348        try:
349            for member in archive.getmembers():
350                self.assertEqual(member.uid, 0)
351                self.assertEqual(member.gid, 0)
352        finally:
353            archive.close()
354
355        # building a sdist again
356        dist, cmd = self.get_cmd()
357
358        # creating a gztar
359        cmd.formats = ['gztar']
360        cmd.ensure_finalized()
361        cmd.run()
362
363        # making sure we have the good rights
364        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
365        archive = tarfile.open(archive_name)
366
367        # note that we are not testing the group ownership here
368        # because, depending on the platforms and the container
369        # rights (see #7408)
370        try:
371            for member in archive.getmembers():
372                self.assertEqual(member.uid, os.getuid())
373        finally:
374            archive.close()
375
376    # the following tests make sure there is a nice error message instead
377    # of a traceback when parsing an invalid manifest template
378
379    def _check_template(self, content):
380        dist, cmd = self.get_cmd()
381        os.chdir(self.tmp_dir)
382        self.write_file('MANIFEST.in', content)
383        cmd.ensure_finalized()
384        cmd.filelist = FileList()
385        cmd.read_template()
386        warnings = self.get_logs(WARN)
387        self.assertEqual(len(warnings), 1)
388
389    def test_invalid_template_unknown_command(self):
390        self._check_template('taunt knights *')
391
392    def test_invalid_template_wrong_arguments(self):
393        # this manifest command takes one argument
394        self._check_template('prune')
395
396    @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only')
397    def test_invalid_template_wrong_path(self):
398        # on Windows, trailing slashes are not allowed
399        # this used to crash instead of raising a warning: #8286
400        self._check_template('include examples/')
401
402    @unittest.skipUnless(zlib, "requires zlib")
403    def test_get_file_list(self):
404        # make sure MANIFEST is recalculated
405        dist, cmd = self.get_cmd()
406
407        # filling data_files by pointing files in package_data
408        dist.package_data = {'somecode': ['*.txt']}
409        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
410        cmd.formats = ['gztar']
411        cmd.ensure_finalized()
412        cmd.run()
413
414        f = open(cmd.manifest)
415        try:
416            manifest = [line.strip() for line in f.read().split('\n')
417                        if line.strip() != '']
418        finally:
419            f.close()
420
421        self.assertEqual(len(manifest), 5)
422
423        # adding a file
424        self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#')
425
426        # make sure build_py is reinitialized, like a fresh run
427        build_py = dist.get_command_obj('build_py')
428        build_py.finalized = False
429        build_py.ensure_finalized()
430
431        cmd.run()
432
433        f = open(cmd.manifest)
434        try:
435            manifest2 = [line.strip() for line in f.read().split('\n')
436                         if line.strip() != '']
437        finally:
438            f.close()
439
440        # do we have the new file in MANIFEST ?
441        self.assertEqual(len(manifest2), 6)
442        self.assertIn('doc2.txt', manifest2[-1])
443
444    @unittest.skipUnless(zlib, "requires zlib")
445    def test_manifest_marker(self):
446        # check that autogenerated MANIFESTs have a marker
447        dist, cmd = self.get_cmd()
448        cmd.ensure_finalized()
449        cmd.run()
450
451        f = open(cmd.manifest)
452        try:
453            manifest = [line.strip() for line in f.read().split('\n')
454                        if line.strip() != '']
455        finally:
456            f.close()
457
458        self.assertEqual(manifest[0],
459                         '# file GENERATED by distutils, do NOT edit')
460
461    @unittest.skipUnless(zlib, 'requires zlib')
462    def test_manifest_comments(self):
463        # make sure comments don't cause exceptions or wrong includes
464        contents = dedent("""\
465            # bad.py
466            #bad.py
467            good.py
468            """)
469        dist, cmd = self.get_cmd()
470        cmd.ensure_finalized()
471        self.write_file((self.tmp_dir, cmd.manifest), contents)
472        self.write_file((self.tmp_dir, 'good.py'), '# pick me!')
473        self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!")
474        self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!")
475        cmd.run()
476        self.assertEqual(cmd.filelist.files, ['good.py'])
477
478    @unittest.skipUnless(zlib, "requires zlib")
479    def test_manual_manifest(self):
480        # check that a MANIFEST without a marker is left alone
481        dist, cmd = self.get_cmd()
482        cmd.formats = ['gztar']
483        cmd.ensure_finalized()
484        self.write_file((self.tmp_dir, cmd.manifest), 'README.manual')
485        self.write_file((self.tmp_dir, 'README.manual'),
486                         'This project maintains its MANIFEST file itself.')
487        cmd.run()
488        self.assertEqual(cmd.filelist.files, ['README.manual'])
489
490        f = open(cmd.manifest)
491        try:
492            manifest = [line.strip() for line in f.read().split('\n')
493                        if line.strip() != '']
494        finally:
495            f.close()
496
497        self.assertEqual(manifest, ['README.manual'])
498
499        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
500        archive = tarfile.open(archive_name)
501        try:
502            filenames = [tarinfo.name for tarinfo in archive]
503        finally:
504            archive.close()
505        self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO',
506                                             'fake-1.0/README.manual'])
507
508def test_suite():
509    return unittest.makeSuite(SDistTestCase)
510
511if __name__ == "__main__":
512    run_unittest(test_suite())
513