1"""distutils.command.sdist
2
3Implements the Distutils 'sdist' command (create a source distribution)."""
4
5import os
6import sys
7from glob import glob
8from warnings import warn
9
10from distutils.core import Command
11from distutils import dir_util
12from distutils import file_util
13from distutils import archive_util
14from distutils.text_file import TextFile
15from distutils.filelist import FileList
16from distutils import log
17from distutils.util import convert_path
18from distutils.errors import DistutilsTemplateError, DistutilsOptionError
19
20
21def show_formats():
22    """Print all possible values for the 'formats' option (used by
23    the "--help-formats" command-line option).
24    """
25    from distutils.fancy_getopt import FancyGetopt
26    from distutils.archive_util import ARCHIVE_FORMATS
27    formats = []
28    for format in ARCHIVE_FORMATS.keys():
29        formats.append(("formats=" + format, None,
30                        ARCHIVE_FORMATS[format][2]))
31    formats.sort()
32    FancyGetopt(formats).print_help(
33        "List of available source distribution formats:")
34
35
36class sdist(Command):
37
38    description = "create a source distribution (tarball, zip file, etc.)"
39
40    def checking_metadata(self):
41        """Callable used for the check sub-command.
42
43        Placed here so user_options can view it"""
44        return self.metadata_check
45
46    user_options = [
47        ('template=', 't',
48         "name of manifest template file [default: MANIFEST.in]"),
49        ('manifest=', 'm',
50         "name of manifest file [default: MANIFEST]"),
51        ('use-defaults', None,
52         "include the default file set in the manifest "
53         "[default; disable with --no-defaults]"),
54        ('no-defaults', None,
55         "don't include the default file set"),
56        ('prune', None,
57         "specifically exclude files/directories that should not be "
58         "distributed (build tree, RCS/CVS dirs, etc.) "
59         "[default; disable with --no-prune]"),
60        ('no-prune', None,
61         "don't automatically exclude anything"),
62        ('manifest-only', 'o',
63         "just regenerate the manifest and then stop "
64         "(implies --force-manifest)"),
65        ('force-manifest', 'f',
66         "forcibly regenerate the manifest and carry on as usual. "
67         "Deprecated: now the manifest is always regenerated."),
68        ('formats=', None,
69         "formats for source distribution (comma-separated list)"),
70        ('keep-temp', 'k',
71         "keep the distribution tree around after creating " +
72         "archive file(s)"),
73        ('dist-dir=', 'd',
74         "directory to put the source distribution archive(s) in "
75         "[default: dist]"),
76        ('metadata-check', None,
77         "Ensure that all required elements of meta-data "
78         "are supplied. Warn if any missing. [default]"),
79        ('owner=', 'u',
80         "Owner name used when creating a tar file [default: current user]"),
81        ('group=', 'g',
82         "Group name used when creating a tar file [default: current group]"),
83        ]
84
85    boolean_options = ['use-defaults', 'prune',
86                       'manifest-only', 'force-manifest',
87                       'keep-temp', 'metadata-check']
88
89    help_options = [
90        ('help-formats', None,
91         "list available distribution formats", show_formats),
92        ]
93
94    negative_opt = {'no-defaults': 'use-defaults',
95                    'no-prune': 'prune' }
96
97    sub_commands = [('check', checking_metadata)]
98
99    READMES = ('README', 'README.txt', 'README.rst')
100
101    def initialize_options(self):
102        # 'template' and 'manifest' are, respectively, the names of
103        # the manifest template and manifest file.
104        self.template = None
105        self.manifest = None
106
107        # 'use_defaults': if true, we will include the default file set
108        # in the manifest
109        self.use_defaults = 1
110        self.prune = 1
111
112        self.manifest_only = 0
113        self.force_manifest = 0
114
115        self.formats = ['gztar']
116        self.keep_temp = 0
117        self.dist_dir = None
118
119        self.archive_files = None
120        self.metadata_check = 1
121        self.owner = None
122        self.group = None
123
124    def finalize_options(self):
125        if self.manifest is None:
126            self.manifest = "MANIFEST"
127        if self.template is None:
128            self.template = "MANIFEST.in"
129
130        self.ensure_string_list('formats')
131
132        bad_format = archive_util.check_archive_formats(self.formats)
133        if bad_format:
134            raise DistutilsOptionError(
135                  "unknown archive format '%s'" % bad_format)
136
137        if self.dist_dir is None:
138            self.dist_dir = "dist"
139
140    def run(self):
141        # 'filelist' contains the list of files that will make up the
142        # manifest
143        self.filelist = FileList()
144
145        # Run sub commands
146        for cmd_name in self.get_sub_commands():
147            self.run_command(cmd_name)
148
149        # Do whatever it takes to get the list of files to process
150        # (process the manifest template, read an existing manifest,
151        # whatever).  File list is accumulated in 'self.filelist'.
152        self.get_file_list()
153
154        # If user just wanted us to regenerate the manifest, stop now.
155        if self.manifest_only:
156            return
157
158        # Otherwise, go ahead and create the source distribution tarball,
159        # or zipfile, or whatever.
160        self.make_distribution()
161
162    def check_metadata(self):
163        """Deprecated API."""
164        warn("distutils.command.sdist.check_metadata is deprecated, \
165              use the check command instead", PendingDeprecationWarning)
166        check = self.distribution.get_command_obj('check')
167        check.ensure_finalized()
168        check.run()
169
170    def get_file_list(self):
171        """Figure out the list of files to include in the source
172        distribution, and put it in 'self.filelist'.  This might involve
173        reading the manifest template (and writing the manifest), or just
174        reading the manifest, or just using the default file set -- it all
175        depends on the user's options.
176        """
177        # new behavior when using a template:
178        # the file list is recalculated every time because
179        # even if MANIFEST.in or setup.py are not changed
180        # the user might have added some files in the tree that
181        # need to be included.
182        #
183        #  This makes --force the default and only behavior with templates.
184        template_exists = os.path.isfile(self.template)
185        if not template_exists and self._manifest_is_not_generated():
186            self.read_manifest()
187            self.filelist.sort()
188            self.filelist.remove_duplicates()
189            return
190
191        if not template_exists:
192            self.warn(("manifest template '%s' does not exist " +
193                        "(using default file list)") %
194                        self.template)
195        self.filelist.findall()
196
197        if self.use_defaults:
198            self.add_defaults()
199
200        if template_exists:
201            self.read_template()
202
203        if self.prune:
204            self.prune_file_list()
205
206        self.filelist.sort()
207        self.filelist.remove_duplicates()
208        self.write_manifest()
209
210    def add_defaults(self):
211        """Add all the default files to self.filelist:
212          - README or README.txt
213          - setup.py
214          - test/test*.py
215          - all pure Python modules mentioned in setup script
216          - all files pointed by package_data (build_py)
217          - all files defined in data_files.
218          - all files defined as scripts.
219          - all C sources listed as part of extensions or C libraries
220            in the setup script (doesn't catch C headers!)
221        Warns if (README or README.txt) or setup.py are missing; everything
222        else is optional.
223        """
224        self._add_defaults_standards()
225        self._add_defaults_optional()
226        self._add_defaults_python()
227        self._add_defaults_data_files()
228        self._add_defaults_ext()
229        self._add_defaults_c_libs()
230        self._add_defaults_scripts()
231
232    @staticmethod
233    def _cs_path_exists(fspath):
234        """
235        Case-sensitive path existence check
236
237        >>> sdist._cs_path_exists(__file__)
238        True
239        >>> sdist._cs_path_exists(__file__.upper())
240        False
241        """
242        if not os.path.exists(fspath):
243            return False
244        # make absolute so we always have a directory
245        abspath = os.path.abspath(fspath)
246        directory, filename = os.path.split(abspath)
247        return filename in os.listdir(directory)
248
249    def _add_defaults_standards(self):
250        standards = [self.READMES, self.distribution.script_name]
251        for fn in standards:
252            if isinstance(fn, tuple):
253                alts = fn
254                got_it = False
255                for fn in alts:
256                    if self._cs_path_exists(fn):
257                        got_it = True
258                        self.filelist.append(fn)
259                        break
260
261                if not got_it:
262                    self.warn("standard file not found: should have one of " +
263                              ', '.join(alts))
264            else:
265                if self._cs_path_exists(fn):
266                    self.filelist.append(fn)
267                else:
268                    self.warn("standard file '%s' not found" % fn)
269
270    def _add_defaults_optional(self):
271        optional = ['test/test*.py', 'setup.cfg']
272        for pattern in optional:
273            files = filter(os.path.isfile, glob(pattern))
274            self.filelist.extend(files)
275
276    def _add_defaults_python(self):
277        # build_py is used to get:
278        #  - python modules
279        #  - files defined in package_data
280        build_py = self.get_finalized_command('build_py')
281
282        # getting python files
283        if self.distribution.has_pure_modules():
284            self.filelist.extend(build_py.get_source_files())
285
286        # getting package_data files
287        # (computed in build_py.data_files by build_py.finalize_options)
288        for pkg, src_dir, build_dir, filenames in build_py.data_files:
289            for filename in filenames:
290                self.filelist.append(os.path.join(src_dir, filename))
291
292    def _add_defaults_data_files(self):
293        # getting distribution.data_files
294        if self.distribution.has_data_files():
295            for item in self.distribution.data_files:
296                if isinstance(item, str):
297                    # plain file
298                    item = convert_path(item)
299                    if os.path.isfile(item):
300                        self.filelist.append(item)
301                else:
302                    # a (dirname, filenames) tuple
303                    dirname, filenames = item
304                    for f in filenames:
305                        f = convert_path(f)
306                        if os.path.isfile(f):
307                            self.filelist.append(f)
308
309    def _add_defaults_ext(self):
310        if self.distribution.has_ext_modules():
311            build_ext = self.get_finalized_command('build_ext')
312            self.filelist.extend(build_ext.get_source_files())
313
314    def _add_defaults_c_libs(self):
315        if self.distribution.has_c_libraries():
316            build_clib = self.get_finalized_command('build_clib')
317            self.filelist.extend(build_clib.get_source_files())
318
319    def _add_defaults_scripts(self):
320        if self.distribution.has_scripts():
321            build_scripts = self.get_finalized_command('build_scripts')
322            self.filelist.extend(build_scripts.get_source_files())
323
324    def read_template(self):
325        """Read and parse manifest template file named by self.template.
326
327        (usually "MANIFEST.in") The parsing and processing is done by
328        'self.filelist', which updates itself accordingly.
329        """
330        log.info("reading manifest template '%s'", self.template)
331        template = TextFile(self.template, strip_comments=1, skip_blanks=1,
332                            join_lines=1, lstrip_ws=1, rstrip_ws=1,
333                            collapse_join=1)
334
335        try:
336            while True:
337                line = template.readline()
338                if line is None:            # end of file
339                    break
340
341                try:
342                    self.filelist.process_template_line(line)
343                # the call above can raise a DistutilsTemplateError for
344                # malformed lines, or a ValueError from the lower-level
345                # convert_path function
346                except (DistutilsTemplateError, ValueError) as msg:
347                    self.warn("%s, line %d: %s" % (template.filename,
348                                                   template.current_line,
349                                                   msg))
350        finally:
351            template.close()
352
353    def prune_file_list(self):
354        """Prune off branches that might slip into the file list as created
355        by 'read_template()', but really don't belong there:
356          * the build tree (typically "build")
357          * the release tree itself (only an issue if we ran "sdist"
358            previously with --keep-temp, or it aborted)
359          * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories
360        """
361        build = self.get_finalized_command('build')
362        base_dir = self.distribution.get_fullname()
363
364        self.filelist.exclude_pattern(None, prefix=build.build_base)
365        self.filelist.exclude_pattern(None, prefix=base_dir)
366
367        if sys.platform == 'win32':
368            seps = r'/|\\'
369        else:
370            seps = '/'
371
372        vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr',
373                    '_darcs']
374        vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps)
375        self.filelist.exclude_pattern(vcs_ptrn, is_regex=1)
376
377    def write_manifest(self):
378        """Write the file list in 'self.filelist' (presumably as filled in
379        by 'add_defaults()' and 'read_template()') to the manifest file
380        named by 'self.manifest'.
381        """
382        if self._manifest_is_not_generated():
383            log.info("not writing to manually maintained "
384                     "manifest file '%s'" % self.manifest)
385            return
386
387        content = self.filelist.files[:]
388        content.insert(0, '# file GENERATED by distutils, do NOT edit')
389        self.execute(file_util.write_file, (self.manifest, content),
390                     "writing manifest file '%s'" % self.manifest)
391
392    def _manifest_is_not_generated(self):
393        # check for special comment used in 3.1.3 and higher
394        if not os.path.isfile(self.manifest):
395            return False
396
397        fp = open(self.manifest)
398        try:
399            first_line = fp.readline()
400        finally:
401            fp.close()
402        return first_line != '# file GENERATED by distutils, do NOT edit\n'
403
404    def read_manifest(self):
405        """Read the manifest file (named by 'self.manifest') and use it to
406        fill in 'self.filelist', the list of files to include in the source
407        distribution.
408        """
409        log.info("reading manifest file '%s'", self.manifest)
410        with open(self.manifest) as manifest:
411            for line in manifest:
412                # ignore comments and blank lines
413                line = line.strip()
414                if line.startswith('#') or not line:
415                    continue
416                self.filelist.append(line)
417
418    def make_release_tree(self, base_dir, files):
419        """Create the directory tree that will become the source
420        distribution archive.  All directories implied by the filenames in
421        'files' are created under 'base_dir', and then we hard link or copy
422        (if hard linking is unavailable) those files into place.
423        Essentially, this duplicates the developer's source tree, but in a
424        directory named after the distribution, containing only the files
425        to be distributed.
426        """
427        # Create all the directories under 'base_dir' necessary to
428        # put 'files' there; the 'mkpath()' is just so we don't die
429        # if the manifest happens to be empty.
430        self.mkpath(base_dir)
431        dir_util.create_tree(base_dir, files, dry_run=self.dry_run)
432
433        # And walk over the list of files, either making a hard link (if
434        # os.link exists) to each one that doesn't already exist in its
435        # corresponding location under 'base_dir', or copying each file
436        # that's out-of-date in 'base_dir'.  (Usually, all files will be
437        # out-of-date, because by default we blow away 'base_dir' when
438        # we're done making the distribution archives.)
439
440        if hasattr(os, 'link'):        # can make hard links on this system
441            link = 'hard'
442            msg = "making hard links in %s..." % base_dir
443        else:                           # nope, have to copy
444            link = None
445            msg = "copying files to %s..." % base_dir
446
447        if not files:
448            log.warn("no files to distribute -- empty manifest?")
449        else:
450            log.info(msg)
451        for file in files:
452            if not os.path.isfile(file):
453                log.warn("'%s' not a regular file -- skipping", file)
454            else:
455                dest = os.path.join(base_dir, file)
456                self.copy_file(file, dest, link=link)
457
458        self.distribution.metadata.write_pkg_info(base_dir)
459
460    def make_distribution(self):
461        """Create the source distribution(s).  First, we create the release
462        tree with 'make_release_tree()'; then, we create all required
463        archive files (according to 'self.formats') from the release tree.
464        Finally, we clean up by blowing away the release tree (unless
465        'self.keep_temp' is true).  The list of archive files created is
466        stored so it can be retrieved later by 'get_archive_files()'.
467        """
468        # Don't warn about missing meta-data here -- should be (and is!)
469        # done elsewhere.
470        base_dir = self.distribution.get_fullname()
471        base_name = os.path.join(self.dist_dir, base_dir)
472
473        self.make_release_tree(base_dir, self.filelist.files)
474        archive_files = []              # remember names of files we create
475        # tar archive must be created last to avoid overwrite and remove
476        if 'tar' in self.formats:
477            self.formats.append(self.formats.pop(self.formats.index('tar')))
478
479        for fmt in self.formats:
480            file = self.make_archive(base_name, fmt, base_dir=base_dir,
481                                     owner=self.owner, group=self.group)
482            archive_files.append(file)
483            self.distribution.dist_files.append(('sdist', '', file))
484
485        self.archive_files = archive_files
486
487        if not self.keep_temp:
488            dir_util.remove_tree(base_dir, dry_run=self.dry_run)
489
490    def get_archive_files(self):
491        """Return the list of archive files created when the command
492        was run, or None if the command hasn't run yet.
493        """
494        return self.archive_files
495