1"""distutils.filelist
2
3Provides the FileList class, used for poking about the filesystem
4and building lists of files.
5"""
6
7__revision__ = "$Id$"
8
9import os, re
10import fnmatch
11from distutils.util import convert_path
12from distutils.errors import DistutilsTemplateError, DistutilsInternalError
13from distutils import log
14
15class FileList:
16    """A list of files built by on exploring the filesystem and filtered by
17    applying various patterns to what we find there.
18
19    Instance attributes:
20      dir
21        directory from which files will be taken -- only used if
22        'allfiles' not supplied to constructor
23      files
24        list of filenames currently being built/filtered/manipulated
25      allfiles
26        complete list of files under consideration (ie. without any
27        filtering applied)
28    """
29
30    def __init__(self, warn=None, debug_print=None):
31        # ignore argument to FileList, but keep them for backwards
32        # compatibility
33        self.allfiles = None
34        self.files = []
35
36    def set_allfiles(self, allfiles):
37        self.allfiles = allfiles
38
39    def findall(self, dir=os.curdir):
40        self.allfiles = findall(dir)
41
42    def debug_print(self, msg):
43        """Print 'msg' to stdout if the global DEBUG (taken from the
44        DISTUTILS_DEBUG environment variable) flag is true.
45        """
46        from distutils.debug import DEBUG
47        if DEBUG:
48            print msg
49
50    # -- List-like methods ---------------------------------------------
51
52    def append(self, item):
53        self.files.append(item)
54
55    def extend(self, items):
56        self.files.extend(items)
57
58    def sort(self):
59        # Not a strict lexical sort!
60        sortable_files = map(os.path.split, self.files)
61        sortable_files.sort()
62        self.files = []
63        for sort_tuple in sortable_files:
64            self.files.append(os.path.join(*sort_tuple))
65
66
67    # -- Other miscellaneous utility methods ---------------------------
68
69    def remove_duplicates(self):
70        # Assumes list has been sorted!
71        for i in range(len(self.files) - 1, 0, -1):
72            if self.files[i] == self.files[i - 1]:
73                del self.files[i]
74
75
76    # -- "File template" methods ---------------------------------------
77
78    def _parse_template_line(self, line):
79        words = line.split()
80        action = words[0]
81
82        patterns = dir = dir_pattern = None
83
84        if action in ('include', 'exclude',
85                      'global-include', 'global-exclude'):
86            if len(words) < 2:
87                raise DistutilsTemplateError, \
88                      "'%s' expects <pattern1> <pattern2> ..." % action
89
90            patterns = map(convert_path, words[1:])
91
92        elif action in ('recursive-include', 'recursive-exclude'):
93            if len(words) < 3:
94                raise DistutilsTemplateError, \
95                      "'%s' expects <dir> <pattern1> <pattern2> ..." % action
96
97            dir = convert_path(words[1])
98            patterns = map(convert_path, words[2:])
99
100        elif action in ('graft', 'prune'):
101            if len(words) != 2:
102                raise DistutilsTemplateError, \
103                     "'%s' expects a single <dir_pattern>" % action
104
105            dir_pattern = convert_path(words[1])
106
107        else:
108            raise DistutilsTemplateError, "unknown action '%s'" % action
109
110        return (action, patterns, dir, dir_pattern)
111
112    def process_template_line(self, line):
113        # Parse the line: split it up, make sure the right number of words
114        # is there, and return the relevant words.  'action' is always
115        # defined: it's the first word of the line.  Which of the other
116        # three are defined depends on the action; it'll be either
117        # patterns, (dir and patterns), or (dir_pattern).
118        action, patterns, dir, dir_pattern = self._parse_template_line(line)
119
120        # OK, now we know that the action is valid and we have the
121        # right number of words on the line for that action -- so we
122        # can proceed with minimal error-checking.
123        if action == 'include':
124            self.debug_print("include " + ' '.join(patterns))
125            for pattern in patterns:
126                if not self.include_pattern(pattern, anchor=1):
127                    log.warn("warning: no files found matching '%s'",
128                             pattern)
129
130        elif action == 'exclude':
131            self.debug_print("exclude " + ' '.join(patterns))
132            for pattern in patterns:
133                if not self.exclude_pattern(pattern, anchor=1):
134                    log.warn(("warning: no previously-included files "
135                              "found matching '%s'"), pattern)
136
137        elif action == 'global-include':
138            self.debug_print("global-include " + ' '.join(patterns))
139            for pattern in patterns:
140                if not self.include_pattern(pattern, anchor=0):
141                    log.warn(("warning: no files found matching '%s' " +
142                              "anywhere in distribution"), pattern)
143
144        elif action == 'global-exclude':
145            self.debug_print("global-exclude " + ' '.join(patterns))
146            for pattern in patterns:
147                if not self.exclude_pattern(pattern, anchor=0):
148                    log.warn(("warning: no previously-included files matching "
149                              "'%s' found anywhere in distribution"),
150                             pattern)
151
152        elif action == 'recursive-include':
153            self.debug_print("recursive-include %s %s" %
154                             (dir, ' '.join(patterns)))
155            for pattern in patterns:
156                if not self.include_pattern(pattern, prefix=dir):
157                    log.warn(("warning: no files found matching '%s' " +
158                                "under directory '%s'"),
159                             pattern, dir)
160
161        elif action == 'recursive-exclude':
162            self.debug_print("recursive-exclude %s %s" %
163                             (dir, ' '.join(patterns)))
164            for pattern in patterns:
165                if not self.exclude_pattern(pattern, prefix=dir):
166                    log.warn(("warning: no previously-included files matching "
167                              "'%s' found under directory '%s'"),
168                             pattern, dir)
169
170        elif action == 'graft':
171            self.debug_print("graft " + dir_pattern)
172            if not self.include_pattern(None, prefix=dir_pattern):
173                log.warn("warning: no directories found matching '%s'",
174                         dir_pattern)
175
176        elif action == 'prune':
177            self.debug_print("prune " + dir_pattern)
178            if not self.exclude_pattern(None, prefix=dir_pattern):
179                log.warn(("no previously-included directories found " +
180                          "matching '%s'"), dir_pattern)
181        else:
182            raise DistutilsInternalError, \
183                  "this cannot happen: invalid action '%s'" % action
184
185    # -- Filtering/selection methods -----------------------------------
186
187    def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
188        """Select strings (presumably filenames) from 'self.files' that
189        match 'pattern', a Unix-style wildcard (glob) pattern.
190
191        Patterns are not quite the same as implemented by the 'fnmatch'
192        module: '*' and '?'  match non-special characters, where "special"
193        is platform-dependent: slash on Unix; colon, slash, and backslash on
194        DOS/Windows; and colon on Mac OS.
195
196        If 'anchor' is true (the default), then the pattern match is more
197        stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If
198        'anchor' is false, both of these will match.
199
200        If 'prefix' is supplied, then only filenames starting with 'prefix'
201        (itself a pattern) and ending with 'pattern', with anything in between
202        them, will match.  'anchor' is ignored in this case.
203
204        If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
205        'pattern' is assumed to be either a string containing a regex or a
206        regex object -- no translation is done, the regex is just compiled
207        and used as-is.
208
209        Selected strings will be added to self.files.
210
211        Return 1 if files are found.
212        """
213        # XXX docstring lying about what the special chars are?
214        files_found = 0
215        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
216        self.debug_print("include_pattern: applying regex r'%s'" %
217                         pattern_re.pattern)
218
219        # delayed loading of allfiles list
220        if self.allfiles is None:
221            self.findall()
222
223        for name in self.allfiles:
224            if pattern_re.search(name):
225                self.debug_print(" adding " + name)
226                self.files.append(name)
227                files_found = 1
228
229        return files_found
230
231
232    def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
233        """Remove strings (presumably filenames) from 'files' that match
234        'pattern'.
235
236        Other parameters are the same as for 'include_pattern()', above.
237        The list 'self.files' is modified in place. Return 1 if files are
238        found.
239        """
240        files_found = 0
241        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
242        self.debug_print("exclude_pattern: applying regex r'%s'" %
243                         pattern_re.pattern)
244        for i in range(len(self.files)-1, -1, -1):
245            if pattern_re.search(self.files[i]):
246                self.debug_print(" removing " + self.files[i])
247                del self.files[i]
248                files_found = 1
249
250        return files_found
251
252
253# ----------------------------------------------------------------------
254# Utility functions
255
256def findall(dir = os.curdir):
257    """Find all files under 'dir' and return the list of full filenames
258    (relative to 'dir').
259    """
260    from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK
261
262    list = []
263    stack = [dir]
264    pop = stack.pop
265    push = stack.append
266
267    while stack:
268        dir = pop()
269        names = os.listdir(dir)
270
271        for name in names:
272            if dir != os.curdir:        # avoid the dreaded "./" syndrome
273                fullname = os.path.join(dir, name)
274            else:
275                fullname = name
276
277            # Avoid excess stat calls -- just one will do, thank you!
278            stat = os.stat(fullname)
279            mode = stat[ST_MODE]
280            if S_ISREG(mode):
281                list.append(fullname)
282            elif S_ISDIR(mode) and not S_ISLNK(mode):
283                push(fullname)
284
285    return list
286
287
288def glob_to_re(pattern):
289    """Translate a shell-like glob pattern to a regular expression.
290
291    Return a string containing the regex.  Differs from
292    'fnmatch.translate()' in that '*' does not match "special characters"
293    (which are platform-specific).
294    """
295    pattern_re = fnmatch.translate(pattern)
296
297    # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
298    # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
299    # and by extension they shouldn't match such "special characters" under
300    # any OS.  So change all non-escaped dots in the RE to match any
301    # character except the special characters (currently: just os.sep).
302    sep = os.sep
303    if os.sep == '\\':
304        # we're using a regex to manipulate a regex, so we need
305        # to escape the backslash twice
306        sep = r'\\\\'
307    escaped = r'\1[^%s]' % sep
308    pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
309    return pattern_re
310
311
312def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
313    """Translate a shell-like wildcard pattern to a compiled regular
314    expression.
315
316    Return the compiled regex.  If 'is_regex' true,
317    then 'pattern' is directly compiled to a regex (if it's a string)
318    or just returned as-is (assumes it's a regex object).
319    """
320    if is_regex:
321        if isinstance(pattern, str):
322            return re.compile(pattern)
323        else:
324            return pattern
325
326    if pattern:
327        pattern_re = glob_to_re(pattern)
328    else:
329        pattern_re = ''
330
331    if prefix is not None:
332        # ditch end of pattern character
333        empty_pattern = glob_to_re('')
334        prefix_re = glob_to_re(prefix)[:-len(empty_pattern)]
335        sep = os.sep
336        if os.sep == '\\':
337            sep = r'\\'
338        pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
339    else:                               # no prefix -- respect anchor flag
340        if anchor:
341            pattern_re = "^" + pattern_re
342
343    return re.compile(pattern_re)
344