1import fnmatch
2import glob
3import os.path
4import sys
5
6from _pydev_bundle import pydev_log
7import pydevd_file_utils
8import json
9from collections import namedtuple
10from _pydev_imps._pydev_saved_modules import threading
11from pydevd_file_utils import normcase
12from _pydevd_bundle.pydevd_constants import USER_CODE_BASENAMES_STARTING_WITH, \
13    LIBRARY_CODE_BASENAMES_STARTING_WITH, IS_PYPY, IS_WINDOWS
14from _pydevd_bundle import pydevd_constants
15
16try:
17    xrange  # noqa
18except NameError:
19    xrange = range  # noqa
20
21ExcludeFilter = namedtuple('ExcludeFilter', 'name, exclude, is_path')
22
23
24def _convert_to_str_and_clear_empty(roots):
25    if sys.version_info[0] <= 2:
26        # In py2 we need bytes for the files.
27        roots = [
28            root if not isinstance(root, unicode) else root.encode(sys.getfilesystemencoding())
29            for root in roots
30        ]
31
32    new_roots = []
33    for root in roots:
34        assert isinstance(root, str), '%s not str (found: %s)' % (root, type(root))
35        if root:
36            new_roots.append(root)
37    return new_roots
38
39
40def _check_matches(patterns, paths):
41    if not patterns and not paths:
42        # Matched to the end.
43        return True
44
45    if (not patterns and paths) or (patterns and not paths):
46        return False
47
48    pattern = normcase(patterns[0])
49    path = normcase(paths[0])
50
51    if not glob.has_magic(pattern):
52
53        if pattern != path:
54            return False
55
56    elif pattern == '**':
57        if len(patterns) == 1:
58            return True  # if ** is the last one it matches anything to the right.
59
60        for i in xrange(len(paths)):
61            # Recursively check the remaining patterns as the
62            # current pattern could match any number of paths.
63            if _check_matches(patterns[1:], paths[i:]):
64                return True
65
66    elif not fnmatch.fnmatch(path, pattern):
67        # Current part doesn't match.
68        return False
69
70    return _check_matches(patterns[1:], paths[1:])
71
72
73def glob_matches_path(path, pattern, sep=os.sep, altsep=os.altsep):
74    if altsep:
75        pattern = pattern.replace(altsep, sep)
76        path = path.replace(altsep, sep)
77
78    drive = ''
79    if len(path) > 1 and path[1] == ':':
80        drive, path = path[0], path[2:]
81
82    if drive and len(pattern) > 1:
83        if pattern[1] == ':':
84            if drive.lower() != pattern[0].lower():
85                return False
86            pattern = pattern[2:]
87
88    patterns = pattern.split(sep)
89    paths = path.split(sep)
90    if paths:
91        if paths[0] == '':
92            paths = paths[1:]
93    if patterns:
94        if patterns[0] == '':
95            patterns = patterns[1:]
96
97    return _check_matches(patterns, paths)
98
99
100class FilesFiltering(object):
101    '''
102    Note: calls at FilesFiltering are uncached.
103
104    The actual API used should be through PyDB.
105    '''
106
107    def __init__(self):
108        self._exclude_filters = []
109        self._project_roots = []
110        self._library_roots = []
111
112        # Filter out libraries?
113        self._use_libraries_filter = False
114        self.require_module = False  # True if some exclude filter filters by the module.
115
116        self.set_use_libraries_filter(os.getenv('PYDEVD_FILTER_LIBRARIES') is not None)
117
118        project_roots = os.getenv('IDE_PROJECT_ROOTS', None)
119        if project_roots is not None:
120            project_roots = project_roots.split(os.pathsep)
121        else:
122            project_roots = []
123        self.set_project_roots(project_roots)
124
125        library_roots = os.getenv('LIBRARY_ROOTS', None)
126        if library_roots is not None:
127            library_roots = library_roots.split(os.pathsep)
128        else:
129            library_roots = self._get_default_library_roots()
130        self.set_library_roots(library_roots)
131
132        # Stepping filters.
133        pydevd_filters = os.getenv('PYDEVD_FILTERS', '')
134        # To filter out it's something as: {'**/not_my_code/**': True}
135        if pydevd_filters:
136            pydev_log.debug("PYDEVD_FILTERS %s", (pydevd_filters,))
137            if pydevd_filters.startswith('{'):
138                # dict(glob_pattern (str) -> exclude(True or False))
139                exclude_filters = []
140                for key, val in json.loads(pydevd_filters).items():
141                    exclude_filters.append(ExcludeFilter(key, val, True))
142                self._exclude_filters = exclude_filters
143            else:
144                # A ';' separated list of strings with globs for the
145                # list of excludes.
146                filters = pydevd_filters.split(';')
147                new_filters = []
148                for new_filter in filters:
149                    if new_filter.strip():
150                        new_filters.append(ExcludeFilter(new_filter.strip(), True, True))
151                self._exclude_filters = new_filters
152
153    @classmethod
154    def _get_default_library_roots(cls):
155        pydev_log.debug("Collecting default library roots.")
156        # Provide sensible defaults if not in env vars.
157        import site
158
159        roots = []
160
161        try:
162            import sysconfig  # Python 2.7 onwards only.
163        except ImportError:
164            pass
165        else:
166            for path_name in set(('stdlib', 'platstdlib', 'purelib', 'platlib')) & set(sysconfig.get_path_names()):
167                roots.append(sysconfig.get_path(path_name))
168
169        # Make sure we always get at least the standard library location (based on the `os` and
170        # `threading` modules -- it's a bit weird that it may be different on the ci, but it happens).
171        roots.append(os.path.dirname(os.__file__))
172        roots.append(os.path.dirname(threading.__file__))
173        if IS_PYPY:
174            # On PyPy 3.6 (7.3.1) it wrongly says that sysconfig.get_path('stdlib') is
175            # <install>/lib-pypy when the installed version is <install>/lib_pypy.
176            try:
177                import _pypy_wait
178            except ImportError:
179                pydev_log.debug("Unable to import _pypy_wait on PyPy when collecting default library roots.")
180            else:
181                pypy_lib_dir = os.path.dirname(_pypy_wait.__file__)
182                pydev_log.debug("Adding %s to default library roots.", pypy_lib_dir)
183                roots.append(pypy_lib_dir)
184
185        if hasattr(site, 'getusersitepackages'):
186            site_paths = site.getusersitepackages()
187            if isinstance(site_paths, (list, tuple)):
188                for site_path in site_paths:
189                    roots.append(site_path)
190            else:
191                roots.append(site_paths)
192
193        if hasattr(site, 'getsitepackages'):
194            site_paths = site.getsitepackages()
195            if isinstance(site_paths, (list, tuple)):
196                for site_path in site_paths:
197                    roots.append(site_path)
198            else:
199                roots.append(site_paths)
200
201        for path in sys.path:
202            if os.path.exists(path) and os.path.basename(path) in ('site-packages', 'pip-global'):
203                roots.append(path)
204
205        roots.extend([os.path.realpath(path) for path in roots])
206
207        return sorted(set(roots))
208
209    def _fix_roots(self, roots):
210        roots = _convert_to_str_and_clear_empty(roots)
211        new_roots = []
212        for root in roots:
213            path = self._absolute_normalized_path(root)
214            if pydevd_constants.IS_WINDOWS:
215                new_roots.append(path + '\\')
216            else:
217                new_roots.append(path + '/')
218        return new_roots
219
220    def _absolute_normalized_path(self, filename):
221        '''
222        Provides a version of the filename that's absolute and normalized.
223        '''
224        return normcase(pydevd_file_utils.absolute_path(filename))
225
226    def set_project_roots(self, project_roots):
227        self._project_roots = self._fix_roots(project_roots)
228        pydev_log.debug("IDE_PROJECT_ROOTS %s\n" % project_roots)
229
230    def _get_project_roots(self):
231        return self._project_roots
232
233    def set_library_roots(self, roots):
234        self._library_roots = self._fix_roots(roots)
235        pydev_log.debug("LIBRARY_ROOTS %s\n" % roots)
236
237    def _get_library_roots(self):
238        return self._library_roots
239
240    def in_project_roots(self, received_filename):
241        '''
242        Note: don't call directly. Use PyDb.in_project_scope (there's no caching here and it doesn't
243        handle all possibilities for knowing whether a project is actually in the scope, it
244        just handles the heuristics based on the absolute_normalized_filename without the actual frame).
245        '''
246        DEBUG = False
247
248        if received_filename.startswith(USER_CODE_BASENAMES_STARTING_WITH):
249            if DEBUG:
250                pydev_log.debug('In in_project_roots - user basenames - starts with %s (%s)', received_filename, USER_CODE_BASENAMES_STARTING_WITH)
251            return True
252
253        if received_filename.startswith(LIBRARY_CODE_BASENAMES_STARTING_WITH):
254            if DEBUG:
255                pydev_log.debug('Not in in_project_roots - library basenames - starts with %s (%s)', received_filename, LIBRARY_CODE_BASENAMES_STARTING_WITH)
256            return False
257
258        project_roots = self._get_project_roots()  # roots are absolute/normalized.
259
260        absolute_normalized_filename = self._absolute_normalized_path(received_filename)
261        absolute_normalized_filename_as_dir = absolute_normalized_filename + ('\\' if IS_WINDOWS else '/')
262
263        found_in_project = []
264        for root in project_roots:
265            if root and (absolute_normalized_filename.startswith(root) or root == absolute_normalized_filename_as_dir):
266                if DEBUG:
267                    pydev_log.debug('In project: %s (%s)', absolute_normalized_filename, root)
268                found_in_project.append(root)
269
270        found_in_library = []
271        library_roots = self._get_library_roots()
272        for root in library_roots:
273            if root and (absolute_normalized_filename.startswith(root) or root == absolute_normalized_filename_as_dir):
274                found_in_library.append(root)
275                if DEBUG:
276                    pydev_log.debug('In library: %s (%s)', absolute_normalized_filename, root)
277            else:
278                if DEBUG:
279                    pydev_log.debug('Not in library: %s (%s)', absolute_normalized_filename, root)
280
281        if not project_roots:
282            # If we have no project roots configured, consider it being in the project
283            # roots if it's not found in site-packages (because we have defaults for those
284            # and not the other way around).
285            in_project = not found_in_library
286            if DEBUG:
287                pydev_log.debug('Final in project (no project roots): %s (%s)', absolute_normalized_filename, in_project)
288
289        else:
290            in_project = False
291            if found_in_project:
292                if not found_in_library:
293                    if DEBUG:
294                        pydev_log.debug('Final in project (in_project and not found_in_library): %s (True)', absolute_normalized_filename)
295                    in_project = True
296                else:
297                    # Found in both, let's see which one has the bigger path matched.
298                    if max(len(x) for x in found_in_project) > max(len(x) for x in found_in_library):
299                        in_project = True
300                    if DEBUG:
301                        pydev_log.debug('Final in project (found in both): %s (%s)', absolute_normalized_filename, in_project)
302
303        return in_project
304
305    def use_libraries_filter(self):
306        '''
307        Should we debug only what's inside project folders?
308        '''
309        return self._use_libraries_filter
310
311    def set_use_libraries_filter(self, use):
312        pydev_log.debug("pydevd: Use libraries filter: %s\n" % use)
313        self._use_libraries_filter = use
314
315    def use_exclude_filters(self):
316        # Enabled if we have any filters registered.
317        return len(self._exclude_filters) > 0
318
319    def exclude_by_filter(self, absolute_filename, module_name):
320        '''
321        :return: True if it should be excluded, False if it should be included and None
322            if no rule matched the given file.
323        '''
324        for exclude_filter in self._exclude_filters:  # : :type exclude_filter: ExcludeFilter
325            if exclude_filter.is_path:
326                if glob_matches_path(absolute_filename, exclude_filter.name):
327                    return exclude_filter.exclude
328            else:
329                # Module filter.
330                if exclude_filter.name == module_name or module_name.startswith(exclude_filter.name + '.'):
331                    return exclude_filter.exclude
332        return None
333
334    def set_exclude_filters(self, exclude_filters):
335        '''
336        :param list(ExcludeFilter) exclude_filters:
337        '''
338        self._exclude_filters = exclude_filters
339        self.require_module = False
340        for exclude_filter in exclude_filters:
341            if not exclude_filter.is_path:
342                self.require_module = True
343                break
344