1import fnmatch 2import glob 3import os.path 4import sys 5 6from _pydev_bundle import pydev_log 7import pydevd_file_utils 8import json 9from collections import namedtuple 10from _pydev_imps._pydev_saved_modules import threading 11from pydevd_file_utils import normcase 12from _pydevd_bundle.pydevd_constants import USER_CODE_BASENAMES_STARTING_WITH, \ 13 LIBRARY_CODE_BASENAMES_STARTING_WITH, IS_PYPY, IS_WINDOWS 14from _pydevd_bundle import pydevd_constants 15 16try: 17 xrange # noqa 18except NameError: 19 xrange = range # noqa 20 21ExcludeFilter = namedtuple('ExcludeFilter', 'name, exclude, is_path') 22 23 24def _convert_to_str_and_clear_empty(roots): 25 if sys.version_info[0] <= 2: 26 # In py2 we need bytes for the files. 27 roots = [ 28 root if not isinstance(root, unicode) else root.encode(sys.getfilesystemencoding()) 29 for root in roots 30 ] 31 32 new_roots = [] 33 for root in roots: 34 assert isinstance(root, str), '%s not str (found: %s)' % (root, type(root)) 35 if root: 36 new_roots.append(root) 37 return new_roots 38 39 40def _check_matches(patterns, paths): 41 if not patterns and not paths: 42 # Matched to the end. 43 return True 44 45 if (not patterns and paths) or (patterns and not paths): 46 return False 47 48 pattern = normcase(patterns[0]) 49 path = normcase(paths[0]) 50 51 if not glob.has_magic(pattern): 52 53 if pattern != path: 54 return False 55 56 elif pattern == '**': 57 if len(patterns) == 1: 58 return True # if ** is the last one it matches anything to the right. 59 60 for i in xrange(len(paths)): 61 # Recursively check the remaining patterns as the 62 # current pattern could match any number of paths. 63 if _check_matches(patterns[1:], paths[i:]): 64 return True 65 66 elif not fnmatch.fnmatch(path, pattern): 67 # Current part doesn't match. 68 return False 69 70 return _check_matches(patterns[1:], paths[1:]) 71 72 73def glob_matches_path(path, pattern, sep=os.sep, altsep=os.altsep): 74 if altsep: 75 pattern = pattern.replace(altsep, sep) 76 path = path.replace(altsep, sep) 77 78 drive = '' 79 if len(path) > 1 and path[1] == ':': 80 drive, path = path[0], path[2:] 81 82 if drive and len(pattern) > 1: 83 if pattern[1] == ':': 84 if drive.lower() != pattern[0].lower(): 85 return False 86 pattern = pattern[2:] 87 88 patterns = pattern.split(sep) 89 paths = path.split(sep) 90 if paths: 91 if paths[0] == '': 92 paths = paths[1:] 93 if patterns: 94 if patterns[0] == '': 95 patterns = patterns[1:] 96 97 return _check_matches(patterns, paths) 98 99 100class FilesFiltering(object): 101 ''' 102 Note: calls at FilesFiltering are uncached. 103 104 The actual API used should be through PyDB. 105 ''' 106 107 def __init__(self): 108 self._exclude_filters = [] 109 self._project_roots = [] 110 self._library_roots = [] 111 112 # Filter out libraries? 113 self._use_libraries_filter = False 114 self.require_module = False # True if some exclude filter filters by the module. 115 116 self.set_use_libraries_filter(os.getenv('PYDEVD_FILTER_LIBRARIES') is not None) 117 118 project_roots = os.getenv('IDE_PROJECT_ROOTS', None) 119 if project_roots is not None: 120 project_roots = project_roots.split(os.pathsep) 121 else: 122 project_roots = [] 123 self.set_project_roots(project_roots) 124 125 library_roots = os.getenv('LIBRARY_ROOTS', None) 126 if library_roots is not None: 127 library_roots = library_roots.split(os.pathsep) 128 else: 129 library_roots = self._get_default_library_roots() 130 self.set_library_roots(library_roots) 131 132 # Stepping filters. 133 pydevd_filters = os.getenv('PYDEVD_FILTERS', '') 134 # To filter out it's something as: {'**/not_my_code/**': True} 135 if pydevd_filters: 136 pydev_log.debug("PYDEVD_FILTERS %s", (pydevd_filters,)) 137 if pydevd_filters.startswith('{'): 138 # dict(glob_pattern (str) -> exclude(True or False)) 139 exclude_filters = [] 140 for key, val in json.loads(pydevd_filters).items(): 141 exclude_filters.append(ExcludeFilter(key, val, True)) 142 self._exclude_filters = exclude_filters 143 else: 144 # A ';' separated list of strings with globs for the 145 # list of excludes. 146 filters = pydevd_filters.split(';') 147 new_filters = [] 148 for new_filter in filters: 149 if new_filter.strip(): 150 new_filters.append(ExcludeFilter(new_filter.strip(), True, True)) 151 self._exclude_filters = new_filters 152 153 @classmethod 154 def _get_default_library_roots(cls): 155 pydev_log.debug("Collecting default library roots.") 156 # Provide sensible defaults if not in env vars. 157 import site 158 159 roots = [] 160 161 try: 162 import sysconfig # Python 2.7 onwards only. 163 except ImportError: 164 pass 165 else: 166 for path_name in set(('stdlib', 'platstdlib', 'purelib', 'platlib')) & set(sysconfig.get_path_names()): 167 roots.append(sysconfig.get_path(path_name)) 168 169 # Make sure we always get at least the standard library location (based on the `os` and 170 # `threading` modules -- it's a bit weird that it may be different on the ci, but it happens). 171 roots.append(os.path.dirname(os.__file__)) 172 roots.append(os.path.dirname(threading.__file__)) 173 if IS_PYPY: 174 # On PyPy 3.6 (7.3.1) it wrongly says that sysconfig.get_path('stdlib') is 175 # <install>/lib-pypy when the installed version is <install>/lib_pypy. 176 try: 177 import _pypy_wait 178 except ImportError: 179 pydev_log.debug("Unable to import _pypy_wait on PyPy when collecting default library roots.") 180 else: 181 pypy_lib_dir = os.path.dirname(_pypy_wait.__file__) 182 pydev_log.debug("Adding %s to default library roots.", pypy_lib_dir) 183 roots.append(pypy_lib_dir) 184 185 if hasattr(site, 'getusersitepackages'): 186 site_paths = site.getusersitepackages() 187 if isinstance(site_paths, (list, tuple)): 188 for site_path in site_paths: 189 roots.append(site_path) 190 else: 191 roots.append(site_paths) 192 193 if hasattr(site, 'getsitepackages'): 194 site_paths = site.getsitepackages() 195 if isinstance(site_paths, (list, tuple)): 196 for site_path in site_paths: 197 roots.append(site_path) 198 else: 199 roots.append(site_paths) 200 201 for path in sys.path: 202 if os.path.exists(path) and os.path.basename(path) in ('site-packages', 'pip-global'): 203 roots.append(path) 204 205 roots.extend([os.path.realpath(path) for path in roots]) 206 207 return sorted(set(roots)) 208 209 def _fix_roots(self, roots): 210 roots = _convert_to_str_and_clear_empty(roots) 211 new_roots = [] 212 for root in roots: 213 path = self._absolute_normalized_path(root) 214 if pydevd_constants.IS_WINDOWS: 215 new_roots.append(path + '\\') 216 else: 217 new_roots.append(path + '/') 218 return new_roots 219 220 def _absolute_normalized_path(self, filename): 221 ''' 222 Provides a version of the filename that's absolute and normalized. 223 ''' 224 return normcase(pydevd_file_utils.absolute_path(filename)) 225 226 def set_project_roots(self, project_roots): 227 self._project_roots = self._fix_roots(project_roots) 228 pydev_log.debug("IDE_PROJECT_ROOTS %s\n" % project_roots) 229 230 def _get_project_roots(self): 231 return self._project_roots 232 233 def set_library_roots(self, roots): 234 self._library_roots = self._fix_roots(roots) 235 pydev_log.debug("LIBRARY_ROOTS %s\n" % roots) 236 237 def _get_library_roots(self): 238 return self._library_roots 239 240 def in_project_roots(self, received_filename): 241 ''' 242 Note: don't call directly. Use PyDb.in_project_scope (there's no caching here and it doesn't 243 handle all possibilities for knowing whether a project is actually in the scope, it 244 just handles the heuristics based on the absolute_normalized_filename without the actual frame). 245 ''' 246 DEBUG = False 247 248 if received_filename.startswith(USER_CODE_BASENAMES_STARTING_WITH): 249 if DEBUG: 250 pydev_log.debug('In in_project_roots - user basenames - starts with %s (%s)', received_filename, USER_CODE_BASENAMES_STARTING_WITH) 251 return True 252 253 if received_filename.startswith(LIBRARY_CODE_BASENAMES_STARTING_WITH): 254 if DEBUG: 255 pydev_log.debug('Not in in_project_roots - library basenames - starts with %s (%s)', received_filename, LIBRARY_CODE_BASENAMES_STARTING_WITH) 256 return False 257 258 project_roots = self._get_project_roots() # roots are absolute/normalized. 259 260 absolute_normalized_filename = self._absolute_normalized_path(received_filename) 261 absolute_normalized_filename_as_dir = absolute_normalized_filename + ('\\' if IS_WINDOWS else '/') 262 263 found_in_project = [] 264 for root in project_roots: 265 if root and (absolute_normalized_filename.startswith(root) or root == absolute_normalized_filename_as_dir): 266 if DEBUG: 267 pydev_log.debug('In project: %s (%s)', absolute_normalized_filename, root) 268 found_in_project.append(root) 269 270 found_in_library = [] 271 library_roots = self._get_library_roots() 272 for root in library_roots: 273 if root and (absolute_normalized_filename.startswith(root) or root == absolute_normalized_filename_as_dir): 274 found_in_library.append(root) 275 if DEBUG: 276 pydev_log.debug('In library: %s (%s)', absolute_normalized_filename, root) 277 else: 278 if DEBUG: 279 pydev_log.debug('Not in library: %s (%s)', absolute_normalized_filename, root) 280 281 if not project_roots: 282 # If we have no project roots configured, consider it being in the project 283 # roots if it's not found in site-packages (because we have defaults for those 284 # and not the other way around). 285 in_project = not found_in_library 286 if DEBUG: 287 pydev_log.debug('Final in project (no project roots): %s (%s)', absolute_normalized_filename, in_project) 288 289 else: 290 in_project = False 291 if found_in_project: 292 if not found_in_library: 293 if DEBUG: 294 pydev_log.debug('Final in project (in_project and not found_in_library): %s (True)', absolute_normalized_filename) 295 in_project = True 296 else: 297 # Found in both, let's see which one has the bigger path matched. 298 if max(len(x) for x in found_in_project) > max(len(x) for x in found_in_library): 299 in_project = True 300 if DEBUG: 301 pydev_log.debug('Final in project (found in both): %s (%s)', absolute_normalized_filename, in_project) 302 303 return in_project 304 305 def use_libraries_filter(self): 306 ''' 307 Should we debug only what's inside project folders? 308 ''' 309 return self._use_libraries_filter 310 311 def set_use_libraries_filter(self, use): 312 pydev_log.debug("pydevd: Use libraries filter: %s\n" % use) 313 self._use_libraries_filter = use 314 315 def use_exclude_filters(self): 316 # Enabled if we have any filters registered. 317 return len(self._exclude_filters) > 0 318 319 def exclude_by_filter(self, absolute_filename, module_name): 320 ''' 321 :return: True if it should be excluded, False if it should be included and None 322 if no rule matched the given file. 323 ''' 324 for exclude_filter in self._exclude_filters: # : :type exclude_filter: ExcludeFilter 325 if exclude_filter.is_path: 326 if glob_matches_path(absolute_filename, exclude_filter.name): 327 return exclude_filter.exclude 328 else: 329 # Module filter. 330 if exclude_filter.name == module_name or module_name.startswith(exclude_filter.name + '.'): 331 return exclude_filter.exclude 332 return None 333 334 def set_exclude_filters(self, exclude_filters): 335 ''' 336 :param list(ExcludeFilter) exclude_filters: 337 ''' 338 self._exclude_filters = exclude_filters 339 self.require_module = False 340 for exclude_filter in exclude_filters: 341 if not exclude_filter.is_path: 342 self.require_module = True 343 break 344