1"""Low-level infrastructure to find modules. 2 3This builds on fscache.py; find_sources.py builds on top of this. 4""" 5 6import ast 7import collections 8import functools 9import os 10import re 11import subprocess 12import sys 13from enum import Enum 14 15from typing import Dict, Iterator, List, NamedTuple, Optional, Set, Tuple, Union 16from typing_extensions import Final 17 18from mypy.fscache import FileSystemCache 19from mypy.options import Options 20from mypy.stubinfo import is_legacy_bundled_package 21from mypy import sitepkgs 22 23# Paths to be searched in find_module(). 24SearchPaths = NamedTuple( 25 'SearchPaths', 26 [('python_path', Tuple[str, ...]), # where user code is found 27 ('mypy_path', Tuple[str, ...]), # from $MYPYPATH or config variable 28 ('package_path', Tuple[str, ...]), # from get_site_packages_dirs() 29 ('typeshed_path', Tuple[str, ...]), # paths in typeshed 30 ]) 31 32# Package dirs are a two-tuple of path to search and whether to verify the module 33OnePackageDir = Tuple[str, bool] 34PackageDirs = List[OnePackageDir] 35 36PYTHON_EXTENSIONS = ['.pyi', '.py'] # type: Final 37 38PYTHON2_STUB_DIR = '@python2' # type: Final 39 40 41# TODO: Consider adding more reasons here? 42# E.g. if we deduce a module would likely be found if the user were 43# to set the --namespace-packages flag. 44class ModuleNotFoundReason(Enum): 45 # The module was not found: we found neither stubs nor a plausible code 46 # implementation (with or without a py.typed file). 47 NOT_FOUND = 0 48 49 # The implementation for this module plausibly exists (e.g. we 50 # found a matching folder or *.py file), but either the parent package 51 # did not contain a py.typed file or we were unable to find a 52 # corresponding *-stubs package. 53 FOUND_WITHOUT_TYPE_HINTS = 1 54 55 # The module was not found in the current working directory, but 56 # was able to be found in the parent directory. 57 WRONG_WORKING_DIRECTORY = 2 58 59 # Stub PyPI package (typically types-pkgname) known to exist but not installed. 60 APPROVED_STUBS_NOT_INSTALLED = 3 61 62 def error_message_templates(self, daemon: bool) -> Tuple[str, List[str]]: 63 doc_link = "See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports" 64 if self is ModuleNotFoundReason.NOT_FOUND: 65 msg = 'Cannot find implementation or library stub for module named "{module}"' 66 notes = [doc_link] 67 elif self is ModuleNotFoundReason.WRONG_WORKING_DIRECTORY: 68 msg = 'Cannot find implementation or library stub for module named "{module}"' 69 notes = ["You may be running mypy in a subpackage, " 70 "mypy should be run on the package root"] 71 elif self is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: 72 msg = 'Skipping analyzing "{module}": found module but no type hints or library stubs' 73 notes = [doc_link] 74 elif self is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: 75 msg = ( 76 'Library stubs not installed for "{module}" (or incompatible with Python {pyver})' 77 ) 78 notes = ['Hint: "python3 -m pip install {stub_dist}"'] 79 if not daemon: 80 notes.append( 81 '(or run "mypy --install-types" to install all missing stub packages)') 82 notes.append(doc_link) 83 else: 84 assert False 85 return msg, notes 86 87 88# If we found the module, returns the path to the module as a str. 89# Otherwise, returns the reason why the module wasn't found. 90ModuleSearchResult = Union[str, ModuleNotFoundReason] 91 92 93class BuildSource: 94 """A single source file.""" 95 96 def __init__(self, path: Optional[str], module: Optional[str], 97 text: Optional[str] = None, base_dir: Optional[str] = None) -> None: 98 self.path = path # File where it's found (e.g. 'xxx/yyy/foo/bar.py') 99 self.module = module or '__main__' # Module name (e.g. 'foo.bar') 100 self.text = text # Source code, if initially supplied, else None 101 self.base_dir = base_dir # Directory where the package is rooted (e.g. 'xxx/yyy') 102 103 def __repr__(self) -> str: 104 return 'BuildSource(path=%r, module=%r, has_text=%s, base_dir=%r)' % ( 105 self.path, 106 self.module, 107 self.text is not None, 108 self.base_dir) 109 110 111class FindModuleCache: 112 """Module finder with integrated cache. 113 114 Module locations and some intermediate results are cached internally 115 and can be cleared with the clear() method. 116 117 All file system accesses are performed through a FileSystemCache, 118 which is not ever cleared by this class. If necessary it must be 119 cleared by client code. 120 """ 121 122 def __init__(self, 123 search_paths: SearchPaths, 124 fscache: Optional[FileSystemCache], 125 options: Optional[Options]) -> None: 126 self.search_paths = search_paths 127 self.fscache = fscache or FileSystemCache() 128 # Cache for get_toplevel_possibilities: 129 # search_paths -> (toplevel_id -> list(package_dirs)) 130 self.initial_components = {} # type: Dict[Tuple[str, ...], Dict[str, List[str]]] 131 # Cache find_module: id -> result 132 self.results = {} # type: Dict[str, ModuleSearchResult] 133 self.ns_ancestors = {} # type: Dict[str, str] 134 self.options = options 135 custom_typeshed_dir = None 136 if options: 137 custom_typeshed_dir = options.custom_typeshed_dir 138 self.stdlib_py_versions = load_stdlib_py_versions(custom_typeshed_dir) 139 self.python_major_ver = 3 if options is None else options.python_version[0] 140 141 def clear(self) -> None: 142 self.results.clear() 143 self.initial_components.clear() 144 self.ns_ancestors.clear() 145 146 def find_lib_path_dirs(self, id: str, lib_path: Tuple[str, ...]) -> PackageDirs: 147 """Find which elements of a lib_path have the directory a module needs to exist. 148 149 This is run for the python_path, mypy_path, and typeshed_path search paths. 150 """ 151 components = id.split('.') 152 dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' 153 154 dirs = [] 155 for pathitem in self.get_toplevel_possibilities(lib_path, components[0]): 156 # e.g., '/usr/lib/python3.4/foo/bar' 157 dir = os.path.normpath(os.path.join(pathitem, dir_chain)) 158 if self.fscache.isdir(dir): 159 dirs.append((dir, True)) 160 return dirs 161 162 def get_toplevel_possibilities(self, lib_path: Tuple[str, ...], id: str) -> List[str]: 163 """Find which elements of lib_path could contain a particular top-level module. 164 165 In practice, almost all modules can be routed to the correct entry in 166 lib_path by looking at just the first component of the module name. 167 168 We take advantage of this by enumerating the contents of all of the 169 directories on the lib_path and building a map of which entries in 170 the lib_path could contain each potential top-level module that appears. 171 """ 172 173 if lib_path in self.initial_components: 174 return self.initial_components[lib_path].get(id, []) 175 176 # Enumerate all the files in the directories on lib_path and produce the map 177 components = {} # type: Dict[str, List[str]] 178 for dir in lib_path: 179 try: 180 contents = self.fscache.listdir(dir) 181 except OSError: 182 contents = [] 183 # False positives are fine for correctness here, since we will check 184 # precisely later, so we only look at the root of every filename without 185 # any concern for the exact details. 186 for name in contents: 187 name = os.path.splitext(name)[0] 188 components.setdefault(name, []).append(dir) 189 190 if self.python_major_ver == 2: 191 components = {id: filter_redundant_py2_dirs(dirs) 192 for id, dirs in components.items()} 193 194 self.initial_components[lib_path] = components 195 return components.get(id, []) 196 197 def find_module(self, id: str, *, fast_path: bool = False) -> ModuleSearchResult: 198 """Return the path of the module source file or why it wasn't found. 199 200 If fast_path is True, prioritize performance over generating detailed 201 error descriptions. 202 """ 203 if id not in self.results: 204 top_level = id.partition('.')[0] 205 use_typeshed = True 206 if top_level in self.stdlib_py_versions: 207 use_typeshed = self._typeshed_has_version(top_level) 208 self.results[id] = self._find_module(id, use_typeshed) 209 if (not fast_path 210 and self.results[id] is ModuleNotFoundReason.NOT_FOUND 211 and self._can_find_module_in_parent_dir(id)): 212 self.results[id] = ModuleNotFoundReason.WRONG_WORKING_DIRECTORY 213 return self.results[id] 214 215 def _typeshed_has_version(self, module: str) -> bool: 216 if not self.options: 217 return True 218 version = typeshed_py_version(self.options) 219 min_version, max_version = self.stdlib_py_versions[module] 220 return version >= min_version and (max_version is None or version <= max_version) 221 222 def _find_module_non_stub_helper(self, components: List[str], 223 pkg_dir: str) -> Union[OnePackageDir, ModuleNotFoundReason]: 224 plausible_match = False 225 dir_path = pkg_dir 226 for index, component in enumerate(components): 227 dir_path = os.path.join(dir_path, component) 228 if self.fscache.isfile(os.path.join(dir_path, 'py.typed')): 229 return os.path.join(pkg_dir, *components[:-1]), index == 0 230 elif not plausible_match and (self.fscache.isdir(dir_path) 231 or self.fscache.isfile(dir_path + ".py")): 232 plausible_match = True 233 if is_legacy_bundled_package(components[0], self.python_major_ver): 234 if (len(components) == 1 235 or (self.find_module(components[0]) is 236 ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED)): 237 return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED 238 if is_legacy_bundled_package('.'.join(components[:2]), self.python_major_ver): 239 return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED 240 if plausible_match: 241 return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS 242 else: 243 return ModuleNotFoundReason.NOT_FOUND 244 245 def _update_ns_ancestors(self, components: List[str], match: Tuple[str, bool]) -> None: 246 path, verify = match 247 for i in range(1, len(components)): 248 pkg_id = '.'.join(components[:-i]) 249 if pkg_id not in self.ns_ancestors and self.fscache.isdir(path): 250 self.ns_ancestors[pkg_id] = path 251 path = os.path.dirname(path) 252 253 def _can_find_module_in_parent_dir(self, id: str) -> bool: 254 """Test if a module can be found by checking the parent directories 255 of the current working directory. 256 """ 257 working_dir = os.getcwd() 258 parent_search = FindModuleCache(SearchPaths((), (), (), ()), self.fscache, self.options) 259 while any(file.endswith(("__init__.py", "__init__.pyi")) 260 for file in os.listdir(working_dir)): 261 working_dir = os.path.dirname(working_dir) 262 parent_search.search_paths = SearchPaths((working_dir,), (), (), ()) 263 if not isinstance(parent_search._find_module(id, False), ModuleNotFoundReason): 264 return True 265 return False 266 267 def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: 268 fscache = self.fscache 269 270 # If we're looking for a module like 'foo.bar.baz', it's likely that most of the 271 # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover 272 # that only once and cache it for when we look for modules like 'foo.bar.blah' 273 # that will require the same subdirectory. 274 components = id.split('.') 275 dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' 276 277 # We have two sets of folders so that we collect *all* stubs folders and 278 # put them in the front of the search path 279 third_party_inline_dirs = [] # type: PackageDirs 280 third_party_stubs_dirs = [] # type: PackageDirs 281 found_possible_third_party_missing_type_hints = False 282 need_installed_stubs = False 283 # Third-party stub/typed packages 284 for pkg_dir in self.search_paths.package_path: 285 stub_name = components[0] + '-stubs' 286 stub_dir = os.path.join(pkg_dir, stub_name) 287 if self.python_major_ver == 2: 288 alt_stub_name = components[0] + '-python2-stubs' 289 alt_stub_dir = os.path.join(pkg_dir, alt_stub_name) 290 if fscache.isdir(alt_stub_dir): 291 stub_name = alt_stub_name 292 stub_dir = alt_stub_dir 293 if fscache.isdir(stub_dir) and self._is_compatible_stub_package(stub_dir): 294 stub_typed_file = os.path.join(stub_dir, 'py.typed') 295 stub_components = [stub_name] + components[1:] 296 path = os.path.join(pkg_dir, *stub_components[:-1]) 297 if fscache.isdir(path): 298 if fscache.isfile(stub_typed_file): 299 # Stub packages can have a py.typed file, which must include 300 # 'partial\n' to make the package partial 301 # Partial here means that mypy should look at the runtime 302 # package if installed. 303 if fscache.read(stub_typed_file).decode().strip() == 'partial': 304 runtime_path = os.path.join(pkg_dir, dir_chain) 305 third_party_inline_dirs.append((runtime_path, True)) 306 # if the package is partial, we don't verify the module, as 307 # the partial stub package may not have a __init__.pyi 308 third_party_stubs_dirs.append((path, False)) 309 else: 310 # handle the edge case where people put a py.typed file 311 # in a stub package, but it isn't partial 312 third_party_stubs_dirs.append((path, True)) 313 else: 314 third_party_stubs_dirs.append((path, True)) 315 non_stub_match = self._find_module_non_stub_helper(components, pkg_dir) 316 if isinstance(non_stub_match, ModuleNotFoundReason): 317 if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: 318 found_possible_third_party_missing_type_hints = True 319 elif non_stub_match is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: 320 need_installed_stubs = True 321 else: 322 third_party_inline_dirs.append(non_stub_match) 323 self._update_ns_ancestors(components, non_stub_match) 324 if self.options and self.options.use_builtins_fixtures: 325 # Everything should be in fixtures. 326 third_party_inline_dirs.clear() 327 third_party_stubs_dirs.clear() 328 found_possible_third_party_missing_type_hints = False 329 python_mypy_path = self.search_paths.mypy_path + self.search_paths.python_path 330 candidate_base_dirs = self.find_lib_path_dirs(id, python_mypy_path) 331 if use_typeshed: 332 # Search for stdlib stubs in typeshed before installed 333 # stubs to avoid picking up backports (dataclasses, for 334 # example) when the library is included in stdlib. 335 candidate_base_dirs += self.find_lib_path_dirs(id, self.search_paths.typeshed_path) 336 candidate_base_dirs += third_party_stubs_dirs + third_party_inline_dirs 337 338 # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now 339 # contains just the subdirectories 'foo/bar' that actually exist under the 340 # elements of lib_path. This is probably much shorter than lib_path itself. 341 # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. 342 seplast = os.sep + components[-1] # so e.g. '/baz' 343 sepinit = os.sep + '__init__' 344 near_misses = [] # Collect near misses for namespace mode (see below). 345 for base_dir, verify in candidate_base_dirs: 346 base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' 347 has_init = False 348 dir_prefix = base_dir 349 for _ in range(len(components) - 1): 350 dir_prefix = os.path.dirname(dir_prefix) 351 # Prefer package over module, i.e. baz/__init__.py* over baz.py*. 352 for extension in PYTHON_EXTENSIONS: 353 path = base_path + sepinit + extension 354 suffix = '-stubs' 355 if self.python_major_ver == 2: 356 if os.path.isdir(base_path + '-python2-stubs'): 357 suffix = '-python2-stubs' 358 path_stubs = base_path + suffix + sepinit + extension 359 if fscache.isfile_case(path, dir_prefix): 360 has_init = True 361 if verify and not verify_module(fscache, id, path, dir_prefix): 362 near_misses.append((path, dir_prefix)) 363 continue 364 return path 365 elif fscache.isfile_case(path_stubs, dir_prefix): 366 if verify and not verify_module(fscache, id, path_stubs, dir_prefix): 367 near_misses.append((path_stubs, dir_prefix)) 368 continue 369 return path_stubs 370 371 # In namespace mode, register a potential namespace package 372 if self.options and self.options.namespace_packages: 373 if fscache.isdir(base_path) and not has_init: 374 near_misses.append((base_path, dir_prefix)) 375 376 # No package, look for module. 377 for extension in PYTHON_EXTENSIONS: 378 path = base_path + extension 379 if fscache.isfile_case(path, dir_prefix): 380 if verify and not verify_module(fscache, id, path, dir_prefix): 381 near_misses.append((path, dir_prefix)) 382 continue 383 return path 384 385 # In namespace mode, re-check those entries that had 'verify'. 386 # Assume search path entries xxx, yyy and zzz, and we're 387 # looking for foo.bar.baz. Suppose near_misses has: 388 # 389 # - xxx/foo/bar/baz.py 390 # - yyy/foo/bar/baz/__init__.py 391 # - zzz/foo/bar/baz.pyi 392 # 393 # If any of the foo directories has __init__.py[i], it wins. 394 # Else, we look for foo/bar/__init__.py[i], etc. If there are 395 # none, the first hit wins. Note that this does not take into 396 # account whether the lowest-level module is a file (baz.py), 397 # a package (baz/__init__.py), or a stub file (baz.pyi) -- for 398 # these the first one encountered along the search path wins. 399 # 400 # The helper function highest_init_level() returns an int that 401 # indicates the highest level at which a __init__.py[i] file 402 # is found; if no __init__ was found it returns 0, if we find 403 # only foo/bar/__init__.py it returns 1, and if we have 404 # foo/__init__.py it returns 2 (regardless of what's in 405 # foo/bar). It doesn't look higher than that. 406 if self.options and self.options.namespace_packages and near_misses: 407 levels = [highest_init_level(fscache, id, path, dir_prefix) 408 for path, dir_prefix in near_misses] 409 index = levels.index(max(levels)) 410 return near_misses[index][0] 411 412 # Finally, we may be asked to produce an ancestor for an 413 # installed package with a py.typed marker that is a 414 # subpackage of a namespace package. We only fess up to these 415 # if we would otherwise return "not found". 416 ancestor = self.ns_ancestors.get(id) 417 if ancestor is not None: 418 return ancestor 419 420 if need_installed_stubs: 421 return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED 422 elif found_possible_third_party_missing_type_hints: 423 return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS 424 else: 425 return ModuleNotFoundReason.NOT_FOUND 426 427 def _is_compatible_stub_package(self, stub_dir: str) -> bool: 428 """Does a stub package support the target Python version? 429 430 Stub packages may contain a metadata file which specifies 431 whether the stubs are compatible with Python 2 and 3. 432 """ 433 metadata_fnam = os.path.join(stub_dir, 'METADATA.toml') 434 if os.path.isfile(metadata_fnam): 435 # Delay import for a possible minor performance win. 436 import toml 437 with open(metadata_fnam, 'r') as f: 438 metadata = toml.load(f) 439 if self.python_major_ver == 2: 440 return bool(metadata.get('python2', False)) 441 else: 442 return bool(metadata.get('python3', True)) 443 return True 444 445 def find_modules_recursive(self, module: str) -> List[BuildSource]: 446 module_path = self.find_module(module) 447 if isinstance(module_path, ModuleNotFoundReason): 448 return [] 449 sources = [BuildSource(module_path, module, None)] 450 451 package_path = None 452 if module_path.endswith(('__init__.py', '__init__.pyi')): 453 package_path = os.path.dirname(module_path) 454 elif self.fscache.isdir(module_path): 455 package_path = module_path 456 if package_path is None: 457 return sources 458 459 # This logic closely mirrors that in find_sources. One small but important difference is 460 # that we do not sort names with keyfunc. The recursive call to find_modules_recursive 461 # calls find_module, which will handle the preference between packages, pyi and py. 462 # Another difference is it doesn't handle nested search paths / package roots. 463 464 seen = set() # type: Set[str] 465 names = sorted(self.fscache.listdir(package_path)) 466 for name in names: 467 # Skip certain names altogether 468 if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): 469 continue 470 subpath = os.path.join(package_path, name) 471 472 if self.options and matches_exclude( 473 subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2 474 ): 475 continue 476 477 if self.fscache.isdir(subpath): 478 # Only recurse into packages 479 if (self.options and self.options.namespace_packages) or ( 480 self.fscache.isfile(os.path.join(subpath, "__init__.py")) 481 or self.fscache.isfile(os.path.join(subpath, "__init__.pyi")) 482 ): 483 seen.add(name) 484 sources.extend(self.find_modules_recursive(module + '.' + name)) 485 else: 486 stem, suffix = os.path.splitext(name) 487 if stem == '__init__': 488 continue 489 if stem not in seen and '.' not in stem and suffix in PYTHON_EXTENSIONS: 490 # (If we sorted names by keyfunc) we could probably just make the BuildSource 491 # ourselves, but this ensures compatibility with find_module / the cache 492 seen.add(stem) 493 sources.extend(self.find_modules_recursive(module + '.' + stem)) 494 return sources 495 496 497def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbose: bool) -> bool: 498 if not exclude: 499 return False 500 subpath_str = os.path.relpath(subpath).replace(os.sep, "/") 501 if fscache.isdir(subpath): 502 subpath_str += "/" 503 if re.search(exclude, subpath_str): 504 if verbose: 505 print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) 506 return True 507 return False 508 509 510def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool: 511 """Check that all packages containing id have a __init__ file.""" 512 if path.endswith(('__init__.py', '__init__.pyi')): 513 path = os.path.dirname(path) 514 for i in range(id.count('.')): 515 path = os.path.dirname(path) 516 if not any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension)), 517 prefix) 518 for extension in PYTHON_EXTENSIONS): 519 return False 520 return True 521 522 523def highest_init_level(fscache: FileSystemCache, id: str, path: str, prefix: str) -> int: 524 """Compute the highest level where an __init__ file is found.""" 525 if path.endswith(('__init__.py', '__init__.pyi')): 526 path = os.path.dirname(path) 527 level = 0 528 for i in range(id.count('.')): 529 path = os.path.dirname(path) 530 if any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension)), 531 prefix) 532 for extension in PYTHON_EXTENSIONS): 533 level = i + 1 534 return level 535 536 537def mypy_path() -> List[str]: 538 path_env = os.getenv('MYPYPATH') 539 if not path_env: 540 return [] 541 return path_env.split(os.pathsep) 542 543 544def default_lib_path(data_dir: str, 545 pyversion: Tuple[int, int], 546 custom_typeshed_dir: Optional[str]) -> List[str]: 547 """Return default standard library search paths.""" 548 path = [] # type: List[str] 549 550 if custom_typeshed_dir: 551 typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib") 552 mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions") 553 versions_file = os.path.join(typeshed_dir, "VERSIONS") 554 if not os.path.isdir(typeshed_dir) or not os.path.isfile(versions_file): 555 print("error: --custom-typeshed-dir does not point to a valid typeshed ({})".format( 556 custom_typeshed_dir)) 557 sys.exit(2) 558 else: 559 auto = os.path.join(data_dir, 'stubs-auto') 560 if os.path.isdir(auto): 561 data_dir = auto 562 typeshed_dir = os.path.join(data_dir, "typeshed", "stdlib") 563 mypy_extensions_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-extensions") 564 if pyversion[0] == 2: 565 # Python 2 variants of certain stdlib modules are in a separate directory. 566 python2_dir = os.path.join(typeshed_dir, PYTHON2_STUB_DIR) 567 path.append(python2_dir) 568 path.append(typeshed_dir) 569 570 # Get mypy-extensions stubs from typeshed, since we treat it as an 571 # "internal" library, similar to typing and typing-extensions. 572 path.append(mypy_extensions_dir) 573 574 # Add fallback path that can be used if we have a broken installation. 575 if sys.platform != 'win32': 576 path.append('/usr/local/lib/mypy') 577 if not path: 578 print("Could not resolve typeshed subdirectories. Your mypy install is broken.\n" 579 "Python executable is located at {0}.\nMypy located at {1}".format( 580 sys.executable, data_dir), file=sys.stderr) 581 sys.exit(1) 582 return path 583 584 585@functools.lru_cache(maxsize=None) 586def get_site_packages_dirs(python_executable: Optional[str]) -> Tuple[List[str], List[str]]: 587 """Find package directories for given python. 588 589 This runs a subprocess call, which generates a list of the egg directories, and the site 590 package directories. To avoid repeatedly calling a subprocess (which can be slow!) we 591 lru_cache the results. 592 """ 593 594 if python_executable is None: 595 return [], [] 596 elif python_executable == sys.executable: 597 # Use running Python's package dirs 598 site_packages = sitepkgs.getsitepackages() 599 else: 600 # Use subprocess to get the package directory of given Python 601 # executable 602 site_packages = ast.literal_eval( 603 subprocess.check_output([python_executable, sitepkgs.__file__], 604 stderr=subprocess.PIPE).decode()) 605 return expand_site_packages(site_packages) 606 607 608def expand_site_packages(site_packages: List[str]) -> Tuple[List[str], List[str]]: 609 """Expands .pth imports in site-packages directories""" 610 egg_dirs = [] # type: List[str] 611 for dir in site_packages: 612 if not os.path.isdir(dir): 613 continue 614 pth_filenames = sorted(name for name in os.listdir(dir) if name.endswith(".pth")) 615 for pth_filename in pth_filenames: 616 egg_dirs.extend(_parse_pth_file(dir, pth_filename)) 617 618 return egg_dirs, site_packages 619 620 621def _parse_pth_file(dir: str, pth_filename: str) -> Iterator[str]: 622 """ 623 Mimics a subset of .pth import hook from Lib/site.py 624 See https://github.com/python/cpython/blob/3.5/Lib/site.py#L146-L185 625 """ 626 627 pth_file = os.path.join(dir, pth_filename) 628 try: 629 f = open(pth_file, "r") 630 except OSError: 631 return 632 with f: 633 for line in f.readlines(): 634 if line.startswith("#"): 635 # Skip comment lines 636 continue 637 if line.startswith(("import ", "import\t")): 638 # import statements in .pth files are not supported 639 continue 640 641 yield _make_abspath(line.rstrip(), dir) 642 643 644def _make_abspath(path: str, root: str) -> str: 645 """Take a path and make it absolute relative to root if not already absolute.""" 646 if os.path.isabs(path): 647 return os.path.normpath(path) 648 else: 649 return os.path.join(root, os.path.normpath(path)) 650 651 652def add_py2_mypypath_entries(mypypath: List[str]) -> List[str]: 653 """Add corresponding @python2 subdirectories to mypypath. 654 655 For each path entry 'x', add 'x/@python2' before 'x' if the latter is 656 a directory. 657 """ 658 result = [] 659 for item in mypypath: 660 python2_dir = os.path.join(item, PYTHON2_STUB_DIR) 661 if os.path.isdir(python2_dir): 662 # @python2 takes precedence, but we also look into the parent 663 # directory. 664 result.append(python2_dir) 665 result.append(item) 666 else: 667 result.append(item) 668 return result 669 670 671def compute_search_paths(sources: List[BuildSource], 672 options: Options, 673 data_dir: str, 674 alt_lib_path: Optional[str] = None) -> SearchPaths: 675 """Compute the search paths as specified in PEP 561. 676 677 There are the following 4 members created: 678 - User code (from `sources`) 679 - MYPYPATH (set either via config or environment variable) 680 - installed package directories (which will later be split into stub-only and inline) 681 - typeshed 682 """ 683 # Determine the default module search path. 684 lib_path = collections.deque( 685 default_lib_path(data_dir, 686 options.python_version, 687 custom_typeshed_dir=options.custom_typeshed_dir)) 688 689 if options.use_builtins_fixtures: 690 # Use stub builtins (to speed up test cases and to make them easier to 691 # debug). This is a test-only feature, so assume our files are laid out 692 # as in the source tree. 693 # We also need to allow overriding where to look for it. Argh. 694 root_dir = os.getenv('MYPY_TEST_PREFIX', None) 695 if not root_dir: 696 root_dir = os.path.dirname(os.path.dirname(__file__)) 697 lib_path.appendleft(os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) 698 # alt_lib_path is used by some tests to bypass the normal lib_path mechanics. 699 # If we don't have one, grab directories of source files. 700 python_path = [] # type: List[str] 701 if not alt_lib_path: 702 for source in sources: 703 # Include directory of the program file in the module search path. 704 if source.base_dir: 705 dir = source.base_dir 706 if dir not in python_path: 707 python_path.append(dir) 708 709 # Do this even if running as a file, for sanity (mainly because with 710 # multiple builds, there could be a mix of files/modules, so its easier 711 # to just define the semantics that we always add the current director 712 # to the lib_path 713 # TODO: Don't do this in some cases; for motivation see see 714 # https://github.com/python/mypy/issues/4195#issuecomment-341915031 715 if options.bazel: 716 dir = '.' 717 else: 718 dir = os.getcwd() 719 if dir not in lib_path: 720 python_path.insert(0, dir) 721 722 # Start with a MYPYPATH environment variable at the front of the mypy_path, if defined. 723 mypypath = mypy_path() 724 725 # Add a config-defined mypy path. 726 mypypath.extend(options.mypy_path) 727 728 # If provided, insert the caller-supplied extra module path to the 729 # beginning (highest priority) of the search path. 730 if alt_lib_path: 731 mypypath.insert(0, alt_lib_path) 732 733 # When type checking in Python 2 module, add @python2 subdirectories of 734 # path items into the search path. 735 if options.python_version[0] == 2: 736 mypypath = add_py2_mypypath_entries(mypypath) 737 738 egg_dirs, site_packages = get_site_packages_dirs(options.python_executable) 739 for site_dir in site_packages: 740 assert site_dir not in lib_path 741 if (site_dir in mypypath or 742 any(p.startswith(site_dir + os.path.sep) for p in mypypath) or 743 os.path.altsep and any(p.startswith(site_dir + os.path.altsep) for p in mypypath)): 744 print("{} is in the MYPYPATH. Please remove it.".format(site_dir), file=sys.stderr) 745 print("See https://mypy.readthedocs.io/en/stable/running_mypy.html" 746 "#how-mypy-handles-imports for more info", file=sys.stderr) 747 sys.exit(1) 748 elif site_dir in python_path: 749 print("{} is in the PYTHONPATH. Please change directory" 750 " so it is not.".format(site_dir), 751 file=sys.stderr) 752 sys.exit(1) 753 754 return SearchPaths(python_path=tuple(reversed(python_path)), 755 mypy_path=tuple(mypypath), 756 package_path=tuple(egg_dirs + site_packages), 757 typeshed_path=tuple(lib_path)) 758 759 760def load_stdlib_py_versions(custom_typeshed_dir: Optional[str] 761 ) -> Dict[str, Tuple[Tuple[int, int], Optional[Tuple[int, int]]]]: 762 """Return dict with minimum and maximum Python versions of stdlib modules. 763 764 The contents look like 765 {..., 'secrets': ((3, 6), None), 'symbol': ((2, 7), (3, 9)), ...} 766 767 None means there is no maximum version. 768 """ 769 typeshed_dir = custom_typeshed_dir or os.path.join(os.path.dirname(__file__), "typeshed") 770 stdlib_dir = os.path.join(typeshed_dir, "stdlib") 771 result = {} 772 773 versions_path = os.path.join(stdlib_dir, "VERSIONS") 774 assert os.path.isfile(versions_path), (custom_typeshed_dir, versions_path, __file__) 775 with open(versions_path) as f: 776 for line in f: 777 line = line.split("#")[0].strip() 778 if line == "": 779 continue 780 module, version_range = line.split(":") 781 versions = version_range.split("-") 782 min_version = parse_version(versions[0]) 783 max_version = (parse_version(versions[1]) 784 if len(versions) >= 2 and versions[1].strip() else None) 785 result[module] = min_version, max_version 786 787 # Modules that are Python 2 only or have separate Python 2 stubs 788 # have stubs in @python2/ and may need an override. 789 python2_dir = os.path.join(stdlib_dir, PYTHON2_STUB_DIR) 790 try: 791 for fnam in os.listdir(python2_dir): 792 fnam = fnam.replace(".pyi", "") 793 max_version = result.get(fnam, ((2, 7), None))[1] 794 result[fnam] = (2, 7), max_version 795 except FileNotFoundError: 796 # Ignore error to support installations where Python 2 stubs aren't available. 797 pass 798 799 return result 800 801 802def parse_version(version: str) -> Tuple[int, int]: 803 major, minor = version.strip().split(".") 804 return int(major), int(minor) 805 806 807def typeshed_py_version(options: Options) -> Tuple[int, int]: 808 """Return Python version used for checking whether module supports typeshed.""" 809 # Typeshed no longer covers Python 3.x versions before 3.6, so 3.6 is 810 # the earliest we can support. 811 if options.python_version[0] >= 3: 812 return max(options.python_version, (3, 6)) 813 else: 814 return options.python_version 815 816 817def filter_redundant_py2_dirs(dirs: List[str]) -> List[str]: 818 """If dirs has <dir>/@python2 followed by <dir>, filter out the latter.""" 819 if len(dirs) <= 1 or not any(d.endswith(PYTHON2_STUB_DIR) for d in dirs): 820 # Fast path -- nothing to do 821 return dirs 822 seen = [] 823 result = [] 824 for d in dirs: 825 if d.endswith(PYTHON2_STUB_DIR): 826 seen.append(os.path.dirname(d)) 827 result.append(d) 828 elif d not in seen: 829 result.append(d) 830 return result 831