1"""Core implementation of path-based import. 2 3This module is NOT meant to be directly imported! It has been designed such 4that it can be bootstrapped into Python as the implementation of import. As 5such it requires the injection of specific modules and attributes in order to 6work. One should use importlib as the public-facing version of this module. 7 8""" 9# IMPORTANT: Whenever making changes to this module, be sure to run a top-level 10# `make regen-importlib` followed by `make` in order to get the frozen version 11# of the module updated. Not doing so will result in the Makefile to fail for 12# all others who don't have a ./python around to freeze the module in the early 13# stages of compilation. 14# 15 16# See importlib._setup() for what is injected into the global namespace. 17 18# When editing this code be aware that code executed at import time CANNOT 19# reference any injected objects! This includes not only global code but also 20# anything specified at the class level. 21 22# Import builtin modules 23import _imp 24import _io 25import sys 26import _warnings 27import marshal 28 29 30_MS_WINDOWS = (sys.platform == 'win32') 31if _MS_WINDOWS: 32 import nt as _os 33 import winreg 34else: 35 import posix as _os 36 37 38if _MS_WINDOWS: 39 path_separators = ['\\', '/'] 40else: 41 path_separators = ['/'] 42# Assumption made in _path_join() 43assert all(len(sep) == 1 for sep in path_separators) 44path_sep = path_separators[0] 45path_sep_tuple = tuple(path_separators) 46path_separators = ''.join(path_separators) 47_pathseps_with_colon = {f':{s}' for s in path_separators} 48 49 50# Bootstrap-related code ###################################################### 51_CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', 52_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' 53_CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY 54 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) 55 56 57def _make_relax_case(): 58 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): 59 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY): 60 key = 'PYTHONCASEOK' 61 else: 62 key = b'PYTHONCASEOK' 63 64 def _relax_case(): 65 """True if filenames must be checked case-insensitively.""" 66 return key in _os.environ 67 else: 68 def _relax_case(): 69 """True if filenames must be checked case-insensitively.""" 70 return False 71 return _relax_case 72 73 74def _pack_uint32(x): 75 """Convert a 32-bit integer to little-endian.""" 76 return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') 77 78 79def _unpack_uint32(data): 80 """Convert 4 bytes in little-endian to an integer.""" 81 assert len(data) == 4 82 return int.from_bytes(data, 'little') 83 84def _unpack_uint16(data): 85 """Convert 2 bytes in little-endian to an integer.""" 86 assert len(data) == 2 87 return int.from_bytes(data, 'little') 88 89 90if _MS_WINDOWS: 91 def _path_join(*path_parts): 92 """Replacement for os.path.join().""" 93 if not path_parts: 94 return "" 95 if len(path_parts) == 1: 96 return path_parts[0] 97 root = "" 98 path = [] 99 for new_root, tail in map(_os._path_splitroot, path_parts): 100 if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple): 101 root = new_root.rstrip(path_separators) or root 102 path = [path_sep + tail] 103 elif new_root.endswith(':'): 104 if root.casefold() != new_root.casefold(): 105 # Drive relative paths have to be resolved by the OS, so we reset the 106 # tail but do not add a path_sep prefix. 107 root = new_root 108 path = [tail] 109 else: 110 path.append(tail) 111 else: 112 root = new_root or root 113 path.append(tail) 114 path = [p.rstrip(path_separators) for p in path if p] 115 if len(path) == 1 and not path[0]: 116 # Avoid losing the root's trailing separator when joining with nothing 117 return root + path_sep 118 return root + path_sep.join(path) 119 120else: 121 def _path_join(*path_parts): 122 """Replacement for os.path.join().""" 123 return path_sep.join([part.rstrip(path_separators) 124 for part in path_parts if part]) 125 126 127def _path_split(path): 128 """Replacement for os.path.split().""" 129 i = max(path.rfind(p) for p in path_separators) 130 if i < 0: 131 return '', path 132 return path[:i], path[i + 1:] 133 134 135def _path_stat(path): 136 """Stat the path. 137 138 Made a separate function to make it easier to override in experiments 139 (e.g. cache stat results). 140 141 """ 142 return _os.stat(path) 143 144 145def _path_is_mode_type(path, mode): 146 """Test whether the path is the specified mode type.""" 147 try: 148 stat_info = _path_stat(path) 149 except OSError: 150 return False 151 return (stat_info.st_mode & 0o170000) == mode 152 153 154def _path_isfile(path): 155 """Replacement for os.path.isfile.""" 156 return _path_is_mode_type(path, 0o100000) 157 158 159def _path_isdir(path): 160 """Replacement for os.path.isdir.""" 161 if not path: 162 path = _os.getcwd() 163 return _path_is_mode_type(path, 0o040000) 164 165 166if _MS_WINDOWS: 167 def _path_isabs(path): 168 """Replacement for os.path.isabs.""" 169 if not path: 170 return False 171 root = _os._path_splitroot(path)[0].replace('/', '\\') 172 return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\')) 173 174else: 175 def _path_isabs(path): 176 """Replacement for os.path.isabs.""" 177 return path.startswith(path_separators) 178 179 180def _write_atomic(path, data, mode=0o666): 181 """Best-effort function to write data to a path atomically. 182 Be prepared to handle a FileExistsError if concurrent writing of the 183 temporary file is attempted.""" 184 # id() is used to generate a pseudo-random filename. 185 path_tmp = '{}.{}'.format(path, id(path)) 186 fd = _os.open(path_tmp, 187 _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666) 188 try: 189 # We first write data to a temporary file, and then use os.replace() to 190 # perform an atomic rename. 191 with _io.FileIO(fd, 'wb') as file: 192 file.write(data) 193 _os.replace(path_tmp, path) 194 except OSError: 195 try: 196 _os.unlink(path_tmp) 197 except OSError: 198 pass 199 raise 200 201 202_code_type = type(_write_atomic.__code__) 203 204 205# Finder/loader utility code ############################################### 206 207# Magic word to reject .pyc files generated by other Python versions. 208# It should change for each incompatible change to the bytecode. 209# 210# The value of CR and LF is incorporated so if you ever read or write 211# a .pyc file in text mode the magic number will be wrong; also, the 212# Apple MPW compiler swaps their values, botching string constants. 213# 214# There were a variety of old schemes for setting the magic number. 215# The current working scheme is to increment the previous value by 216# 10. 217# 218# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic 219# number also includes a new "magic tag", i.e. a human readable string used 220# to represent the magic number in __pycache__ directories. When you change 221# the magic number, you must also set a new unique magic tag. Generally this 222# can be named after the Python major version of the magic number bump, but 223# it can really be anything, as long as it's different than anything else 224# that's come before. The tags are included in the following table, starting 225# with Python 3.2a0. 226# 227# Known values: 228# Python 1.5: 20121 229# Python 1.5.1: 20121 230# Python 1.5.2: 20121 231# Python 1.6: 50428 232# Python 2.0: 50823 233# Python 2.0.1: 50823 234# Python 2.1: 60202 235# Python 2.1.1: 60202 236# Python 2.1.2: 60202 237# Python 2.2: 60717 238# Python 2.3a0: 62011 239# Python 2.3a0: 62021 240# Python 2.3a0: 62011 (!) 241# Python 2.4a0: 62041 242# Python 2.4a3: 62051 243# Python 2.4b1: 62061 244# Python 2.5a0: 62071 245# Python 2.5a0: 62081 (ast-branch) 246# Python 2.5a0: 62091 (with) 247# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) 248# Python 2.5b3: 62101 (fix wrong code: for x, in ...) 249# Python 2.5b3: 62111 (fix wrong code: x += yield) 250# Python 2.5c1: 62121 (fix wrong lnotab with for loops and 251# storing constants that should have been removed) 252# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) 253# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) 254# Python 2.6a1: 62161 (WITH_CLEANUP optimization) 255# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) 256# Python 2.7a0: 62181 (optimize conditional branches: 257# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) 258# Python 2.7a0 62191 (introduce SETUP_WITH) 259# Python 2.7a0 62201 (introduce BUILD_SET) 260# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) 261# Python 3000: 3000 262# 3010 (removed UNARY_CONVERT) 263# 3020 (added BUILD_SET) 264# 3030 (added keyword-only parameters) 265# 3040 (added signature annotations) 266# 3050 (print becomes a function) 267# 3060 (PEP 3115 metaclass syntax) 268# 3061 (string literals become unicode) 269# 3071 (PEP 3109 raise changes) 270# 3081 (PEP 3137 make __file__ and __name__ unicode) 271# 3091 (kill str8 interning) 272# 3101 (merge from 2.6a0, see 62151) 273# 3103 (__file__ points to source file) 274# Python 3.0a4: 3111 (WITH_CLEANUP optimization). 275# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT 276 #3021) 277# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: 278# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) 279# Python 3.1a1: 3151 (optimize conditional branches: 280# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE 281 #4715) 282# Python 3.2a1: 3160 (add SETUP_WITH #6101) 283# tag: cpython-32 284# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) 285# tag: cpython-32 286# Python 3.2a3 3180 (add DELETE_DEREF #4617) 287# Python 3.3a1 3190 (__class__ super closure changed) 288# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) 289# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) 290# Python 3.3a2 3220 (changed PEP 380 implementation #14230) 291# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) 292# Python 3.4a1 3250 (evaluate positional default arguments before 293# keyword-only defaults #16967) 294# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override 295# free vars #17853) 296# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) 297# Python 3.4a1 3280 (remove implicit class argument) 298# Python 3.4a4 3290 (changes to __qualname__ computation #19301) 299# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) 300# Python 3.4rc2 3310 (alter __qualname__ computation #20625) 301# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) 302# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) 303# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) 304# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) 305# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) 306# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) 307# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) 308# Python 3.6a2 3370 (16 bit wordcode #26647) 309# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) 310# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE 311# #27095) 312# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) 313# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes 314# #27985) 315# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL 316 #27213) 317# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) 318# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) 319# Python 3.6rc1 3379 (more thorough __class__ validation #23722) 320# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) 321# Python 3.7a2 3391 (update GET_AITER #31709) 322# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) 323# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) 324# Python 3.7b5 3394 (restored docstring as the first stmt in the body; 325# this might affected the first line number #32911) 326# Python 3.8a1 3400 (move frame block handling to compiler #17611) 327# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) 328# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) 329# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict 330# comprehensions #35224) 331# Python 3.8b2 3412 (Swap the position of positional args and positional 332# only args in ast.arguments #37593) 333# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) 334# 335# MAGIC must change whenever the bytecode emitted by the compiler may no 336# longer be understood by older implementations of the eval loop (usually 337# due to the addition of new opcodes). 338# 339# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array 340# in PC/launcher.c must also be updated. 341 342MAGIC_NUMBER = (3413).to_bytes(2, 'little') + b'\r\n' 343_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c 344 345_PYCACHE = '__pycache__' 346_OPT = 'opt-' 347 348SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. 349 350BYTECODE_SUFFIXES = ['.pyc'] 351# Deprecated. 352DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES 353 354def cache_from_source(path, debug_override=None, *, optimization=None): 355 """Given the path to a .py file, return the path to its .pyc file. 356 357 The .py file does not need to exist; this simply returns the path to the 358 .pyc file calculated as if the .py file were imported. 359 360 The 'optimization' parameter controls the presumed optimization level of 361 the bytecode file. If 'optimization' is not None, the string representation 362 of the argument is taken and verified to be alphanumeric (else ValueError 363 is raised). 364 365 The debug_override parameter is deprecated. If debug_override is not None, 366 a True value is the same as setting 'optimization' to the empty string 367 while a False value is equivalent to setting 'optimization' to '1'. 368 369 If sys.implementation.cache_tag is None then NotImplementedError is raised. 370 371 """ 372 if debug_override is not None: 373 _warnings.warn('the debug_override parameter is deprecated; use ' 374 "'optimization' instead", DeprecationWarning) 375 if optimization is not None: 376 message = 'debug_override or optimization must be set to None' 377 raise TypeError(message) 378 optimization = '' if debug_override else 1 379 path = _os.fspath(path) 380 head, tail = _path_split(path) 381 base, sep, rest = tail.rpartition('.') 382 tag = sys.implementation.cache_tag 383 if tag is None: 384 raise NotImplementedError('sys.implementation.cache_tag is None') 385 almost_filename = ''.join([(base if base else rest), sep, tag]) 386 if optimization is None: 387 if sys.flags.optimize == 0: 388 optimization = '' 389 else: 390 optimization = sys.flags.optimize 391 optimization = str(optimization) 392 if optimization != '': 393 if not optimization.isalnum(): 394 raise ValueError('{!r} is not alphanumeric'.format(optimization)) 395 almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization) 396 filename = almost_filename + BYTECODE_SUFFIXES[0] 397 if sys.pycache_prefix is not None: 398 # We need an absolute path to the py file to avoid the possibility of 399 # collisions within sys.pycache_prefix, if someone has two different 400 # `foo/bar.py` on their system and they import both of them using the 401 # same sys.pycache_prefix. Let's say sys.pycache_prefix is 402 # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first 403 # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative 404 # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an 405 # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`. 406 if not _path_isabs(head): 407 head = _path_join(_os.getcwd(), head) 408 409 # Strip initial drive from a Windows path. We know we have an absolute 410 # path here, so the second part of the check rules out a POSIX path that 411 # happens to contain a colon at the second character. 412 if head[1] == ':' and head[0] not in path_separators: 413 head = head[2:] 414 415 # Strip initial path separator from `head` to complete the conversion 416 # back to a root-relative path before joining. 417 return _path_join( 418 sys.pycache_prefix, 419 head.lstrip(path_separators), 420 filename, 421 ) 422 return _path_join(head, _PYCACHE, filename) 423 424 425def source_from_cache(path): 426 """Given the path to a .pyc. file, return the path to its .py file. 427 428 The .pyc file does not need to exist; this simply returns the path to 429 the .py file calculated to correspond to the .pyc file. If path does 430 not conform to PEP 3147/488 format, ValueError will be raised. If 431 sys.implementation.cache_tag is None then NotImplementedError is raised. 432 433 """ 434 if sys.implementation.cache_tag is None: 435 raise NotImplementedError('sys.implementation.cache_tag is None') 436 path = _os.fspath(path) 437 head, pycache_filename = _path_split(path) 438 found_in_pycache_prefix = False 439 if sys.pycache_prefix is not None: 440 stripped_path = sys.pycache_prefix.rstrip(path_separators) 441 if head.startswith(stripped_path + path_sep): 442 head = head[len(stripped_path):] 443 found_in_pycache_prefix = True 444 if not found_in_pycache_prefix: 445 head, pycache = _path_split(head) 446 if pycache != _PYCACHE: 447 raise ValueError(f'{_PYCACHE} not bottom-level directory in ' 448 f'{path!r}') 449 dot_count = pycache_filename.count('.') 450 if dot_count not in {2, 3}: 451 raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}') 452 elif dot_count == 3: 453 optimization = pycache_filename.rsplit('.', 2)[-2] 454 if not optimization.startswith(_OPT): 455 raise ValueError("optimization portion of filename does not start " 456 f"with {_OPT!r}") 457 opt_level = optimization[len(_OPT):] 458 if not opt_level.isalnum(): 459 raise ValueError(f"optimization level {optimization!r} is not an " 460 "alphanumeric value") 461 base_filename = pycache_filename.partition('.')[0] 462 return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) 463 464 465def _get_sourcefile(bytecode_path): 466 """Convert a bytecode file path to a source path (if possible). 467 468 This function exists purely for backwards-compatibility for 469 PyImport_ExecCodeModuleWithFilenames() in the C API. 470 471 """ 472 if len(bytecode_path) == 0: 473 return None 474 rest, _, extension = bytecode_path.rpartition('.') 475 if not rest or extension.lower()[-3:-1] != 'py': 476 return bytecode_path 477 try: 478 source_path = source_from_cache(bytecode_path) 479 except (NotImplementedError, ValueError): 480 source_path = bytecode_path[:-1] 481 return source_path if _path_isfile(source_path) else bytecode_path 482 483 484def _get_cached(filename): 485 if filename.endswith(tuple(SOURCE_SUFFIXES)): 486 try: 487 return cache_from_source(filename) 488 except NotImplementedError: 489 pass 490 elif filename.endswith(tuple(BYTECODE_SUFFIXES)): 491 return filename 492 else: 493 return None 494 495 496def _calc_mode(path): 497 """Calculate the mode permissions for a bytecode file.""" 498 try: 499 mode = _path_stat(path).st_mode 500 except OSError: 501 mode = 0o666 502 # We always ensure write access so we can update cached files 503 # later even when the source files are read-only on Windows (#6074) 504 mode |= 0o200 505 return mode 506 507 508def _check_name(method): 509 """Decorator to verify that the module being requested matches the one the 510 loader can handle. 511 512 The first argument (self) must define _name which the second argument is 513 compared against. If the comparison fails then ImportError is raised. 514 515 """ 516 def _check_name_wrapper(self, name=None, *args, **kwargs): 517 if name is None: 518 name = self.name 519 elif self.name != name: 520 raise ImportError('loader for %s cannot handle %s' % 521 (self.name, name), name=name) 522 return method(self, name, *args, **kwargs) 523 try: 524 _wrap = _bootstrap._wrap 525 except NameError: 526 # XXX yuck 527 def _wrap(new, old): 528 for replace in ['__module__', '__name__', '__qualname__', '__doc__']: 529 if hasattr(old, replace): 530 setattr(new, replace, getattr(old, replace)) 531 new.__dict__.update(old.__dict__) 532 _wrap(_check_name_wrapper, method) 533 return _check_name_wrapper 534 535 536def _find_module_shim(self, fullname): 537 """Try to find a loader for the specified module by delegating to 538 self.find_loader(). 539 540 This method is deprecated in favor of finder.find_spec(). 541 542 """ 543 # Call find_loader(). If it returns a string (indicating this 544 # is a namespace package portion), generate a warning and 545 # return None. 546 loader, portions = self.find_loader(fullname) 547 if loader is None and len(portions): 548 msg = 'Not importing directory {}: missing __init__' 549 _warnings.warn(msg.format(portions[0]), ImportWarning) 550 return loader 551 552 553def _classify_pyc(data, name, exc_details): 554 """Perform basic validity checking of a pyc header and return the flags field, 555 which determines how the pyc should be further validated against the source. 556 557 *data* is the contents of the pyc file. (Only the first 16 bytes are 558 required, though.) 559 560 *name* is the name of the module being imported. It is used for logging. 561 562 *exc_details* is a dictionary passed to ImportError if it raised for 563 improved debugging. 564 565 ImportError is raised when the magic number is incorrect or when the flags 566 field is invalid. EOFError is raised when the data is found to be truncated. 567 568 """ 569 magic = data[:4] 570 if magic != MAGIC_NUMBER: 571 message = f'bad magic number in {name!r}: {magic!r}' 572 _bootstrap._verbose_message('{}', message) 573 raise ImportError(message, **exc_details) 574 if len(data) < 16: 575 message = f'reached EOF while reading pyc header of {name!r}' 576 _bootstrap._verbose_message('{}', message) 577 raise EOFError(message) 578 flags = _unpack_uint32(data[4:8]) 579 # Only the first two flags are defined. 580 if flags & ~0b11: 581 message = f'invalid flags {flags!r} in {name!r}' 582 raise ImportError(message, **exc_details) 583 return flags 584 585 586def _validate_timestamp_pyc(data, source_mtime, source_size, name, 587 exc_details): 588 """Validate a pyc against the source last-modified time. 589 590 *data* is the contents of the pyc file. (Only the first 16 bytes are 591 required.) 592 593 *source_mtime* is the last modified timestamp of the source file. 594 595 *source_size* is None or the size of the source file in bytes. 596 597 *name* is the name of the module being imported. It is used for logging. 598 599 *exc_details* is a dictionary passed to ImportError if it raised for 600 improved debugging. 601 602 An ImportError is raised if the bytecode is stale. 603 604 """ 605 if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF): 606 message = f'bytecode is stale for {name!r}' 607 _bootstrap._verbose_message('{}', message) 608 raise ImportError(message, **exc_details) 609 if (source_size is not None and 610 _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)): 611 raise ImportError(f'bytecode is stale for {name!r}', **exc_details) 612 613 614def _validate_hash_pyc(data, source_hash, name, exc_details): 615 """Validate a hash-based pyc by checking the real source hash against the one in 616 the pyc header. 617 618 *data* is the contents of the pyc file. (Only the first 16 bytes are 619 required.) 620 621 *source_hash* is the importlib.util.source_hash() of the source file. 622 623 *name* is the name of the module being imported. It is used for logging. 624 625 *exc_details* is a dictionary passed to ImportError if it raised for 626 improved debugging. 627 628 An ImportError is raised if the bytecode is stale. 629 630 """ 631 if data[8:16] != source_hash: 632 raise ImportError( 633 f'hash in bytecode doesn\'t match hash of source {name!r}', 634 **exc_details, 635 ) 636 637 638def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): 639 """Compile bytecode as found in a pyc.""" 640 code = marshal.loads(data) 641 if isinstance(code, _code_type): 642 _bootstrap._verbose_message('code object from {!r}', bytecode_path) 643 if source_path is not None: 644 _imp._fix_co_filename(code, source_path) 645 return code 646 else: 647 raise ImportError('Non-code object in {!r}'.format(bytecode_path), 648 name=name, path=bytecode_path) 649 650 651def _code_to_timestamp_pyc(code, mtime=0, source_size=0): 652 "Produce the data for a timestamp-based pyc." 653 data = bytearray(MAGIC_NUMBER) 654 data.extend(_pack_uint32(0)) 655 data.extend(_pack_uint32(mtime)) 656 data.extend(_pack_uint32(source_size)) 657 data.extend(marshal.dumps(code)) 658 return data 659 660 661def _code_to_hash_pyc(code, source_hash, checked=True): 662 "Produce the data for a hash-based pyc." 663 data = bytearray(MAGIC_NUMBER) 664 flags = 0b1 | checked << 1 665 data.extend(_pack_uint32(flags)) 666 assert len(source_hash) == 8 667 data.extend(source_hash) 668 data.extend(marshal.dumps(code)) 669 return data 670 671 672def decode_source(source_bytes): 673 """Decode bytes representing source code and return the string. 674 675 Universal newline support is used in the decoding. 676 """ 677 import tokenize # To avoid bootstrap issues. 678 source_bytes_readline = _io.BytesIO(source_bytes).readline 679 encoding = tokenize.detect_encoding(source_bytes_readline) 680 newline_decoder = _io.IncrementalNewlineDecoder(None, True) 681 return newline_decoder.decode(source_bytes.decode(encoding[0])) 682 683 684# Module specifications ####################################################### 685 686_POPULATE = object() 687 688 689def spec_from_file_location(name, location=None, *, loader=None, 690 submodule_search_locations=_POPULATE): 691 """Return a module spec based on a file location. 692 693 To indicate that the module is a package, set 694 submodule_search_locations to a list of directory paths. An 695 empty list is sufficient, though its not otherwise useful to the 696 import system. 697 698 The loader must take a spec as its only __init__() arg. 699 700 """ 701 if location is None: 702 # The caller may simply want a partially populated location- 703 # oriented spec. So we set the location to a bogus value and 704 # fill in as much as we can. 705 location = '<unknown>' 706 if hasattr(loader, 'get_filename'): 707 # ExecutionLoader 708 try: 709 location = loader.get_filename(name) 710 except ImportError: 711 pass 712 else: 713 location = _os.fspath(location) 714 715 # If the location is on the filesystem, but doesn't actually exist, 716 # we could return None here, indicating that the location is not 717 # valid. However, we don't have a good way of testing since an 718 # indirect location (e.g. a zip file or URL) will look like a 719 # non-existent file relative to the filesystem. 720 721 spec = _bootstrap.ModuleSpec(name, loader, origin=location) 722 spec._set_fileattr = True 723 724 # Pick a loader if one wasn't provided. 725 if loader is None: 726 for loader_class, suffixes in _get_supported_file_loaders(): 727 if location.endswith(tuple(suffixes)): 728 loader = loader_class(name, location) 729 spec.loader = loader 730 break 731 else: 732 return None 733 734 # Set submodule_search_paths appropriately. 735 if submodule_search_locations is _POPULATE: 736 # Check the loader. 737 if hasattr(loader, 'is_package'): 738 try: 739 is_package = loader.is_package(name) 740 except ImportError: 741 pass 742 else: 743 if is_package: 744 spec.submodule_search_locations = [] 745 else: 746 spec.submodule_search_locations = submodule_search_locations 747 if spec.submodule_search_locations == []: 748 if location: 749 dirname = _path_split(location)[0] 750 spec.submodule_search_locations.append(dirname) 751 752 return spec 753 754 755# Loaders ##################################################################### 756 757class WindowsRegistryFinder: 758 759 """Meta path finder for modules declared in the Windows registry.""" 760 761 REGISTRY_KEY = ( 762 'Software\\Python\\PythonCore\\{sys_version}' 763 '\\Modules\\{fullname}') 764 REGISTRY_KEY_DEBUG = ( 765 'Software\\Python\\PythonCore\\{sys_version}' 766 '\\Modules\\{fullname}\\Debug') 767 DEBUG_BUILD = False # Changed in _setup() 768 769 @classmethod 770 def _open_registry(cls, key): 771 try: 772 return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) 773 except OSError: 774 return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) 775 776 @classmethod 777 def _search_registry(cls, fullname): 778 if cls.DEBUG_BUILD: 779 registry_key = cls.REGISTRY_KEY_DEBUG 780 else: 781 registry_key = cls.REGISTRY_KEY 782 key = registry_key.format(fullname=fullname, 783 sys_version='%d.%d' % sys.version_info[:2]) 784 try: 785 with cls._open_registry(key) as hkey: 786 filepath = _winreg.QueryValue(hkey, '') 787 except OSError: 788 return None 789 return filepath 790 791 @classmethod 792 def find_spec(cls, fullname, path=None, target=None): 793 filepath = cls._search_registry(fullname) 794 if filepath is None: 795 return None 796 try: 797 _path_stat(filepath) 798 except OSError: 799 return None 800 for loader, suffixes in _get_supported_file_loaders(): 801 if filepath.endswith(tuple(suffixes)): 802 spec = _bootstrap.spec_from_loader(fullname, 803 loader(fullname, filepath), 804 origin=filepath) 805 return spec 806 807 @classmethod 808 def find_module(cls, fullname, path=None): 809 """Find module named in the registry. 810 811 This method is deprecated. Use exec_module() instead. 812 813 """ 814 spec = cls.find_spec(fullname, path) 815 if spec is not None: 816 return spec.loader 817 else: 818 return None 819 820 821class _LoaderBasics: 822 823 """Base class of common code needed by both SourceLoader and 824 SourcelessFileLoader.""" 825 826 def is_package(self, fullname): 827 """Concrete implementation of InspectLoader.is_package by checking if 828 the path returned by get_filename has a filename of '__init__.py'.""" 829 filename = _path_split(self.get_filename(fullname))[1] 830 filename_base = filename.rsplit('.', 1)[0] 831 tail_name = fullname.rpartition('.')[2] 832 return filename_base == '__init__' and tail_name != '__init__' 833 834 def create_module(self, spec): 835 """Use default semantics for module creation.""" 836 837 def exec_module(self, module): 838 """Execute the module.""" 839 code = self.get_code(module.__name__) 840 if code is None: 841 raise ImportError('cannot load module {!r} when get_code() ' 842 'returns None'.format(module.__name__)) 843 _bootstrap._call_with_frames_removed(exec, code, module.__dict__) 844 845 def load_module(self, fullname): 846 """This module is deprecated.""" 847 return _bootstrap._load_module_shim(self, fullname) 848 849 850class SourceLoader(_LoaderBasics): 851 852 def path_mtime(self, path): 853 """Optional method that returns the modification time (an int) for the 854 specified path (a str). 855 856 Raises OSError when the path cannot be handled. 857 """ 858 raise OSError 859 860 def path_stats(self, path): 861 """Optional method returning a metadata dict for the specified 862 path (a str). 863 864 Possible keys: 865 - 'mtime' (mandatory) is the numeric timestamp of last source 866 code modification; 867 - 'size' (optional) is the size in bytes of the source code. 868 869 Implementing this method allows the loader to read bytecode files. 870 Raises OSError when the path cannot be handled. 871 """ 872 return {'mtime': self.path_mtime(path)} 873 874 def _cache_bytecode(self, source_path, cache_path, data): 875 """Optional method which writes data (bytes) to a file path (a str). 876 877 Implementing this method allows for the writing of bytecode files. 878 879 The source path is needed in order to correctly transfer permissions 880 """ 881 # For backwards compatibility, we delegate to set_data() 882 return self.set_data(cache_path, data) 883 884 def set_data(self, path, data): 885 """Optional method which writes data (bytes) to a file path (a str). 886 887 Implementing this method allows for the writing of bytecode files. 888 """ 889 890 891 def get_source(self, fullname): 892 """Concrete implementation of InspectLoader.get_source.""" 893 path = self.get_filename(fullname) 894 try: 895 source_bytes = self.get_data(path) 896 except OSError as exc: 897 raise ImportError('source not available through get_data()', 898 name=fullname) from exc 899 return decode_source(source_bytes) 900 901 def source_to_code(self, data, path, *, _optimize=-1): 902 """Return the code object compiled from source. 903 904 The 'data' argument can be any object type that compile() supports. 905 """ 906 return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', 907 dont_inherit=True, optimize=_optimize) 908 909 def get_code(self, fullname): 910 """Concrete implementation of InspectLoader.get_code. 911 912 Reading of bytecode requires path_stats to be implemented. To write 913 bytecode, set_data must also be implemented. 914 915 """ 916 source_path = self.get_filename(fullname) 917 source_mtime = None 918 source_bytes = None 919 source_hash = None 920 hash_based = False 921 check_source = True 922 try: 923 bytecode_path = cache_from_source(source_path) 924 except NotImplementedError: 925 bytecode_path = None 926 else: 927 try: 928 st = self.path_stats(source_path) 929 except OSError: 930 pass 931 else: 932 source_mtime = int(st['mtime']) 933 try: 934 data = self.get_data(bytecode_path) 935 except OSError: 936 pass 937 else: 938 exc_details = { 939 'name': fullname, 940 'path': bytecode_path, 941 } 942 try: 943 flags = _classify_pyc(data, fullname, exc_details) 944 bytes_data = memoryview(data)[16:] 945 hash_based = flags & 0b1 != 0 946 if hash_based: 947 check_source = flags & 0b10 != 0 948 if (_imp.check_hash_based_pycs != 'never' and 949 (check_source or 950 _imp.check_hash_based_pycs == 'always')): 951 source_bytes = self.get_data(source_path) 952 source_hash = _imp.source_hash( 953 _RAW_MAGIC_NUMBER, 954 source_bytes, 955 ) 956 _validate_hash_pyc(data, source_hash, fullname, 957 exc_details) 958 else: 959 _validate_timestamp_pyc( 960 data, 961 source_mtime, 962 st['size'], 963 fullname, 964 exc_details, 965 ) 966 except (ImportError, EOFError): 967 pass 968 else: 969 _bootstrap._verbose_message('{} matches {}', bytecode_path, 970 source_path) 971 return _compile_bytecode(bytes_data, name=fullname, 972 bytecode_path=bytecode_path, 973 source_path=source_path) 974 if source_bytes is None: 975 source_bytes = self.get_data(source_path) 976 code_object = self.source_to_code(source_bytes, source_path) 977 _bootstrap._verbose_message('code object from {}', source_path) 978 if (not sys.dont_write_bytecode and bytecode_path is not None and 979 source_mtime is not None): 980 if hash_based: 981 if source_hash is None: 982 source_hash = _imp.source_hash(source_bytes) 983 data = _code_to_hash_pyc(code_object, source_hash, check_source) 984 else: 985 data = _code_to_timestamp_pyc(code_object, source_mtime, 986 len(source_bytes)) 987 try: 988 self._cache_bytecode(source_path, bytecode_path, data) 989 except NotImplementedError: 990 pass 991 return code_object 992 993 994class FileLoader: 995 996 """Base file loader class which implements the loader protocol methods that 997 require file system usage.""" 998 999 def __init__(self, fullname, path): 1000 """Cache the module name and the path to the file found by the 1001 finder.""" 1002 self.name = fullname 1003 self.path = path 1004 1005 def __eq__(self, other): 1006 return (self.__class__ == other.__class__ and 1007 self.__dict__ == other.__dict__) 1008 1009 def __hash__(self): 1010 return hash(self.name) ^ hash(self.path) 1011 1012 @_check_name 1013 def load_module(self, fullname): 1014 """Load a module from a file. 1015 1016 This method is deprecated. Use exec_module() instead. 1017 1018 """ 1019 # The only reason for this method is for the name check. 1020 # Issue #14857: Avoid the zero-argument form of super so the implementation 1021 # of that form can be updated without breaking the frozen module 1022 return super(FileLoader, self).load_module(fullname) 1023 1024 @_check_name 1025 def get_filename(self, fullname): 1026 """Return the path to the source file as found by the finder.""" 1027 return self.path 1028 1029 def get_data(self, path): 1030 """Return the data from path as raw bytes.""" 1031 if isinstance(self, (SourceLoader, ExtensionFileLoader)): 1032 with _io.open_code(str(path)) as file: 1033 return file.read() 1034 else: 1035 with _io.FileIO(path, 'r') as file: 1036 return file.read() 1037 1038 # ResourceReader ABC API. 1039 1040 @_check_name 1041 def get_resource_reader(self, module): 1042 if self.is_package(module): 1043 return self 1044 return None 1045 1046 def open_resource(self, resource): 1047 path = _path_join(_path_split(self.path)[0], resource) 1048 return _io.FileIO(path, 'r') 1049 1050 def resource_path(self, resource): 1051 if not self.is_resource(resource): 1052 raise FileNotFoundError 1053 path = _path_join(_path_split(self.path)[0], resource) 1054 return path 1055 1056 def is_resource(self, name): 1057 if path_sep in name: 1058 return False 1059 path = _path_join(_path_split(self.path)[0], name) 1060 return _path_isfile(path) 1061 1062 def contents(self): 1063 return iter(_os.listdir(_path_split(self.path)[0])) 1064 1065 1066class SourceFileLoader(FileLoader, SourceLoader): 1067 1068 """Concrete implementation of SourceLoader using the file system.""" 1069 1070 def path_stats(self, path): 1071 """Return the metadata for the path.""" 1072 st = _path_stat(path) 1073 return {'mtime': st.st_mtime, 'size': st.st_size} 1074 1075 def _cache_bytecode(self, source_path, bytecode_path, data): 1076 # Adapt between the two APIs 1077 mode = _calc_mode(source_path) 1078 return self.set_data(bytecode_path, data, _mode=mode) 1079 1080 def set_data(self, path, data, *, _mode=0o666): 1081 """Write bytes data to a file.""" 1082 parent, filename = _path_split(path) 1083 path_parts = [] 1084 # Figure out what directories are missing. 1085 while parent and not _path_isdir(parent): 1086 parent, part = _path_split(parent) 1087 path_parts.append(part) 1088 # Create needed directories. 1089 for part in reversed(path_parts): 1090 parent = _path_join(parent, part) 1091 try: 1092 _os.mkdir(parent) 1093 except FileExistsError: 1094 # Probably another Python process already created the dir. 1095 continue 1096 except OSError as exc: 1097 # Could be a permission error, read-only filesystem: just forget 1098 # about writing the data. 1099 _bootstrap._verbose_message('could not create {!r}: {!r}', 1100 parent, exc) 1101 return 1102 try: 1103 _write_atomic(path, data, _mode) 1104 _bootstrap._verbose_message('created {!r}', path) 1105 except OSError as exc: 1106 # Same as above: just don't write the bytecode. 1107 _bootstrap._verbose_message('could not create {!r}: {!r}', path, 1108 exc) 1109 1110 1111class SourcelessFileLoader(FileLoader, _LoaderBasics): 1112 1113 """Loader which handles sourceless file imports.""" 1114 1115 def get_code(self, fullname): 1116 path = self.get_filename(fullname) 1117 data = self.get_data(path) 1118 # Call _classify_pyc to do basic validation of the pyc but ignore the 1119 # result. There's no source to check against. 1120 exc_details = { 1121 'name': fullname, 1122 'path': path, 1123 } 1124 _classify_pyc(data, fullname, exc_details) 1125 return _compile_bytecode( 1126 memoryview(data)[16:], 1127 name=fullname, 1128 bytecode_path=path, 1129 ) 1130 1131 def get_source(self, fullname): 1132 """Return None as there is no source code.""" 1133 return None 1134 1135 1136# Filled in by _setup(). 1137EXTENSION_SUFFIXES = [] 1138 1139 1140class ExtensionFileLoader(FileLoader, _LoaderBasics): 1141 1142 """Loader for extension modules. 1143 1144 The constructor is designed to work with FileFinder. 1145 1146 """ 1147 1148 def __init__(self, name, path): 1149 self.name = name 1150 if not _path_isabs(path): 1151 try: 1152 path = _path_join(_os.getcwd(), path) 1153 except OSError: 1154 pass 1155 self.path = path 1156 1157 def __eq__(self, other): 1158 return (self.__class__ == other.__class__ and 1159 self.__dict__ == other.__dict__) 1160 1161 def __hash__(self): 1162 return hash(self.name) ^ hash(self.path) 1163 1164 def create_module(self, spec): 1165 """Create an unitialized extension module""" 1166 module = _bootstrap._call_with_frames_removed( 1167 _imp.create_dynamic, spec) 1168 _bootstrap._verbose_message('extension module {!r} loaded from {!r}', 1169 spec.name, self.path) 1170 return module 1171 1172 def exec_module(self, module): 1173 """Initialize an extension module""" 1174 _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) 1175 _bootstrap._verbose_message('extension module {!r} executed from {!r}', 1176 self.name, self.path) 1177 1178 def is_package(self, fullname): 1179 """Return True if the extension module is a package.""" 1180 file_name = _path_split(self.path)[1] 1181 return any(file_name == '__init__' + suffix 1182 for suffix in EXTENSION_SUFFIXES) 1183 1184 def get_code(self, fullname): 1185 """Return None as an extension module cannot create a code object.""" 1186 return None 1187 1188 def get_source(self, fullname): 1189 """Return None as extension modules have no source code.""" 1190 return None 1191 1192 @_check_name 1193 def get_filename(self, fullname): 1194 """Return the path to the source file as found by the finder.""" 1195 return self.path 1196 1197 1198class _NamespacePath: 1199 """Represents a namespace package's path. It uses the module name 1200 to find its parent module, and from there it looks up the parent's 1201 __path__. When this changes, the module's own path is recomputed, 1202 using path_finder. For top-level modules, the parent module's path 1203 is sys.path.""" 1204 1205 def __init__(self, name, path, path_finder): 1206 self._name = name 1207 self._path = path 1208 self._last_parent_path = tuple(self._get_parent_path()) 1209 self._path_finder = path_finder 1210 1211 def _find_parent_path_names(self): 1212 """Returns a tuple of (parent-module-name, parent-path-attr-name)""" 1213 parent, dot, me = self._name.rpartition('.') 1214 if dot == '': 1215 # This is a top-level module. sys.path contains the parent path. 1216 return 'sys', 'path' 1217 # Not a top-level module. parent-module.__path__ contains the 1218 # parent path. 1219 return parent, '__path__' 1220 1221 def _get_parent_path(self): 1222 parent_module_name, path_attr_name = self._find_parent_path_names() 1223 return getattr(sys.modules[parent_module_name], path_attr_name) 1224 1225 def _recalculate(self): 1226 # If the parent's path has changed, recalculate _path 1227 parent_path = tuple(self._get_parent_path()) # Make a copy 1228 if parent_path != self._last_parent_path: 1229 spec = self._path_finder(self._name, parent_path) 1230 # Note that no changes are made if a loader is returned, but we 1231 # do remember the new parent path 1232 if spec is not None and spec.loader is None: 1233 if spec.submodule_search_locations: 1234 self._path = spec.submodule_search_locations 1235 self._last_parent_path = parent_path # Save the copy 1236 return self._path 1237 1238 def __iter__(self): 1239 return iter(self._recalculate()) 1240 1241 def __getitem__(self, index): 1242 return self._recalculate()[index] 1243 1244 def __setitem__(self, index, path): 1245 self._path[index] = path 1246 1247 def __len__(self): 1248 return len(self._recalculate()) 1249 1250 def __repr__(self): 1251 return '_NamespacePath({!r})'.format(self._path) 1252 1253 def __contains__(self, item): 1254 return item in self._recalculate() 1255 1256 def append(self, item): 1257 self._path.append(item) 1258 1259 1260# We use this exclusively in module_from_spec() for backward-compatibility. 1261class _NamespaceLoader: 1262 def __init__(self, name, path, path_finder): 1263 self._path = _NamespacePath(name, path, path_finder) 1264 1265 @classmethod 1266 def module_repr(cls, module): 1267 """Return repr for the module. 1268 1269 The method is deprecated. The import machinery does the job itself. 1270 1271 """ 1272 return '<module {!r} (namespace)>'.format(module.__name__) 1273 1274 def is_package(self, fullname): 1275 return True 1276 1277 def get_source(self, fullname): 1278 return '' 1279 1280 def get_code(self, fullname): 1281 return compile('', '<string>', 'exec', dont_inherit=True) 1282 1283 def create_module(self, spec): 1284 """Use default semantics for module creation.""" 1285 1286 def exec_module(self, module): 1287 pass 1288 1289 def load_module(self, fullname): 1290 """Load a namespace module. 1291 1292 This method is deprecated. Use exec_module() instead. 1293 1294 """ 1295 # The import system never calls this method. 1296 _bootstrap._verbose_message('namespace module loaded with path {!r}', 1297 self._path) 1298 return _bootstrap._load_module_shim(self, fullname) 1299 1300 1301# Finders ##################################################################### 1302 1303class PathFinder: 1304 1305 """Meta path finder for sys.path and package __path__ attributes.""" 1306 1307 @classmethod 1308 def invalidate_caches(cls): 1309 """Call the invalidate_caches() method on all path entry finders 1310 stored in sys.path_importer_caches (where implemented).""" 1311 for name, finder in list(sys.path_importer_cache.items()): 1312 if finder is None: 1313 del sys.path_importer_cache[name] 1314 elif hasattr(finder, 'invalidate_caches'): 1315 finder.invalidate_caches() 1316 1317 @classmethod 1318 def _path_hooks(cls, path): 1319 """Search sys.path_hooks for a finder for 'path'.""" 1320 if sys.path_hooks is not None and not sys.path_hooks: 1321 _warnings.warn('sys.path_hooks is empty', ImportWarning) 1322 for hook in sys.path_hooks: 1323 try: 1324 return hook(path) 1325 except ImportError: 1326 continue 1327 else: 1328 return None 1329 1330 @classmethod 1331 def _path_importer_cache(cls, path): 1332 """Get the finder for the path entry from sys.path_importer_cache. 1333 1334 If the path entry is not in the cache, find the appropriate finder 1335 and cache it. If no finder is available, store None. 1336 1337 """ 1338 if path == '': 1339 try: 1340 path = _os.getcwd() 1341 except FileNotFoundError: 1342 # Don't cache the failure as the cwd can easily change to 1343 # a valid directory later on. 1344 return None 1345 try: 1346 finder = sys.path_importer_cache[path] 1347 except KeyError: 1348 finder = cls._path_hooks(path) 1349 sys.path_importer_cache[path] = finder 1350 return finder 1351 1352 @classmethod 1353 def _legacy_get_spec(cls, fullname, finder): 1354 # This would be a good place for a DeprecationWarning if 1355 # we ended up going that route. 1356 if hasattr(finder, 'find_loader'): 1357 loader, portions = finder.find_loader(fullname) 1358 else: 1359 loader = finder.find_module(fullname) 1360 portions = [] 1361 if loader is not None: 1362 return _bootstrap.spec_from_loader(fullname, loader) 1363 spec = _bootstrap.ModuleSpec(fullname, None) 1364 spec.submodule_search_locations = portions 1365 return spec 1366 1367 @classmethod 1368 def _get_spec(cls, fullname, path, target=None): 1369 """Find the loader or namespace_path for this module/package name.""" 1370 # If this ends up being a namespace package, namespace_path is 1371 # the list of paths that will become its __path__ 1372 namespace_path = [] 1373 for entry in path: 1374 if not isinstance(entry, (str, bytes)): 1375 continue 1376 finder = cls._path_importer_cache(entry) 1377 if finder is not None: 1378 if hasattr(finder, 'find_spec'): 1379 spec = finder.find_spec(fullname, target) 1380 else: 1381 spec = cls._legacy_get_spec(fullname, finder) 1382 if spec is None: 1383 continue 1384 if spec.loader is not None: 1385 return spec 1386 portions = spec.submodule_search_locations 1387 if portions is None: 1388 raise ImportError('spec missing loader') 1389 # This is possibly part of a namespace package. 1390 # Remember these path entries (if any) for when we 1391 # create a namespace package, and continue iterating 1392 # on path. 1393 namespace_path.extend(portions) 1394 else: 1395 spec = _bootstrap.ModuleSpec(fullname, None) 1396 spec.submodule_search_locations = namespace_path 1397 return spec 1398 1399 @classmethod 1400 def find_spec(cls, fullname, path=None, target=None): 1401 """Try to find a spec for 'fullname' on sys.path or 'path'. 1402 1403 The search is based on sys.path_hooks and sys.path_importer_cache. 1404 """ 1405 if path is None: 1406 path = sys.path 1407 spec = cls._get_spec(fullname, path, target) 1408 if spec is None: 1409 return None 1410 elif spec.loader is None: 1411 namespace_path = spec.submodule_search_locations 1412 if namespace_path: 1413 # We found at least one namespace path. Return a spec which 1414 # can create the namespace package. 1415 spec.origin = None 1416 spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) 1417 return spec 1418 else: 1419 return None 1420 else: 1421 return spec 1422 1423 @classmethod 1424 def find_module(cls, fullname, path=None): 1425 """find the module on sys.path or 'path' based on sys.path_hooks and 1426 sys.path_importer_cache. 1427 1428 This method is deprecated. Use find_spec() instead. 1429 1430 """ 1431 spec = cls.find_spec(fullname, path) 1432 if spec is None: 1433 return None 1434 return spec.loader 1435 1436 @classmethod 1437 def find_distributions(cls, *args, **kwargs): 1438 """ 1439 Find distributions. 1440 1441 Return an iterable of all Distribution instances capable of 1442 loading the metadata for packages matching ``context.name`` 1443 (or all names if ``None`` indicated) along the paths in the list 1444 of directories ``context.path``. 1445 """ 1446 from importlib.metadata import MetadataPathFinder 1447 return MetadataPathFinder.find_distributions(*args, **kwargs) 1448 1449 1450class FileFinder: 1451 1452 """File-based finder. 1453 1454 Interactions with the file system are cached for performance, being 1455 refreshed when the directory the finder is handling has been modified. 1456 1457 """ 1458 1459 def __init__(self, path, *loader_details): 1460 """Initialize with the path to search on and a variable number of 1461 2-tuples containing the loader and the file suffixes the loader 1462 recognizes.""" 1463 loaders = [] 1464 for loader, suffixes in loader_details: 1465 loaders.extend((suffix, loader) for suffix in suffixes) 1466 self._loaders = loaders 1467 # Base (directory) path 1468 self.path = path or '.' 1469 if not _path_isabs(self.path): 1470 self.path = _path_join(_os.getcwd(), self.path) 1471 self._path_mtime = -1 1472 self._path_cache = set() 1473 self._relaxed_path_cache = set() 1474 1475 def invalidate_caches(self): 1476 """Invalidate the directory mtime.""" 1477 self._path_mtime = -1 1478 1479 find_module = _find_module_shim 1480 1481 def find_loader(self, fullname): 1482 """Try to find a loader for the specified module, or the namespace 1483 package portions. Returns (loader, list-of-portions). 1484 1485 This method is deprecated. Use find_spec() instead. 1486 1487 """ 1488 spec = self.find_spec(fullname) 1489 if spec is None: 1490 return None, [] 1491 return spec.loader, spec.submodule_search_locations or [] 1492 1493 def _get_spec(self, loader_class, fullname, path, smsl, target): 1494 loader = loader_class(fullname, path) 1495 return spec_from_file_location(fullname, path, loader=loader, 1496 submodule_search_locations=smsl) 1497 1498 def find_spec(self, fullname, target=None): 1499 """Try to find a spec for the specified module. 1500 1501 Returns the matching spec, or None if not found. 1502 """ 1503 is_namespace = False 1504 tail_module = fullname.rpartition('.')[2] 1505 try: 1506 mtime = _path_stat(self.path or _os.getcwd()).st_mtime 1507 except OSError: 1508 mtime = -1 1509 if mtime != self._path_mtime: 1510 self._fill_cache() 1511 self._path_mtime = mtime 1512 # tail_module keeps the original casing, for __file__ and friends 1513 if _relax_case(): 1514 cache = self._relaxed_path_cache 1515 cache_module = tail_module.lower() 1516 else: 1517 cache = self._path_cache 1518 cache_module = tail_module 1519 # Check if the module is the name of a directory (and thus a package). 1520 if cache_module in cache: 1521 base_path = _path_join(self.path, tail_module) 1522 for suffix, loader_class in self._loaders: 1523 init_filename = '__init__' + suffix 1524 full_path = _path_join(base_path, init_filename) 1525 if _path_isfile(full_path): 1526 return self._get_spec(loader_class, fullname, full_path, [base_path], target) 1527 else: 1528 # If a namespace package, return the path if we don't 1529 # find a module in the next section. 1530 is_namespace = _path_isdir(base_path) 1531 # Check for a file w/ a proper suffix exists. 1532 for suffix, loader_class in self._loaders: 1533 try: 1534 full_path = _path_join(self.path, tail_module + suffix) 1535 except ValueError: 1536 return None 1537 _bootstrap._verbose_message('trying {}', full_path, verbosity=2) 1538 if cache_module + suffix in cache: 1539 if _path_isfile(full_path): 1540 return self._get_spec(loader_class, fullname, full_path, 1541 None, target) 1542 if is_namespace: 1543 _bootstrap._verbose_message('possible namespace for {}', base_path) 1544 spec = _bootstrap.ModuleSpec(fullname, None) 1545 spec.submodule_search_locations = [base_path] 1546 return spec 1547 return None 1548 1549 def _fill_cache(self): 1550 """Fill the cache of potential modules and packages for this directory.""" 1551 path = self.path 1552 try: 1553 contents = _os.listdir(path or _os.getcwd()) 1554 except (FileNotFoundError, PermissionError, NotADirectoryError): 1555 # Directory has either been removed, turned into a file, or made 1556 # unreadable. 1557 contents = [] 1558 # We store two cached versions, to handle runtime changes of the 1559 # PYTHONCASEOK environment variable. 1560 if not sys.platform.startswith('win'): 1561 self._path_cache = set(contents) 1562 else: 1563 # Windows users can import modules with case-insensitive file 1564 # suffixes (for legacy reasons). Make the suffix lowercase here 1565 # so it's done once instead of for every import. This is safe as 1566 # the specified suffixes to check against are always specified in a 1567 # case-sensitive manner. 1568 lower_suffix_contents = set() 1569 for item in contents: 1570 name, dot, suffix = item.partition('.') 1571 if dot: 1572 new_name = '{}.{}'.format(name, suffix.lower()) 1573 else: 1574 new_name = name 1575 lower_suffix_contents.add(new_name) 1576 self._path_cache = lower_suffix_contents 1577 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): 1578 self._relaxed_path_cache = {fn.lower() for fn in contents} 1579 1580 @classmethod 1581 def path_hook(cls, *loader_details): 1582 """A class method which returns a closure to use on sys.path_hook 1583 which will return an instance using the specified loaders and the path 1584 called on the closure. 1585 1586 If the path called on the closure is not a directory, ImportError is 1587 raised. 1588 1589 """ 1590 def path_hook_for_FileFinder(path): 1591 """Path hook for importlib.machinery.FileFinder.""" 1592 if not _path_isdir(path): 1593 raise ImportError('only directories are supported', path=path) 1594 return cls(path, *loader_details) 1595 1596 return path_hook_for_FileFinder 1597 1598 def __repr__(self): 1599 return 'FileFinder({!r})'.format(self.path) 1600 1601 1602# Import setup ############################################################### 1603 1604def _fix_up_module(ns, name, pathname, cpathname=None): 1605 # This function is used by PyImport_ExecCodeModuleObject(). 1606 loader = ns.get('__loader__') 1607 spec = ns.get('__spec__') 1608 if not loader: 1609 if spec: 1610 loader = spec.loader 1611 elif pathname == cpathname: 1612 loader = SourcelessFileLoader(name, pathname) 1613 else: 1614 loader = SourceFileLoader(name, pathname) 1615 if not spec: 1616 spec = spec_from_file_location(name, pathname, loader=loader) 1617 try: 1618 ns['__spec__'] = spec 1619 ns['__loader__'] = loader 1620 ns['__file__'] = pathname 1621 ns['__cached__'] = cpathname 1622 except Exception: 1623 # Not important enough to report. 1624 pass 1625 1626 1627def _get_supported_file_loaders(): 1628 """Returns a list of file-based module loaders. 1629 1630 Each item is a tuple (loader, suffixes). 1631 """ 1632 extensions = ExtensionFileLoader, _imp.extension_suffixes() 1633 source = SourceFileLoader, SOURCE_SUFFIXES 1634 bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES 1635 return [extensions, source, bytecode] 1636 1637 1638def _setup(_bootstrap_module): 1639 """Setup the path-based importers for importlib by importing needed 1640 built-in modules and injecting them into the global namespace. 1641 1642 Other components are extracted from the core bootstrap module. 1643 1644 """ 1645 global sys, _imp, _bootstrap 1646 _bootstrap = _bootstrap_module 1647 sys = _bootstrap.sys 1648 _imp = _bootstrap._imp 1649 1650 # Directly load built-in modules needed during bootstrap. 1651 self_module = sys.modules[__name__] 1652 for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): 1653 if builtin_name not in sys.modules: 1654 builtin_module = _bootstrap._builtin_from_name(builtin_name) 1655 else: 1656 builtin_module = sys.modules[builtin_name] 1657 setattr(self_module, builtin_name, builtin_module) 1658 1659 # Directly load the os module (needed during bootstrap). 1660 os_details = ('posix', ['/']), ('nt', ['\\', '/']) 1661 for builtin_os, path_separators in os_details: 1662 # Assumption made in _path_join() 1663 assert all(len(sep) == 1 for sep in path_separators) 1664 path_sep = path_separators[0] 1665 if builtin_os in sys.modules: 1666 os_module = sys.modules[builtin_os] 1667 break 1668 else: 1669 try: 1670 os_module = _bootstrap._builtin_from_name(builtin_os) 1671 break 1672 except ImportError: 1673 continue 1674 else: 1675 raise ImportError('importlib requires posix or nt') 1676 setattr(self_module, '_os', os_module) 1677 setattr(self_module, 'path_sep', path_sep) 1678 setattr(self_module, 'path_separators', ''.join(path_separators)) 1679 setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators}) 1680 1681 # Directly load the _thread module (needed during bootstrap). 1682 thread_module = _bootstrap._builtin_from_name('_thread') 1683 setattr(self_module, '_thread', thread_module) 1684 1685 # Directly load the _weakref module (needed during bootstrap). 1686 weakref_module = _bootstrap._builtin_from_name('_weakref') 1687 setattr(self_module, '_weakref', weakref_module) 1688 1689 # Directly load the winreg module (needed during bootstrap). 1690 if builtin_os == 'nt': 1691 winreg_module = _bootstrap._builtin_from_name('winreg') 1692 setattr(self_module, '_winreg', winreg_module) 1693 1694 # Constants 1695 setattr(self_module, '_relax_case', _make_relax_case()) 1696 EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) 1697 if builtin_os == 'nt': 1698 SOURCE_SUFFIXES.append('.pyw') 1699 if '_d.pyd' in EXTENSION_SUFFIXES: 1700 WindowsRegistryFinder.DEBUG_BUILD = True 1701 1702 1703def _install(_bootstrap_module): 1704 """Install the path-based import components.""" 1705 _setup(_bootstrap_module) 1706 supported_loaders = _get_supported_file_loaders() 1707 sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) 1708 sys.meta_path.append(PathFinder) 1709