1""" 2Find modules used by a script, using bytecode analysis. 3 4Based on the stdlib modulefinder by Thomas Heller and Just van Rossum, 5but uses a graph data structure and 2.3 features 6 7XXX: Verify all calls to _import_hook (and variants) to ensure that 8imports are done in the right way. 9""" 10from __future__ import absolute_import, print_function 11 12#FIXME: To decrease the likelihood of ModuleGraph exceeding the recursion limit 13#and hence unpredictably raising fatal exceptions, increase the recursion 14#limit at PyInstaller startup (i.e., in the 15#PyInstaller.building.build_main.build() function). For details, see: 16# https://github.com/pyinstaller/pyinstaller/issues/1919#issuecomment-216016176 17 18import pkg_resources 19 20import ast 21import codecs 22import dis 23import imp 24import marshal 25import os 26import pkgutil 27import sys 28import re 29from collections import deque, namedtuple 30from struct import unpack 31import warnings 32 33from altgraph.ObjectGraph import ObjectGraph 34from altgraph import GraphError 35 36from . import util 37from . import zipio 38from ._compat import get_instructions, BytesIO, StringIO, \ 39 pathname2url, _cOrd, _READ_MODE 40 41 42BOM = codecs.BOM_UTF8.decode('utf-8') 43 44 45#FIXME: Leverage this rather than magic numbers below. 46ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL = -1 47""" 48Constant instructing the builtin `__import__()` function to attempt both 49absolute and relative imports. 50""" 51 52 53#FIXME: Leverage this rather than magic numbers below. 54ABSOLUTE_IMPORT_LEVEL = 0 55""" 56Constant instructing the builtin `__import__()` function to attempt only 57absolute imports. 58""" 59 60 61#FIXME: Leverage this rather than magic numbers below. 62DEFAULT_IMPORT_LEVEL = ( 63 ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL if sys.version_info[0] == 2 else 64 ABSOLUTE_IMPORT_LEVEL) 65""" 66Constant instructing the builtin `__import__()` function to attempt the default 67import style specific to the active Python interpreter. 68 69Specifically, under: 70 71* Python 2, this defaults to attempting both absolute and relative imports. 72* Python 3, this defaults to attempting only absolute imports. 73""" 74 75# TODO: Refactor all uses of explicit filetypes in this module *AND* of the 76# imp.get_suffixes() function to use this dictionary instead. Unfortunately, 77# tests for explicit filetypes (e.g., ".py") are non-portable. Under Windows, 78# for example, both the ".py" *AND* ".pyw" filetypes signify valid uncompiled 79# Python modules. 80# TODO: The imp.get_suffixes() function (in fact, the entire "imp" package) has 81# been deprecated as of Python 3.3 by the importlib.machinery.all_suffixes() 82# function, which largely performs the same role. Unfortunately, the latter 83# function was only introduced with Python 3.3. Since PyInstaller requires 84# Python >= 3.3 when running under Python 3, refactor this as follows: 85# 86# * Under Python 2, continue calling imp.get_suffixes(). 87# * Under Python 3, call importlib.machinery.all_suffixes() instead. 88_IMPORTABLE_FILETYPE_TO_METADATA = { 89 filetype: (filetype, open_mode, imp_type) 90 for filetype, open_mode, imp_type in imp.get_suffixes() 91} 92""" 93Dictionary mapping the filetypes of importable files to the 3-tuple of metadata 94describing such files returned by the `imp.get_suffixes()` function whose first 95element is that filetype. 96 97This dictionary simplifies platform-portable importation of importable files, 98including: 99 100* Uncompiled modules suffixed by `.py` (as well as `.pyw` under Windows). 101* Compiled modules suffixed by either `.pyc` or `.pyo`. 102* C extensions suffixed by the platform-specific shared library filetype (e.g., 103 `.so` under Linux, `.dll` under Windows). 104 105The keys of this dictionary are `.`-prefixed filetypes (e.g., `.py`, `.so'); 106the values of this dictionary are 3-tuples whose: 107 1081. First element is the same `.`-prefixed filetype. 1091. Second element is the mode to be passed to the `open()` built-in to open 110 files of that filetype under the current platform and Python interpreter 111 (e.g., `rU` for the `.py` filetype under Python 2, `r` for the same 112 filetype under Python 3). 1131. Third element is a magic number specific to the `imp` module (e.g., 114 `imp.C_EXTENSION` for filetypes corresponding to C extensions). 115""" 116 117 118 119# Modulegraph does a good job at simulating Python's, but it can not 120# handle packagepath modifications packages make at runtime. Therefore there 121# is a mechanism whereby you can register extra paths in this map for a 122# package, and it will be honored. 123# 124# Note this is a mapping is lists of paths. 125_packagePathMap = {} 126 127# Prefix used in magic .pth files used by setuptools to create namespace 128# packages without an __init__.py file. 129# 130# The value is a list of such prefixes as the prefix varies with versions of 131# setuptools. 132_SETUPTOOLS_NAMESPACEPKG_PTHs=( 133 # setuptools 31.0.0 134 ("import sys, types, os;has_mfs = sys.version_info > (3, 5);" 135 "p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('"), 136 # distribute 0.6.10 137 ("import sys,types,os; p = os.path.join(" 138 "sys._getframe(1).f_locals['sitedir'], *('"), 139 # setuptools 0.6c9, distribute 0.6.12 140 ("import sys,new,os; p = os.path.join(sys._getframe(" 141 "1).f_locals['sitedir'], *('"), 142 # setuptools 28.1.0 143 ("import sys, types, os;p = os.path.join(" 144 "sys._getframe(1).f_locals['sitedir'], *('"), 145 # setuptools 28.7.0 146 ("import sys, types, os;pep420 = sys.version_info > (3, 3);" 147 "p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('"), 148) 149 150 151class InvalidRelativeImportError (ImportError): 152 pass 153 154 155def _namespace_package_path(fqname, pathnames, path=None): 156 """ 157 Return the __path__ for the python package in *fqname*. 158 159 This function uses setuptools metadata to extract information 160 about namespace packages from installed eggs. 161 """ 162 working_set = pkg_resources.WorkingSet(path) 163 164 path = list(pathnames) 165 166 for dist in working_set: 167 if dist.has_metadata('namespace_packages.txt'): 168 namespaces = dist.get_metadata( 169 'namespace_packages.txt').splitlines() 170 if fqname in namespaces: 171 nspath = os.path.join(dist.location, *fqname.split('.')) 172 if nspath not in path: 173 path.append(nspath) 174 175 return path 176 177_strs = re.compile(r'''^\s*["']([A-Za-z0-9_]+)["'],?\s*''') # "<- emacs happy 178 179 180def _eval_str_tuple(value): 181 """ 182 Input is the repr of a tuple of strings, output 183 is that tuple. 184 185 This only works with a tuple where the members are 186 python identifiers. 187 """ 188 if not (value.startswith('(') and value.endswith(')')): 189 raise ValueError(value) 190 191 orig_value = value 192 value = value[1:-1] 193 194 result = [] 195 while value: 196 m = _strs.match(value) 197 if m is None: 198 raise ValueError(orig_value) 199 200 result.append(m.group(1)) 201 value = value[len(m.group(0)):] 202 203 return tuple(result) 204 205 206def _path_from_importerror(exc, default): 207 # This is a hack, but sadly enough the necessary information 208 # isn't available otherwise. 209 m = re.match(r'^No module named (\S+)$', str(exc)) 210 if m is not None: 211 return m.group(1) 212 213 return default 214 215 216def os_listdir(path): 217 """ 218 Deprecated name 219 """ 220 warnings.warn( 221 "Use zipio.listdir instead of os_listdir", 222 DeprecationWarning) 223 return zipio.listdir(path) 224 225 226def _code_to_file(co): 227 """ Convert code object to a .pyc pseudo-file """ 228 if sys.version_info >= (3, 7): 229 header = imp.get_magic() + (b'\0' * 12) 230 elif sys.version_info >= (3, 4): 231 header = imp.get_magic() + (b'\0' * 8) 232 else: 233 header = imp.get_magic() + (b'\0' * 4) 234 return BytesIO(header + marshal.dumps(co)) 235 236 237def moduleInfoForPath(path): 238 for (ext, readmode, typ) in imp.get_suffixes(): 239 if path.endswith(ext): 240 return os.path.basename(path)[:-len(ext)], readmode, typ 241 return None 242 243 244def AddPackagePath(packagename, path): 245 warnings.warn( 246 "Use addPackagePath instead of AddPackagePath", 247 DeprecationWarning) 248 addPackagePath(packagename, path) 249 250 251def addPackagePath(packagename, path): 252 paths = _packagePathMap.get(packagename, []) 253 paths.append(path) 254 _packagePathMap[packagename] = paths 255 256 257_replacePackageMap = {} 258 259 260# This ReplacePackage mechanism allows modulefinder to work around the 261# way the _xmlplus package injects itself under the name "xml" into 262# sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml") 263# before running ModuleGraph. 264def ReplacePackage(oldname, newname): 265 warnings.warn("use replacePackage instead of ReplacePackage", 266 DeprecationWarning) 267 replacePackage(oldname, newname) 268 269 270def replacePackage(oldname, newname): 271 _replacePackageMap[oldname] = newname 272 273 274#FIXME: What is this? Do we actually need this? This appears to provide 275#significantly more fine-grained metadata than PyInstaller will ever require. 276#It consumes a great deal of space (slots or no slots), since we store an 277#instance of this class for each edge of the graph. 278class DependencyInfo (namedtuple("DependencyInfo", 279 ["conditional", "function", "tryexcept", "fromlist"])): 280 __slots__ = () 281 282 def _merged(self, other): 283 if (not self.conditional and not self.function and not self.tryexcept) \ 284 or (not other.conditional and not other.function and not other.tryexcept): 285 return DependencyInfo( 286 conditional=False, 287 function=False, 288 tryexcept=False, 289 fromlist=self.fromlist and other.fromlist) 290 291 else: 292 return DependencyInfo( 293 conditional=self.conditional or other.conditional, 294 function=self.function or other.function, 295 tryexcept=self.tryexcept or other.tryexcept, 296 fromlist=self.fromlist and other.fromlist) 297 298 299#FIXME: Shift the following Node class hierarchy into a new 300#"PyInstaller.lib.modulegraph.node" module. This module is much too long. 301#FIXME: Refactor "_deferred_imports" from a tuple into a proper lightweight 302#class leveraging "__slots__". If not for backward compatibility, we'd just 303#leverage a named tuple -- but this should do just as well. 304#FIXME: Move the "packagepath" attribute into the "Package" class. Only 305#packages define the "__path__" special attribute. The codebase currently 306#erroneously tests whether "module.packagepath is not None" to determine 307#whether a node is a package or not. However, "isinstance(module, Package)" is 308#a significantly more reliable test. Refactor the former into the latter. 309class Node(object): 310 """ 311 Abstract base class (ABC) of all objects added to a `ModuleGraph`. 312 313 Attributes 314 ---------- 315 code : codeobject 316 Code object of the pure-Python module corresponding to this graph node 317 if any _or_ `None` otherwise. 318 graphident : str 319 Synonym of `identifier` required by the `ObjectGraph` superclass of the 320 `ModuleGraph` class. For readability, the `identifier` attribute should 321 typically be used instead. 322 filename : str 323 Absolute path of this graph node's corresponding module, package, or C 324 extension if any _or_ `None` otherwise. 325 identifier : str 326 Fully-qualified name of this graph node's corresponding module, 327 package, or C extension. 328 packagepath : str 329 List of the absolute paths of all directories comprising this graph 330 node's corresponding package. If this is a: 331 * Non-namespace package, this list contains exactly one path. 332 * Namespace package, this list contains one or more paths. 333 _deferred_imports : list 334 List of all target modules imported by the source module corresponding 335 to this graph node whole importations have been deferred for subsequent 336 processing in between calls to the `_ModuleGraph._scan_code()` and 337 `_ModuleGraph._process_imports()` methods for this source module _or_ 338 `None` otherwise. Each element of this list is a 3-tuple 339 `(have_star, _safe_import_hook_args, _safe_import_hook_kwargs)` 340 collecting the importation of a target module from this source module 341 for subsequent processing, where: 342 * `have_star` is a boolean `True` only if this is a `from`-style star 343 import (e.g., resembling `from {target_module_name} import *`). 344 * `_safe_import_hook_args` is a (typically non-empty) sequence of all 345 positional arguments to be passed to the `_safe_import_hook()` method 346 to add this importation to the graph. 347 * `_safe_import_hook_kwargs` is a (typically empty) dictionary of all 348 keyword arguments to be passed to the `_safe_import_hook()` method 349 to add this importation to the graph. 350 Unlike functional languages, Python imposes a maximum depth on the 351 interpreter stack (and hence recursion). On breaching this depth, 352 Python raises a fatal `RuntimeError` exception. Since `ModuleGraph` 353 parses imports recursively rather than iteratively, this depth _was_ 354 commonly breached before the introduction of this list. Python 355 environments installing a large number of modules (e.g., Anaconda) were 356 particularly susceptible. Why? Because `ModuleGraph` concurrently 357 descended through both the abstract syntax trees (ASTs) of all source 358 modules being parsed _and_ the graph of all target modules imported by 359 these source modules being built. The stack thus consisted of 360 alternating layers of AST and graph traversal. To unwind such 361 alternation and effectively halve the stack depth, `ModuleGraph` now 362 descends through the abstract syntax tree (AST) of each source module 363 being parsed and adds all importations originating within this module 364 to this list _before_ descending into the graph of these importations. 365 See pyinstaller/pyinstaller/#1289 for further details. 366 _global_attr_names : set 367 Set of the unqualified names of all global attributes (e.g., classes, 368 variables) defined in the pure-Python module corresponding to this 369 graph node if any _or_ the empty set otherwise. This includes the names 370 of all attributes imported via `from`-style star imports from other 371 existing modules (e.g., `from {target_module_name} import *`). This 372 set is principally used to differentiate the non-ignorable importation 373 of non-existent submodules in a package from the ignorable importation 374 of existing global attributes defined in that package's pure-Python 375 `__init__` submodule in `from`-style imports (e.g., `bar` in 376 `from foo import bar`, which may be either a submodule or attribute of 377 `foo`), as such imports ambiguously allow both. This set is _not_ used 378 to differentiate submodules from attributes in `import`-style imports 379 (e.g., `bar` in `import foo.bar`, which _must_ be a submodule of 380 `foo`), as such imports unambiguously allow only submodules. 381 _starimported_ignored_module_names : set 382 Set of the fully-qualified names of all existing unparsable modules 383 that the existing parsable module corresponding to this graph node 384 attempted to perform one or more "star imports" from. If this module 385 either does _not_ exist or does but is unparsable, this is the empty 386 set. Equivalently, this set contains each fully-qualified name 387 `{trg_module_name}` for which: 388 * This module contains an import statement of the form 389 `from {trg_module_name} import *`. 390 * The module whose name is `{trg_module_name}` exists but is _not_ 391 parsable by `ModuleGraph` (e.g., due to _not_ being pure-Python). 392 **This set is currently defined but otherwise ignored.** 393 _submodule_basename_to_node : dict 394 Dictionary mapping from the unqualified name of each submodule 395 contained by the parent module corresponding to this graph node to that 396 submodule's graph node. If this dictionary is non-empty, this parent 397 module is typically but _not_ always a package (e.g., the non-package 398 `os` module containing the `os.path` submodule). 399 """ 400 401 __slots__ = [ 402 'code', 403 'filename', 404 'graphident', 405 'identifier', 406 'packagepath', 407 '_deferred_imports', 408 '_global_attr_names', 409 '_starimported_ignored_module_names', 410 '_submodule_basename_to_node', 411 ] 412 413 def __init__(self, identifier): 414 """ 415 Initialize this graph node. 416 417 Parameters 418 ---------- 419 identifier : str 420 Fully-qualified name of this graph node's corresponding module, 421 package, or C extension. 422 """ 423 424 self.code = None 425 self.filename = None 426 self.graphident = identifier 427 self.identifier = identifier 428 self.packagepath = None 429 self._deferred_imports = None 430 self._global_attr_names = set() 431 self._starimported_ignored_module_names = set() 432 self._submodule_basename_to_node = dict() 433 434 435 def is_global_attr(self, attr_name): 436 """ 437 `True` only if the pure-Python module corresponding to this graph node 438 defines a global attribute (e.g., class, variable) with the passed 439 name. 440 441 If this module is actually a package, this method instead returns 442 `True` only if this package's pure-Python `__init__` submodule defines 443 such a global attribute. In this case, note that this package may still 444 contain an importable submodule of the same name. Callers should 445 attempt to import this attribute as a submodule of this package 446 _before_ assuming this attribute to be an ignorable global. See 447 "Examples" below for further details. 448 449 Parameters 450 ---------- 451 attr_name : str 452 Unqualified name of the attribute to be tested. 453 454 Returns 455 ---------- 456 bool 457 `True` only if this module defines this global attribute. 458 459 Examples 460 ---------- 461 Consider a hypothetical module `foo` containing submodules `bar` and 462 `__init__` where the latter assigns `bar` to be a global variable 463 (possibly star-exported via the special `__all__` global variable): 464 465 >>> # In "foo.__init__": 466 >>> bar = 3.1415 467 468 Python 2 and 3 both permissively permit this. This method returns 469 `True` in this case (i.e., when called on the `foo` package's graph 470 node, passed the attribute name `bar`) despite the importability of the 471 `foo.bar` submodule. 472 """ 473 474 return attr_name in self._global_attr_names 475 476 477 def is_submodule(self, submodule_basename): 478 """ 479 `True` only if the parent module corresponding to this graph node 480 contains the submodule with the passed name. 481 482 If `True`, this parent module is typically but _not_ always a package 483 (e.g., the non-package `os` module containing the `os.path` submodule). 484 485 Parameters 486 ---------- 487 submodule_basename : str 488 Unqualified name of the submodule to be tested. 489 490 Returns 491 ---------- 492 bool 493 `True` only if this parent module contains this submodule. 494 """ 495 496 return submodule_basename in self._submodule_basename_to_node 497 498 499 def add_global_attr(self, attr_name): 500 """ 501 Record the global attribute (e.g., class, variable) with the passed 502 name to be defined by the pure-Python module corresponding to this 503 graph node. 504 505 If this module is actually a package, this method instead records this 506 attribute to be defined by this package's pure-Python `__init__` 507 submodule. 508 509 Parameters 510 ---------- 511 attr_name : str 512 Unqualified name of the attribute to be added. 513 """ 514 515 self._global_attr_names.add(attr_name) 516 517 518 def add_global_attrs_from_module(self, target_module): 519 """ 520 Record all global attributes (e.g., classes, variables) defined by the 521 target module corresponding to the passed graph node to also be defined 522 by the source module corresponding to this graph node. 523 524 If the source module is actually a package, this method instead records 525 these attributes to be defined by this package's pure-Python `__init__` 526 submodule. 527 528 Parameters 529 ---------- 530 target_module : Node 531 Graph node of the target module to import attributes from. 532 """ 533 534 self._global_attr_names.update(target_module._global_attr_names) 535 536 537 def add_submodule(self, submodule_basename, submodule_node): 538 """ 539 Add the submodule with the passed name and previously imported graph 540 node to the parent module corresponding to this graph node. 541 542 This parent module is typically but _not_ always a package (e.g., the 543 non-package `os` module containing the `os.path` submodule). 544 545 Parameters 546 ---------- 547 submodule_basename : str 548 Unqualified name of the submodule to add to this parent module. 549 submodule_node : Node 550 Graph node of this submodule. 551 """ 552 553 self._submodule_basename_to_node[submodule_basename] = submodule_node 554 555 556 def get_submodule(self, submodule_basename): 557 """ 558 Graph node of the submodule with the passed name in the parent module 559 corresponding to this graph node. 560 561 If this parent module does _not_ contain this submodule, an exception 562 is raised. Else, this parent module is typically but _not_ always a 563 package (e.g., the non-package `os` module containing the `os.path` 564 submodule). 565 566 Parameters 567 ---------- 568 module_basename : str 569 Unqualified name of the submodule to retrieve. 570 571 Returns 572 ---------- 573 Node 574 Graph node of this submodule. 575 """ 576 577 return self._submodule_basename_to_node[submodule_basename] 578 579 580 def get_submodule_or_none(self, submodule_basename): 581 """ 582 Graph node of the submodule with the passed unqualified name in the 583 parent module corresponding to this graph node if this module contains 584 this submodule _or_ `None`. 585 586 This parent module is typically but _not_ always a package (e.g., the 587 non-package `os` module containing the `os.path` submodule). 588 589 Parameters 590 ---------- 591 submodule_basename : str 592 Unqualified name of the submodule to retrieve. 593 594 Returns 595 ---------- 596 Node 597 Graph node of this submodule if this parent module contains this 598 submodule _or_ `None`. 599 """ 600 601 return self._submodule_basename_to_node.get(submodule_basename) 602 603 604 def remove_global_attr_if_found(self, attr_name): 605 """ 606 Record the global attribute (e.g., class, variable) with the passed 607 name if previously recorded as defined by the pure-Python module 608 corresponding to this graph node to be subsequently undefined by the 609 same module. 610 611 If this module is actually a package, this method instead records this 612 attribute to be undefined by this package's pure-Python `__init__` 613 submodule. 614 615 This method is intended to be called on globals previously defined by 616 this module that are subsequently undefined via the `del` built-in by 617 this module, thus "forgetting" or "undoing" these globals. 618 619 For safety, there exists no corresponding `remove_global_attr()` 620 method. While defining this method is trivial, doing so would invite 621 `KeyError` exceptions on scanning valid Python that lexically deletes a 622 global in a scope under this module's top level (e.g., in a function) 623 _before_ defining this global at this top level. Since `ModuleGraph` 624 cannot and should not (re)implement a full-blown Python interpreter, 625 ignoring out-of-order deletions is the only sane policy. 626 627 Parameters 628 ---------- 629 attr_name : str 630 Unqualified name of the attribute to be removed. 631 """ 632 633 if self.is_global_attr(attr_name): 634 self._global_attr_names.remove(attr_name) 635 636 637 def __cmp__(self, other): 638 try: 639 otherIdent = getattr(other, 'graphident') 640 except AttributeError: 641 return NotImplemented 642 643 return cmp(self.graphident, otherIdent) # noqa: F821 644 645 def __eq__(self, other): 646 try: 647 otherIdent = getattr(other, 'graphident') 648 except AttributeError: 649 return False 650 651 return self.graphident == otherIdent 652 653 def __ne__(self, other): 654 try: 655 otherIdent = getattr(other, 'graphident') 656 except AttributeError: 657 return True 658 659 return self.graphident != otherIdent 660 661 def __lt__(self, other): 662 try: 663 otherIdent = getattr(other, 'graphident') 664 except AttributeError: 665 return NotImplemented 666 667 return self.graphident < otherIdent 668 669 def __le__(self, other): 670 try: 671 otherIdent = getattr(other, 'graphident') 672 except AttributeError: 673 return NotImplemented 674 675 return self.graphident <= otherIdent 676 677 def __gt__(self, other): 678 try: 679 otherIdent = getattr(other, 'graphident') 680 except AttributeError: 681 return NotImplemented 682 683 return self.graphident > otherIdent 684 685 def __ge__(self, other): 686 try: 687 otherIdent = getattr(other, 'graphident') 688 except AttributeError: 689 return NotImplemented 690 691 return self.graphident >= otherIdent 692 693 def __hash__(self): 694 return hash(self.graphident) 695 696 def infoTuple(self): 697 return (self.identifier,) 698 699 def __repr__(self): 700 return '%s%r' % (type(self).__name__, self.infoTuple()) 701 702 703# TODO: This indirection is, frankly, unnecessary. The 704# ModuleGraph.alias_module() should directly add the desired AliasNode instance 705# to the graph rather than indirectly adding an Alias instance to the 706# "lazynodes" dictionary. 707class Alias(str): 708 """ 709 Placeholder aliasing an existing source module to a non-existent target 710 module (i.e., the desired alias). 711 712 For obscure reasons, this class subclasses `str`. Each instance of this 713 class is the fully-qualified name of the existing source module being 714 aliased. Unlike the related `AliasNode` class, instances of this class are 715 _not_ actual nodes and hence _not_ added to the graph; they only facilitate 716 communication between the `ModuleGraph.alias_module()` and 717 `ModuleGraph.findNode()` methods. 718 """ 719 720 721class AliasNode(Node): 722 """ 723 Graph node representing the aliasing of an existing source module under a 724 non-existent target module name (i.e., the desired alias). 725 """ 726 727 def __init__(self, name, node): 728 """ 729 Initialize this alias. 730 731 Parameters 732 ---------- 733 name : str 734 Fully-qualified name of the non-existent target module to be 735 created (as an alias of the existing source module). 736 node : Node 737 Graph node of the existing source module being aliased. 738 """ 739 super(AliasNode, self).__init__(name) 740 741 #FIXME: Why only some? Why not *EVERYTHING* except "graphident", which 742 #must remain equal to "name" for lookup purposes? This is, after all, 743 #an alias. The idea is for the two nodes to effectively be the same. 744 745 # Copy some attributes from this source module into this target alias. 746 for attr_name in ( 747 'identifier', 'packagepath', 748 '_global_attr_names', '_starimported_ignored_module_names', 749 '_submodule_basename_to_node'): 750 if hasattr(node, attr_name): 751 setattr(self, attr_name, getattr(node, attr_name)) 752 753 754 def infoTuple(self): 755 return (self.graphident, self.identifier) 756 757 758class BadModule(Node): 759 pass 760 761 762class ExcludedModule(BadModule): 763 pass 764 765 766class MissingModule(BadModule): 767 pass 768 769 770class InvalidRelativeImport (BadModule): 771 def __init__(self, relative_path, from_name): 772 identifier = relative_path 773 if relative_path.endswith('.'): 774 identifier += from_name 775 else: 776 identifier += '.' + from_name 777 super(InvalidRelativeImport, self).__init__(identifier) 778 self.relative_path = relative_path 779 self.from_name = from_name 780 781 def infoTuple(self): 782 return (self.relative_path, self.from_name) 783 784 785class Script(Node): 786 def __init__(self, filename): 787 super(Script, self).__init__(filename) 788 self.filename = filename 789 790 def infoTuple(self): 791 return (self.filename,) 792 793 794class BaseModule(Node): 795 def __init__(self, name, filename=None, path=None): 796 super(BaseModule, self).__init__(name) 797 self.filename = filename 798 self.packagepath = path 799 800 def infoTuple(self): 801 return tuple(filter(None, (self.identifier, self.filename, self.packagepath))) 802 803 804class BuiltinModule(BaseModule): 805 pass 806 807 808class SourceModule(BaseModule): 809 pass 810 811 812class InvalidSourceModule(SourceModule): 813 pass 814 815 816class CompiledModule(BaseModule): 817 pass 818 819 820class InvalidCompiledModule(BaseModule): 821 pass 822 823 824class Extension(BaseModule): 825 pass 826 827 828class Package(BaseModule): 829 """ 830 Graph node representing a non-namespace package. 831 """ 832 pass 833 834 835class NamespacePackage(Package): 836 """ 837 Graph node representing a namespace package. 838 """ 839 pass 840 841 842class RuntimeModule(BaseModule): 843 """ 844 Graph node representing a non-package Python module dynamically defined at 845 runtime. 846 847 Most modules are statically defined on-disk as standard Python files. 848 Some modules, however, are dynamically defined in-memory at runtime 849 (e.g., `gi.repository.Gst`, dynamically defined by the statically 850 defined `gi.repository.__init__` module). 851 852 This node represents such a runtime module. Since this is _not_ a package, 853 all attempts to import submodules from this module in `from`-style import 854 statements (e.g., the `queue` submodule in `from six.moves import queue`) 855 will be silently ignored. 856 857 To ensure that the parent package of this module if any is also imported 858 and added to the graph, this node is typically added to the graph by 859 calling the `ModuleGraph.add_module()` method. 860 """ 861 pass 862 863 864class RuntimePackage(Package): 865 """ 866 Graph node representing a non-namespace Python package dynamically defined 867 at runtime. 868 869 Most packages are statically defined on-disk as standard subdirectories 870 containing `__init__.py` files. Some packages, however, are dynamically 871 defined in-memory at runtime (e.g., `six.moves`, dynamically defined by 872 the statically defined `six` module). 873 874 This node represents such a runtime package. All attributes imported from 875 this package in `from`-style import statements that are submodules of this 876 package (e.g., the `queue` submodule in `from six.moves import queue`) will 877 be imported rather than ignored. 878 879 To ensure that the parent package of this package if any is also imported 880 and added to the graph, this node is typically added to the graph by 881 calling the `ModuleGraph.add_module()` method. 882 """ 883 pass 884 885 886#FIXME: Safely removable. We don't actually use this anywhere. After removing 887#this class, remove the corresponding entry from "compat". 888class FlatPackage(BaseModule): 889 def __init__(self, *args, **kwds): 890 warnings.warn( 891 "This class will be removed in a future version of modulegraph", 892 DeprecationWarning) 893 super(FlatPackage, *args, **kwds) 894 895 896#FIXME: Safely removable. We don't actually use this anywhere. After removing 897#this class, remove the corresponding entry from "compat". 898class ArchiveModule(BaseModule): 899 def __init__(self, *args, **kwds): 900 warnings.warn( 901 "This class will be removed in a future version of modulegraph", 902 DeprecationWarning) 903 super(FlatPackage, *args, **kwds) 904 905 906# HTML templates for ModuleGraph generator 907header = """\ 908<!DOCTYPE html> 909<html> 910 <head> 911 <meta charset="UTF-8"> 912 <title>%(TITLE)s</title> 913 <style> 914 .node { padding: 0.5em 0 0.5em; border-top: thin grey dotted; } 915 .moduletype { font: smaller italic } 916 .node a { text-decoration: none; color: #006699; } 917 .node a:visited { text-decoration: none; color: #2f0099; } 918 </style> 919 </head> 920 <body> 921 <h1>%(TITLE)s</h1>""" 922entry = """ 923<div class="node"> 924 <a name="%(NAME)s"></a> 925 %(CONTENT)s 926</div>""" 927contpl = """<tt>%(NAME)s</tt> <span class="moduletype">%(TYPE)s</span>""" 928contpl_linked = """\ 929<a target="code" href="%(URL)s" type="text/plain"><tt>%(NAME)s</tt></a> 930<span class="moduletype">%(TYPE)s</span>""" 931imports = """\ 932 <div class="import"> 933%(HEAD)s: 934 %(LINKS)s 935 </div> 936""" 937footer = """ 938 </body> 939</html>""" 940 941 942def _ast_names(names): 943 result = [] 944 for nm in names: 945 if isinstance(nm, ast.alias): 946 result.append(nm.name) 947 else: 948 result.append(nm) 949 950 result = [r for r in result if r != '__main__'] 951 return result 952 953 954def uniq(seq): 955 """Remove duplicates from a list, preserving order""" 956 # Taken from https://stackoverflow.com/questions/480214 957 seen = set() 958 seen_add = seen.add 959 return [x for x in seq if not (x in seen or seen_add(x))] 960 961 962if sys.version_info[0] == 2: 963 DEFAULT_IMPORT_LEVEL = -1 964else: 965 DEFAULT_IMPORT_LEVEL = 0 966 967 968class _Visitor(ast.NodeVisitor): 969 def __init__(self, graph, module): 970 self._graph = graph 971 self._module = module 972 self._level = DEFAULT_IMPORT_LEVEL 973 self._in_if = [False] 974 self._in_def = [False] 975 self._in_tryexcept = [False] 976 977 @property 978 def in_if(self): 979 return self._in_if[-1] 980 981 @property 982 def in_def(self): 983 return self._in_def[-1] 984 985 @property 986 def in_tryexcept(self): 987 return self._in_tryexcept[-1] 988 989 990 def _collect_import(self, name, fromlist, level): 991 if sys.version_info[0] == 2: 992 if name == '__future__' and 'absolute_import' in (fromlist or ()): 993 self._level = 0 994 995 have_star = False 996 if fromlist is not None: 997 fromlist = uniq(fromlist) 998 if '*' in fromlist: 999 fromlist.remove('*') 1000 have_star = True 1001 1002 # Record this import as originating from this module for subsequent 1003 # handling by the _process_imports() method. 1004 self._module._deferred_imports.append( 1005 (have_star, 1006 (name, self._module, fromlist, level), 1007 {'edge_attr': DependencyInfo( 1008 conditional=self.in_if, 1009 tryexcept=self.in_tryexcept, 1010 function=self.in_def, 1011 fromlist=False)})) 1012 1013 1014 def visit_Import(self, node): 1015 for nm in _ast_names(node.names): 1016 self._collect_import(nm, None, self._level) 1017 1018 def visit_ImportFrom(self, node): 1019 level = node.level if node.level != 0 else self._level 1020 self._collect_import(node.module or '', _ast_names(node.names), level) 1021 1022 def visit_If(self, node): 1023 self._in_if.append(True) 1024 self.generic_visit(node) 1025 self._in_if.pop() 1026 1027 def visit_FunctionDef(self, node): 1028 self._in_def.append(True) 1029 self.generic_visit(node) 1030 self._in_def.pop() 1031 1032 visit_AsyncFunctionDef = visit_FunctionDef 1033 1034 def visit_Try(self, node): 1035 self._in_tryexcept.append(True) 1036 self.generic_visit(node) 1037 self._in_tryexcept.pop() 1038 1039 def visit_TryExcept(self, node): 1040 self._in_tryexcept.append(True) 1041 self.generic_visit(node) 1042 self._in_tryexcept.pop() 1043 1044 def visit_Expression(self, node): 1045 # Expression node's cannot contain import statements or 1046 # other nodes that are relevant for us. 1047 pass 1048 1049 # Expression isn't actually used as such in AST trees, 1050 # therefore define visitors for all kinds of expression nodes. 1051 visit_BoolOp = visit_Expression 1052 visit_BinOp = visit_Expression 1053 visit_UnaryOp = visit_Expression 1054 visit_Lambda = visit_Expression 1055 visit_IfExp = visit_Expression 1056 visit_Dict = visit_Expression 1057 visit_Set = visit_Expression 1058 visit_ListComp = visit_Expression 1059 visit_SetComp = visit_Expression 1060 visit_ListComp = visit_Expression 1061 visit_GeneratorExp = visit_Expression 1062 visit_Compare = visit_Expression 1063 visit_Yield = visit_Expression 1064 visit_YieldFrom = visit_Expression 1065 visit_Await = visit_Expression 1066 visit_Call = visit_Expression 1067 visit_Await = visit_Expression 1068 1069 1070class ModuleGraph(ObjectGraph): 1071 """ 1072 Directed graph whose nodes represent modules and edges represent 1073 dependencies between these modules. 1074 """ 1075 1076 1077 def createNode(self, cls, name, *args, **kw): 1078 m = self.findNode(name) 1079 1080 if m is None: 1081 #assert m is None, m 1082 m = super(ModuleGraph, self).createNode(cls, name, *args, **kw) 1083 1084 return m 1085 1086 1087 def __init__(self, path=None, excludes=(), replace_paths=(), implies=(), graph=None, debug=0): 1088 super(ModuleGraph, self).__init__(graph=graph, debug=debug) 1089 if path is None: 1090 path = sys.path 1091 self.path = path 1092 self.lazynodes = {} 1093 # excludes is stronger than implies 1094 self.lazynodes.update(dict(implies)) 1095 for m in excludes: 1096 self.lazynodes[m] = None 1097 self.replace_paths = replace_paths 1098 1099 self.set_setuptools_nspackages() 1100 # Maintain own list of package path mappings in the scope of Modulegraph 1101 # object. 1102 self._package_path_map = _packagePathMap 1103 1104 def set_setuptools_nspackages(self): 1105 # This is used when running in the test-suite 1106 self.nspackages = self._calc_setuptools_nspackages() 1107 1108 def _calc_setuptools_nspackages(self): 1109 # Setuptools has some magic handling for namespace 1110 # packages when using 'install --single-version-externally-managed' 1111 # (used by system packagers and also by pip) 1112 # 1113 # When this option is used namespace packages are writting to 1114 # disk *without* an __init__.py file, which means the regular 1115 # import machinery will not find them. 1116 # 1117 # We therefore explicitly look for the hack used by 1118 # setuptools to get this kind of namespace packages to work. 1119 1120 pkgmap = {} 1121 1122 try: 1123 from pkgutil import ImpImporter 1124 except ImportError: 1125 try: 1126 from _pkgutil import ImpImporter 1127 except ImportError: 1128 ImpImporter = pkg_resources.ImpWrapper 1129 1130 if sys.version_info[:2] >= (3, 3): 1131 import importlib.machinery 1132 ImpImporter = importlib.machinery.FileFinder 1133 1134 for entry in self.path: 1135 importer = pkg_resources.get_importer(entry) 1136 1137 if isinstance(importer, ImpImporter): 1138 try: 1139 ldir = os.listdir(entry) 1140 except os.error: 1141 continue 1142 1143 for fn in ldir: 1144 if fn.endswith('-nspkg.pth'): 1145 with open(os.path.join(entry, fn), _READ_MODE) as fp: 1146 for ln in fp: 1147 for pfx in _SETUPTOOLS_NAMESPACEPKG_PTHs: 1148 if ln.startswith(pfx): 1149 try: 1150 start = len(pfx)-2 1151 stop = ln.index(')', start)+1 1152 except ValueError: 1153 continue 1154 1155 pkg = _eval_str_tuple(ln[start:stop]) 1156 identifier = ".".join(pkg) 1157 subdir = os.path.join(entry, *pkg) 1158 if os.path.exists(os.path.join(subdir, '__init__.py')): 1159 # There is a real __init__.py, 1160 # ignore the setuptools hack 1161 continue 1162 1163 if identifier in pkgmap: 1164 pkgmap[identifier].append(subdir) 1165 else: 1166 pkgmap[identifier] = [subdir] 1167 break 1168 1169 return pkgmap 1170 1171 def implyNodeReference(self, node, other, edge_data=None): 1172 """ 1173 Create a reference from the passed source node to the passed other node, 1174 implying the former to depend upon the latter. 1175 1176 While the source node _must_ be an existing graph node, the target node 1177 may be either an existing graph node _or_ a fully-qualified module name. 1178 In the latter case, the module with that name and all parent packages of 1179 that module will be imported _without_ raising exceptions and for each 1180 newly imported module or package: 1181 1182 * A new graph node will be created for that module or package. 1183 * A reference from the passed source node to that module or package will 1184 be created. 1185 1186 This method allows dependencies between Python objects _not_ importable 1187 with standard techniques (e.g., module aliases, C extensions). 1188 1189 Parameters 1190 ---------- 1191 node : str 1192 Graph node for this reference's source module or package. 1193 other : {Node, str} 1194 Either a graph node _or_ fully-qualified name for this reference's 1195 target module or package. 1196 """ 1197 1198 if isinstance(other, Node): 1199 self._updateReference(node, other, edge_data) 1200 else: 1201 if isinstance(other, tuple): 1202 raise ValueError(other) 1203 others = self._safe_import_hook(other, node, None) 1204 for other in others: 1205 self._updateReference(node, other, edge_data) 1206 1207 def getReferences(self, fromnode): 1208 """ 1209 Yield all nodes that `fromnode` dependes on (that is, 1210 all modules that `fromnode` imports. 1211 """ 1212 1213 node = self.findNode(fromnode) 1214 out_edges, _ = self.get_edges(node) 1215 return out_edges 1216 1217 def getReferers(self, tonode, collapse_missing_modules=True): 1218 node = self.findNode(tonode) 1219 _, in_edges = self.get_edges(node) 1220 1221 if collapse_missing_modules: 1222 for n in in_edges: 1223 if isinstance(n, MissingModule): 1224 for n in self.getReferers(n, False): 1225 yield n 1226 1227 else: 1228 yield n 1229 1230 else: 1231 for n in in_edges: 1232 yield n 1233 1234 def hasEdge(self, fromnode, tonode): 1235 """ Return True iff there is an edge from 'fromnode' to 'tonode' """ 1236 fromnode = self.findNode(fromnode) 1237 tonode = self.findNode(tonode) 1238 1239 return self.graph.edge_by_node(fromnode, tonode) is not None 1240 1241 def foldReferences(self, packagenode): 1242 """ 1243 Create edges to/from `packagenode` based on the edges to/from all 1244 submodules of that package _and_ then hide the graph nodes 1245 corresponding to those submodules. 1246 """ 1247 1248 pkg = self.findNode(packagenode) 1249 1250 for n in self.nodes(): 1251 if not n.identifier.startswith(pkg.identifier + '.'): 1252 continue 1253 1254 iter_out, iter_inc = self.get_edges(n) 1255 for other in iter_out: 1256 if other.identifier.startswith(pkg.identifier + '.'): 1257 continue 1258 1259 if not self.hasEdge(pkg, other): 1260 # Ignore circular dependencies 1261 self._updateReference(pkg, other, 'pkg-internal-import') 1262 1263 for other in iter_inc: 1264 if other.identifier.startswith(pkg.identifier + '.'): 1265 # Ignore circular dependencies 1266 continue 1267 1268 if not self.hasEdge(other, pkg): 1269 self._updateReference(other, pkg, 'pkg-import') 1270 1271 self.graph.hide_node(n) 1272 1273 # TODO: unfoldReferences(pkg) that restore the submodule nodes and 1274 # removes 'pkg-import' and 'pkg-internal-import' edges. Care should 1275 # be taken to ensure that references are correct if multiple packages 1276 # are folded and then one of them in unfolded 1277 1278 def _updateReference(self, fromnode, tonode, edge_data): 1279 try: 1280 ed = self.edgeData(fromnode, tonode) 1281 except (KeyError, GraphError): # XXX: Why 'GraphError' 1282 return self.createReference(fromnode, tonode, edge_data) 1283 1284 if not (isinstance(ed, DependencyInfo) and isinstance(edge_data, DependencyInfo)): 1285 self.updateEdgeData(fromnode, tonode, edge_data) 1286 else: 1287 self.updateEdgeData(fromnode, tonode, ed._merged(edge_data)) 1288 1289 def createReference(self, fromnode, tonode, edge_data='direct'): 1290 """ 1291 Create a reference from fromnode to tonode 1292 """ 1293 return super(ModuleGraph, self).createReference(fromnode, tonode, edge_data=edge_data) 1294 1295 def findNode(self, name, create_nspkg=True): 1296 """ 1297 Graph node uniquely identified by the passed fully-qualified module 1298 name if this module has been added to the graph _or_ `None` otherwise. 1299 1300 If (in order): 1301 1302 . A namespace package with this identifier exists _and_ the passed 1303 `create_nspkg` parameter is `True`, this package will be 1304 instantiated and returned. 1305 . A lazy node with this identifier and: 1306 * No dependencies exists, this node will be instantiated and 1307 returned. 1308 * Dependencies exists, this node and all transitive dependencies of 1309 this node be instantiated and this node returned. 1310 . A non-lazy node with this identifier exists, this node will be 1311 returned as is. 1312 1313 Parameters 1314 ---------- 1315 name : str 1316 Fully-qualified name of the module whose graph node is to be found. 1317 create_nspkg : bool 1318 Whether or not to implicitly instantiate namespace packages. If 1319 `True` _and_ this name is that of a previously registered namespace 1320 package (i.e., in `self.nspackages`) not already added to the 1321 graph, this package will be added to the graph. Defaults to `True`. 1322 1323 Returns 1324 ---------- 1325 Node 1326 Graph node of this module if added to the graph _or_ `None` 1327 otherwise. 1328 """ 1329 1330 data = super(ModuleGraph, self).findNode(name) 1331 1332 if data is not None: 1333 return data 1334 1335 if name in self.lazynodes: 1336 deps = self.lazynodes.pop(name) 1337 1338 if deps is None: 1339 # excluded module 1340 m = self.createNode(ExcludedModule, name) 1341 elif isinstance(deps, Alias): 1342 other = self._safe_import_hook(deps, None, None).pop() 1343 m = self.createNode(AliasNode, name, other) 1344 self.implyNodeReference(m, other) 1345 else: 1346 m = self._safe_import_hook(name, None, None).pop() 1347 for dep in deps: 1348 self.implyNodeReference(m, dep) 1349 1350 return m 1351 1352 if name in self.nspackages and create_nspkg: 1353 # name is a --single-version-externally-managed 1354 # namespace package (setuptools/distribute) 1355 pathnames = self.nspackages.pop(name) 1356 m = self.createNode(NamespacePackage, name) 1357 1358 # FIXME: The filename must be set to a string to ensure that py2app 1359 # works, it is not clear yet why that is. Setting to None would be 1360 # cleaner. 1361 m.filename = '-' 1362 m.packagepath = _namespace_package_path(name, pathnames, self.path) 1363 1364 # As per comment at top of file, simulate runtime packagepath additions. 1365 m.packagepath = m.packagepath + self._package_path_map.get(name, []) 1366 return m 1367 1368 return None 1369 1370 1371 def run_script(self, pathname, caller=None): 1372 """ 1373 Create a node by path (not module name). It is expected to be a Python 1374 source file, and will be scanned for dependencies. 1375 """ 1376 self.msg(2, "run_script", pathname) 1377 1378 pathname = os.path.realpath(pathname) 1379 m = self.findNode(pathname) 1380 if m is not None: 1381 return m 1382 1383 if sys.version_info[0] != 2: 1384 with open(pathname, 'rb') as fp: 1385 encoding = util.guess_encoding(fp) 1386 1387 with open(pathname, _READ_MODE, encoding=encoding) as fp: 1388 contents = fp.read() + '\n' 1389 if contents.startswith(BOM): 1390 # Ignore BOM at start of input 1391 contents = contents[1:] 1392 1393 else: 1394 with open(pathname, _READ_MODE) as fp: 1395 contents = fp.read() + '\n' 1396 1397 co_ast = compile(contents, pathname, 'exec', ast.PyCF_ONLY_AST, True) 1398 co = compile(co_ast, pathname, 'exec', 0, True) 1399 m = self.createNode(Script, pathname) 1400 self._updateReference(caller, m, None) 1401 self._scan_code(m, co, co_ast) 1402 m.code = co 1403 if self.replace_paths: 1404 m.code = self._replace_paths_in_code(m.code) 1405 return m 1406 1407 1408 #FIXME: For safety, the "source_module" parameter should default to the 1409 #root node of the current graph if unpassed. This parameter currently 1410 #defaults to None, thus disconnected modules imported in this manner (e.g., 1411 #hidden imports imported by depend.analysis.initialize_modgraph()). 1412 def import_hook( 1413 self, 1414 target_module_partname, 1415 source_module=None, 1416 target_attr_names=None, 1417 level=DEFAULT_IMPORT_LEVEL, 1418 edge_attr=None, 1419 ): 1420 """ 1421 Import the module with the passed name, all parent packages of this 1422 module, _and_ all submodules and attributes in this module with the 1423 passed names from the previously imported caller module signified by 1424 the passed graph node. 1425 1426 Unlike most import methods (e.g., `_safe_import_hook()`), this method 1427 is designed to be publicly called by both external and internal 1428 callers and hence is public. 1429 1430 Parameters 1431 ---------- 1432 target_module_partname : str 1433 Partially-qualified name of the target module to be imported. See 1434 `_safe_import_hook()` for further details. 1435 source_module : Node 1436 Graph node for the previously imported **source module** (i.e., 1437 module containing the `import` statement triggering the call to 1438 this method) _or_ `None` if this module is to be imported in a 1439 "disconnected" manner. **Passing `None` is _not_ recommended.** 1440 Doing so produces a disconnected graph in which the graph node 1441 created for the module to be imported will be disconnected and 1442 hence unreachable from all other nodes -- which frequently causes 1443 subtle issues in external callers (namely PyInstaller, which 1444 silently ignores unreachable nodes). 1445 target_attr_names : list 1446 List of the unqualified names of all submodules and attributes to 1447 be imported from the module to be imported if this is a "from"- 1448 style import (e.g., `[encode_base64, encode_noop]` for the import 1449 `from email.encoders import encode_base64, encode_noop`) _or_ 1450 `None` otherwise. 1451 level : int 1452 Whether to perform an absolute or relative import. See 1453 `_safe_import_hook()` for further details. 1454 1455 Returns 1456 ---------- 1457 list 1458 List of the graph nodes created for all modules explicitly imported 1459 by this call, including the passed module and all submodules listed 1460 in `target_attr_names` _but_ excluding all parent packages 1461 implicitly imported by this call. If `target_attr_names` is `None` 1462 or the empty list, this is guaranteed to be a list of one element: 1463 the graph node created for the passed module. 1464 1465 Raises 1466 ---------- 1467 ImportError 1468 If the target module to be imported is unimportable. 1469 """ 1470 self.msg(3, "_import_hook", target_module_partname, source_module, source_module, level) 1471 1472 source_package = self._determine_parent(source_module) 1473 target_package, target_module_partname = self._find_head_package( 1474 source_package, target_module_partname, level) 1475 target_module = self._load_tail(target_package, target_module_partname) 1476 target_modules = [target_module] 1477 1478 # If this is a "from"-style import *AND* this target module is 1479 # actually a package, import all submodules of this package specified 1480 # by the "import" half of this import (e.g., the submodules "bar" and 1481 # "car" of the target package "foo" in "from foo import bar, car"). 1482 # 1483 # If this target module is a non-package, it could still contain 1484 # importable submodules (e.g., the non-package `os` module containing 1485 # the `os.path` submodule). In this case, these submodules are already 1486 # imported by this target module's pure-Python code. Since our import 1487 # scanner already detects such imports, these submodules need *NOT* be 1488 # reimported here. 1489 if target_attr_names and isinstance(target_module, Package): 1490 for target_submodule in self._import_importable_package_submodules( 1491 target_module, target_attr_names): 1492 if target_submodule not in target_modules: 1493 target_modules.append(target_submodule) 1494 1495 # Add an edge from this source module to each target module. 1496 for target_module in target_modules: 1497 self._updateReference( 1498 source_module, target_module, edge_data=edge_attr) 1499 1500 return target_modules 1501 1502 1503 def _determine_parent(self, caller): 1504 """ 1505 Determine the package containing a node. 1506 """ 1507 self.msgin(4, "determine_parent", caller) 1508 1509 parent = None 1510 if caller: 1511 pname = caller.identifier 1512 1513 if isinstance(caller, Package): 1514 parent = caller 1515 1516 elif '.' in pname: 1517 pname = pname[:pname.rfind('.')] 1518 parent = self.findNode(pname) 1519 1520 elif caller.packagepath: 1521 # XXX: I have no idea why this line 1522 # is necessary. 1523 parent = self.findNode(pname) 1524 1525 self.msgout(4, "determine_parent ->", parent) 1526 return parent 1527 1528 1529 def _find_head_package( 1530 self, 1531 source_package, 1532 target_module_partname, 1533 level=DEFAULT_IMPORT_LEVEL): 1534 """ 1535 Import the target package providing the target module with the passed 1536 name to be subsequently imported from the previously imported source 1537 package corresponding to the passed graph node. 1538 1539 Parameters 1540 ---------- 1541 source_package : Package 1542 Graph node for the previously imported **source package** (i.e., 1543 package containing the module containing the `import` statement 1544 triggering the call to this method) _or_ `None` if this module is 1545 to be imported in a "disconnected" manner. **Passing `None` is 1546 _not_ recommended.** See the `_import_hook()` method for further 1547 details. 1548 target_module_partname : str 1549 Partially-qualified name of the target module to be imported. See 1550 `_safe_import_hook()` for further details. 1551 level : int 1552 Whether to perform absolute or relative imports. See the 1553 `_safe_import_hook()` method for further details. 1554 1555 Returns 1556 ---------- 1557 (target_package, target_module_tailname) 1558 2-tuple describing the imported target package, where: 1559 * `target_package` is the graph node created for this package. 1560 * `target_module_tailname` is the unqualified name of the target 1561 module to be subsequently imported (e.g., `text` when passed a 1562 `target_module_partname` of `email.mime.text`). 1563 1564 Raises 1565 ---------- 1566 ImportError 1567 If the package to be imported is unimportable. 1568 """ 1569 self.msgin(4, "find_head_package", source_package, target_module_partname, level) 1570 1571 #FIXME: Rename all local variable names to something sensible. No, 1572 #"p_fqdn" is not a sensible name. 1573 1574 # If this target module is a submodule... 1575 if '.' in target_module_partname: 1576 target_module_headname, target_module_tailname = ( 1577 target_module_partname.split('.', 1)) 1578 # Else, this target module is a top-level module. 1579 else: 1580 target_module_headname = target_module_partname 1581 target_module_tailname = '' 1582 1583 # If attempting both absolute and relative imports... 1584 if level == ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL: 1585 if source_package: 1586 target_package_name = source_package.identifier + '.' + target_module_headname 1587 else: 1588 target_package_name = target_module_headname 1589 # Else if attempting only absolute imports... 1590 elif level == ABSOLUTE_IMPORT_LEVEL: 1591 target_package_name = target_module_headname 1592 1593 # Absolute import, ignore the parent 1594 source_package = None 1595 # Else if attempting only relative imports... 1596 else: 1597 if source_package is None: 1598 self.msg(2, "Relative import outside of package") 1599 raise InvalidRelativeImportError( 1600 "Relative import outside of package (name=%r, parent=%r, level=%r)" % ( 1601 target_module_partname, source_package, level)) 1602 1603 for i in range(level - 1): 1604 if '.' not in source_package.identifier: 1605 self.msg(2, "Relative import outside of package") 1606 raise InvalidRelativeImportError( 1607 "Relative import outside of package (name=%r, parent=%r, level=%r)" % ( 1608 target_module_partname, source_package, level)) 1609 1610 p_fqdn = source_package.identifier.rsplit('.', 1)[0] 1611 new_parent = self.findNode(p_fqdn) 1612 if new_parent is None: 1613 #FIXME: Repetition detected. Exterminate. Exterminate. 1614 self.msg(2, "Relative import outside of package") 1615 raise InvalidRelativeImportError( 1616 "Relative import outside of package (name=%r, parent=%r, level=%r)" % ( 1617 target_module_partname, source_package, level)) 1618 1619 assert new_parent is not source_package, ( 1620 new_parent, source_package) 1621 source_package = new_parent 1622 1623 if target_module_headname: 1624 target_package_name = ( 1625 source_package.identifier + '.' + target_module_headname) 1626 else: 1627 target_package_name = source_package.identifier 1628 1629 # Graph node of this target package. 1630 target_package = self._safe_import_module( 1631 target_module_headname, target_package_name, source_package) 1632 1633 #FIXME: Why exactly is this necessary again? This doesn't quite seem 1634 #right but maybe it is. Shouldn't absolute imports only be performed if 1635 #the passed "level" is either "ABSOLUTE_IMPORT_LEVEL" or 1636 #"ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL" -- or, more succinctly: 1637 # 1638 # if level < 1: 1639 1640 # If this target package is *NOT* importable and a source package was 1641 # passed, attempt to import this target package as an absolute import. 1642 if target_package is None and source_package is not None: 1643 target_package_name = target_module_headname 1644 source_package = None 1645 1646 # Graph node for the target package, again. 1647 target_package = self._safe_import_module( 1648 target_module_headname, target_package_name, source_package) 1649 1650 # If this target package is importable, return this package. 1651 if target_package is not None: 1652 self.msgout(4, "find_head_package ->", (target_package, target_module_tailname)) 1653 return target_package, target_module_tailname 1654 1655 # Else, raise an exception. 1656 self.msgout(4, "raise ImportError: No module named", target_package_name) 1657 raise ImportError("No module named " + target_package_name) 1658 1659 1660 def _load_tail(self, package, submodule_name): 1661 """ 1662 Import the submodule with the passed name and all parent packages of 1663 this module from the previously imported parent package corresponding 1664 to the passed graph node. 1665 1666 Parameters 1667 ---------- 1668 package : Package 1669 Graph node of the previously imported package containing this 1670 submodule. 1671 submodule_name : str 1672 Name of the submodule to be imported in either qualified (e.g., 1673 `email.mime.base`) or unqualified (e.g., `base`) form. 1674 1675 Returns 1676 ---------- 1677 Node 1678 Graph node created for this submodule. 1679 1680 Raises 1681 ---------- 1682 ImportError 1683 If this submodule is unimportable. 1684 """ 1685 self.msgin(4, "load_tail", package, submodule_name) 1686 1687 submodule = package 1688 while submodule_name: 1689 i = submodule_name.find('.') 1690 if i < 0: 1691 i = len(submodule_name) 1692 head, submodule_name = submodule_name[:i], submodule_name[i+1:] 1693 mname = "%s.%s" % (submodule.identifier, head) 1694 submodule = self._safe_import_module(head, mname, submodule) 1695 1696 if submodule is None: 1697 # FIXME: Why do we no longer return a MissingModule instance? 1698 # result = self.createNode(MissingModule, mname) 1699 self.msgout(4, "raise ImportError: No module named", mname) 1700 raise ImportError("No module named " + repr(mname)) 1701 1702 self.msgout(4, "load_tail ->", submodule) 1703 return submodule 1704 1705 1706 #FIXME: Refactor from a generator yielding graph nodes into a non-generator 1707 #returning a list or tuple of all yielded graph nodes. This method is only 1708 #called once above and the return value of that call is only iterated over 1709 #as a list or tuple. There's no demonstrable reason for this to be a 1710 #generator. Generators are great for their intended purposes (e.g., as 1711 #continuations). This isn't one of those purposes. 1712 def _import_importable_package_submodules(self, package, attr_names): 1713 """ 1714 Generator importing and yielding each importable submodule (of the 1715 previously imported package corresponding to the passed graph node) 1716 whose unqualified name is in the passed list. 1717 1718 Elements of this list that are _not_ importable submodules of this 1719 package are either: 1720 1721 * Ignorable attributes (e.g., classes, globals) defined at the top 1722 level of this package's `__init__` submodule, which will be ignored. 1723 * Else, unignorable unimportable submodules, in which case an 1724 exception is raised. 1725 1726 Parameters 1727 ---------- 1728 package : Package 1729 Graph node of the previously imported package containing the 1730 modules to be imported and yielded. 1731 1732 attr_names : list 1733 List of the unqualified names of all attributes of this package to 1734 attempt to import as submodules. This list will be internally 1735 converted into a set, safely ignoring any duplicates in this list 1736 (e.g., reducing the "from"-style import 1737 `from foo import bar, car, far, bar, car, far` to merely 1738 `from foo import bar, car, far`). 1739 1740 Yields 1741 ---------- 1742 Node 1743 Graph node created for the currently imported submodule. 1744 1745 Raises 1746 ---------- 1747 ImportError 1748 If any attribute whose name is in `attr_names` is neither: 1749 * An importable submodule of this package. 1750 * An ignorable global attribute (e.g., class, variable) defined at 1751 the top level of this package's `__init__` submodule. 1752 In this case, this attribute _must_ be an unimportable submodule of 1753 this package. 1754 """ 1755 1756 # Ignore duplicate submodule names in the passed list. 1757 attr_names = set(attr_names) 1758 self.msgin(4, "_import_importable_package_submodules", package, attr_names) 1759 1760 #FIXME: This test *SHOULD* be superfluous and hence safely removable. 1761 #The higher-level _scan_bytecode() and _collect_import() methods 1762 #already guarantee "*" characters to be removed from fromlists. 1763 if '*' in attr_names: 1764 attr_names.update(self._find_all_submodules(package)) 1765 attr_names.remove('*') 1766 1767 # self.msg(4, '_import_importable_package_submodules (global attrs)', package.identifier, package._global_attr_names) 1768 1769 # For the name of each attribute to be imported from this package... 1770 for attr_name in attr_names: 1771 # self.msg(4, '_import_importable_package_submodules (fromlist attr)', package.identifier, attr_name) 1772 1773 # Graph node of this attribute if this attribute is a previously 1774 # imported module or None otherwise. 1775 submodule = package.get_submodule_or_none(attr_name) 1776 1777 # If this attribute is *NOT* a previously imported module, attempt 1778 # to import this attribute as a submodule of this package. 1779 if submodule is None: 1780 # Fully-qualified name of this submodule. 1781 submodule_name = package.identifier + '.' + attr_name 1782 1783 # Graph node of this submodule if importable or None otherwise. 1784 submodule = self._safe_import_module( 1785 attr_name, submodule_name, package) 1786 1787 # If this submodule is unimportable... 1788 if submodule is None: 1789 # If this attribute is a global (e.g., class, variable) 1790 # defined at the top level of this package's "__init__" 1791 # submodule, this importation is safely ignorable. Do so 1792 # and skip to the next attribute. 1793 # 1794 # This behaviour is non-conformant with Python behaviour, 1795 # which is bad, but is required to sanely handle all 1796 # possible edge cases, which is good. In Python, a global 1797 # attribute defined at the top level of a package's 1798 # "__init__" submodule shadows a submodule of the same name 1799 # in that package. Attempting to import that submodule 1800 # instead imports that attribute; thus, that submodule is 1801 # effectively unimportable. In this method and elsewhere, 1802 # that submodule is tested for first and hence shadows that 1803 # attribute -- the opposite logic. Attempts to import that 1804 # attribute are mistakenly seen as attempts to import that 1805 # submodule! Why? 1806 # 1807 # Edge cases. PyInstaller (and by extension ModuleGraph) 1808 # only cares about module imports. Global attribute imports 1809 # are parsed only as the means to this ends and are 1810 # otherwise ignorable. The cost of erroneously shadowing: 1811 # 1812 # * Submodules by attributes is significant. Doing so 1813 # prevents such submodules from being frozen and hence 1814 # imported at application runtime. 1815 # * Attributes by submodules is insignificant. Doing so 1816 # could erroneously freeze such submodules despite their 1817 # never being imported at application runtime. However, 1818 # ModuleGraph is incapable of determining with certainty 1819 # that Python logic in another module other than the 1820 # "__init__" submodule containing these attributes does 1821 # *NOT* delete these attributes and hence unshadow these 1822 # submodules, which would then become importable at 1823 # runtime and require freezing. Hence, ModuleGraph *MUST* 1824 # permissively assume submodules of the same name as 1825 # attributes to be unshadowed elsewhere and require 1826 # freezing -- even if they do not. 1827 # 1828 # It is practically difficult (albeit technically feasible) 1829 # for ModuleGraph to determine whether or not the target 1830 # attribute names of "from"-style import statements (e.g., 1831 # "bar" and "car" in "from foo import bar, car") refer to 1832 # non-ignorable submodules or ignorable non-module globals 1833 # during opcode scanning. Distinguishing these two cases 1834 # during opcode scanning would require a costly call to the 1835 # _find_module() method, which would subsequently be 1836 # repeated during import-graph construction. This could be 1837 # ameliorated with caching, which itself would require 1838 # costly space consumption and developer time. 1839 # 1840 # Since opcode scanning fails to distinguish these two 1841 # cases, this and other methods subsequently called at 1842 # import-graph construction time (e.g., 1843 # _safe_import_hook()) must do so. Since submodules of the 1844 # same name as attributes must assume to be unshadowed 1845 # elsewhere and require freezing, the only solution is to 1846 # attempt to import an attribute as a non-ignorable module 1847 # *BEFORE* assuming an attribute to be an ignorable 1848 # non-module. Which is what this and other methods do. 1849 # 1850 # See Package.is_global_attr() for similar discussion. 1851 if package.is_global_attr(attr_name): 1852 self.msg(4, '_import_importable_package_submodules: ignoring from-imported global', package.identifier, attr_name) 1853 continue 1854 # Else, this attribute is an unimportable submodule. Since 1855 # this is *NOT* safely ignorable, raise an exception. 1856 else: 1857 raise ImportError("No module named " + submodule_name) 1858 1859 # Yield this submodule's graph node to the caller. 1860 yield submodule 1861 1862 self.msgin(4, "_import_importable_package_submodules ->") 1863 1864 1865 def _find_all_submodules(self, m): 1866 if not m.packagepath: 1867 return 1868 # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. 1869 # But we must also collect Python extension modules - although 1870 # we cannot separate normal dlls from Python extensions. 1871 for path in m.packagepath: 1872 try: 1873 names = zipio.listdir(path) 1874 except (os.error, IOError): 1875 self.msg(2, "can't list directory", path) 1876 continue 1877 for info in (moduleInfoForPath(p) for p in names): 1878 if info is None: 1879 continue 1880 if info[0] != '__init__': 1881 yield info[0] 1882 1883 1884 def alias_module(self, src_module_name, trg_module_name): 1885 """ 1886 Alias the source module to the target module with the passed names. 1887 1888 This method ensures that the next call to findNode() given the target 1889 module name will resolve this alias. This includes importing and adding 1890 a graph node for the source module if needed as well as adding a 1891 reference from the target to source module. 1892 1893 Parameters 1894 ---------- 1895 src_module_name : str 1896 Fully-qualified name of the existing **source module** (i.e., the 1897 module being aliased). 1898 trg_module_name : str 1899 Fully-qualified name of the non-existent **target module** (i.e., 1900 the alias to be created). 1901 """ 1902 self.msg(3, 'alias_module "%s" -> "%s"' % (src_module_name, trg_module_name)) 1903 # print('alias_module "%s" -> "%s"' % (src_module_name, trg_module_name)) 1904 assert isinstance(src_module_name, str), '"%s" not a module name.' % str(src_module_name) 1905 assert isinstance(trg_module_name, str), '"%s" not a module name.' % str(trg_module_name) 1906 1907 # If the target module has already been added to the graph as either a 1908 # non-alias or as a different alias, raise an exception. 1909 trg_module = self.findNode(trg_module_name) 1910 if trg_module is not None and not ( 1911 isinstance(trg_module, AliasNode) and 1912 trg_module.identifier == src_module_name): 1913 raise ValueError( 1914 'Target module "%s" already imported as "%s".' % ( 1915 trg_module_name, trg_module)) 1916 1917 # See findNode() for details. 1918 self.lazynodes[trg_module_name] = Alias(src_module_name) 1919 1920 1921 def add_module(self, module): 1922 """ 1923 Add the passed module node to the graph if not already added. 1924 1925 If that module has a parent module or package with a previously added 1926 node, this method also adds a reference from this module node to its 1927 parent node and adds this module node to its parent node's namespace. 1928 1929 This high-level method wraps the low-level `addNode()` method, but is 1930 typically _only_ called by graph hooks adding runtime module nodes. For 1931 all other node types, the `import_module()` method should be called. 1932 1933 Parameters 1934 ---------- 1935 module : BaseModule 1936 Graph node of the module to be added. 1937 """ 1938 self.msg(3, 'add_module', module) 1939 1940 # If no node exists for this module, add such a node. 1941 module_added = self.findNode(module.identifier) 1942 if module_added is None: 1943 self.addNode(module) 1944 else: 1945 assert module == module_added, 'New module %r != previous %r.' % (module, module_added) 1946 1947 # If this module has a previously added parent, reference this module to 1948 # its parent and add this module to its parent's namespace. 1949 parent_name, _, module_basename = module.identifier.rpartition('.') 1950 if parent_name: 1951 parent = self.findNode(parent_name) 1952 if parent is None: 1953 self.msg(4, 'add_module parent not found:', parent_name) 1954 else: 1955 self.createReference(module, parent) 1956 parent.add_submodule(module_basename, module) 1957 1958 1959 def append_package_path(self, package_name, directory): 1960 """ 1961 Modulegraph does a good job at simulating Python's, but it can not 1962 handle packagepath '__path__' modifications packages make at runtime. 1963 1964 Therefore there is a mechanism whereby you can register extra paths 1965 in this map for a package, and it will be honored. 1966 1967 NOTE: This method has to be called before a package is resolved by 1968 modulegraph. 1969 1970 Parameters 1971 ---------- 1972 module : str 1973 Fully-qualified module name. 1974 directory : str 1975 Absolute or relative path of the directory to append to the 1976 '__path__' attribute. 1977 """ 1978 1979 paths = self._package_path_map.setdefault(package_name, []) 1980 paths.append(directory) 1981 1982 1983 def _safe_import_module( 1984 self, module_partname, module_name, parent_module): 1985 """ 1986 Create a new graph node for the module with the passed name under the 1987 parent package signified by the passed graph node _without_ raising 1988 `ImportError` exceptions. 1989 1990 If this module has already been imported, this module's existing graph 1991 node will be returned; else if this module is importable, a new graph 1992 node will be added for this module and returned; else this module is 1993 unimportable, in which case `None` will be returned. Like the 1994 `_safe_import_hook()` method, this method does _not_ raise 1995 `ImportError` exceptions when this module is unimportable. 1996 1997 Parameters 1998 ---------- 1999 module_partname : str 2000 Unqualified name of the module to be imported (e.g., `text`). 2001 module_name : str 2002 Fully-qualified name of this module (e.g., `email.mime.text`). 2003 parent_module : Package 2004 Graph node of the previously imported parent module containing this 2005 submodule _or_ `None` if this is a top-level module (i.e., 2006 `module_name` contains no `.` delimiters). This parent module is 2007 typically but _not_ always a package (e.g., the `os.path` submodule 2008 contained by the `os` module). 2009 2010 Returns 2011 ---------- 2012 Node 2013 Graph node created for this module _or_ `None` if this module is 2014 unimportable. 2015 """ 2016 self.msgin(3, "safe_import_module", module_partname, module_name, parent_module) 2017 2018 # If this module has *NOT* already been imported, do so. 2019 module = self.findNode(module_name) 2020 if module is None: 2021 # List of the absolute paths of all directories to be searched for 2022 # this module. This effectively defaults to "sys.path". 2023 search_dirs = None 2024 2025 # Open file handle providing the physical contents of this module. 2026 file_handle = None 2027 2028 # If this module has a parent package... 2029 if parent_module is not None: 2030 # ...with a list of the absolute paths of all directories 2031 # comprising this package, prefer that to "sys.path". 2032 if parent_module.packagepath is not None: 2033 search_dirs = parent_module.packagepath 2034 # Else, something is horribly wrong. Return emptiness. 2035 else: 2036 self.msgout(3, "safe_import_module -> None (parent_parent.packagepath is None)") 2037 return None 2038 2039 try: 2040 try: 2041 file_handle, pathname, metadata = self._find_module( 2042 module_partname, search_dirs, parent_module) 2043 except ImportError as exc: 2044 self.msgout(3, "safe_import_module -> None (%r)" % exc) 2045 return None 2046 2047 module = self._load_module( 2048 module_name, file_handle, pathname, metadata) 2049 finally: 2050 if file_handle is not None: 2051 file_handle.close() 2052 2053 # If this is a submodule rather than top-level module... 2054 if parent_module is not None: 2055 self.msg(4, "safe_import_module create reference", module, "->", parent_module) 2056 2057 # Add an edge from this submodule to its parent module. 2058 self._updateReference( 2059 module, parent_module, edge_data=DependencyInfo( 2060 conditional=False, 2061 fromlist=False, 2062 function=False, 2063 tryexcept=False, 2064 )) 2065 2066 # Add this submodule to its parent module. 2067 parent_module.add_submodule(module_partname, module) 2068 2069 # Return this module. 2070 self.msgout(3, "safe_import_module ->", module) 2071 return module 2072 2073 2074 #FIXME: For clarity, rename method parameters to: 2075 # def _load_module(self, module_name, file_handle, pathname, imp_info): 2076 def _load_module(self, fqname, fp, pathname, info): 2077 suffix, mode, typ = info 2078 self.msgin(2, "load_module", fqname, fp and "fp", pathname) 2079 2080 if typ == imp.PKG_DIRECTORY: 2081 if isinstance(mode, (list, tuple)): 2082 packagepath = mode 2083 else: 2084 packagepath = [] 2085 2086 m = self._load_package(fqname, pathname, packagepath) 2087 self.msgout(2, "load_module ->", m) 2088 return m 2089 2090 if typ == imp.PY_SOURCE: 2091 contents = fp.read() 2092 if isinstance(contents, bytes): 2093 contents += b'\n' 2094 else: 2095 contents += '\n' 2096 2097 try: 2098 co = compile(contents, pathname, 'exec', ast.PyCF_ONLY_AST, True) 2099 if sys.version_info[:2] == (3, 5): 2100 # In Python 3.5 some syntax problems with async 2101 # functions are only reported when compiling to bytecode 2102 compile(co, '-', 'exec', 0, True) 2103 except SyntaxError: 2104 co = None 2105 cls = InvalidSourceModule 2106 self.msg(2, "load_module: InvalidSourceModule", pathname) 2107 2108 else: 2109 cls = SourceModule 2110 2111 elif typ == imp.PY_COMPILED: 2112 data = fp.read(4) 2113 magic = imp.get_magic() 2114 if data != magic: 2115 self.msg(2, "load_module: InvalidCompiledModule, " 2116 "bad magic number", pathname, data, magic) 2117 co = None 2118 cls = InvalidCompiledModule 2119 else: 2120 if sys.version_info >= (3, 7): 2121 fp.read(12) 2122 elif sys.version_info >= (3, 4): 2123 fp.read(8) 2124 else: 2125 fp.read(4) 2126 try: 2127 co = marshal.loads(fp.read()) 2128 cls = CompiledModule 2129 except Exception as exc: 2130 self.msg(2, "load_module: InvalidCompiledModule, " 2131 "Cannot load code", pathname, exc) 2132 co = None 2133 cls = InvalidCompiledModule 2134 2135 elif typ == imp.C_BUILTIN: 2136 cls = BuiltinModule 2137 co = None 2138 2139 else: 2140 cls = Extension 2141 co = None 2142 2143 m = self.createNode(cls, fqname) 2144 m.filename = pathname 2145 if co is not None: 2146 try: 2147 if isinstance(co, ast.AST): 2148 co_ast = co 2149 co = compile(co_ast, pathname, 'exec', 0, True) 2150 else: 2151 co_ast = None 2152 self._scan_code(m, co, co_ast) 2153 2154 if self.replace_paths: 2155 co = self._replace_paths_in_code(co) 2156 m.code = co 2157 except SyntaxError: 2158 self.msg(1, "load_module: SyntaxError in ", pathname) 2159 cls = InvalidSourceModule 2160 m = self.createNode(cls, fqname) 2161 2162 self.msgout(2, "load_module ->", m) 2163 return m 2164 2165 2166 def _safe_import_hook( 2167 self, target_module_partname, source_module, target_attr_names, 2168 level=DEFAULT_IMPORT_LEVEL, edge_attr=None): 2169 """ 2170 Import the module with the passed name and all parent packages of this 2171 module from the previously imported caller module signified by the 2172 passed graph node _without_ raising `ImportError` exceptions. 2173 2174 This method wraps the lowel-level `_import_hook()` method. On catching 2175 an `ImportError` exception raised by that method, this method creates 2176 and adds a `MissingNode` instance describing the unimportable module to 2177 the graph instead. 2178 2179 Parameters 2180 ---------- 2181 target_module_partname : str 2182 Partially-qualified name of the module to be imported. If `level` 2183 is: 2184 * `ABSOLUTE_OR_RELATIVE_IMPORT_LEVEL` (e.g., the Python 2 default) 2185 or a positive integer (e.g., an explicit relative import), the 2186 fully-qualified name of this module is the concatenation of the 2187 fully-qualified name of the caller module's package and this 2188 parameter. 2189 * `ABSOLUTE_IMPORT_LEVEL` (e.g., the Python 3 default), this name 2190 is already fully-qualified. 2191 * A non-negative integer (e.g., `1`), this name is typically the 2192 empty string. In this case, this is a "from"-style relative 2193 import (e.g., "from . import bar") and the fully-qualified name 2194 of this module is dynamically resolved by import machinery. 2195 source_module : Node 2196 Graph node for the previously imported **caller module** (i.e., 2197 module containing the `import` statement triggering the call to 2198 this method) _or_ `None` if this module is to be imported in a 2199 "disconnected" manner. **Passing `None` is _not_ recommended.** 2200 Doing so produces a disconnected graph in which the graph node 2201 created for the module to be imported will be disconnected and 2202 hence unreachable from all other nodes -- which frequently causes 2203 subtle issues in external callers (e.g., PyInstaller, which 2204 silently ignores unreachable nodes). 2205 target_attr_names : list 2206 List of the unqualified names of all submodules and attributes to 2207 be imported via a `from`-style import statement from this target 2208 module if any (e.g., the list `[encode_base64, encode_noop]` for 2209 the import `from email.encoders import encode_base64, encode_noop`) 2210 _or_ `None` otherwise. Ignored unless `source_module` is the graph 2211 node of a package (i.e., is an instance of the `Package` class). 2212 Why? Because: 2213 * Consistency. The `_import_importable_package_submodules()` 2214 method accepts a similar list applicable only to packages. 2215 * Efficiency. Unlike packages, modules cannot physically contain 2216 submodules. Hence, any target module imported via a `from`-style 2217 import statement as an attribute from another target parent 2218 module must itself have been imported in that target parent 2219 module. The import statement responsible for that import must 2220 already have been previously parsed by `ModuleGraph`, in which 2221 case that target module will already be frozen by PyInstaller. 2222 These imports are safely ignorable here. 2223 level : int 2224 Whether to perform an absolute or relative import. This parameter 2225 corresponds exactly to the parameter of the same name accepted by 2226 the `__import__()` built-in: "The default is -1 which indicates 2227 both absolute and relative imports will be attempted. 0 means only 2228 perform absolute imports. Positive values for level indicate the 2229 number of parent directories to search relative to the directory of 2230 the module calling `__import__()`." Defaults to -1 under Python 2 2231 and 0 under Python 3. Since this default depends on the major 2232 version of the current Python interpreter, depending on this 2233 default can result in unpredictable and non-portable behaviour. 2234 Callers are strongly recommended to explicitly pass this parameter 2235 rather than implicitly accept this default. 2236 2237 Returns 2238 ---------- 2239 list 2240 List of the graph nodes created for all modules explicitly imported 2241 by this call, including the passed module and all submodules listed 2242 in `target_attr_names` _but_ excluding all parent packages 2243 implicitly imported by this call. If `target_attr_names` is either 2244 `None` or the empty list, this is guaranteed to be a list of one 2245 element: the graph node created for the passed module. As above, 2246 `MissingNode` instances are created for all unimportable modules. 2247 """ 2248 self.msg(3, "_safe_import_hook", target_module_partname, source_module, target_attr_names, level) 2249 2250 def is_swig_candidate(): 2251 return (source_module is not None and 2252 target_attr_names is None and 2253 level == ABSOLUTE_IMPORT_LEVEL and 2254 type(source_module) is SourceModule and 2255 target_module_partname == 2256 '_' + source_module.identifier.rpartition('.')[2] and 2257 sys.version_info[0] == 3) 2258 2259 def is_swig_wrapper(source_module): 2260 # TODO Define a new function util.open_text_file() performing 2261 # this logic, which is repeated numerous times in this module. 2262 # FIXME: Actually, can't we just use the new compat.open() 2263 # function to reliably open text files in a portable manner? 2264 with open(source_module.filename, 'rb') as source_module_file: 2265 encoding = util.guess_encoding(source_module_file) 2266 with open(source_module.filename, _READ_MODE, encoding=encoding) \ 2267 as source_module_file: 2268 first_line = source_module_file.readline() 2269 self.msg(5, 'SWIG wrapper candidate first line: %r' % (first_line)) 2270 return "automatically generated by SWIG" in first_line 2271 2272 2273 # List of the graph nodes created for all target modules both 2274 # imported by and returned from this call, whose: 2275 # 2276 # * First element is the graph node for the core target module 2277 # specified by the "target_module_partname" parameter. 2278 # * Remaining elements are the graph nodes for all target submodules 2279 # specified by the "target_attr_names" parameter. 2280 target_modules = None 2281 2282 # True if this is a Python 2-style implicit relative import of a 2283 # SWIG-generated C extension. False if we checked and it is not SWIG. 2284 # None if we haven't checked yet. 2285 is_swig_import = None 2286 2287 # Attempt to import this target module in the customary way. 2288 try: 2289 target_modules = self.import_hook( 2290 target_module_partname, source_module, 2291 target_attr_names=None, level=level, edge_attr=edge_attr) 2292 # Failing that, defer to custom module importers handling non-standard 2293 # import schemes (namely, SWIG). 2294 except InvalidRelativeImportError: 2295 self.msgout(2, "Invalid relative import", level, 2296 target_module_partname, target_attr_names) 2297 result = [] 2298 for sub in target_attr_names or '*': 2299 m = self.createNode(InvalidRelativeImport, 2300 '.' * level + target_module_partname, sub) 2301 self._updateReference(source_module, m, edge_data=edge_attr) 2302 result.append(m) 2303 return result 2304 except ImportError as msg: 2305 # If this is an absolute top-level import under Python 3 and if the 2306 # name to be imported is the caller's name prefixed by "_", this 2307 # could be a SWIG-generated Python 2-style implicit relative import. 2308 # SWIG-generated files contain functions named swig_import_helper() 2309 # importing dynamic libraries residing in the same directory. For 2310 # example, a SWIG-generated caller module "csr.py" might resemble: 2311 # 2312 # # This file was automatically generated by SWIG (http://www.swig.org). 2313 # ... 2314 # def swig_import_helper(): 2315 # ... 2316 # try: 2317 # fp, pathname, description = imp.find_module('_csr', 2318 # [dirname(__file__)]) 2319 # except ImportError: 2320 # import _csr 2321 # return _csr 2322 # 2323 # While there exists no reasonable means for modulegraph to parse 2324 # the call to imp.find_module(), the subsequent implicit relative 2325 # import is trivially parsable. This import is prohibited under 2326 # Python 3, however, and thus parsed only if the caller's file is 2327 # parsable plaintext (as indicated by a filetype of ".py") and the 2328 # first line of this file is the above SWIG header comment. 2329 # 2330 # The constraint that this library's name be the caller's name 2331 # prefixed by '_' is explicitly mandated by SWIG and thus a 2332 # reliable indicator of "SWIG-ness". The SWIG documentation states: 2333 # "When linking the module, the name of the output file has to match 2334 # the name of the module prefixed by an underscore." 2335 # 2336 # Only source modules (e.g., ".py"-suffixed files) are SWIG import 2337 # candidates. All other node types are safely ignorable. 2338 if is_swig_candidate(): 2339 self.msg( 2340 4, 2341 'SWIG import candidate (name=%r, caller=%r, level=%r)' % ( 2342 target_module_partname, source_module, level)) 2343 is_swig_import = is_swig_wrapper(source_module) 2344 if is_swig_import: 2345 # Convert this Python 2-compliant implicit relative 2346 # import prohibited by Python 3 into a Python 2347 # 3-compliant explicit relative "from"-style import for 2348 # the duration of this function call by overwriting the 2349 # original parameters passed to this call. 2350 target_attr_names = [target_module_partname] 2351 target_module_partname = '' 2352 level = 1 2353 self.msg(2, 2354 'SWIG import (caller=%r, fromlist=%r, level=%r)' 2355 % (source_module, target_attr_names, level)) 2356 # Import this target SWIG C extension's package. 2357 try: 2358 target_modules = self.import_hook( 2359 target_module_partname, source_module, 2360 target_attr_names=None, 2361 level=level, 2362 edge_attr=edge_attr) 2363 except ImportError as msg: 2364 self.msg(2, "SWIG ImportError:", str(msg)) 2365 2366 # If this module remains unimportable... 2367 if target_modules is None: 2368 self.msg(2, "ImportError:", str(msg)) 2369 2370 # Add this module as a MissingModule node. 2371 target_module = self.createNode( 2372 MissingModule, 2373 _path_from_importerror(msg, target_module_partname)) 2374 self._updateReference( 2375 source_module, target_module, edge_data=edge_attr) 2376 2377 # Initialize this list to this node. 2378 target_modules = [target_module] 2379 2380 # Ensure that the above logic imported exactly one target module. 2381 assert len(target_modules) == 1, ( 2382 'Expected import_hook() to' 2383 'return only one module but received: {}'.format(target_modules)) 2384 2385 # Target module imported above. 2386 target_module = target_modules[0] 2387 2388 if isinstance(target_module, MissingModule) \ 2389 and is_swig_import is None and is_swig_candidate() \ 2390 and is_swig_wrapper(source_module): 2391 # if this possible swig C module was previously imported from 2392 # a python module other than its corresponding swig python 2393 # module, then it may have been considered a MissingModule. 2394 # Try to reimport it now. For details see pull-request #2578 2395 # and issue #1522. 2396 # 2397 # If this module was takes as a SWIG candidate above, but failed 2398 # to import, this would be a MissingModule, too. Thus check if 2399 # this was the case (is_swig_import would be not None) to avoid 2400 # recursion error. If `is_swig_import` is None and we are still a 2401 # swig candidate then that means we haven't properly imported this 2402 # swig module yet so do that below. 2403 # 2404 # Remove the MissingModule node from the graph so that we can 2405 # attempt a reimport and avoid collisions. This node should be 2406 # fine to remove because the proper module will be imported and 2407 # added to the graph in the next line (call to _safe_import_hook). 2408 self.removeNode(target_module) 2409 # Reimport the SWIG C module relative to the wrapper 2410 target_modules = self._safe_import_hook( 2411 target_module_partname, source_module, 2412 target_attr_names=None, level=1, edge_attr=edge_attr) 2413 # return the output regardless because it would just be 2414 # duplicating the processing below 2415 return target_modules 2416 2417 if isinstance(edge_attr, DependencyInfo): 2418 edge_attr = edge_attr._replace(fromlist=True) 2419 2420 # If this is a "from"-style import *AND* this target module is a 2421 # package, import all attributes listed by the "import" clause of this 2422 # import that are submodules of this package. If this target module is 2423 # *NOT* a package, these attributes are always ignorable globals (e.g., 2424 # classes, variables) defined at the top level of this module. 2425 # 2426 # If this target module is a non-package, it could still contain 2427 # importable submodules (e.g., the non-package `os` module containing 2428 # the `os.path` submodule). In this case, these submodules are already 2429 # imported by this target module's pure-Python code. Since our import 2430 # scanner already detects these imports, these submodules need *NOT* be 2431 # reimported here. (Doing so would be harmless but inefficient.) 2432 if target_attr_names and isinstance(target_module, Package): 2433 # For the name of each attribute imported from this target package 2434 # into this source module... 2435 for target_submodule_partname in target_attr_names: 2436 #FIXME: Is this optimization *REALLY* an optimization or at all 2437 #necessary? The findNode() method called below should already 2438 #be heavily optimized, in which case this optimization here is 2439 #premature, senseless, and should be eliminated. 2440 2441 # If this attribute is a previously imported submodule of this 2442 # target module, optimize this edge case. 2443 if target_module.is_submodule(target_submodule_partname): 2444 # Graph node for this submodule. 2445 target_submodule = target_module.get_submodule( 2446 target_submodule_partname) 2447 2448 #FIXME: What? Shouldn't "target_submodule" *ALWAYS* be 2449 #non-None here? Assert this to be non-None instead. 2450 if target_submodule is not None: 2451 #FIXME: Why does duplication matter? List searches are 2452 #mildly expensive. 2453 2454 # If this submodule has not already been added to the 2455 # list of submodules to be returned, do so. 2456 if target_submodule not in target_modules: 2457 self._updateReference( 2458 source_module, 2459 target_submodule, 2460 edge_data=edge_attr) 2461 target_modules.append(target_submodule) 2462 continue 2463 2464 # Fully-qualified name of this submodule. 2465 target_submodule_name = ( 2466 target_module.identifier + '.' + target_submodule_partname) 2467 2468 # Graph node of this submodule if previously imported or None. 2469 target_submodule = self.findNode(target_submodule_name) 2470 2471 # If this submodule has not been imported, do so as if this 2472 # submodule were the only attribute listed by the "import" 2473 # clause of this import (e.g., as "from foo import bar" rather 2474 # than "from foo import car, far, bar"). 2475 if target_submodule is None: 2476 # Attempt to import this submodule. 2477 try: 2478 # Ignore the list of graph nodes returned by this 2479 # method. If both this submodule's package and this 2480 # submodule are importable, this method returns a 2481 # 2-element list whose second element is this 2482 # submodule's graph node. However, if this submodule's 2483 # package is importable but this submodule is not, 2484 # this submodule is either: 2485 # 2486 # * An ignorable global attribute defined at the top 2487 # level of this package's "__init__" submodule. In 2488 # this case, this method returns a 1-element list 2489 # without raising an exception. 2490 # * A non-ignorable unimportable submodule. In this 2491 # case, this method raises an "ImportError". 2492 # 2493 # While the first two cases are disambiguatable by the 2494 # length of this list, doing so would render this code 2495 # dependent on import_hook() details subject to change. 2496 # Instead, call findNode() to decide the truthiness. 2497 self.import_hook( 2498 target_module_partname, source_module, 2499 target_attr_names=[target_submodule_partname], 2500 level=level, 2501 edge_attr=edge_attr) 2502 2503 # Graph node of this submodule imported by the prior 2504 # call if importable or None otherwise. 2505 target_submodule = self.findNode(target_submodule_name) 2506 2507 # If this submodule does not exist, this *MUST* be an 2508 # ignorable global attribute defined at the top level 2509 # of this package's "__init__" submodule. 2510 if target_submodule is None: 2511 # Assert this to actually be the case. 2512 assert target_module.is_global_attr( 2513 target_submodule_partname), ( 2514 'No global named {} in {}.__init__'.format( 2515 target_submodule_partname, 2516 target_module.identifier)) 2517 2518 # Skip this safely ignorable importation to the 2519 # next attribute. See similar logic in the body of 2520 # _import_importable_package_submodules(). 2521 self.msg(4, '_safe_import_hook', 'ignoring imported non-module global', target_module.identifier, target_submodule_partname) 2522 continue 2523 2524 # If this is a SWIG C extension, instruct PyInstaller 2525 # to freeze this extension under its unqualified rather 2526 # than qualified name (e.g., as "_csr" rather than 2527 # "scipy.sparse.sparsetools._csr"), permitting the 2528 # implicit relative import in its parent SWIG module to 2529 # successfully find this extension. 2530 if is_swig_import: 2531 # If a graph node with this name already exists, 2532 # avoid collisions by emitting an error instead. 2533 if self.findNode(target_submodule_partname): 2534 self.msg( 2535 2, 2536 'SWIG import error: %r basename %r ' 2537 'already exists' % ( 2538 target_submodule_name, 2539 target_submodule_partname)) 2540 else: 2541 self.msg( 2542 4, 2543 'SWIG import renamed from %r to %r' % ( 2544 target_submodule_name, 2545 target_submodule_partname)) 2546 target_submodule.identifier = ( 2547 target_submodule_partname) 2548 # If this submodule is unimportable, add a MissingModule. 2549 except ImportError as msg: 2550 self.msg(2, "ImportError:", str(msg)) 2551 target_submodule = self.createNode( 2552 MissingModule, target_submodule_name) 2553 2554 # Add this submodule to its package. 2555 target_module.add_submodule( 2556 target_submodule_partname, target_submodule) 2557 if target_submodule is not None: 2558 self._updateReference( 2559 target_module, target_submodule, edge_data=edge_attr) 2560 self._updateReference( 2561 source_module, target_submodule, edge_data=edge_attr) 2562 2563 if target_submodule not in target_modules: 2564 target_modules.append(target_submodule) 2565 2566 # Return the list of all target modules imported by this call. 2567 return target_modules 2568 2569 2570 def _scan_code( 2571 self, 2572 module, 2573 module_code_object, 2574 module_code_object_ast=None): 2575 """ 2576 Parse and add all import statements from the passed code object of the 2577 passed source module to this graph, recursively. 2578 2579 **This method is at the root of all `ModuleGraph` recursion.** 2580 Recursion begins here and ends when all import statements in all code 2581 objects of all modules transitively imported by the source module 2582 passed to the first call to this method have been added to the graph. 2583 Specifically, this method: 2584 2585 1. If the passed `module_code_object_ast` parameter is non-`None`, 2586 parses all import statements from this object. 2587 2. Else, parses all import statements from the passed 2588 `module_code_object` parameter. 2589 1. For each such import statement: 2590 1. Adds to this `ModuleGraph` instance: 2591 1. Nodes for all target modules of these imports. 2592 1. Directed edges from this source module to these target 2593 modules. 2594 2. Recursively calls this method with these target modules. 2595 2596 Parameters 2597 ---------- 2598 module : Node 2599 Graph node of the module to be parsed. 2600 module_code_object : PyCodeObject 2601 Code object providing this module's disassembled Python bytecode. 2602 Ignored unless `module_code_object_ast` is `None`. 2603 module_code_object_ast : optional[ast.AST] 2604 Optional abstract syntax tree (AST) of this module if any or `None` 2605 otherwise. Defaults to `None`, in which case the passed 2606 `module_code_object` is parsed instead. 2607 """ 2608 2609 # For safety, guard against multiple scans of the same module by 2610 # resetting this module's list of deferred target imports. While 2611 # uncommon, this edge case can occur due to: 2612 # 2613 # * Dynamic package replacement via the replacePackage() function. For 2614 # example, the real "_xmlplus" package dynamically replaces itself 2615 # with the fake "xml" package into the "sys.modules" cache of all 2616 # currently loaded modules at runtime. 2617 module._deferred_imports = [] 2618 2619 # Parse all imports from this module *BEFORE* adding these imports to 2620 # the graph. If an AST is provided, parse that rather than this 2621 # module's code object. 2622 if module_code_object_ast is not None: 2623 # Parse this module's AST for imports. 2624 self._scan_ast(module, module_code_object_ast) 2625 2626 # Parse this module's code object for all relevant non-imports 2627 # (e.g., global variable declarations and undeclarations). 2628 self._scan_bytecode( 2629 module, module_code_object, is_scanning_imports=False) 2630 # Else, parse this module's code object for imports. 2631 else: 2632 self._scan_bytecode( 2633 module, module_code_object, is_scanning_imports=True) 2634 2635 # Add all imports parsed above to this graph. 2636 self._process_imports(module) 2637 2638 2639 def _scan_ast(self, module, module_code_object_ast): 2640 """ 2641 Parse and add all import statements from the passed abstract syntax 2642 tree (AST) of the passed source module to this graph, non-recursively. 2643 2644 Parameters 2645 ---------- 2646 module : Node 2647 Graph node of the module to be parsed. 2648 module_code_object_ast : ast.AST 2649 Abstract syntax tree (AST) of this module to be parsed. 2650 """ 2651 2652 visitor = _Visitor(self, module) 2653 visitor.visit(module_code_object_ast) 2654 2655 #FIXME: Optimize. Global attributes added by this method are tested by 2656 #other methods *ONLY* for packages, implying this method should scan and 2657 #handle opcodes pertaining to global attributes (e.g., 2658 #"STORE_NAME", "DELETE_GLOBAL") only if the passed "module" 2659 #object is an instance of the "Package" class. For all other module types, 2660 #these opcodes should simply be ignored. 2661 # 2662 #After doing so, the "Node._global_attr_names" attribute and all methods 2663 #using this attribute (e.g., Node.is_global()) should be moved from the 2664 #"Node" superclass to the "Package" subclass. 2665 def _scan_bytecode( 2666 self, module, module_code_object, is_scanning_imports): 2667 """ 2668 Parse and add all import statements from the passed code object of the 2669 passed source module to this graph, non-recursively. 2670 2671 This method parses all reasonably parsable operations (i.e., operations 2672 that are both syntactically and semantically parsable _without_ 2673 requiring Turing-complete interpretation) directly or indirectly 2674 involving module importation from this code object. This includes: 2675 2676 * `IMPORT_NAME`, denoting an import statement. Ignored unless 2677 the passed `is_scanning_imports` parameter is `True`. 2678 * `STORE_NAME` and `STORE_GLOBAL`, denoting the 2679 declaration of a global attribute (e.g., class, variable) in this 2680 module. This method stores each such declaration for subsequent 2681 lookup. While global attributes are usually irrelevant to import 2682 parsing, they remain the only means of distinguishing erroneous 2683 non-ignorable attempts to import non-existent submodules of a package 2684 from successful ignorable attempts to import existing global 2685 attributes of a package's `__init__` submodule (e.g., the `bar` in 2686 `from foo import bar`, which is either a non-ignorable submodule of 2687 `foo` or an ignorable global attribute of `foo.__init__`). 2688 * `DELETE_NAME` and `DELETE_GLOBAL`, denoting the 2689 undeclaration of a previously declared global attribute in this 2690 module. 2691 2692 Since `ModuleGraph` is _not_ intended to replicate the behaviour of a 2693 full-featured Turing-complete Python interpreter, this method ignores 2694 operations that are _not_ reasonably parsable from this code object -- 2695 even those directly or indirectly involving module importation. This 2696 includes: 2697 2698 * `STORE_ATTR(namei)`, implementing `TOS.name = TOS1`. If `TOS` is the 2699 name of a target module currently imported into the namespace of the 2700 passed source module, this opcode would ideally be parsed to add that 2701 global attribute to that target module. Since this addition only 2702 conditionally occurs on the importation of this source module and 2703 execution of the code branch in this module performing this addition, 2704 however, that global _cannot_ be unconditionally added to that target 2705 module. In short, only Turing-complete behaviour suffices. 2706 * `DELETE_ATTR(namei)`, implementing `del TOS.name`. If `TOS` is the 2707 name of a target module currently imported into the namespace of the 2708 passed source module, this opcode would ideally be parsed to remove 2709 that global attribute from that target module. Again, however, only 2710 Turing-complete behaviour suffices. 2711 2712 Parameters 2713 ---------- 2714 module : Node 2715 Graph node of the module to be parsed. 2716 module_code_object : PyCodeObject 2717 Code object of the module to be parsed. 2718 is_scanning_imports : bool 2719 `True` only if this method is parsing import statements from 2720 `IMPORT_NAME` opcodes. If `False`, no import statements will be 2721 parsed. This parameter is typically: 2722 * `True` when parsing this module's code object for such imports. 2723 * `False` when parsing this module's abstract syntax tree (AST) 2724 (rather than code object) for such imports. In this case, that 2725 parsing will have already parsed import statements, which this 2726 parsing must avoid repeating. 2727 """ 2728 level = None 2729 fromlist = None 2730 2731 # 'deque' is a list-like container with fast appends, pops on 2732 # either end, and automatically discarding elements too much. 2733 prev_insts = deque(maxlen=2) 2734 for inst in util.iterate_instructions(module_code_object): 2735 if not inst: 2736 continue 2737 # If this is an import statement originating from this module, 2738 # parse this import. 2739 # 2740 # Note that the related "IMPORT_FROM" opcode need *NOT* be parsed. 2741 # "IMPORT_NAME" suffices. For further details, see 2742 # http://probablyprogramming.com/2008/04/14/python-import_name 2743 if inst.opname == 'IMPORT_NAME': 2744 # If this method is ignoring import statements, skip to the 2745 # next opcode. 2746 if not is_scanning_imports: 2747 continue 2748 2749 assert prev_insts[-2].opname == 'LOAD_CONST' 2750 assert prev_insts[-1].opname == 'LOAD_CONST' 2751 2752 # Python >=2.5: LOAD_CONST flags, LOAD_CONST names, IMPORT_NAME name 2753 level = prev_insts[-2].argval 2754 fromlist = prev_insts[-1].argval 2755 2756 assert fromlist is None or type(fromlist) is tuple 2757 target_module_partname = inst.argval 2758 2759 #FIXME: The exact same logic appears in _collect_import(), 2760 #which isn't particularly helpful. Instead, defer this logic 2761 #until later by: 2762 # 2763 #* Refactor the "_deferred_imports" list to contain 2-tuples 2764 # "(_safe_import_hook_args, _safe_import_hook_kwargs)" rather 2765 # than 3-tuples "(have_star, _safe_import_hook_args, 2766 # _safe_import_hook_kwargs)". 2767 #* Stop prepending these tuples by a "have_star" boolean both 2768 # here, in _collect_import(), and in _process_imports(). 2769 #* Shift the logic below to _process_imports(). 2770 #* Remove the same logic from _collect_import(). 2771 have_star = False 2772 if fromlist is not None: 2773 fromlist = uniq(fromlist) 2774 if '*' in fromlist: 2775 fromlist.remove('*') 2776 have_star = True 2777 2778 # Record this import as originating from this module for 2779 # subsequent handling by the _process_imports() method. 2780 module._deferred_imports.append(( 2781 have_star, 2782 (target_module_partname, module, fromlist, level), 2783 {} 2784 )) 2785 2786 elif inst.opname in ('STORE_NAME', 'STORE_GLOBAL'): 2787 # If this is the declaration of a global attribute (e.g., 2788 # class, variable) in this module, store this declaration for 2789 # subsequent lookup. See method docstring for further details. 2790 # 2791 # Global attributes are usually irrelevant to import parsing, but 2792 # remain the only means of distinguishing erroneous non-ignorable 2793 # attempts to import non-existent submodules of a package from 2794 # successful ignorable attempts to import existing global 2795 # attributes of a package's "__init__" submodule (e.g., the "bar" 2796 # in "from foo import bar", which is either a non-ignorable 2797 # submodule of "foo" or an ignorable global attribute of 2798 # "foo.__init__"). 2799 name = inst.argval 2800 module.add_global_attr(name) 2801 2802 elif inst.opname in ('DELETE_NAME', 'DELETE_GLOBAL'): 2803 # If this is the undeclaration of a previously declared global 2804 # attribute (e.g., class, variable) in this module, remove that 2805 # declaration to prevent subsequent lookup. See method docstring 2806 # for further details. 2807 name = inst.argval 2808 module.remove_global_attr_if_found(name) 2809 2810 prev_insts.append(inst) 2811 2812 2813 def _process_imports(self, source_module): 2814 """ 2815 Graph all target modules whose importations were previously parsed from 2816 the passed source module by a prior call to the `_scan_code()` method 2817 and methods call by that method (e.g., `_scan_ast()`, 2818 `_scan_bytecode()`, `_scan_bytecode_stores()`). 2819 2820 Parameters 2821 ---------- 2822 source_module : Node 2823 Graph node of the source module to graph target imports for. 2824 """ 2825 2826 # If this source module imported no target modules, noop. 2827 if not source_module._deferred_imports: 2828 return 2829 2830 # For each target module imported by this source module... 2831 for have_star, import_info, kwargs in source_module._deferred_imports: 2832 # Graph node of the target module specified by the "from" portion 2833 # of this "from"-style star import (e.g., an import resembling 2834 # "from {target_module_name} import *") or ignored otherwise. 2835 target_module = self._safe_import_hook(*import_info, **kwargs)[0] 2836 2837 # If this is a "from"-style star import, process this import. 2838 if have_star: 2839 #FIXME: Sadly, the current approach to importing attributes 2840 #from "from"-style star imports is... simplistic. This should 2841 #be revised as follows. If this target module is: 2842 # 2843 #* A package: 2844 # * Whose "__init__" submodule defines the "__all__" global 2845 # attribute, only attributes listed by this attribute should 2846 # be imported. 2847 # * Else, *NO* attributes should be imported. 2848 #* A non-package: 2849 # * Defining the "__all__" global attribute, only attributes 2850 # listed by this attribute should be imported. 2851 # * Else, only public attributes whose names are *NOT* 2852 # prefixed by "_" should be imported. 2853 source_module.add_global_attrs_from_module(target_module) 2854 2855 source_module._starimported_ignored_module_names.update( 2856 target_module._starimported_ignored_module_names) 2857 2858 # If this target module has no code object and hence is 2859 # unparsable, record its name for posterity. 2860 if target_module.code is None: 2861 target_module_name = import_info[0] 2862 source_module._starimported_ignored_module_names.add( 2863 target_module_name) 2864 2865 # For safety, prevent these imports from being reprocessed. 2866 source_module._deferred_imports = None 2867 2868 2869 def _load_package(self, fqname, pathname, pkgpath): 2870 """ 2871 Called only when an imp.PKG_DIRECTORY is found 2872 """ 2873 self.msgin(2, "load_package", fqname, pathname, pkgpath) 2874 newname = _replacePackageMap.get(fqname) 2875 if newname: 2876 fqname = newname 2877 2878 ns_pkgpath = _namespace_package_path(fqname, pkgpath or [], self.path) 2879 if ns_pkgpath or pkgpath: 2880 # this is a namespace package 2881 m = self.createNode(NamespacePackage, fqname) 2882 m.filename = '-' 2883 m.packagepath = ns_pkgpath 2884 else: 2885 m = self.createNode(Package, fqname) 2886 m.filename = pathname 2887 # PEP-302-compliant loaders return the pathname of the 2888 # `__init__`-file, not the packge directory. 2889 if os.path.basename(pathname).startswith('__init__.'): 2890 pathname = os.path.dirname(pathname) 2891 m.packagepath = [pathname] + ns_pkgpath 2892 2893 # As per comment at top of file, simulate runtime packagepath additions. 2894 m.packagepath = m.packagepath + self._package_path_map.get(fqname, []) 2895 2896 try: 2897 self.msg(2, "find __init__ for %s"%(m.packagepath,)) 2898 fp, buf, stuff = self._find_module("__init__", m.packagepath, parent=m) 2899 except ImportError: 2900 pass 2901 2902 else: 2903 try: 2904 self.msg(2, "load __init__ for %s"%(m.packagepath,)) 2905 self._load_module(fqname, fp, buf, stuff) 2906 finally: 2907 if fp is not None: 2908 fp.close() 2909 self.msgout(2, "load_package ->", m) 2910 return m 2911 2912 2913 def _find_module(self, name, path, parent=None): 2914 """ 2915 3-tuple describing the physical location of the module with the passed 2916 name if this module is physically findable _or_ raise `ImportError`. 2917 2918 This high-level method wraps the low-level `modulegraph.find_module()` 2919 function with additional support for graph-based module caching. 2920 2921 Parameters 2922 ---------- 2923 name : str 2924 Fully-qualified name of the Python module to be found. 2925 path : list 2926 List of the absolute paths of all directories to search for this 2927 module _or_ `None` if the default path list `self.path` is to be 2928 searched. 2929 parent : Node 2930 Package containing this module if this module is a submodule of a 2931 package _or_ `None` if this is a top-level module. 2932 2933 Returns 2934 ---------- 2935 (file_handle, filename, metadata) 2936 See `modulegraph._find_module()` for details. 2937 2938 Raises 2939 ---------- 2940 ImportError 2941 If this module is _not_ found. 2942 """ 2943 2944 if parent is not None: 2945 # assert path is not None 2946 fullname = parent.identifier + '.' + name 2947 else: 2948 fullname = name 2949 2950 node = self.findNode(fullname) 2951 if node is not None: 2952 self.msg(3, "find_module: already included?", node) 2953 raise ImportError(name) 2954 2955 if path is None: 2956 if name in sys.builtin_module_names: 2957 return (None, None, ("", "", imp.C_BUILTIN)) 2958 2959 path = self.path 2960 2961 return self._find_module_path(fullname, name, path) 2962 2963 2964 def _find_module_path(self, fullname, module_name, search_dirs): 2965 """ 2966 3-tuple describing the physical location of the module with the passed 2967 name if this module is physically findable _or_ raise `ImportError`. 2968 2969 This low-level function is a variant on the standard `imp.find_module()` 2970 function with additional support for: 2971 2972 * Multiple search paths. The passed list of absolute paths will be 2973 iteratively searched for the first directory containing a file 2974 corresponding to this module. 2975 * Compressed (e.g., zipped) packages. 2976 2977 For efficiency, the higher level `ModuleGraph._find_module()` method 2978 wraps this function with support for module caching. 2979 2980 Parameters 2981 ---------- 2982 module_name : str 2983 Fully-qualified name of the module to be found. 2984 search_dirs : list 2985 List of the absolute paths of all directories to search for this 2986 module (in order). Searching will halt at the first directory 2987 containing this module. 2988 2989 Returns 2990 ---------- 2991 (file_handle, filename, metadata) 2992 3-tuple describing the physical location of this module, where: 2993 * `file_handle` is an open read-only file handle from which the 2994 on-disk contents of this module may be read if this is a 2995 pure-Python module or `None` otherwise (e.g., if this is a 2996 package or C extension). 2997 * `filename` is the absolute path of this file. 2998 * `metadata` is itself a 3-tuple `(filetype, open_mode, imp_type)`. 2999 See the `_IMPORTABLE_FILETYPE_TO_METADATA` dictionary for 3000 details. 3001 3002 Raises 3003 ---------- 3004 ImportError 3005 If this module is _not_ found. 3006 """ 3007 self.msgin(4, "_find_module_path <-", fullname, search_dirs) 3008 3009 # TODO: Under: 3010 # 3011 # * Python 3.3, the following logic should be replaced by logic 3012 # leveraging only the "importlib" module. 3013 # * Python 3.4, the following logic should be replaced by a call to 3014 # importlib.util.find_spec(). 3015 3016 # Top-level 3-tuple to be returned. 3017 path_data = None 3018 3019 # File handle to be returned. 3020 file_handle = None 3021 3022 # List of the absolute paths of all directories comprising the 3023 # namespace package to which this module belongs if any. 3024 namespace_dirs = [] 3025 3026 try: 3027 for search_dir in search_dirs: 3028 # PEP 302-compliant importer making loaders for this directory. 3029 importer = pkgutil.get_importer(search_dir) 3030 3031 # If this directory is not importable, continue. 3032 if importer is None: 3033 # self.msg(4, "_find_module_path importer not found", search_dir) 3034 continue 3035 3036 # Get the PEP 302-compliant loader object loading this module. 3037 # 3038 # If this importer defines the PEP 302-compliant find_loader() 3039 # method, prefer that. 3040 if hasattr(importer, 'find_loader'): 3041 loader, loader_namespace_dirs = importer.find_loader( 3042 module_name) 3043 namespace_dirs.extend(loader_namespace_dirs) 3044 # Else if this importer defines the Python 2-specific 3045 # find_module() method, fall back to that. Despite the method 3046 # name, this method returns a loader rather than a module. 3047 elif hasattr(importer, 'find_module'): 3048 loader = importer.find_module(module_name) 3049 # Else, raise an exception. 3050 else: 3051 raise ImportError( 3052 "Module %r importer %r loader unobtainable" % (module_name, importer)) 3053 3054 # If this module is not loadable from this directory, continue. 3055 if loader is None: 3056 # self.msg(4, "_find_module_path loader not found", search_dir) 3057 continue 3058 3059 # 3-tuple of metadata to be returned. 3060 metadata = None 3061 3062 # Absolute path of this module. If this module resides in a 3063 # compressed archive, this is the absolute path of this module 3064 # after extracting this module from that archive and hence 3065 # should not exist; else, this path should typically exist. 3066 pathname = None 3067 3068 # If this loader defines the PEP 302-compliant get_filename() 3069 # method, preferably call that method first. Most if not all 3070 # loaders (including zipimporter objects) define this method. 3071 if hasattr(loader, 'get_filename'): 3072 pathname = loader.get_filename(module_name) 3073 # Else if this loader provides a "path" attribute, defer to that. 3074 elif hasattr(loader, 'path'): 3075 pathname = loader.path 3076 # Else, raise an exception. 3077 else: 3078 raise ImportError( 3079 "Module %r loader %r path unobtainable" % (module_name, loader)) 3080 3081 # If no path was found, this is probably a namespace package. In 3082 # such case, continue collecting namespace directories. 3083 if pathname is None: 3084 self.msg(4, "_find_module_path path not found", pathname) 3085 continue 3086 3087 # If this loader defines the PEP 302-compliant is_package() 3088 # method returning True, this is a non-namespace package. 3089 if hasattr(loader, 'is_package') and loader.is_package(module_name): 3090 metadata = ('', '', imp.PKG_DIRECTORY) 3091 # Else, this is either a module or C extension. 3092 else: 3093 # In either case, this path must have a filetype. 3094 filetype = os.path.splitext(pathname)[1] 3095 if not filetype: 3096 raise ImportError( 3097 'Non-package module %r path %r has no filetype' % (module_name, pathname)) 3098 3099 # 3-tuple of metadata specific to this filetype. 3100 metadata = _IMPORTABLE_FILETYPE_TO_METADATA.get( 3101 filetype, None) 3102 if metadata is None: 3103 raise ImportError( 3104 'Non-package module %r filetype %r unrecognized' % (pathname, filetype)) 3105 3106 # See "_IMPORTABLE_FILETYPE_TO_METADATA" for details. 3107 open_mode = metadata[1] 3108 imp_type = metadata[2] 3109 3110 # If this is a C extension, leave this path unopened. 3111 if imp_type == imp.C_EXTENSION: 3112 pass 3113 # Else, this is a module. 3114 # 3115 # If this loader defines the PEP 302-compliant get_source() 3116 # method, open the returned string as a file-like buffer. 3117 elif imp_type == imp.PY_SOURCE and hasattr(loader, 'get_source'): 3118 file_handle = StringIO(loader.get_source(module_name)) 3119 # If this loader defines the PEP 302-compliant get_code() 3120 # method, open the returned object as a file-like buffer. 3121 elif imp_type == imp.PY_COMPILED and hasattr(loader, 'get_code'): 3122 try: 3123 code_object = loader.get_code(module_name) 3124 if code_object is None: 3125 file_handle = BytesIO(b'\0\0\0\0\0\0\0\0') 3126 else: 3127 file_handle = _code_to_file(code_object) 3128 except ImportError: 3129 # post-bone the ImportError until load_module 3130 file_handle = BytesIO(b'\0\0\0\0\0\0\0\0') 3131 # If this is an uncompiled file under Python 3, open this 3132 # file for encoding-aware text reading. 3133 elif imp_type == imp.PY_SOURCE and sys.version_info[0] == 3: 3134 with open(pathname, 'rb') as file_handle: 3135 encoding = util.guess_encoding(file_handle) 3136 file_handle = open( 3137 pathname, open_mode, encoding=encoding) 3138 # Else, this is either a compiled file or an uncompiled 3139 # file under Python 2. In either case, open this file. 3140 else: 3141 file_handle = open(pathname, open_mode) 3142 3143 # Return such metadata. 3144 path_data = (file_handle, pathname, metadata) 3145 break 3146 # Else if this is a namespace package, return such metadata. 3147 else: 3148 if namespace_dirs: 3149 path_data = (None, namespace_dirs[0], ( 3150 '', namespace_dirs, imp.PKG_DIRECTORY)) 3151 except UnicodeDecodeError as exc: 3152 self.msgout(1, "_find_module_path -> unicode error", exc) 3153 # Ensure that exceptions are logged, as this function is typically 3154 # called by the import_module() method which squelches ImportErrors. 3155 except Exception as exc: 3156 self.msgout(4, "_find_module_path -> exception", exc) 3157 raise 3158 3159 # If this module was not found, raise an exception. 3160 self.msgout(4, "_find_module_path ->", path_data) 3161 if path_data is None: 3162 raise ImportError("No module named " + repr(module_name)) 3163 3164 return path_data 3165 3166 3167 def create_xref(self, out=None): 3168 global header, footer, entry, contpl, contpl_linked, imports 3169 if out is None: 3170 out = sys.stdout 3171 scripts = [] 3172 mods = [] 3173 for mod in self.flatten(): 3174 name = os.path.basename(mod.identifier) 3175 if isinstance(mod, Script): 3176 scripts.append((name, mod)) 3177 else: 3178 mods.append((name, mod)) 3179 scripts.sort() 3180 mods.sort() 3181 scriptnames = [sn for sn, m in scripts] 3182 scripts.extend(mods) 3183 mods = scripts 3184 3185 title = "modulegraph cross reference for " + ', '.join(scriptnames) 3186 print(header % {"TITLE": title}, file=out) 3187 3188 def sorted_namelist(mods): 3189 lst = [os.path.basename(mod.identifier) for mod in mods if mod] 3190 lst.sort() 3191 return lst 3192 for name, m in mods: 3193 content = "" 3194 if isinstance(m, BuiltinModule): 3195 content = contpl % {"NAME": name, 3196 "TYPE": "<i>(builtin module)</i>"} 3197 elif isinstance(m, Extension): 3198 content = contpl % {"NAME": name, 3199 "TYPE": "<tt>%s</tt>" % m.filename} 3200 else: 3201 url = pathname2url(m.filename or "") 3202 content = contpl_linked % {"NAME": name, "URL": url, 3203 'TYPE': m.__class__.__name__} 3204 oute, ince = map(sorted_namelist, self.get_edges(m)) 3205 if oute: 3206 links = [] 3207 for n in oute: 3208 links.append(""" <a href="#%s">%s</a>\n""" % (n, n)) 3209 # #8226 = bullet-point; can't use html-entities since the 3210 # test-suite uses xml.etree.ElementTree.XMLParser, which 3211 # does't supprot them. 3212 links = " • ".join(links) 3213 content += imports % {"HEAD": "imports", "LINKS": links} 3214 if ince: 3215 links = [] 3216 for n in ince: 3217 links.append(""" <a href="#%s">%s</a>\n""" % (n, n)) 3218 # #8226 = bullet-point; can't use html-entities since the 3219 # test-suite uses xml.etree.ElementTree.XMLParser, which 3220 # does't supprot them. 3221 links = " • ".join(links) 3222 content += imports % {"HEAD": "imported by", "LINKS": links} 3223 print(entry % {"NAME": name, "CONTENT": content}, file=out) 3224 print(footer, file=out) 3225 3226 def itergraphreport(self, name='G', flatpackages=()): 3227 # XXX: Can this be implemented using Dot()? 3228 nodes = list(map(self.graph.describe_node, self.graph.iterdfs(self))) 3229 describe_edge = self.graph.describe_edge 3230 edges = deque() 3231 packagenodes = set() 3232 packageidents = {} 3233 nodetoident = {} 3234 inpackages = {} 3235 mainedges = set() 3236 3237 # XXX - implement 3238 flatpackages = dict(flatpackages) 3239 3240 def nodevisitor(node, data, outgoing, incoming): 3241 if not isinstance(data, Node): 3242 return {'label': str(node)} 3243 #if isinstance(d, (ExcludedModule, MissingModule, BadModule)): 3244 # return None 3245 s = '<f0> ' + type(data).__name__ 3246 for i, v in enumerate(data.infoTuple()[:1], 1): 3247 s += '| <f%d> %s' % (i, v) 3248 return {'label': s, 'shape': 'record'} 3249 3250 3251 def edgevisitor(edge, data, head, tail): 3252 # XXX: This method nonsense, the edge 3253 # data is never initialized. 3254 if data == 'orphan': 3255 return {'style': 'dashed'} 3256 elif data == 'pkgref': 3257 return {'style': 'dotted'} 3258 return {} 3259 3260 yield 'digraph %s {\ncharset="UTF-8";\n' % (name,) 3261 attr = dict(rankdir='LR', concentrate='true') 3262 cpatt = '%s="%s"' 3263 for item in attr.items(): 3264 yield '\t%s;\n' % (cpatt % item,) 3265 3266 # find all packages (subgraphs) 3267 for (node, data, outgoing, incoming) in nodes: 3268 nodetoident[node] = getattr(data, 'identifier', None) 3269 if isinstance(data, Package): 3270 packageidents[data.identifier] = node 3271 inpackages[node] = set([node]) 3272 packagenodes.add(node) 3273 3274 # create sets for subgraph, write out descriptions 3275 for (node, data, outgoing, incoming) in nodes: 3276 # update edges 3277 for edge in (describe_edge(e) for e in outgoing): 3278 edges.append(edge) 3279 3280 # describe node 3281 yield '\t"%s" [%s];\n' % ( 3282 node, 3283 ','.join([ 3284 (cpatt % item) for item in 3285 nodevisitor(node, data, outgoing, incoming).items() 3286 ]), 3287 ) 3288 3289 inside = inpackages.get(node) 3290 if inside is None: 3291 inside = inpackages[node] = set() 3292 ident = nodetoident[node] 3293 if ident is None: 3294 continue 3295 pkgnode = packageidents.get(ident[:ident.rfind('.')]) 3296 if pkgnode is not None: 3297 inside.add(pkgnode) 3298 3299 graph = [] 3300 subgraphs = {} 3301 for key in packagenodes: 3302 subgraphs[key] = [] 3303 3304 while edges: 3305 edge, data, head, tail = edges.popleft() 3306 if ((head, tail)) in mainedges: 3307 continue 3308 mainedges.add((head, tail)) 3309 tailpkgs = inpackages[tail] 3310 common = inpackages[head] & tailpkgs 3311 if not common and tailpkgs: 3312 usepkgs = sorted(tailpkgs) 3313 if len(usepkgs) != 1 or usepkgs[0] != tail: 3314 edges.append((edge, data, head, usepkgs[0])) 3315 edges.append((edge, 'pkgref', usepkgs[-1], tail)) 3316 continue 3317 if common: 3318 common = common.pop() 3319 if tail == common: 3320 edges.append((edge, data, tail, head)) 3321 elif head == common: 3322 subgraphs[common].append((edge, 'pkgref', head, tail)) 3323 else: 3324 edges.append((edge, data, common, head)) 3325 edges.append((edge, data, common, tail)) 3326 3327 else: 3328 graph.append((edge, data, head, tail)) 3329 3330 def do_graph(edges, tabs): 3331 edgestr = tabs + '"%s" -> "%s" [%s];\n' 3332 # describe edge 3333 for (edge, data, head, tail) in edges: 3334 attribs = edgevisitor(edge, data, head, tail) 3335 yield edgestr % ( 3336 head, 3337 tail, 3338 ','.join([(cpatt % item) for item in attribs.items()]), 3339 ) 3340 3341 for g, edges in subgraphs.items(): 3342 yield '\tsubgraph "cluster_%s" {\n' % (g,) 3343 yield '\t\tlabel="%s";\n' % (nodetoident[g],) 3344 for s in do_graph(edges, '\t\t'): 3345 yield s 3346 yield '\t}\n' 3347 3348 for s in do_graph(graph, '\t'): 3349 yield s 3350 3351 yield '}\n' 3352 3353 def graphreport(self, fileobj=None, flatpackages=()): 3354 if fileobj is None: 3355 fileobj = sys.stdout 3356 fileobj.writelines(self.itergraphreport(flatpackages=flatpackages)) 3357 3358 def report(self): 3359 """Print a report to stdout, listing the found modules with their 3360 paths, as well as modules that are missing, or seem to be missing. 3361 """ 3362 print() 3363 print("%-15s %-25s %s" % ("Class", "Name", "File")) 3364 print("%-15s %-25s %s" % ("-----", "----", "----")) 3365 for m in sorted(self.flatten(), key=lambda n: n.identifier): 3366 print("%-15s %-25s %s" % (type(m).__name__, m.identifier, m.filename or "")) 3367 3368 def _replace_paths_in_code(self, co): 3369 new_filename = original_filename = os.path.normpath(co.co_filename) 3370 for f, r in self.replace_paths: 3371 f = os.path.join(f, '') 3372 r = os.path.join(r, '') 3373 if original_filename.startswith(f): 3374 new_filename = r + original_filename[len(f):] 3375 break 3376 3377 else: 3378 return co 3379 3380 consts = list(co.co_consts) 3381 for i in range(len(consts)): 3382 if isinstance(consts[i], type(co)): 3383 consts[i] = self._replace_paths_in_code(consts[i]) 3384 3385 code_func = type(co) 3386 3387 if hasattr(co, 'co_kwonlyargcount'): 3388 return code_func( 3389 co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, 3390 co.co_stacksize, co.co_flags, co.co_code, 3391 tuple(consts), co.co_names, co.co_varnames, 3392 new_filename, co.co_name, co.co_firstlineno, 3393 co.co_lnotab, co.co_freevars, co.co_cellvars) 3394 else: 3395 return code_func( 3396 co.co_argcount, co.co_nlocals, co.co_stacksize, 3397 co.co_flags, co.co_code, tuple(consts), co.co_names, 3398 co.co_varnames, new_filename, co.co_name, 3399 co.co_firstlineno, co.co_lnotab, 3400 co.co_freevars, co.co_cellvars) 3401