1# Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com 2# 3# Part of "Nuitka", an optimizing Python compiler that is compatible and 4# integrates with CPython, but also works on its own. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18""" Recursion into other modules. 19 20""" 21 22import glob 23import os 24 25from nuitka import ModuleRegistry, Options 26from nuitka.importing import ImportCache, Importing, StandardLibrary 27from nuitka.plugins.Plugins import Plugins 28from nuitka.PythonVersions import python_version 29from nuitka.Tracing import recursion_logger 30from nuitka.utils.FileOperations import listDir, relpath 31from nuitka.utils.ModuleNames import ModuleName 32 33 34def _recurseTo(module_package, module_filename, module_kind): 35 from nuitka.tree import Building 36 37 module, is_added = Building.buildModule( 38 module_filename=module_filename, 39 module_package=module_package, 40 source_code=None, 41 is_top=False, 42 is_main=False, 43 is_shlib=module_kind == "shlib", 44 is_fake=False, 45 hide_syntax_error=True, 46 ) 47 48 ImportCache.addImportedModule(module) 49 50 return module, is_added 51 52 53def recurseTo( 54 signal_change, module_package, module_filename, module_relpath, module_kind, reason 55): 56 if ImportCache.isImportedModuleByPath(module_relpath): 57 try: 58 module = ImportCache.getImportedModuleByPath(module_relpath, module_package) 59 except KeyError: 60 module = None 61 else: 62 module = None 63 64 if module is None: 65 module, added_flag = _recurseTo( 66 module_package=module_package, 67 module_filename=module_filename, 68 module_kind=module_kind, 69 ) 70 71 if added_flag and signal_change is not None: 72 signal_change("new_code", module.getSourceReference(), reason) 73 74 return module 75 76 77def decideRecursion(module_filename, module_name, module_kind, extra_recursion=False): 78 # Many branches, which make decisions immediately, by returning 79 # pylint: disable=too-many-return-statements 80 if module_name == "__main__": 81 return False, "Main program is not recursed to again." 82 83 plugin_decision = Plugins.onModuleEncounter( 84 module_filename, module_name, module_kind 85 ) 86 87 if plugin_decision is not None: 88 return plugin_decision 89 90 if module_kind == "shlib": 91 if Options.isStandaloneMode(): 92 return True, "Extension module needed for standalone mode." 93 else: 94 return False, "Shared library cannot be inspected." 95 96 no_case, reason = module_name.matchesToShellPatterns( 97 patterns=Options.getShallFollowInNoCase() 98 ) 99 100 if no_case: 101 return (False, "Module %s instructed by user to not recurse to." % reason) 102 103 any_case, reason = module_name.matchesToShellPatterns( 104 patterns=Options.getShallFollowModules() 105 ) 106 107 if any_case: 108 return (True, "Module %s instructed by user to recurse to." % reason) 109 110 if Options.shallFollowNoImports(): 111 return (False, "Requested to not recurse at all.") 112 113 if StandardLibrary.isStandardLibraryPath(module_filename): 114 return ( 115 Options.shallFollowStandardLibrary(), 116 "Requested to %srecurse to standard library." 117 % ("" if Options.shallFollowStandardLibrary() else "not "), 118 ) 119 120 if Options.shallFollowAllImports(): 121 return (True, "Requested to recurse to all non-standard library modules.") 122 123 # Means, we were not given instructions how to handle things. 124 if extra_recursion: 125 return (True, "Lives in plug-in directory.") 126 127 if Options.shallMakeModule(): 128 return (False, "Making a module, not following any imports by default.") 129 130 return (None, "Default behavior, not recursing without request.") 131 132 133def considerFilename(module_filename): 134 module_filename = os.path.normpath(module_filename) 135 136 if os.path.isdir(module_filename): 137 module_filename = os.path.abspath(module_filename) 138 139 module_name = os.path.basename(module_filename) 140 module_relpath = relpath(module_filename) 141 142 return module_filename, module_relpath, module_name 143 elif module_filename.endswith(".py"): 144 module_name = os.path.basename(module_filename)[:-3] 145 module_relpath = relpath(module_filename) 146 147 return module_filename, module_relpath, module_name 148 elif module_filename.endswith(".pyw"): 149 module_name = os.path.basename(module_filename)[:-4] 150 module_relpath = relpath(module_filename) 151 152 return module_filename, module_relpath, module_name 153 else: 154 return None 155 156 157def isSameModulePath(path1, path2): 158 if os.path.basename(path1) == "__init__.py": 159 path1 = os.path.dirname(path1) 160 if os.path.basename(path2) == "__init__.py": 161 path2 = os.path.dirname(path2) 162 163 return os.path.abspath(path1) == os.path.abspath(path2) 164 165 166def checkPluginSinglePath(plugin_filename, module_package): 167 # Many branches, for the decision is very complex, pylint: disable=too-many-branches 168 169 if Options.isShowInclusion(): 170 recursion_logger.info( 171 "Checking detail plug-in path '%s' '%s':" 172 % (plugin_filename, module_package) 173 ) 174 175 module_name, module_kind = Importing.getModuleNameAndKindFromFilename( 176 plugin_filename 177 ) 178 179 module_name = ModuleName.makeModuleNameInPackage(module_name, module_package) 180 181 if module_kind is not None: 182 decision, reason = decideRecursion( 183 module_filename=plugin_filename, 184 module_name=module_name, 185 module_kind=module_kind, 186 extra_recursion=True, 187 ) 188 189 if decision: 190 module_relpath = relpath(plugin_filename) 191 192 module = recurseTo( 193 signal_change=None, 194 module_filename=plugin_filename, 195 module_relpath=module_relpath, 196 module_package=module_package, 197 module_kind=module_kind, 198 reason=reason, 199 ) 200 201 if module: 202 if Options.isShowInclusion(): 203 recursion_logger.info( 204 "Included '%s' as '%s'." 205 % ( 206 module.getFullName(), 207 module, 208 ) 209 ) 210 211 ImportCache.addImportedModule(module) 212 213 if module.isCompiledPythonPackage(): 214 package_filename = module.getFilename() 215 216 if os.path.isdir(package_filename): 217 # Must be a namespace package. 218 assert python_version >= 0x300 219 220 package_dir = package_filename 221 222 # Only include it, if it contains actual modules, which will 223 # recurse to this one and find it again. 224 else: 225 package_dir = os.path.dirname(package_filename) 226 227 # Real packages will always be included. 228 ModuleRegistry.addRootModule(module) 229 230 if Options.isShowInclusion(): 231 recursion_logger.info("Package directory '%s'." % package_dir) 232 233 for sub_path, sub_filename in listDir(package_dir): 234 if sub_filename in ("__init__.py", "__pycache__"): 235 continue 236 237 assert sub_path != plugin_filename 238 239 if Importing.isPackageDir(sub_path) and not os.path.exists( 240 sub_path + ".py" 241 ): 242 checkPluginSinglePath( 243 sub_path, module_package=module.getFullName() 244 ) 245 elif sub_path.endswith(".py"): 246 checkPluginSinglePath( 247 sub_path, module_package=module.getFullName() 248 ) 249 250 elif module.isCompiledPythonModule(): 251 ModuleRegistry.addRootModule(module) 252 elif module.isPythonShlibModule(): 253 if Options.isStandaloneMode(): 254 ModuleRegistry.addRootModule(module) 255 256 else: 257 recursion_logger.warning( 258 "Failed to include module from '%s'." % plugin_filename 259 ) 260 261 262def checkPluginPath(plugin_filename, module_package): 263 plugin_filename = os.path.normpath(plugin_filename) 264 265 if Options.isShowInclusion(): 266 recursion_logger.info( 267 "Checking top level plug-in path %s %s" % (plugin_filename, module_package) 268 ) 269 270 plugin_info = considerFilename(module_filename=plugin_filename) 271 272 if plugin_info is not None: 273 # File or package makes a difference, handle that 274 if os.path.isfile(plugin_info[0]) or Importing.isPackageDir(plugin_info[0]): 275 checkPluginSinglePath(plugin_filename, module_package=module_package) 276 elif os.path.isdir(plugin_info[0]): 277 for sub_path, sub_filename in listDir(plugin_info[0]): 278 assert sub_filename != "__init__.py" 279 280 if Importing.isPackageDir(sub_path) or sub_path.endswith(".py"): 281 checkPluginSinglePath(sub_path, module_package=None) 282 else: 283 recursion_logger.warning( 284 "Failed to include module from %r." % plugin_info[0] 285 ) 286 else: 287 recursion_logger.warning("Failed to recurse to directory %r." % plugin_filename) 288 289 290def checkPluginFilenamePattern(pattern): 291 if Options.isShowInclusion(): 292 recursion_logger.info("Checking plug-in pattern '%s':" % pattern) 293 294 assert not os.path.isdir(pattern), pattern 295 296 found = False 297 298 for filename in glob.iglob(pattern): 299 if filename.endswith(".pyc"): 300 continue 301 302 if not os.path.isfile(filename): 303 continue 304 305 found = True 306 checkPluginSinglePath(filename, module_package=None) 307 308 if not found: 309 recursion_logger.warning("Didn't match any files against pattern %r." % pattern) 310