1""" 2:mod:`jedi.inference.imports` is here to resolve import statements and return 3the modules/classes/functions/whatever, which they stand for. However there's 4not any actual importing done. This module is about finding modules in the 5filesystem. This can be quite tricky sometimes, because Python imports are not 6always that simple. 7 8This module also supports import autocompletion, which means to complete 9statements like ``from datetim`` (cursor at the end would return ``datetime``). 10""" 11import os 12from pathlib import Path 13 14from parso.python import tree 15from parso.tree import search_ancestor 16 17from jedi import debug 18from jedi import settings 19from jedi.file_io import FolderIO 20from jedi.parser_utils import get_cached_code_lines 21from jedi.inference import sys_path 22from jedi.inference import helpers 23from jedi.inference import compiled 24from jedi.inference import analysis 25from jedi.inference.utils import unite 26from jedi.inference.cache import inference_state_method_cache 27from jedi.inference.names import ImportName, SubModuleName 28from jedi.inference.base_value import ValueSet, NO_VALUES 29from jedi.inference.gradual.typeshed import import_module_decorator, \ 30 create_stub_module, parse_stub_module 31from jedi.inference.compiled.subprocess.functions import ImplicitNSInfo 32from jedi.plugins import plugin_manager 33 34 35class ModuleCache: 36 def __init__(self): 37 self._name_cache = {} 38 39 def add(self, string_names, value_set): 40 if string_names is not None: 41 self._name_cache[string_names] = value_set 42 43 def get(self, string_names): 44 return self._name_cache.get(string_names) 45 46 47# This memoization is needed, because otherwise we will infinitely loop on 48# certain imports. 49@inference_state_method_cache(default=NO_VALUES) 50def infer_import(context, tree_name): 51 module_context = context.get_root_context() 52 from_import_name, import_path, level, values = \ 53 _prepare_infer_import(module_context, tree_name) 54 if values: 55 56 if from_import_name is not None: 57 values = values.py__getattribute__( 58 from_import_name, 59 name_context=context, 60 analysis_errors=False 61 ) 62 63 if not values: 64 path = import_path + (from_import_name,) 65 importer = Importer(context.inference_state, path, module_context, level) 66 values = importer.follow() 67 debug.dbg('after import: %s', values) 68 return values 69 70 71@inference_state_method_cache(default=[]) 72def goto_import(context, tree_name): 73 module_context = context.get_root_context() 74 from_import_name, import_path, level, values = \ 75 _prepare_infer_import(module_context, tree_name) 76 if not values: 77 return [] 78 79 if from_import_name is not None: 80 names = unite([ 81 c.goto( 82 from_import_name, 83 name_context=context, 84 analysis_errors=False 85 ) for c in values 86 ]) 87 # Avoid recursion on the same names. 88 if names and not any(n.tree_name is tree_name for n in names): 89 return names 90 91 path = import_path + (from_import_name,) 92 importer = Importer(context.inference_state, path, module_context, level) 93 values = importer.follow() 94 return set(s.name for s in values) 95 96 97def _prepare_infer_import(module_context, tree_name): 98 import_node = search_ancestor(tree_name, 'import_name', 'import_from') 99 import_path = import_node.get_path_for_name(tree_name) 100 from_import_name = None 101 try: 102 from_names = import_node.get_from_names() 103 except AttributeError: 104 # Is an import_name 105 pass 106 else: 107 if len(from_names) + 1 == len(import_path): 108 # We have to fetch the from_names part first and then check 109 # if from_names exists in the modules. 110 from_import_name = import_path[-1] 111 import_path = from_names 112 113 importer = Importer(module_context.inference_state, tuple(import_path), 114 module_context, import_node.level) 115 116 return from_import_name, tuple(import_path), import_node.level, importer.follow() 117 118 119def _add_error(value, name, message): 120 if hasattr(name, 'parent') and value is not None: 121 analysis.add(value, 'import-error', name, message) 122 else: 123 debug.warning('ImportError without origin: ' + message) 124 125 126def _level_to_base_import_path(project_path, directory, level): 127 """ 128 In case the level is outside of the currently known package (something like 129 import .....foo), we can still try our best to help the user for 130 completions. 131 """ 132 for i in range(level - 1): 133 old = directory 134 directory = os.path.dirname(directory) 135 if old == directory: 136 return None, None 137 138 d = directory 139 level_import_paths = [] 140 # Now that we are on the level that the user wants to be, calculate the 141 # import path for it. 142 while True: 143 if d == project_path: 144 return level_import_paths, d 145 dir_name = os.path.basename(d) 146 if dir_name: 147 level_import_paths.insert(0, dir_name) 148 d = os.path.dirname(d) 149 else: 150 return None, directory 151 152 153class Importer: 154 def __init__(self, inference_state, import_path, module_context, level=0): 155 """ 156 An implementation similar to ``__import__``. Use `follow` 157 to actually follow the imports. 158 159 *level* specifies whether to use absolute or relative imports. 0 (the 160 default) means only perform absolute imports. Positive values for level 161 indicate the number of parent directories to search relative to the 162 directory of the module calling ``__import__()`` (see PEP 328 for the 163 details). 164 165 :param import_path: List of namespaces (strings or Names). 166 """ 167 debug.speed('import %s %s' % (import_path, module_context)) 168 self._inference_state = inference_state 169 self.level = level 170 self._module_context = module_context 171 172 self._fixed_sys_path = None 173 self._infer_possible = True 174 if level: 175 base = module_context.get_value().py__package__() 176 # We need to care for two cases, the first one is if it's a valid 177 # Python import. This import has a properly defined module name 178 # chain like `foo.bar.baz` and an import in baz is made for 179 # `..lala.` It can then resolve to `foo.bar.lala`. 180 # The else here is a heuristic for all other cases, if for example 181 # in `foo` you search for `...bar`, it's obviously out of scope. 182 # However since Jedi tries to just do it's best, we help the user 183 # here, because he might have specified something wrong in his 184 # project. 185 if level <= len(base): 186 # Here we basically rewrite the level to 0. 187 base = tuple(base) 188 if level > 1: 189 base = base[:-level + 1] 190 import_path = base + tuple(import_path) 191 else: 192 path = module_context.py__file__() 193 project_path = self._inference_state.project.path 194 import_path = list(import_path) 195 if path is None: 196 # If no path is defined, our best guess is that the current 197 # file is edited by a user on the current working 198 # directory. We need to add an initial path, because it 199 # will get removed as the name of the current file. 200 directory = project_path 201 else: 202 directory = os.path.dirname(path) 203 204 base_import_path, base_directory = _level_to_base_import_path( 205 project_path, directory, level, 206 ) 207 if base_directory is None: 208 # Everything is lost, the relative import does point 209 # somewhere out of the filesystem. 210 self._infer_possible = False 211 else: 212 self._fixed_sys_path = [base_directory] 213 214 if base_import_path is None: 215 if import_path: 216 _add_error( 217 module_context, import_path[0], 218 message='Attempted relative import beyond top-level package.' 219 ) 220 else: 221 import_path = base_import_path + import_path 222 self.import_path = import_path 223 224 @property 225 def _str_import_path(self): 226 """Returns the import path as pure strings instead of `Name`.""" 227 return tuple( 228 name.value if isinstance(name, tree.Name) else name 229 for name in self.import_path 230 ) 231 232 def _sys_path_with_modifications(self, is_completion): 233 if self._fixed_sys_path is not None: 234 return self._fixed_sys_path 235 236 return ( 237 # For import completions we don't want to see init paths, but for 238 # inference we want to show the user as much as possible. 239 # See GH #1446. 240 self._inference_state.get_sys_path(add_init_paths=not is_completion) 241 + [ 242 str(p) for p 243 in sys_path.check_sys_path_modifications(self._module_context) 244 ] 245 ) 246 247 def follow(self): 248 if not self.import_path: 249 if self._fixed_sys_path: 250 # This is a bit of a special case, that maybe should be 251 # revisited. If the project path is wrong or the user uses 252 # relative imports the wrong way, we might end up here, where 253 # the `fixed_sys_path == project.path` in that case we kind of 254 # use the project.path.parent directory as our path. This is 255 # usually not a problem, except if imports in other places are 256 # using the same names. Example: 257 # 258 # foo/ < #1 259 # - setup.py 260 # - foo/ < #2 261 # - __init__.py 262 # - foo.py < #3 263 # 264 # If the top foo is our project folder and somebody uses 265 # `from . import foo` in `setup.py`, it will resolve to foo #2, 266 # which means that the import for foo.foo is cached as 267 # `__init__.py` (#2) and not as `foo.py` (#3). This is usually 268 # not an issue, because this case is probably pretty rare, but 269 # might be an issue for some people. 270 # 271 # However for most normal cases where we work with different 272 # file names, this code path hits where we basically change the 273 # project path to an ancestor of project path. 274 from jedi.inference.value.namespace import ImplicitNamespaceValue 275 import_path = (os.path.basename(self._fixed_sys_path[0]),) 276 ns = ImplicitNamespaceValue( 277 self._inference_state, 278 string_names=import_path, 279 paths=self._fixed_sys_path, 280 ) 281 return ValueSet({ns}) 282 return NO_VALUES 283 if not self._infer_possible: 284 return NO_VALUES 285 286 # Check caches first 287 from_cache = self._inference_state.stub_module_cache.get(self._str_import_path) 288 if from_cache is not None: 289 return ValueSet({from_cache}) 290 from_cache = self._inference_state.module_cache.get(self._str_import_path) 291 if from_cache is not None: 292 return from_cache 293 294 sys_path = self._sys_path_with_modifications(is_completion=False) 295 296 return import_module_by_names( 297 self._inference_state, self.import_path, sys_path, self._module_context 298 ) 299 300 def _get_module_names(self, search_path=None, in_module=None): 301 """ 302 Get the names of all modules in the search_path. This means file names 303 and not names defined in the files. 304 """ 305 if search_path is None: 306 sys_path = self._sys_path_with_modifications(is_completion=True) 307 else: 308 sys_path = search_path 309 return list(iter_module_names( 310 self._inference_state, self._module_context, sys_path, 311 module_cls=ImportName if in_module is None else SubModuleName, 312 add_builtin_modules=search_path is None and in_module is None, 313 )) 314 315 def completion_names(self, inference_state, only_modules=False): 316 """ 317 :param only_modules: Indicates wheter it's possible to import a 318 definition that is not defined in a module. 319 """ 320 if not self._infer_possible: 321 return [] 322 323 names = [] 324 if self.import_path: 325 # flask 326 if self._str_import_path == ('flask', 'ext'): 327 # List Flask extensions like ``flask_foo`` 328 for mod in self._get_module_names(): 329 modname = mod.string_name 330 if modname.startswith('flask_'): 331 extname = modname[len('flask_'):] 332 names.append(ImportName(self._module_context, extname)) 333 # Now the old style: ``flaskext.foo`` 334 for dir in self._sys_path_with_modifications(is_completion=True): 335 flaskext = os.path.join(dir, 'flaskext') 336 if os.path.isdir(flaskext): 337 names += self._get_module_names([flaskext]) 338 339 values = self.follow() 340 for value in values: 341 # Non-modules are not completable. 342 if value.api_type != 'module': # not a module 343 continue 344 if not value.is_compiled(): 345 # sub_modules_dict is not implemented for compiled modules. 346 names += value.sub_modules_dict().values() 347 348 if not only_modules: 349 from jedi.inference.gradual.conversion import convert_values 350 351 both_values = values | convert_values(values) 352 for c in both_values: 353 for filter in c.get_filters(): 354 names += filter.values() 355 else: 356 if self.level: 357 # We only get here if the level cannot be properly calculated. 358 names += self._get_module_names(self._fixed_sys_path) 359 else: 360 # This is just the list of global imports. 361 names += self._get_module_names() 362 return names 363 364 365def import_module_by_names(inference_state, import_names, sys_path=None, 366 module_context=None, prefer_stubs=True): 367 if sys_path is None: 368 sys_path = inference_state.get_sys_path() 369 370 str_import_names = tuple( 371 i.value if isinstance(i, tree.Name) else i 372 for i in import_names 373 ) 374 value_set = [None] 375 for i, name in enumerate(import_names): 376 value_set = ValueSet.from_sets([ 377 import_module( 378 inference_state, 379 str_import_names[:i+1], 380 parent_module_value, 381 sys_path, 382 prefer_stubs=prefer_stubs, 383 ) for parent_module_value in value_set 384 ]) 385 if not value_set: 386 message = 'No module named ' + '.'.join(str_import_names) 387 if module_context is not None: 388 _add_error(module_context, name, message) 389 else: 390 debug.warning(message) 391 return NO_VALUES 392 return value_set 393 394 395@plugin_manager.decorate() 396@import_module_decorator 397def import_module(inference_state, import_names, parent_module_value, sys_path): 398 """ 399 This method is very similar to importlib's `_gcd_import`. 400 """ 401 if import_names[0] in settings.auto_import_modules: 402 module = _load_builtin_module(inference_state, import_names, sys_path) 403 if module is None: 404 return NO_VALUES 405 return ValueSet([module]) 406 407 module_name = '.'.join(import_names) 408 if parent_module_value is None: 409 # Override the sys.path. It works only good that way. 410 # Injecting the path directly into `find_module` did not work. 411 file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info( 412 string=import_names[-1], 413 full_name=module_name, 414 sys_path=sys_path, 415 is_global_search=True, 416 ) 417 if is_pkg is None: 418 return NO_VALUES 419 else: 420 paths = parent_module_value.py__path__() 421 if paths is None: 422 # The module might not be a package. 423 return NO_VALUES 424 425 for path in paths: 426 # At the moment we are only using one path. So this is 427 # not important to be correct. 428 if not isinstance(path, list): 429 path = [path] 430 file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info( 431 string=import_names[-1], 432 path=path, 433 full_name=module_name, 434 is_global_search=False, 435 ) 436 if is_pkg is not None: 437 break 438 else: 439 return NO_VALUES 440 441 if isinstance(file_io_or_ns, ImplicitNSInfo): 442 from jedi.inference.value.namespace import ImplicitNamespaceValue 443 module = ImplicitNamespaceValue( 444 inference_state, 445 string_names=tuple(file_io_or_ns.name.split('.')), 446 paths=file_io_or_ns.paths, 447 ) 448 elif file_io_or_ns is None: 449 module = _load_builtin_module(inference_state, import_names, sys_path) 450 if module is None: 451 return NO_VALUES 452 else: 453 module = _load_python_module( 454 inference_state, file_io_or_ns, 455 import_names=import_names, 456 is_package=is_pkg, 457 ) 458 459 if parent_module_value is None: 460 debug.dbg('global search_module %s: %s', import_names[-1], module) 461 else: 462 debug.dbg('search_module %s in paths %s: %s', module_name, paths, module) 463 return ValueSet([module]) 464 465 466def _load_python_module(inference_state, file_io, 467 import_names=None, is_package=False): 468 module_node = inference_state.parse( 469 file_io=file_io, 470 cache=True, 471 diff_cache=settings.fast_parser, 472 cache_path=settings.cache_directory, 473 ) 474 475 from jedi.inference.value import ModuleValue 476 return ModuleValue( 477 inference_state, module_node, 478 file_io=file_io, 479 string_names=import_names, 480 code_lines=get_cached_code_lines(inference_state.grammar, file_io.path), 481 is_package=is_package, 482 ) 483 484 485def _load_builtin_module(inference_state, import_names=None, sys_path=None): 486 project = inference_state.project 487 if sys_path is None: 488 sys_path = inference_state.get_sys_path() 489 if not project._load_unsafe_extensions: 490 safe_paths = project._get_base_sys_path(inference_state) 491 sys_path = [p for p in sys_path if p in safe_paths] 492 493 dotted_name = '.'.join(import_names) 494 assert dotted_name is not None 495 module = compiled.load_module(inference_state, dotted_name=dotted_name, sys_path=sys_path) 496 if module is None: 497 # The file might raise an ImportError e.g. and therefore not be 498 # importable. 499 return None 500 return module 501 502 503def load_module_from_path(inference_state, file_io, import_names=None, is_package=None): 504 """ 505 This should pretty much only be used for get_modules_containing_name. It's 506 here to ensure that a random path is still properly loaded into the Jedi 507 module structure. 508 """ 509 path = Path(file_io.path) 510 if import_names is None: 511 e_sys_path = inference_state.get_sys_path() 512 import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path) 513 else: 514 assert isinstance(is_package, bool) 515 516 is_stub = path.suffix == '.pyi' 517 if is_stub: 518 folder_io = file_io.get_parent_folder() 519 if folder_io.path.endswith('-stubs'): 520 folder_io = FolderIO(folder_io.path[:-6]) 521 if path.name == '__init__.pyi': 522 python_file_io = folder_io.get_file_io('__init__.py') 523 else: 524 python_file_io = folder_io.get_file_io(import_names[-1] + '.py') 525 526 try: 527 v = load_module_from_path( 528 inference_state, python_file_io, 529 import_names, is_package=is_package 530 ) 531 values = ValueSet([v]) 532 except FileNotFoundError: 533 values = NO_VALUES 534 535 return create_stub_module( 536 inference_state, inference_state.latest_grammar, values, 537 parse_stub_module(inference_state, file_io), file_io, import_names 538 ) 539 else: 540 module = _load_python_module( 541 inference_state, file_io, 542 import_names=import_names, 543 is_package=is_package, 544 ) 545 inference_state.module_cache.add(import_names, ValueSet([module])) 546 return module 547 548 549def load_namespace_from_path(inference_state, folder_io): 550 import_names, is_package = sys_path.transform_path_to_dotted( 551 inference_state.get_sys_path(), 552 Path(folder_io.path) 553 ) 554 from jedi.inference.value.namespace import ImplicitNamespaceValue 555 return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path]) 556 557 558def follow_error_node_imports_if_possible(context, name): 559 error_node = tree.search_ancestor(name, 'error_node') 560 if error_node is not None: 561 # Get the first command start of a started simple_stmt. The error 562 # node is sometimes a small_stmt and sometimes a simple_stmt. Check 563 # for ; leaves that start a new statements. 564 start_index = 0 565 for index, n in enumerate(error_node.children): 566 if n.start_pos > name.start_pos: 567 break 568 if n == ';': 569 start_index = index + 1 570 nodes = error_node.children[start_index:] 571 first_name = nodes[0].get_first_leaf().value 572 573 # Make it possible to infer stuff like `import foo.` or 574 # `from foo.bar`. 575 if first_name in ('from', 'import'): 576 is_import_from = first_name == 'from' 577 level, names = helpers.parse_dotted_names( 578 nodes, 579 is_import_from=is_import_from, 580 until_node=name, 581 ) 582 return Importer( 583 context.inference_state, names, context.get_root_context(), level).follow() 584 return None 585 586 587def iter_module_names(inference_state, module_context, search_path, 588 module_cls=ImportName, add_builtin_modules=True): 589 """ 590 Get the names of all modules in the search_path. This means file names 591 and not names defined in the files. 592 """ 593 # add builtin module names 594 if add_builtin_modules: 595 for name in inference_state.compiled_subprocess.get_builtin_module_names(): 596 yield module_cls(module_context, name) 597 598 for name in inference_state.compiled_subprocess.iter_module_names(search_path): 599 yield module_cls(module_context, name) 600