1"""Code for translating between type systems.""" 2 3import logging 4import types 5 6from pytype import abstract 7from pytype import abstract_utils 8from pytype import blocks 9from pytype import class_mixin 10from pytype import datatypes 11from pytype import function 12from pytype import overlay_dict 13from pytype import mixin 14from pytype import output 15from pytype import special_builtins 16from pytype import utils 17from pytype.overlays import typing_overlay 18from pytype.pyc import loadmarshal 19from pytype.pytd import mro 20from pytype.pytd import pytd 21from pytype.pytd import pytd_utils 22from pytype.pytd.parse import parser_constants 23from pytype.typegraph import cfg 24 25 26log = logging.getLogger(__name__) 27 28 29MAX_IMPORT_DEPTH = 12 30 31 32# types not exposed as python classes 33NoneType = type(None) 34EllipsisType = type(Ellipsis) 35 36 37class IteratorType: 38 pass 39 40 41class CoroutineType: 42 pass 43 44 45class AwaitableType: 46 pass 47 48 49class AsyncGeneratorType: 50 pass 51 52 53class Converter(utils.VirtualMachineWeakrefMixin): 54 """Functions for creating the classes in abstract.py.""" 55 56 unsolvable: abstract.Unsolvable 57 58 # Define this error inside Converter so that it is exposed to abstract.py 59 class TypeParameterError(Exception): 60 61 def __init__(self, type_param_name): 62 super().__init__(type_param_name) 63 self.type_param_name = type_param_name 64 65 def __init__(self, vm): 66 super().__init__(vm) 67 self.vm.convert = self # to make constant_to_value calls below work 68 self.pytd_convert = output.Converter(vm) 69 70 # If set, allow construction of recursive values, setting the 71 # self-referential field to Any 72 self.recursion_allowed = False 73 74 self._convert_cache = {} 75 self._resolved_late_types = {} # performance cache 76 77 # Initialize primitive_classes to empty to allow constant_to_value to run. 78 self.primitive_classes = () 79 80 # object_type is needed to initialize the primitive class values. 81 self.object_type = self.constant_to_value(object) 82 83 self.unsolvable = abstract.Unsolvable(self.vm) 84 self.empty = abstract.Empty(self.vm) 85 self.no_return = typing_overlay.NoReturn(self.vm) 86 87 # Now fill primitive_classes with the real values using constant_to_value. 88 primitive_classes = [ 89 int, float, str, bytes, object, NoneType, complex, bool, slice, 90 types.CodeType, EllipsisType, super, 91 ] 92 self.primitive_classes = { 93 v: self.constant_to_value(v) for v in primitive_classes 94 } 95 self.primitive_class_names = [ 96 self._type_to_name(x) for x in self.primitive_classes] 97 self.none = abstract.ConcreteValue(None, 98 self.primitive_classes[NoneType], 99 self.vm) 100 self.true = abstract.ConcreteValue(True, self.primitive_classes[bool], 101 self.vm) 102 self.false = abstract.ConcreteValue(False, self.primitive_classes[bool], 103 self.vm) 104 self.ellipsis = abstract.ConcreteValue( 105 Ellipsis, self.primitive_classes[EllipsisType], self.vm) 106 107 self.primitive_class_instances = {} 108 for name, cls in self.primitive_classes.items(): 109 if name == NoneType: 110 # This is possible because all None instances are the same. 111 # Without it pytype could not reason that "x is None" is always true, if 112 # x is indeed None. 113 instance = self.none 114 elif name == EllipsisType: 115 instance = self.ellipsis 116 else: 117 instance = abstract.Instance(cls, self.vm) 118 self.primitive_class_instances[name] = instance 119 self._convert_cache[(abstract.Instance, cls.pytd_cls)] = instance 120 121 self.none_type = self.primitive_classes[NoneType] 122 self.super_type = self.primitive_classes[super] 123 self.str_type = self.primitive_classes[str] 124 self.int_type = self.primitive_classes[int] 125 self.bool_type = self.primitive_classes[bool] 126 # TODO(b/195453869): get rid of unicode_type 127 self.unicode_type = self.str_type 128 self.bytes_type = self.primitive_classes[bytes] 129 130 self.list_type = self.constant_to_value(list) 131 self.set_type = self.constant_to_value(set) 132 self.frozenset_type = self.constant_to_value(frozenset) 133 self.dict_type = self.constant_to_value(dict) 134 self.type_type = self.constant_to_value(type) 135 self.module_type = self.constant_to_value(types.ModuleType) 136 self.function_type = self.constant_to_value(types.FunctionType) 137 self.tuple_type = self.constant_to_value(tuple) 138 self.generator_type = self.constant_to_value(types.GeneratorType) 139 self.iterator_type = self.constant_to_value(IteratorType) 140 self.coroutine_type = self.constant_to_value(CoroutineType) 141 self.awaitable_type = self.constant_to_value(AwaitableType) 142 if self.vm.python_version >= (3, 6): 143 self.async_generator_type = self.constant_to_value( 144 AsyncGeneratorType) 145 self.bool_values = { 146 True: self.true, 147 False: self.false, 148 None: self.primitive_class_instances[bool], 149 } 150 self.next_attr = "__next__" 151 152 def constant_name(self, constant_type): 153 if constant_type is None: 154 return "constant" 155 elif isinstance(constant_type, tuple): 156 return "(%s)" % ", ".join(self.constant_name(c) for c in constant_type) 157 else: 158 return constant_type.__name__ 159 160 def _type_to_name(self, t): 161 """Convert a type to its name.""" 162 assert t.__class__ is type 163 if t is types.FunctionType: 164 return "typing.Callable" 165 elif t is IteratorType: 166 return "builtins.object" 167 elif t is CoroutineType: 168 return "builtins.coroutine" 169 elif t is AwaitableType: 170 return "typing.Awaitable" 171 elif t is AsyncGeneratorType: 172 return "builtins.asyncgenerator" 173 else: 174 return "builtins." + t.__name__ 175 176 def value_to_constant(self, val, constant_type): 177 if (abstract_utils.is_concrete(val) and 178 isinstance(val.pyval, constant_type or object)): 179 return val.pyval 180 name = self.constant_name(constant_type) 181 raise abstract_utils.ConversionError("%s is not of type %s" % (val, name)) 182 183 def name_to_value(self, name, subst=None, ast=None): 184 if ast is None: 185 pytd_cls = self.vm.lookup_builtin(name) 186 else: 187 pytd_cls = ast.Lookup(name) 188 subst = subst or datatypes.AliasingDict() 189 return self.constant_to_value(pytd_cls, subst, self.vm.root_node) 190 191 def tuple_to_value(self, content): 192 """Create a VM tuple from the given sequence.""" 193 content = tuple(content) # content might be a generator 194 value = abstract.Tuple(content, self.vm) 195 return value 196 197 def build_none(self, node): 198 return self.none.to_variable(node) 199 200 def build_bool(self, node, value=None): 201 # pylint: disable=g-bool-id-comparison 202 if value is None: 203 return self.primitive_class_instances[bool].to_variable(node) 204 elif value is True: 205 return self.true.to_variable(node) 206 elif value is False: 207 return self.false.to_variable(node) 208 else: 209 raise ValueError("Invalid bool value: %r" % value) 210 211 def build_int(self, node): 212 i = self.primitive_class_instances[int] 213 return i.to_variable(node) 214 215 def build_string(self, node, s): 216 del node 217 return self.constant_to_var(s) 218 219 def build_content(self, elements): 220 if len(elements) == 1: 221 return next(iter(elements)) 222 var = self.vm.program.NewVariable() 223 for v in elements: 224 var.PasteVariable(v) 225 return var 226 227 def build_slice(self, node, start, stop, step=None): 228 const_types = (int, type(None)) 229 try: 230 if start: 231 start = abstract_utils.get_atomic_python_constant(start, const_types) 232 if stop: 233 stop = abstract_utils.get_atomic_python_constant(stop, const_types) 234 if step: 235 step = abstract_utils.get_atomic_python_constant(step, const_types) 236 except abstract_utils.ConversionError: 237 return self.primitive_class_instances[slice].to_variable(node) 238 return abstract.ConcreteValue( 239 slice(start, stop, step), self.primitive_classes[slice], 240 self.vm).to_variable(node) 241 242 def build_list(self, node, content): 243 """Create a VM list from the given sequence.""" 244 # TODO(rechen): set T to empty if there is nothing in content 245 content = [var.AssignToNewVariable(node) for var in content] 246 return abstract.List(content, self.vm).to_variable(node) 247 248 def build_collection_of_type(self, node, typ, var): 249 """Create a collection Typ[T] with T derived from the given variable.""" 250 ret = abstract.Instance(typ, self.vm) 251 ret.merge_instance_type_parameter(node, abstract_utils.T, var) 252 return ret.to_variable(node) 253 254 def build_list_of_type(self, node, var): 255 """Create a VM list with element type derived from the given variable.""" 256 return self.build_collection_of_type(node, self.list_type, var) 257 258 def build_set(self, node, content): 259 """Create a VM set from the given sequence.""" 260 content = list(content) # content might be a generator 261 value = abstract.Instance(self.set_type, self.vm) 262 value.merge_instance_type_parameter( 263 node, abstract_utils.T, self.build_content(content)) 264 return value.to_variable(node) 265 266 def build_map(self, node): 267 """Create an empty VM dict.""" 268 return abstract.Dict(self.vm).to_variable(node) 269 270 def build_tuple(self, node, content): 271 """Create a VM tuple from the given sequence.""" 272 return self.tuple_to_value(content).to_variable(node) 273 274 def get_maybe_abstract_instance(self, data): 275 """Get an instance of the same type as the given data, abstract if possible. 276 277 Get an abstract instance of primitive data stored as a 278 ConcreteValue. Return any other data as-is. This is used by 279 constant_to_var to discard concrete values that have been kept 280 around for InterpreterFunction. 281 282 Arguments: 283 data: The data. 284 285 Returns: 286 An instance of the same type as the data, abstract if possible. 287 """ 288 if isinstance(data, mixin.PythonConstant): 289 data_type = type(data.pyval) 290 if data_type in self.primitive_class_instances: 291 return self.primitive_class_instances[data_type] 292 return data 293 294 def _create_new_unknown_value(self, action): 295 if not action or not self.vm.frame: 296 return abstract.Unknown(self.vm) 297 # We allow only one Unknown at each point in the program, regardless of 298 # what the call stack is. 299 key = ("unknown", self.vm.frame.current_opcode, action) 300 if key not in self._convert_cache: 301 self._convert_cache[key] = abstract.Unknown(self.vm) 302 return self._convert_cache[key] 303 304 def create_new_unknown(self, node, source=None, action=None, force=False): 305 """Create a new variable containing unknown.""" 306 if not force and not self.vm.generate_unknowns: 307 # unsolvable instances are cheaper than unknown, so use those for --quick. 308 return self.unsolvable.to_variable(node) 309 unknown = self._create_new_unknown_value(action) 310 v = self.vm.program.NewVariable() 311 val = v.AddBinding( 312 unknown, source_set=[source] if source else [], where=node) 313 unknown.owner = val 314 self.vm.trace_unknown(unknown.class_name, val) 315 return v 316 317 def create_new_varargs_value(self, arg_type): 318 """Create a varargs argument given its element type.""" 319 params = {abstract_utils.T: arg_type} 320 return abstract.ParameterizedClass(self.tuple_type, params, self.vm) 321 322 def create_new_kwargs_value(self, arg_type): 323 """Create a kwargs argument given its element type.""" 324 params = {abstract_utils.K: self.str_type, abstract_utils.V: arg_type} 325 return abstract.ParameterizedClass(self.dict_type, params, self.vm) 326 327 def get_element_type(self, arg_type): 328 """Extract the element type of a vararg or kwarg.""" 329 if not isinstance(arg_type, abstract.ParameterizedClass): 330 assert (isinstance(arg_type, class_mixin.Class) and 331 arg_type.full_name in ("builtins.dict", "builtins.tuple")) 332 return None 333 elif arg_type.base_cls is self.dict_type: 334 return arg_type.get_formal_type_parameter(abstract_utils.V) 335 else: 336 assert arg_type.base_cls is self.tuple_type 337 return arg_type.get_formal_type_parameter(abstract_utils.T) 338 339 def _copy_type_parameters(self, old_container, new_container_name): 340 new_container = self.name_to_value(new_container_name) 341 if isinstance(old_container, abstract.ParameterizedClass): 342 return abstract.ParameterizedClass( 343 new_container, old_container.formal_type_parameters, self.vm) 344 else: 345 assert isinstance(old_container, class_mixin.Class) 346 return new_container 347 348 def widen_type(self, container): 349 """Widen a tuple to an iterable, or a dict to a mapping.""" 350 if container.full_name == "builtins.tuple": 351 return self._copy_type_parameters(container, "typing.Iterable") 352 else: 353 assert container.full_name == "builtins.dict", container.full_name 354 return self._copy_type_parameters(container, "typing.Mapping") 355 356 def merge_classes(self, instances): 357 """Merge the classes of the given instances. 358 359 Args: 360 instances: An iterable of instances. 361 Returns: 362 An abstract.BaseValue created by merging the instances' classes. 363 """ 364 classes = set() 365 for v in instances: 366 cls = v.get_class() 367 if cls and cls != self.empty: 368 classes.add(cls) 369 return self.vm.merge_values(classes) 370 371 def constant_to_var(self, pyval, subst=None, node=None, source_sets=None, 372 discard_concrete_values=False): 373 """Convert a constant to a Variable. 374 375 This converts a constant to a cfg.Variable. Unlike constant_to_value, it 376 can handle things that need to be represented as a Variable with multiple 377 possible values (i.e., a union type), like pytd.Function. 378 379 Args: 380 pyval: The Python constant to convert. Can be a PyTD definition or a 381 builtin constant. 382 subst: The current type parameters. 383 node: The current CFG node. (For instances) 384 source_sets: An iterator over instances of SourceSet (or just tuples). 385 discard_concrete_values: Whether concrete values should be discarded from 386 type parameters. 387 Returns: 388 A cfg.Variable. 389 Raises: 390 TypeParameterError: if conversion is attempted on a type parameter without 391 a substitution. 392 ValueError: if pytype is not of a known type. 393 """ 394 source_sets = source_sets or [[]] 395 node = node or self.vm.root_node 396 if isinstance(pyval, pytd.NothingType): 397 return self.vm.program.NewVariable([], [], self.vm.root_node) 398 elif isinstance(pyval, pytd.Alias): 399 return self.constant_to_var(pyval.type, subst, node, source_sets, 400 discard_concrete_values) 401 elif isinstance(pyval, abstract_utils.AsInstance): 402 cls = pyval.cls 403 if isinstance(cls, pytd.AnythingType): 404 return self.unsolvable.to_variable(node) 405 elif (isinstance(pyval, abstract_utils.AsReturnValue) and 406 isinstance(cls, pytd.NothingType)): 407 return self.no_return.to_variable(node) 408 var = self.vm.program.NewVariable() 409 for t in pytd_utils.UnpackUnion(cls): 410 if isinstance(t, pytd.TypeParameter): 411 if not subst or t.full_name not in subst: 412 raise self.TypeParameterError(t.full_name) 413 else: 414 for v in subst[t.full_name].bindings: 415 for source_set in source_sets: 416 var.AddBinding(self.get_maybe_abstract_instance(v.data) 417 if discard_concrete_values else v.data, 418 source_set + [v], node) 419 elif isinstance(t, pytd.NothingType): 420 pass 421 else: 422 value = self.constant_to_value( 423 abstract_utils.AsInstance(t), subst, node) 424 for source_set in source_sets: 425 var.AddBinding(value, source_set, node) 426 return var 427 elif isinstance(pyval, pytd.Constant): 428 return self.constant_to_var(abstract_utils.AsInstance(pyval.type), subst, 429 node, source_sets, discard_concrete_values) 430 result = self.constant_to_value(pyval, subst, node) 431 if result is not None: 432 return result.to_variable(node) 433 # There might still be bugs on the abstract intepreter when it returns, 434 # e.g. a list of values instead of a list of types: 435 assert pyval.__class__ != cfg.Variable, pyval 436 if pyval.__class__ == tuple: 437 # This case needs to go at the end because many things are actually also 438 # tuples. 439 return self.build_tuple(self.vm.root_node, (self.constant_to_var( 440 v, subst, node, source_sets, discard_concrete_values) 441 for i, v in enumerate(pyval))) 442 raise ValueError( 443 "Cannot convert {} to an abstract value".format(pyval.__class__)) 444 445 def constant_to_value(self, pyval, subst=None, node=None): 446 """Like constant_to_var, but convert to an abstract.BaseValue. 447 448 This also memoizes the results. We don't memoize on name, as builtin types 449 like str or list might be reinitialized under different names (e.g. "param 450 1"), but we want the canonical name and type. We *do* memoize on the type 451 as well, to make sure that e.g. "1.0" and "1" get converted to different 452 constants. Memoization is an optimization, but an important one - mapping 453 constants like "None" to the same AbstractValue greatly simplifies the 454 cfg structures we're building. 455 456 Args: 457 pyval: The constant to convert. 458 subst: The current type parameters. 459 node: The current CFG node. (For instances) 460 461 Returns: 462 The converted constant. (Instance of BaseValue) 463 """ 464 node = node or self.vm.root_node 465 if pyval.__class__ is tuple: 466 type_key = tuple(type(v) for v in pyval) 467 else: 468 type_key = type(pyval) 469 key = ("constant", pyval, type_key) 470 if key in self._convert_cache: 471 if self._convert_cache[key] is None: 472 self._convert_cache[key] = self.unsolvable 473 # This error is triggered by, e.g., classes inheriting from each other. 474 if not self.recursion_allowed: 475 name = getattr(pyval, "name", None) or pyval.__class__.__name__ 476 self.vm.errorlog.recursion_error(self.vm.frames, name) 477 return self._convert_cache[key] 478 else: 479 self._convert_cache[key] = None # for recursion detection 480 need_node = [False] # mutable value that can be modified by get_node 481 def get_node(): 482 need_node[0] = True 483 return node 484 value = self._constant_to_value(pyval, subst, get_node) 485 if not need_node[0] or node is self.vm.root_node: 486 # Values that contain a non-root node cannot be cached. Otherwise, 487 # we'd introduce bugs such as the following: 488 # if <condition>: 489 # d = {"a": 1} # "a" is cached here 490 # else: 491 # # the cached value of "a", which contains a node that is only 492 # # visible inside the "if", is used, which will eventually lead 493 # # pytype to think that the V->complex binding isn't visible. 494 # d = {"a": 1j} 495 self._convert_cache[key] = value 496 return value 497 498 def _load_late_type(self, late_type): 499 """Resolve a late type, possibly by loading a module.""" 500 if late_type.name not in self._resolved_late_types: 501 module, dot, _ = late_type.name.rpartition(".") 502 assert dot 503 ast = self.vm.loader.import_name(module) 504 if ast is not None: 505 try: 506 cls = ast.Lookup(late_type.name) 507 except KeyError: 508 if "__getattr__" not in ast: 509 log.warning("Couldn't resolve %s", late_type.name) 510 t = pytd.AnythingType() 511 else: 512 t = pytd.ToType(cls, allow_functions=True) 513 else: 514 # A pickle file refers to a module that went away in the mean time. 515 log.error("During dependency resolution, couldn't import %r", module) 516 t = pytd.AnythingType() 517 self._resolved_late_types[late_type.name] = t 518 return self._resolved_late_types[late_type.name] 519 520 def _create_module(self, ast): 521 data = (ast.constants + ast.type_params + ast.classes + 522 ast.functions + ast.aliases) 523 members = {val.name.rsplit(".")[-1]: val for val in data} 524 return abstract.Module(self.vm, ast.name, members, ast) 525 526 def _get_literal_value(self, pyval): 527 if pyval == self.vm.lookup_builtin("builtins.True"): 528 return True 529 elif pyval == self.vm.lookup_builtin("builtins.False"): 530 return False 531 elif isinstance(pyval, str): 532 prefix, value = parser_constants.STRING_RE.match(pyval).groups()[:2] 533 value = value[1:-1] # remove quotation marks 534 if "b" in prefix: 535 value = str(value).encode("utf-8") 536 return value 537 else: 538 return pyval 539 540 def _constant_to_value(self, pyval, subst, get_node): 541 """Create a BaseValue that represents a python constant. 542 543 This supports both constant from code constant pools and PyTD constants such 544 as classes. This also supports builtin python objects such as int and float. 545 546 Args: 547 pyval: The python or PyTD value to convert. 548 subst: The current type parameters. 549 get_node: A getter function for the current node. 550 551 Returns: 552 A Value that represents the constant, or None if we couldn't convert. 553 Raises: 554 NotImplementedError: If we don't know how to convert a value. 555 TypeParameterError: If we can't find a substitution for a type parameter. 556 """ 557 if isinstance(pyval, str): 558 return abstract.ConcreteValue(pyval, self.str_type, self.vm) 559 elif isinstance(pyval, bytes): 560 return abstract.ConcreteValue(pyval, self.bytes_type, self.vm) 561 elif isinstance(pyval, bool): 562 return self.true if pyval else self.false 563 elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: 564 # For small integers, preserve the actual value (for things like the 565 # level in IMPORT_NAME). 566 return abstract.ConcreteValue(pyval, self.int_type, self.vm) 567 elif pyval.__class__ in self.primitive_classes: 568 return self.primitive_class_instances[pyval.__class__] 569 elif pyval.__class__ is frozenset: 570 instance = abstract.Instance(self.frozenset_type, self.vm) 571 for element in pyval: 572 instance.merge_instance_type_parameter( 573 self.vm.root_node, abstract_utils.T, self.constant_to_var( 574 element, subst, self.vm.root_node)) 575 return instance 576 elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): 577 return abstract.ConcreteValue(pyval, 578 self.primitive_classes[types.CodeType], 579 self.vm) 580 elif pyval is super: 581 return special_builtins.Super(self.vm) 582 elif pyval is object: 583 return special_builtins.Object(self.vm) 584 elif pyval.__class__ is type: 585 try: 586 return self.name_to_value(self._type_to_name(pyval), subst) 587 except (KeyError, AttributeError): 588 log.debug("Failed to find pytd", exc_info=True) 589 raise 590 elif isinstance(pyval, pytd.LateType): 591 actual = self._load_late_type(pyval) 592 return self._constant_to_value(actual, subst, get_node) 593 elif isinstance(pyval, pytd.TypeDeclUnit): 594 return self._create_module(pyval) 595 elif isinstance(pyval, pytd.Module): 596 mod = self.vm.loader.import_name(pyval.module_name) 597 return self._create_module(mod) 598 elif isinstance(pyval, pytd.Class): 599 if pyval.name == "builtins.super": 600 return self.vm.special_builtins["super"] 601 elif pyval.name == "builtins.object": 602 return self.object_type 603 elif pyval.name == "types.ModuleType": 604 return self.module_type 605 elif pyval.name == "_importlib_modulespec.ModuleType": 606 # Python 3's typeshed uses a stub file indirection to define ModuleType 607 # even though it is exported via types.pyi. 608 return self.module_type 609 elif pyval.name == "types.FunctionType": 610 return self.function_type 611 else: 612 module, dot, base_name = pyval.name.rpartition(".") 613 # typing.TypingContainer intentionally loads the underlying pytd types. 614 if (module not in ("typing", "typing_extensions") and 615 module in overlay_dict.overlays): 616 overlay = self.vm.import_module(module, module, 0) 617 if overlay.get_module(base_name) is overlay: 618 overlay.load_lazy_attribute(base_name) 619 return abstract_utils.get_atomic_value(overlay.members[base_name]) 620 try: 621 cls = abstract.PyTDClass(base_name, pyval, self.vm) 622 except mro.MROError as e: 623 self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) 624 cls = self.unsolvable 625 else: 626 if dot: 627 cls.module = module 628 cls.call_metaclass_init(get_node()) 629 return cls 630 elif isinstance(pyval, pytd.Function): 631 signatures = [function.PyTDSignature(pyval.name, sig, self.vm) 632 for sig in pyval.signatures] 633 type_new = self.vm.lookup_builtin("builtins.type").Lookup("__new__") 634 if pyval is type_new: 635 f_cls = special_builtins.TypeNew 636 else: 637 f_cls = abstract.PyTDFunction 638 f = f_cls(pyval.name, signatures, pyval.kind, self.vm) 639 f.is_abstract = pyval.is_abstract 640 return f 641 elif isinstance(pyval, pytd.ClassType): 642 if pyval.cls: 643 cls = pyval.cls 644 else: 645 # If pyval is a reference to a class in builtins or typing, we can fill 646 # in the class ourselves. lookup_builtin raises a KeyError if the name 647 # is not found. 648 cls = self.vm.lookup_builtin(pyval.name) 649 assert isinstance(cls, pytd.Class) 650 return self.constant_to_value(cls, subst, self.vm.root_node) 651 elif isinstance(pyval, pytd.NothingType): 652 return self.empty 653 elif isinstance(pyval, pytd.AnythingType): 654 return self.unsolvable 655 elif (isinstance(pyval, pytd.Constant) and 656 isinstance(pyval.type, pytd.AnythingType)): 657 # We allow "X = ... # type: Any" to declare X as a type. 658 return self.unsolvable 659 elif (isinstance(pyval, pytd.Constant) and 660 isinstance(pyval.type, pytd.GenericType) and 661 pyval.type.name == "builtins.type"): 662 # `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`. 663 param, = pyval.type.parameters 664 return self.constant_to_value(param, subst, self.vm.root_node) 665 elif isinstance(pyval, pytd.UnionType): 666 options = [ 667 self.constant_to_value(t, subst, self.vm.root_node) 668 for t in pyval.type_list 669 ] 670 if len(options) > 1: 671 return abstract.Union(options, self.vm) 672 else: 673 return options[0] 674 elif isinstance(pyval, pytd.TypeParameter): 675 constraints = tuple( 676 self.constant_to_value(c, {}, self.vm.root_node) 677 for c in pyval.constraints) 678 bound = ( 679 pyval.bound and 680 self.constant_to_value(pyval.bound, {}, self.vm.root_node)) 681 return abstract.TypeParameter( 682 pyval.name, self.vm, constraints=constraints, 683 bound=bound, module=pyval.scope) 684 elif isinstance(pyval, abstract_utils.AsInstance): 685 cls = pyval.cls 686 if isinstance(cls, pytd.LateType): 687 actual = self._load_late_type(cls) 688 if not isinstance(actual, pytd.ClassType): 689 return self.unsolvable 690 cls = actual.cls 691 if isinstance(cls, pytd.ClassType): 692 cls = cls.cls 693 if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar": 694 param, = cls.parameters 695 return self.constant_to_value( 696 abstract_utils.AsInstance(param), subst, self.vm.root_node) 697 elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and 698 cls.template): 699 # If we're converting a generic Class, need to create a new instance of 700 # it. See test_classes.testGenericReinstantiated. 701 if isinstance(cls, pytd.Class): 702 params = tuple(t.type_param.upper_value for t in cls.template) 703 cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls), 704 parameters=params) 705 if isinstance(cls.base_type, pytd.LateType): 706 actual = self._load_late_type(cls.base_type) 707 if not isinstance(actual, pytd.ClassType): 708 return self.unsolvable 709 base_cls = actual.cls 710 else: 711 base_type = cls.base_type 712 assert isinstance(base_type, pytd.ClassType) 713 base_cls = base_type.cls 714 assert isinstance(base_cls, pytd.Class), base_cls 715 if base_cls.name == "builtins.type": 716 c, = cls.parameters 717 if isinstance(c, pytd.TypeParameter): 718 if not subst or c.full_name not in subst: 719 raise self.TypeParameterError(c.full_name) 720 # deformalize gets rid of any unexpected TypeVars, which can appear 721 # if something is annotated as Type[T]. 722 return self.vm.annotations_util.deformalize( 723 self.merge_classes(subst[c.full_name].data)) 724 else: 725 return self.constant_to_value(c, subst, self.vm.root_node) 726 elif isinstance(cls, pytd.TupleType): 727 content = tuple(self.constant_to_var(abstract_utils.AsInstance(p), 728 subst, get_node()) 729 for p in cls.parameters) 730 return abstract.Tuple(content, self.vm) 731 elif isinstance(cls, pytd.CallableType): 732 clsval = self.constant_to_value(cls, subst, self.vm.root_node) 733 return abstract.Instance(clsval, self.vm) 734 else: 735 clsval = self.constant_to_value(base_cls, subst, self.vm.root_node) 736 instance = abstract.Instance(clsval, self.vm) 737 num_params = len(cls.parameters) 738 assert num_params <= len(base_cls.template) 739 for i, formal in enumerate(base_cls.template): 740 if i < num_params: 741 node = get_node() 742 p = self.constant_to_var( 743 abstract_utils.AsInstance(cls.parameters[i]), subst, node) 744 else: 745 # An omitted type parameter implies `Any`. 746 node = self.vm.root_node 747 p = self.unsolvable.to_variable(node) 748 instance.merge_instance_type_parameter(node, formal.name, p) 749 return instance 750 elif isinstance(cls, pytd.Class): 751 assert not cls.template 752 # This key is also used in __init__ 753 key = (abstract.Instance, cls) 754 if key not in self._convert_cache: 755 if cls.name in ["builtins.type", "builtins.property"]: 756 # An instance of "type" or of an anonymous property can be anything. 757 instance = self._create_new_unknown_value("type") 758 else: 759 mycls = self.constant_to_value(cls, subst, self.vm.root_node) 760 instance = abstract.Instance(mycls, self.vm) 761 log.info("New pytd instance for %s: %r", cls.name, instance) 762 self._convert_cache[key] = instance 763 return self._convert_cache[key] 764 elif isinstance(cls, pytd.Literal): 765 return self.constant_to_value( 766 self._get_literal_value(cls.value), subst, self.vm.root_node) 767 else: 768 return self.constant_to_value(cls, subst, self.vm.root_node) 769 elif (isinstance(pyval, pytd.GenericType) and 770 pyval.name == "typing.ClassVar"): 771 param, = pyval.parameters 772 return self.constant_to_value(param, subst, self.vm.root_node) 773 elif isinstance(pyval, pytd.GenericType): 774 if isinstance(pyval.base_type, pytd.LateType): 775 actual = self._load_late_type(pyval.base_type) 776 if not isinstance(actual, pytd.ClassType): 777 return self.unsolvable 778 base = actual.cls 779 else: 780 assert isinstance(pyval.base_type, pytd.ClassType), pyval 781 base = pyval.base_type.cls 782 assert isinstance(base, pytd.Class), base 783 base_cls = self.constant_to_value(base, subst, self.vm.root_node) 784 if not isinstance(base_cls, class_mixin.Class): 785 # base_cls can be, e.g., an unsolvable due to an mro error. 786 return self.unsolvable 787 if isinstance(pyval, pytd.TupleType): 788 abstract_class = abstract.TupleClass 789 template = list(range(len(pyval.parameters))) + [abstract_utils.T] 790 combined_parameter = pytd_utils.JoinTypes(pyval.parameters) 791 parameters = pyval.parameters + (combined_parameter,) 792 elif isinstance(pyval, pytd.CallableType): 793 abstract_class = abstract.CallableClass 794 template = list(range(len(pyval.args))) + [abstract_utils.ARGS, 795 abstract_utils.RET] 796 parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret) 797 else: 798 abstract_class = abstract.ParameterizedClass 799 if pyval.name == "typing.Generic": 800 pyval_template = pyval.parameters 801 else: 802 pyval_template = base.template 803 template = tuple(t.name for t in pyval_template) 804 parameters = pyval.parameters 805 assert (pyval.name in ("typing.Generic", "typing.Protocol") or 806 len(parameters) <= len(template)) 807 # Delay type parameter loading to handle recursive types. 808 # See the ParameterizedClass.formal_type_parameters() property. 809 type_parameters = abstract_utils.LazyFormalTypeParameters( 810 template, parameters, subst) 811 return abstract_class(base_cls, type_parameters, self.vm) 812 elif isinstance(pyval, pytd.Literal): 813 value = self.constant_to_value( 814 self._get_literal_value(pyval.value), subst, self.vm.root_node) 815 return abstract.LiteralClass(value, self.vm) 816 elif isinstance(pyval, pytd.Annotated): 817 return self.constant_to_value(pyval.base_type, subst, self.vm.root_node) 818 elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node 819 return self.tuple_to_value([ 820 self.constant_to_var(item, subst, self.vm.root_node) 821 for i, item in enumerate(pyval) 822 ]) 823 else: 824 raise NotImplementedError("Can't convert constant %s %r" % 825 (type(pyval), pyval)) 826