1import re 2import sys 3import warnings 4 5import typing # for typing.Type, which conflicts with types.Type 6from typing import ( 7 Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List, overload 8) 9from typing_extensions import Final, Literal, overload 10 11from mypy.sharedparse import ( 12 special_function_elide_names, argument_elide_name, 13) 14from mypy.nodes import ( 15 MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, 16 OverloadedFuncDef, OverloadPart, 17 ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, 18 ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, 19 DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, 20 WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, 21 TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, 22 DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, 23 FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, 24 UnaryExpr, LambdaExpr, ComparisonExpr, AssignmentExpr, 25 StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, 26 SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, 27 AwaitExpr, TempNode, Expression, Statement, 28 ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2, 29 check_arg_names, 30 FakeInfo, 31) 32from mypy.types import ( 33 Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, CallableArgument, 34 TypeOfAny, Instance, RawExpressionType, ProperType, UnionType, 35) 36from mypy import defaults 37from mypy import message_registry, errorcodes as codes 38from mypy.errors import Errors 39from mypy.options import Options 40from mypy.reachability import mark_block_unreachable 41 42try: 43 # pull this into a final variable to make mypyc be quiet about the 44 # the default argument warning 45 PY_MINOR_VERSION = sys.version_info[1] # type: Final 46 47 # Check if we can use the stdlib ast module instead of typed_ast. 48 if sys.version_info >= (3, 8): 49 import ast as ast3 50 assert 'kind' in ast3.Constant._fields, \ 51 "This 3.8.0 alpha (%s) is too old; 3.8.0a3 required" % sys.version.split()[0] 52 # TODO: Num, Str, Bytes, NameConstant, Ellipsis are deprecated in 3.8. 53 # TODO: Index, ExtSlice are deprecated in 3.9. 54 from ast import ( 55 AST, 56 Call, 57 FunctionType, 58 Name, 59 Attribute, 60 Ellipsis as ast3_Ellipsis, 61 Starred, 62 NameConstant, 63 Expression as ast3_Expression, 64 Str, 65 Bytes, 66 Index, 67 Num, 68 UnaryOp, 69 USub, 70 ) 71 72 def ast3_parse(source: Union[str, bytes], filename: str, mode: str, 73 feature_version: int = PY_MINOR_VERSION) -> AST: 74 return ast3.parse(source, filename, mode, 75 type_comments=True, # This works the magic 76 feature_version=feature_version) 77 78 NamedExpr = ast3.NamedExpr 79 Constant = ast3.Constant 80 else: 81 from typed_ast import ast3 82 from typed_ast.ast3 import ( 83 AST, 84 Call, 85 FunctionType, 86 Name, 87 Attribute, 88 Ellipsis as ast3_Ellipsis, 89 Starred, 90 NameConstant, 91 Expression as ast3_Expression, 92 Str, 93 Bytes, 94 Index, 95 Num, 96 UnaryOp, 97 USub, 98 ) 99 100 def ast3_parse(source: Union[str, bytes], filename: str, mode: str, 101 feature_version: int = PY_MINOR_VERSION) -> AST: 102 return ast3.parse(source, filename, mode, feature_version=feature_version) 103 104 # These don't exist before 3.8 105 NamedExpr = Any 106 Constant = Any 107except ImportError: 108 try: 109 from typed_ast import ast35 # type: ignore[attr-defined] # noqa: F401 110 except ImportError: 111 print('The typed_ast package is not installed.\n' 112 'You can install it with `python3 -m pip install typed-ast`.', 113 file=sys.stderr) 114 else: 115 print('You need a more recent version of the typed_ast package.\n' 116 'You can update to the latest version with ' 117 '`python3 -m pip install -U typed-ast`.', 118 file=sys.stderr) 119 sys.exit(1) 120 121N = TypeVar('N', bound=Node) 122 123# There is no way to create reasonable fallbacks at this stage, 124# they must be patched later. 125MISSING_FALLBACK = FakeInfo("fallback can't be filled out until semanal") # type: Final 126_dummy_fallback = Instance(MISSING_FALLBACK, [], -1) # type: Final 127 128TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final 129 130INVALID_TYPE_IGNORE = 'Invalid "type: ignore" comment' # type: Final 131 132TYPE_IGNORE_PATTERN = re.compile(r'[^#]*#\s*type:\s*ignore\s*(.*)') 133 134 135def parse(source: Union[str, bytes], 136 fnam: str, 137 module: Optional[str], 138 errors: Optional[Errors] = None, 139 options: Optional[Options] = None) -> MypyFile: 140 141 """Parse a source file, without doing any semantic analysis. 142 143 Return the parse tree. If errors is not provided, raise ParseError 144 on failure. Otherwise, use the errors object to report parse errors. 145 """ 146 raise_on_error = False 147 if errors is None: 148 errors = Errors() 149 raise_on_error = True 150 if options is None: 151 options = Options() 152 errors.set_file(fnam, module) 153 is_stub_file = fnam.endswith('.pyi') 154 try: 155 if is_stub_file: 156 feature_version = defaults.PYTHON3_VERSION[1] 157 else: 158 assert options.python_version[0] >= 3 159 feature_version = options.python_version[1] 160 # Disable deprecation warnings about \u 161 with warnings.catch_warnings(): 162 warnings.filterwarnings("ignore", category=DeprecationWarning) 163 ast = ast3_parse(source, fnam, 'exec', feature_version=feature_version) 164 165 tree = ASTConverter(options=options, 166 is_stub=is_stub_file, 167 errors=errors, 168 ).visit(ast) 169 tree.path = fnam 170 tree.is_stub = is_stub_file 171 except SyntaxError as e: 172 # alias to please mypyc 173 is_py38_or_earlier = sys.version_info < (3, 9) 174 if is_py38_or_earlier and e.filename == "<fstring>": 175 # In Python 3.8 and earlier, syntax errors in f-strings have lineno relative to the 176 # start of the f-string. This would be misleading, as mypy will report the error as the 177 # lineno within the file. 178 e.lineno = None 179 errors.report(e.lineno if e.lineno is not None else -1, e.offset, e.msg, blocker=True, 180 code=codes.SYNTAX) 181 tree = MypyFile([], [], False, {}) 182 183 if raise_on_error and errors.is_errors(): 184 errors.raise_error() 185 186 return tree 187 188 189def parse_type_ignore_tag(tag: Optional[str]) -> Optional[List[str]]: 190 """Parse optional "[code, ...]" tag after "# type: ignore". 191 192 Return: 193 * [] if no tag was found (ignore all errors) 194 * list of ignored error codes if a tag was found 195 * None if the tag was invalid. 196 """ 197 if not tag or tag.strip() == '' or tag.strip().startswith('#'): 198 # No tag -- ignore all errors. 199 return [] 200 m = re.match(r'\s*\[([^]#]*)\]\s*(#.*)?$', tag) 201 if m is None: 202 # Invalid "# type: ignore" comment. 203 return None 204 return [code.strip() for code in m.group(1).split(',')] 205 206 207def parse_type_comment(type_comment: str, 208 line: int, 209 column: int, 210 errors: Optional[Errors], 211 assume_str_is_unicode: bool = True, 212 ) -> Tuple[Optional[List[str]], Optional[ProperType]]: 213 """Parse type portion of a type comment (+ optional type ignore). 214 215 Return (ignore info, parsed type). 216 """ 217 try: 218 typ = ast3_parse(type_comment, '<type_comment>', 'eval') 219 except SyntaxError: 220 if errors is not None: 221 stripped_type = type_comment.split("#", 2)[0].strip() 222 err_msg = '{} "{}"'.format(TYPE_COMMENT_SYNTAX_ERROR, stripped_type) 223 errors.report(line, column, err_msg, blocker=True, code=codes.SYNTAX) 224 return None, None 225 else: 226 raise 227 else: 228 extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) 229 if extra_ignore: 230 # Typeshed has a non-optional return type for group! 231 tag = cast(Any, extra_ignore).group(1) # type: Optional[str] 232 ignored = parse_type_ignore_tag(tag) # type: Optional[List[str]] 233 if ignored is None: 234 if errors is not None: 235 errors.report(line, column, INVALID_TYPE_IGNORE, code=codes.SYNTAX) 236 else: 237 raise SyntaxError 238 else: 239 ignored = None 240 assert isinstance(typ, ast3_Expression) 241 converted = TypeConverter(errors, 242 line=line, 243 override_column=column, 244 assume_str_is_unicode=assume_str_is_unicode, 245 is_evaluated=False).visit(typ.body) 246 return ignored, converted 247 248 249def parse_type_string(expr_string: str, expr_fallback_name: str, 250 line: int, column: int, assume_str_is_unicode: bool = True) -> ProperType: 251 """Parses a type that was originally present inside of an explicit string, 252 byte string, or unicode string. 253 254 For example, suppose we have the type `Foo["blah"]`. We should parse the 255 string expression "blah" using this function. 256 257 If `assume_str_is_unicode` is set to true, this function will assume that 258 `Foo["blah"]` is equivalent to `Foo[u"blah"]`. Otherwise, it assumes it's 259 equivalent to `Foo[b"blah"]`. 260 261 The caller is responsible for keeping track of the context in which the 262 type string was encountered (e.g. in Python 3 code, Python 2 code, Python 2 263 code with unicode_literals...) and setting `assume_str_is_unicode` accordingly. 264 """ 265 try: 266 _, node = parse_type_comment(expr_string.strip(), line=line, column=column, errors=None, 267 assume_str_is_unicode=assume_str_is_unicode) 268 if isinstance(node, UnboundType) and node.original_str_expr is None: 269 node.original_str_expr = expr_string 270 node.original_str_fallback = expr_fallback_name 271 return node 272 elif isinstance(node, UnionType): 273 return node 274 else: 275 return RawExpressionType(expr_string, expr_fallback_name, line, column) 276 except (SyntaxError, ValueError): 277 # Note: the parser will raise a `ValueError` instead of a SyntaxError if 278 # the string happens to contain things like \x00. 279 return RawExpressionType(expr_string, expr_fallback_name, line, column) 280 281 282def is_no_type_check_decorator(expr: ast3.expr) -> bool: 283 if isinstance(expr, Name): 284 return expr.id == 'no_type_check' 285 elif isinstance(expr, Attribute): 286 if isinstance(expr.value, Name): 287 return expr.value.id == 'typing' and expr.attr == 'no_type_check' 288 return False 289 290 291class ASTConverter: 292 def __init__(self, 293 options: Options, 294 is_stub: bool, 295 errors: Errors) -> None: 296 # 'C' for class, 'F' for function 297 self.class_and_function_stack = [] # type: List[Literal['C', 'F']] 298 self.imports = [] # type: List[ImportBase] 299 300 self.options = options 301 self.is_stub = is_stub 302 self.errors = errors 303 304 self.type_ignores = {} # type: Dict[int, List[str]] 305 306 # Cache of visit_X methods keyed by type of visited object 307 self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] 308 309 def note(self, msg: str, line: int, column: int) -> None: 310 self.errors.report(line, column, msg, severity='note', code=codes.SYNTAX) 311 312 def fail(self, 313 msg: str, 314 line: int, 315 column: int, 316 blocker: bool = True) -> None: 317 if blocker or not self.options.ignore_errors: 318 self.errors.report(line, column, msg, blocker=blocker, code=codes.SYNTAX) 319 320 def visit(self, node: Optional[AST]) -> Any: 321 if node is None: 322 return None 323 typeobj = type(node) 324 visitor = self.visitor_cache.get(typeobj) 325 if visitor is None: 326 method = 'visit_' + node.__class__.__name__ 327 visitor = getattr(self, method) 328 self.visitor_cache[typeobj] = visitor 329 return visitor(node) 330 331 def set_line(self, node: N, n: Union[ast3.expr, ast3.stmt, ast3.ExceptHandler]) -> N: 332 node.line = n.lineno 333 node.column = n.col_offset 334 node.end_line = getattr(n, "end_lineno", None) if isinstance(n, ast3.expr) else None 335 return node 336 337 def translate_opt_expr_list(self, l: Sequence[Optional[AST]]) -> List[Optional[Expression]]: 338 res = [] # type: List[Optional[Expression]] 339 for e in l: 340 exp = self.visit(e) 341 res.append(exp) 342 return res 343 344 def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: 345 return cast(List[Expression], self.translate_opt_expr_list(l)) 346 347 def get_lineno(self, node: Union[ast3.expr, ast3.stmt]) -> int: 348 if (isinstance(node, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef)) 349 and node.decorator_list): 350 return node.decorator_list[0].lineno 351 return node.lineno 352 353 def translate_stmt_list(self, 354 stmts: Sequence[ast3.stmt], 355 ismodule: bool = False) -> List[Statement]: 356 # A "# type: ignore" comment before the first statement of a module 357 # ignores the whole module: 358 if (ismodule and stmts and self.type_ignores 359 and min(self.type_ignores) < self.get_lineno(stmts[0])): 360 self.errors.used_ignored_lines[self.errors.file].add(min(self.type_ignores)) 361 block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) 362 mark_block_unreachable(block) 363 return [block] 364 365 res = [] # type: List[Statement] 366 for stmt in stmts: 367 node = self.visit(stmt) 368 res.append(node) 369 370 return res 371 372 def translate_type_comment(self, 373 n: Union[ast3.stmt, ast3.arg], 374 type_comment: Optional[str]) -> Optional[ProperType]: 375 if type_comment is None: 376 return None 377 else: 378 lineno = n.lineno 379 extra_ignore, typ = parse_type_comment(type_comment, 380 lineno, 381 n.col_offset, 382 self.errors) 383 if extra_ignore is not None: 384 self.type_ignores[lineno] = extra_ignore 385 return typ 386 387 op_map = { 388 ast3.Add: '+', 389 ast3.Sub: '-', 390 ast3.Mult: '*', 391 ast3.MatMult: '@', 392 ast3.Div: '/', 393 ast3.Mod: '%', 394 ast3.Pow: '**', 395 ast3.LShift: '<<', 396 ast3.RShift: '>>', 397 ast3.BitOr: '|', 398 ast3.BitXor: '^', 399 ast3.BitAnd: '&', 400 ast3.FloorDiv: '//' 401 } # type: Final[Dict[typing.Type[AST], str]] 402 403 def from_operator(self, op: ast3.operator) -> str: 404 op_name = ASTConverter.op_map.get(type(op)) 405 if op_name is None: 406 raise RuntimeError('Unknown operator ' + str(type(op))) 407 else: 408 return op_name 409 410 comp_op_map = { 411 ast3.Gt: '>', 412 ast3.Lt: '<', 413 ast3.Eq: '==', 414 ast3.GtE: '>=', 415 ast3.LtE: '<=', 416 ast3.NotEq: '!=', 417 ast3.Is: 'is', 418 ast3.IsNot: 'is not', 419 ast3.In: 'in', 420 ast3.NotIn: 'not in' 421 } # type: Final[Dict[typing.Type[AST], str]] 422 423 def from_comp_operator(self, op: ast3.cmpop) -> str: 424 op_name = ASTConverter.comp_op_map.get(type(op)) 425 if op_name is None: 426 raise RuntimeError('Unknown comparison operator ' + str(type(op))) 427 else: 428 return op_name 429 430 def as_block(self, stmts: List[ast3.stmt], lineno: int) -> Optional[Block]: 431 b = None 432 if stmts: 433 b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) 434 b.set_line(lineno) 435 return b 436 437 def as_required_block(self, stmts: List[ast3.stmt], lineno: int) -> Block: 438 assert stmts # must be non-empty 439 b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) 440 b.set_line(lineno) 441 return b 442 443 def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]: 444 ret = [] # type: List[Statement] 445 current_overload = [] # type: List[OverloadPart] 446 current_overload_name = None # type: Optional[str] 447 for stmt in stmts: 448 if (current_overload_name is not None 449 and isinstance(stmt, (Decorator, FuncDef)) 450 and stmt.name == current_overload_name): 451 current_overload.append(stmt) 452 else: 453 if len(current_overload) == 1: 454 ret.append(current_overload[0]) 455 elif len(current_overload) > 1: 456 ret.append(OverloadedFuncDef(current_overload)) 457 458 if isinstance(stmt, Decorator): 459 current_overload = [stmt] 460 current_overload_name = stmt.name 461 else: 462 current_overload = [] 463 current_overload_name = None 464 ret.append(stmt) 465 466 if len(current_overload) == 1: 467 ret.append(current_overload[0]) 468 elif len(current_overload) > 1: 469 ret.append(OverloadedFuncDef(current_overload)) 470 return ret 471 472 def in_method_scope(self) -> bool: 473 return self.class_and_function_stack[-2:] == ['C', 'F'] 474 475 def translate_module_id(self, id: str) -> str: 476 """Return the actual, internal module id for a source text id. 477 478 For example, translate '__builtin__' in Python 2 to 'builtins'. 479 """ 480 if id == self.options.custom_typing_module: 481 return 'typing' 482 elif id == '__builtin__' and self.options.python_version[0] == 2: 483 # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation 484 # is named __builtin__.py (there is another layer of translation elsewhere). 485 return 'builtins' 486 return id 487 488 def visit_Module(self, mod: ast3.Module) -> MypyFile: 489 self.type_ignores = {} 490 for ti in mod.type_ignores: 491 parsed = parse_type_ignore_tag(ti.tag) # type: ignore[attr-defined] 492 if parsed is not None: 493 self.type_ignores[ti.lineno] = parsed 494 else: 495 self.fail(INVALID_TYPE_IGNORE, ti.lineno, -1) 496 body = self.fix_function_overloads(self.translate_stmt_list(mod.body, ismodule=True)) 497 return MypyFile(body, 498 self.imports, 499 False, 500 self.type_ignores, 501 ) 502 503 # --- stmt --- 504 # FunctionDef(identifier name, arguments args, 505 # stmt* body, expr* decorator_list, expr? returns, string? type_comment) 506 # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, 507 # arg? kwarg, expr* defaults) 508 def visit_FunctionDef(self, n: ast3.FunctionDef) -> Union[FuncDef, Decorator]: 509 return self.do_func_def(n) 510 511 # AsyncFunctionDef(identifier name, arguments args, 512 # stmt* body, expr* decorator_list, expr? returns, string? type_comment) 513 def visit_AsyncFunctionDef(self, n: ast3.AsyncFunctionDef) -> Union[FuncDef, Decorator]: 514 return self.do_func_def(n, is_coroutine=True) 515 516 def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef], 517 is_coroutine: bool = False) -> Union[FuncDef, Decorator]: 518 """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef.""" 519 self.class_and_function_stack.append('F') 520 no_type_check = bool(n.decorator_list and 521 any(is_no_type_check_decorator(d) for d in n.decorator_list)) 522 523 lineno = n.lineno 524 args = self.transform_args(n.args, lineno, no_type_check=no_type_check) 525 526 posonlyargs = [arg.arg for arg in getattr(n.args, "posonlyargs", [])] 527 arg_kinds = [arg.kind for arg in args] 528 arg_names = [arg.variable.name for arg in args] # type: List[Optional[str]] 529 arg_names = [None if argument_elide_name(name) or name in posonlyargs else name 530 for name in arg_names] 531 if special_function_elide_names(n.name): 532 arg_names = [None] * len(arg_names) 533 arg_types = [] # type: List[Optional[Type]] 534 if no_type_check: 535 arg_types = [None] * len(args) 536 return_type = None 537 elif n.type_comment is not None: 538 try: 539 func_type_ast = ast3_parse(n.type_comment, '<func_type>', 'func_type') 540 assert isinstance(func_type_ast, FunctionType) 541 # for ellipsis arg 542 if (len(func_type_ast.argtypes) == 1 and 543 isinstance(func_type_ast.argtypes[0], ast3_Ellipsis)): 544 if n.returns: 545 # PEP 484 disallows both type annotations and type comments 546 self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) 547 arg_types = [a.type_annotation 548 if a.type_annotation is not None 549 else AnyType(TypeOfAny.unannotated) 550 for a in args] 551 else: 552 # PEP 484 disallows both type annotations and type comments 553 if n.returns or any(a.type_annotation is not None for a in args): 554 self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) 555 translated_args = (TypeConverter(self.errors, 556 line=lineno, 557 override_column=n.col_offset) 558 .translate_expr_list(func_type_ast.argtypes)) 559 arg_types = [a if a is not None else AnyType(TypeOfAny.unannotated) 560 for a in translated_args] 561 return_type = TypeConverter(self.errors, 562 line=lineno).visit(func_type_ast.returns) 563 564 # add implicit self type 565 if self.in_method_scope() and len(arg_types) < len(args): 566 arg_types.insert(0, AnyType(TypeOfAny.special_form)) 567 except SyntaxError: 568 stripped_type = n.type_comment.split("#", 2)[0].strip() 569 err_msg = '{} "{}"'.format(TYPE_COMMENT_SYNTAX_ERROR, stripped_type) 570 self.fail(err_msg, lineno, n.col_offset) 571 if n.type_comment and n.type_comment[0] not in ["(", "#"]: 572 self.note('Suggestion: wrap argument types in parentheses', 573 lineno, n.col_offset) 574 arg_types = [AnyType(TypeOfAny.from_error)] * len(args) 575 return_type = AnyType(TypeOfAny.from_error) 576 else: 577 arg_types = [a.type_annotation for a in args] 578 return_type = TypeConverter(self.errors, line=n.returns.lineno 579 if n.returns else lineno).visit(n.returns) 580 581 for arg, arg_type in zip(args, arg_types): 582 self.set_type_optional(arg_type, arg.initializer) 583 584 func_type = None 585 if any(arg_types) or return_type: 586 if len(arg_types) != 1 and any(isinstance(t, EllipsisType) 587 for t in arg_types): 588 self.fail("Ellipses cannot accompany other argument types " 589 "in function type signature", lineno, n.col_offset) 590 elif len(arg_types) > len(arg_kinds): 591 self.fail('Type signature has too many arguments', lineno, n.col_offset, 592 blocker=False) 593 elif len(arg_types) < len(arg_kinds): 594 self.fail('Type signature has too few arguments', lineno, n.col_offset, 595 blocker=False) 596 else: 597 func_type = CallableType([a if a is not None else 598 AnyType(TypeOfAny.unannotated) for a in arg_types], 599 arg_kinds, 600 arg_names, 601 return_type if return_type is not None else 602 AnyType(TypeOfAny.unannotated), 603 _dummy_fallback) 604 605 func_def = FuncDef(n.name, 606 args, 607 self.as_required_block(n.body, lineno), 608 func_type) 609 if isinstance(func_def.type, CallableType): 610 # semanal.py does some in-place modifications we want to avoid 611 func_def.unanalyzed_type = func_def.type.copy_modified() 612 if is_coroutine: 613 func_def.is_coroutine = True 614 if func_type is not None: 615 func_type.definition = func_def 616 func_type.line = lineno 617 618 if n.decorator_list: 619 if sys.version_info < (3, 8): 620 # Before 3.8, [typed_]ast the line number points to the first decorator. 621 # In 3.8, it points to the 'def' line, where we want it. 622 lineno += len(n.decorator_list) 623 end_lineno = None # type: Optional[int] 624 else: 625 # Set end_lineno to the old pre-3.8 lineno, in order to keep 626 # existing "# type: ignore" comments working: 627 end_lineno = n.decorator_list[0].lineno + len(n.decorator_list) 628 629 var = Var(func_def.name) 630 var.is_ready = False 631 var.set_line(lineno) 632 633 func_def.is_decorated = True 634 func_def.set_line(lineno, n.col_offset, end_lineno) 635 func_def.body.set_line(lineno) # TODO: Why? 636 637 deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) 638 first = n.decorator_list[0] 639 deco.set_line(first.lineno, first.col_offset) 640 retval = deco # type: Union[FuncDef, Decorator] 641 else: 642 # FuncDef overrides set_line -- can't use self.set_line 643 func_def.set_line(lineno, n.col_offset) 644 retval = func_def 645 self.class_and_function_stack.pop() 646 return retval 647 648 def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: 649 if self.options.no_implicit_optional: 650 return 651 # Indicate that type should be wrapped in an Optional if arg is initialized to None. 652 optional = isinstance(initializer, NameExpr) and initializer.name == 'None' 653 if isinstance(type, UnboundType): 654 type.optional = optional 655 656 def transform_args(self, 657 args: ast3.arguments, 658 line: int, 659 no_type_check: bool = False, 660 ) -> List[Argument]: 661 new_args = [] 662 names = [] # type: List[ast3.arg] 663 args_args = getattr(args, "posonlyargs", []) + args.args 664 args_defaults = args.defaults 665 num_no_defaults = len(args_args) - len(args_defaults) 666 # positional arguments without defaults 667 for a in args_args[:num_no_defaults]: 668 new_args.append(self.make_argument(a, None, ARG_POS, no_type_check)) 669 names.append(a) 670 671 # positional arguments with defaults 672 for a, d in zip(args_args[num_no_defaults:], args_defaults): 673 new_args.append(self.make_argument(a, d, ARG_OPT, no_type_check)) 674 names.append(a) 675 676 # *arg 677 if args.vararg is not None: 678 new_args.append(self.make_argument(args.vararg, None, ARG_STAR, no_type_check)) 679 names.append(args.vararg) 680 681 # keyword-only arguments with defaults 682 for a, kd in zip(args.kwonlyargs, args.kw_defaults): 683 new_args.append(self.make_argument( 684 a, 685 kd, 686 ARG_NAMED if kd is None else ARG_NAMED_OPT, 687 no_type_check)) 688 names.append(a) 689 690 # **kwarg 691 if args.kwarg is not None: 692 new_args.append(self.make_argument(args.kwarg, None, ARG_STAR2, no_type_check)) 693 names.append(args.kwarg) 694 695 check_arg_names([arg.variable.name for arg in new_args], names, self.fail_arg) 696 697 return new_args 698 699 def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int, 700 no_type_check: bool) -> Argument: 701 if no_type_check: 702 arg_type = None 703 else: 704 annotation = arg.annotation 705 type_comment = arg.type_comment 706 if annotation is not None and type_comment is not None: 707 self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, arg.lineno, arg.col_offset) 708 arg_type = None 709 if annotation is not None: 710 arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation) 711 else: 712 arg_type = self.translate_type_comment(arg, type_comment) 713 return Argument(Var(arg.arg), arg_type, self.visit(default), kind) 714 715 def fail_arg(self, msg: str, arg: ast3.arg) -> None: 716 self.fail(msg, arg.lineno, arg.col_offset) 717 718 # ClassDef(identifier name, 719 # expr* bases, 720 # keyword* keywords, 721 # stmt* body, 722 # expr* decorator_list) 723 def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: 724 self.class_and_function_stack.append('C') 725 keywords = [(kw.arg, self.visit(kw.value)) 726 for kw in n.keywords if kw.arg] 727 728 cdef = ClassDef(n.name, 729 self.as_required_block(n.body, n.lineno), 730 None, 731 self.translate_expr_list(n.bases), 732 metaclass=dict(keywords).get('metaclass'), 733 keywords=keywords) 734 cdef.decorators = self.translate_expr_list(n.decorator_list) 735 # Set end_lineno to the old mypy 0.700 lineno, in order to keep 736 # existing "# type: ignore" comments working: 737 if sys.version_info < (3, 8): 738 cdef.line = n.lineno + len(n.decorator_list) 739 cdef.end_line = n.lineno 740 else: 741 cdef.line = n.lineno 742 cdef.end_line = n.decorator_list[0].lineno if n.decorator_list else None 743 cdef.column = n.col_offset 744 self.class_and_function_stack.pop() 745 return cdef 746 747 # Return(expr? value) 748 def visit_Return(self, n: ast3.Return) -> ReturnStmt: 749 node = ReturnStmt(self.visit(n.value)) 750 return self.set_line(node, n) 751 752 # Delete(expr* targets) 753 def visit_Delete(self, n: ast3.Delete) -> DelStmt: 754 if len(n.targets) > 1: 755 tup = TupleExpr(self.translate_expr_list(n.targets)) 756 tup.set_line(n.lineno) 757 node = DelStmt(tup) 758 else: 759 node = DelStmt(self.visit(n.targets[0])) 760 return self.set_line(node, n) 761 762 # Assign(expr* targets, expr? value, string? type_comment, expr? annotation) 763 def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt: 764 lvalues = self.translate_expr_list(n.targets) 765 rvalue = self.visit(n.value) 766 typ = self.translate_type_comment(n, n.type_comment) 767 s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False) 768 return self.set_line(s, n) 769 770 # AnnAssign(expr target, expr annotation, expr? value, int simple) 771 def visit_AnnAssign(self, n: ast3.AnnAssign) -> AssignmentStmt: 772 line = n.lineno 773 if n.value is None: # always allow 'x: int' 774 rvalue = TempNode(AnyType(TypeOfAny.special_form), no_rhs=True) # type: Expression 775 rvalue.line = line 776 rvalue.column = n.col_offset 777 else: 778 rvalue = self.visit(n.value) 779 typ = TypeConverter(self.errors, line=line).visit(n.annotation) 780 assert typ is not None 781 typ.column = n.annotation.col_offset 782 s = AssignmentStmt([self.visit(n.target)], rvalue, type=typ, new_syntax=True) 783 return self.set_line(s, n) 784 785 # AugAssign(expr target, operator op, expr value) 786 def visit_AugAssign(self, n: ast3.AugAssign) -> OperatorAssignmentStmt: 787 s = OperatorAssignmentStmt(self.from_operator(n.op), 788 self.visit(n.target), 789 self.visit(n.value)) 790 return self.set_line(s, n) 791 792 # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) 793 def visit_For(self, n: ast3.For) -> ForStmt: 794 target_type = self.translate_type_comment(n, n.type_comment) 795 node = ForStmt(self.visit(n.target), 796 self.visit(n.iter), 797 self.as_required_block(n.body, n.lineno), 798 self.as_block(n.orelse, n.lineno), 799 target_type) 800 return self.set_line(node, n) 801 802 # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) 803 def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt: 804 target_type = self.translate_type_comment(n, n.type_comment) 805 node = ForStmt(self.visit(n.target), 806 self.visit(n.iter), 807 self.as_required_block(n.body, n.lineno), 808 self.as_block(n.orelse, n.lineno), 809 target_type) 810 node.is_async = True 811 return self.set_line(node, n) 812 813 # While(expr test, stmt* body, stmt* orelse) 814 def visit_While(self, n: ast3.While) -> WhileStmt: 815 node = WhileStmt(self.visit(n.test), 816 self.as_required_block(n.body, n.lineno), 817 self.as_block(n.orelse, n.lineno)) 818 return self.set_line(node, n) 819 820 # If(expr test, stmt* body, stmt* orelse) 821 def visit_If(self, n: ast3.If) -> IfStmt: 822 lineno = n.lineno 823 node = IfStmt([self.visit(n.test)], 824 [self.as_required_block(n.body, lineno)], 825 self.as_block(n.orelse, lineno)) 826 return self.set_line(node, n) 827 828 # With(withitem* items, stmt* body, string? type_comment) 829 def visit_With(self, n: ast3.With) -> WithStmt: 830 target_type = self.translate_type_comment(n, n.type_comment) 831 node = WithStmt([self.visit(i.context_expr) for i in n.items], 832 [self.visit(i.optional_vars) for i in n.items], 833 self.as_required_block(n.body, n.lineno), 834 target_type) 835 return self.set_line(node, n) 836 837 # AsyncWith(withitem* items, stmt* body, string? type_comment) 838 def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt: 839 target_type = self.translate_type_comment(n, n.type_comment) 840 s = WithStmt([self.visit(i.context_expr) for i in n.items], 841 [self.visit(i.optional_vars) for i in n.items], 842 self.as_required_block(n.body, n.lineno), 843 target_type) 844 s.is_async = True 845 return self.set_line(s, n) 846 847 # Raise(expr? exc, expr? cause) 848 def visit_Raise(self, n: ast3.Raise) -> RaiseStmt: 849 node = RaiseStmt(self.visit(n.exc), self.visit(n.cause)) 850 return self.set_line(node, n) 851 852 # Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) 853 def visit_Try(self, n: ast3.Try) -> TryStmt: 854 vs = [ 855 self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers 856 ] 857 types = [self.visit(h.type) for h in n.handlers] 858 handlers = [self.as_required_block(h.body, h.lineno) for h in n.handlers] 859 860 node = TryStmt(self.as_required_block(n.body, n.lineno), 861 vs, 862 types, 863 handlers, 864 self.as_block(n.orelse, n.lineno), 865 self.as_block(n.finalbody, n.lineno)) 866 return self.set_line(node, n) 867 868 # Assert(expr test, expr? msg) 869 def visit_Assert(self, n: ast3.Assert) -> AssertStmt: 870 node = AssertStmt(self.visit(n.test), self.visit(n.msg)) 871 return self.set_line(node, n) 872 873 # Import(alias* names) 874 def visit_Import(self, n: ast3.Import) -> Import: 875 names = [] # type: List[Tuple[str, Optional[str]]] 876 for alias in n.names: 877 name = self.translate_module_id(alias.name) 878 asname = alias.asname 879 if asname is None and name != alias.name: 880 # if the module name has been translated (and it's not already 881 # an explicit import-as), make it an implicit import-as the 882 # original name 883 asname = alias.name 884 names.append((name, asname)) 885 i = Import(names) 886 self.imports.append(i) 887 return self.set_line(i, n) 888 889 # ImportFrom(identifier? module, alias* names, int? level) 890 def visit_ImportFrom(self, n: ast3.ImportFrom) -> ImportBase: 891 assert n.level is not None 892 if len(n.names) == 1 and n.names[0].name == '*': 893 mod = n.module if n.module is not None else '' 894 i = ImportAll(mod, n.level) # type: ImportBase 895 else: 896 i = ImportFrom(self.translate_module_id(n.module) if n.module is not None else '', 897 n.level, 898 [(a.name, a.asname) for a in n.names]) 899 self.imports.append(i) 900 return self.set_line(i, n) 901 902 # Global(identifier* names) 903 def visit_Global(self, n: ast3.Global) -> GlobalDecl: 904 g = GlobalDecl(n.names) 905 return self.set_line(g, n) 906 907 # Nonlocal(identifier* names) 908 def visit_Nonlocal(self, n: ast3.Nonlocal) -> NonlocalDecl: 909 d = NonlocalDecl(n.names) 910 return self.set_line(d, n) 911 912 # Expr(expr value) 913 def visit_Expr(self, n: ast3.Expr) -> ExpressionStmt: 914 value = self.visit(n.value) 915 node = ExpressionStmt(value) 916 return self.set_line(node, n) 917 918 # Pass 919 def visit_Pass(self, n: ast3.Pass) -> PassStmt: 920 s = PassStmt() 921 return self.set_line(s, n) 922 923 # Break 924 def visit_Break(self, n: ast3.Break) -> BreakStmt: 925 s = BreakStmt() 926 return self.set_line(s, n) 927 928 # Continue 929 def visit_Continue(self, n: ast3.Continue) -> ContinueStmt: 930 s = ContinueStmt() 931 return self.set_line(s, n) 932 933 # --- expr --- 934 935 def visit_NamedExpr(self, n: NamedExpr) -> AssignmentExpr: 936 s = AssignmentExpr(self.visit(n.target), self.visit(n.value)) 937 return self.set_line(s, n) 938 939 # BoolOp(boolop op, expr* values) 940 def visit_BoolOp(self, n: ast3.BoolOp) -> OpExpr: 941 # mypy translates (1 and 2 and 3) as (1 and (2 and 3)) 942 assert len(n.values) >= 2 943 op_node = n.op 944 if isinstance(op_node, ast3.And): 945 op = 'and' 946 elif isinstance(op_node, ast3.Or): 947 op = 'or' 948 else: 949 raise RuntimeError('unknown BoolOp ' + str(type(n))) 950 951 # potentially inefficient! 952 return self.group(op, self.translate_expr_list(n.values), n) 953 954 def group(self, op: str, vals: List[Expression], n: ast3.expr) -> OpExpr: 955 if len(vals) == 2: 956 e = OpExpr(op, vals[0], vals[1]) 957 else: 958 e = OpExpr(op, vals[0], self.group(op, vals[1:], n)) 959 return self.set_line(e, n) 960 961 # BinOp(expr left, operator op, expr right) 962 def visit_BinOp(self, n: ast3.BinOp) -> OpExpr: 963 op = self.from_operator(n.op) 964 965 if op is None: 966 raise RuntimeError('cannot translate BinOp ' + str(type(n.op))) 967 968 e = OpExpr(op, self.visit(n.left), self.visit(n.right)) 969 return self.set_line(e, n) 970 971 # UnaryOp(unaryop op, expr operand) 972 def visit_UnaryOp(self, n: ast3.UnaryOp) -> UnaryExpr: 973 op = None 974 if isinstance(n.op, ast3.Invert): 975 op = '~' 976 elif isinstance(n.op, ast3.Not): 977 op = 'not' 978 elif isinstance(n.op, ast3.UAdd): 979 op = '+' 980 elif isinstance(n.op, ast3.USub): 981 op = '-' 982 983 if op is None: 984 raise RuntimeError('cannot translate UnaryOp ' + str(type(n.op))) 985 986 e = UnaryExpr(op, self.visit(n.operand)) 987 return self.set_line(e, n) 988 989 # Lambda(arguments args, expr body) 990 def visit_Lambda(self, n: ast3.Lambda) -> LambdaExpr: 991 body = ast3.Return(n.body) 992 body.lineno = n.body.lineno 993 body.col_offset = n.body.col_offset 994 995 e = LambdaExpr(self.transform_args(n.args, n.lineno), 996 self.as_required_block([body], n.lineno)) 997 e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line 998 return e 999 1000 # IfExp(expr test, expr body, expr orelse) 1001 def visit_IfExp(self, n: ast3.IfExp) -> ConditionalExpr: 1002 e = ConditionalExpr(self.visit(n.test), 1003 self.visit(n.body), 1004 self.visit(n.orelse)) 1005 return self.set_line(e, n) 1006 1007 # Dict(expr* keys, expr* values) 1008 def visit_Dict(self, n: ast3.Dict) -> DictExpr: 1009 e = DictExpr(list(zip(self.translate_opt_expr_list(n.keys), 1010 self.translate_expr_list(n.values)))) 1011 return self.set_line(e, n) 1012 1013 # Set(expr* elts) 1014 def visit_Set(self, n: ast3.Set) -> SetExpr: 1015 e = SetExpr(self.translate_expr_list(n.elts)) 1016 return self.set_line(e, n) 1017 1018 # ListComp(expr elt, comprehension* generators) 1019 def visit_ListComp(self, n: ast3.ListComp) -> ListComprehension: 1020 e = ListComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n))) 1021 return self.set_line(e, n) 1022 1023 # SetComp(expr elt, comprehension* generators) 1024 def visit_SetComp(self, n: ast3.SetComp) -> SetComprehension: 1025 e = SetComprehension(self.visit_GeneratorExp(cast(ast3.GeneratorExp, n))) 1026 return self.set_line(e, n) 1027 1028 # DictComp(expr key, expr value, comprehension* generators) 1029 def visit_DictComp(self, n: ast3.DictComp) -> DictionaryComprehension: 1030 targets = [self.visit(c.target) for c in n.generators] 1031 iters = [self.visit(c.iter) for c in n.generators] 1032 ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] 1033 is_async = [bool(c.is_async) for c in n.generators] 1034 e = DictionaryComprehension(self.visit(n.key), 1035 self.visit(n.value), 1036 targets, 1037 iters, 1038 ifs_list, 1039 is_async) 1040 return self.set_line(e, n) 1041 1042 # GeneratorExp(expr elt, comprehension* generators) 1043 def visit_GeneratorExp(self, n: ast3.GeneratorExp) -> GeneratorExpr: 1044 targets = [self.visit(c.target) for c in n.generators] 1045 iters = [self.visit(c.iter) for c in n.generators] 1046 ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] 1047 is_async = [bool(c.is_async) for c in n.generators] 1048 e = GeneratorExpr(self.visit(n.elt), 1049 targets, 1050 iters, 1051 ifs_list, 1052 is_async) 1053 return self.set_line(e, n) 1054 1055 # Await(expr value) 1056 def visit_Await(self, n: ast3.Await) -> AwaitExpr: 1057 v = self.visit(n.value) 1058 e = AwaitExpr(v) 1059 return self.set_line(e, n) 1060 1061 # Yield(expr? value) 1062 def visit_Yield(self, n: ast3.Yield) -> YieldExpr: 1063 e = YieldExpr(self.visit(n.value)) 1064 return self.set_line(e, n) 1065 1066 # YieldFrom(expr value) 1067 def visit_YieldFrom(self, n: ast3.YieldFrom) -> YieldFromExpr: 1068 e = YieldFromExpr(self.visit(n.value)) 1069 return self.set_line(e, n) 1070 1071 # Compare(expr left, cmpop* ops, expr* comparators) 1072 def visit_Compare(self, n: ast3.Compare) -> ComparisonExpr: 1073 operators = [self.from_comp_operator(o) for o in n.ops] 1074 operands = self.translate_expr_list([n.left] + n.comparators) 1075 e = ComparisonExpr(operators, operands) 1076 return self.set_line(e, n) 1077 1078 # Call(expr func, expr* args, keyword* keywords) 1079 # keyword = (identifier? arg, expr value) 1080 def visit_Call(self, n: Call) -> CallExpr: 1081 args = n.args 1082 keywords = n.keywords 1083 keyword_names = [k.arg for k in keywords] 1084 arg_types = self.translate_expr_list( 1085 [a.value if isinstance(a, Starred) else a for a in args] + 1086 [k.value for k in keywords]) 1087 arg_kinds = ([ARG_STAR if type(a) is Starred else ARG_POS for a in args] + 1088 [ARG_STAR2 if arg is None else ARG_NAMED for arg in keyword_names]) 1089 e = CallExpr(self.visit(n.func), 1090 arg_types, 1091 arg_kinds, 1092 cast('List[Optional[str]]', [None] * len(args)) + keyword_names) 1093 return self.set_line(e, n) 1094 1095 # Constant(object value) -- a constant, in Python 3.8. 1096 def visit_Constant(self, n: Constant) -> Any: 1097 val = n.value 1098 e = None # type: Any 1099 if val is None: 1100 e = NameExpr('None') 1101 elif isinstance(val, str): 1102 e = StrExpr(n.s) 1103 elif isinstance(val, bytes): 1104 e = BytesExpr(bytes_to_human_readable_repr(n.s)) 1105 elif isinstance(val, bool): # Must check before int! 1106 e = NameExpr(str(val)) 1107 elif isinstance(val, int): 1108 e = IntExpr(val) 1109 elif isinstance(val, float): 1110 e = FloatExpr(val) 1111 elif isinstance(val, complex): 1112 e = ComplexExpr(val) 1113 elif val is Ellipsis: 1114 e = EllipsisExpr() 1115 else: 1116 raise RuntimeError('Constant not implemented for ' + str(type(val))) 1117 return self.set_line(e, n) 1118 1119 # Num(object n) -- a number as a PyObject. 1120 def visit_Num(self, n: ast3.Num) -> Union[IntExpr, FloatExpr, ComplexExpr]: 1121 # The n field has the type complex, but complex isn't *really* 1122 # a parent of int and float, and this causes isinstance below 1123 # to think that the complex branch is always picked. Avoid 1124 # this by throwing away the type. 1125 val = n.n # type: object 1126 if isinstance(val, int): 1127 e = IntExpr(val) # type: Union[IntExpr, FloatExpr, ComplexExpr] 1128 elif isinstance(val, float): 1129 e = FloatExpr(val) 1130 elif isinstance(val, complex): 1131 e = ComplexExpr(val) 1132 else: 1133 raise RuntimeError('num not implemented for ' + str(type(val))) 1134 return self.set_line(e, n) 1135 1136 # Str(string s) 1137 def visit_Str(self, n: Str) -> Union[UnicodeExpr, StrExpr]: 1138 # Hack: assume all string literals in Python 2 stubs are normal 1139 # strs (i.e. not unicode). All stubs are parsed with the Python 3 1140 # parser, which causes unprefixed string literals to be interpreted 1141 # as unicode instead of bytes. This hack is generally okay, 1142 # because mypy considers str literals to be compatible with 1143 # unicode. 1144 e = StrExpr(n.s) 1145 return self.set_line(e, n) 1146 1147 # JoinedStr(expr* values) 1148 def visit_JoinedStr(self, n: ast3.JoinedStr) -> Expression: 1149 # Each of n.values is a str or FormattedValue; we just concatenate 1150 # them all using ''.join. 1151 empty_string = StrExpr('') 1152 empty_string.set_line(n.lineno, n.col_offset) 1153 strs_to_join = ListExpr(self.translate_expr_list(n.values)) 1154 strs_to_join.set_line(empty_string) 1155 # Don't make unnecessary join call if there is only one str to join 1156 if len(strs_to_join.items) == 1: 1157 return self.set_line(strs_to_join.items[0], n) 1158 join_method = MemberExpr(empty_string, 'join') 1159 join_method.set_line(empty_string) 1160 result_expression = CallExpr(join_method, 1161 [strs_to_join], 1162 [ARG_POS], 1163 [None]) 1164 return self.set_line(result_expression, n) 1165 1166 # FormattedValue(expr value) 1167 def visit_FormattedValue(self, n: ast3.FormattedValue) -> Expression: 1168 # A FormattedValue is a component of a JoinedStr, or it can exist 1169 # on its own. We translate them to individual '{}'.format(value) 1170 # calls. Format specifier and conversion information is passed along 1171 # to allow mypyc to support f-strings with format specifiers and conversions. 1172 val_exp = self.visit(n.value) 1173 val_exp.set_line(n.lineno, n.col_offset) 1174 conv_str = '' if n.conversion is None or n.conversion < 0 else '!' + chr(n.conversion) 1175 format_string = StrExpr('{' + conv_str + ':{}}') 1176 format_spec_exp = self.visit(n.format_spec) if n.format_spec is not None else StrExpr('') 1177 format_string.set_line(n.lineno, n.col_offset) 1178 format_method = MemberExpr(format_string, 'format') 1179 format_method.set_line(format_string) 1180 result_expression = CallExpr(format_method, 1181 [val_exp, format_spec_exp], 1182 [ARG_POS, ARG_POS], 1183 [None, None]) 1184 return self.set_line(result_expression, n) 1185 1186 # Bytes(bytes s) 1187 def visit_Bytes(self, n: ast3.Bytes) -> Union[BytesExpr, StrExpr]: 1188 e = BytesExpr(bytes_to_human_readable_repr(n.s)) 1189 return self.set_line(e, n) 1190 1191 # NameConstant(singleton value) 1192 def visit_NameConstant(self, n: NameConstant) -> NameExpr: 1193 e = NameExpr(str(n.value)) 1194 return self.set_line(e, n) 1195 1196 # Ellipsis 1197 def visit_Ellipsis(self, n: ast3_Ellipsis) -> EllipsisExpr: 1198 e = EllipsisExpr() 1199 return self.set_line(e, n) 1200 1201 # Attribute(expr value, identifier attr, expr_context ctx) 1202 def visit_Attribute(self, n: Attribute) -> Union[MemberExpr, SuperExpr]: 1203 value = n.value 1204 member_expr = MemberExpr(self.visit(value), n.attr) 1205 obj = member_expr.expr 1206 if (isinstance(obj, CallExpr) and 1207 isinstance(obj.callee, NameExpr) and 1208 obj.callee.name == 'super'): 1209 e = SuperExpr(member_expr.name, obj) # type: Union[MemberExpr, SuperExpr] 1210 else: 1211 e = member_expr 1212 return self.set_line(e, n) 1213 1214 # Subscript(expr value, slice slice, expr_context ctx) 1215 def visit_Subscript(self, n: ast3.Subscript) -> IndexExpr: 1216 e = IndexExpr(self.visit(n.value), self.visit(n.slice)) 1217 self.set_line(e, n) 1218 # alias to please mypyc 1219 is_py38_or_earlier = sys.version_info < (3, 9) 1220 if ( 1221 isinstance(n.slice, ast3.Slice) or 1222 (is_py38_or_earlier and isinstance(n.slice, ast3.ExtSlice)) 1223 ): 1224 # Before Python 3.9, Slice has no line/column in the raw ast. To avoid incompatibility 1225 # visit_Slice doesn't set_line, even in Python 3.9 on. 1226 # ExtSlice also has no line/column info. In Python 3.9 on, line/column is set for 1227 # e.index when visiting n.slice. 1228 e.index.line = e.line 1229 e.index.column = e.column 1230 return e 1231 1232 # Starred(expr value, expr_context ctx) 1233 def visit_Starred(self, n: Starred) -> StarExpr: 1234 e = StarExpr(self.visit(n.value)) 1235 return self.set_line(e, n) 1236 1237 # Name(identifier id, expr_context ctx) 1238 def visit_Name(self, n: Name) -> NameExpr: 1239 e = NameExpr(n.id) 1240 return self.set_line(e, n) 1241 1242 # List(expr* elts, expr_context ctx) 1243 def visit_List(self, n: ast3.List) -> Union[ListExpr, TupleExpr]: 1244 expr_list = [self.visit(e) for e in n.elts] # type: List[Expression] 1245 if isinstance(n.ctx, ast3.Store): 1246 # [x, y] = z and (x, y) = z means exactly the same thing 1247 e = TupleExpr(expr_list) # type: Union[ListExpr, TupleExpr] 1248 else: 1249 e = ListExpr(expr_list) 1250 return self.set_line(e, n) 1251 1252 # Tuple(expr* elts, expr_context ctx) 1253 def visit_Tuple(self, n: ast3.Tuple) -> TupleExpr: 1254 e = TupleExpr(self.translate_expr_list(n.elts)) 1255 return self.set_line(e, n) 1256 1257 # --- slice --- 1258 1259 # Slice(expr? lower, expr? upper, expr? step) 1260 def visit_Slice(self, n: ast3.Slice) -> SliceExpr: 1261 return SliceExpr(self.visit(n.lower), 1262 self.visit(n.upper), 1263 self.visit(n.step)) 1264 1265 # ExtSlice(slice* dims) 1266 def visit_ExtSlice(self, n: ast3.ExtSlice) -> TupleExpr: 1267 # cast for mypyc's benefit on Python 3.9 1268 return TupleExpr(self.translate_expr_list(cast(Any, n).dims)) 1269 1270 # Index(expr value) 1271 def visit_Index(self, n: Index) -> Node: 1272 # cast for mypyc's benefit on Python 3.9 1273 return self.visit(cast(Any, n).value) 1274 1275 1276class TypeConverter: 1277 def __init__(self, 1278 errors: Optional[Errors], 1279 line: int = -1, 1280 override_column: int = -1, 1281 assume_str_is_unicode: bool = True, 1282 is_evaluated: bool = True, 1283 ) -> None: 1284 self.errors = errors 1285 self.line = line 1286 self.override_column = override_column 1287 self.node_stack = [] # type: List[AST] 1288 self.assume_str_is_unicode = assume_str_is_unicode 1289 self.is_evaluated = is_evaluated 1290 1291 def convert_column(self, column: int) -> int: 1292 """Apply column override if defined; otherwise return column. 1293 1294 Column numbers are sometimes incorrect in the AST and the column 1295 override can be used to work around that. 1296 """ 1297 if self.override_column < 0: 1298 return column 1299 else: 1300 return self.override_column 1301 1302 def invalid_type(self, node: AST, note: Optional[str] = None) -> RawExpressionType: 1303 """Constructs a type representing some expression that normally forms an invalid type. 1304 For example, if we see a type hint that says "3 + 4", we would transform that 1305 expression into a RawExpressionType. 1306 1307 The semantic analysis layer will report an "Invalid type" error when it 1308 encounters this type, along with the given note if one is provided. 1309 1310 See RawExpressionType's docstring for more details on how it's used. 1311 """ 1312 return RawExpressionType( 1313 None, 1314 'typing.Any', 1315 line=self.line, 1316 column=getattr(node, 'col_offset', -1), 1317 note=note, 1318 ) 1319 1320 @overload 1321 def visit(self, node: ast3.expr) -> ProperType: ... 1322 1323 @overload 1324 def visit(self, node: Optional[AST]) -> Optional[ProperType]: ... 1325 1326 def visit(self, node: Optional[AST]) -> Optional[ProperType]: 1327 """Modified visit -- keep track of the stack of nodes""" 1328 if node is None: 1329 return None 1330 self.node_stack.append(node) 1331 try: 1332 method = 'visit_' + node.__class__.__name__ 1333 visitor = getattr(self, method, None) 1334 if visitor is not None: 1335 return visitor(node) 1336 else: 1337 return self.invalid_type(node) 1338 finally: 1339 self.node_stack.pop() 1340 1341 def parent(self) -> Optional[AST]: 1342 """Return the AST node above the one we are processing""" 1343 if len(self.node_stack) < 2: 1344 return None 1345 return self.node_stack[-2] 1346 1347 def fail(self, msg: str, line: int, column: int) -> None: 1348 if self.errors: 1349 self.errors.report(line, column, msg, blocker=True, code=codes.SYNTAX) 1350 1351 def note(self, msg: str, line: int, column: int) -> None: 1352 if self.errors: 1353 self.errors.report(line, column, msg, severity='note', code=codes.SYNTAX) 1354 1355 def translate_expr_list(self, l: Sequence[ast3.expr]) -> List[Type]: 1356 return [self.visit(e) for e in l] 1357 1358 def visit_raw_str(self, s: str) -> Type: 1359 # An escape hatch that allows the AST walker in fastparse2 to 1360 # directly hook into the Python 3.5 type converter in some cases 1361 # without needing to create an intermediary `Str` object. 1362 _, typ = parse_type_comment(s.strip(), 1363 self.line, 1364 -1, 1365 self.errors, 1366 self.assume_str_is_unicode) 1367 return typ or AnyType(TypeOfAny.from_error) 1368 1369 def visit_Call(self, e: Call) -> Type: 1370 # Parse the arg constructor 1371 f = e.func 1372 constructor = stringify_name(f) 1373 1374 if not isinstance(self.parent(), ast3.List): 1375 note = None 1376 if constructor: 1377 note = "Suggestion: use {0}[...] instead of {0}(...)".format(constructor) 1378 return self.invalid_type(e, note=note) 1379 if not constructor: 1380 self.fail("Expected arg constructor name", e.lineno, e.col_offset) 1381 1382 name = None # type: Optional[str] 1383 default_type = AnyType(TypeOfAny.special_form) 1384 typ = default_type # type: Type 1385 for i, arg in enumerate(e.args): 1386 if i == 0: 1387 converted = self.visit(arg) 1388 assert converted is not None 1389 typ = converted 1390 elif i == 1: 1391 name = self._extract_argument_name(arg) 1392 else: 1393 self.fail("Too many arguments for argument constructor", 1394 f.lineno, f.col_offset) 1395 for k in e.keywords: 1396 value = k.value 1397 if k.arg == "name": 1398 if name is not None: 1399 self.fail('"{}" gets multiple values for keyword argument "name"'.format( 1400 constructor), f.lineno, f.col_offset) 1401 name = self._extract_argument_name(value) 1402 elif k.arg == "type": 1403 if typ is not default_type: 1404 self.fail('"{}" gets multiple values for keyword argument "type"'.format( 1405 constructor), f.lineno, f.col_offset) 1406 converted = self.visit(value) 1407 assert converted is not None 1408 typ = converted 1409 else: 1410 self.fail( 1411 'Unexpected argument "{}" for argument constructor'.format(k.arg), 1412 value.lineno, value.col_offset) 1413 return CallableArgument(typ, name, constructor, e.lineno, e.col_offset) 1414 1415 def translate_argument_list(self, l: Sequence[ast3.expr]) -> TypeList: 1416 return TypeList([self.visit(e) for e in l], line=self.line) 1417 1418 def _extract_argument_name(self, n: ast3.expr) -> Optional[str]: 1419 if isinstance(n, Str): 1420 return n.s.strip() 1421 elif isinstance(n, NameConstant) and str(n.value) == 'None': 1422 return None 1423 self.fail('Expected string literal for argument name, got {}'.format( 1424 type(n).__name__), self.line, 0) 1425 return None 1426 1427 def visit_Name(self, n: Name) -> Type: 1428 return UnboundType(n.id, line=self.line, column=self.convert_column(n.col_offset)) 1429 1430 def visit_BinOp(self, n: ast3.BinOp) -> Type: 1431 if not isinstance(n.op, ast3.BitOr): 1432 return self.invalid_type(n) 1433 1434 left = self.visit(n.left) 1435 right = self.visit(n.right) 1436 return UnionType([left, right], 1437 line=self.line, 1438 column=self.convert_column(n.col_offset), 1439 is_evaluated=self.is_evaluated, 1440 uses_pep604_syntax=True) 1441 1442 def visit_NameConstant(self, n: NameConstant) -> Type: 1443 if isinstance(n.value, bool): 1444 return RawExpressionType(n.value, 'builtins.bool', line=self.line) 1445 else: 1446 return UnboundType(str(n.value), line=self.line, column=n.col_offset) 1447 1448 # Only for 3.8 and newer 1449 def visit_Constant(self, n: Constant) -> Type: 1450 val = n.value 1451 if val is None: 1452 # None is a type. 1453 return UnboundType('None', line=self.line) 1454 if isinstance(val, str): 1455 # Parse forward reference. 1456 if (n.kind and 'u' in n.kind) or self.assume_str_is_unicode: 1457 return parse_type_string(n.s, 'builtins.unicode', self.line, n.col_offset, 1458 assume_str_is_unicode=self.assume_str_is_unicode) 1459 else: 1460 return parse_type_string(n.s, 'builtins.str', self.line, n.col_offset, 1461 assume_str_is_unicode=self.assume_str_is_unicode) 1462 if val is Ellipsis: 1463 # '...' is valid in some types. 1464 return EllipsisType(line=self.line) 1465 if isinstance(val, bool): 1466 # Special case for True/False. 1467 return RawExpressionType(val, 'builtins.bool', line=self.line) 1468 if isinstance(val, (int, float, complex)): 1469 return self.numeric_type(val, n) 1470 if isinstance(val, bytes): 1471 contents = bytes_to_human_readable_repr(val) 1472 return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset) 1473 # Everything else is invalid. 1474 return self.invalid_type(n) 1475 1476 # UnaryOp(op, operand) 1477 def visit_UnaryOp(self, n: UnaryOp) -> Type: 1478 # We support specifically Literal[-4] and nothing else. 1479 # For example, Literal[+4] or Literal[~6] is not supported. 1480 typ = self.visit(n.operand) 1481 if isinstance(typ, RawExpressionType) and isinstance(n.op, USub): 1482 if isinstance(typ.literal_value, int): 1483 typ.literal_value *= -1 1484 return typ 1485 return self.invalid_type(n) 1486 1487 def numeric_type(self, value: object, n: AST) -> Type: 1488 # The node's field has the type complex, but complex isn't *really* 1489 # a parent of int and float, and this causes isinstance below 1490 # to think that the complex branch is always picked. Avoid 1491 # this by throwing away the type. 1492 if isinstance(value, int): 1493 numeric_value = value # type: Optional[int] 1494 type_name = 'builtins.int' 1495 else: 1496 # Other kinds of numbers (floats, complex) are not valid parameters for 1497 # RawExpressionType so we just pass in 'None' for now. We'll report the 1498 # appropriate error at a later stage. 1499 numeric_value = None 1500 type_name = 'builtins.{}'.format(type(value).__name__) 1501 return RawExpressionType( 1502 numeric_value, 1503 type_name, 1504 line=self.line, 1505 column=getattr(n, 'col_offset', -1), 1506 ) 1507 1508 # These next three methods are only used if we are on python < 1509 # 3.8, using typed_ast. They are defined unconditionally because 1510 # mypyc can't handle conditional method definitions. 1511 1512 # Num(number n) 1513 def visit_Num(self, n: Num) -> Type: 1514 return self.numeric_type(n.n, n) 1515 1516 # Str(string s) 1517 def visit_Str(self, n: Str) -> Type: 1518 # Note: we transform these fallback types into the correct types in 1519 # 'typeanal.py' -- specifically in the named_type_with_normalized_str method. 1520 # If we're analyzing Python 3, that function will translate 'builtins.unicode' 1521 # into 'builtins.str'. In contrast, if we're analyzing Python 2 code, we'll 1522 # translate 'builtins.bytes' in the method below into 'builtins.str'. 1523 1524 # Do a getattr because the field doesn't exist in 3.8 (where 1525 # this method doesn't actually ever run.) We can't just do 1526 # an attribute access with a `# type: ignore` because it would be 1527 # unused on < 3.8. 1528 kind = getattr(n, 'kind') # type: str # noqa 1529 1530 if 'u' in kind or self.assume_str_is_unicode: 1531 return parse_type_string(n.s, 'builtins.unicode', self.line, n.col_offset, 1532 assume_str_is_unicode=self.assume_str_is_unicode) 1533 else: 1534 return parse_type_string(n.s, 'builtins.str', self.line, n.col_offset, 1535 assume_str_is_unicode=self.assume_str_is_unicode) 1536 1537 # Bytes(bytes s) 1538 def visit_Bytes(self, n: Bytes) -> Type: 1539 contents = bytes_to_human_readable_repr(n.s) 1540 return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset) 1541 1542 # Subscript(expr value, slice slice, expr_context ctx) # Python 3.8 and before 1543 # Subscript(expr value, expr slice, expr_context ctx) # Python 3.9 and later 1544 def visit_Subscript(self, n: ast3.Subscript) -> Type: 1545 if sys.version_info >= (3, 9): # Really 3.9a5 or later 1546 sliceval = n.slice # type: Any 1547 if (isinstance(sliceval, ast3.Slice) or 1548 (isinstance(sliceval, ast3.Tuple) and 1549 any(isinstance(x, ast3.Slice) for x in sliceval.elts))): 1550 self.fail(TYPE_COMMENT_SYNTAX_ERROR, self.line, getattr(n, 'col_offset', -1)) 1551 return AnyType(TypeOfAny.from_error) 1552 else: 1553 # Python 3.8 or earlier use a different AST structure for subscripts 1554 if not isinstance(n.slice, Index): 1555 self.fail(TYPE_COMMENT_SYNTAX_ERROR, self.line, getattr(n, 'col_offset', -1)) 1556 return AnyType(TypeOfAny.from_error) 1557 sliceval = n.slice.value 1558 1559 empty_tuple_index = False 1560 if isinstance(sliceval, ast3.Tuple): 1561 params = self.translate_expr_list(sliceval.elts) 1562 if len(sliceval.elts) == 0: 1563 empty_tuple_index = True 1564 else: 1565 params = [self.visit(sliceval)] 1566 1567 value = self.visit(n.value) 1568 if isinstance(value, UnboundType) and not value.args: 1569 return UnboundType(value.name, params, line=self.line, column=value.column, 1570 empty_tuple_index=empty_tuple_index) 1571 else: 1572 return self.invalid_type(n) 1573 1574 def visit_Tuple(self, n: ast3.Tuple) -> Type: 1575 return TupleType(self.translate_expr_list(n.elts), _dummy_fallback, 1576 implicit=True, line=self.line, column=self.convert_column(n.col_offset)) 1577 1578 # Attribute(expr value, identifier attr, expr_context ctx) 1579 def visit_Attribute(self, n: Attribute) -> Type: 1580 before_dot = self.visit(n.value) 1581 1582 if isinstance(before_dot, UnboundType) and not before_dot.args: 1583 return UnboundType("{}.{}".format(before_dot.name, n.attr), line=self.line) 1584 else: 1585 return self.invalid_type(n) 1586 1587 # Ellipsis 1588 def visit_Ellipsis(self, n: ast3_Ellipsis) -> Type: 1589 return EllipsisType(line=self.line) 1590 1591 # List(expr* elts, expr_context ctx) 1592 def visit_List(self, n: ast3.List) -> Type: 1593 assert isinstance(n.ctx, ast3.Load) 1594 return self.translate_argument_list(n.elts) 1595 1596 1597def stringify_name(n: AST) -> Optional[str]: 1598 if isinstance(n, Name): 1599 return n.id 1600 elif isinstance(n, Attribute): 1601 sv = stringify_name(n.value) 1602 if sv is not None: 1603 return "{}.{}".format(sv, n.attr) 1604 return None # Can't do it. 1605 1606 1607def bytes_to_human_readable_repr(b: bytes) -> str: 1608 """Converts bytes into some human-readable representation. Unprintable 1609 bytes such as the nul byte are escaped. For example: 1610 1611 >>> b = bytes([102, 111, 111, 10, 0]) 1612 >>> s = bytes_to_human_readable_repr(b) 1613 >>> print(s) 1614 foo\n\x00 1615 >>> print(repr(s)) 1616 'foo\\n\\x00' 1617 """ 1618 return repr(b)[2:-1] 1619