1from typing import Any, Callable, Generator, Iterable, List, NamedTuple, Optional, Union, Sequence, TextIO, Tuple 2from builtins import open as _builtin_open 3import sys 4from token import * # noqa: F403 5 6if sys.version_info < (3, 7): 7 COMMENT: int 8 NL: int 9 ENCODING: int 10 11_Position = Tuple[int, int] 12 13class _TokenInfo(NamedTuple): 14 type: int 15 string: str 16 start: _Position 17 end: _Position 18 line: str 19 20class TokenInfo(_TokenInfo): 21 @property 22 def exact_type(self) -> int: ... 23 24# Backwards compatible tokens can be sequences of a shorter length too 25_Token = Union[TokenInfo, Sequence[Union[int, str, _Position]]] 26 27class TokenError(Exception): ... 28class StopTokenizing(Exception): ... 29 30class Untokenizer: 31 tokens: List[str] 32 prev_row: int 33 prev_col: int 34 encoding: Optional[str] 35 def __init__(self) -> None: ... 36 def add_whitespace(self, start: _Position) -> None: ... 37 def untokenize(self, iterable: Iterable[_Token]) -> str: ... 38 def compat(self, token: Sequence[Union[int, str]], iterable: Iterable[_Token]) -> None: ... 39 40def untokenize(iterable: Iterable[_Token]) -> Any: ... 41def detect_encoding(readline: Callable[[], bytes]) -> Tuple[str, Sequence[bytes]]: ... 42def tokenize(readline: Callable[[], bytes]) -> Generator[TokenInfo, None, None]: ... 43def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, None]: ... # undocumented 44 45if sys.version_info >= (3, 6): 46 from os import PathLike 47 def open(filename: Union[str, bytes, int, PathLike[Any]]) -> TextIO: ... 48else: 49 def open(filename: Union[str, bytes, int]) -> TextIO: ... 50 51# Names in __all__ with no definition: 52# AMPER 53# AMPEREQUAL 54# ASYNC 55# AT 56# ATEQUAL 57# AWAIT 58# CIRCUMFLEX 59# CIRCUMFLEXEQUAL 60# COLON 61# COMMA 62# DEDENT 63# DOT 64# DOUBLESLASH 65# DOUBLESLASHEQUAL 66# DOUBLESTAR 67# DOUBLESTAREQUAL 68# ELLIPSIS 69# ENDMARKER 70# EQEQUAL 71# EQUAL 72# ERRORTOKEN 73# GREATER 74# GREATEREQUAL 75# INDENT 76# ISEOF 77# ISNONTERMINAL 78# ISTERMINAL 79# LBRACE 80# LEFTSHIFT 81# LEFTSHIFTEQUAL 82# LESS 83# LESSEQUAL 84# LPAR 85# LSQB 86# MINEQUAL 87# MINUS 88# NAME 89# NEWLINE 90# NOTEQUAL 91# NT_OFFSET 92# NUMBER 93# N_TOKENS 94# OP 95# PERCENT 96# PERCENTEQUAL 97# PLUS 98# PLUSEQUAL 99# RARROW 100# RBRACE 101# RIGHTSHIFT 102# RIGHTSHIFTEQUAL 103# RPAR 104# RSQB 105# SEMI 106# SLASH 107# SLASHEQUAL 108# STAR 109# STAREQUAL 110# STRING 111# TILDE 112# VBAR 113# VBAREQUAL 114# tok_name 115