1import argparse 2from collections import OrderedDict 3import configparser 4import glob as fileglob 5from io import StringIO 6import os 7import re 8import sys 9 10import toml 11from typing import (Any, Callable, Dict, List, Mapping, MutableMapping, Optional, Sequence, 12 TextIO, Tuple, Union, cast) 13from typing_extensions import Final 14 15from mypy import defaults 16from mypy.options import Options, PER_MODULE_OPTIONS 17 18_CONFIG_VALUE_TYPES = Union[str, bool, int, float, Dict[str, str], List[str], Tuple[int, int]] 19_INI_PARSER_CALLABLE = Callable[[Any], _CONFIG_VALUE_TYPES] 20 21 22def parse_version(v: str) -> Tuple[int, int]: 23 m = re.match(r'\A(\d)\.(\d+)\Z', v) 24 if not m: 25 raise argparse.ArgumentTypeError( 26 "Invalid python version '{}' (expected format: 'x.y')".format(v)) 27 major, minor = int(m.group(1)), int(m.group(2)) 28 if major == 2: 29 if minor != 7: 30 raise argparse.ArgumentTypeError( 31 "Python 2.{} is not supported (must be 2.7)".format(minor)) 32 elif major == 3: 33 if minor < defaults.PYTHON3_VERSION_MIN[1]: 34 raise argparse.ArgumentTypeError( 35 "Python 3.{0} is not supported (must be {1}.{2} or higher)".format(minor, 36 *defaults.PYTHON3_VERSION_MIN)) 37 else: 38 raise argparse.ArgumentTypeError( 39 "Python major version '{}' out of range (must be 2 or 3)".format(major)) 40 return major, minor 41 42 43def try_split(v: Union[str, Sequence[str]], split_regex: str = '[,]') -> List[str]: 44 """Split and trim a str or list of str into a list of str""" 45 if isinstance(v, str): 46 return [p.strip() for p in re.split(split_regex, v)] 47 48 return [p.strip() for p in v] 49 50 51def expand_path(path: str) -> str: 52 """Expand the user home directory and any environment variables contained within 53 the provided path. 54 """ 55 56 return os.path.expandvars(os.path.expanduser(path)) 57 58 59def split_and_match_files_list(paths: Sequence[str]) -> List[str]: 60 """Take a list of files/directories (with support for globbing through the glob library). 61 62 Where a path/glob matches no file, we still include the raw path in the resulting list. 63 64 Returns a list of file paths 65 """ 66 expanded_paths = [] 67 68 for path in paths: 69 path = expand_path(path.strip()) 70 globbed_files = fileglob.glob(path, recursive=True) 71 if globbed_files: 72 expanded_paths.extend(globbed_files) 73 else: 74 expanded_paths.append(path) 75 76 return expanded_paths 77 78 79def split_and_match_files(paths: str) -> List[str]: 80 """Take a string representing a list of files/directories (with support for globbing 81 through the glob library). 82 83 Where a path/glob matches no file, we still include the raw path in the resulting list. 84 85 Returns a list of file paths 86 """ 87 88 return split_and_match_files_list(paths.split(',')) 89 90 91def check_follow_imports(choice: str) -> str: 92 choices = ['normal', 'silent', 'skip', 'error'] 93 if choice not in choices: 94 raise argparse.ArgumentTypeError( 95 "invalid choice '{}' (choose from {})".format( 96 choice, 97 ', '.join("'{}'".format(x) for x in choices))) 98 return choice 99 100 101# For most options, the type of the default value set in options.py is 102# sufficient, and we don't have to do anything here. This table 103# exists to specify types for values initialized to None or container 104# types. 105ini_config_types = { 106 'python_version': parse_version, 107 'strict_optional_whitelist': lambda s: s.split(), 108 'custom_typing_module': str, 109 'custom_typeshed_dir': expand_path, 110 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], 111 'files': split_and_match_files, 112 'quickstart_file': expand_path, 113 'junit_xml': expand_path, 114 # These two are for backwards compatibility 115 'silent_imports': bool, 116 'almost_silent': bool, 117 'follow_imports': check_follow_imports, 118 'no_site_packages': bool, 119 'plugins': lambda s: [p.strip() for p in s.split(',')], 120 'always_true': lambda s: [p.strip() for p in s.split(',')], 121 'always_false': lambda s: [p.strip() for p in s.split(',')], 122 'disable_error_code': lambda s: [p.strip() for p in s.split(',')], 123 'enable_error_code': lambda s: [p.strip() for p in s.split(',')], 124 'package_root': lambda s: [p.strip() for p in s.split(',')], 125 'cache_dir': expand_path, 126 'python_executable': expand_path, 127 'strict': bool, 128} # type: Final[Dict[str, _INI_PARSER_CALLABLE]] 129 130# Reuse the ini_config_types and overwrite the diff 131toml_config_types = ini_config_types.copy() # type: Final[Dict[str, _INI_PARSER_CALLABLE]] 132toml_config_types.update({ 133 'python_version': lambda s: parse_version(str(s)), 134 'strict_optional_whitelist': try_split, 135 'mypy_path': lambda s: [expand_path(p) for p in try_split(s, '[,:]')], 136 'files': lambda s: split_and_match_files_list(try_split(s)), 137 'follow_imports': lambda s: check_follow_imports(str(s)), 138 'plugins': try_split, 139 'always_true': try_split, 140 'always_false': try_split, 141 'disable_error_code': try_split, 142 'enable_error_code': try_split, 143 'package_root': try_split, 144}) 145 146 147def parse_config_file(options: Options, set_strict_flags: Callable[[], None], 148 filename: Optional[str], 149 stdout: Optional[TextIO] = None, 150 stderr: Optional[TextIO] = None) -> None: 151 """Parse a config file into an Options object. 152 153 Errors are written to stderr but are not fatal. 154 155 If filename is None, fall back to default config files. 156 """ 157 stdout = stdout or sys.stdout 158 stderr = stderr or sys.stderr 159 160 if filename is not None: 161 config_files = (filename,) # type: Tuple[str, ...] 162 else: 163 config_files = tuple(map(os.path.expanduser, defaults.CONFIG_FILES)) 164 165 config_parser = configparser.RawConfigParser() 166 167 for config_file in config_files: 168 if not os.path.exists(config_file): 169 continue 170 try: 171 if is_toml(config_file): 172 toml_data = cast("OrderedDict[str, Any]", 173 toml.load(config_file, _dict=OrderedDict)) 174 # Filter down to just mypy relevant toml keys 175 toml_data = toml_data.get('tool', {}) 176 if 'mypy' not in toml_data: 177 continue 178 toml_data = OrderedDict({'mypy': toml_data['mypy']}) 179 parser = destructure_overrides(toml_data) # type: MutableMapping[str, Any] 180 config_types = toml_config_types 181 else: 182 config_parser.read(config_file) 183 parser = config_parser 184 config_types = ini_config_types 185 except (toml.TomlDecodeError, configparser.Error, ConfigTOMLValueError) as err: 186 print("%s: %s" % (config_file, err), file=stderr) 187 else: 188 if config_file in defaults.SHARED_CONFIG_FILES and 'mypy' not in parser: 189 continue 190 file_read = config_file 191 options.config_file = file_read 192 break 193 else: 194 return 195 196 os.environ['MYPY_CONFIG_FILE_DIR'] = os.path.dirname( 197 os.path.abspath(config_file)) 198 199 if 'mypy' not in parser: 200 if filename or file_read not in defaults.SHARED_CONFIG_FILES: 201 print("%s: No [mypy] section in config file" % file_read, file=stderr) 202 else: 203 section = parser['mypy'] 204 prefix = '%s: [%s]: ' % (file_read, 'mypy') 205 updates, report_dirs = parse_section( 206 prefix, options, set_strict_flags, section, config_types, stderr) 207 for k, v in updates.items(): 208 setattr(options, k, v) 209 options.report_dirs.update(report_dirs) 210 211 for name, section in parser.items(): 212 if name.startswith('mypy-'): 213 prefix = get_prefix(file_read, name) 214 updates, report_dirs = parse_section( 215 prefix, options, set_strict_flags, section, config_types, stderr) 216 if report_dirs: 217 print("%sPer-module sections should not specify reports (%s)" % 218 (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))), 219 file=stderr) 220 if set(updates) - PER_MODULE_OPTIONS: 221 print("%sPer-module sections should only specify per-module flags (%s)" % 222 (prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))), 223 file=stderr) 224 updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS} 225 globs = name[5:] 226 for glob in globs.split(','): 227 # For backwards compatibility, replace (back)slashes with dots. 228 glob = glob.replace(os.sep, '.') 229 if os.altsep: 230 glob = glob.replace(os.altsep, '.') 231 232 if (any(c in glob for c in '?[]!') or 233 any('*' in x and x != '*' for x in glob.split('.'))): 234 print("%sPatterns must be fully-qualified module names, optionally " 235 "with '*' in some components (e.g spam.*.eggs.*)" 236 % prefix, 237 file=stderr) 238 else: 239 options.per_module_options[glob] = updates 240 241 242def get_prefix(file_read: str, name: str) -> str: 243 if is_toml(file_read): 244 module_name_str = 'module = "%s"' % '-'.join(name.split('-')[1:]) 245 else: 246 module_name_str = name 247 248 return '%s: [%s]: ' % (file_read, module_name_str) 249 250 251def is_toml(filename: str) -> bool: 252 return filename.lower().endswith('.toml') 253 254 255def destructure_overrides(toml_data: "OrderedDict[str, Any]") -> "OrderedDict[str, Any]": 256 """Take the new [[tool.mypy.overrides]] section array in the pyproject.toml file, 257 and convert it back to a flatter structure that the existing config_parser can handle. 258 259 E.g. the following pyproject.toml file: 260 261 [[tool.mypy.overrides]] 262 module = [ 263 "a.b", 264 "b.*" 265 ] 266 disallow_untyped_defs = true 267 268 [[tool.mypy.overrides]] 269 module = 'c' 270 disallow_untyped_defs = false 271 272 Would map to the following config dict that it would have gotten from parsing an equivalent 273 ini file: 274 275 { 276 "mypy-a.b": { 277 disallow_untyped_defs = true, 278 }, 279 "mypy-b.*": { 280 disallow_untyped_defs = true, 281 }, 282 "mypy-c": { 283 disallow_untyped_defs: false, 284 }, 285 } 286 """ 287 if 'overrides' not in toml_data['mypy']: 288 return toml_data 289 290 if not isinstance(toml_data['mypy']['overrides'], list): 291 raise ConfigTOMLValueError("tool.mypy.overrides sections must be an array. Please make " 292 "sure you are using double brackets like so: [[tool.mypy.overrides]]") 293 294 result = toml_data.copy() 295 for override in result['mypy']['overrides']: 296 if 'module' not in override: 297 raise ConfigTOMLValueError("toml config file contains a [[tool.mypy.overrides]] " 298 "section, but no module to override was specified.") 299 300 if isinstance(override['module'], str): 301 modules = [override['module']] 302 elif isinstance(override['module'], list): 303 modules = override['module'] 304 else: 305 raise ConfigTOMLValueError("toml config file contains a [[tool.mypy.overrides]] " 306 "section with a module value that is not a string or a list of " 307 "strings") 308 309 for module in modules: 310 module_overrides = override.copy() 311 del module_overrides['module'] 312 old_config_name = 'mypy-%s' % module 313 if old_config_name not in result: 314 result[old_config_name] = module_overrides 315 else: 316 for new_key, new_value in module_overrides.items(): 317 if (new_key in result[old_config_name] and 318 result[old_config_name][new_key] != new_value): 319 raise ConfigTOMLValueError("toml config file contains " 320 "[[tool.mypy.overrides]] sections with conflicting " 321 "values. Module '%s' has two different values for '%s'" 322 % (module, new_key)) 323 result[old_config_name][new_key] = new_value 324 325 del result['mypy']['overrides'] 326 return result 327 328 329def parse_section(prefix: str, template: Options, 330 set_strict_flags: Callable[[], None], 331 section: Mapping[str, Any], 332 config_types: Dict[str, Any], 333 stderr: TextIO = sys.stderr 334 ) -> Tuple[Dict[str, object], Dict[str, str]]: 335 """Parse one section of a config file. 336 337 Returns a dict of option values encountered, and a dict of report directories. 338 """ 339 results = {} # type: Dict[str, object] 340 report_dirs = {} # type: Dict[str, str] 341 for key in section: 342 invert = False 343 options_key = key 344 if key in config_types: 345 ct = config_types[key] 346 else: 347 dv = None 348 # We have to keep new_semantic_analyzer in Options 349 # for plugin compatibility but it is not a valid option anymore. 350 assert hasattr(template, 'new_semantic_analyzer') 351 if key != 'new_semantic_analyzer': 352 dv = getattr(template, key, None) 353 if dv is None: 354 if key.endswith('_report'): 355 report_type = key[:-7].replace('_', '-') 356 if report_type in defaults.REPORTER_NAMES: 357 report_dirs[report_type] = str(section[key]) 358 else: 359 print("%sUnrecognized report type: %s" % (prefix, key), 360 file=stderr) 361 continue 362 if key.startswith('x_'): 363 pass # Don't complain about `x_blah` flags 364 elif key.startswith('no_') and hasattr(template, key[3:]): 365 options_key = key[3:] 366 invert = True 367 elif key.startswith('allow') and hasattr(template, 'dis' + key): 368 options_key = 'dis' + key 369 invert = True 370 elif key.startswith('disallow') and hasattr(template, key[3:]): 371 options_key = key[3:] 372 invert = True 373 elif key == 'strict': 374 pass # Special handling below 375 else: 376 print("%sUnrecognized option: %s = %s" % (prefix, key, section[key]), 377 file=stderr) 378 if invert: 379 dv = getattr(template, options_key, None) 380 else: 381 continue 382 ct = type(dv) 383 v = None # type: Any 384 try: 385 if ct is bool: 386 if isinstance(section, dict): 387 v = convert_to_boolean(section.get(key)) 388 else: 389 v = section.getboolean(key) # type: ignore[attr-defined] # Until better stub 390 if invert: 391 v = not v 392 elif callable(ct): 393 if invert: 394 print("%sCan not invert non-boolean key %s" % (prefix, options_key), 395 file=stderr) 396 continue 397 try: 398 v = ct(section.get(key)) 399 except argparse.ArgumentTypeError as err: 400 print("%s%s: %s" % (prefix, key, err), file=stderr) 401 continue 402 else: 403 print("%sDon't know what type %s should have" % (prefix, key), file=stderr) 404 continue 405 except ValueError as err: 406 print("%s%s: %s" % (prefix, key, err), file=stderr) 407 continue 408 if key == 'strict': 409 if v: 410 set_strict_flags() 411 continue 412 if key == 'silent_imports': 413 print("%ssilent_imports has been replaced by " 414 "ignore_missing_imports=True; follow_imports=skip" % prefix, file=stderr) 415 if v: 416 if 'ignore_missing_imports' not in results: 417 results['ignore_missing_imports'] = True 418 if 'follow_imports' not in results: 419 results['follow_imports'] = 'skip' 420 if key == 'almost_silent': 421 print("%salmost_silent has been replaced by " 422 "follow_imports=error" % prefix, file=stderr) 423 if v: 424 if 'follow_imports' not in results: 425 results['follow_imports'] = 'error' 426 results[options_key] = v 427 return results, report_dirs 428 429 430def convert_to_boolean(value: Optional[Any]) -> bool: 431 """Return a boolean value translating from other types if necessary.""" 432 if isinstance(value, bool): 433 return value 434 if not isinstance(value, str): 435 value = str(value) 436 if value.lower() not in configparser.RawConfigParser.BOOLEAN_STATES: 437 raise ValueError('Not a boolean: %s' % value) 438 return configparser.RawConfigParser.BOOLEAN_STATES[value.lower()] 439 440 441def split_directive(s: str) -> Tuple[List[str], List[str]]: 442 """Split s on commas, except during quoted sections. 443 444 Returns the parts and a list of error messages.""" 445 parts = [] 446 cur = [] # type: List[str] 447 errors = [] 448 i = 0 449 while i < len(s): 450 if s[i] == ',': 451 parts.append(''.join(cur).strip()) 452 cur = [] 453 elif s[i] == '"': 454 i += 1 455 while i < len(s) and s[i] != '"': 456 cur.append(s[i]) 457 i += 1 458 if i == len(s): 459 errors.append("Unterminated quote in configuration comment") 460 cur.clear() 461 else: 462 cur.append(s[i]) 463 i += 1 464 if cur: 465 parts.append(''.join(cur).strip()) 466 467 return parts, errors 468 469 470def mypy_comments_to_config_map(line: str, 471 template: Options) -> Tuple[Dict[str, str], List[str]]: 472 """Rewrite the mypy comment syntax into ini file syntax. 473 474 Returns 475 """ 476 options = {} 477 entries, errors = split_directive(line) 478 for entry in entries: 479 if '=' not in entry: 480 name = entry 481 value = None 482 else: 483 name, value = [x.strip() for x in entry.split('=', 1)] 484 485 name = name.replace('-', '_') 486 if value is None: 487 value = 'True' 488 options[name] = value 489 490 return options, errors 491 492 493def parse_mypy_comments( 494 args: List[Tuple[int, str]], 495 template: Options) -> Tuple[Dict[str, object], List[Tuple[int, str]]]: 496 """Parse a collection of inline mypy: configuration comments. 497 498 Returns a dictionary of options to be applied and a list of error messages 499 generated. 500 """ 501 502 errors = [] # type: List[Tuple[int, str]] 503 sections = {} 504 505 for lineno, line in args: 506 # In order to easily match the behavior for bools, we abuse configparser. 507 # Oddly, the only way to get the SectionProxy object with the getboolean 508 # method is to create a config parser. 509 parser = configparser.RawConfigParser() 510 options, parse_errors = mypy_comments_to_config_map(line, template) 511 parser['dummy'] = options 512 errors.extend((lineno, x) for x in parse_errors) 513 514 stderr = StringIO() 515 strict_found = False 516 517 def set_strict_flags() -> None: 518 nonlocal strict_found 519 strict_found = True 520 521 new_sections, reports = parse_section( 522 '', template, set_strict_flags, parser['dummy'], ini_config_types, stderr=stderr) 523 errors.extend((lineno, x) for x in stderr.getvalue().strip().split('\n') if x) 524 if reports: 525 errors.append((lineno, "Reports not supported in inline configuration")) 526 if strict_found: 527 errors.append((lineno, 528 'Setting "strict" not supported in inline configuration: specify it in ' 529 'a configuration file instead, or set individual inline flags ' 530 '(see "mypy -h" for the list of flags enabled in strict mode)')) 531 532 sections.update(new_sections) 533 534 return sections, errors 535 536 537def get_config_module_names(filename: Optional[str], modules: List[str]) -> str: 538 if not filename or not modules: 539 return '' 540 541 if not is_toml(filename): 542 return ", ".join("[mypy-%s]" % module for module in modules) 543 544 return "module = ['%s']" % ("', '".join(sorted(modules))) 545 546 547class ConfigTOMLValueError(ValueError): 548 pass 549