1""" 2Collect various information about Python to help debugging test failures. 3""" 4from __future__ import print_function 5import errno 6import re 7import sys 8import traceback 9import warnings 10 11 12def normalize_text(text): 13 if text is None: 14 return None 15 text = str(text) 16 text = re.sub(r'\s+', ' ', text) 17 return text.strip() 18 19 20class PythonInfo: 21 def __init__(self): 22 self.info = {} 23 24 def add(self, key, value): 25 if key in self.info: 26 raise ValueError("duplicate key: %r" % key) 27 28 if value is None: 29 return 30 31 if not isinstance(value, int): 32 if not isinstance(value, str): 33 # convert other objects like sys.flags to string 34 value = str(value) 35 36 value = value.strip() 37 if not value: 38 return 39 40 self.info[key] = value 41 42 def get_infos(self): 43 """ 44 Get information as a key:value dictionary where values are strings. 45 """ 46 return {key: str(value) for key, value in self.info.items()} 47 48 49def copy_attributes(info_add, obj, name_fmt, attributes, formatter=None): 50 for attr in attributes: 51 value = getattr(obj, attr, None) 52 if value is None: 53 continue 54 name = name_fmt % attr 55 if formatter is not None: 56 value = formatter(attr, value) 57 info_add(name, value) 58 59 60def copy_attr(info_add, name, mod, attr_name): 61 try: 62 value = getattr(mod, attr_name) 63 except AttributeError: 64 return 65 info_add(name, value) 66 67 68def call_func(info_add, name, mod, func_name, formatter=None): 69 try: 70 func = getattr(mod, func_name) 71 except AttributeError: 72 return 73 value = func() 74 if formatter is not None: 75 value = formatter(value) 76 info_add(name, value) 77 78 79def collect_sys(info_add): 80 attributes = ( 81 '_framework', 82 'abiflags', 83 'api_version', 84 'builtin_module_names', 85 'byteorder', 86 'dont_write_bytecode', 87 'executable', 88 'flags', 89 'float_info', 90 'float_repr_style', 91 'hash_info', 92 'hexversion', 93 'implementation', 94 'int_info', 95 'maxsize', 96 'maxunicode', 97 'path', 98 'platform', 99 'prefix', 100 'thread_info', 101 'version', 102 'version_info', 103 'winver', 104 ) 105 copy_attributes(info_add, sys, 'sys.%s', attributes) 106 107 call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') 108 call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') 109 110 encoding = sys.getfilesystemencoding() 111 if hasattr(sys, 'getfilesystemencodeerrors'): 112 encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors()) 113 info_add('sys.filesystem_encoding', encoding) 114 115 for name in ('stdin', 'stdout', 'stderr'): 116 stream = getattr(sys, name) 117 if stream is None: 118 continue 119 encoding = getattr(stream, 'encoding', None) 120 if not encoding: 121 continue 122 errors = getattr(stream, 'errors', None) 123 if errors: 124 encoding = '%s/%s' % (encoding, errors) 125 info_add('sys.%s.encoding' % name, encoding) 126 127 # Were we compiled --with-pydebug or with #define Py_DEBUG? 128 Py_DEBUG = hasattr(sys, 'gettotalrefcount') 129 if Py_DEBUG: 130 text = 'Yes (sys.gettotalrefcount() present)' 131 else: 132 text = 'No (sys.gettotalrefcount() missing)' 133 info_add('Py_DEBUG', text) 134 135 136def collect_platform(info_add): 137 import platform 138 139 arch = platform.architecture() 140 arch = ' '.join(filter(bool, arch)) 141 info_add('platform.architecture', arch) 142 143 info_add('platform.python_implementation', 144 platform.python_implementation()) 145 info_add('platform.platform', 146 platform.platform(aliased=True)) 147 148 libc_ver = ('%s %s' % platform.libc_ver()).strip() 149 if libc_ver: 150 info_add('platform.libc_ver', libc_ver) 151 152 153def collect_locale(info_add): 154 import locale 155 156 info_add('locale.encoding', locale.getpreferredencoding(False)) 157 158 159def collect_builtins(info_add): 160 info_add('builtins.float.float_format', float.__getformat__("float")) 161 info_add('builtins.float.double_format', float.__getformat__("double")) 162 163 164def collect_os(info_add): 165 import os 166 167 def format_attr(attr, value): 168 if attr in ('supports_follow_symlinks', 'supports_fd', 169 'supports_effective_ids'): 170 return str(sorted(func.__name__ for func in value)) 171 else: 172 return value 173 174 attributes = ( 175 'name', 176 'supports_bytes_environ', 177 'supports_effective_ids', 178 'supports_fd', 179 'supports_follow_symlinks', 180 ) 181 copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) 182 183 call_func(info_add, 'os.cwd', os, 'getcwd') 184 185 call_func(info_add, 'os.uid', os, 'getuid') 186 call_func(info_add, 'os.gid', os, 'getgid') 187 call_func(info_add, 'os.uname', os, 'uname') 188 189 def format_groups(groups): 190 return ', '.join(map(str, groups)) 191 192 call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups) 193 194 if hasattr(os, 'getlogin'): 195 try: 196 login = os.getlogin() 197 except OSError: 198 # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl 199 # for device" on Travis CI 200 pass 201 else: 202 info_add("os.login", login) 203 204 call_func(info_add, 'os.cpu_count', os, 'cpu_count') 205 call_func(info_add, 'os.loadavg', os, 'getloadavg') 206 207 # Environment variables used by the stdlib and tests. Don't log the full 208 # environment: filter to list to not leak sensitive information. 209 # 210 # HTTP_PROXY is not logged because it can contain a password. 211 ENV_VARS = frozenset(( 212 "APPDATA", 213 "AR", 214 "ARCHFLAGS", 215 "ARFLAGS", 216 "AUDIODEV", 217 "CC", 218 "CFLAGS", 219 "COLUMNS", 220 "COMPUTERNAME", 221 "COMSPEC", 222 "CPP", 223 "CPPFLAGS", 224 "DISPLAY", 225 "DISTUTILS_DEBUG", 226 "DISTUTILS_USE_SDK", 227 "DYLD_LIBRARY_PATH", 228 "ENSUREPIP_OPTIONS", 229 "HISTORY_FILE", 230 "HOME", 231 "HOMEDRIVE", 232 "HOMEPATH", 233 "IDLESTARTUP", 234 "LANG", 235 "LDFLAGS", 236 "LDSHARED", 237 "LD_LIBRARY_PATH", 238 "LINES", 239 "MACOSX_DEPLOYMENT_TARGET", 240 "MAILCAPS", 241 "MAKEFLAGS", 242 "MIXERDEV", 243 "MSSDK", 244 "PATH", 245 "PATHEXT", 246 "PIP_CONFIG_FILE", 247 "PLAT", 248 "POSIXLY_CORRECT", 249 "PY_SAX_PARSER", 250 "ProgramFiles", 251 "ProgramFiles(x86)", 252 "RUNNING_ON_VALGRIND", 253 "SDK_TOOLS_BIN", 254 "SERVER_SOFTWARE", 255 "SHELL", 256 "SOURCE_DATE_EPOCH", 257 "SYSTEMROOT", 258 "TEMP", 259 "TERM", 260 "TILE_LIBRARY", 261 "TIX_LIBRARY", 262 "TMP", 263 "TMPDIR", 264 "TRAVIS", 265 "TZ", 266 "USERPROFILE", 267 "VIRTUAL_ENV", 268 "WAYLAND_DISPLAY", 269 "WINDIR", 270 "_PYTHON_HOST_PLATFORM", 271 "_PYTHON_PROJECT_BASE", 272 "_PYTHON_SYSCONFIGDATA_NAME", 273 "__PYVENV_LAUNCHER__", 274 )) 275 for name, value in os.environ.items(): 276 uname = name.upper() 277 if (uname in ENV_VARS 278 # Copy PYTHON* and LC_* variables 279 or uname.startswith(("PYTHON", "LC_")) 280 # Visual Studio: VS140COMNTOOLS 281 or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): 282 info_add('os.environ[%s]' % name, value) 283 284 if hasattr(os, 'umask'): 285 mask = os.umask(0) 286 os.umask(mask) 287 info_add("os.umask", '%03o' % mask) 288 289 if hasattr(os, 'getrandom'): 290 # PEP 524: Check if system urandom is initialized 291 try: 292 try: 293 os.getrandom(1, os.GRND_NONBLOCK) 294 state = 'ready (initialized)' 295 except BlockingIOError as exc: 296 state = 'not seeded yet (%s)' % exc 297 info_add('os.getrandom', state) 298 except OSError as exc: 299 # Python was compiled on a more recent Linux version 300 # than the current Linux kernel: ignore OSError(ENOSYS) 301 if exc.errno != errno.ENOSYS: 302 raise 303 304 305def collect_readline(info_add): 306 try: 307 import readline 308 except ImportError: 309 return 310 311 def format_attr(attr, value): 312 if isinstance(value, int): 313 return "%#x" % value 314 else: 315 return value 316 317 attributes = ( 318 "_READLINE_VERSION", 319 "_READLINE_RUNTIME_VERSION", 320 "_READLINE_LIBRARY_VERSION", 321 ) 322 copy_attributes(info_add, readline, 'readline.%s', attributes, 323 formatter=format_attr) 324 325 if not hasattr(readline, "_READLINE_LIBRARY_VERSION"): 326 # _READLINE_LIBRARY_VERSION has been added to CPython 3.7 327 doc = getattr(readline, '__doc__', '') 328 if 'libedit readline' in doc: 329 info_add('readline.library', 'libedit readline') 330 elif 'GNU readline' in doc: 331 info_add('readline.library', 'GNU readline') 332 333 334def collect_gdb(info_add): 335 import subprocess 336 337 try: 338 proc = subprocess.Popen(["gdb", "-nx", "--version"], 339 stdout=subprocess.PIPE, 340 stderr=subprocess.PIPE, 341 universal_newlines=True) 342 version = proc.communicate()[0] 343 except OSError: 344 return 345 346 # Only keep the first line 347 version = version.splitlines()[0] 348 info_add('gdb_version', version) 349 350 351def collect_tkinter(info_add): 352 try: 353 import _tkinter 354 except ImportError: 355 pass 356 else: 357 attributes = ('TK_VERSION', 'TCL_VERSION') 358 copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes) 359 360 try: 361 import tkinter 362 except ImportError: 363 pass 364 else: 365 tcl = tkinter.Tcl() 366 patchlevel = tcl.call('info', 'patchlevel') 367 info_add('tkinter.info_patchlevel', patchlevel) 368 369 370def collect_time(info_add): 371 import time 372 373 info_add('time.time', time.time()) 374 375 attributes = ( 376 'altzone', 377 'daylight', 378 'timezone', 379 'tzname', 380 ) 381 copy_attributes(info_add, time, 'time.%s', attributes) 382 383 if hasattr(time, 'get_clock_info'): 384 for clock in ('clock', 'monotonic', 'perf_counter', 385 'process_time', 'thread_time', 'time'): 386 try: 387 # prevent DeprecatingWarning on get_clock_info('clock') 388 with warnings.catch_warnings(record=True): 389 clock_info = time.get_clock_info(clock) 390 except ValueError: 391 # missing clock like time.thread_time() 392 pass 393 else: 394 info_add('time.get_clock_info(%s)' % clock, clock_info) 395 396 397def collect_datetime(info_add): 398 try: 399 import datetime 400 except ImportError: 401 return 402 403 info_add('datetime.datetime.now', datetime.datetime.now()) 404 405 406def collect_sysconfig(info_add): 407 import sysconfig 408 409 for name in ( 410 'ABIFLAGS', 411 'ANDROID_API_LEVEL', 412 'CC', 413 'CCSHARED', 414 'CFLAGS', 415 'CFLAGSFORSHARED', 416 'CONFIG_ARGS', 417 'HOST_GNU_TYPE', 418 'MACHDEP', 419 'MULTIARCH', 420 'OPT', 421 'PY_CFLAGS', 422 'PY_CFLAGS_NODIST', 423 'PY_CORE_LDFLAGS', 424 'PY_LDFLAGS', 425 'PY_LDFLAGS_NODIST', 426 'PY_STDMODULE_CFLAGS', 427 'Py_DEBUG', 428 'Py_ENABLE_SHARED', 429 'SHELL', 430 'SOABI', 431 'prefix', 432 ): 433 value = sysconfig.get_config_var(name) 434 if name == 'ANDROID_API_LEVEL' and not value: 435 # skip ANDROID_API_LEVEL=0 436 continue 437 value = normalize_text(value) 438 info_add('sysconfig[%s]' % name, value) 439 440 441def collect_ssl(info_add): 442 import os 443 try: 444 import ssl 445 except ImportError: 446 return 447 try: 448 import _ssl 449 except ImportError: 450 _ssl = None 451 452 def format_attr(attr, value): 453 if attr.startswith('OP_'): 454 return '%#8x' % value 455 else: 456 return value 457 458 attributes = ( 459 'OPENSSL_VERSION', 460 'OPENSSL_VERSION_INFO', 461 'HAS_SNI', 462 'OP_ALL', 463 'OP_NO_TLSv1_1', 464 ) 465 copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr) 466 467 options_names = [] 468 protocol_names = {} 469 verify_modes = {} 470 for name in dir(ssl): 471 if name.startswith('OP_'): 472 options_names.append((name, getattr(ssl, name))) 473 elif name.startswith('PROTOCOL_'): 474 protocol_names[getattr(ssl, name)] = name 475 elif name.startswith('CERT_'): 476 verify_modes[getattr(ssl, name)] = name 477 options_names.sort(key=lambda item: item[1], reverse=True) 478 479 def formatter(attr_name, value): 480 if attr_name == 'options': 481 options_text = [] 482 for opt_name, opt_value in options_names: 483 if value & opt_value: 484 options_text.append(opt_name) 485 value &= ~opt_value 486 if value: 487 options_text.append(str(value)) 488 return '|' .join(options_text) 489 elif attr_name == 'verify_mode': 490 return verify_modes.get(value, value) 491 elif attr_name == 'protocol': 492 return protocol_names.get(value, value) 493 else: 494 return value 495 496 for name, ctx in ( 497 ('SSLContext(PROTOCOL_TLS)', ssl.SSLContext(ssl.PROTOCOL_TLS)), 498 ('default_https_context', ssl._create_default_https_context()), 499 ('stdlib_context', ssl._create_stdlib_context()), 500 ): 501 attributes = ( 502 'minimum_version', 503 'maximum_version', 504 'protocol', 505 'options', 506 'verify_mode', 507 ) 508 copy_attributes(info_add, ctx, 'ssl.%s.%%s' % name, attributes, formatter=formatter) 509 510 env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"] 511 if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'): 512 parts = _ssl.get_default_verify_paths() 513 env_names.extend((parts[0], parts[2])) 514 515 for name in env_names: 516 try: 517 value = os.environ[name] 518 except KeyError: 519 continue 520 info_add('ssl.environ[%s]' % name, value) 521 522 523def collect_socket(info_add): 524 import socket 525 526 hostname = socket.gethostname() 527 info_add('socket.hostname', hostname) 528 529 530def collect_sqlite(info_add): 531 try: 532 import sqlite3 533 except ImportError: 534 return 535 536 attributes = ('version', 'sqlite_version') 537 copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes) 538 539 540def collect_zlib(info_add): 541 try: 542 import zlib 543 except ImportError: 544 return 545 546 attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION') 547 copy_attributes(info_add, zlib, 'zlib.%s', attributes) 548 549 550def collect_expat(info_add): 551 try: 552 from xml.parsers import expat 553 except ImportError: 554 return 555 556 attributes = ('EXPAT_VERSION',) 557 copy_attributes(info_add, expat, 'expat.%s', attributes) 558 559 560def collect_decimal(info_add): 561 try: 562 import _decimal 563 except ImportError: 564 return 565 566 attributes = ('__libmpdec_version__',) 567 copy_attributes(info_add, _decimal, '_decimal.%s', attributes) 568 569 570def collect_testcapi(info_add): 571 try: 572 import _testcapi 573 except ImportError: 574 return 575 576 call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname') 577 copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC') 578 579 580def collect_resource(info_add): 581 try: 582 import resource 583 except ImportError: 584 return 585 586 limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')] 587 for name in limits: 588 key = getattr(resource, name) 589 value = resource.getrlimit(key) 590 info_add('resource.%s' % name, value) 591 592 call_func(info_add, 'resource.pagesize', resource, 'getpagesize') 593 594 595def collect_test_socket(info_add): 596 try: 597 from test import test_socket 598 except ImportError: 599 return 600 601 # all check attributes like HAVE_SOCKET_CAN 602 attributes = [name for name in dir(test_socket) 603 if name.startswith('HAVE_')] 604 copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) 605 606 607def collect_test_support(info_add): 608 try: 609 from test import support 610 except ImportError: 611 return 612 613 attributes = ('IPV6_ENABLED',) 614 copy_attributes(info_add, support, 'test_support.%s', attributes) 615 616 call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available') 617 call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized') 618 619 620def collect_cc(info_add): 621 import subprocess 622 import sysconfig 623 624 CC = sysconfig.get_config_var('CC') 625 if not CC: 626 return 627 628 try: 629 import shlex 630 args = shlex.split(CC) 631 except ImportError: 632 args = CC.split() 633 args.append('--version') 634 try: 635 proc = subprocess.Popen(args, 636 stdout=subprocess.PIPE, 637 stderr=subprocess.STDOUT, 638 universal_newlines=True) 639 except OSError: 640 # Cannot run the compiler, for example when Python has been 641 # cross-compiled and installed on the target platform where the 642 # compiler is missing. 643 return 644 645 stdout = proc.communicate()[0] 646 if proc.returncode: 647 # CC --version failed: ignore error 648 return 649 650 text = stdout.splitlines()[0] 651 text = normalize_text(text) 652 info_add('CC.version', text) 653 654 655def collect_gdbm(info_add): 656 try: 657 from _gdbm import _GDBM_VERSION 658 except ImportError: 659 return 660 661 info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION))) 662 663 664def collect_get_config(info_add): 665 # Dump global configuration variables, _PyCoreConfig 666 # and _PyMainInterpreterConfig 667 try: 668 from _testinternalcapi import get_configs 669 except ImportError: 670 return 671 672 all_configs = get_configs() 673 for config_type in sorted(all_configs): 674 config = all_configs[config_type] 675 for key in sorted(config): 676 info_add('%s[%s]' % (config_type, key), repr(config[key])) 677 678 679def collect_subprocess(info_add): 680 import subprocess 681 copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) 682 683 684def collect_info(info): 685 error = False 686 info_add = info.add 687 688 for collect_func in ( 689 # collect_os() should be the first, to check the getrandom() status 690 collect_os, 691 692 collect_builtins, 693 collect_gdb, 694 collect_locale, 695 collect_platform, 696 collect_readline, 697 collect_socket, 698 collect_sqlite, 699 collect_ssl, 700 collect_sys, 701 collect_sysconfig, 702 collect_time, 703 collect_datetime, 704 collect_tkinter, 705 collect_zlib, 706 collect_expat, 707 collect_decimal, 708 collect_testcapi, 709 collect_resource, 710 collect_cc, 711 collect_gdbm, 712 collect_get_config, 713 collect_subprocess, 714 715 # Collecting from tests should be last as they have side effects. 716 collect_test_socket, 717 collect_test_support, 718 ): 719 try: 720 collect_func(info_add) 721 except Exception as exc: 722 error = True 723 print("ERROR: %s() failed" % (collect_func.__name__), 724 file=sys.stderr) 725 traceback.print_exc(file=sys.stderr) 726 print(file=sys.stderr) 727 sys.stderr.flush() 728 729 return error 730 731 732def dump_info(info, file=None): 733 title = "Python debug information" 734 print(title) 735 print("=" * len(title)) 736 print() 737 738 infos = info.get_infos() 739 infos = sorted(infos.items()) 740 for key, value in infos: 741 value = value.replace("\n", " ") 742 print("%s: %s" % (key, value)) 743 print() 744 745 746def main(): 747 info = PythonInfo() 748 error = collect_info(info) 749 dump_info(info) 750 751 if error: 752 print("Collection failed: exit with error", file=sys.stderr) 753 sys.exit(1) 754 755 756if __name__ == "__main__": 757 main() 758