1"""
2Collect various information about Python to help debugging test failures.
3"""
4from __future__ import print_function
5import errno
6import re
7import sys
8import traceback
9import warnings
10
11
12def normalize_text(text):
13    if text is None:
14        return None
15    text = str(text)
16    text = re.sub(r'\s+', ' ', text)
17    return text.strip()
18
19
20class PythonInfo:
21    def __init__(self):
22        self.info = {}
23
24    def add(self, key, value):
25        if key in self.info:
26            raise ValueError("duplicate key: %r" % key)
27
28        if value is None:
29            return
30
31        if not isinstance(value, int):
32            if not isinstance(value, str):
33                # convert other objects like sys.flags to string
34                value = str(value)
35
36            value = value.strip()
37            if not value:
38                return
39
40        self.info[key] = value
41
42    def get_infos(self):
43        """
44        Get information as a key:value dictionary where values are strings.
45        """
46        return {key: str(value) for key, value in self.info.items()}
47
48
49def copy_attributes(info_add, obj, name_fmt, attributes, formatter=None):
50    for attr in attributes:
51        value = getattr(obj, attr, None)
52        if value is None:
53            continue
54        name = name_fmt % attr
55        if formatter is not None:
56            value = formatter(attr, value)
57        info_add(name, value)
58
59
60def copy_attr(info_add, name, mod, attr_name):
61    try:
62        value = getattr(mod, attr_name)
63    except AttributeError:
64        return
65    info_add(name, value)
66
67
68def call_func(info_add, name, mod, func_name, formatter=None):
69    try:
70        func = getattr(mod, func_name)
71    except AttributeError:
72        return
73    value = func()
74    if formatter is not None:
75        value = formatter(value)
76    info_add(name, value)
77
78
79def collect_sys(info_add):
80    attributes = (
81        '_framework',
82        'abiflags',
83        'api_version',
84        'builtin_module_names',
85        'byteorder',
86        'dont_write_bytecode',
87        'executable',
88        'flags',
89        'float_info',
90        'float_repr_style',
91        'hash_info',
92        'hexversion',
93        'implementation',
94        'int_info',
95        'maxsize',
96        'maxunicode',
97        'path',
98        'platform',
99        'prefix',
100        'thread_info',
101        'version',
102        'version_info',
103        'winver',
104    )
105    copy_attributes(info_add, sys, 'sys.%s', attributes)
106
107    call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel')
108    call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion')
109
110    encoding = sys.getfilesystemencoding()
111    if hasattr(sys, 'getfilesystemencodeerrors'):
112        encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors())
113    info_add('sys.filesystem_encoding', encoding)
114
115    for name in ('stdin', 'stdout', 'stderr'):
116        stream = getattr(sys, name)
117        if stream is None:
118            continue
119        encoding = getattr(stream, 'encoding', None)
120        if not encoding:
121            continue
122        errors = getattr(stream, 'errors', None)
123        if errors:
124            encoding = '%s/%s' % (encoding, errors)
125        info_add('sys.%s.encoding' % name, encoding)
126
127    # Were we compiled --with-pydebug or with #define Py_DEBUG?
128    Py_DEBUG = hasattr(sys, 'gettotalrefcount')
129    if Py_DEBUG:
130        text = 'Yes (sys.gettotalrefcount() present)'
131    else:
132        text = 'No (sys.gettotalrefcount() missing)'
133    info_add('Py_DEBUG', text)
134
135
136def collect_platform(info_add):
137    import platform
138
139    arch = platform.architecture()
140    arch = ' '.join(filter(bool, arch))
141    info_add('platform.architecture', arch)
142
143    info_add('platform.python_implementation',
144             platform.python_implementation())
145    info_add('platform.platform',
146             platform.platform(aliased=True))
147
148    libc_ver = ('%s %s' % platform.libc_ver()).strip()
149    if libc_ver:
150        info_add('platform.libc_ver', libc_ver)
151
152
153def collect_locale(info_add):
154    import locale
155
156    info_add('locale.encoding', locale.getpreferredencoding(False))
157
158
159def collect_builtins(info_add):
160    info_add('builtins.float.float_format', float.__getformat__("float"))
161    info_add('builtins.float.double_format', float.__getformat__("double"))
162
163
164def collect_os(info_add):
165    import os
166
167    def format_attr(attr, value):
168        if attr in ('supports_follow_symlinks', 'supports_fd',
169                    'supports_effective_ids'):
170            return str(sorted(func.__name__ for func in value))
171        else:
172            return value
173
174    attributes = (
175        'name',
176        'supports_bytes_environ',
177        'supports_effective_ids',
178        'supports_fd',
179        'supports_follow_symlinks',
180    )
181    copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
182
183    call_func(info_add, 'os.cwd', os, 'getcwd')
184
185    call_func(info_add, 'os.uid', os, 'getuid')
186    call_func(info_add, 'os.gid', os, 'getgid')
187    call_func(info_add, 'os.uname', os, 'uname')
188
189    def format_groups(groups):
190        return ', '.join(map(str, groups))
191
192    call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups)
193
194    if hasattr(os, 'getlogin'):
195        try:
196            login = os.getlogin()
197        except OSError:
198            # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl
199            # for device" on Travis CI
200            pass
201        else:
202            info_add("os.login", login)
203
204    call_func(info_add, 'os.cpu_count', os, 'cpu_count')
205    call_func(info_add, 'os.loadavg', os, 'getloadavg')
206
207    # Environment variables used by the stdlib and tests. Don't log the full
208    # environment: filter to list to not leak sensitive information.
209    #
210    # HTTP_PROXY is not logged because it can contain a password.
211    ENV_VARS = frozenset((
212        "APPDATA",
213        "AR",
214        "ARCHFLAGS",
215        "ARFLAGS",
216        "AUDIODEV",
217        "CC",
218        "CFLAGS",
219        "COLUMNS",
220        "COMPUTERNAME",
221        "COMSPEC",
222        "CPP",
223        "CPPFLAGS",
224        "DISPLAY",
225        "DISTUTILS_DEBUG",
226        "DISTUTILS_USE_SDK",
227        "DYLD_LIBRARY_PATH",
228        "ENSUREPIP_OPTIONS",
229        "HISTORY_FILE",
230        "HOME",
231        "HOMEDRIVE",
232        "HOMEPATH",
233        "IDLESTARTUP",
234        "LANG",
235        "LDFLAGS",
236        "LDSHARED",
237        "LD_LIBRARY_PATH",
238        "LINES",
239        "MACOSX_DEPLOYMENT_TARGET",
240        "MAILCAPS",
241        "MAKEFLAGS",
242        "MIXERDEV",
243        "MSSDK",
244        "PATH",
245        "PATHEXT",
246        "PIP_CONFIG_FILE",
247        "PLAT",
248        "POSIXLY_CORRECT",
249        "PY_SAX_PARSER",
250        "ProgramFiles",
251        "ProgramFiles(x86)",
252        "RUNNING_ON_VALGRIND",
253        "SDK_TOOLS_BIN",
254        "SERVER_SOFTWARE",
255        "SHELL",
256        "SOURCE_DATE_EPOCH",
257        "SYSTEMROOT",
258        "TEMP",
259        "TERM",
260        "TILE_LIBRARY",
261        "TIX_LIBRARY",
262        "TMP",
263        "TMPDIR",
264        "TRAVIS",
265        "TZ",
266        "USERPROFILE",
267        "VIRTUAL_ENV",
268        "WAYLAND_DISPLAY",
269        "WINDIR",
270        "_PYTHON_HOST_PLATFORM",
271        "_PYTHON_PROJECT_BASE",
272        "_PYTHON_SYSCONFIGDATA_NAME",
273        "__PYVENV_LAUNCHER__",
274    ))
275    for name, value in os.environ.items():
276        uname = name.upper()
277        if (uname in ENV_VARS
278           # Copy PYTHON* and LC_* variables
279           or uname.startswith(("PYTHON", "LC_"))
280           # Visual Studio: VS140COMNTOOLS
281           or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))):
282            info_add('os.environ[%s]' % name, value)
283
284    if hasattr(os, 'umask'):
285        mask = os.umask(0)
286        os.umask(mask)
287        info_add("os.umask", '%03o' % mask)
288
289    if hasattr(os, 'getrandom'):
290        # PEP 524: Check if system urandom is initialized
291        try:
292            try:
293                os.getrandom(1, os.GRND_NONBLOCK)
294                state = 'ready (initialized)'
295            except BlockingIOError as exc:
296                state = 'not seeded yet (%s)' % exc
297            info_add('os.getrandom', state)
298        except OSError as exc:
299            # Python was compiled on a more recent Linux version
300            # than the current Linux kernel: ignore OSError(ENOSYS)
301            if exc.errno != errno.ENOSYS:
302                raise
303
304
305def collect_readline(info_add):
306    try:
307        import readline
308    except ImportError:
309        return
310
311    def format_attr(attr, value):
312        if isinstance(value, int):
313            return "%#x" % value
314        else:
315            return value
316
317    attributes = (
318        "_READLINE_VERSION",
319        "_READLINE_RUNTIME_VERSION",
320        "_READLINE_LIBRARY_VERSION",
321    )
322    copy_attributes(info_add, readline, 'readline.%s', attributes,
323                    formatter=format_attr)
324
325    if not hasattr(readline, "_READLINE_LIBRARY_VERSION"):
326        # _READLINE_LIBRARY_VERSION has been added to CPython 3.7
327        doc = getattr(readline, '__doc__', '')
328        if 'libedit readline' in doc:
329            info_add('readline.library', 'libedit readline')
330        elif 'GNU readline' in doc:
331            info_add('readline.library', 'GNU readline')
332
333
334def collect_gdb(info_add):
335    import subprocess
336
337    try:
338        proc = subprocess.Popen(["gdb", "-nx", "--version"],
339                                stdout=subprocess.PIPE,
340                                stderr=subprocess.PIPE,
341                                universal_newlines=True)
342        version = proc.communicate()[0]
343    except OSError:
344        return
345
346    # Only keep the first line
347    version = version.splitlines()[0]
348    info_add('gdb_version', version)
349
350
351def collect_tkinter(info_add):
352    try:
353        import _tkinter
354    except ImportError:
355        pass
356    else:
357        attributes = ('TK_VERSION', 'TCL_VERSION')
358        copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes)
359
360    try:
361        import tkinter
362    except ImportError:
363        pass
364    else:
365        tcl = tkinter.Tcl()
366        patchlevel = tcl.call('info', 'patchlevel')
367        info_add('tkinter.info_patchlevel', patchlevel)
368
369
370def collect_time(info_add):
371    import time
372
373    info_add('time.time', time.time())
374
375    attributes = (
376        'altzone',
377        'daylight',
378        'timezone',
379        'tzname',
380    )
381    copy_attributes(info_add, time, 'time.%s', attributes)
382
383    if hasattr(time, 'get_clock_info'):
384        for clock in ('clock', 'monotonic', 'perf_counter',
385                      'process_time', 'thread_time', 'time'):
386            try:
387                # prevent DeprecatingWarning on get_clock_info('clock')
388                with warnings.catch_warnings(record=True):
389                    clock_info = time.get_clock_info(clock)
390            except ValueError:
391                # missing clock like time.thread_time()
392                pass
393            else:
394                info_add('time.get_clock_info(%s)' % clock, clock_info)
395
396
397def collect_datetime(info_add):
398    try:
399        import datetime
400    except ImportError:
401        return
402
403    info_add('datetime.datetime.now', datetime.datetime.now())
404
405
406def collect_sysconfig(info_add):
407    import sysconfig
408
409    for name in (
410        'ABIFLAGS',
411        'ANDROID_API_LEVEL',
412        'CC',
413        'CCSHARED',
414        'CFLAGS',
415        'CFLAGSFORSHARED',
416        'CONFIG_ARGS',
417        'HOST_GNU_TYPE',
418        'MACHDEP',
419        'MULTIARCH',
420        'OPT',
421        'PY_CFLAGS',
422        'PY_CFLAGS_NODIST',
423        'PY_CORE_LDFLAGS',
424        'PY_LDFLAGS',
425        'PY_LDFLAGS_NODIST',
426        'PY_STDMODULE_CFLAGS',
427        'Py_DEBUG',
428        'Py_ENABLE_SHARED',
429        'SHELL',
430        'SOABI',
431        'prefix',
432    ):
433        value = sysconfig.get_config_var(name)
434        if name == 'ANDROID_API_LEVEL' and not value:
435            # skip ANDROID_API_LEVEL=0
436            continue
437        value = normalize_text(value)
438        info_add('sysconfig[%s]' % name, value)
439
440
441def collect_ssl(info_add):
442    import os
443    try:
444        import ssl
445    except ImportError:
446        return
447    try:
448        import _ssl
449    except ImportError:
450        _ssl = None
451
452    def format_attr(attr, value):
453        if attr.startswith('OP_'):
454            return '%#8x' % value
455        else:
456            return value
457
458    attributes = (
459        'OPENSSL_VERSION',
460        'OPENSSL_VERSION_INFO',
461        'HAS_SNI',
462        'OP_ALL',
463        'OP_NO_TLSv1_1',
464    )
465    copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr)
466
467    options_names = []
468    protocol_names = {}
469    verify_modes = {}
470    for name in dir(ssl):
471        if name.startswith('OP_'):
472            options_names.append((name, getattr(ssl, name)))
473        elif name.startswith('PROTOCOL_'):
474            protocol_names[getattr(ssl, name)] = name
475        elif name.startswith('CERT_'):
476            verify_modes[getattr(ssl, name)] = name
477    options_names.sort(key=lambda item: item[1], reverse=True)
478
479    def formatter(attr_name, value):
480        if attr_name == 'options':
481            options_text = []
482            for opt_name, opt_value in options_names:
483                if value & opt_value:
484                    options_text.append(opt_name)
485                    value &= ~opt_value
486            if value:
487                options_text.append(str(value))
488            return '|' .join(options_text)
489        elif attr_name == 'verify_mode':
490            return verify_modes.get(value, value)
491        elif attr_name == 'protocol':
492            return protocol_names.get(value, value)
493        else:
494            return value
495
496    for name, ctx in (
497        ('SSLContext(PROTOCOL_TLS)', ssl.SSLContext(ssl.PROTOCOL_TLS)),
498        ('default_https_context', ssl._create_default_https_context()),
499        ('stdlib_context', ssl._create_stdlib_context()),
500    ):
501        attributes = (
502            'minimum_version',
503            'maximum_version',
504            'protocol',
505            'options',
506            'verify_mode',
507        )
508        copy_attributes(info_add, ctx, 'ssl.%s.%%s' % name, attributes, formatter=formatter)
509
510    env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"]
511    if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'):
512        parts = _ssl.get_default_verify_paths()
513        env_names.extend((parts[0], parts[2]))
514
515    for name in env_names:
516        try:
517            value = os.environ[name]
518        except KeyError:
519            continue
520        info_add('ssl.environ[%s]' % name, value)
521
522
523def collect_socket(info_add):
524    import socket
525
526    hostname = socket.gethostname()
527    info_add('socket.hostname', hostname)
528
529
530def collect_sqlite(info_add):
531    try:
532        import sqlite3
533    except ImportError:
534        return
535
536    attributes = ('version', 'sqlite_version')
537    copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes)
538
539
540def collect_zlib(info_add):
541    try:
542        import zlib
543    except ImportError:
544        return
545
546    attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION')
547    copy_attributes(info_add, zlib, 'zlib.%s', attributes)
548
549
550def collect_expat(info_add):
551    try:
552        from xml.parsers import expat
553    except ImportError:
554        return
555
556    attributes = ('EXPAT_VERSION',)
557    copy_attributes(info_add, expat, 'expat.%s', attributes)
558
559
560def collect_decimal(info_add):
561    try:
562        import _decimal
563    except ImportError:
564        return
565
566    attributes = ('__libmpdec_version__',)
567    copy_attributes(info_add, _decimal, '_decimal.%s', attributes)
568
569
570def collect_testcapi(info_add):
571    try:
572        import _testcapi
573    except ImportError:
574        return
575
576    call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname')
577    copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC')
578
579
580def collect_resource(info_add):
581    try:
582        import resource
583    except ImportError:
584        return
585
586    limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')]
587    for name in limits:
588        key = getattr(resource, name)
589        value = resource.getrlimit(key)
590        info_add('resource.%s' % name, value)
591
592    call_func(info_add, 'resource.pagesize', resource, 'getpagesize')
593
594
595def collect_test_socket(info_add):
596    try:
597        from test import test_socket
598    except ImportError:
599        return
600
601    # all check attributes like HAVE_SOCKET_CAN
602    attributes = [name for name in dir(test_socket)
603                  if name.startswith('HAVE_')]
604    copy_attributes(info_add, test_socket, 'test_socket.%s', attributes)
605
606
607def collect_test_support(info_add):
608    try:
609        from test import support
610    except ImportError:
611        return
612
613    attributes = ('IPV6_ENABLED',)
614    copy_attributes(info_add, support, 'test_support.%s', attributes)
615
616    call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available')
617    call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized')
618
619
620def collect_cc(info_add):
621    import subprocess
622    import sysconfig
623
624    CC = sysconfig.get_config_var('CC')
625    if not CC:
626        return
627
628    try:
629        import shlex
630        args = shlex.split(CC)
631    except ImportError:
632        args = CC.split()
633    args.append('--version')
634    try:
635        proc = subprocess.Popen(args,
636                                stdout=subprocess.PIPE,
637                                stderr=subprocess.STDOUT,
638                                universal_newlines=True)
639    except OSError:
640        # Cannot run the compiler, for example when Python has been
641        # cross-compiled and installed on the target platform where the
642        # compiler is missing.
643        return
644
645    stdout = proc.communicate()[0]
646    if proc.returncode:
647        # CC --version failed: ignore error
648        return
649
650    text = stdout.splitlines()[0]
651    text = normalize_text(text)
652    info_add('CC.version', text)
653
654
655def collect_gdbm(info_add):
656    try:
657        from _gdbm import _GDBM_VERSION
658    except ImportError:
659        return
660
661    info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION)))
662
663
664def collect_get_config(info_add):
665    # Dump global configuration variables, _PyCoreConfig
666    # and _PyMainInterpreterConfig
667    try:
668        from _testinternalcapi import get_configs
669    except ImportError:
670        return
671
672    all_configs = get_configs()
673    for config_type in sorted(all_configs):
674        config = all_configs[config_type]
675        for key in sorted(config):
676            info_add('%s[%s]' % (config_type, key), repr(config[key]))
677
678
679def collect_subprocess(info_add):
680    import subprocess
681    copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
682
683
684def collect_info(info):
685    error = False
686    info_add = info.add
687
688    for collect_func in (
689        # collect_os() should be the first, to check the getrandom() status
690        collect_os,
691
692        collect_builtins,
693        collect_gdb,
694        collect_locale,
695        collect_platform,
696        collect_readline,
697        collect_socket,
698        collect_sqlite,
699        collect_ssl,
700        collect_sys,
701        collect_sysconfig,
702        collect_time,
703        collect_datetime,
704        collect_tkinter,
705        collect_zlib,
706        collect_expat,
707        collect_decimal,
708        collect_testcapi,
709        collect_resource,
710        collect_cc,
711        collect_gdbm,
712        collect_get_config,
713        collect_subprocess,
714
715        # Collecting from tests should be last as they have side effects.
716        collect_test_socket,
717        collect_test_support,
718    ):
719        try:
720            collect_func(info_add)
721        except Exception as exc:
722            error = True
723            print("ERROR: %s() failed" % (collect_func.__name__),
724                  file=sys.stderr)
725            traceback.print_exc(file=sys.stderr)
726            print(file=sys.stderr)
727            sys.stderr.flush()
728
729    return error
730
731
732def dump_info(info, file=None):
733    title = "Python debug information"
734    print(title)
735    print("=" * len(title))
736    print()
737
738    infos = info.get_infos()
739    infos = sorted(infos.items())
740    for key, value in infos:
741        value = value.replace("\n", " ")
742        print("%s: %s" % (key, value))
743    print()
744
745
746def main():
747    info = PythonInfo()
748    error = collect_info(info)
749    dump_info(info)
750
751    if error:
752        print("Collection failed: exit with error", file=sys.stderr)
753        sys.exit(1)
754
755
756if __name__ == "__main__":
757    main()
758