1# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) 2from test import support 3from test.support import import_helper 4from test.support import os_helper 5import unittest 6 7from collections import namedtuple 8import contextlib 9import json 10import os 11import os.path 12import re 13import shutil 14import subprocess 15import sys 16import sysconfig 17import tempfile 18import textwrap 19 20 21MS_WINDOWS = (os.name == 'nt') 22MACOS = (sys.platform == 'darwin') 23 24PYMEM_ALLOCATOR_NOT_SET = 0 25PYMEM_ALLOCATOR_DEBUG = 2 26PYMEM_ALLOCATOR_MALLOC = 3 27 28# _PyCoreConfig_InitCompatConfig() 29API_COMPAT = 1 30# _PyCoreConfig_InitPythonConfig() 31API_PYTHON = 2 32# _PyCoreConfig_InitIsolatedConfig() 33API_ISOLATED = 3 34 35INIT_LOOPS = 16 36MAX_HASH_SEED = 4294967295 37 38 39# If we are running from a build dir, but the stdlib has been installed, 40# some tests need to expect different results. 41STDLIB_INSTALL = os.path.join(sys.prefix, sys.platlibdir, 42 f'python{sys.version_info.major}.{sys.version_info.minor}') 43if not os.path.isfile(os.path.join(STDLIB_INSTALL, 'os.py')): 44 STDLIB_INSTALL = None 45 46def debug_build(program): 47 program = os.path.basename(program) 48 name = os.path.splitext(program)[0] 49 return name.casefold().endswith("_d".casefold()) 50 51 52def remove_python_envvars(): 53 env = dict(os.environ) 54 # Remove PYTHON* environment variables to get deterministic environment 55 for key in list(env): 56 if key.startswith('PYTHON'): 57 del env[key] 58 return env 59 60 61class EmbeddingTestsMixin: 62 def setUp(self): 63 exename = "_testembed" 64 builddir = os.path.dirname(sys.executable) 65 if MS_WINDOWS: 66 ext = ("_d" if debug_build(sys.executable) else "") + ".exe" 67 exename += ext 68 exepath = builddir 69 expecteddir = os.path.join(support.REPO_ROOT, builddir) 70 else: 71 exepath = os.path.join(builddir, 'Programs') 72 expecteddir = os.path.join(support.REPO_ROOT, 'Programs') 73 self.test_exe = exe = os.path.join(exepath, exename) 74 if exepath != expecteddir or not os.path.exists(exe): 75 self.skipTest("%r doesn't exist" % exe) 76 # This is needed otherwise we get a fatal error: 77 # "Py_Initialize: Unable to get the locale encoding 78 # LookupError: no codec search functions registered: can't find encoding" 79 self.oldcwd = os.getcwd() 80 os.chdir(support.REPO_ROOT) 81 82 def tearDown(self): 83 os.chdir(self.oldcwd) 84 85 def run_embedded_interpreter(self, *args, env=None, 86 timeout=None, returncode=0, input=None, 87 cwd=None): 88 """Runs a test in the embedded interpreter""" 89 cmd = [self.test_exe] 90 cmd.extend(args) 91 if env is not None and MS_WINDOWS: 92 # Windows requires at least the SYSTEMROOT environment variable to 93 # start Python. 94 env = env.copy() 95 env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] 96 97 p = subprocess.Popen(cmd, 98 stdout=subprocess.PIPE, 99 stderr=subprocess.PIPE, 100 universal_newlines=True, 101 env=env, 102 cwd=cwd) 103 try: 104 (out, err) = p.communicate(input=input, timeout=timeout) 105 except: 106 p.terminate() 107 p.wait() 108 raise 109 if p.returncode != returncode and support.verbose: 110 print(f"--- {cmd} failed ---") 111 print(f"stdout:\n{out}") 112 print(f"stderr:\n{err}") 113 print(f"------") 114 115 self.assertEqual(p.returncode, returncode, 116 "bad returncode %d, stderr is %r" % 117 (p.returncode, err)) 118 return out, err 119 120 def run_repeated_init_and_subinterpreters(self): 121 out, err = self.run_embedded_interpreter("test_repeated_init_and_subinterpreters") 122 self.assertEqual(err, "") 123 124 # The output from _testembed looks like this: 125 # --- Pass 1 --- 126 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 127 # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 128 # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 129 # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 130 # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 131 # --- Pass 2 --- 132 # ... 133 134 interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " 135 r"thread state <(0x[\dA-F]+)>: " 136 r"id\(modules\) = ([\d]+)$") 137 Interp = namedtuple("Interp", "id interp tstate modules") 138 139 numloops = 1 140 current_run = [] 141 for line in out.splitlines(): 142 if line == "--- Pass {} ---".format(numloops): 143 self.assertEqual(len(current_run), 0) 144 if support.verbose > 1: 145 print(line) 146 numloops += 1 147 continue 148 149 self.assertLess(len(current_run), 5) 150 match = re.match(interp_pat, line) 151 if match is None: 152 self.assertRegex(line, interp_pat) 153 154 # Parse the line from the loop. The first line is the main 155 # interpreter and the 3 afterward are subinterpreters. 156 interp = Interp(*match.groups()) 157 if support.verbose > 1: 158 print(interp) 159 self.assertTrue(interp.interp) 160 self.assertTrue(interp.tstate) 161 self.assertTrue(interp.modules) 162 current_run.append(interp) 163 164 # The last line in the loop should be the same as the first. 165 if len(current_run) == 5: 166 main = current_run[0] 167 self.assertEqual(interp, main) 168 yield current_run 169 current_run = [] 170 171 172class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): 173 maxDiff = 100 * 50 174 175 def test_subinterps_main(self): 176 for run in self.run_repeated_init_and_subinterpreters(): 177 main = run[0] 178 179 self.assertEqual(main.id, '0') 180 181 def test_subinterps_different_ids(self): 182 for run in self.run_repeated_init_and_subinterpreters(): 183 main, *subs, _ = run 184 185 mainid = int(main.id) 186 for i, sub in enumerate(subs): 187 self.assertEqual(sub.id, str(mainid + i + 1)) 188 189 def test_subinterps_distinct_state(self): 190 for run in self.run_repeated_init_and_subinterpreters(): 191 main, *subs, _ = run 192 193 if '0x0' in main: 194 # XXX Fix on Windows (and other platforms): something 195 # is going on with the pointers in Programs/_testembed.c. 196 # interp.interp is 0x0 and interp.modules is the same 197 # between interpreters. 198 raise unittest.SkipTest('platform prints pointers as 0x0') 199 200 for sub in subs: 201 # A new subinterpreter may have the same 202 # PyInterpreterState pointer as a previous one if 203 # the earlier one has already been destroyed. So 204 # we compare with the main interpreter. The same 205 # applies to tstate. 206 self.assertNotEqual(sub.interp, main.interp) 207 self.assertNotEqual(sub.tstate, main.tstate) 208 self.assertNotEqual(sub.modules, main.modules) 209 210 def test_repeated_init_and_inittab(self): 211 out, err = self.run_embedded_interpreter("test_repeated_init_and_inittab") 212 self.assertEqual(err, "") 213 214 lines = [f"--- Pass {i} ---" for i in range(1, INIT_LOOPS+1)] 215 lines = "\n".join(lines) + "\n" 216 self.assertEqual(out, lines) 217 218 def test_forced_io_encoding(self): 219 # Checks forced configuration of embedded interpreter IO streams 220 env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") 221 out, err = self.run_embedded_interpreter("test_forced_io_encoding", env=env) 222 if support.verbose > 1: 223 print() 224 print(out) 225 print(err) 226 expected_stream_encoding = "utf-8" 227 expected_errors = "surrogateescape" 228 expected_output = '\n'.join([ 229 "--- Use defaults ---", 230 "Expected encoding: default", 231 "Expected errors: default", 232 "stdin: {in_encoding}:{errors}", 233 "stdout: {out_encoding}:{errors}", 234 "stderr: {out_encoding}:backslashreplace", 235 "--- Set errors only ---", 236 "Expected encoding: default", 237 "Expected errors: ignore", 238 "stdin: {in_encoding}:ignore", 239 "stdout: {out_encoding}:ignore", 240 "stderr: {out_encoding}:backslashreplace", 241 "--- Set encoding only ---", 242 "Expected encoding: iso8859-1", 243 "Expected errors: default", 244 "stdin: iso8859-1:{errors}", 245 "stdout: iso8859-1:{errors}", 246 "stderr: iso8859-1:backslashreplace", 247 "--- Set encoding and errors ---", 248 "Expected encoding: iso8859-1", 249 "Expected errors: replace", 250 "stdin: iso8859-1:replace", 251 "stdout: iso8859-1:replace", 252 "stderr: iso8859-1:backslashreplace"]) 253 expected_output = expected_output.format( 254 in_encoding=expected_stream_encoding, 255 out_encoding=expected_stream_encoding, 256 errors=expected_errors) 257 # This is useful if we ever trip over odd platform behaviour 258 self.maxDiff = None 259 self.assertEqual(out.strip(), expected_output) 260 261 def test_pre_initialization_api(self): 262 """ 263 Checks some key parts of the C-API that need to work before the runtime 264 is initialized (via Py_Initialize()). 265 """ 266 env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) 267 out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env) 268 if MS_WINDOWS: 269 expected_path = self.test_exe 270 else: 271 expected_path = os.path.join(os.getcwd(), "spam") 272 expected_output = f"sys.executable: {expected_path}\n" 273 self.assertIn(expected_output, out) 274 self.assertEqual(err, '') 275 276 def test_pre_initialization_sys_options(self): 277 """ 278 Checks that sys.warnoptions and sys._xoptions can be set before the 279 runtime is initialized (otherwise they won't be effective). 280 """ 281 env = remove_python_envvars() 282 env['PYTHONPATH'] = os.pathsep.join(sys.path) 283 out, err = self.run_embedded_interpreter( 284 "test_pre_initialization_sys_options", env=env) 285 expected_output = ( 286 "sys.warnoptions: ['once', 'module', 'default']\n" 287 "sys._xoptions: {'dev': '2', 'utf8': '1'}\n" 288 "warnings.filters[:3]: ['default', 'module', 'once']\n" 289 ) 290 self.assertIn(expected_output, out) 291 self.assertEqual(err, '') 292 293 def test_bpo20891(self): 294 """ 295 bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not 296 crash. 297 """ 298 out, err = self.run_embedded_interpreter("test_bpo20891") 299 self.assertEqual(out, '') 300 self.assertEqual(err, '') 301 302 def test_initialize_twice(self): 303 """ 304 bpo-33932: Calling Py_Initialize() twice should do nothing (and not 305 crash!). 306 """ 307 out, err = self.run_embedded_interpreter("test_initialize_twice") 308 self.assertEqual(out, '') 309 self.assertEqual(err, '') 310 311 def test_initialize_pymain(self): 312 """ 313 bpo-34008: Calling Py_Main() after Py_Initialize() must not fail. 314 """ 315 out, err = self.run_embedded_interpreter("test_initialize_pymain") 316 self.assertEqual(out.rstrip(), "Py_Main() after Py_Initialize: sys.argv=['-c', 'arg2']") 317 self.assertEqual(err, '') 318 319 def test_run_main(self): 320 out, err = self.run_embedded_interpreter("test_run_main") 321 self.assertEqual(out.rstrip(), "Py_RunMain(): sys.argv=['-c', 'arg2']") 322 self.assertEqual(err, '') 323 324 def test_run_main_loop(self): 325 # bpo-40413: Calling Py_InitializeFromConfig()+Py_RunMain() multiple 326 # times must not crash. 327 nloop = 5 328 out, err = self.run_embedded_interpreter("test_run_main_loop") 329 self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop) 330 self.assertEqual(err, '') 331 332 333class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 334 maxDiff = 4096 335 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') 336 337 # Marker to read the default configuration: get_default_config() 338 GET_DEFAULT_CONFIG = object() 339 340 # Marker to ignore a configuration parameter 341 IGNORE_CONFIG = object() 342 343 PRE_CONFIG_COMPAT = { 344 '_config_init': API_COMPAT, 345 'allocator': PYMEM_ALLOCATOR_NOT_SET, 346 'parse_argv': 0, 347 'configure_locale': 1, 348 'coerce_c_locale': 0, 349 'coerce_c_locale_warn': 0, 350 'utf8_mode': 0, 351 } 352 if MS_WINDOWS: 353 PRE_CONFIG_COMPAT.update({ 354 'legacy_windows_fs_encoding': 0, 355 }) 356 PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT, 357 _config_init=API_PYTHON, 358 parse_argv=1, 359 coerce_c_locale=GET_DEFAULT_CONFIG, 360 utf8_mode=GET_DEFAULT_CONFIG, 361 ) 362 PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT, 363 _config_init=API_ISOLATED, 364 configure_locale=0, 365 isolated=1, 366 use_environment=0, 367 utf8_mode=0, 368 dev_mode=0, 369 coerce_c_locale=0, 370 ) 371 372 COPY_PRE_CONFIG = [ 373 'dev_mode', 374 'isolated', 375 'use_environment', 376 ] 377 378 CONFIG_COMPAT = { 379 '_config_init': API_COMPAT, 380 'isolated': 0, 381 'use_environment': 1, 382 'dev_mode': 0, 383 384 'install_signal_handlers': 1, 385 'use_hash_seed': 0, 386 'hash_seed': 0, 387 'faulthandler': 0, 388 'tracemalloc': 0, 389 'import_time': 0, 390 'code_debug_ranges': 1, 391 'show_ref_count': 0, 392 'dump_refs': 0, 393 'malloc_stats': 0, 394 395 'filesystem_encoding': GET_DEFAULT_CONFIG, 396 'filesystem_errors': GET_DEFAULT_CONFIG, 397 398 'pycache_prefix': None, 399 'program_name': GET_DEFAULT_CONFIG, 400 'parse_argv': 0, 401 'argv': [""], 402 'orig_argv': [], 403 404 'xoptions': [], 405 'warnoptions': [], 406 407 'pythonpath_env': None, 408 'home': None, 409 'executable': GET_DEFAULT_CONFIG, 410 'base_executable': GET_DEFAULT_CONFIG, 411 412 'prefix': GET_DEFAULT_CONFIG, 413 'base_prefix': GET_DEFAULT_CONFIG, 414 'exec_prefix': GET_DEFAULT_CONFIG, 415 'base_exec_prefix': GET_DEFAULT_CONFIG, 416 'module_search_paths': GET_DEFAULT_CONFIG, 417 'module_search_paths_set': 1, 418 'platlibdir': sys.platlibdir, 419 'stdlib_dir': GET_DEFAULT_CONFIG, 420 421 'site_import': 1, 422 'bytes_warning': 0, 423 'warn_default_encoding': 0, 424 'inspect': 0, 425 'interactive': 0, 426 'optimization_level': 0, 427 'parser_debug': 0, 428 'write_bytecode': 1, 429 'verbose': 0, 430 'quiet': 0, 431 'user_site_directory': 1, 432 'configure_c_stdio': 0, 433 'buffered_stdio': 1, 434 435 'stdio_encoding': GET_DEFAULT_CONFIG, 436 'stdio_errors': GET_DEFAULT_CONFIG, 437 438 'skip_source_first_line': 0, 439 'run_command': None, 440 'run_module': None, 441 'run_filename': None, 442 443 '_install_importlib': 1, 444 'check_hash_pycs_mode': 'default', 445 'pathconfig_warnings': 1, 446 '_init_main': 1, 447 '_isolated_interpreter': 0, 448 'use_frozen_modules': 1, 449 '_is_python_build': IGNORE_CONFIG, 450 } 451 if MS_WINDOWS: 452 CONFIG_COMPAT.update({ 453 'legacy_windows_stdio': 0, 454 }) 455 456 CONFIG_PYTHON = dict(CONFIG_COMPAT, 457 _config_init=API_PYTHON, 458 configure_c_stdio=1, 459 parse_argv=2, 460 ) 461 CONFIG_ISOLATED = dict(CONFIG_COMPAT, 462 _config_init=API_ISOLATED, 463 isolated=1, 464 use_environment=0, 465 user_site_directory=0, 466 dev_mode=0, 467 install_signal_handlers=0, 468 use_hash_seed=0, 469 faulthandler=0, 470 tracemalloc=0, 471 pathconfig_warnings=0, 472 ) 473 if MS_WINDOWS: 474 CONFIG_ISOLATED['legacy_windows_stdio'] = 0 475 476 # global config 477 DEFAULT_GLOBAL_CONFIG = { 478 'Py_HasFileSystemDefaultEncoding': 0, 479 'Py_HashRandomizationFlag': 1, 480 '_Py_HasFileSystemDefaultEncodeErrors': 0, 481 } 482 COPY_GLOBAL_PRE_CONFIG = [ 483 ('Py_UTF8Mode', 'utf8_mode'), 484 ] 485 COPY_GLOBAL_CONFIG = [ 486 # Copy core config to global config for expected values 487 # True means that the core config value is inverted (0 => 1 and 1 => 0) 488 ('Py_BytesWarningFlag', 'bytes_warning'), 489 ('Py_DebugFlag', 'parser_debug'), 490 ('Py_DontWriteBytecodeFlag', 'write_bytecode', True), 491 ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), 492 ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), 493 ('Py_FrozenFlag', 'pathconfig_warnings', True), 494 ('Py_IgnoreEnvironmentFlag', 'use_environment', True), 495 ('Py_InspectFlag', 'inspect'), 496 ('Py_InteractiveFlag', 'interactive'), 497 ('Py_IsolatedFlag', 'isolated'), 498 ('Py_NoSiteFlag', 'site_import', True), 499 ('Py_NoUserSiteDirectory', 'user_site_directory', True), 500 ('Py_OptimizeFlag', 'optimization_level'), 501 ('Py_QuietFlag', 'quiet'), 502 ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), 503 ('Py_VerboseFlag', 'verbose'), 504 ] 505 if MS_WINDOWS: 506 COPY_GLOBAL_PRE_CONFIG.extend(( 507 ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), 508 )) 509 COPY_GLOBAL_CONFIG.extend(( 510 ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), 511 )) 512 513 EXPECTED_CONFIG = None 514 515 @classmethod 516 def tearDownClass(cls): 517 # clear cache 518 cls.EXPECTED_CONFIG = None 519 520 def main_xoptions(self, xoptions_list): 521 xoptions = {} 522 for opt in xoptions_list: 523 if '=' in opt: 524 key, value = opt.split('=', 1) 525 xoptions[key] = value 526 else: 527 xoptions[opt] = True 528 return xoptions 529 530 def _get_expected_config_impl(self): 531 env = remove_python_envvars() 532 code = textwrap.dedent(''' 533 import json 534 import sys 535 import _testinternalcapi 536 537 configs = _testinternalcapi.get_configs() 538 539 data = json.dumps(configs) 540 data = data.encode('utf-8') 541 sys.stdout.buffer.write(data) 542 sys.stdout.buffer.flush() 543 ''') 544 545 # Use -S to not import the site module: get the proper configuration 546 # when test_embed is run from a venv (bpo-35313) 547 args = [sys.executable, '-S', '-c', code] 548 proc = subprocess.run(args, env=env, 549 stdout=subprocess.PIPE, 550 stderr=subprocess.PIPE) 551 if proc.returncode: 552 raise Exception(f"failed to get the default config: " 553 f"stdout={proc.stdout!r} stderr={proc.stderr!r}") 554 stdout = proc.stdout.decode('utf-8') 555 # ignore stderr 556 try: 557 return json.loads(stdout) 558 except json.JSONDecodeError: 559 self.fail(f"fail to decode stdout: {stdout!r}") 560 561 def _get_expected_config(self): 562 cls = InitConfigTests 563 if cls.EXPECTED_CONFIG is None: 564 cls.EXPECTED_CONFIG = self._get_expected_config_impl() 565 566 # get a copy 567 configs = {} 568 for config_key, config_value in cls.EXPECTED_CONFIG.items(): 569 config = {} 570 for key, value in config_value.items(): 571 if isinstance(value, list): 572 value = value.copy() 573 config[key] = value 574 configs[config_key] = config 575 return configs 576 577 def get_expected_config(self, expected_preconfig, expected, 578 env, api, modify_path_cb=None, cwd=None): 579 configs = self._get_expected_config() 580 581 pre_config = configs['pre_config'] 582 for key, value in expected_preconfig.items(): 583 if value is self.GET_DEFAULT_CONFIG: 584 expected_preconfig[key] = pre_config[key] 585 586 if not expected_preconfig['configure_locale'] or api == API_COMPAT: 587 # there is no easy way to get the locale encoding before 588 # setlocale(LC_CTYPE, "") is called: don't test encodings 589 for key in ('filesystem_encoding', 'filesystem_errors', 590 'stdio_encoding', 'stdio_errors'): 591 expected[key] = self.IGNORE_CONFIG 592 593 if not expected_preconfig['configure_locale']: 594 # UTF-8 Mode depends on the locale. There is no easy way 595 # to guess if UTF-8 Mode will be enabled or not if the locale 596 # is not configured. 597 expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG 598 599 if expected_preconfig['utf8_mode'] == 1: 600 if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG: 601 expected['filesystem_encoding'] = 'utf-8' 602 if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG: 603 expected['filesystem_errors'] = self.UTF8_MODE_ERRORS 604 if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG: 605 expected['stdio_encoding'] = 'utf-8' 606 if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: 607 expected['stdio_errors'] = 'surrogateescape' 608 609 if MS_WINDOWS: 610 default_executable = self.test_exe 611 elif expected['program_name'] is not self.GET_DEFAULT_CONFIG: 612 default_executable = os.path.abspath(expected['program_name']) 613 else: 614 default_executable = os.path.join(os.getcwd(), '_testembed') 615 if expected['executable'] is self.GET_DEFAULT_CONFIG: 616 expected['executable'] = default_executable 617 if expected['base_executable'] is self.GET_DEFAULT_CONFIG: 618 expected['base_executable'] = default_executable 619 if expected['program_name'] is self.GET_DEFAULT_CONFIG: 620 expected['program_name'] = './_testembed' 621 if MS_WINDOWS: 622 # follow the calculation in getpath.py 623 tmpname = expected['program_name'] + '.exe' 624 if cwd: 625 tmpname = os.path.join(cwd, tmpname) 626 if os.path.isfile(tmpname): 627 expected['program_name'] += '.exe' 628 del tmpname 629 630 config = configs['config'] 631 for key, value in expected.items(): 632 if value is self.GET_DEFAULT_CONFIG: 633 expected[key] = config[key] 634 635 if expected['module_search_paths'] is not self.IGNORE_CONFIG: 636 pythonpath_env = expected['pythonpath_env'] 637 if pythonpath_env is not None: 638 paths = pythonpath_env.split(os.path.pathsep) 639 expected['module_search_paths'] = [*paths, *expected['module_search_paths']] 640 if modify_path_cb is not None: 641 expected['module_search_paths'] = expected['module_search_paths'].copy() 642 modify_path_cb(expected['module_search_paths']) 643 644 for key in self.COPY_PRE_CONFIG: 645 if key not in expected_preconfig: 646 expected_preconfig[key] = expected[key] 647 648 def check_pre_config(self, configs, expected): 649 pre_config = dict(configs['pre_config']) 650 for key, value in list(expected.items()): 651 if value is self.IGNORE_CONFIG: 652 pre_config.pop(key, None) 653 del expected[key] 654 self.assertEqual(pre_config, expected) 655 656 def check_config(self, configs, expected): 657 config = dict(configs['config']) 658 for key, value in list(expected.items()): 659 if value is self.IGNORE_CONFIG: 660 config.pop(key, None) 661 del expected[key] 662 self.assertEqual(config, expected) 663 664 def check_global_config(self, configs): 665 pre_config = configs['pre_config'] 666 config = configs['config'] 667 668 expected = dict(self.DEFAULT_GLOBAL_CONFIG) 669 for item in self.COPY_GLOBAL_CONFIG: 670 if len(item) == 3: 671 global_key, core_key, opposite = item 672 expected[global_key] = 0 if config[core_key] else 1 673 else: 674 global_key, core_key = item 675 expected[global_key] = config[core_key] 676 for item in self.COPY_GLOBAL_PRE_CONFIG: 677 if len(item) == 3: 678 global_key, core_key, opposite = item 679 expected[global_key] = 0 if pre_config[core_key] else 1 680 else: 681 global_key, core_key = item 682 expected[global_key] = pre_config[core_key] 683 684 self.assertEqual(configs['global_config'], expected) 685 686 def check_all_configs(self, testname, expected_config=None, 687 expected_preconfig=None, 688 modify_path_cb=None, 689 stderr=None, *, api, preconfig_api=None, 690 env=None, ignore_stderr=False, cwd=None): 691 new_env = remove_python_envvars() 692 if env is not None: 693 new_env.update(env) 694 env = new_env 695 696 if preconfig_api is None: 697 preconfig_api = api 698 if preconfig_api == API_ISOLATED: 699 default_preconfig = self.PRE_CONFIG_ISOLATED 700 elif preconfig_api == API_PYTHON: 701 default_preconfig = self.PRE_CONFIG_PYTHON 702 else: 703 default_preconfig = self.PRE_CONFIG_COMPAT 704 if expected_preconfig is None: 705 expected_preconfig = {} 706 expected_preconfig = dict(default_preconfig, **expected_preconfig) 707 708 if expected_config is None: 709 expected_config = {} 710 711 if api == API_PYTHON: 712 default_config = self.CONFIG_PYTHON 713 elif api == API_ISOLATED: 714 default_config = self.CONFIG_ISOLATED 715 else: 716 default_config = self.CONFIG_COMPAT 717 expected_config = dict(default_config, **expected_config) 718 719 self.get_expected_config(expected_preconfig, 720 expected_config, 721 env, 722 api, modify_path_cb, cwd) 723 724 out, err = self.run_embedded_interpreter(testname, 725 env=env, cwd=cwd) 726 if stderr is None and not expected_config['verbose']: 727 stderr = "" 728 if stderr is not None and not ignore_stderr: 729 self.assertEqual(err.rstrip(), stderr) 730 try: 731 configs = json.loads(out) 732 except json.JSONDecodeError: 733 self.fail(f"fail to decode stdout: {out!r}") 734 735 self.check_pre_config(configs, expected_preconfig) 736 self.check_config(configs, expected_config) 737 self.check_global_config(configs) 738 return configs 739 740 def test_init_default_config(self): 741 self.check_all_configs("test_init_initialize_config", api=API_COMPAT) 742 743 def test_preinit_compat_config(self): 744 self.check_all_configs("test_preinit_compat_config", api=API_COMPAT) 745 746 def test_init_compat_config(self): 747 self.check_all_configs("test_init_compat_config", api=API_COMPAT) 748 749 def test_init_global_config(self): 750 preconfig = { 751 'utf8_mode': 1, 752 } 753 config = { 754 'program_name': './globalvar', 755 'site_import': 0, 756 'bytes_warning': 1, 757 'warnoptions': ['default::BytesWarning'], 758 'inspect': 1, 759 'interactive': 1, 760 'optimization_level': 2, 761 'write_bytecode': 0, 762 'verbose': 1, 763 'quiet': 1, 764 'buffered_stdio': 0, 765 766 'user_site_directory': 0, 767 'pathconfig_warnings': 0, 768 } 769 self.check_all_configs("test_init_global_config", config, preconfig, 770 api=API_COMPAT) 771 772 def test_init_from_config(self): 773 preconfig = { 774 'allocator': PYMEM_ALLOCATOR_MALLOC, 775 'utf8_mode': 1, 776 } 777 config = { 778 'install_signal_handlers': 0, 779 'use_hash_seed': 1, 780 'hash_seed': 123, 781 'tracemalloc': 2, 782 'import_time': 1, 783 'code_debug_ranges': 0, 784 'show_ref_count': 1, 785 'malloc_stats': 1, 786 787 'stdio_encoding': 'iso8859-1', 788 'stdio_errors': 'replace', 789 790 'pycache_prefix': 'conf_pycache_prefix', 791 'program_name': './conf_program_name', 792 'argv': ['-c', 'arg2'], 793 'orig_argv': ['python3', 794 '-W', 'cmdline_warnoption', 795 '-X', 'dev', 796 '-c', 'pass', 797 'arg2'], 798 'parse_argv': 2, 799 'xoptions': [ 800 'dev=3', 801 'utf8', 802 'dev', 803 ], 804 'warnoptions': [ 805 'cmdline_warnoption', 806 'default::BytesWarning', 807 'config_warnoption', 808 ], 809 'run_command': 'pass\n', 810 811 'site_import': 0, 812 'bytes_warning': 1, 813 'inspect': 1, 814 'interactive': 1, 815 'optimization_level': 2, 816 'write_bytecode': 0, 817 'verbose': 1, 818 'quiet': 1, 819 'configure_c_stdio': 1, 820 'buffered_stdio': 0, 821 'user_site_directory': 0, 822 'faulthandler': 1, 823 'platlibdir': 'my_platlibdir', 824 'module_search_paths': self.IGNORE_CONFIG, 825 826 'check_hash_pycs_mode': 'always', 827 'pathconfig_warnings': 0, 828 829 '_isolated_interpreter': 1, 830 } 831 self.check_all_configs("test_init_from_config", config, preconfig, 832 api=API_COMPAT) 833 834 def test_init_compat_env(self): 835 preconfig = { 836 'allocator': PYMEM_ALLOCATOR_MALLOC, 837 } 838 config = { 839 'use_hash_seed': 1, 840 'hash_seed': 42, 841 'tracemalloc': 2, 842 'import_time': 1, 843 'code_debug_ranges': 0, 844 'malloc_stats': 1, 845 'inspect': 1, 846 'optimization_level': 2, 847 'pythonpath_env': '/my/path', 848 'pycache_prefix': 'env_pycache_prefix', 849 'write_bytecode': 0, 850 'verbose': 1, 851 'buffered_stdio': 0, 852 'stdio_encoding': 'iso8859-1', 853 'stdio_errors': 'replace', 854 'user_site_directory': 0, 855 'faulthandler': 1, 856 'warnoptions': ['EnvVar'], 857 'platlibdir': 'env_platlibdir', 858 'module_search_paths': self.IGNORE_CONFIG, 859 } 860 self.check_all_configs("test_init_compat_env", config, preconfig, 861 api=API_COMPAT) 862 863 def test_init_python_env(self): 864 preconfig = { 865 'allocator': PYMEM_ALLOCATOR_MALLOC, 866 'utf8_mode': 1, 867 } 868 config = { 869 'use_hash_seed': 1, 870 'hash_seed': 42, 871 'tracemalloc': 2, 872 'import_time': 1, 873 'code_debug_ranges': 0, 874 'malloc_stats': 1, 875 'inspect': 1, 876 'optimization_level': 2, 877 'pythonpath_env': '/my/path', 878 'pycache_prefix': 'env_pycache_prefix', 879 'write_bytecode': 0, 880 'verbose': 1, 881 'buffered_stdio': 0, 882 'stdio_encoding': 'iso8859-1', 883 'stdio_errors': 'replace', 884 'user_site_directory': 0, 885 'faulthandler': 1, 886 'warnoptions': ['EnvVar'], 887 'platlibdir': 'env_platlibdir', 888 'module_search_paths': self.IGNORE_CONFIG, 889 } 890 self.check_all_configs("test_init_python_env", config, preconfig, 891 api=API_PYTHON) 892 893 def test_init_env_dev_mode(self): 894 preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) 895 config = dict(dev_mode=1, 896 faulthandler=1, 897 warnoptions=['default']) 898 self.check_all_configs("test_init_env_dev_mode", config, preconfig, 899 api=API_COMPAT) 900 901 def test_init_env_dev_mode_alloc(self): 902 preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC) 903 config = dict(dev_mode=1, 904 faulthandler=1, 905 warnoptions=['default']) 906 self.check_all_configs("test_init_env_dev_mode_alloc", config, preconfig, 907 api=API_COMPAT) 908 909 def test_init_dev_mode(self): 910 preconfig = { 911 'allocator': PYMEM_ALLOCATOR_DEBUG, 912 } 913 config = { 914 'faulthandler': 1, 915 'dev_mode': 1, 916 'warnoptions': ['default'], 917 } 918 self.check_all_configs("test_init_dev_mode", config, preconfig, 919 api=API_PYTHON) 920 921 def test_preinit_parse_argv(self): 922 # Pre-initialize implicitly using argv: make sure that -X dev 923 # is used to configure the allocation in preinitialization 924 preconfig = { 925 'allocator': PYMEM_ALLOCATOR_DEBUG, 926 } 927 config = { 928 'argv': ['script.py'], 929 'orig_argv': ['python3', '-X', 'dev', 'script.py'], 930 'run_filename': os.path.abspath('script.py'), 931 'dev_mode': 1, 932 'faulthandler': 1, 933 'warnoptions': ['default'], 934 'xoptions': ['dev'], 935 } 936 self.check_all_configs("test_preinit_parse_argv", config, preconfig, 937 api=API_PYTHON) 938 939 def test_preinit_dont_parse_argv(self): 940 # -X dev must be ignored by isolated preconfiguration 941 preconfig = { 942 'isolated': 0, 943 } 944 argv = ["python3", 945 "-E", "-I", 946 "-X", "dev", 947 "-X", "utf8", 948 "script.py"] 949 config = { 950 'argv': argv, 951 'orig_argv': argv, 952 'isolated': 0, 953 } 954 self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig, 955 api=API_ISOLATED) 956 957 def test_init_isolated_flag(self): 958 config = { 959 'isolated': 1, 960 'use_environment': 0, 961 'user_site_directory': 0, 962 } 963 self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON) 964 965 def test_preinit_isolated1(self): 966 # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set 967 config = { 968 'isolated': 1, 969 'use_environment': 0, 970 'user_site_directory': 0, 971 } 972 self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT) 973 974 def test_preinit_isolated2(self): 975 # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1 976 config = { 977 'isolated': 1, 978 'use_environment': 0, 979 'user_site_directory': 0, 980 } 981 self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT) 982 983 def test_preinit_isolated_config(self): 984 self.check_all_configs("test_preinit_isolated_config", api=API_ISOLATED) 985 986 def test_init_isolated_config(self): 987 self.check_all_configs("test_init_isolated_config", api=API_ISOLATED) 988 989 def test_preinit_python_config(self): 990 self.check_all_configs("test_preinit_python_config", api=API_PYTHON) 991 992 def test_init_python_config(self): 993 self.check_all_configs("test_init_python_config", api=API_PYTHON) 994 995 def test_init_dont_configure_locale(self): 996 # _PyPreConfig.configure_locale=0 997 preconfig = { 998 'configure_locale': 0, 999 'coerce_c_locale': 0, 1000 } 1001 self.check_all_configs("test_init_dont_configure_locale", {}, preconfig, 1002 api=API_PYTHON) 1003 1004 @unittest.skip('as of 3.11 this test no longer works because ' 1005 'path calculations do not occur on read') 1006 def test_init_read_set(self): 1007 config = { 1008 'program_name': './init_read_set', 1009 'executable': 'my_executable', 1010 'base_executable': 'my_executable', 1011 } 1012 def modify_path(path): 1013 path.insert(1, "test_path_insert1") 1014 path.append("test_path_append") 1015 self.check_all_configs("test_init_read_set", config, 1016 api=API_PYTHON, 1017 modify_path_cb=modify_path) 1018 1019 def test_init_sys_add(self): 1020 config = { 1021 'faulthandler': 1, 1022 'xoptions': [ 1023 'dev', 1024 'utf8', 1025 'faulthandler', 1026 ], 1027 'warnoptions': [ 1028 'ignore:::cmdline_warnoption', 1029 'ignore:::sysadd_warnoption', 1030 'ignore:::config_warnoption', 1031 ], 1032 'orig_argv': ['python3', 1033 '-W', 'ignore:::cmdline_warnoption', 1034 '-X', 'utf8'], 1035 } 1036 preconfig = {'utf8_mode': 1} 1037 self.check_all_configs("test_init_sys_add", config, 1038 expected_preconfig=preconfig, 1039 api=API_PYTHON) 1040 1041 def test_init_run_main(self): 1042 code = ('import _testinternalcapi, json; ' 1043 'print(json.dumps(_testinternalcapi.get_configs()))') 1044 config = { 1045 'argv': ['-c', 'arg2'], 1046 'orig_argv': ['python3', '-c', code, 'arg2'], 1047 'program_name': './python3', 1048 'run_command': code + '\n', 1049 'parse_argv': 2, 1050 } 1051 self.check_all_configs("test_init_run_main", config, api=API_PYTHON) 1052 1053 def test_init_main(self): 1054 code = ('import _testinternalcapi, json; ' 1055 'print(json.dumps(_testinternalcapi.get_configs()))') 1056 config = { 1057 'argv': ['-c', 'arg2'], 1058 'orig_argv': ['python3', 1059 '-c', code, 1060 'arg2'], 1061 'program_name': './python3', 1062 'run_command': code + '\n', 1063 'parse_argv': 2, 1064 '_init_main': 0, 1065 } 1066 self.check_all_configs("test_init_main", config, 1067 api=API_PYTHON, 1068 stderr="Run Python code before _Py_InitializeMain") 1069 1070 def test_init_parse_argv(self): 1071 config = { 1072 'parse_argv': 2, 1073 'argv': ['-c', 'arg1', '-v', 'arg3'], 1074 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 1075 'program_name': './argv0', 1076 'run_command': 'pass\n', 1077 'use_environment': 0, 1078 } 1079 self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON) 1080 1081 def test_init_dont_parse_argv(self): 1082 pre_config = { 1083 'parse_argv': 0, 1084 } 1085 config = { 1086 'parse_argv': 0, 1087 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 1088 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 1089 'program_name': './argv0', 1090 } 1091 self.check_all_configs("test_init_dont_parse_argv", config, pre_config, 1092 api=API_PYTHON) 1093 1094 def default_program_name(self, config): 1095 if MS_WINDOWS: 1096 program_name = 'python' 1097 executable = self.test_exe 1098 else: 1099 program_name = 'python3' 1100 if MACOS: 1101 executable = self.test_exe 1102 else: 1103 executable = shutil.which(program_name) or '' 1104 config.update({ 1105 'program_name': program_name, 1106 'base_executable': executable, 1107 'executable': executable, 1108 }) 1109 1110 def test_init_setpath(self): 1111 # Test Py_SetPath() 1112 config = self._get_expected_config() 1113 paths = config['config']['module_search_paths'] 1114 1115 config = { 1116 'module_search_paths': paths, 1117 'prefix': '', 1118 'base_prefix': '', 1119 'exec_prefix': '', 1120 'base_exec_prefix': '', 1121 # The current getpath.c doesn't determine the stdlib dir 1122 # in this case. 1123 'stdlib_dir': '', 1124 } 1125 self.default_program_name(config) 1126 env = {'TESTPATH': os.path.pathsep.join(paths)} 1127 1128 self.check_all_configs("test_init_setpath", config, 1129 api=API_COMPAT, env=env, 1130 ignore_stderr=True) 1131 1132 def test_init_setpath_config(self): 1133 # Test Py_SetPath() with PyConfig 1134 config = self._get_expected_config() 1135 paths = config['config']['module_search_paths'] 1136 1137 config = { 1138 # set by Py_SetPath() 1139 'module_search_paths': paths, 1140 'prefix': '', 1141 'base_prefix': '', 1142 'exec_prefix': '', 1143 'base_exec_prefix': '', 1144 # The current getpath.c doesn't determine the stdlib dir 1145 # in this case. 1146 'stdlib_dir': '', 1147 'use_frozen_modules': 1, 1148 # overridden by PyConfig 1149 'program_name': 'conf_program_name', 1150 'base_executable': 'conf_executable', 1151 'executable': 'conf_executable', 1152 } 1153 env = {'TESTPATH': os.path.pathsep.join(paths)} 1154 self.check_all_configs("test_init_setpath_config", config, 1155 api=API_PYTHON, env=env, ignore_stderr=True) 1156 1157 def module_search_paths(self, prefix=None, exec_prefix=None): 1158 config = self._get_expected_config() 1159 if prefix is None: 1160 prefix = config['config']['prefix'] 1161 if exec_prefix is None: 1162 exec_prefix = config['config']['prefix'] 1163 if MS_WINDOWS: 1164 return config['config']['module_search_paths'] 1165 else: 1166 ver = sys.version_info 1167 return [ 1168 os.path.join(prefix, sys.platlibdir, 1169 f'python{ver.major}{ver.minor}.zip'), 1170 os.path.join(prefix, sys.platlibdir, 1171 f'python{ver.major}.{ver.minor}'), 1172 os.path.join(exec_prefix, sys.platlibdir, 1173 f'python{ver.major}.{ver.minor}', 'lib-dynload'), 1174 ] 1175 1176 @contextlib.contextmanager 1177 def tmpdir_with_python(self, subdir=None): 1178 # Temporary directory with a copy of the Python program 1179 with tempfile.TemporaryDirectory() as tmpdir: 1180 # bpo-38234: On macOS and FreeBSD, the temporary directory 1181 # can be symbolic link. For example, /tmp can be a symbolic link 1182 # to /var/tmp. Call realpath() to resolve all symbolic links. 1183 tmpdir = os.path.realpath(tmpdir) 1184 if subdir: 1185 tmpdir = os.path.normpath(os.path.join(tmpdir, subdir)) 1186 os.makedirs(tmpdir) 1187 1188 if MS_WINDOWS: 1189 # Copy pythonXY.dll (or pythonXY_d.dll) 1190 ver = sys.version_info 1191 dll = f'python{ver.major}{ver.minor}' 1192 dll3 = f'python{ver.major}' 1193 if debug_build(sys.executable): 1194 dll += '_d' 1195 dll3 += '_d' 1196 dll += '.dll' 1197 dll3 += '.dll' 1198 dll = os.path.join(os.path.dirname(self.test_exe), dll) 1199 dll3 = os.path.join(os.path.dirname(self.test_exe), dll3) 1200 dll_copy = os.path.join(tmpdir, os.path.basename(dll)) 1201 dll3_copy = os.path.join(tmpdir, os.path.basename(dll3)) 1202 shutil.copyfile(dll, dll_copy) 1203 shutil.copyfile(dll3, dll3_copy) 1204 1205 # Copy Python program 1206 exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe)) 1207 shutil.copyfile(self.test_exe, exec_copy) 1208 shutil.copystat(self.test_exe, exec_copy) 1209 self.test_exe = exec_copy 1210 1211 yield tmpdir 1212 1213 def test_init_setpythonhome(self): 1214 # Test Py_SetPythonHome(home) with PYTHONPATH env var 1215 config = self._get_expected_config() 1216 paths = config['config']['module_search_paths'] 1217 paths_str = os.path.pathsep.join(paths) 1218 1219 for path in paths: 1220 if not os.path.isdir(path): 1221 continue 1222 if os.path.exists(os.path.join(path, 'os.py')): 1223 home = os.path.dirname(path) 1224 break 1225 else: 1226 self.fail(f"Unable to find home in {paths!r}") 1227 1228 prefix = exec_prefix = home 1229 if MS_WINDOWS: 1230 stdlib = os.path.join(home, "Lib") 1231 # Because we are specifying 'home', module search paths 1232 # are fairly static 1233 expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] 1234 else: 1235 version = f'{sys.version_info.major}.{sys.version_info.minor}' 1236 stdlib = os.path.join(home, sys.platlibdir, f'python{version}') 1237 expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) 1238 1239 config = { 1240 'home': home, 1241 'module_search_paths': expected_paths, 1242 'prefix': prefix, 1243 'base_prefix': prefix, 1244 'exec_prefix': exec_prefix, 1245 'base_exec_prefix': exec_prefix, 1246 'pythonpath_env': paths_str, 1247 'stdlib_dir': stdlib, 1248 } 1249 self.default_program_name(config) 1250 if not config['executable']: 1251 config['use_frozen_modules'] = -1 1252 env = {'TESTHOME': home, 'PYTHONPATH': paths_str} 1253 self.check_all_configs("test_init_setpythonhome", config, 1254 api=API_COMPAT, env=env) 1255 1256 def copy_paths_by_env(self, config): 1257 all_configs = self._get_expected_config() 1258 paths = all_configs['config']['module_search_paths'] 1259 paths_str = os.path.pathsep.join(paths) 1260 config['pythonpath_env'] = paths_str 1261 env = {'PYTHONPATH': paths_str} 1262 return env 1263 1264 @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32') 1265 def test_init_pybuilddir(self): 1266 # Test path configuration with pybuilddir.txt configuration file 1267 1268 with self.tmpdir_with_python() as tmpdir: 1269 # pybuilddir.txt is a sub-directory relative to the current 1270 # directory (tmpdir) 1271 subdir = 'libdir' 1272 libdir = os.path.join(tmpdir, subdir) 1273 # The stdlib dir is dirname(executable) + VPATH + 'Lib' 1274 stdlibdir = os.path.join(tmpdir, 'Lib') 1275 os.mkdir(libdir) 1276 1277 filename = os.path.join(tmpdir, 'pybuilddir.txt') 1278 with open(filename, "w", encoding="utf8") as fp: 1279 fp.write(subdir) 1280 1281 module_search_paths = self.module_search_paths() 1282 module_search_paths[-2] = stdlibdir 1283 module_search_paths[-1] = libdir 1284 1285 executable = self.test_exe 1286 config = { 1287 'base_exec_prefix': sysconfig.get_config_var("exec_prefix"), 1288 'base_prefix': sysconfig.get_config_var("prefix"), 1289 'base_executable': executable, 1290 'executable': executable, 1291 'module_search_paths': module_search_paths, 1292 'stdlib_dir': stdlibdir, 1293 } 1294 env = self.copy_paths_by_env(config) 1295 self.check_all_configs("test_init_compat_config", config, 1296 api=API_COMPAT, env=env, 1297 ignore_stderr=True, cwd=tmpdir) 1298 1299 @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir') 1300 def test_init_pybuilddir_win32(self): 1301 # Test path configuration with pybuilddir.txt configuration file 1302 1303 with self.tmpdir_with_python(r'PCbuild\arch') as tmpdir: 1304 # The prefix is dirname(executable) + VPATH 1305 prefix = os.path.normpath(os.path.join(tmpdir, r'..\..')) 1306 # The stdlib dir is dirname(executable) + VPATH + 'Lib' 1307 stdlibdir = os.path.normpath(os.path.join(tmpdir, r'..\..\Lib')) 1308 1309 filename = os.path.join(tmpdir, 'pybuilddir.txt') 1310 with open(filename, "w", encoding="utf8") as fp: 1311 fp.write(tmpdir) 1312 1313 module_search_paths = self.module_search_paths() 1314 module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) 1315 module_search_paths[-2] = stdlibdir 1316 module_search_paths[-1] = tmpdir 1317 1318 executable = self.test_exe 1319 config = { 1320 'base_exec_prefix': prefix, 1321 'base_prefix': prefix, 1322 'base_executable': executable, 1323 'executable': executable, 1324 'prefix': prefix, 1325 'exec_prefix': prefix, 1326 'module_search_paths': module_search_paths, 1327 'stdlib_dir': stdlibdir, 1328 } 1329 env = self.copy_paths_by_env(config) 1330 self.check_all_configs("test_init_compat_config", config, 1331 api=API_COMPAT, env=env, 1332 ignore_stderr=False, cwd=tmpdir) 1333 1334 def test_init_pyvenv_cfg(self): 1335 # Test path configuration with pyvenv.cfg configuration file 1336 1337 with self.tmpdir_with_python() as tmpdir, \ 1338 tempfile.TemporaryDirectory() as pyvenv_home: 1339 ver = sys.version_info 1340 1341 if not MS_WINDOWS: 1342 lib_dynload = os.path.join(pyvenv_home, 1343 sys.platlibdir, 1344 f'python{ver.major}.{ver.minor}', 1345 'lib-dynload') 1346 os.makedirs(lib_dynload) 1347 else: 1348 lib_folder = os.path.join(pyvenv_home, 'Lib') 1349 os.makedirs(lib_folder) 1350 # getpath.py uses Lib\os.py as the LANDMARK 1351 shutil.copyfile( 1352 os.path.join(support.STDLIB_DIR, 'os.py'), 1353 os.path.join(lib_folder, 'os.py'), 1354 ) 1355 1356 filename = os.path.join(tmpdir, 'pyvenv.cfg') 1357 with open(filename, "w", encoding="utf8") as fp: 1358 print("home = %s" % pyvenv_home, file=fp) 1359 print("include-system-site-packages = false", file=fp) 1360 1361 paths = self.module_search_paths() 1362 if not MS_WINDOWS: 1363 paths[-1] = lib_dynload 1364 else: 1365 # Include DLLs directory as well 1366 paths.insert(1, '.\\DLLs') 1367 for index, path in enumerate(paths): 1368 if index == 0: 1369 # Because we copy the DLLs into tmpdir as well, the zip file 1370 # entry in sys.path will be there. For a regular venv, it will 1371 # usually be in the home directory. 1372 paths[index] = os.path.join(tmpdir, os.path.basename(path)) 1373 else: 1374 paths[index] = os.path.join(pyvenv_home, os.path.basename(path)) 1375 paths[-1] = pyvenv_home 1376 1377 executable = self.test_exe 1378 base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) 1379 exec_prefix = pyvenv_home 1380 config = { 1381 'base_prefix': sysconfig.get_config_var("prefix"), 1382 'base_exec_prefix': exec_prefix, 1383 'exec_prefix': exec_prefix, 1384 'base_executable': base_executable, 1385 'executable': executable, 1386 'module_search_paths': paths, 1387 } 1388 if MS_WINDOWS: 1389 config['base_prefix'] = pyvenv_home 1390 config['prefix'] = pyvenv_home 1391 config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') 1392 config['use_frozen_modules'] = 1 1393 else: 1394 # cannot reliably assume stdlib_dir here because it 1395 # depends too much on our build. But it ought to be found 1396 config['stdlib_dir'] = self.IGNORE_CONFIG 1397 config['use_frozen_modules'] = 1 1398 1399 env = self.copy_paths_by_env(config) 1400 self.check_all_configs("test_init_compat_config", config, 1401 api=API_COMPAT, env=env, 1402 ignore_stderr=True, cwd=tmpdir) 1403 1404 def test_global_pathconfig(self): 1405 # Test C API functions getting the path configuration: 1406 # 1407 # - Py_GetExecPrefix() 1408 # - Py_GetPath() 1409 # - Py_GetPrefix() 1410 # - Py_GetProgramFullPath() 1411 # - Py_GetProgramName() 1412 # - Py_GetPythonHome() 1413 # 1414 # The global path configuration (_Py_path_config) must be a copy 1415 # of the path configuration of PyInterpreter.config (PyConfig). 1416 ctypes = import_helper.import_module('ctypes') 1417 _testinternalcapi = import_helper.import_module('_testinternalcapi') 1418 1419 def get_func(name): 1420 func = getattr(ctypes.pythonapi, name) 1421 func.argtypes = () 1422 func.restype = ctypes.c_wchar_p 1423 return func 1424 1425 Py_GetPath = get_func('Py_GetPath') 1426 Py_GetPrefix = get_func('Py_GetPrefix') 1427 Py_GetExecPrefix = get_func('Py_GetExecPrefix') 1428 Py_GetProgramName = get_func('Py_GetProgramName') 1429 Py_GetProgramFullPath = get_func('Py_GetProgramFullPath') 1430 Py_GetPythonHome = get_func('Py_GetPythonHome') 1431 1432 config = _testinternalcapi.get_configs()['config'] 1433 1434 self.assertEqual(Py_GetPath().split(os.path.pathsep), 1435 config['module_search_paths']) 1436 self.assertEqual(Py_GetPrefix(), config['prefix']) 1437 self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) 1438 self.assertEqual(Py_GetProgramName(), config['program_name']) 1439 self.assertEqual(Py_GetProgramFullPath(), config['executable']) 1440 self.assertEqual(Py_GetPythonHome(), config['home']) 1441 1442 def test_init_warnoptions(self): 1443 # lowest to highest priority 1444 warnoptions = [ 1445 'ignore:::PyConfig_Insert0', # PyWideStringList_Insert(0) 1446 'default', # PyConfig.dev_mode=1 1447 'ignore:::env1', # PYTHONWARNINGS env var 1448 'ignore:::env2', # PYTHONWARNINGS env var 1449 'ignore:::cmdline1', # -W opt command line option 1450 'ignore:::cmdline2', # -W opt command line option 1451 'default::BytesWarning', # PyConfig.bytes_warnings=1 1452 'ignore:::PySys_AddWarnOption1', # PySys_AddWarnOption() 1453 'ignore:::PySys_AddWarnOption2', # PySys_AddWarnOption() 1454 'ignore:::PyConfig_BeforeRead', # PyConfig.warnoptions 1455 'ignore:::PyConfig_AfterRead'] # PyWideStringList_Append() 1456 preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) 1457 config = { 1458 'dev_mode': 1, 1459 'faulthandler': 1, 1460 'bytes_warning': 1, 1461 'warnoptions': warnoptions, 1462 'orig_argv': ['python3', 1463 '-Wignore:::cmdline1', 1464 '-Wignore:::cmdline2'], 1465 } 1466 self.check_all_configs("test_init_warnoptions", config, preconfig, 1467 api=API_PYTHON) 1468 1469 def test_init_set_config(self): 1470 config = { 1471 '_init_main': 0, 1472 'bytes_warning': 2, 1473 'warnoptions': ['error::BytesWarning'], 1474 } 1475 self.check_all_configs("test_init_set_config", config, 1476 api=API_ISOLATED) 1477 1478 def test_get_argc_argv(self): 1479 self.run_embedded_interpreter("test_get_argc_argv") 1480 # ignore output 1481 1482 def test_init_use_frozen_modules(self): 1483 tests = { 1484 ('=on', 1), 1485 ('=off', 0), 1486 ('=', 1), 1487 ('', 1), 1488 } 1489 for raw, expected in tests: 1490 optval = f'frozen_modules{raw}' 1491 config = { 1492 'parse_argv': 2, 1493 'argv': ['-c'], 1494 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'], 1495 'program_name': './argv0', 1496 'run_command': 'pass\n', 1497 'use_environment': 1, 1498 'xoptions': [optval], 1499 'use_frozen_modules': expected, 1500 } 1501 env = {'TESTFROZEN': raw[1:]} if raw else None 1502 with self.subTest(repr(raw)): 1503 self.check_all_configs("test_init_use_frozen_modules", config, 1504 api=API_PYTHON, env=env) 1505 1506 1507class SetConfigTests(unittest.TestCase): 1508 def test_set_config(self): 1509 # bpo-42260: Test _PyInterpreterState_SetConfig() 1510 import_helper.import_module('_testcapi') 1511 cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config'] 1512 proc = subprocess.run(cmd, 1513 stdout=subprocess.PIPE, 1514 stderr=subprocess.PIPE, 1515 encoding='utf-8', errors='backslashreplace') 1516 if proc.returncode and support.verbose: 1517 print(proc.stdout) 1518 print(proc.stderr) 1519 self.assertEqual(proc.returncode, 0, 1520 (proc.returncode, proc.stdout, proc.stderr)) 1521 1522 1523class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): 1524 def test_open_code_hook(self): 1525 self.run_embedded_interpreter("test_open_code_hook") 1526 1527 def test_audit(self): 1528 self.run_embedded_interpreter("test_audit") 1529 1530 def test_audit_subinterpreter(self): 1531 self.run_embedded_interpreter("test_audit_subinterpreter") 1532 1533 def test_audit_run_command(self): 1534 self.run_embedded_interpreter("test_audit_run_command", 1535 timeout=support.SHORT_TIMEOUT, 1536 returncode=1) 1537 1538 def test_audit_run_file(self): 1539 self.run_embedded_interpreter("test_audit_run_file", 1540 timeout=support.SHORT_TIMEOUT, 1541 returncode=1) 1542 1543 def test_audit_run_interactivehook(self): 1544 startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py" 1545 with open(startup, "w", encoding="utf-8") as f: 1546 print("import sys", file=f) 1547 print("sys.__interactivehook__ = lambda: None", file=f) 1548 try: 1549 env = {**remove_python_envvars(), "PYTHONSTARTUP": startup} 1550 self.run_embedded_interpreter("test_audit_run_interactivehook", 1551 timeout=support.SHORT_TIMEOUT, 1552 returncode=10, env=env) 1553 finally: 1554 os.unlink(startup) 1555 1556 def test_audit_run_startup(self): 1557 startup = os.path.join(self.oldcwd, os_helper.TESTFN) + ".py" 1558 with open(startup, "w", encoding="utf-8") as f: 1559 print("pass", file=f) 1560 try: 1561 env = {**remove_python_envvars(), "PYTHONSTARTUP": startup} 1562 self.run_embedded_interpreter("test_audit_run_startup", 1563 timeout=support.SHORT_TIMEOUT, 1564 returncode=10, env=env) 1565 finally: 1566 os.unlink(startup) 1567 1568 def test_audit_run_stdin(self): 1569 self.run_embedded_interpreter("test_audit_run_stdin", 1570 timeout=support.SHORT_TIMEOUT, 1571 returncode=1) 1572 1573 1574class MiscTests(EmbeddingTestsMixin, unittest.TestCase): 1575 def test_unicode_id_init(self): 1576 # bpo-42882: Test that _PyUnicode_FromId() works 1577 # when Python is initialized multiples times. 1578 self.run_embedded_interpreter("test_unicode_id_init") 1579 1580 # See bpo-44133 1581 @unittest.skipIf(os.name == 'nt', 1582 'Py_FrozenMain is not exported on Windows') 1583 def test_frozenmain(self): 1584 env = dict(os.environ) 1585 env['PYTHONUNBUFFERED'] = '1' 1586 out, err = self.run_embedded_interpreter("test_frozenmain", env=env) 1587 executable = os.path.realpath('./argv0') 1588 expected = textwrap.dedent(f""" 1589 Frozen Hello World 1590 sys.argv ['./argv0', '-E', 'arg1', 'arg2'] 1591 config program_name: ./argv0 1592 config executable: {executable} 1593 config use_environment: 1 1594 config configure_c_stdio: 1 1595 config buffered_stdio: 0 1596 """).lstrip() 1597 self.assertEqual(out, expected) 1598 1599 1600class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): 1601 # Test PyStdPrinter_Type which is used by _PySys_SetPreliminaryStderr(): 1602 # "Set up a preliminary stderr printer until we have enough 1603 # infrastructure for the io module in place." 1604 1605 STDOUT_FD = 1 1606 1607 def create_printer(self, fd): 1608 ctypes = import_helper.import_module('ctypes') 1609 PyFile_NewStdPrinter = ctypes.pythonapi.PyFile_NewStdPrinter 1610 PyFile_NewStdPrinter.argtypes = (ctypes.c_int,) 1611 PyFile_NewStdPrinter.restype = ctypes.py_object 1612 return PyFile_NewStdPrinter(fd) 1613 1614 def test_write(self): 1615 message = "unicode:\xe9-\u20ac-\udc80!\n" 1616 1617 stdout_fd = self.STDOUT_FD 1618 stdout_fd_copy = os.dup(stdout_fd) 1619 self.addCleanup(os.close, stdout_fd_copy) 1620 1621 rfd, wfd = os.pipe() 1622 self.addCleanup(os.close, rfd) 1623 self.addCleanup(os.close, wfd) 1624 try: 1625 # PyFile_NewStdPrinter() only accepts fileno(stdout) 1626 # or fileno(stderr) file descriptor. 1627 os.dup2(wfd, stdout_fd) 1628 1629 printer = self.create_printer(stdout_fd) 1630 printer.write(message) 1631 finally: 1632 os.dup2(stdout_fd_copy, stdout_fd) 1633 1634 data = os.read(rfd, 100) 1635 self.assertEqual(data, message.encode('utf8', 'backslashreplace')) 1636 1637 def test_methods(self): 1638 fd = self.STDOUT_FD 1639 printer = self.create_printer(fd) 1640 self.assertEqual(printer.fileno(), fd) 1641 self.assertEqual(printer.isatty(), os.isatty(fd)) 1642 printer.flush() # noop 1643 printer.close() # noop 1644 1645 def test_disallow_instantiation(self): 1646 fd = self.STDOUT_FD 1647 printer = self.create_printer(fd) 1648 support.check_disallow_instantiation(self, type(printer)) 1649 1650 1651if __name__ == "__main__": 1652 unittest.main() 1653