1# Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com 2# 3# Part of "Nuitka", an optimizing Python compiler that is compatible and 4# integrates with CPython, but also works on its own. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18""" Common test infrastructure functions. To be used by test runners. """ 19 20from __future__ import print_function 21 22import ast 23import atexit 24import gc 25import hashlib 26import os 27import shutil 28import signal 29import sys 30import tempfile 31import threading 32import time 33from contextlib import contextmanager 34from optparse import OptionGroup, OptionParser 35 36from nuitka.__past__ import subprocess 37from nuitka.PythonVersions import ( 38 getPartiallySupportedPythonVersions, 39 getSupportedPythonVersions, 40) 41from nuitka.Tracing import OurLogger, my_print 42from nuitka.tree.SourceReading import readSourceCodeFromFilename 43from nuitka.utils.AppDirs import getCacheDir 44from nuitka.utils.Execution import check_output, getNullInput 45from nuitka.utils.FileOperations import ( 46 areSamePaths, 47 getExternalUsePath, 48 getFileContentByLine, 49 getFileContents, 50 getFileList, 51 isPathBelowOrSameAs, 52 makePath, 53 removeDirectory, 54) 55from nuitka.utils.Utils import getOS 56 57from .SearchModes import ( 58 SearchModeAll, 59 SearchModeByPattern, 60 SearchModeCoverage, 61 SearchModeImmediate, 62 SearchModeOnly, 63 SearchModeResume, 64) 65 66test_logger = OurLogger("", base_style="blue") 67 68 69def check_result(*popenargs, **kwargs): 70 if "stdout" in kwargs: 71 raise ValueError("stdout argument not allowed, it will be overridden.") 72 73 process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) 74 _unused_output, _unused_err = process.communicate() 75 retcode = process.poll() 76 77 if retcode: 78 return False 79 else: 80 return True 81 82 83def goMainDir(): 84 # Go its own directory, to have it easy with path knowledge. 85 os.chdir(os.path.dirname(os.path.abspath(sys.modules["__main__"].__file__))) 86 87 88_python_version_str = None 89_python_version = None 90_python_arch = None 91_python_executable = None 92_python_vendor = None 93 94 95def setup(suite="", needs_io_encoding=False, silent=False, go_main=True): 96 if go_main: 97 goMainDir() 98 99 if "PYTHON" not in os.environ: 100 os.environ["PYTHON"] = sys.executable 101 102 # Allow test code to use this to make caching specific. 103 os.environ["NUITKA_TEST_SUITE"] = suite 104 105 # Allow providing 33, 27, and expand that to python2.7 106 if ( 107 len(os.environ["PYTHON"]) == 2 108 and os.environ["PYTHON"].isdigit() 109 and os.name != "nt" 110 ): 111 112 os.environ["PYTHON"] = "python%s.%s" % ( 113 os.environ["PYTHON"][0], 114 os.environ["PYTHON"][1], 115 ) 116 117 if needs_io_encoding and "PYTHONIOENCODING" not in os.environ: 118 os.environ["PYTHONIOENCODING"] = "utf-8" 119 120 version_output = check_output( 121 ( 122 os.environ["PYTHON"], 123 "-c", 124 """\ 125import sys, os;\ 126print(".".join(str(s) for s in list(sys.version_info)[:3]));\ 127print(("x86_64" if "AMD64" in sys.version else "x86") if os.name == "nt" else os.uname()[4]);\ 128print(sys.executable);\ 129print("Anaconda" if os.path.exists(os.path.join(sys.prefix, 'conda-meta')) else "Unknown")\ 130""", 131 ), 132 stderr=subprocess.STDOUT, 133 ) 134 135 global _python_version_str, _python_version, _python_arch, _python_executable, _python_vendor # singleton, pylint: disable=global-statement 136 137 _python_version_str = version_output.split(b"\n")[0].strip() 138 _python_arch = version_output.split(b"\n")[1].strip() 139 _python_executable = version_output.split(b"\n")[2].strip() 140 _python_vendor = version_output.split(b"\n")[3].strip() 141 142 if str is not bytes: 143 _python_version_str = _python_version_str.decode("utf8") 144 _python_arch = _python_arch.decode("utf8") 145 _python_executable = _python_executable.decode("utf8") 146 _python_vendor = _python_vendor.decode("utf8") 147 148 assert type(_python_version_str) is str, repr(_python_version_str) 149 assert type(_python_arch) is str, repr(_python_arch) 150 assert type(_python_executable) is str, repr(_python_executable) 151 152 if not silent: 153 my_print("Using concrete python", _python_version_str, "on", _python_arch) 154 155 if "COVERAGE_FILE" not in os.environ: 156 os.environ["COVERAGE_FILE"] = os.path.join( 157 os.path.dirname(__file__), "..", "..", "..", ".coverage" 158 ) 159 160 _python_version = tuple(int(d) for d in _python_version_str.split(".")) 161 162 return _python_version 163 164 165def getPythonArch(): 166 return _python_arch 167 168 169def getPythonVendor(): 170 return _python_vendor 171 172 173def getPythonVersionString(): 174 return _python_version_str 175 176 177tmp_dir = None 178 179 180def getTempDir(): 181 # Create a temporary directory to work in, automatically remove it in case 182 # it is empty in the end. 183 global tmp_dir # singleton, pylint: disable=global-statement 184 185 if tmp_dir is None: 186 tmp_dir = tempfile.mkdtemp( 187 prefix=os.path.basename( 188 os.path.dirname(os.path.abspath(sys.modules["__main__"].__file__)) 189 ) 190 + "-", 191 dir=tempfile.gettempdir() if not os.path.exists("/var/tmp") else "/var/tmp", 192 ) 193 194 def removeTempDir(): 195 removeDirectory(path=tmp_dir, ignore_errors=True) 196 197 atexit.register(removeTempDir) 198 199 return tmp_dir 200 201 202def convertUsing2to3(path, force=False): 203 command = [os.environ["PYTHON"], "-m", "py_compile", path] 204 205 if not force: 206 with open(path) as source_file: 207 if "xrange" not in source_file.read(): 208 with open(os.devnull, "w") as stderr: 209 if check_result(command, stderr=stderr): 210 return path, False 211 212 filename = os.path.basename(path) 213 214 new_path = os.path.join(getTempDir(), filename) 215 216 # This may already be a temp file, e.g. because of construct creation. 217 try: 218 shutil.copy(path, new_path) 219 except shutil.Error: 220 pass 221 222 # For Python2.6 and 3.2 the -m lib2to3 was not yet supported. 223 use_binary = sys.version_info[:2] in ((2, 6), (3, 2)) 224 225 if use_binary: 226 # On Windows, we cannot rely on 2to3 to be in the path. 227 if os.name == "nt": 228 command = [ 229 sys.executable, 230 os.path.join(os.path.dirname(sys.executable), "Tools/Scripts/2to3.py"), 231 ] 232 else: 233 command = ["2to3"] 234 else: 235 command = [sys.executable, "-m", "lib2to3"] 236 237 command += ("-w", "-n", "--no-diffs", new_path) 238 239 with open(os.devnull, "w") as devnull: 240 try: 241 check_output(command, stderr=devnull) 242 243 except subprocess.CalledProcessError: 244 if os.name == "nt": 245 raise 246 247 command[0:3] = ["2to3"] 248 249 check_output(command, stderr=devnull) 250 251 with open(new_path) as result_file: 252 data = result_file.read() 253 254 with open(new_path, "w") as result_file: 255 result_file.write("__file__ = %r\n" % os.path.abspath(path)) 256 result_file.write(data) 257 258 return new_path, True 259 260 261def decideFilenameVersionSkip(filename): 262 """Make decision whether to skip based on filename and Python version. 263 264 This codifies certain rules that files can have as suffixes or prefixes 265 to make them be part of the set of tests executed for a version or not. 266 267 Generally, an ening of "<major><minor>.py" indicates that it must be that 268 Python version or higher. There is no need for ending in "26.py" as this 269 is the minimum version anyway. 270 271 The "_2.py" indicates a maxmimum version of 2.7, i.e. not Python 3.x, for 272 language syntax no more supported. 273 """ 274 275 # This will make many decisions with immediate returns. 276 # pylint: disable=too-many-branches,too-many-return-statements 277 278 assert type(filename) is str, repr(filename) 279 280 # Skip runner scripts by default. 281 if filename.startswith("run_"): 282 return False 283 284 if filename.endswith(".j2"): 285 filename = filename[:-3] 286 287 # Skip tests that require Python 2.7 at least. 288 if filename.endswith("27.py") and _python_version < (2, 7): 289 return False 290 291 # Skip tests that require Python 2 at maximum. 292 if filename.endswith("_2.py") and _python_version >= (3,): 293 return False 294 295 # Skip tests that require Python 3.7 at maximum. 296 if filename.endswith("_37.py") and _python_version >= (3, 8): 297 return False 298 299 # Skip tests that require Python 3.2 at least. 300 if filename.endswith("32.py") and _python_version < (3, 2): 301 return False 302 303 # Skip tests that require Python 3.3 at least. 304 if filename.endswith("33.py") and _python_version < (3, 3): 305 return False 306 307 # Skip tests that require Python 3.4 at least. 308 if filename.endswith("34.py") and _python_version < (3, 4): 309 return False 310 311 # Skip tests that require Python 3.5 at least. 312 if filename.endswith("35.py") and _python_version < (3, 5): 313 return False 314 315 # Skip tests that require Python 3.6 at least. 316 if filename.endswith("36.py") and _python_version < (3, 6): 317 return False 318 319 # Skip tests that require Python 3.7 at least. 320 if filename.endswith("37.py") and _python_version < (3, 7): 321 return False 322 323 # Skip tests that require Python 3.8 at least. 324 if filename.endswith("38.py") and _python_version < (3, 8): 325 return False 326 327 # Skip tests that require Python 3.9 at least. 328 if filename.endswith("39.py") and _python_version < (3, 9): 329 return False 330 331 return True 332 333 334def decideNeeds2to3(filename): 335 return _python_version >= (3,) and not filename.endswith( 336 ("32.py", "33.py", "34.py", "35.py", "36.py", "37.py", "38.py", "39.py") 337 ) 338 339 340def _removeCPythonTestSuiteDir(): 341 # Cleanup, some tests apparently forget that. 342 try: 343 if os.path.isdir("@test"): 344 removeDirectory("@test", ignore_errors=False) 345 elif os.path.isfile("@test"): 346 os.unlink("@test") 347 except OSError: 348 # TODO: Move this into removeDirectory maybe. Doing an external 349 # call as last resort could be a good idea. 350 351 # This seems to work for broken "lnk" files. 352 if os.name == "nt": 353 os.system("rmdir /S /Q @test") 354 355 if os.path.exists("@test"): 356 raise 357 358 359def compareWithCPython( 360 dirname, filename, extra_flags, search_mode, needs_2to3, on_error=None 361): 362 """Call the comparison tool. For a given directory filename. 363 364 The search mode decides if the test case aborts on error or gets extra 365 flags that are exceptions. 366 367 """ 368 369 # Many cases to consider here, pylint: disable=too-many-branches 370 371 if dirname is None: 372 path = filename 373 else: 374 path = os.path.join(dirname, filename) 375 376 # Apply 2to3 conversion if necessary. 377 if needs_2to3: 378 path, converted = convertUsing2to3(path) 379 else: 380 converted = False 381 382 if os.getenv("NUITKA_TEST_INSTALLED", "") == "1": 383 command = [ 384 sys.executable, 385 "-m", 386 "nuitka.tools.testing.compare_with_cpython", 387 path, 388 "silent", 389 ] 390 else: 391 compare_with_cpython = os.path.join("..", "..", "bin", "compare_with_cpython") 392 if os.path.exists(compare_with_cpython): 393 command = [sys.executable, compare_with_cpython, path, "silent"] 394 else: 395 test_logger.sysexit("Error, cannot locate Nuitka comparison runner.") 396 397 if extra_flags is not None: 398 command += extra_flags 399 400 command += search_mode.getExtraFlags(dirname, filename) 401 402 # Cleanup before and after test stage directory. 403 _removeCPythonTestSuiteDir() 404 405 try: 406 result = subprocess.call(command) 407 except KeyboardInterrupt: 408 result = 2 409 410 # Cleanup before and after test stage directory. 411 _removeCPythonTestSuiteDir() 412 413 if result != 0 and result != 2 and search_mode.abortOnFinding(dirname, filename): 414 if on_error is not None: 415 on_error(dirname, filename) 416 417 search_mode.onErrorDetected("Error exit! %s" % result) 418 419 if converted: 420 os.unlink(path) 421 422 if result == 2: 423 test_logger.sysexit("Interrupted, with CTRL-C\n", exit_code=2) 424 425 426def checkCompilesNotWithCPython(dirname, filename, search_mode): 427 if dirname is None: 428 path = filename 429 else: 430 path = os.path.join(dirname, filename) 431 432 command = [_python_executable, "-mcompileall", path] 433 434 try: 435 result = subprocess.call(command) 436 except KeyboardInterrupt: 437 result = 2 438 439 if result != 1 and result != 2 and search_mode.abortOnFinding(dirname, filename): 440 search_mode.onErrorDetected("Error exit! %s" % result) 441 442 443def checkSucceedsWithCPython(filename): 444 command = [_python_executable, filename] 445 446 with open(os.devnull, "w") as devnull: 447 result = subprocess.call(command, stdout=devnull, stderr=subprocess.STDOUT) 448 449 return result == 0 450 451 452def hasDebugPython(): 453 # On Debian systems, these work. 454 debug_python = os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg") 455 if os.path.exists(debug_python): 456 return True 457 458 # On Windows systems, these work. 459 debug_python = os.environ["PYTHON"] 460 if debug_python.lower().endswith(".exe"): 461 debug_python = debug_python[:-4] 462 debug_python = debug_python + "_d.exe" 463 if os.path.exists(debug_python): 464 return True 465 466 # For other Python, if it's the one also executing the runner, which is 467 # very probably the case, we check that. We don't check the provided 468 # binary here, this could be done as well. 469 if sys.executable == os.environ["PYTHON"] and hasattr(sys, "gettotalrefcount"): 470 return True 471 472 # Otherwise no. 473 return False 474 475 476def displayRuntimeTraces(logger, path): 477 if not os.path.exists(path): 478 # TODO: Have a logger package passed. 479 logger.sysexit("Error, cannot find %r (%r)." % (path, os.path.abspath(path))) 480 481 path = os.path.abspath(path) 482 483 # TODO: Merge code for building command with below function, this is otherwise 484 # horribly bad. 485 486 if os.name == "posix": 487 # Run with traces to help debugging, specifically in CI environment. 488 if getOS() in ("Darwin", "FreeBSD"): 489 test_logger.info("dtruss:") 490 os.system("sudo dtruss %s" % path) 491 else: 492 test_logger.info("strace:") 493 os.system("strace -s4096 -e file %s" % path) 494 495 496def checkRuntimeLoadedFilesForOutsideAccesses(loaded_filenames, white_list): 497 # A lot of special white listing is required. 498 # pylint: disable=too-many-branches 499 500 result = [] 501 502 for loaded_filename in loaded_filenames: 503 loaded_filename = os.path.normpath(loaded_filename) 504 loaded_filename = os.path.normcase(loaded_filename) 505 loaded_basename = os.path.basename(loaded_filename) 506 507 ok = False 508 for entry in white_list: 509 if loaded_filename.startswith(entry): 510 ok = True 511 512 while entry: 513 old_entry = entry 514 entry = os.path.dirname(entry) 515 516 if old_entry == entry: 517 break 518 519 if loaded_filename == entry: 520 ok = True 521 break 522 if ok: 523 continue 524 525 ignore = True 526 for ignored_dir in ( 527 # System configuration is OK 528 "/etc", 529 "/usr/etc", 530 "/usr/local/etc", 531 # Runtime user state and kernel information is OK. 532 "/proc", 533 "/dev", 534 "/run", 535 "/sys", 536 "/tmp", 537 # Locals may of course be loaded. 538 "/usr/lib/locale", 539 "/usr/share/locale", 540 "/usr/share/X11/locale", 541 # Themes may of course be loaded. 542 "/usr/share/themes", 543 # Terminal info files are OK too. 544 "/lib/terminfo", 545 ): 546 if isPathBelowOrSameAs(ignored_dir, loaded_filename): 547 ignore = False 548 break 549 if not ignore: 550 continue 551 552 if "gtk" in loaded_filename and "/engines/" in loaded_filename: 553 continue 554 555 # System C libraries are to be expected. 556 if loaded_basename.startswith( 557 ( 558 "ld-linux-x86-64.so", 559 "libc.so.", 560 "libpthread.so.", 561 "libm.so.", 562 "libdl.so.", 563 "libBrokenLocale.so.", 564 "libSegFault.so", 565 "libanl.so.", 566 "libcidn.so.", 567 "libcrypt.so.", 568 "libmemusage.so", 569 "libmvec.so.", 570 "libnsl.so.", 571 "libnss_compat.so.", 572 "libnss_db.so.", 573 "libnss_dns.so.", 574 "libnss_files.so.", 575 "libnss_hesiod.so.", 576 "libnss_nis.so.", 577 "libnss_nisplus.so.", 578 "libpcprofile.so", 579 "libresolv.so.", 580 "librt.so.", 581 "libthread_db-1.0.so", 582 "libthread_db.so.", 583 "libutil.so.", 584 ) 585 ): 586 continue 587 588 # Taking these from system is harmless and desirable 589 if loaded_basename.startswith(("libz.so", "libgcc_s.so")): 590 continue 591 592 # TODO: Unclear, loading gconv from filesystem of installed system 593 # may be OK or not. I think it should be. 594 if loaded_basename == "gconv-modules.cache": 595 continue 596 if "/gconv/" in loaded_filename: 597 continue 598 if loaded_basename.startswith("libicu"): 599 continue 600 601 # GTK may access X files. 602 if loaded_basename == ".Xauthority": 603 continue 604 605 result.append(loaded_filename) 606 607 return result 608 609 610def hasModule(module_name): 611 with open(os.devnull, "w") as devnull: 612 result = subprocess.call( 613 (os.environ["PYTHON"], "-c", "import %s" % module_name), 614 stdout=devnull, 615 stderr=subprocess.STDOUT, 616 ) 617 618 return result == 0 619 620 621m1 = {} 622m2 = {} 623 624 625def cleanObjRefCntMaps(): 626 m1.clear() 627 m2.clear() 628 629 # Warm out repr 630 for x in gc.get_objects(): 631 try: 632 str(x) 633 except Exception: # Catch all the things, pylint: disable=broad-except 634 pass 635 636 637def snapObjRefCntMap(before): 638 # Inherently complex, pylint: disable=too-many-branches 639 640 if before: 641 m = m1 642 else: 643 m = m2 644 645 m.clear() 646 gc.collect() 647 648 for x in gc.get_objects(): 649 # The dictionary is cyclic, and contains itself, avoid that. 650 if x is m1 or x is m2: 651 continue 652 653 if type(x) is str and (x in m1 or x in m2): 654 continue 655 656 if type(x) is not str and isinstance(x, str): 657 k = "str_overload_" + x.__class__.__name__ + str(x) 658 elif type(x) is dict: 659 if "__builtins__" in x: 660 k = "<module dict %s>" % x["__name__"] 661 elif "__spec__" in x and "__name__" in x: 662 k = "<module dict %s>" % x["__name__"] 663 else: 664 k = str(x) 665 elif x.__class__.__name__ == "compiled_frame": 666 k = "<compiled_frame at xxx, line %d code %s" % (x.f_lineno, x.f_code) 667 else: 668 k = str(x) 669 670 c = sys.getrefcount(x) 671 672 if k in m: 673 m[k] += c 674 else: 675 m[k] = c 676 677 678orig_print = None 679 680 681def disablePrinting(): 682 # Singleton, pylint: disable=global-statement 683 global orig_print 684 685 if orig_print is None: 686 orig_print = __builtins__["print"] 687 __builtins__["print"] = lambda *args, **kwargs: None 688 689 690def reenablePrinting(): 691 # Singleton, pylint: disable=global-statement 692 global orig_print 693 694 if orig_print is not None: 695 __builtins__["print"] = orig_print 696 orig_print = None 697 698 699_debug_python = hasattr(sys, "gettotalrefcount") 700 701 702def getTotalReferenceCount(): 703 if _debug_python: 704 gc.collect() 705 return sys.gettotalrefcount() 706 else: 707 gc.collect() 708 all_objects = gc.get_objects() 709 710 # Sum object reference twice, once without the sum value type, then switch 711 # the type, and use the type used to avoid the integers before that. 712 result = 0.0 713 for obj in all_objects: 714 if type(obj) is float: 715 continue 716 717 result += sys.getrefcount(obj) 718 719 result = int(result) 720 721 for obj in all_objects: 722 if type(obj) is not float: 723 continue 724 725 result += sys.getrefcount(obj) 726 727 return result 728 729 730def checkReferenceCount(checked_function, max_rounds=20, explain=False): 731 # This is obviously going to be complex, pylint: disable=too-many-branches 732 733 # Clean start conditions. 734 assert sys.exc_info() == (None, None, None), sys.exc_info() 735 736 print(checked_function.__name__ + ": ", end="") 737 sys.stdout.flush() 738 739 disablePrinting() 740 741 # Make sure reference for these are already taken at the start. 742 ref_count1 = 17 743 ref_count2 = 17 744 745 if explain: 746 cleanObjRefCntMaps() 747 748 assert max_rounds > 0 749 750 result = False 751 752 for count in range(max_rounds): 753 if explain and count == max_rounds - 1: 754 snapObjRefCntMap(before=True) 755 756 ref_count1 = getTotalReferenceCount() 757 758 checked_function() 759 760 ref_count2 = getTotalReferenceCount() 761 762 # Not allowed, but happens when bugs occur. 763 assert sys.exc_info() == (None, None, None), sys.exc_info() 764 765 if ref_count1 == ref_count2: 766 result = True 767 break 768 769 if explain and count == max_rounds - 1: 770 snapObjRefCntMap(before=False) 771 772 reenablePrinting() 773 774 if result: 775 print("PASSED") 776 else: 777 print("FAILED", ref_count1, ref_count2, "leaked", ref_count2 - ref_count1) 778 779 if explain: 780 print("REPORT of differences:") 781 assert m1 782 assert m2 783 784 for key in m1: 785 if key not in m2: 786 print("*" * 80) 787 print("extra:", m1[key], key) 788 elif m1[key] != m2[key]: 789 print("*" * 80) 790 print(m1[key], "->", m2[key], key) 791 else: 792 pass 793 794 for key in m2: 795 if key not in m1: 796 print("*" * 80) 797 print("missing:", m2[key], key) 798 799 # print m1[key] 800 801 assert sys.exc_info() == (None, None, None), sys.exc_info() 802 803 gc.collect() 804 sys.stdout.flush() 805 806 return result 807 808 809def createSearchMode(): 810 # Dealing with many options, pylint: disable=too-many-branches 811 812 parser = OptionParser() 813 814 select_group = OptionGroup(parser, "Select Tests") 815 816 select_group.add_option( 817 "--pattern", 818 action="store", 819 dest="pattern", 820 default="", 821 help="""\ 822Execute only tests matching the pattern. Defaults to all tests.""", 823 ) 824 select_group.add_option( 825 "--all", 826 action="store_true", 827 dest="all", 828 default=False, 829 help="""\ 830Execute all tests, continue execution even after failure of one.""", 831 ) 832 833 parser.add_option_group(select_group) 834 835 debug_group = OptionGroup(parser, "Test features") 836 837 debug_group.add_option( 838 "--debug", 839 action="store_true", 840 dest="debug", 841 default=False, 842 help="""\ 843Executing all self checks possible to find errors in Nuitka, good for test coverage. 844Defaults to off.""", 845 ) 846 847 debug_group.add_option( 848 "--commands", 849 action="store_true", 850 dest="show_commands", 851 default=False, 852 help="""Output commands being done in output comparison. 853Defaults to off.""", 854 ) 855 856 parser.add_option_group(debug_group) 857 858 options, positional_args = parser.parse_args() 859 860 if options.debug: 861 addExtendedExtraOptions("--debug") 862 863 if options.show_commands: 864 os.environ["NUITKA_TRACE_COMMANDS"] = "1" 865 866 # Default to searching. 867 mode = positional_args[0] if positional_args else "search" 868 869 # Avoid having to use options style. 870 if mode in ("search", "only"): 871 if len(positional_args) >= 2 and not options.pattern: 872 options.pattern = positional_args[1] 873 874 if mode == "search": 875 if options.all: 876 return SearchModeAll() 877 elif options.pattern: 878 pattern = options.pattern.replace("/", os.path.sep) 879 return SearchModeByPattern(pattern) 880 else: 881 return SearchModeImmediate() 882 elif mode == "resume": 883 return SearchModeResume(sys.modules["__main__"].__file__) 884 elif mode == "only": 885 if options.pattern: 886 pattern = options.pattern.replace("/", os.path.sep) 887 return SearchModeOnly(pattern) 888 else: 889 assert False 890 elif mode == "coverage": 891 return SearchModeCoverage() 892 else: 893 test_logger.sysexit("Error, using unknown search mode %r" % mode) 894 895 896def reportSkip(reason, dirname, filename): 897 case = os.path.join(dirname, filename) 898 case = os.path.normpath(case) 899 900 my_print("Skipped, %s (%s)." % (case, reason)) 901 902 903def executeReferenceChecked(prefix, names, tests_skipped, tests_stderr, explain=False): 904 gc.disable() 905 906 extract_number = lambda name: int(name.replace(prefix, "")) 907 908 # Find the function names. 909 matching_names = tuple( 910 name for name in names if name.startswith(prefix) and name[-1].isdigit() 911 ) 912 913 old_stderr = sys.stderr 914 915 # Everything passed 916 result = True 917 918 for name in sorted(matching_names, key=extract_number): 919 number = extract_number(name) 920 921 # print(tests_skipped) 922 if number in tests_skipped: 923 my_print(name + ": SKIPPED (%s)" % tests_skipped[number]) 924 continue 925 926 # Avoid unraisable output. 927 try: 928 if number in tests_stderr: 929 sys.stderr = open(os.devnull, "wb") 930 except OSError: # Windows 931 if not checkReferenceCount(names[name], explain=explain): 932 result = False 933 else: 934 if not checkReferenceCount(names[name], explain=explain): 935 result = False 936 937 if number in tests_stderr: 938 new_stderr = sys.stderr 939 sys.stderr = old_stderr 940 new_stderr.close() 941 942 gc.enable() 943 return result 944 945 946def addToPythonPath(python_path, in_front=False): 947 if type(python_path) in (tuple, list): 948 python_path = os.pathsep.join(python_path) 949 950 if python_path: 951 if "PYTHONPATH" in os.environ: 952 if in_front: 953 os.environ["PYTHONPATH"] = ( 954 python_path + os.pathsep + os.environ["PYTHONPATH"] 955 ) 956 else: 957 os.environ["PYTHONPATH"] += os.pathsep + python_path 958 else: 959 os.environ["PYTHONPATH"] = python_path 960 961 962@contextmanager 963def withPythonPathChange(python_path): 964 if python_path: 965 if type(python_path) not in (tuple, list): 966 python_path = python_path.split(os.pathsep) 967 968 python_path = [ 969 os.path.normpath(os.path.abspath(element)) for element in python_path 970 ] 971 972 python_path = os.pathsep.join(python_path) 973 974 if "PYTHONPATH" in os.environ: 975 old_path = os.environ["PYTHONPATH"] 976 os.environ["PYTHONPATH"] += os.pathsep + python_path 977 else: 978 old_path = None 979 os.environ["PYTHONPATH"] = python_path 980 981 # print( 982 # "Effective PYTHONPATH in %s is %r" % ( 983 # sys.modules["__main__"], 984 # os.environ.get("PYTHONPATH", "") 985 # ) 986 # ) 987 988 yield 989 990 if python_path: 991 if old_path is None: 992 del os.environ["PYTHONPATH"] 993 else: 994 os.environ["PYTHONPATH"] = old_path 995 996 997def addExtendedExtraOptions(*args): 998 old_value = os.environ.get("NUITKA_EXTRA_OPTIONS") 999 1000 value = old_value 1001 1002 for arg in args: 1003 if value is None: 1004 value = arg 1005 else: 1006 value += " " + arg 1007 1008 os.environ["NUITKA_EXTRA_OPTIONS"] = value 1009 1010 return old_value 1011 1012 1013@contextmanager 1014def withExtendedExtraOptions(*args): 1015 assert args 1016 1017 old_value = addExtendedExtraOptions(*args) 1018 1019 yield 1020 1021 if old_value is None: 1022 del os.environ["NUITKA_EXTRA_OPTIONS"] 1023 else: 1024 os.environ["NUITKA_EXTRA_OPTIONS"] = old_value 1025 1026 1027def indentedCode(codes, count): 1028 """Indent code, used for generating test codes.""" 1029 return "\n".join(" " * count + line if line else "" for line in codes) 1030 1031 1032def convertToPython(doctests, line_filter=None): 1033 """Convert give doctest string to static Python code.""" 1034 # This is convoluted, but it just needs to work, pylint: disable=too-many-branches 1035 1036 import doctest 1037 1038 code = doctest.script_from_examples(doctests) 1039 1040 if code.endswith("\n"): 1041 code += "#\n" 1042 else: 1043 assert False 1044 1045 output = [] 1046 inside = False 1047 1048 def getPrintPrefixed(evaluated, line_number): 1049 try: 1050 node = ast.parse(evaluated.lstrip(), "eval") 1051 except SyntaxError: 1052 return evaluated 1053 1054 if node.body[0].__class__.__name__ == "Expr": 1055 count = 0 1056 1057 while evaluated.startswith(" " * count): 1058 count += 1 1059 1060 if sys.version_info < (3,): 1061 modified = (count - 1) * " " + "print " + evaluated 1062 return ( 1063 (count - 1) * " " 1064 + ("print 'Line %d'" % line_number) 1065 + "\n" 1066 + modified 1067 ) 1068 else: 1069 modified = (count - 1) * " " + "print(" + evaluated + "\n)\n" 1070 return ( 1071 (count - 1) * " " 1072 + ("print('Line %d'" % line_number) 1073 + ")\n" 1074 + modified 1075 ) 1076 else: 1077 return evaluated 1078 1079 def getTried(evaluated, line_number): 1080 if sys.version_info < (3,): 1081 return """ 1082try: 1083%(evaluated)s 1084except Exception as __e: 1085 print "Occurred", type(__e), __e 1086""" % { 1087 "evaluated": indentedCode( 1088 getPrintPrefixed(evaluated, line_number).split("\n"), 4 1089 ) 1090 } 1091 else: 1092 return """ 1093try: 1094%(evaluated)s 1095except Exception as __e: 1096 print("Occurred", type(__e), __e) 1097""" % { 1098 "evaluated": indentedCode( 1099 getPrintPrefixed(evaluated, line_number).split("\n"), 4 1100 ) 1101 } 1102 1103 def isOpener(evaluated): 1104 evaluated = evaluated.lstrip() 1105 1106 if evaluated == "": 1107 return False 1108 1109 return evaluated.split()[0] in ( 1110 "def", 1111 "with", 1112 "class", 1113 "for", 1114 "while", 1115 "try:", 1116 "except", 1117 "except:", 1118 "finally:", 1119 "else:", 1120 ) 1121 1122 chunk = None 1123 for line_number, line in enumerate(code.split("\n")): 1124 # print "->", inside, line 1125 1126 if line_filter is not None and line_filter(line): 1127 continue 1128 1129 if inside and line and line[0].isalnum() and not isOpener(line): 1130 output.append(getTried("\n".join(chunk), line_number)) 1131 1132 chunk = [] 1133 inside = False 1134 1135 if inside and not (line.startswith("#") and line.find("SyntaxError:") != -1): 1136 chunk.append(line) 1137 elif line.startswith("#"): 1138 if line.find("SyntaxError:") != -1: 1139 # print "Syntax error detected" 1140 1141 if inside: 1142 # print "Dropping chunk", chunk 1143 1144 chunk = [] 1145 inside = False 1146 else: 1147 del output[-1] 1148 elif isOpener(line): 1149 inside = True 1150 chunk = [line] 1151 elif line.strip() == "": 1152 output.append(line) 1153 else: 1154 output.append(getTried(line, line_number)) 1155 1156 return "\n".join(output).rstrip() + "\n" 1157 1158 1159def compileLibraryPath(search_mode, path, stage_dir, decide, action): 1160 my_print("Checking standard library path:", path) 1161 1162 for root, dirnames, filenames in os.walk(path): 1163 dirnames_to_remove = [dirname for dirname in dirnames if "-" in dirname] 1164 1165 for dirname in dirnames_to_remove: 1166 dirnames.remove(dirname) 1167 1168 dirnames.sort() 1169 1170 filenames = [filename for filename in filenames if decide(root, filename)] 1171 1172 for filename in sorted(filenames): 1173 if not search_mode.consider(root, filename): 1174 continue 1175 1176 full_path = os.path.join(root, filename) 1177 1178 my_print(full_path, ":", end=" ") 1179 sys.stdout.flush() 1180 1181 action(stage_dir, path, full_path) 1182 1183 1184def compileLibraryTest(search_mode, stage_dir, decide, action): 1185 if not os.path.exists(stage_dir): 1186 os.makedirs(stage_dir) 1187 1188 my_dirname = os.path.join(os.path.dirname(__file__), "../../..") 1189 my_dirname = os.path.normpath(my_dirname) 1190 1191 paths = [path for path in sys.path if not path.startswith(my_dirname)] 1192 1193 my_print("Using standard library paths:") 1194 for path in paths: 1195 my_print(path) 1196 1197 for path in paths: 1198 print("Checking path:", path) 1199 compileLibraryPath( 1200 search_mode=search_mode, 1201 path=path, 1202 stage_dir=stage_dir, 1203 decide=decide, 1204 action=action, 1205 ) 1206 1207 search_mode.finish() 1208 1209 1210def run_async(coro): 1211 """Execute a coroutine until it's done.""" 1212 1213 values = [] 1214 result = None 1215 while True: 1216 try: 1217 values.append(coro.send(None)) 1218 except StopIteration as ex: 1219 result = ex.args[0] if ex.args else None 1220 break 1221 return values, result 1222 1223 1224def async_iterate(g): 1225 """Execute async generator until it's done.""" 1226 1227 # Test code for Python3, catches all kinds of exceptions. 1228 # pylint: disable=broad-except 1229 1230 # Also Python3 only, pylint: disable=I0021,undefined-variable 1231 1232 res = [] 1233 while True: 1234 try: 1235 g.__anext__().__next__() 1236 except StopAsyncIteration: 1237 res.append("STOP") 1238 break 1239 except StopIteration as ex: 1240 if ex.args: 1241 res.append("ex arg %s" % ex.args[0]) 1242 else: 1243 res.append("EMPTY StopIteration") 1244 break 1245 except Exception as ex: 1246 res.append(str(type(ex))) 1247 1248 return res 1249 1250 1251def getTestingCacheDir(): 1252 cache_dir = getCacheDir() 1253 1254 result = os.path.join(cache_dir, "tests_state") 1255 makePath(result) 1256 return result 1257 1258 1259def getTestingCPythonOutputsCacheDir(): 1260 cache_dir = getCacheDir() 1261 1262 result = os.path.join( 1263 cache_dir, "cpython_outputs", os.environ.get("NUITKA_TEST_SUITE", "") 1264 ) 1265 1266 makePath(result) 1267 return result 1268 1269 1270def scanDirectoryForTestCases(dirname, template_context=None): 1271 filenames = os.listdir(dirname) 1272 1273 filenames = [ 1274 filename 1275 for filename in filenames 1276 if (filename.endswith(".py") and not filename + ".j2" in filenames) 1277 or filename.endswith(".j2") 1278 ] 1279 1280 # Jinja2 environment is optional. 1281 env = None 1282 1283 for filename in sorted(filenames): 1284 if not decideFilenameVersionSkip(filename): 1285 continue 1286 1287 if filename.endswith(".j2"): 1288 # Needs to be a dictionary with template arguments. 1289 assert template_context is not None 1290 1291 if env is None: 1292 import jinja2 1293 1294 env = jinja2.Environment( 1295 loader=jinja2.FileSystemLoader("."), 1296 trim_blocks=True, 1297 lstrip_blocks=True, 1298 ) 1299 env.undefined = jinja2.StrictUndefined 1300 1301 template = env.get_template(filename) 1302 1303 code = template.render(name=template.name, **template_context) 1304 1305 filename = filename[:-3] 1306 with open(filename, "w") as output: 1307 output.write( 1308 "'''Automatically generated test, not part of releases or git.\n\n'''\n" 1309 ) 1310 1311 output.write(code) 1312 1313 yield filename 1314 1315 1316def setupCacheHashSalt(test_code_path): 1317 assert os.path.exists(test_code_path) 1318 1319 if os.path.exists(os.path.join(test_code_path, ".git")): 1320 git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path] 1321 1322 process = subprocess.Popen( 1323 args=git_cmd, 1324 stdin=getNullInput(), 1325 stdout=subprocess.PIPE, 1326 stderr=subprocess.PIPE, 1327 ) 1328 1329 stdout_git, stderr_git = process.communicate() 1330 assert process.returncode == 0, stderr_git 1331 1332 salt_value = hashlib.md5(stdout_git) 1333 else: 1334 salt_value = hashlib.md5() 1335 1336 for filename in getFileList(test_code_path): 1337 if filename.endswith(".py"): 1338 salt_value.update(getFileContents(filename, mode="rb")) 1339 1340 os.environ["NUITKA_HASH_SALT"] = salt_value.hexdigest() 1341 1342 1343def displayFolderContents(name, path): 1344 test_logger.info("Listing of %s %r:" % (name, path)) 1345 1346 if os.path.exists(path): 1347 if os.name == "nt": 1348 command = "dir /b /s /a:-D %s" % path 1349 else: 1350 command = "ls -Rla %s" % path 1351 1352 os.system(command) 1353 else: 1354 test_logger.info("Does not exist.") 1355 1356 1357def displayFileContents(name, path): 1358 test_logger.info("Contents of %s %r:" % (name, path)) 1359 1360 if os.path.exists(path): 1361 for line in getFileContentByLine(path): 1362 my_print(line) 1363 else: 1364 test_logger.info("Does not exist.") 1365 1366 1367def someGenerator(): 1368 yield 1 1369 yield 2 1370 yield 3 1371 1372 1373def someGeneratorRaising(): 1374 yield 1 1375 raise TypeError(2) 1376 1377 1378# checks requirements needed to run each test module, according to the specified special comment 1379# special comments are in the following formats: 1380# "# nuitka-skip-unless-expression: expression to be evaluated" 1381# OR 1382# "# nuitka-skip-unless-imports: module1,module2,..." 1383def checkRequirements(filename): 1384 for line in readSourceCodeFromFilename(None, filename).splitlines(): 1385 if line.startswith("# nuitka-skip-unless-"): 1386 if line[21:33] == "expression: ": 1387 expression = line[33:] 1388 with open(os.devnull, "w") as devnull: 1389 result = subprocess.call( 1390 ( 1391 os.environ["PYTHON"], 1392 "-c", 1393 "import sys, os; sys.exit(not bool(%s))" % expression, 1394 ), 1395 stdout=devnull, 1396 stderr=subprocess.STDOUT, 1397 ) 1398 if result != 0: 1399 return (False, "Expression '%s' evaluated to false" % expression) 1400 1401 elif line[21:30] == "imports: ": 1402 imports_needed = line[30:].rstrip().split(",") 1403 for i in imports_needed: 1404 if not hasModule(i): 1405 return ( 1406 False, 1407 i 1408 + " not installed for this Python version, but test needs it", 1409 ) 1410 # default return value 1411 return (True, "") 1412 1413 1414class DelayedExecutionThread(threading.Thread): 1415 def __init__(self, timeout, func): 1416 threading.Thread.__init__(self) 1417 self.timeout = timeout 1418 1419 self.func = func 1420 1421 def run(self): 1422 time.sleep(self.timeout) 1423 self.func() 1424 1425 1426def executeAfterTimePassed(timeout, func): 1427 alarm = DelayedExecutionThread(timeout=timeout, func=func) 1428 alarm.start() 1429 1430 1431def killProcess(name, pid): 1432 """Kill a process in a portable way. 1433 1434 Right now SIGINT is used, unclear what to do on Windows 1435 with Python2 or non-related processes. 1436 """ 1437 1438 if str is bytes and os.name == "nt": 1439 test_logger.info("Using taskkill on test process %r." % name) 1440 os.system("taskkill.exe /PID %d" % pid) 1441 else: 1442 test_logger.info("Killing test process %r." % name) 1443 os.kill(pid, signal.SIGINT) 1444 1445 1446def checkLoadedFileAccesses(loaded_filenames, current_dir): 1447 # Many details to consider, pylint: disable=too-many-branches,too-many-statements 1448 1449 current_dir = os.path.normpath(current_dir) 1450 current_dir = os.path.normcase(current_dir) 1451 current_dir_ext = os.path.normcase(getExternalUsePath(current_dir)) 1452 1453 illegal_accesses = [] 1454 1455 for loaded_filename in loaded_filenames: 1456 orig_loaded_filename = loaded_filename 1457 1458 loaded_filename = os.path.normpath(loaded_filename) 1459 loaded_filename = os.path.normcase(loaded_filename) 1460 loaded_basename = os.path.basename(loaded_filename) 1461 1462 if os.name == "nt": 1463 if areSamePaths( 1464 os.path.dirname(loaded_filename), 1465 os.path.normpath(os.path.join(os.environ["SYSTEMROOT"], "System32")), 1466 ): 1467 continue 1468 if areSamePaths( 1469 os.path.dirname(loaded_filename), 1470 os.path.normpath(os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")), 1471 ): 1472 continue 1473 1474 if r"windows\winsxs" in loaded_filename: 1475 continue 1476 1477 # Github actions have these in PATH overriding SYSTEMROOT 1478 if r"windows performance toolkit" in loaded_filename: 1479 continue 1480 if r"powershell" in loaded_filename: 1481 continue 1482 if r"azure dev spaces cli" in loaded_filename: 1483 continue 1484 if r"tortoisesvn" in loaded_filename: 1485 continue 1486 1487 if loaded_filename.startswith(current_dir): 1488 continue 1489 1490 if loaded_filename.startswith(os.path.abspath(current_dir)): 1491 continue 1492 1493 if loaded_filename.startswith(current_dir_ext): 1494 continue 1495 1496 ignore = True 1497 for ignored_dir in ( 1498 # System configuration is OK 1499 "/etc", 1500 "/usr/etc", 1501 "/usr/local/etc", 1502 # Runtime user state and kernel information is OK. 1503 "/proc", 1504 "/dev", 1505 "/run", 1506 "/sys", 1507 "/tmp", 1508 # Locals may of course be loaded. 1509 "/usr/lib/locale", 1510 "/usr/share/locale", 1511 "/usr/share/X11/locale", 1512 # Themes may of course be loaded. 1513 "/usr/share/themes", 1514 # Terminal info files are OK too. 1515 "/lib/terminfo", 1516 ): 1517 if isPathBelowOrSameAs(ignored_dir, loaded_filename): 1518 ignore = False 1519 break 1520 if not ignore: 1521 continue 1522 1523 # Themes may of course be loaded. 1524 if loaded_filename.startswith("/usr/share/themes"): 1525 continue 1526 if "gtk" in loaded_filename and "/engines/" in loaded_filename: 1527 continue 1528 1529 if loaded_filename in ( 1530 "/usr", 1531 "/usr/local", 1532 "/usr/local/lib", 1533 "/usr/share", 1534 "/usr/local/share", 1535 "/usr/lib64", 1536 ): 1537 continue 1538 1539 # TCL/tk for tkinter for non-Windows is OK. 1540 if loaded_filename.startswith( 1541 ( 1542 "/usr/lib/tcltk/", 1543 "/usr/share/tcltk/", 1544 "/usr/lib/tcl/", 1545 "/usr/lib64/tcl/", 1546 ) 1547 ): 1548 continue 1549 if loaded_filename in ( 1550 "/usr/lib/tcltk", 1551 "/usr/share/tcltk", 1552 "/usr/lib/tcl", 1553 "/usr/lib64/tcl", 1554 ): 1555 continue 1556 1557 if loaded_filename in ( 1558 "/lib", 1559 "/lib64", 1560 "/lib/sse2", 1561 "/lib/tls", 1562 "/lib64/tls", 1563 "/usr/lib/sse2", 1564 "/usr/lib/tls", 1565 "/usr/lib64/tls", 1566 ): 1567 continue 1568 1569 if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"): 1570 continue 1571 if loaded_filename in ( 1572 "/usr/share/tcl8.6/init.tcl", 1573 "/usr/share/tcl8.5/init.tcl", 1574 ): 1575 continue 1576 if loaded_filename in ( 1577 "/usr/share/tcl8.6/encoding", 1578 "/usr/share/tcl8.5/encoding", 1579 ): 1580 continue 1581 1582 # System SSL config on Linux. TODO: Should this not be included and 1583 # read from dist folder. 1584 if loaded_basename == "openssl.cnf": 1585 continue 1586 1587 # Taking these from system is harmless and desirable 1588 if loaded_basename.startswith(("libz.so", "libgcc_s.so")): 1589 continue 1590 1591 # System C libraries are to be expected. 1592 if loaded_basename.startswith( 1593 ( 1594 "ld-linux-x86-64.so", 1595 "libc.so.", 1596 "libpthread.so.", 1597 "libm.so.", 1598 "libdl.so.", 1599 "libBrokenLocale.so.", 1600 "libSegFault.so", 1601 "libanl.so.", 1602 "libcidn.so.", 1603 "libcrypt.so.", 1604 "libmemusage.so", 1605 "libmvec.so.", 1606 "libnsl.so.", 1607 "libnss_compat.so.", 1608 "libnss_db.so.", 1609 "libnss_dns.so.", 1610 "libnss_files.so.", 1611 "libnss_hesiod.so.", 1612 "libnss_nis.so.", 1613 "libnss_nisplus.so.", 1614 "libpcprofile.so", 1615 "libresolv.so.", 1616 "librt.so.", 1617 "libthread_db-1.0.so", 1618 "libthread_db.so.", 1619 "libutil.so.", 1620 ) 1621 ): 1622 continue 1623 1624 # System C++ standard library is also OK. 1625 if loaded_basename.startswith("libstdc++."): 1626 continue 1627 1628 # Curses library is OK from system too. 1629 if loaded_basename.startswith("libtinfo.so."): 1630 continue 1631 1632 # Loaded by C library potentially for DNS lookups. 1633 if loaded_basename.startswith( 1634 ( 1635 "libnss_", 1636 "libnsl", 1637 # Some systems load a lot more, this is CentOS 7 on OBS 1638 "libattr.so.", 1639 "libbz2.so.", 1640 "libcap.so.", 1641 "libdw.so.", 1642 "libelf.so.", 1643 "liblzma.so.", 1644 # Some systems load a lot more, this is Fedora 26 on OBS 1645 "libselinux.so.", 1646 "libpcre.so.", 1647 # And this is Fedora 29 on OBS 1648 "libblkid.so.", 1649 "libmount.so.", 1650 "libpcre2-8.so.", 1651 # CentOS 8 on OBS 1652 "libuuid.so.", 1653 ) 1654 ): 1655 continue 1656 1657 # Loaded by dtruss on macOS X. 1658 if loaded_filename.startswith("/usr/lib/dtrace/"): 1659 continue 1660 1661 # Loaded by cowbuilder and pbuilder on Debian 1662 if loaded_basename == ".ilist": 1663 continue 1664 if "cowdancer" in loaded_filename: 1665 continue 1666 if "eatmydata" in loaded_filename: 1667 continue 1668 1669 # Loading from home directories is OK too. 1670 if ( 1671 loaded_filename.startswith("/home/") 1672 or loaded_filename.startswith("/data/") 1673 or loaded_filename.startswith("/root/") 1674 or loaded_filename in ("/home", "/data", "/root") 1675 ): 1676 continue 1677 1678 # For Debian builders, /build is OK too. 1679 if loaded_filename.startswith("/build/") or loaded_filename == "/build": 1680 continue 1681 1682 # TODO: Unclear, loading gconv from filesystem of installed system 1683 # may be OK or not. I think it should be. 1684 if loaded_basename == "gconv-modules.cache": 1685 continue 1686 if "/gconv/" in loaded_filename: 1687 continue 1688 if loaded_basename.startswith("libicu"): 1689 continue 1690 if loaded_filename.startswith("/usr/share/icu/"): 1691 continue 1692 1693 # Loading from caches is OK. 1694 if loaded_filename.startswith("/var/cache/"): 1695 continue 1696 1697 # At least Python3.7 considers the default Python3 path and checks it. 1698 if loaded_filename == "/usr/bin/python3": 1699 continue 1700 1701 # Accessing the versioned Python3.x binary is also happening. 1702 if loaded_filename in ( 1703 "/usr/bin/python3." + version for version in ("5", "6", "7", "8", "9", "10") 1704 ): 1705 continue 1706 1707 binary_path = _python_executable 1708 1709 found = False 1710 while binary_path: 1711 if loaded_filename == binary_path: 1712 found = True 1713 break 1714 1715 if binary_path == os.path.dirname(binary_path): 1716 break 1717 1718 binary_path = os.path.dirname(binary_path) 1719 1720 if loaded_filename == os.path.join( 1721 binary_path, 1722 "python" + ("%d%d" % (_python_version[0], _python_version[1])), 1723 ): 1724 found = True 1725 break 1726 1727 if found: 1728 continue 1729 1730 lib_prefix_dir = "/usr/lib/python%d.%s" % ( 1731 _python_version[0], 1732 _python_version[1], 1733 ) 1734 1735 # PySide accesses its directory. 1736 if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"): 1737 continue 1738 1739 # GTK accesses package directories only. 1740 if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gtk-2.0/gtk"): 1741 continue 1742 if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"): 1743 continue 1744 if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gtk-2.0/gio"): 1745 continue 1746 if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"): 1747 continue 1748 1749 # PyQt5 seems to do this, but won't use contents then. 1750 if loaded_filename in ( 1751 "/usr/lib/qt5/plugins", 1752 "/usr/lib/qt5", 1753 "/usr/lib64/qt5/plugins", 1754 "/usr/lib64/qt5", 1755 "/usr/lib/x86_64-linux-gnu/qt5/plugins", 1756 "/usr/lib/x86_64-linux-gnu/qt5", 1757 "/usr/lib/x86_64-linux-gnu", 1758 "/usr/lib", 1759 ): 1760 continue 1761 1762 # Can look at the interpreters of the system. 1763 if loaded_basename in "python3": 1764 continue 1765 if loaded_basename in ( 1766 "python%s" + supported_version 1767 for supported_version in ( 1768 getSupportedPythonVersions() + getPartiallySupportedPythonVersions() 1769 ) 1770 ): 1771 continue 1772 1773 # Current Python executable can actually be a symlink and 1774 # the real executable which it points to will be on the 1775 # loaded_filenames list. This is all fine, let's ignore it. 1776 # Also, because the loaded_filename can be yet another symlink 1777 # (this is weird, but it's true), let's better resolve its real 1778 # path too. 1779 if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable): 1780 continue 1781 1782 # Accessing SE-Linux is OK. 1783 if loaded_filename in ("/sys/fs/selinux", "/selinux"): 1784 continue 1785 1786 # Looking at device is OK. 1787 if loaded_filename.startswith("/sys/devices/"): 1788 continue 1789 1790 # Allow reading time zone info of local system. 1791 if loaded_filename.startswith("/usr/share/zoneinfo/"): 1792 continue 1793 1794 # The access to .pth files has no effect. 1795 if loaded_filename.endswith(".pth"): 1796 continue 1797 1798 # Looking at site-package dir alone is alone. 1799 if loaded_filename.endswith(("site-packages", "dist-packages")): 1800 continue 1801 1802 # QtNetwork insist on doing this it seems. 1803 if loaded_basename.startswith(("libcrypto.so", "libssl.so")): 1804 continue 1805 1806 # macOS uses these: 1807 if loaded_basename in ( 1808 "libcrypto.1.0.0.dylib", 1809 "libssl.1.0.0.dylib", 1810 "libcrypto.1.1.dylib", 1811 ): 1812 continue 1813 1814 # Linux onefile uses this 1815 if loaded_basename.startswith("libfuse.so."): 1816 continue 1817 1818 # MSVC run time DLLs, due to SxS come from system. 1819 if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"): 1820 continue 1821 1822 illegal_accesses.append(orig_loaded_filename) 1823 1824 return illegal_accesses 1825