1#!/usr/bin/env python3 2 3# Copyright 2012-2021 The Meson development team 4 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8 9# http://www.apache.org/licenses/LICENSE-2.0 10 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17# Work around some pathlib bugs... 18from mesonbuild import _pathlib 19import sys 20sys.modules['pathlib'] = _pathlib 21 22from concurrent.futures import ProcessPoolExecutor, CancelledError 23from enum import Enum 24from io import StringIO 25from pathlib import Path, PurePath 26import argparse 27import functools 28import itertools 29import json 30import multiprocessing 31import os 32import re 33import shlex 34import shutil 35import signal 36import subprocess 37import tempfile 38import time 39import typing as T 40import xml.etree.ElementTree as ET 41import collections 42 43from mesonbuild import build 44from mesonbuild import environment 45from mesonbuild import compilers 46from mesonbuild import mesonlib 47from mesonbuild import mlog 48from mesonbuild import mtest 49from mesonbuild.compilers import compiler_from_language, detect_objc_compiler, detect_objcpp_compiler 50from mesonbuild.build import ConfigurationData 51from mesonbuild.mesonlib import MachineChoice, Popen_safe, TemporaryDirectoryWinProof 52from mesonbuild.mlog import blue, bold, cyan, green, red, yellow, normal_green 53from mesonbuild.coredata import backendlist, version as meson_version 54from mesonbuild.mesonmain import setup_vsenv 55from mesonbuild.modules.python import PythonExternalProgram 56from run_tests import get_fake_options, run_configure, get_meson_script 57from run_tests import get_backend_commands, get_backend_args_for_dir, Backend 58from run_tests import ensure_backend_detects_changes 59from run_tests import guess_backend 60 61if T.TYPE_CHECKING: 62 from types import FrameType 63 from mesonbuild.environment import Environment 64 from mesonbuild._typing import Protocol 65 from concurrent.futures import Future 66 from mesonbuild.modules.python import PythonIntrospectionDict 67 68 class CompilerArgumentType(Protocol): 69 cross_file: str 70 native_file: str 71 use_tmpdir: bool 72 73 74 class ArgumentType(CompilerArgumentType): 75 76 """Typing information for command line arguments.""" 77 78 extra_args: T.List[str] 79 backend: str 80 num_workers: int 81 failfast: bool 82 no_unittests: bool 83 only: T.List[str] 84 85ALL_TESTS = ['cmake', 'common', 'native', 'warning-meson', 'failing-meson', 'failing-build', 'failing-test', 86 'keyval', 'platform-osx', 'platform-windows', 'platform-linux', 87 'java', 'C#', 'vala', 'cython', 'rust', 'd', 'objective c', 'objective c++', 88 'fortran', 'swift', 'cuda', 'python3', 'python', 'fpga', 'frameworks', 'nasm', 'wasm', 89 ] 90 91 92class BuildStep(Enum): 93 configure = 1 94 build = 2 95 test = 3 96 install = 4 97 clean = 5 98 validate = 6 99 100 101class TestResult(BaseException): 102 def __init__(self, cicmds: T.List[str]) -> None: 103 self.msg = '' # empty msg indicates test success 104 self.stdo = '' 105 self.stde = '' 106 self.mlog = '' 107 self.cicmds = cicmds 108 self.conftime: float = 0 109 self.buildtime: float = 0 110 self.testtime: float = 0 111 112 def add_step(self, step: BuildStep, stdo: str, stde: str, mlog: str = '', time: float = 0) -> None: 113 self.step = step 114 self.stdo += stdo 115 self.stde += stde 116 self.mlog += mlog 117 if step == BuildStep.configure: 118 self.conftime = time 119 elif step == BuildStep.build: 120 self.buildtime = time 121 elif step == BuildStep.test: 122 self.testtime = time 123 124 def fail(self, msg: str) -> None: 125 self.msg = msg 126 127python = PythonExternalProgram(sys.executable) 128python.sanity() 129 130class InstalledFile: 131 def __init__(self, raw: T.Dict[str, str]): 132 self.path = raw['file'] 133 self.typ = raw['type'] 134 self.platform = raw.get('platform', None) 135 self.language = raw.get('language', 'c') # type: str 136 137 version = raw.get('version', '') # type: str 138 if version: 139 self.version = version.split('.') # type: T.List[str] 140 else: 141 # split on '' will return [''], we want an empty list though 142 self.version = [] 143 144 def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Path]: 145 p = Path(self.path) 146 canonical_compiler = compiler 147 if ((compiler in ['clang-cl', 'intel-cl']) or 148 (env.machines.host.is_windows() and compiler in {'pgi', 'dmd', 'ldc'})): 149 canonical_compiler = 'msvc' 150 151 python_suffix = python.info['suffix'] 152 153 has_pdb = False 154 if self.language in {'c', 'cpp'}: 155 has_pdb = canonical_compiler == 'msvc' 156 elif self.language == 'd': 157 # dmd's optlink does not genearte pdb iles 158 has_pdb = env.coredata.compilers.host['d'].linker.id in {'link', 'lld-link'} 159 160 # Abort if the platform does not match 161 matches = { 162 'msvc': canonical_compiler == 'msvc', 163 'gcc': canonical_compiler != 'msvc', 164 'cygwin': env.machines.host.is_cygwin(), 165 '!cygwin': not env.machines.host.is_cygwin(), 166 }.get(self.platform or '', True) 167 if not matches: 168 return None 169 170 # Handle the different types 171 if self.typ in {'py_implib', 'python_lib', 'python_file'}: 172 val = p.as_posix() 173 val = val.replace('@PYTHON_PLATLIB@', python.platlib) 174 val = val.replace('@PYTHON_PURELIB@', python.purelib) 175 p = Path(val) 176 if self.typ == 'python_file': 177 return p 178 if self.typ == 'python_lib': 179 return p.with_suffix(python_suffix) 180 if self.typ in ['file', 'dir']: 181 return p 182 elif self.typ == 'shared_lib': 183 if env.machines.host.is_windows() or env.machines.host.is_cygwin(): 184 # Windows only has foo.dll and foo-X.dll 185 if len(self.version) > 1: 186 return None 187 if self.version: 188 p = p.with_name('{}-{}'.format(p.name, self.version[0])) 189 return p.with_suffix('.dll') 190 191 p = p.with_name(f'lib{p.name}') 192 if env.machines.host.is_darwin(): 193 # MacOS only has libfoo.dylib and libfoo.X.dylib 194 if len(self.version) > 1: 195 return None 196 197 # pathlib.Path.with_suffix replaces, not appends 198 suffix = '.dylib' 199 if self.version: 200 suffix = '.{}{}'.format(self.version[0], suffix) 201 else: 202 # pathlib.Path.with_suffix replaces, not appends 203 suffix = '.so' 204 if self.version: 205 suffix = '{}.{}'.format(suffix, '.'.join(self.version)) 206 return p.with_suffix(suffix) 207 elif self.typ == 'exe': 208 if env.machines.host.is_windows() or env.machines.host.is_cygwin(): 209 return p.with_suffix('.exe') 210 elif self.typ == 'pdb': 211 if self.version: 212 p = p.with_name('{}-{}'.format(p.name, self.version[0])) 213 return p.with_suffix('.pdb') if has_pdb else None 214 elif self.typ in {'implib', 'implibempty', 'py_implib'}: 215 if env.machines.host.is_windows() and canonical_compiler == 'msvc': 216 # only MSVC doesn't generate empty implibs 217 if self.typ == 'implibempty' and compiler == 'msvc': 218 return None 219 return p.parent / (re.sub(r'^lib', '', p.name) + '.lib') 220 elif env.machines.host.is_windows() or env.machines.host.is_cygwin(): 221 if self.typ == 'py_implib': 222 p = p.with_suffix(python_suffix) 223 return p.with_suffix('.dll.a') 224 else: 225 return None 226 elif self.typ == 'expr': 227 return Path(platform_fix_name(p.as_posix(), canonical_compiler, env)) 228 else: 229 raise RuntimeError(f'Invalid installed file type {self.typ}') 230 231 return p 232 233 def get_paths(self, compiler: str, env: environment.Environment, installdir: Path) -> T.List[Path]: 234 p = self.get_path(compiler, env) 235 if not p: 236 return [] 237 if self.typ == 'dir': 238 abs_p = installdir / p 239 if not abs_p.exists(): 240 raise RuntimeError(f'{p} does not exist') 241 if not abs_p.is_dir(): 242 raise RuntimeError(f'{p} is not a directory') 243 return [x.relative_to(installdir) for x in abs_p.rglob('*') if x.is_file() or x.is_symlink()] 244 else: 245 return [p] 246 247@functools.total_ordering 248class TestDef: 249 def __init__(self, path: Path, name: T.Optional[str], args: T.List[str], skip: bool = False): 250 self.category = path.parts[1] 251 self.path = path 252 self.name = name 253 self.args = args 254 self.skip = skip 255 self.env = os.environ.copy() 256 self.installed_files = [] # type: T.List[InstalledFile] 257 self.do_not_set_opts = [] # type: T.List[str] 258 self.stdout = [] # type: T.List[T.Dict[str, str]] 259 self.skip_expected = False 260 261 # Always print a stack trace for Meson exceptions 262 self.env['MESON_FORCE_BACKTRACE'] = '1' 263 264 def __repr__(self) -> str: 265 return '<{}: {:<48} [{}: {}] -- {}>'.format(type(self).__name__, str(self.path), self.name, self.args, self.skip) 266 267 def display_name(self) -> mlog.TV_LoggableList: 268 # Remove the redundant 'test cases' part 269 section, id = self.path.parts[1:3] 270 res: mlog.TV_LoggableList = [f'{section}:', bold(id)] 271 if self.name: 272 res += [f' ({self.name})'] 273 return res 274 275 def __lt__(self, other: object) -> bool: 276 if isinstance(other, TestDef): 277 # None is not sortable, so replace it with an empty string 278 s_id = int(self.path.name.split(' ')[0]) 279 o_id = int(other.path.name.split(' ')[0]) 280 return (s_id, self.path, self.name or '') < (o_id, other.path, other.name or '') 281 return NotImplemented 282 283failing_logs: T.List[str] = [] 284print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ 285under_ci = 'CI' in os.environ 286ci_jobname = os.environ.get('MESON_CI_JOBNAME', None) 287do_debug = under_ci or print_debug 288no_meson_log_msg = 'No meson-log.txt found.' 289 290host_c_compiler: T.Optional[str] = None 291compiler_id_map: T.Dict[str, str] = {} 292tool_vers_map: T.Dict[str, str] = {} 293 294compile_commands: T.List[str] 295clean_commands: T.List[str] 296test_commands: T.List[str] 297install_commands: T.List[str] 298uninstall_commands: T.List[str] 299 300backend: 'Backend' 301backend_flags: T.List[str] 302 303stop: bool = False 304is_worker_process: bool = False 305 306# Let's have colors in our CI output 307if under_ci: 308 def _ci_colorize_console() -> bool: 309 return not is_worker_process 310 311 mlog.colorize_console = _ci_colorize_console 312 313class StopException(Exception): 314 def __init__(self) -> None: 315 super().__init__('Stopped by user') 316 317def stop_handler(signal: signal.Signals, frame: T.Optional['FrameType']) -> None: 318 global stop 319 stop = True 320signal.signal(signal.SIGINT, stop_handler) 321signal.signal(signal.SIGTERM, stop_handler) 322 323def setup_commands(optbackend: str) -> None: 324 global do_debug, backend, backend_flags 325 global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands 326 backend, backend_flags = guess_backend(optbackend, shutil.which('msbuild')) 327 compile_commands, clean_commands, test_commands, install_commands, \ 328 uninstall_commands = get_backend_commands(backend, do_debug) 329 330# TODO try to eliminate or at least reduce this function 331def platform_fix_name(fname: str, canonical_compiler: str, env: environment.Environment) -> str: 332 if '?lib' in fname: 333 if env.machines.host.is_windows() and canonical_compiler == 'msvc': 334 fname = re.sub(r'lib/\?lib(.*)\.', r'bin/\1.', fname) 335 fname = re.sub(r'/\?lib/', r'/bin/', fname) 336 elif env.machines.host.is_windows(): 337 fname = re.sub(r'lib/\?lib(.*)\.', r'bin/lib\1.', fname) 338 fname = re.sub(r'\?lib(.*)\.dll$', r'lib\1.dll', fname) 339 fname = re.sub(r'/\?lib/', r'/bin/', fname) 340 elif env.machines.host.is_cygwin(): 341 fname = re.sub(r'lib/\?lib(.*)\.so$', r'bin/cyg\1.dll', fname) 342 fname = re.sub(r'lib/\?lib(.*)\.', r'bin/cyg\1.', fname) 343 fname = re.sub(r'\?lib(.*)\.dll$', r'cyg\1.dll', fname) 344 fname = re.sub(r'/\?lib/', r'/bin/', fname) 345 else: 346 fname = re.sub(r'\?lib', 'lib', fname) 347 348 if fname.endswith('?so'): 349 if env.machines.host.is_windows() and canonical_compiler == 'msvc': 350 fname = re.sub(r'lib/([^/]*)\?so$', r'bin/\1.dll', fname) 351 fname = re.sub(r'/(?:lib|)([^/]*?)\?so$', r'/\1.dll', fname) 352 return fname 353 elif env.machines.host.is_windows(): 354 fname = re.sub(r'lib/([^/]*)\?so$', r'bin/\1.dll', fname) 355 fname = re.sub(r'/([^/]*?)\?so$', r'/\1.dll', fname) 356 return fname 357 elif env.machines.host.is_cygwin(): 358 fname = re.sub(r'lib/([^/]*)\?so$', r'bin/\1.dll', fname) 359 fname = re.sub(r'/lib([^/]*?)\?so$', r'/cyg\1.dll', fname) 360 fname = re.sub(r'/([^/]*?)\?so$', r'/\1.dll', fname) 361 return fname 362 elif env.machines.host.is_darwin(): 363 return fname[:-3] + '.dylib' 364 else: 365 return fname[:-3] + '.so' 366 367 return fname 368 369def validate_install(test: TestDef, installdir: Path, env: environment.Environment) -> str: 370 ret_msg = '' 371 expected_raw = [] # type: T.List[Path] 372 for i in test.installed_files: 373 try: 374 expected_raw += i.get_paths(host_c_compiler, env, installdir) 375 except RuntimeError as err: 376 ret_msg += f'Expected path error: {err}\n' 377 expected = {x: False for x in expected_raw} 378 found = [x.relative_to(installdir) for x in installdir.rglob('*') if x.is_file() or x.is_symlink()] 379 # Mark all found files as found and detect unexpected files 380 for fname in found: 381 if fname not in expected: 382 ret_msg += f'Extra file {fname} found.\n' 383 continue 384 expected[fname] = True 385 # Check if expected files were found 386 for p, f in expected.items(): 387 if not f: 388 ret_msg += f'Expected file {p} missing.\n' 389 # List dir content on error 390 if ret_msg != '': 391 ret_msg += '\nInstall dir contents:\n' 392 for p in found: 393 ret_msg += f' - {p}\n' 394 return ret_msg 395 396def log_text_file(logfile: T.TextIO, testdir: Path, result: TestResult) -> None: 397 logfile.write('%s\nstdout\n\n---\n' % testdir.as_posix()) 398 logfile.write(result.stdo) 399 logfile.write('\n\n---\n\nstderr\n\n---\n') 400 logfile.write(result.stde) 401 logfile.write('\n\n---\n\n') 402 if print_debug: 403 try: 404 print(result.stdo) 405 except UnicodeError: 406 sanitized_out = result.stdo.encode('ascii', errors='replace').decode() 407 print(sanitized_out) 408 try: 409 print(result.stde, file=sys.stderr) 410 except UnicodeError: 411 sanitized_err = result.stde.encode('ascii', errors='replace').decode() 412 print(sanitized_err, file=sys.stderr) 413 414 415def _run_ci_include(args: T.List[str]) -> str: 416 if not args: 417 return 'At least one parameter required' 418 try: 419 data = Path(args[0]).read_text(errors='ignore', encoding='utf-8') 420 return 'Included file {}:\n{}\n'.format(args[0], data) 421 except Exception: 422 return 'Failed to open {}'.format(args[0]) 423 424ci_commands = { 425 'ci_include': _run_ci_include 426} 427 428def run_ci_commands(raw_log: str) -> T.List[str]: 429 res = [] 430 for l in raw_log.splitlines(): 431 if not l.startswith('!meson_ci!/'): 432 continue 433 cmd = shlex.split(l[11:]) 434 if not cmd or cmd[0] not in ci_commands: 435 continue 436 res += ['CI COMMAND {}:\n{}\n'.format(cmd[0], ci_commands[cmd[0]](cmd[1:]))] 437 return res 438 439class OutputMatch: 440 def __init__(self, how: str, expected: str, count: int) -> None: 441 self.how = how 442 self.expected = expected 443 self.count = count 444 445 def match(self, actual: str) -> bool: 446 if self.how == "re": 447 return bool(re.match(self.expected, actual)) 448 return self.expected == actual 449 450def _compare_output(expected: T.List[T.Dict[str, str]], output: str, desc: str) -> str: 451 if expected: 452 matches: T.List[OutputMatch] = [] 453 nomatches: T.List[OutputMatch] = [] 454 for item in expected: 455 how = item.get('match', 'literal') 456 expected_line = item.get('line') 457 count = int(item.get('count', -1)) 458 459 # Simple heuristic to automatically convert path separators for 460 # Windows: 461 # 462 # Any '/' appearing before 'WARNING' or 'ERROR' (i.e. a path in a 463 # filename part of a location) is replaced with '\' (in a re: '\\' 464 # which matches a literal '\') 465 # 466 # (There should probably be a way to turn this off for more complex 467 # cases which don't fit this) 468 if mesonlib.is_windows(): 469 if how != "re": 470 sub = r'\\' 471 else: 472 sub = r'\\\\' 473 expected_line = re.sub(r'/(?=.*(WARNING|ERROR))', sub, expected_line) 474 475 m = OutputMatch(how, expected_line, count) 476 if count == 0: 477 nomatches.append(m) 478 else: 479 matches.append(m) 480 481 482 i = 0 483 for actual in output.splitlines(): 484 # Verify this line does not match any unexpected lines (item.count == 0) 485 for match in nomatches: 486 if match.match(actual): 487 return f'unexpected "{match.expected}" found in {desc}' 488 # If we matched all expected lines, continue to verify there are 489 # no unexpected line. If nomatches is empty then we are done already. 490 if i >= len(matches): 491 if not nomatches: 492 break 493 continue 494 # Check if this line match current expected line 495 match = matches[i] 496 if match.match(actual): 497 if match.count < 0: 498 # count was not specified, continue with next expected line, 499 # it does not matter if this line will be matched again or 500 # not. 501 i += 1 502 else: 503 # count was specified (must be >0), continue expecting this 504 # same line. If count reached 0 we continue with next 505 # expected line but remember that this one must not match 506 # anymore. 507 match.count -= 1 508 if match.count == 0: 509 nomatches.append(match) 510 i += 1 511 512 if i < len(matches): 513 # reached the end of output without finding expected 514 return f'expected "{matches[i].expected}" not found in {desc}' 515 516 return '' 517 518def validate_output(test: TestDef, stdo: str, stde: str) -> str: 519 return _compare_output(test.stdout, stdo, 'stdout') 520 521# There are some class variables and such that cahce 522# information. Clear all of these. The better solution 523# would be to change the code so that no state is persisted 524# but that would be a lot of work given that Meson was originally 525# coded to run as a batch process. 526def clear_internal_caches() -> None: 527 import mesonbuild.interpreterbase 528 from mesonbuild.dependencies import CMakeDependency 529 from mesonbuild.mesonlib import PerMachine 530 mesonbuild.interpreterbase.FeatureNew.feature_registry = {} 531 CMakeDependency.class_cmakeinfo = PerMachine(None, None) 532 533def run_test_inprocess(testdir: str) -> T.Tuple[int, str, str, str]: 534 old_stdout = sys.stdout 535 sys.stdout = mystdout = StringIO() 536 old_stderr = sys.stderr 537 sys.stderr = mystderr = StringIO() 538 old_cwd = os.getcwd() 539 os.chdir(testdir) 540 test_log_fname = Path('meson-logs', 'testlog.txt') 541 try: 542 returncode_test = mtest.run_with_args(['--no-rebuild']) 543 if test_log_fname.exists(): 544 test_log = test_log_fname.open(encoding='utf-8', errors='ignore').read() 545 else: 546 test_log = '' 547 returncode_benchmark = mtest.run_with_args(['--no-rebuild', '--benchmark', '--logbase', 'benchmarklog']) 548 finally: 549 sys.stdout = old_stdout 550 sys.stderr = old_stderr 551 os.chdir(old_cwd) 552 return max(returncode_test, returncode_benchmark), mystdout.getvalue(), mystderr.getvalue(), test_log 553 554# Build directory name must be the same so Ccache works over 555# consecutive invocations. 556def create_deterministic_builddir(test: TestDef, use_tmpdir: bool) -> str: 557 import hashlib 558 src_dir = test.path.as_posix() 559 if test.name: 560 src_dir += test.name 561 rel_dirname = 'b ' + hashlib.sha256(src_dir.encode(errors='ignore')).hexdigest()[0:10] 562 abs_pathname = os.path.join(tempfile.gettempdir() if use_tmpdir else os.getcwd(), rel_dirname) 563 if os.path.exists(abs_pathname): 564 mesonlib.windows_proof_rmtree(abs_pathname) 565 os.mkdir(abs_pathname) 566 return abs_pathname 567 568def format_parameter_file(file_basename: str, test: TestDef, test_build_dir: str) -> Path: 569 confdata = ConfigurationData() 570 confdata.values = {'MESON_TEST_ROOT': (str(test.path.absolute()), 'base directory of current test')} 571 572 template = test.path / (file_basename + '.in') 573 destination = Path(test_build_dir) / file_basename 574 mesonlib.do_conf_file(str(template), str(destination), confdata, 'meson') 575 576 return destination 577 578def detect_parameter_files(test: TestDef, test_build_dir: str) -> T.Tuple[Path, Path]: 579 nativefile = test.path / 'nativefile.ini' 580 crossfile = test.path / 'crossfile.ini' 581 582 if os.path.exists(str(test.path / 'nativefile.ini.in')): 583 nativefile = format_parameter_file('nativefile.ini', test, test_build_dir) 584 585 if os.path.exists(str(test.path / 'crossfile.ini.in')): 586 crossfile = format_parameter_file('crossfile.ini', test, test_build_dir) 587 588 return nativefile, crossfile 589 590# In previous python versions the global variables are lost in ProcessPoolExecutor. 591# So, we use this tuple to restore some of them 592class GlobalState(T.NamedTuple): 593 compile_commands: T.List[str] 594 clean_commands: T.List[str] 595 test_commands: T.List[str] 596 install_commands: T.List[str] 597 uninstall_commands: T.List[str] 598 599 backend: 'Backend' 600 backend_flags: T.List[str] 601 602 host_c_compiler: T.Optional[str] 603 604def run_test(test: TestDef, 605 extra_args: T.List[str], 606 should_fail: str, 607 use_tmp: bool, 608 state: T.Optional[GlobalState] = None) -> T.Optional[TestResult]: 609 # Unpack the global state 610 global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands, backend, backend_flags, host_c_compiler 611 if state is not None: 612 compile_commands, clean_commands, test_commands, install_commands, uninstall_commands, backend, backend_flags, host_c_compiler = state 613 # Store that this is a worker process 614 global is_worker_process 615 is_worker_process = True 616 # Setup the test environment 617 assert not test.skip, 'Skipped thest should not be run' 618 build_dir = create_deterministic_builddir(test, use_tmp) 619 try: 620 with TemporaryDirectoryWinProof(prefix='i ', dir=None if use_tmp else os.getcwd()) as install_dir: 621 try: 622 return _run_test(test, build_dir, install_dir, extra_args, should_fail) 623 except TestResult as r: 624 return r 625 finally: 626 mlog.shutdown() # Close the log file because otherwise Windows wets itself. 627 finally: 628 mesonlib.windows_proof_rmtree(build_dir) 629 630def _run_test(test: TestDef, 631 test_build_dir: str, 632 install_dir: str, 633 extra_args: T.List[str], 634 should_fail: str) -> TestResult: 635 gen_start = time.time() 636 # Configure in-process 637 gen_args = [] # type: T.List[str] 638 if 'prefix' not in test.do_not_set_opts: 639 gen_args += ['--prefix', 'x:/usr'] if mesonlib.is_windows() else ['--prefix', '/usr'] 640 if 'libdir' not in test.do_not_set_opts: 641 gen_args += ['--libdir', 'lib'] 642 gen_args += [test.path.as_posix(), test_build_dir] + backend_flags + extra_args 643 644 nativefile, crossfile = detect_parameter_files(test, test_build_dir) 645 646 if nativefile.exists(): 647 gen_args.extend(['--native-file', nativefile.as_posix()]) 648 if crossfile.exists(): 649 gen_args.extend(['--cross-file', crossfile.as_posix()]) 650 (returncode, stdo, stde) = run_configure(gen_args, env=test.env, catch_exception=True) 651 try: 652 logfile = Path(test_build_dir, 'meson-logs', 'meson-log.txt') 653 mesonlog = logfile.open(errors='ignore', encoding='utf-8').read() 654 except Exception: 655 mesonlog = no_meson_log_msg 656 cicmds = run_ci_commands(mesonlog) 657 testresult = TestResult(cicmds) 658 testresult.add_step(BuildStep.configure, stdo, stde, mesonlog, time.time() - gen_start) 659 output_msg = validate_output(test, stdo, stde) 660 testresult.mlog += output_msg 661 if output_msg: 662 testresult.fail('Unexpected output while configuring.') 663 return testresult 664 if should_fail == 'meson': 665 if returncode == 1: 666 return testresult 667 elif returncode != 0: 668 testresult.fail(f'Test exited with unexpected status {returncode}.') 669 return testresult 670 else: 671 testresult.fail('Test that should have failed succeeded.') 672 return testresult 673 if returncode != 0: 674 testresult.fail('Generating the build system failed.') 675 return testresult 676 builddata = build.load(test_build_dir) 677 dir_args = get_backend_args_for_dir(backend, test_build_dir) 678 679 # Build with subprocess 680 def build_step() -> None: 681 build_start = time.time() 682 pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir) 683 testresult.add_step(BuildStep.build, o, e, '', time.time() - build_start) 684 if should_fail == 'build': 685 if pc.returncode != 0: 686 raise testresult 687 testresult.fail('Test that should have failed to build succeeded.') 688 raise testresult 689 if pc.returncode != 0: 690 testresult.fail('Compiling source code failed.') 691 raise testresult 692 693 # Touch the meson.build file to force a regenerate 694 def force_regenerate() -> None: 695 ensure_backend_detects_changes(backend) 696 os.utime(str(test.path / 'meson.build')) 697 698 # just test building 699 build_step() 700 701 # test that regeneration works for build step 702 force_regenerate() 703 build_step() # TBD: assert nothing gets built after the regenerate? 704 705 # test that regeneration works for test step 706 force_regenerate() 707 708 # Test in-process 709 clear_internal_caches() 710 test_start = time.time() 711 (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir) 712 testresult.add_step(BuildStep.test, tstdo, tstde, test_log, time.time() - test_start) 713 if should_fail == 'test': 714 if returncode != 0: 715 return testresult 716 testresult.fail('Test that should have failed to run unit tests succeeded.') 717 return testresult 718 if returncode != 0: 719 testresult.fail('Running unit tests failed.') 720 return testresult 721 722 # Do installation, if the backend supports it 723 if install_commands: 724 env = test.env.copy() 725 env['DESTDIR'] = install_dir 726 # Install with subprocess 727 pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env) 728 testresult.add_step(BuildStep.install, o, e) 729 if pi.returncode != 0: 730 testresult.fail('Running install failed.') 731 return testresult 732 733 # Clean with subprocess 734 env = test.env.copy() 735 pi, o, e = Popen_safe(clean_commands + dir_args, cwd=test_build_dir, env=env) 736 testresult.add_step(BuildStep.clean, o, e) 737 if pi.returncode != 0: 738 testresult.fail('Running clean failed.') 739 return testresult 740 741 # Validate installed files 742 testresult.add_step(BuildStep.install, '', '') 743 if not install_commands: 744 return testresult 745 install_msg = validate_install(test, Path(install_dir), builddata.environment) 746 if install_msg: 747 testresult.fail('\n' + install_msg) 748 return testresult 749 750 return testresult 751 752 753# processing of test.json 'skip_*' keys, which can appear at top level, or in 754# matrix: 755def _skip_keys(test_def: T.Dict) -> T.Tuple[bool, bool]: 756 skip_expected = False 757 758 # Test is expected to skip if MESON_CI_JOBNAME contains any of the list of 759 # substrings 760 if ('skip_on_jobname' in test_def) and (ci_jobname is not None): 761 skip_expected = any(s in ci_jobname for s in test_def['skip_on_jobname']) 762 763 # Test is expected to skip if os matches 764 if 'skip_on_os' in test_def: 765 mesonenv = environment.Environment(None, None, get_fake_options('/')) 766 for skip_os in test_def['skip_on_os']: 767 if skip_os.startswith('!'): 768 if mesonenv.machines.host.system != skip_os[1:]: 769 skip_expected = True 770 else: 771 if mesonenv.machines.host.system == skip_os: 772 skip_expected = True 773 774 # Skip if environment variable is present 775 skip = False 776 if 'skip_on_env' in test_def: 777 for skip_env_var in test_def['skip_on_env']: 778 if skip_env_var in os.environ: 779 skip = True 780 781 return (skip, skip_expected) 782 783 784def load_test_json(t: TestDef, stdout_mandatory: bool) -> T.List[TestDef]: 785 all_tests: T.List[TestDef] = [] 786 test_def = {} 787 test_def_file = t.path / 'test.json' 788 if test_def_file.is_file(): 789 test_def = json.loads(test_def_file.read_text(encoding='utf-8')) 790 791 # Handle additional environment variables 792 env = {} # type: T.Dict[str, str] 793 if 'env' in test_def: 794 assert isinstance(test_def['env'], dict) 795 env = test_def['env'] 796 for key, val in env.items(): 797 val = val.replace('@ROOT@', t.path.resolve().as_posix()) 798 val = val.replace('@PATH@', t.env.get('PATH', '')) 799 env[key] = val 800 801 # Handle installed files 802 installed = [] # type: T.List[InstalledFile] 803 if 'installed' in test_def: 804 installed = [InstalledFile(x) for x in test_def['installed']] 805 806 # Handle expected output 807 stdout = test_def.get('stdout', []) 808 if stdout_mandatory and not stdout: 809 raise RuntimeError(f"{test_def_file} must contain a non-empty stdout key") 810 811 # Handle the do_not_set_opts list 812 do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str] 813 814 (t.skip, t.skip_expected) = _skip_keys(test_def) 815 816 # Skip tests if the tool requirements are not met 817 if 'tools' in test_def: 818 assert isinstance(test_def['tools'], dict) 819 for tool, vers_req in test_def['tools'].items(): 820 if tool not in tool_vers_map: 821 t.skip = True 822 elif not mesonlib.version_compare(tool_vers_map[tool], vers_req): 823 t.skip = True 824 825 # Skip the matrix code and just update the existing test 826 if 'matrix' not in test_def: 827 t.env.update(env) 828 t.installed_files = installed 829 t.do_not_set_opts = do_not_set_opts 830 t.stdout = stdout 831 return [t] 832 833 new_opt_list: T.List[T.List[T.Tuple[str, bool, bool]]] 834 835 # 'matrix; entry is present, so build multiple tests from matrix definition 836 opt_list = [] # type: T.List[T.List[T.Tuple[str, bool, bool]]] 837 matrix = test_def['matrix'] 838 assert "options" in matrix 839 for key, val in matrix["options"].items(): 840 assert isinstance(val, list) 841 tmp_opts = [] # type: T.List[T.Tuple[str, bool, bool]] 842 for i in val: 843 assert isinstance(i, dict) 844 assert "val" in i 845 846 (skip, skip_expected) = _skip_keys(i) 847 848 # Only run the test if all compiler ID's match 849 if 'compilers' in i: 850 for lang, id_list in i['compilers'].items(): 851 if lang not in compiler_id_map or compiler_id_map[lang] not in id_list: 852 skip = True 853 break 854 855 # Add an empty matrix entry 856 if i['val'] is None: 857 tmp_opts += [(None, skip, skip_expected)] 858 continue 859 860 tmp_opts += [('{}={}'.format(key, i['val']), skip, skip_expected)] 861 862 if opt_list: 863 new_opt_list = [] 864 for i in opt_list: 865 for j in tmp_opts: 866 new_opt_list += [[*i, j]] 867 opt_list = new_opt_list 868 else: 869 opt_list = [[x] for x in tmp_opts] 870 871 # Exclude specific configurations 872 if 'exclude' in matrix: 873 assert isinstance(matrix['exclude'], list) 874 new_opt_list = [] 875 for i in opt_list: 876 exclude = False 877 opt_names = [x[0] for x in i] 878 for j in matrix['exclude']: 879 ex_list = [f'{k}={v}' for k, v in j.items()] 880 if all([x in opt_names for x in ex_list]): 881 exclude = True 882 break 883 884 if not exclude: 885 new_opt_list += [i] 886 887 opt_list = new_opt_list 888 889 for i in opt_list: 890 name = ' '.join([x[0] for x in i if x[0] is not None]) 891 opts = ['-D' + x[0] for x in i if x[0] is not None] 892 skip = any([x[1] for x in i]) 893 skip_expected = any([x[2] for x in i]) 894 test = TestDef(t.path, name, opts, skip or t.skip) 895 test.env.update(env) 896 test.installed_files = installed 897 test.do_not_set_opts = do_not_set_opts 898 test.stdout = stdout 899 test.skip_expected = skip_expected or t.skip_expected 900 all_tests.append(test) 901 902 return all_tests 903 904 905def gather_tests(testdir: Path, stdout_mandatory: bool, only: T.List[str]) -> T.List[TestDef]: 906 all_tests: T.List[TestDef] = [] 907 for t in testdir.iterdir(): 908 # Filter non-tests files (dot files, etc) 909 if not t.is_dir() or t.name.startswith('.'): 910 continue 911 if only and not any(t.name.startswith(prefix) for prefix in only): 912 continue 913 test_def = TestDef(t, None, []) 914 all_tests.extend(load_test_json(test_def, stdout_mandatory)) 915 return sorted(all_tests) 916 917 918def have_d_compiler() -> bool: 919 if shutil.which("ldc2"): 920 return True 921 elif shutil.which("ldc"): 922 return True 923 elif shutil.which("gdc"): 924 return True 925 elif shutil.which("dmd"): 926 # The Windows installer sometimes produces a DMD install 927 # that exists but segfaults every time the compiler is run. 928 # Don't know why. Don't know how to fix. Skip in this case. 929 cp = subprocess.run(['dmd', '--version'], 930 stdout=subprocess.PIPE, 931 stderr=subprocess.PIPE) 932 if cp.stdout == b'': 933 return False 934 return True 935 return False 936 937def have_objc_compiler(use_tmp: bool) -> bool: 938 with TemporaryDirectoryWinProof(prefix='b ', dir=None if use_tmp else '.') as build_dir: 939 env = environment.Environment(None, build_dir, get_fake_options('/')) 940 try: 941 objc_comp = detect_objc_compiler(env, MachineChoice.HOST) 942 except mesonlib.MesonException: 943 return False 944 if not objc_comp: 945 return False 946 env.coredata.process_new_compiler('objc', objc_comp, env) 947 try: 948 objc_comp.sanity_check(env.get_scratch_dir(), env) 949 except mesonlib.MesonException: 950 return False 951 return True 952 953def have_objcpp_compiler(use_tmp: bool) -> bool: 954 with TemporaryDirectoryWinProof(prefix='b ', dir=None if use_tmp else '.') as build_dir: 955 env = environment.Environment(None, build_dir, get_fake_options('/')) 956 try: 957 objcpp_comp = detect_objcpp_compiler(env, MachineChoice.HOST) 958 except mesonlib.MesonException: 959 return False 960 if not objcpp_comp: 961 return False 962 env.coredata.process_new_compiler('objcpp', objcpp_comp, env) 963 try: 964 objcpp_comp.sanity_check(env.get_scratch_dir(), env) 965 except mesonlib.MesonException: 966 return False 967 return True 968 969def have_java() -> bool: 970 if shutil.which('javac') and shutil.which('java'): 971 return True 972 return False 973 974def skip_dont_care(t: TestDef) -> bool: 975 # Everything is optional when not running on CI 976 if not under_ci: 977 return True 978 979 # Non-frameworks test are allowed to determine their own skipping under CI (currently) 980 if not t.category.endswith('frameworks'): 981 return True 982 983 if mesonlib.is_osx() and '6 gettext' in str(t.path): 984 return True 985 986 return False 987 988def skip_csharp(backend: Backend) -> bool: 989 if backend is not Backend.ninja: 990 return True 991 if not shutil.which('resgen'): 992 return True 993 if shutil.which('mcs'): 994 return False 995 if shutil.which('csc'): 996 # Only support VS2017 for now. Earlier versions fail 997 # under CI in mysterious ways. 998 try: 999 stdo = subprocess.check_output(['csc', '/version']) 1000 except subprocess.CalledProcessError: 1001 return True 1002 # Having incrementing version numbers would be too easy. 1003 # Microsoft reset the versioning back to 1.0 (from 4.x) 1004 # when they got the Roslyn based compiler. Thus there 1005 # is NO WAY to reliably do version number comparisons. 1006 # Only support the version that ships with VS2017. 1007 return not stdo.startswith(b'2.') 1008 return True 1009 1010# In Azure some setups have a broken rustc that will error out 1011# on all compilation attempts. 1012 1013def has_broken_rustc() -> bool: 1014 dirname = Path('brokenrusttest') 1015 if dirname.exists(): 1016 mesonlib.windows_proof_rmtree(dirname.as_posix()) 1017 dirname.mkdir() 1018 sanity_file = dirname / 'sanity.rs' 1019 sanity_file.write_text('fn main() {\n}\n', encoding='utf-8') 1020 pc = subprocess.run(['rustc', '-o', 'sanity.exe', 'sanity.rs'], 1021 cwd=dirname.as_posix(), 1022 stdout = subprocess.DEVNULL, 1023 stderr = subprocess.DEVNULL) 1024 mesonlib.windows_proof_rmtree(dirname.as_posix()) 1025 return pc.returncode != 0 1026 1027def should_skip_rust(backend: Backend) -> bool: 1028 if not shutil.which('rustc'): 1029 return True 1030 if backend is not Backend.ninja: 1031 return True 1032 if mesonlib.is_windows(): 1033 if has_broken_rustc(): 1034 return True 1035 return False 1036 1037def detect_tests_to_run(only: T.Dict[str, T.List[str]], use_tmp: bool) -> T.List[T.Tuple[str, T.List[TestDef], bool]]: 1038 """ 1039 Parameters 1040 ---------- 1041 only: dict of categories and list of test cases, optional 1042 specify names of tests to run 1043 1044 Returns 1045 ------- 1046 gathered_tests: list of tuple of str, list of TestDef, bool 1047 tests to run 1048 """ 1049 1050 skip_fortran = not(shutil.which('gfortran') or 1051 shutil.which('flang') or 1052 shutil.which('pgfortran') or 1053 shutil.which('nagfor') or 1054 shutil.which('ifort')) 1055 1056 class TestCategory: 1057 def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandatory: bool = False): 1058 self.category = category # category name 1059 self.subdir = subdir # subdirectory 1060 self.skip = skip # skip condition 1061 self.stdout_mandatory = stdout_mandatory # expected stdout is mandatory for tests in this category 1062 1063 all_tests = [ 1064 TestCategory('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)), 1065 TestCategory('common', 'common'), 1066 TestCategory('native', 'native'), 1067 TestCategory('warning-meson', 'warning', stdout_mandatory=True), 1068 TestCategory('failing-meson', 'failing', stdout_mandatory=True), 1069 TestCategory('failing-build', 'failing build'), 1070 TestCategory('failing-test', 'failing test'), 1071 TestCategory('keyval', 'keyval'), 1072 TestCategory('platform-osx', 'osx', not mesonlib.is_osx()), 1073 TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()), 1074 TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()), 1075 TestCategory('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()), 1076 TestCategory('C#', 'csharp', skip_csharp(backend)), 1077 TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))), 1078 TestCategory('cython', 'cython', backend is not Backend.ninja or not shutil.which(os.environ.get('CYTHON', 'cython'))), 1079 TestCategory('rust', 'rust', should_skip_rust(backend)), 1080 TestCategory('d', 'd', backend is not Backend.ninja or not have_d_compiler()), 1081 TestCategory('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)), 1082 TestCategory('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)), 1083 TestCategory('fortran', 'fortran', skip_fortran or backend != Backend.ninja), 1084 TestCategory('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')), 1085 # CUDA tests on Windows: use Ninja backend: python run_project_tests.py --only cuda --backend ninja 1086 TestCategory('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')), 1087 TestCategory('python3', 'python3', backend is not Backend.ninja), 1088 TestCategory('python', 'python'), 1089 TestCategory('fpga', 'fpga', shutil.which('yosys') is None), 1090 TestCategory('frameworks', 'frameworks'), 1091 TestCategory('nasm', 'nasm'), 1092 TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja), 1093 ] 1094 1095 categories = [t.category for t in all_tests] 1096 assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories' 1097 1098 if only: 1099 for key in only.keys(): 1100 assert key in categories, f'key `{key}` is not a recognized category' 1101 all_tests = [t for t in all_tests if t.category in only.keys()] 1102 1103 gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory, only[t.category]), t.skip) for t in all_tests] 1104 return gathered_tests 1105 1106def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]], 1107 log_name_base: str, 1108 failfast: bool, 1109 extra_args: T.List[str], 1110 use_tmp: bool, 1111 num_workers: int) -> T.Tuple[int, int, int]: 1112 txtname = log_name_base + '.txt' 1113 with open(txtname, 'w', encoding='utf-8', errors='ignore') as lf: 1114 return _run_tests(all_tests, log_name_base, failfast, extra_args, use_tmp, num_workers, lf) 1115 1116class TestStatus(Enum): 1117 OK = normal_green(' [SUCCESS] ') 1118 SKIP = yellow(' [SKIPPED] ') 1119 ERROR = red(' [ERROR] ') 1120 UNEXSKIP = red('[UNEXSKIP] ') 1121 UNEXRUN = red(' [UNEXRUN] ') 1122 CANCELED = cyan('[CANCELED] ') 1123 RUNNING = blue(' [RUNNING] ') # Should never be actually printed 1124 LOG = bold(' [LOG] ') # Should never be actually printed 1125 1126def default_print(*args: mlog.TV_Loggable, sep: str = ' ') -> None: 1127 print(*args, sep=sep) 1128 1129safe_print = default_print 1130 1131class TestRunFuture: 1132 def __init__(self, name: str, testdef: TestDef, future: T.Optional['Future[T.Optional[TestResult]]']) -> None: 1133 super().__init__() 1134 self.name = name 1135 self.testdef = testdef 1136 self.future = future 1137 self.status = TestStatus.RUNNING if self.future is not None else TestStatus.SKIP 1138 1139 @property 1140 def result(self) -> T.Optional[TestResult]: 1141 return self.future.result() if self.future else None 1142 1143 def log(self) -> None: 1144 without_install = '' if install_commands else '(without install)' 1145 safe_print(self.status.value, without_install, *self.testdef.display_name()) 1146 1147 def update_log(self, new_status: TestStatus) -> None: 1148 self.status = new_status 1149 self.log() 1150 1151 def cancel(self) -> None: 1152 if self.future is not None and self.future.cancel(): 1153 self.status = TestStatus.CANCELED 1154 1155class LogRunFuture: 1156 def __init__(self, msgs: mlog.TV_LoggableList) -> None: 1157 self.msgs = msgs 1158 self.status = TestStatus.LOG 1159 1160 def log(self) -> None: 1161 safe_print(*self.msgs, sep='') 1162 1163 def cancel(self) -> None: 1164 pass 1165 1166RunFutureUnion = T.Union[TestRunFuture, LogRunFuture] 1167 1168def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]], 1169 log_name_base: str, 1170 failfast: bool, 1171 extra_args: T.List[str], 1172 use_tmp: bool, 1173 num_workers: int, 1174 logfile: T.TextIO) -> T.Tuple[int, int, int]: 1175 global stop, host_c_compiler 1176 xmlname = log_name_base + '.xml' 1177 junit_root = ET.Element('testsuites') 1178 conf_time: float = 0 1179 build_time: float = 0 1180 test_time: float = 0 1181 passing_tests = 0 1182 failing_tests = 0 1183 skipped_tests = 0 1184 1185 print(f'\nRunning tests with {num_workers} workers') 1186 1187 # Pack the global state 1188 state = (compile_commands, clean_commands, test_commands, install_commands, uninstall_commands, backend, backend_flags, host_c_compiler) 1189 executor = ProcessPoolExecutor(max_workers=num_workers) 1190 1191 futures: T.List[RunFutureUnion] = [] 1192 1193 # First, collect and start all tests and also queue log messages 1194 for name, test_cases, skipped in all_tests: 1195 current_suite = ET.SubElement(junit_root, 'testsuite', {'name': name, 'tests': str(len(test_cases))}) 1196 if skipped: 1197 futures += [LogRunFuture(['\n', bold(f'Not running {name} tests.'), '\n'])] 1198 else: 1199 futures += [LogRunFuture(['\n', bold(f'Running {name} tests.'), '\n'])] 1200 1201 for t in test_cases: 1202 # Jenkins screws us over by automatically sorting test cases by name 1203 # and getting it wrong by not doing logical number sorting. 1204 (testnum, testbase) = t.path.name.split(' ', 1) 1205 testname = '%.3d %s' % (int(testnum), testbase) 1206 if t.name: 1207 testname += f' ({t.name})' 1208 should_fail = '' 1209 suite_args = [] 1210 if name.startswith('failing'): 1211 should_fail = name.split('failing-')[1] 1212 if name.startswith('warning'): 1213 suite_args = ['--fatal-meson-warnings'] 1214 should_fail = name.split('warning-')[1] 1215 1216 if skipped or t.skip: 1217 futures += [TestRunFuture(testname, t, None)] 1218 continue 1219 result_future = executor.submit(run_test, t, extra_args + suite_args + t.args, should_fail, use_tmp, state=state) 1220 futures += [TestRunFuture(testname, t, result_future)] 1221 1222 # Ensure we only cancel once 1223 tests_canceled = False 1224 1225 # Optionally enable the tqdm progress bar 1226 global safe_print 1227 futures_iter: T.Iterable[RunFutureUnion] = futures 1228 try: 1229 from tqdm import tqdm 1230 futures_iter = tqdm(futures, desc='Running tests', unit='test') 1231 1232 def tqdm_print(*args: mlog.TV_Loggable, sep: str = ' ') -> None: 1233 tqdm.write(sep.join([str(x) for x in args])) 1234 1235 safe_print = tqdm_print 1236 except ImportError: 1237 pass 1238 1239 # Wait and handle the test results and print the stored log output 1240 for f in futures_iter: 1241 # Just a log entry to print something to stdout 1242 sys.stdout.flush() 1243 if isinstance(f, LogRunFuture): 1244 f.log() 1245 continue 1246 1247 # Acutal Test run 1248 testname = f.name 1249 t = f.testdef 1250 try: 1251 result = f.result 1252 except (CancelledError, KeyboardInterrupt): 1253 f.status = TestStatus.CANCELED 1254 1255 if stop and not tests_canceled: 1256 num_running = sum([1 if f2.status is TestStatus.RUNNING else 0 for f2 in futures]) 1257 for f2 in futures: 1258 f2.cancel() 1259 executor.shutdown() 1260 num_canceled = sum([1 if f2.status is TestStatus.CANCELED else 0 for f2 in futures]) 1261 safe_print(f'\nCanceled {num_canceled} out of {num_running} running tests.') 1262 safe_print(f'Finishing the remaining {num_running - num_canceled} tests.\n') 1263 tests_canceled = True 1264 1265 # Handle canceled tests 1266 if f.status is TestStatus.CANCELED: 1267 f.log() 1268 continue 1269 1270 # Handle skipped tests 1271 if result is None: 1272 # skipped due to skipped category skip or 'tools:' or 'skip_on_env:' 1273 is_skipped = True 1274 skip_as_expected = True 1275 else: 1276 # skipped due to test outputting 'MESON_SKIP_TEST' 1277 is_skipped = 'MESON_SKIP_TEST' in result.stdo 1278 if not skip_dont_care(t): 1279 skip_as_expected = (is_skipped == t.skip_expected) 1280 else: 1281 skip_as_expected = True 1282 1283 if is_skipped: 1284 skipped_tests += 1 1285 1286 if is_skipped and skip_as_expected: 1287 f.update_log(TestStatus.SKIP) 1288 current_test = ET.SubElement(current_suite, 'testcase', {'name': testname, 'classname': t.category}) 1289 ET.SubElement(current_test, 'skipped', {}) 1290 continue 1291 1292 if not skip_as_expected: 1293 failing_tests += 1 1294 if is_skipped: 1295 skip_msg = 'Test asked to be skipped, but was not expected to' 1296 status = TestStatus.UNEXSKIP 1297 else: 1298 skip_msg = 'Test ran, but was expected to be skipped' 1299 status = TestStatus.UNEXRUN 1300 result.msg = "%s for MESON_CI_JOBNAME '%s'" % (skip_msg, ci_jobname) 1301 1302 f.update_log(status) 1303 current_test = ET.SubElement(current_suite, 'testcase', {'name': testname, 'classname': t.category}) 1304 ET.SubElement(current_test, 'failure', {'message': result.msg}) 1305 continue 1306 1307 # Handle Failed tests 1308 if result.msg != '': 1309 f.update_log(TestStatus.ERROR) 1310 safe_print(bold('During:'), result.step.name) 1311 safe_print(bold('Reason:'), result.msg) 1312 failing_tests += 1 1313 # Append a visual seperator for the different test cases 1314 cols = shutil.get_terminal_size((100, 20)).columns 1315 name_str = ' '.join([str(x) for x in f.testdef.display_name()]) 1316 name_len = len(re.sub(r'\x1B[^m]+m', '', name_str)) # Do not count escape sequences 1317 left_w = (cols // 2) - (name_len // 2) - 1 1318 left_w = max(3, left_w) 1319 right_w = cols - left_w - name_len - 2 1320 right_w = max(3, right_w) 1321 failing_logs.append(f'\n\x1b[31m{"="*left_w}\x1b[0m {name_str} \x1b[31m{"="*right_w}\x1b[0m\n') 1322 if result.step == BuildStep.configure and result.mlog != no_meson_log_msg: 1323 # For configure failures, instead of printing stdout, 1324 # print the meson log if available since it's a superset 1325 # of stdout and often has very useful information. 1326 failing_logs.append(result.mlog) 1327 elif under_ci: 1328 # Always print the complete meson log when running in 1329 # a CI. This helps debugging issues that only occur in 1330 # a hard to reproduce environment 1331 failing_logs.append(result.mlog) 1332 failing_logs.append(result.stdo) 1333 else: 1334 failing_logs.append(result.stdo) 1335 for cmd_res in result.cicmds: 1336 failing_logs.append(cmd_res) 1337 failing_logs.append(result.stde) 1338 if failfast: 1339 safe_print("Cancelling the rest of the tests") 1340 for f2 in futures: 1341 f2.cancel() 1342 else: 1343 f.update_log(TestStatus.OK) 1344 passing_tests += 1 1345 conf_time += result.conftime 1346 build_time += result.buildtime 1347 test_time += result.testtime 1348 total_time = conf_time + build_time + test_time 1349 log_text_file(logfile, t.path, result) 1350 current_test = ET.SubElement( 1351 current_suite, 1352 'testcase', 1353 {'name': testname, 'classname': t.category, 'time': '%.3f' % total_time} 1354 ) 1355 if result.msg != '': 1356 ET.SubElement(current_test, 'failure', {'message': result.msg}) 1357 stdoel = ET.SubElement(current_test, 'system-out') 1358 stdoel.text = result.stdo 1359 stdeel = ET.SubElement(current_test, 'system-err') 1360 stdeel.text = result.stde 1361 1362 # Reset, just in case 1363 safe_print = default_print 1364 1365 print() 1366 print("Total configuration time: %.2fs" % conf_time) 1367 print("Total build time: %.2fs" % build_time) 1368 print("Total test time: %.2fs" % test_time) 1369 ET.ElementTree(element=junit_root).write(xmlname, xml_declaration=True, encoding='UTF-8') 1370 return passing_tests, failing_tests, skipped_tests 1371 1372def check_meson_commands_work(use_tmpdir: bool, extra_args: T.List[str]) -> None: 1373 global backend, compile_commands, test_commands, install_commands 1374 testdir = PurePath('test cases', 'common', '1 trivial').as_posix() 1375 meson_commands = mesonlib.python_command + [get_meson_script()] 1376 with TemporaryDirectoryWinProof(prefix='b ', dir=None if use_tmpdir else '.') as build_dir: 1377 print('Checking that configuring works...') 1378 gen_cmd = meson_commands + [testdir, build_dir] + backend_flags + extra_args 1379 pc, o, e = Popen_safe(gen_cmd) 1380 if pc.returncode != 0: 1381 raise RuntimeError(f'Failed to configure {testdir!r}:\n{e}\n{o}') 1382 print('Checking that introspect works...') 1383 pc, o, e = Popen_safe(meson_commands + ['introspect', '--targets'], cwd=build_dir) 1384 json.loads(o) 1385 if pc.returncode != 0: 1386 raise RuntimeError(f'Failed to introspect --targets {testdir!r}:\n{e}\n{o}') 1387 print('Checking that building works...') 1388 dir_args = get_backend_args_for_dir(backend, build_dir) 1389 pc, o, e = Popen_safe(compile_commands + dir_args, cwd=build_dir) 1390 if pc.returncode != 0: 1391 raise RuntimeError(f'Failed to build {testdir!r}:\n{e}\n{o}') 1392 print('Checking that testing works...') 1393 pc, o, e = Popen_safe(test_commands, cwd=build_dir) 1394 if pc.returncode != 0: 1395 raise RuntimeError(f'Failed to test {testdir!r}:\n{e}\n{o}') 1396 if install_commands: 1397 print('Checking that installing works...') 1398 pc, o, e = Popen_safe(install_commands, cwd=build_dir) 1399 if pc.returncode != 0: 1400 raise RuntimeError(f'Failed to install {testdir!r}:\n{e}\n{o}') 1401 1402 1403def detect_system_compiler(options: 'CompilerArgumentType') -> None: 1404 global host_c_compiler, compiler_id_map 1405 1406 fake_opts = get_fake_options('/') 1407 if options.cross_file: 1408 fake_opts.cross_file = [options.cross_file] 1409 if options.native_file: 1410 fake_opts.native_file = [options.native_file] 1411 1412 env = environment.Environment(None, None, fake_opts) 1413 1414 print_compilers(env, MachineChoice.HOST) 1415 if options.cross_file: 1416 print_compilers(env, MachineChoice.BUILD) 1417 1418 for lang in sorted(compilers.all_languages): 1419 try: 1420 comp = compiler_from_language(env, lang, MachineChoice.HOST) 1421 # note compiler id for later use with test.json matrix 1422 compiler_id_map[lang] = comp.get_id() 1423 except mesonlib.MesonException: 1424 comp = None 1425 1426 # note C compiler for later use by platform_fix_name() 1427 if lang == 'c': 1428 if comp: 1429 host_c_compiler = comp.get_id() 1430 else: 1431 raise RuntimeError("Could not find C compiler.") 1432 1433 1434def print_compilers(env: 'Environment', machine: MachineChoice) -> None: 1435 print() 1436 print(f'{machine.get_lower_case_name()} machine compilers') 1437 print() 1438 for lang in sorted(compilers.all_languages): 1439 try: 1440 comp = compiler_from_language(env, lang, machine) 1441 details = '{:<10} {} {}'.format('[' + comp.get_id() + ']', ' '.join(comp.get_exelist()), comp.get_version_string()) 1442 except mesonlib.MesonException: 1443 details = '[not found]' 1444 print(f'{lang:<7}: {details}') 1445 1446class ToolInfo(T.NamedTuple): 1447 tool: str 1448 args: T.List[str] 1449 regex: T.Pattern 1450 match_group: int 1451 1452def print_tool_versions() -> None: 1453 tools: T.List[ToolInfo] = [ 1454 ToolInfo( 1455 'ninja', 1456 ['--version'], 1457 re.compile(r'^([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'), 1458 1, 1459 ), 1460 ToolInfo( 1461 'cmake', 1462 ['--version'], 1463 re.compile(r'^cmake version ([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'), 1464 1, 1465 ), 1466 ToolInfo( 1467 'hotdoc', 1468 ['--version'], 1469 re.compile(r'^([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'), 1470 1, 1471 ), 1472 ] 1473 1474 def get_version(t: ToolInfo) -> str: 1475 exe = shutil.which(t.tool) 1476 if not exe: 1477 return 'not found' 1478 1479 args = [t.tool] + t.args 1480 pc, o, e = Popen_safe(args) 1481 if pc.returncode != 0: 1482 return '{} (invalid {} executable)'.format(exe, t.tool) 1483 for i in o.split('\n'): 1484 i = i.strip('\n\r\t ') 1485 m = t.regex.match(i) 1486 if m is not None: 1487 tool_vers_map[t.tool] = m.group(t.match_group) 1488 return '{} ({})'.format(exe, m.group(t.match_group)) 1489 1490 return f'{exe} (unknown)' 1491 1492 print() 1493 print('tools') 1494 print() 1495 1496 max_width = max([len(x.tool) for x in tools] + [7]) 1497 for tool in tools: 1498 print('{0:<{2}}: {1}'.format(tool.tool, get_version(tool), max_width)) 1499 print() 1500 1501def clear_transitive_files() -> None: 1502 a = Path('test cases/common') 1503 for d in a.glob('*subproject subdir/subprojects/subsubsub*'): 1504 if d.is_dir(): 1505 mesonlib.windows_proof_rmtree(str(d)) 1506 else: 1507 mesonlib.windows_proof_rm(str(d)) 1508 1509if __name__ == '__main__': 1510 if under_ci and not ci_jobname: 1511 raise SystemExit('Running under CI but MESON_CI_JOBNAME is not set') 1512 1513 setup_vsenv() 1514 1515 try: 1516 # This fails in some CI environments for unknown reasons. 1517 num_workers = multiprocessing.cpu_count() 1518 except Exception as e: 1519 print('Could not determine number of CPUs due to the following reason:' + str(e)) 1520 print('Defaulting to using only two processes') 1521 num_workers = 2 1522 1523 parser = argparse.ArgumentParser(description="Run the test suite of Meson.") 1524 parser.add_argument('extra_args', nargs='*', 1525 help='arguments that are passed directly to Meson (remember to have -- before these).') 1526 parser.add_argument('--backend', dest='backend', choices=backendlist) 1527 parser.add_argument('-j', dest='num_workers', type=int, default=num_workers, 1528 help=f'Maximum number of parallel tests (default {num_workers})') 1529 parser.add_argument('--failfast', action='store_true', 1530 help='Stop running if test case fails') 1531 parser.add_argument('--no-unittests', action='store_true', 1532 help='Not used, only here to simplify run_tests.py') 1533 parser.add_argument('--only', default=[], 1534 help='name of test(s) to run, in format "category[/name]" where category is one of: ' + ', '.join(ALL_TESTS), nargs='+') 1535 parser.add_argument('--cross-file', action='store', help='File describing cross compilation environment.') 1536 parser.add_argument('--native-file', action='store', help='File describing native compilation environment.') 1537 parser.add_argument('--use-tmpdir', action='store_true', help='Use tmp directory for temporary files.') 1538 options = T.cast('ArgumentType', parser.parse_args()) 1539 1540 if options.cross_file: 1541 options.extra_args += ['--cross-file', options.cross_file] 1542 if options.native_file: 1543 options.extra_args += ['--native-file', options.native_file] 1544 1545 clear_transitive_files() 1546 1547 print('Meson build system', meson_version, 'Project Tests') 1548 print('Using python', sys.version.split('\n')[0]) 1549 if 'VSCMD_VER' in os.environ: 1550 print('VSCMD version', os.environ['VSCMD_VER']) 1551 setup_commands(options.backend) 1552 detect_system_compiler(options) 1553 print_tool_versions() 1554 script_dir = os.path.split(__file__)[0] 1555 if script_dir != '': 1556 os.chdir(script_dir) 1557 check_meson_commands_work(options.use_tmpdir, options.extra_args) 1558 only = collections.defaultdict(list) 1559 for i in options.only: 1560 try: 1561 cat, case = i.split('/') 1562 only[cat].append(case) 1563 except ValueError: 1564 only[i].append('') 1565 try: 1566 all_tests = detect_tests_to_run(only, options.use_tmpdir) 1567 res = run_tests(all_tests, 'meson-test-run', options.failfast, options.extra_args, options.use_tmpdir, options.num_workers) 1568 (passing_tests, failing_tests, skipped_tests) = res 1569 except StopException: 1570 pass 1571 print() 1572 print('Total passed tests: ', green(str(passing_tests))) 1573 print('Total failed tests: ', red(str(failing_tests))) 1574 print('Total skipped tests:', yellow(str(skipped_tests))) 1575 if failing_tests > 0: 1576 print('\nMesonlogs of failing tests\n') 1577 for l in failing_logs: 1578 try: 1579 print(l, '\n') 1580 except UnicodeError: 1581 print(l.encode('ascii', errors='replace').decode(), '\n') 1582 for name, dirs, _ in all_tests: 1583 dir_names = list({x.path.name for x in dirs}) 1584 for k, g in itertools.groupby(dir_names, key=lambda x: x.split()[0]): 1585 tests = list(g) 1586 if len(tests) != 1: 1587 print('WARNING: The {} suite contains duplicate "{}" tests: "{}"'.format(name, k, '", "'.join(tests))) 1588 clear_transitive_files() 1589 raise SystemExit(failing_tests) 1590