1#!/usr/bin/env python3 2 3# Copyright 2012-2021 The Meson development team 4 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8 9# http://www.apache.org/licenses/LICENSE-2.0 10 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17# Work around some pathlib bugs... 18from mesonbuild import _pathlib 19import sys 20sys.modules['pathlib'] = _pathlib 21 22from concurrent.futures import ProcessPoolExecutor, CancelledError 23from enum import Enum 24from io import StringIO 25from pathlib import Path, PurePath 26import argparse 27import functools 28import itertools 29import json 30import multiprocessing 31import os 32import re 33import shlex 34import shutil 35import signal 36import subprocess 37import tempfile 38import time 39import typing as T 40import xml.etree.ElementTree as ET 41import collections 42 43from mesonbuild import build 44from mesonbuild import environment 45from mesonbuild import compilers 46from mesonbuild import mesonlib 47from mesonbuild import mlog 48from mesonbuild import mtest 49from mesonbuild.compilers import compiler_from_language, detect_objc_compiler, detect_objcpp_compiler 50from mesonbuild.build import ConfigurationData 51from mesonbuild.mesonlib import MachineChoice, Popen_safe, TemporaryDirectoryWinProof 52from mesonbuild.mlog import blue, bold, cyan, green, red, yellow, normal_green 53from mesonbuild.coredata import backendlist, version as meson_version 54from mesonbuild.mesonmain import setup_vsenv 55from mesonbuild.modules.python import PythonExternalProgram 56from run_tests import get_fake_options, run_configure, get_meson_script 57from run_tests import get_backend_commands, get_backend_args_for_dir, Backend 58from run_tests import ensure_backend_detects_changes 59from run_tests import guess_backend 60 61if T.TYPE_CHECKING: 62 from types import FrameType 63 from mesonbuild.environment import Environment 64 from mesonbuild._typing import Protocol 65 from concurrent.futures import Future 66 67 class CompilerArgumentType(Protocol): 68 cross_file: str 69 native_file: str 70 use_tmpdir: bool 71 72 73 class ArgumentType(CompilerArgumentType): 74 75 """Typing information for command line arguments.""" 76 77 extra_args: T.List[str] 78 backend: str 79 num_workers: int 80 failfast: bool 81 no_unittests: bool 82 only: T.List[str] 83 84ALL_TESTS = ['cmake', 'common', 'native', 'warning-meson', 'failing-meson', 'failing-build', 'failing-test', 85 'keyval', 'platform-osx', 'platform-windows', 'platform-linux', 86 'java', 'C#', 'vala', 'cython', 'rust', 'd', 'objective c', 'objective c++', 87 'fortran', 'swift', 'cuda', 'python3', 'python', 'fpga', 'frameworks', 'nasm', 'wasm', 88 ] 89 90 91class BuildStep(Enum): 92 configure = 1 93 build = 2 94 test = 3 95 install = 4 96 clean = 5 97 validate = 6 98 99 100class TestResult(BaseException): 101 def __init__(self, cicmds: T.List[str]) -> None: 102 self.msg = '' # empty msg indicates test success 103 self.stdo = '' 104 self.stde = '' 105 self.mlog = '' 106 self.cicmds = cicmds 107 self.conftime: float = 0 108 self.buildtime: float = 0 109 self.testtime: float = 0 110 111 def add_step(self, step: BuildStep, stdo: str, stde: str, mlog: str = '', time: float = 0) -> None: 112 self.step = step 113 self.stdo += stdo 114 self.stde += stde 115 self.mlog += mlog 116 if step == BuildStep.configure: 117 self.conftime = time 118 elif step == BuildStep.build: 119 self.buildtime = time 120 elif step == BuildStep.test: 121 self.testtime = time 122 123 def fail(self, msg: str) -> None: 124 self.msg = msg 125 126python = PythonExternalProgram(sys.executable) 127python.sanity() 128 129class InstalledFile: 130 def __init__(self, raw: T.Dict[str, str]): 131 self.path = raw['file'] 132 self.typ = raw['type'] 133 self.platform = raw.get('platform', None) 134 self.language = raw.get('language', 'c') # type: str 135 136 version = raw.get('version', '') # type: str 137 if version: 138 self.version = version.split('.') # type: T.List[str] 139 else: 140 # split on '' will return [''], we want an empty list though 141 self.version = [] 142 143 def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Path]: 144 p = Path(self.path) 145 canonical_compiler = compiler 146 if ((compiler in ['clang-cl', 'intel-cl']) or 147 (env.machines.host.is_windows() and compiler in {'pgi', 'dmd', 'ldc'})): 148 canonical_compiler = 'msvc' 149 150 python_suffix = python.info['suffix'] 151 152 has_pdb = False 153 if self.language in {'c', 'cpp'}: 154 has_pdb = canonical_compiler == 'msvc' 155 elif self.language == 'd': 156 # dmd's optlink does not genearte pdb iles 157 has_pdb = env.coredata.compilers.host['d'].linker.id in {'link', 'lld-link'} 158 159 # Abort if the platform does not match 160 matches = { 161 'msvc': canonical_compiler == 'msvc', 162 'gcc': canonical_compiler != 'msvc', 163 'cygwin': env.machines.host.is_cygwin(), 164 '!cygwin': not env.machines.host.is_cygwin(), 165 }.get(self.platform or '', True) 166 if not matches: 167 return None 168 169 # Handle the different types 170 if self.typ in {'py_implib', 'python_lib', 'python_file'}: 171 val = p.as_posix() 172 val = val.replace('@PYTHON_PLATLIB@', python.platlib) 173 val = val.replace('@PYTHON_PURELIB@', python.purelib) 174 p = Path(val) 175 if self.typ == 'python_file': 176 return p 177 if self.typ == 'python_lib': 178 return p.with_suffix(python_suffix) 179 if self.typ in ['file', 'dir']: 180 return p 181 elif self.typ == 'shared_lib': 182 if env.machines.host.is_windows() or env.machines.host.is_cygwin(): 183 # Windows only has foo.dll and foo-X.dll 184 if len(self.version) > 1: 185 return None 186 if self.version: 187 p = p.with_name('{}-{}'.format(p.name, self.version[0])) 188 return p.with_suffix('.dll') 189 190 p = p.with_name(f'lib{p.name}') 191 if env.machines.host.is_darwin(): 192 # MacOS only has libfoo.dylib and libfoo.X.dylib 193 if len(self.version) > 1: 194 return None 195 196 # pathlib.Path.with_suffix replaces, not appends 197 suffix = '.dylib' 198 if self.version: 199 suffix = '.{}{}'.format(self.version[0], suffix) 200 else: 201 # pathlib.Path.with_suffix replaces, not appends 202 suffix = '.so' 203 if self.version: 204 suffix = '{}.{}'.format(suffix, '.'.join(self.version)) 205 return p.with_suffix(suffix) 206 elif self.typ == 'exe': 207 if env.machines.host.is_windows() or env.machines.host.is_cygwin(): 208 return p.with_suffix('.exe') 209 elif self.typ == 'pdb': 210 if self.version: 211 p = p.with_name('{}-{}'.format(p.name, self.version[0])) 212 return p.with_suffix('.pdb') if has_pdb else None 213 elif self.typ in {'implib', 'implibempty', 'py_implib'}: 214 if env.machines.host.is_windows() and canonical_compiler == 'msvc': 215 # only MSVC doesn't generate empty implibs 216 if self.typ == 'implibempty' and compiler == 'msvc': 217 return None 218 return p.parent / (re.sub(r'^lib', '', p.name) + '.lib') 219 elif env.machines.host.is_windows() or env.machines.host.is_cygwin(): 220 if self.typ == 'py_implib': 221 p = p.with_suffix(python_suffix) 222 return p.with_suffix('.dll.a') 223 else: 224 return None 225 elif self.typ == 'expr': 226 return Path(platform_fix_name(p.as_posix(), canonical_compiler, env)) 227 else: 228 raise RuntimeError(f'Invalid installed file type {self.typ}') 229 230 return p 231 232 def get_paths(self, compiler: str, env: environment.Environment, installdir: Path) -> T.List[Path]: 233 p = self.get_path(compiler, env) 234 if not p: 235 return [] 236 if self.typ == 'dir': 237 abs_p = installdir / p 238 if not abs_p.exists(): 239 raise RuntimeError(f'{p} does not exist') 240 if not abs_p.is_dir(): 241 raise RuntimeError(f'{p} is not a directory') 242 return [x.relative_to(installdir) for x in abs_p.rglob('*') if x.is_file() or x.is_symlink()] 243 else: 244 return [p] 245 246@functools.total_ordering 247class TestDef: 248 def __init__(self, path: Path, name: T.Optional[str], args: T.List[str], skip: bool = False): 249 self.category = path.parts[1] 250 self.path = path 251 self.name = name 252 self.args = args 253 self.skip = skip 254 self.env = os.environ.copy() 255 self.installed_files = [] # type: T.List[InstalledFile] 256 self.do_not_set_opts = [] # type: T.List[str] 257 self.stdout = [] # type: T.List[T.Dict[str, str]] 258 self.skip_expected = False 259 260 # Always print a stack trace for Meson exceptions 261 self.env['MESON_FORCE_BACKTRACE'] = '1' 262 263 def __repr__(self) -> str: 264 return '<{}: {:<48} [{}: {}] -- {}>'.format(type(self).__name__, str(self.path), self.name, self.args, self.skip) 265 266 def display_name(self) -> mlog.TV_LoggableList: 267 # Remove the redundant 'test cases' part 268 section, id = self.path.parts[1:3] 269 res: mlog.TV_LoggableList = [f'{section}:', bold(id)] 270 if self.name: 271 res += [f' ({self.name})'] 272 return res 273 274 def __lt__(self, other: object) -> bool: 275 if isinstance(other, TestDef): 276 # None is not sortable, so replace it with an empty string 277 s_id = int(self.path.name.split(' ')[0]) 278 o_id = int(other.path.name.split(' ')[0]) 279 return (s_id, self.path, self.name or '') < (o_id, other.path, other.name or '') 280 return NotImplemented 281 282failing_logs: T.List[str] = [] 283print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ 284under_ci = 'CI' in os.environ 285ci_jobname = os.environ.get('MESON_CI_JOBNAME', None) 286do_debug = under_ci or print_debug 287no_meson_log_msg = 'No meson-log.txt found.' 288 289host_c_compiler: T.Optional[str] = None 290compiler_id_map: T.Dict[str, str] = {} 291tool_vers_map: T.Dict[str, str] = {} 292 293compile_commands: T.List[str] 294clean_commands: T.List[str] 295test_commands: T.List[str] 296install_commands: T.List[str] 297uninstall_commands: T.List[str] 298 299backend: 'Backend' 300backend_flags: T.List[str] 301 302stop: bool = False 303is_worker_process: bool = False 304 305# Let's have colors in our CI output 306if under_ci: 307 def _ci_colorize_console() -> bool: 308 return not is_worker_process 309 310 mlog.colorize_console = _ci_colorize_console 311 312class StopException(Exception): 313 def __init__(self) -> None: 314 super().__init__('Stopped by user') 315 316def stop_handler(signal: signal.Signals, frame: T.Optional['FrameType']) -> None: 317 global stop 318 stop = True 319signal.signal(signal.SIGINT, stop_handler) 320signal.signal(signal.SIGTERM, stop_handler) 321 322def setup_commands(optbackend: str) -> None: 323 global do_debug, backend, backend_flags 324 global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands 325 backend, backend_flags = guess_backend(optbackend, shutil.which('msbuild')) 326 compile_commands, clean_commands, test_commands, install_commands, \ 327 uninstall_commands = get_backend_commands(backend, do_debug) 328 329# TODO try to eliminate or at least reduce this function 330def platform_fix_name(fname: str, canonical_compiler: str, env: environment.Environment) -> str: 331 if '?lib' in fname: 332 if env.machines.host.is_windows() and canonical_compiler == 'msvc': 333 fname = re.sub(r'lib/\?lib(.*)\.', r'bin/\1.', fname) 334 fname = re.sub(r'/\?lib/', r'/bin/', fname) 335 elif env.machines.host.is_windows(): 336 fname = re.sub(r'lib/\?lib(.*)\.', r'bin/lib\1.', fname) 337 fname = re.sub(r'\?lib(.*)\.dll$', r'lib\1.dll', fname) 338 fname = re.sub(r'/\?lib/', r'/bin/', fname) 339 elif env.machines.host.is_cygwin(): 340 fname = re.sub(r'lib/\?lib(.*)\.so$', r'bin/cyg\1.dll', fname) 341 fname = re.sub(r'lib/\?lib(.*)\.', r'bin/cyg\1.', fname) 342 fname = re.sub(r'\?lib(.*)\.dll$', r'cyg\1.dll', fname) 343 fname = re.sub(r'/\?lib/', r'/bin/', fname) 344 else: 345 fname = re.sub(r'\?lib', 'lib', fname) 346 347 if fname.endswith('?so'): 348 if env.machines.host.is_windows() and canonical_compiler == 'msvc': 349 fname = re.sub(r'lib/([^/]*)\?so$', r'bin/\1.dll', fname) 350 fname = re.sub(r'/(?:lib|)([^/]*?)\?so$', r'/\1.dll', fname) 351 return fname 352 elif env.machines.host.is_windows(): 353 fname = re.sub(r'lib/([^/]*)\?so$', r'bin/\1.dll', fname) 354 fname = re.sub(r'/([^/]*?)\?so$', r'/\1.dll', fname) 355 return fname 356 elif env.machines.host.is_cygwin(): 357 fname = re.sub(r'lib/([^/]*)\?so$', r'bin/\1.dll', fname) 358 fname = re.sub(r'/lib([^/]*?)\?so$', r'/cyg\1.dll', fname) 359 fname = re.sub(r'/([^/]*?)\?so$', r'/\1.dll', fname) 360 return fname 361 elif env.machines.host.is_darwin(): 362 return fname[:-3] + '.dylib' 363 else: 364 return fname[:-3] + '.so' 365 366 return fname 367 368def validate_install(test: TestDef, installdir: Path, env: environment.Environment) -> str: 369 ret_msg = '' 370 expected_raw = [] # type: T.List[Path] 371 for i in test.installed_files: 372 try: 373 expected_raw += i.get_paths(host_c_compiler, env, installdir) 374 except RuntimeError as err: 375 ret_msg += f'Expected path error: {err}\n' 376 expected = {x: False for x in expected_raw} 377 found = [x.relative_to(installdir) for x in installdir.rglob('*') if x.is_file() or x.is_symlink()] 378 # Mark all found files as found and detect unexpected files 379 for fname in found: 380 if fname not in expected: 381 ret_msg += f'Extra file {fname} found.\n' 382 continue 383 expected[fname] = True 384 # Check if expected files were found 385 for p, f in expected.items(): 386 if not f: 387 ret_msg += f'Expected file {p} missing.\n' 388 # List dir content on error 389 if ret_msg != '': 390 ret_msg += '\nInstall dir contents:\n' 391 for p in found: 392 ret_msg += f' - {p}\n' 393 return ret_msg 394 395def log_text_file(logfile: T.TextIO, testdir: Path, result: TestResult) -> None: 396 logfile.write('%s\nstdout\n\n---\n' % testdir.as_posix()) 397 logfile.write(result.stdo) 398 logfile.write('\n\n---\n\nstderr\n\n---\n') 399 logfile.write(result.stde) 400 logfile.write('\n\n---\n\n') 401 if print_debug: 402 try: 403 print(result.stdo) 404 except UnicodeError: 405 sanitized_out = result.stdo.encode('ascii', errors='replace').decode() 406 print(sanitized_out) 407 try: 408 print(result.stde, file=sys.stderr) 409 except UnicodeError: 410 sanitized_err = result.stde.encode('ascii', errors='replace').decode() 411 print(sanitized_err, file=sys.stderr) 412 413 414def _run_ci_include(args: T.List[str]) -> str: 415 if not args: 416 return 'At least one parameter required' 417 try: 418 data = Path(args[0]).read_text(errors='ignore', encoding='utf-8') 419 return 'Included file {}:\n{}\n'.format(args[0], data) 420 except Exception: 421 return 'Failed to open {}'.format(args[0]) 422 423ci_commands = { 424 'ci_include': _run_ci_include 425} 426 427def run_ci_commands(raw_log: str) -> T.List[str]: 428 res = [] 429 for l in raw_log.splitlines(): 430 if not l.startswith('!meson_ci!/'): 431 continue 432 cmd = shlex.split(l[11:]) 433 if not cmd or cmd[0] not in ci_commands: 434 continue 435 res += ['CI COMMAND {}:\n{}\n'.format(cmd[0], ci_commands[cmd[0]](cmd[1:]))] 436 return res 437 438class OutputMatch: 439 def __init__(self, how: str, expected: str, count: int) -> None: 440 self.how = how 441 self.expected = expected 442 self.count = count 443 444 def match(self, actual: str) -> bool: 445 if self.how == "re": 446 return bool(re.match(self.expected, actual)) 447 return self.expected == actual 448 449def _compare_output(expected: T.List[T.Dict[str, str]], output: str, desc: str) -> str: 450 if expected: 451 matches: T.List[OutputMatch] = [] 452 nomatches: T.List[OutputMatch] = [] 453 for item in expected: 454 how = item.get('match', 'literal') 455 expected_line = item.get('line') 456 count = int(item.get('count', -1)) 457 458 # Simple heuristic to automatically convert path separators for 459 # Windows: 460 # 461 # Any '/' appearing before 'WARNING' or 'ERROR' (i.e. a path in a 462 # filename part of a location) is replaced with '\' (in a re: '\\' 463 # which matches a literal '\') 464 # 465 # (There should probably be a way to turn this off for more complex 466 # cases which don't fit this) 467 if mesonlib.is_windows(): 468 if how != "re": 469 sub = r'\\' 470 else: 471 sub = r'\\\\' 472 expected_line = re.sub(r'/(?=.*(WARNING|ERROR))', sub, expected_line) 473 474 m = OutputMatch(how, expected_line, count) 475 if count == 0: 476 nomatches.append(m) 477 else: 478 matches.append(m) 479 480 481 i = 0 482 for actual in output.splitlines(): 483 # Verify this line does not match any unexpected lines (item.count == 0) 484 for match in nomatches: 485 if match.match(actual): 486 return f'unexpected "{match.expected}" found in {desc}' 487 # If we matched all expected lines, continue to verify there are 488 # no unexpected line. If nomatches is empty then we are done already. 489 if i >= len(matches): 490 if not nomatches: 491 break 492 continue 493 # Check if this line match current expected line 494 match = matches[i] 495 if match.match(actual): 496 if match.count < 0: 497 # count was not specified, continue with next expected line, 498 # it does not matter if this line will be matched again or 499 # not. 500 i += 1 501 else: 502 # count was specified (must be >0), continue expecting this 503 # same line. If count reached 0 we continue with next 504 # expected line but remember that this one must not match 505 # anymore. 506 match.count -= 1 507 if match.count == 0: 508 nomatches.append(match) 509 i += 1 510 511 if i < len(matches): 512 # reached the end of output without finding expected 513 return f'expected "{matches[i].expected}" not found in {desc}' 514 515 return '' 516 517def validate_output(test: TestDef, stdo: str, stde: str) -> str: 518 return _compare_output(test.stdout, stdo, 'stdout') 519 520# There are some class variables and such that cahce 521# information. Clear all of these. The better solution 522# would be to change the code so that no state is persisted 523# but that would be a lot of work given that Meson was originally 524# coded to run as a batch process. 525def clear_internal_caches() -> None: 526 import mesonbuild.interpreterbase 527 from mesonbuild.dependencies import CMakeDependency 528 from mesonbuild.mesonlib import PerMachine 529 mesonbuild.interpreterbase.FeatureNew.feature_registry = {} 530 CMakeDependency.class_cmakeinfo = PerMachine(None, None) 531 532def run_test_inprocess(testdir: str) -> T.Tuple[int, str, str, str]: 533 old_stdout = sys.stdout 534 sys.stdout = mystdout = StringIO() 535 old_stderr = sys.stderr 536 sys.stderr = mystderr = StringIO() 537 old_cwd = os.getcwd() 538 os.chdir(testdir) 539 test_log_fname = Path('meson-logs', 'testlog.txt') 540 try: 541 returncode_test = mtest.run_with_args(['--no-rebuild']) 542 if test_log_fname.exists(): 543 test_log = test_log_fname.open(encoding='utf-8', errors='ignore').read() 544 else: 545 test_log = '' 546 returncode_benchmark = mtest.run_with_args(['--no-rebuild', '--benchmark', '--logbase', 'benchmarklog']) 547 finally: 548 sys.stdout = old_stdout 549 sys.stderr = old_stderr 550 os.chdir(old_cwd) 551 return max(returncode_test, returncode_benchmark), mystdout.getvalue(), mystderr.getvalue(), test_log 552 553# Build directory name must be the same so Ccache works over 554# consecutive invocations. 555def create_deterministic_builddir(test: TestDef, use_tmpdir: bool) -> str: 556 import hashlib 557 src_dir = test.path.as_posix() 558 if test.name: 559 src_dir += test.name 560 rel_dirname = 'b ' + hashlib.sha256(src_dir.encode(errors='ignore')).hexdigest()[0:10] 561 abs_pathname = os.path.join(tempfile.gettempdir() if use_tmpdir else os.getcwd(), rel_dirname) 562 if os.path.exists(abs_pathname): 563 mesonlib.windows_proof_rmtree(abs_pathname) 564 os.mkdir(abs_pathname) 565 return abs_pathname 566 567def format_parameter_file(file_basename: str, test: TestDef, test_build_dir: str) -> Path: 568 confdata = ConfigurationData() 569 confdata.values = {'MESON_TEST_ROOT': (str(test.path.absolute()), 'base directory of current test')} 570 571 template = test.path / (file_basename + '.in') 572 destination = Path(test_build_dir) / file_basename 573 mesonlib.do_conf_file(str(template), str(destination), confdata, 'meson') 574 575 return destination 576 577def detect_parameter_files(test: TestDef, test_build_dir: str) -> T.Tuple[Path, Path]: 578 nativefile = test.path / 'nativefile.ini' 579 crossfile = test.path / 'crossfile.ini' 580 581 if os.path.exists(str(test.path / 'nativefile.ini.in')): 582 nativefile = format_parameter_file('nativefile.ini', test, test_build_dir) 583 584 if os.path.exists(str(test.path / 'crossfile.ini.in')): 585 crossfile = format_parameter_file('crossfile.ini', test, test_build_dir) 586 587 return nativefile, crossfile 588 589# In previous python versions the global variables are lost in ProcessPoolExecutor. 590# So, we use this tuple to restore some of them 591class GlobalState(T.NamedTuple): 592 compile_commands: T.List[str] 593 clean_commands: T.List[str] 594 test_commands: T.List[str] 595 install_commands: T.List[str] 596 uninstall_commands: T.List[str] 597 598 backend: 'Backend' 599 backend_flags: T.List[str] 600 601 host_c_compiler: T.Optional[str] 602 603def run_test(test: TestDef, 604 extra_args: T.List[str], 605 should_fail: str, 606 use_tmp: bool, 607 state: T.Optional[GlobalState] = None) -> T.Optional[TestResult]: 608 # Unpack the global state 609 global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands, backend, backend_flags, host_c_compiler 610 if state is not None: 611 compile_commands, clean_commands, test_commands, install_commands, uninstall_commands, backend, backend_flags, host_c_compiler = state 612 # Store that this is a worker process 613 global is_worker_process 614 is_worker_process = True 615 # Setup the test environment 616 assert not test.skip, 'Skipped thest should not be run' 617 build_dir = create_deterministic_builddir(test, use_tmp) 618 try: 619 with TemporaryDirectoryWinProof(prefix='i ', dir=None if use_tmp else os.getcwd()) as install_dir: 620 try: 621 return _run_test(test, build_dir, install_dir, extra_args, should_fail) 622 except TestResult as r: 623 return r 624 finally: 625 mlog.shutdown() # Close the log file because otherwise Windows wets itself. 626 finally: 627 mesonlib.windows_proof_rmtree(build_dir) 628 629def _run_test(test: TestDef, 630 test_build_dir: str, 631 install_dir: str, 632 extra_args: T.List[str], 633 should_fail: str) -> TestResult: 634 gen_start = time.time() 635 # Configure in-process 636 gen_args = [] # type: T.List[str] 637 if 'prefix' not in test.do_not_set_opts: 638 gen_args += ['--prefix', 'x:/usr'] if mesonlib.is_windows() else ['--prefix', '/usr'] 639 if 'libdir' not in test.do_not_set_opts: 640 gen_args += ['--libdir', 'lib'] 641 gen_args += [test.path.as_posix(), test_build_dir] + backend_flags + extra_args 642 643 nativefile, crossfile = detect_parameter_files(test, test_build_dir) 644 645 if nativefile.exists(): 646 gen_args.extend(['--native-file', nativefile.as_posix()]) 647 if crossfile.exists(): 648 gen_args.extend(['--cross-file', crossfile.as_posix()]) 649 (returncode, stdo, stde) = run_configure(gen_args, env=test.env, catch_exception=True) 650 try: 651 logfile = Path(test_build_dir, 'meson-logs', 'meson-log.txt') 652 mesonlog = logfile.open(errors='ignore', encoding='utf-8').read() 653 except Exception: 654 mesonlog = no_meson_log_msg 655 cicmds = run_ci_commands(mesonlog) 656 testresult = TestResult(cicmds) 657 testresult.add_step(BuildStep.configure, stdo, stde, mesonlog, time.time() - gen_start) 658 output_msg = validate_output(test, stdo, stde) 659 testresult.mlog += output_msg 660 if output_msg: 661 testresult.fail('Unexpected output while configuring.') 662 return testresult 663 if should_fail == 'meson': 664 if returncode == 1: 665 return testresult 666 elif returncode != 0: 667 testresult.fail(f'Test exited with unexpected status {returncode}.') 668 return testresult 669 else: 670 testresult.fail('Test that should have failed succeeded.') 671 return testresult 672 if returncode != 0: 673 testresult.fail('Generating the build system failed.') 674 return testresult 675 builddata = build.load(test_build_dir) 676 dir_args = get_backend_args_for_dir(backend, test_build_dir) 677 678 # Build with subprocess 679 def build_step() -> None: 680 build_start = time.time() 681 pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir) 682 testresult.add_step(BuildStep.build, o, e, '', time.time() - build_start) 683 if should_fail == 'build': 684 if pc.returncode != 0: 685 raise testresult 686 testresult.fail('Test that should have failed to build succeeded.') 687 raise testresult 688 if pc.returncode != 0: 689 testresult.fail('Compiling source code failed.') 690 raise testresult 691 692 # Touch the meson.build file to force a regenerate 693 def force_regenerate() -> None: 694 ensure_backend_detects_changes(backend) 695 os.utime(str(test.path / 'meson.build')) 696 697 # just test building 698 build_step() 699 700 # test that regeneration works for build step 701 force_regenerate() 702 build_step() # TBD: assert nothing gets built after the regenerate? 703 704 # test that regeneration works for test step 705 force_regenerate() 706 707 # Test in-process 708 clear_internal_caches() 709 test_start = time.time() 710 (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir) 711 testresult.add_step(BuildStep.test, tstdo, tstde, test_log, time.time() - test_start) 712 if should_fail == 'test': 713 if returncode != 0: 714 return testresult 715 testresult.fail('Test that should have failed to run unit tests succeeded.') 716 return testresult 717 if returncode != 0: 718 testresult.fail('Running unit tests failed.') 719 return testresult 720 721 # Do installation, if the backend supports it 722 if install_commands: 723 env = test.env.copy() 724 env['DESTDIR'] = install_dir 725 # Install with subprocess 726 pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env) 727 testresult.add_step(BuildStep.install, o, e) 728 if pi.returncode != 0: 729 testresult.fail('Running install failed.') 730 return testresult 731 732 # Clean with subprocess 733 env = test.env.copy() 734 pi, o, e = Popen_safe(clean_commands + dir_args, cwd=test_build_dir, env=env) 735 testresult.add_step(BuildStep.clean, o, e) 736 if pi.returncode != 0: 737 testresult.fail('Running clean failed.') 738 return testresult 739 740 # Validate installed files 741 testresult.add_step(BuildStep.install, '', '') 742 if not install_commands: 743 return testresult 744 install_msg = validate_install(test, Path(install_dir), builddata.environment) 745 if install_msg: 746 testresult.fail('\n' + install_msg) 747 return testresult 748 749 return testresult 750 751 752# processing of test.json 'skip_*' keys, which can appear at top level, or in 753# matrix: 754def _skip_keys(test_def: T.Dict) -> T.Tuple[bool, bool]: 755 skip_expected = False 756 757 # Test is expected to skip if MESON_CI_JOBNAME contains any of the list of 758 # substrings 759 if ('skip_on_jobname' in test_def) and (ci_jobname is not None): 760 skip_expected = any(s in ci_jobname for s in test_def['skip_on_jobname']) 761 762 # Test is expected to skip if os matches 763 if 'skip_on_os' in test_def: 764 mesonenv = environment.Environment(None, None, get_fake_options('/')) 765 for skip_os in test_def['skip_on_os']: 766 if skip_os.startswith('!'): 767 if mesonenv.machines.host.system != skip_os[1:]: 768 skip_expected = True 769 else: 770 if mesonenv.machines.host.system == skip_os: 771 skip_expected = True 772 773 # Skip if environment variable is present 774 skip = False 775 if 'skip_on_env' in test_def: 776 for skip_env_var in test_def['skip_on_env']: 777 if skip_env_var in os.environ: 778 skip = True 779 780 return (skip, skip_expected) 781 782 783def load_test_json(t: TestDef, stdout_mandatory: bool) -> T.List[TestDef]: 784 all_tests: T.List[TestDef] = [] 785 test_def = {} 786 test_def_file = t.path / 'test.json' 787 if test_def_file.is_file(): 788 test_def = json.loads(test_def_file.read_text(encoding='utf-8')) 789 790 # Handle additional environment variables 791 env = {} # type: T.Dict[str, str] 792 if 'env' in test_def: 793 assert isinstance(test_def['env'], dict) 794 env = test_def['env'] 795 for key, val in env.items(): 796 val = val.replace('@ROOT@', t.path.resolve().as_posix()) 797 val = val.replace('@PATH@', t.env.get('PATH', '')) 798 env[key] = val 799 800 # Handle installed files 801 installed = [] # type: T.List[InstalledFile] 802 if 'installed' in test_def: 803 installed = [InstalledFile(x) for x in test_def['installed']] 804 805 # Handle expected output 806 stdout = test_def.get('stdout', []) 807 if stdout_mandatory and not stdout: 808 raise RuntimeError(f"{test_def_file} must contain a non-empty stdout key") 809 810 # Handle the do_not_set_opts list 811 do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str] 812 813 (t.skip, t.skip_expected) = _skip_keys(test_def) 814 815 # Skip tests if the tool requirements are not met 816 if 'tools' in test_def: 817 assert isinstance(test_def['tools'], dict) 818 for tool, vers_req in test_def['tools'].items(): 819 if tool not in tool_vers_map: 820 t.skip = True 821 elif not mesonlib.version_compare(tool_vers_map[tool], vers_req): 822 t.skip = True 823 824 # Skip the matrix code and just update the existing test 825 if 'matrix' not in test_def: 826 t.env.update(env) 827 t.installed_files = installed 828 t.do_not_set_opts = do_not_set_opts 829 t.stdout = stdout 830 return [t] 831 832 new_opt_list: T.List[T.List[T.Tuple[str, bool, bool]]] 833 834 # 'matrix; entry is present, so build multiple tests from matrix definition 835 opt_list = [] # type: T.List[T.List[T.Tuple[str, bool, bool]]] 836 matrix = test_def['matrix'] 837 assert "options" in matrix 838 for key, val in matrix["options"].items(): 839 assert isinstance(val, list) 840 tmp_opts = [] # type: T.List[T.Tuple[str, bool, bool]] 841 for i in val: 842 assert isinstance(i, dict) 843 assert "val" in i 844 845 (skip, skip_expected) = _skip_keys(i) 846 847 # Only run the test if all compiler ID's match 848 if 'compilers' in i: 849 for lang, id_list in i['compilers'].items(): 850 if lang not in compiler_id_map or compiler_id_map[lang] not in id_list: 851 skip = True 852 break 853 854 # Add an empty matrix entry 855 if i['val'] is None: 856 tmp_opts += [(None, skip, skip_expected)] 857 continue 858 859 tmp_opts += [('{}={}'.format(key, i['val']), skip, skip_expected)] 860 861 if opt_list: 862 new_opt_list = [] 863 for i in opt_list: 864 for j in tmp_opts: 865 new_opt_list += [[*i, j]] 866 opt_list = new_opt_list 867 else: 868 opt_list = [[x] for x in tmp_opts] 869 870 # Exclude specific configurations 871 if 'exclude' in matrix: 872 assert isinstance(matrix['exclude'], list) 873 new_opt_list = [] 874 for i in opt_list: 875 exclude = False 876 opt_names = [x[0] for x in i] 877 for j in matrix['exclude']: 878 ex_list = [f'{k}={v}' for k, v in j.items()] 879 if all([x in opt_names for x in ex_list]): 880 exclude = True 881 break 882 883 if not exclude: 884 new_opt_list += [i] 885 886 opt_list = new_opt_list 887 888 for i in opt_list: 889 name = ' '.join([x[0] for x in i if x[0] is not None]) 890 opts = ['-D' + x[0] for x in i if x[0] is not None] 891 skip = any([x[1] for x in i]) 892 skip_expected = any([x[2] for x in i]) 893 test = TestDef(t.path, name, opts, skip or t.skip) 894 test.env.update(env) 895 test.installed_files = installed 896 test.do_not_set_opts = do_not_set_opts 897 test.stdout = stdout 898 test.skip_expected = skip_expected or t.skip_expected 899 all_tests.append(test) 900 901 return all_tests 902 903 904def gather_tests(testdir: Path, stdout_mandatory: bool, only: T.List[str]) -> T.List[TestDef]: 905 all_tests: T.List[TestDef] = [] 906 for t in testdir.iterdir(): 907 # Filter non-tests files (dot files, etc) 908 if not t.is_dir() or t.name.startswith('.'): 909 continue 910 if only and not any(t.name.startswith(prefix) for prefix in only): 911 continue 912 test_def = TestDef(t, None, []) 913 all_tests.extend(load_test_json(test_def, stdout_mandatory)) 914 return sorted(all_tests) 915 916 917def have_d_compiler() -> bool: 918 if shutil.which("ldc2"): 919 return True 920 elif shutil.which("ldc"): 921 return True 922 elif shutil.which("gdc"): 923 return True 924 elif shutil.which("dmd"): 925 # The Windows installer sometimes produces a DMD install 926 # that exists but segfaults every time the compiler is run. 927 # Don't know why. Don't know how to fix. Skip in this case. 928 cp = subprocess.run(['dmd', '--version'], 929 stdout=subprocess.PIPE, 930 stderr=subprocess.PIPE) 931 if cp.stdout == b'': 932 return False 933 return True 934 return False 935 936def have_objc_compiler(use_tmp: bool) -> bool: 937 with TemporaryDirectoryWinProof(prefix='b ', dir=None if use_tmp else '.') as build_dir: 938 env = environment.Environment(None, build_dir, get_fake_options('/')) 939 try: 940 objc_comp = detect_objc_compiler(env, MachineChoice.HOST) 941 except mesonlib.MesonException: 942 return False 943 if not objc_comp: 944 return False 945 env.coredata.process_new_compiler('objc', objc_comp, env) 946 try: 947 objc_comp.sanity_check(env.get_scratch_dir(), env) 948 except mesonlib.MesonException: 949 return False 950 return True 951 952def have_objcpp_compiler(use_tmp: bool) -> bool: 953 with TemporaryDirectoryWinProof(prefix='b ', dir=None if use_tmp else '.') as build_dir: 954 env = environment.Environment(None, build_dir, get_fake_options('/')) 955 try: 956 objcpp_comp = detect_objcpp_compiler(env, MachineChoice.HOST) 957 except mesonlib.MesonException: 958 return False 959 if not objcpp_comp: 960 return False 961 env.coredata.process_new_compiler('objcpp', objcpp_comp, env) 962 try: 963 objcpp_comp.sanity_check(env.get_scratch_dir(), env) 964 except mesonlib.MesonException: 965 return False 966 return True 967 968def have_java() -> bool: 969 if shutil.which('javac') and shutil.which('java'): 970 return True 971 return False 972 973def skip_dont_care(t: TestDef) -> bool: 974 # Everything is optional when not running on CI 975 if not under_ci: 976 return True 977 978 # Non-frameworks test are allowed to determine their own skipping under CI (currently) 979 if not t.category.endswith('frameworks'): 980 return True 981 982 # For the moment, all skips in jobs which don't set MESON_CI_JOBNAME are 983 # treated as expected. In the future, we should make it mandatory to set 984 # MESON_CI_JOBNAME for all CI jobs. 985 if ci_jobname is None: 986 return True 987 988 return False 989 990def skip_csharp(backend: Backend) -> bool: 991 if backend is not Backend.ninja: 992 return True 993 if not shutil.which('resgen'): 994 return True 995 if shutil.which('mcs'): 996 return False 997 if shutil.which('csc'): 998 # Only support VS2017 for now. Earlier versions fail 999 # under CI in mysterious ways. 1000 try: 1001 stdo = subprocess.check_output(['csc', '/version']) 1002 except subprocess.CalledProcessError: 1003 return True 1004 # Having incrementing version numbers would be too easy. 1005 # Microsoft reset the versioning back to 1.0 (from 4.x) 1006 # when they got the Roslyn based compiler. Thus there 1007 # is NO WAY to reliably do version number comparisons. 1008 # Only support the version that ships with VS2017. 1009 return not stdo.startswith(b'2.') 1010 return True 1011 1012# In Azure some setups have a broken rustc that will error out 1013# on all compilation attempts. 1014 1015def has_broken_rustc() -> bool: 1016 dirname = Path('brokenrusttest') 1017 if dirname.exists(): 1018 mesonlib.windows_proof_rmtree(dirname.as_posix()) 1019 dirname.mkdir() 1020 sanity_file = dirname / 'sanity.rs' 1021 sanity_file.write_text('fn main() {\n}\n', encoding='utf-8') 1022 pc = subprocess.run(['rustc', '-o', 'sanity.exe', 'sanity.rs'], 1023 cwd=dirname.as_posix(), 1024 stdout = subprocess.DEVNULL, 1025 stderr = subprocess.DEVNULL) 1026 mesonlib.windows_proof_rmtree(dirname.as_posix()) 1027 return pc.returncode != 0 1028 1029def should_skip_rust(backend: Backend) -> bool: 1030 if not shutil.which('rustc'): 1031 return True 1032 if backend is not Backend.ninja: 1033 return True 1034 if mesonlib.is_windows() and has_broken_rustc(): 1035 return True 1036 return False 1037 1038def detect_tests_to_run(only: T.Dict[str, T.List[str]], use_tmp: bool) -> T.List[T.Tuple[str, T.List[TestDef], bool]]: 1039 """ 1040 Parameters 1041 ---------- 1042 only: dict of categories and list of test cases, optional 1043 specify names of tests to run 1044 1045 Returns 1046 ------- 1047 gathered_tests: list of tuple of str, list of TestDef, bool 1048 tests to run 1049 """ 1050 1051 skip_fortran = not(shutil.which('gfortran') or 1052 shutil.which('flang') or 1053 shutil.which('pgfortran') or 1054 shutil.which('ifort')) 1055 1056 class TestCategory: 1057 def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandatory: bool = False): 1058 self.category = category # category name 1059 self.subdir = subdir # subdirectory 1060 self.skip = skip # skip condition 1061 self.stdout_mandatory = stdout_mandatory # expected stdout is mandatory for tests in this category 1062 1063 all_tests = [ 1064 TestCategory('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)), 1065 TestCategory('common', 'common'), 1066 TestCategory('native', 'native'), 1067 TestCategory('warning-meson', 'warning', stdout_mandatory=True), 1068 TestCategory('failing-meson', 'failing', stdout_mandatory=True), 1069 TestCategory('failing-build', 'failing build'), 1070 TestCategory('failing-test', 'failing test'), 1071 TestCategory('keyval', 'keyval'), 1072 TestCategory('platform-osx', 'osx', not mesonlib.is_osx()), 1073 TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()), 1074 TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()), 1075 TestCategory('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()), 1076 TestCategory('C#', 'csharp', skip_csharp(backend)), 1077 TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))), 1078 TestCategory('cython', 'cython', backend is not Backend.ninja or not shutil.which(os.environ.get('CYTHON', 'cython'))), 1079 TestCategory('rust', 'rust', should_skip_rust(backend)), 1080 TestCategory('d', 'd', backend is not Backend.ninja or not have_d_compiler()), 1081 TestCategory('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)), 1082 TestCategory('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)), 1083 TestCategory('fortran', 'fortran', skip_fortran or backend != Backend.ninja), 1084 TestCategory('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')), 1085 # CUDA tests on Windows: use Ninja backend: python run_project_tests.py --only cuda --backend ninja 1086 TestCategory('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')), 1087 TestCategory('python3', 'python3', backend is not Backend.ninja), 1088 TestCategory('python', 'python'), 1089 TestCategory('fpga', 'fpga', shutil.which('yosys') is None), 1090 TestCategory('frameworks', 'frameworks'), 1091 TestCategory('nasm', 'nasm'), 1092 TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja), 1093 ] 1094 1095 categories = [t.category for t in all_tests] 1096 assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories' 1097 1098 if only: 1099 for key in only.keys(): 1100 assert key in categories, f'key `{key}` is not a recognized category' 1101 all_tests = [t for t in all_tests if t.category in only.keys()] 1102 1103 gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory, only[t.category]), t.skip) for t in all_tests] 1104 return gathered_tests 1105 1106def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]], 1107 log_name_base: str, 1108 failfast: bool, 1109 extra_args: T.List[str], 1110 use_tmp: bool, 1111 num_workers: int) -> T.Tuple[int, int, int]: 1112 txtname = log_name_base + '.txt' 1113 with open(txtname, 'w', encoding='utf-8', errors='ignore') as lf: 1114 return _run_tests(all_tests, log_name_base, failfast, extra_args, use_tmp, num_workers, lf) 1115 1116class TestStatus(Enum): 1117 OK = normal_green(' [SUCCESS] ') 1118 SKIP = yellow(' [SKIPPED] ') 1119 ERROR = red(' [ERROR] ') 1120 UNEXSKIP = red('[UNEXSKIP] ') 1121 UNEXRUN = red(' [UNEXRUN] ') 1122 CANCELED = cyan('[CANCELED] ') 1123 RUNNING = blue(' [RUNNING] ') # Should never be actually printed 1124 LOG = bold(' [LOG] ') # Should never be actually printed 1125 1126def default_print(*args: mlog.TV_Loggable, sep: str = ' ') -> None: 1127 print(*args, sep=sep) 1128 1129safe_print = default_print 1130 1131class TestRunFuture: 1132 def __init__(self, name: str, testdef: TestDef, future: T.Optional['Future[T.Optional[TestResult]]']) -> None: 1133 super().__init__() 1134 self.name = name 1135 self.testdef = testdef 1136 self.future = future 1137 self.status = TestStatus.RUNNING if self.future is not None else TestStatus.SKIP 1138 1139 @property 1140 def result(self) -> T.Optional[TestResult]: 1141 return self.future.result() if self.future else None 1142 1143 def log(self) -> None: 1144 without_install = '' if install_commands else '(without install)' 1145 safe_print(self.status.value, without_install, *self.testdef.display_name()) 1146 1147 def update_log(self, new_status: TestStatus) -> None: 1148 self.status = new_status 1149 self.log() 1150 1151 def cancel(self) -> None: 1152 if self.future is not None and self.future.cancel(): 1153 self.status = TestStatus.CANCELED 1154 1155class LogRunFuture: 1156 def __init__(self, msgs: mlog.TV_LoggableList) -> None: 1157 self.msgs = msgs 1158 self.status = TestStatus.LOG 1159 1160 def log(self) -> None: 1161 safe_print(*self.msgs, sep='') 1162 1163 def cancel(self) -> None: 1164 pass 1165 1166RunFutureUnion = T.Union[TestRunFuture, LogRunFuture] 1167 1168def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]], 1169 log_name_base: str, 1170 failfast: bool, 1171 extra_args: T.List[str], 1172 use_tmp: bool, 1173 num_workers: int, 1174 logfile: T.TextIO) -> T.Tuple[int, int, int]: 1175 global stop, host_c_compiler 1176 xmlname = log_name_base + '.xml' 1177 junit_root = ET.Element('testsuites') 1178 conf_time: float = 0 1179 build_time: float = 0 1180 test_time: float = 0 1181 passing_tests = 0 1182 failing_tests = 0 1183 skipped_tests = 0 1184 1185 print(f'\nRunning tests with {num_workers} workers') 1186 1187 # Pack the global state 1188 state = (compile_commands, clean_commands, test_commands, install_commands, uninstall_commands, backend, backend_flags, host_c_compiler) 1189 executor = ProcessPoolExecutor(max_workers=num_workers) 1190 1191 futures: T.List[RunFutureUnion] = [] 1192 1193 # First, collect and start all tests and also queue log messages 1194 for name, test_cases, skipped in all_tests: 1195 current_suite = ET.SubElement(junit_root, 'testsuite', {'name': name, 'tests': str(len(test_cases))}) 1196 if skipped: 1197 futures += [LogRunFuture(['\n', bold(f'Not running {name} tests.'), '\n'])] 1198 else: 1199 futures += [LogRunFuture(['\n', bold(f'Running {name} tests.'), '\n'])] 1200 1201 for t in test_cases: 1202 # Jenkins screws us over by automatically sorting test cases by name 1203 # and getting it wrong by not doing logical number sorting. 1204 (testnum, testbase) = t.path.name.split(' ', 1) 1205 testname = '%.3d %s' % (int(testnum), testbase) 1206 if t.name: 1207 testname += f' ({t.name})' 1208 should_fail = '' 1209 suite_args = [] 1210 if name.startswith('failing'): 1211 should_fail = name.split('failing-')[1] 1212 if name.startswith('warning'): 1213 suite_args = ['--fatal-meson-warnings'] 1214 should_fail = name.split('warning-')[1] 1215 1216 if skipped or t.skip: 1217 futures += [TestRunFuture(testname, t, None)] 1218 continue 1219 result_future = executor.submit(run_test, t, extra_args + suite_args + t.args, should_fail, use_tmp, state=state) 1220 futures += [TestRunFuture(testname, t, result_future)] 1221 1222 # Ensure we only cancel once 1223 tests_canceled = False 1224 1225 # Optionally enable the tqdm progress bar 1226 global safe_print 1227 futures_iter: T.Iterable[RunFutureUnion] = futures 1228 try: 1229 from tqdm import tqdm 1230 futures_iter = tqdm(futures, desc='Running tests', unit='test') 1231 1232 def tqdm_print(*args: mlog.TV_Loggable, sep: str = ' ') -> None: 1233 tqdm.write(sep.join([str(x) for x in args])) 1234 1235 safe_print = tqdm_print 1236 except ImportError: 1237 pass 1238 1239 # Wait and handle the test results and print the stored log output 1240 for f in futures_iter: 1241 # Just a log entry to print something to stdout 1242 sys.stdout.flush() 1243 if isinstance(f, LogRunFuture): 1244 f.log() 1245 continue 1246 1247 # Acutal Test run 1248 testname = f.name 1249 t = f.testdef 1250 try: 1251 result = f.result 1252 except (CancelledError, KeyboardInterrupt): 1253 f.status = TestStatus.CANCELED 1254 1255 if stop and not tests_canceled: 1256 num_running = sum([1 if f2.status is TestStatus.RUNNING else 0 for f2 in futures]) 1257 for f2 in futures: 1258 f2.cancel() 1259 executor.shutdown() 1260 num_canceled = sum([1 if f2.status is TestStatus.CANCELED else 0 for f2 in futures]) 1261 safe_print(f'\nCanceled {num_canceled} out of {num_running} running tests.') 1262 safe_print(f'Finishing the remaining {num_running - num_canceled} tests.\n') 1263 tests_canceled = True 1264 1265 # Handle canceled tests 1266 if f.status is TestStatus.CANCELED: 1267 f.log() 1268 continue 1269 1270 # Handle skipped tests 1271 if result is None: 1272 # skipped due to skipped category skip or 'tools:' or 'skip_on_env:' 1273 is_skipped = True 1274 skip_as_expected = True 1275 else: 1276 # skipped due to test outputting 'MESON_SKIP_TEST' 1277 is_skipped = 'MESON_SKIP_TEST' in result.stdo 1278 if not skip_dont_care(t): 1279 skip_as_expected = (is_skipped == t.skip_expected) 1280 else: 1281 skip_as_expected = True 1282 1283 if is_skipped: 1284 skipped_tests += 1 1285 1286 if is_skipped and skip_as_expected: 1287 f.update_log(TestStatus.SKIP) 1288 current_test = ET.SubElement(current_suite, 'testcase', {'name': testname, 'classname': t.category}) 1289 ET.SubElement(current_test, 'skipped', {}) 1290 continue 1291 1292 if not skip_as_expected: 1293 failing_tests += 1 1294 if is_skipped: 1295 skip_msg = 'Test asked to be skipped, but was not expected to' 1296 status = TestStatus.UNEXSKIP 1297 else: 1298 skip_msg = 'Test ran, but was expected to be skipped' 1299 status = TestStatus.UNEXRUN 1300 result.msg = "%s for MESON_CI_JOBNAME '%s'" % (skip_msg, ci_jobname) 1301 1302 f.update_log(status) 1303 current_test = ET.SubElement(current_suite, 'testcase', {'name': testname, 'classname': t.category}) 1304 ET.SubElement(current_test, 'failure', {'message': result.msg}) 1305 continue 1306 1307 # Handle Failed tests 1308 if result.msg != '': 1309 f.update_log(TestStatus.ERROR) 1310 safe_print(bold('During:'), result.step.name) 1311 safe_print(bold('Reason:'), result.msg) 1312 failing_tests += 1 1313 # Append a visual seperator for the different test cases 1314 cols = shutil.get_terminal_size((100, 20)).columns 1315 name_str = ' '.join([str(x) for x in f.testdef.display_name()]) 1316 name_len = len(re.sub(r'\x1B[^m]+m', '', name_str)) # Do not count escape sequences 1317 left_w = (cols // 2) - (name_len // 2) - 1 1318 left_w = max(3, left_w) 1319 right_w = cols - left_w - name_len - 2 1320 right_w = max(3, right_w) 1321 failing_logs.append(f'\n\x1b[31m{"="*left_w}\x1b[0m {name_str} \x1b[31m{"="*right_w}\x1b[0m\n') 1322 if result.step == BuildStep.configure and result.mlog != no_meson_log_msg: 1323 # For configure failures, instead of printing stdout, 1324 # print the meson log if available since it's a superset 1325 # of stdout and often has very useful information. 1326 failing_logs.append(result.mlog) 1327 elif under_ci: 1328 # Always print the complete meson log when running in 1329 # a CI. This helps debugging issues that only occur in 1330 # a hard to reproduce environment 1331 failing_logs.append(result.mlog) 1332 failing_logs.append(result.stdo) 1333 else: 1334 failing_logs.append(result.stdo) 1335 for cmd_res in result.cicmds: 1336 failing_logs.append(cmd_res) 1337 failing_logs.append(result.stde) 1338 if failfast: 1339 safe_print("Cancelling the rest of the tests") 1340 for f2 in futures: 1341 f2.cancel() 1342 else: 1343 f.update_log(TestStatus.OK) 1344 passing_tests += 1 1345 conf_time += result.conftime 1346 build_time += result.buildtime 1347 test_time += result.testtime 1348 total_time = conf_time + build_time + test_time 1349 log_text_file(logfile, t.path, result) 1350 current_test = ET.SubElement( 1351 current_suite, 1352 'testcase', 1353 {'name': testname, 'classname': t.category, 'time': '%.3f' % total_time} 1354 ) 1355 if result.msg != '': 1356 ET.SubElement(current_test, 'failure', {'message': result.msg}) 1357 stdoel = ET.SubElement(current_test, 'system-out') 1358 stdoel.text = result.stdo 1359 stdeel = ET.SubElement(current_test, 'system-err') 1360 stdeel.text = result.stde 1361 1362 # Reset, just in case 1363 safe_print = default_print 1364 1365 print() 1366 print("Total configuration time: %.2fs" % conf_time) 1367 print("Total build time: %.2fs" % build_time) 1368 print("Total test time: %.2fs" % test_time) 1369 ET.ElementTree(element=junit_root).write(xmlname, xml_declaration=True, encoding='UTF-8') 1370 return passing_tests, failing_tests, skipped_tests 1371 1372def check_meson_commands_work(use_tmpdir: bool, extra_args: T.List[str]) -> None: 1373 global backend, compile_commands, test_commands, install_commands 1374 testdir = PurePath('test cases', 'common', '1 trivial').as_posix() 1375 meson_commands = mesonlib.python_command + [get_meson_script()] 1376 with TemporaryDirectoryWinProof(prefix='b ', dir=None if use_tmpdir else '.') as build_dir: 1377 print('Checking that configuring works...') 1378 gen_cmd = meson_commands + [testdir, build_dir] + backend_flags + extra_args 1379 pc, o, e = Popen_safe(gen_cmd) 1380 if pc.returncode != 0: 1381 raise RuntimeError(f'Failed to configure {testdir!r}:\n{e}\n{o}') 1382 print('Checking that introspect works...') 1383 pc, o, e = Popen_safe(meson_commands + ['introspect', '--targets'], cwd=build_dir) 1384 json.loads(o) 1385 if pc.returncode != 0: 1386 raise RuntimeError(f'Failed to introspect --targets {testdir!r}:\n{e}\n{o}') 1387 print('Checking that building works...') 1388 dir_args = get_backend_args_for_dir(backend, build_dir) 1389 pc, o, e = Popen_safe(compile_commands + dir_args, cwd=build_dir) 1390 if pc.returncode != 0: 1391 raise RuntimeError(f'Failed to build {testdir!r}:\n{e}\n{o}') 1392 print('Checking that testing works...') 1393 pc, o, e = Popen_safe(test_commands, cwd=build_dir) 1394 if pc.returncode != 0: 1395 raise RuntimeError(f'Failed to test {testdir!r}:\n{e}\n{o}') 1396 if install_commands: 1397 print('Checking that installing works...') 1398 pc, o, e = Popen_safe(install_commands, cwd=build_dir) 1399 if pc.returncode != 0: 1400 raise RuntimeError(f'Failed to install {testdir!r}:\n{e}\n{o}') 1401 1402 1403def detect_system_compiler(options: 'CompilerArgumentType') -> None: 1404 global host_c_compiler, compiler_id_map 1405 1406 fake_opts = get_fake_options('/') 1407 if options.cross_file: 1408 fake_opts.cross_file = [options.cross_file] 1409 if options.native_file: 1410 fake_opts.native_file = [options.native_file] 1411 1412 env = environment.Environment(None, None, fake_opts) 1413 1414 print_compilers(env, MachineChoice.HOST) 1415 if options.cross_file: 1416 print_compilers(env, MachineChoice.BUILD) 1417 1418 for lang in sorted(compilers.all_languages): 1419 try: 1420 comp = compiler_from_language(env, lang, MachineChoice.HOST) 1421 # note compiler id for later use with test.json matrix 1422 compiler_id_map[lang] = comp.get_id() 1423 except mesonlib.MesonException: 1424 comp = None 1425 1426 # note C compiler for later use by platform_fix_name() 1427 if lang == 'c': 1428 if comp: 1429 host_c_compiler = comp.get_id() 1430 else: 1431 raise RuntimeError("Could not find C compiler.") 1432 1433 1434def print_compilers(env: 'Environment', machine: MachineChoice) -> None: 1435 print() 1436 print(f'{machine.get_lower_case_name()} machine compilers') 1437 print() 1438 for lang in sorted(compilers.all_languages): 1439 try: 1440 comp = compiler_from_language(env, lang, machine) 1441 details = '{:<10} {} {}'.format('[' + comp.get_id() + ']', ' '.join(comp.get_exelist()), comp.get_version_string()) 1442 except mesonlib.MesonException: 1443 details = '[not found]' 1444 print(f'{lang:<7}: {details}') 1445 1446class ToolInfo(T.NamedTuple): 1447 tool: str 1448 args: T.List[str] 1449 regex: T.Pattern 1450 match_group: int 1451 1452def print_tool_versions() -> None: 1453 tools: T.List[ToolInfo] = [ 1454 ToolInfo( 1455 'ninja', 1456 ['--version'], 1457 re.compile(r'^([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'), 1458 1, 1459 ), 1460 ToolInfo( 1461 'cmake', 1462 ['--version'], 1463 re.compile(r'^cmake version ([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'), 1464 1, 1465 ), 1466 ToolInfo( 1467 'hotdoc', 1468 ['--version'], 1469 re.compile(r'^([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'), 1470 1, 1471 ), 1472 ] 1473 1474 def get_version(t: ToolInfo) -> str: 1475 exe = shutil.which(t.tool) 1476 if not exe: 1477 return 'not found' 1478 1479 args = [t.tool] + t.args 1480 pc, o, e = Popen_safe(args) 1481 if pc.returncode != 0: 1482 return '{} (invalid {} executable)'.format(exe, t.tool) 1483 for i in o.split('\n'): 1484 i = i.strip('\n\r\t ') 1485 m = t.regex.match(i) 1486 if m is not None: 1487 tool_vers_map[t.tool] = m.group(t.match_group) 1488 return '{} ({})'.format(exe, m.group(t.match_group)) 1489 1490 return f'{exe} (unknown)' 1491 1492 print() 1493 print('tools') 1494 print() 1495 1496 max_width = max([len(x.tool) for x in tools] + [7]) 1497 for tool in tools: 1498 print('{0:<{2}}: {1}'.format(tool.tool, get_version(tool), max_width)) 1499 print() 1500 1501def clear_transitive_files() -> None: 1502 a = Path('test cases/common') 1503 for d in a.glob('*subproject subdir/subprojects/subsubsub*'): 1504 if d.is_dir(): 1505 mesonlib.windows_proof_rmtree(str(d)) 1506 else: 1507 mesonlib.windows_proof_rm(str(d)) 1508 1509if __name__ == '__main__': 1510 setup_vsenv() 1511 1512 try: 1513 # This fails in some CI environments for unknown reasons. 1514 num_workers = multiprocessing.cpu_count() 1515 except Exception as e: 1516 print('Could not determine number of CPUs due to the following reason:' + str(e)) 1517 print('Defaulting to using only two processes') 1518 num_workers = 2 1519 # Due to Ninja deficiency, almost 50% of build time 1520 # is spent waiting. Do something useful instead. 1521 # 1522 # Remove this once the following issue has been resolved: 1523 # https://github.com/mesonbuild/meson/pull/2082 1524 if not mesonlib.is_windows(): # twice as fast on Windows by *not* multiplying by 2. 1525 num_workers *= 2 1526 1527 parser = argparse.ArgumentParser(description="Run the test suite of Meson.") 1528 parser.add_argument('extra_args', nargs='*', 1529 help='arguments that are passed directly to Meson (remember to have -- before these).') 1530 parser.add_argument('--backend', dest='backend', choices=backendlist) 1531 parser.add_argument('-j', dest='num_workers', type=int, default=num_workers, 1532 help=f'Maximum number of parallel tests (default {num_workers})') 1533 parser.add_argument('--failfast', action='store_true', 1534 help='Stop running if test case fails') 1535 parser.add_argument('--no-unittests', action='store_true', 1536 help='Not used, only here to simplify run_tests.py') 1537 parser.add_argument('--only', default=[], 1538 help='name of test(s) to run, in format "category[/name]" where category is one of: ' + ', '.join(ALL_TESTS), nargs='+') 1539 parser.add_argument('--cross-file', action='store', help='File describing cross compilation environment.') 1540 parser.add_argument('--native-file', action='store', help='File describing native compilation environment.') 1541 parser.add_argument('--use-tmpdir', action='store_true', help='Use tmp directory for temporary files.') 1542 options = T.cast('ArgumentType', parser.parse_args()) 1543 1544 if options.cross_file: 1545 options.extra_args += ['--cross-file', options.cross_file] 1546 if options.native_file: 1547 options.extra_args += ['--native-file', options.native_file] 1548 1549 clear_transitive_files() 1550 1551 print('Meson build system', meson_version, 'Project Tests') 1552 print('Using python', sys.version.split('\n')[0]) 1553 if 'VSCMD_VER' in os.environ: 1554 print('VSCMD version', os.environ['VSCMD_VER']) 1555 setup_commands(options.backend) 1556 detect_system_compiler(options) 1557 print_tool_versions() 1558 script_dir = os.path.split(__file__)[0] 1559 if script_dir != '': 1560 os.chdir(script_dir) 1561 check_meson_commands_work(options.use_tmpdir, options.extra_args) 1562 only = collections.defaultdict(list) 1563 for i in options.only: 1564 try: 1565 cat, case = i.split('/') 1566 only[cat].append(case) 1567 except ValueError: 1568 only[i].append('') 1569 try: 1570 all_tests = detect_tests_to_run(only, options.use_tmpdir) 1571 res = run_tests(all_tests, 'meson-test-run', options.failfast, options.extra_args, options.use_tmpdir, options.num_workers) 1572 (passing_tests, failing_tests, skipped_tests) = res 1573 except StopException: 1574 pass 1575 print() 1576 print('Total passed tests: ', green(str(passing_tests))) 1577 print('Total failed tests: ', red(str(failing_tests))) 1578 print('Total skipped tests:', yellow(str(skipped_tests))) 1579 if failing_tests > 0: 1580 print('\nMesonlogs of failing tests\n') 1581 for l in failing_logs: 1582 try: 1583 print(l, '\n') 1584 except UnicodeError: 1585 print(l.encode('ascii', errors='replace').decode(), '\n') 1586 for name, dirs, _ in all_tests: 1587 dir_names = list({x.path.name for x in dirs}) 1588 for k, g in itertools.groupby(dir_names, key=lambda x: x.split()[0]): 1589 tests = list(g) 1590 if len(tests) != 1: 1591 print('WARNING: The {} suite contains duplicate "{}" tests: "{}"'.format(name, k, '", "'.join(tests))) 1592 clear_transitive_files() 1593 raise SystemExit(failing_tests) 1594