1# Class for actually running tests. 2# 3# Copyright (c) 2020-2021 Virtuozzo International GmbH 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program. If not, see <http://www.gnu.org/licenses/>. 17# 18 19import os 20from pathlib import Path 21import datetime 22import time 23import difflib 24import subprocess 25import contextlib 26import json 27import shutil 28import sys 29from multiprocessing import Pool 30from typing import List, Optional, Any, Sequence, Dict, \ 31 ContextManager 32 33from testenv import TestEnv 34 35 36def silent_unlink(path: Path) -> None: 37 try: 38 path.unlink() 39 except OSError: 40 pass 41 42 43def file_diff(file1: str, file2: str) -> List[str]: 44 with open(file1, encoding="utf-8") as f1, \ 45 open(file2, encoding="utf-8") as f2: 46 # We want to ignore spaces at line ends. There are a lot of mess about 47 # it in iotests. 48 # TODO: fix all tests to not produce extra spaces, fix all .out files 49 # and use strict diff here! 50 seq1 = [line.rstrip() for line in f1] 51 seq2 = [line.rstrip() for line in f2] 52 res = [line.rstrip() 53 for line in difflib.unified_diff(seq1, seq2, file1, file2)] 54 return res 55 56 57class LastElapsedTime(ContextManager['LastElapsedTime']): 58 """ Cache for elapsed time for tests, to show it during new test run 59 60 It is safe to use get() at any time. To use update(), you must either 61 use it inside with-block or use save() after update(). 62 """ 63 def __init__(self, cache_file: str, env: TestEnv) -> None: 64 self.env = env 65 self.cache_file = cache_file 66 self.cache: Dict[str, Dict[str, Dict[str, float]]] 67 68 try: 69 with open(cache_file, encoding="utf-8") as f: 70 self.cache = json.load(f) 71 except (OSError, ValueError): 72 self.cache = {} 73 74 def get(self, test: str, 75 default: Optional[float] = None) -> Optional[float]: 76 if test not in self.cache: 77 return default 78 79 if self.env.imgproto not in self.cache[test]: 80 return default 81 82 return self.cache[test][self.env.imgproto].get(self.env.imgfmt, 83 default) 84 85 def update(self, test: str, elapsed: float) -> None: 86 d = self.cache.setdefault(test, {}) 87 d.setdefault(self.env.imgproto, {})[self.env.imgfmt] = elapsed 88 89 def save(self) -> None: 90 with open(self.cache_file, 'w', encoding="utf-8") as f: 91 json.dump(self.cache, f) 92 93 def __enter__(self) -> 'LastElapsedTime': 94 return self 95 96 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 97 self.save() 98 99 100class TestResult: 101 def __init__(self, status: str, description: str = '', 102 elapsed: Optional[float] = None, diff: Sequence[str] = (), 103 casenotrun: str = '', interrupted: bool = False) -> None: 104 self.status = status 105 self.description = description 106 self.elapsed = elapsed 107 self.diff = diff 108 self.casenotrun = casenotrun 109 self.interrupted = interrupted 110 111 112class TestRunner(ContextManager['TestRunner']): 113 shared_self = None 114 115 @staticmethod 116 def proc_run_test(test: str, test_field_width: int) -> TestResult: 117 # We are in a subprocess, we can't change the runner object! 118 runner = TestRunner.shared_self 119 assert runner is not None 120 return runner.run_test(test, test_field_width, mp=True) 121 122 def run_tests_pool(self, tests: List[str], 123 test_field_width: int, jobs: int) -> List[TestResult]: 124 125 # passing self directly to Pool.starmap() just doesn't work, because 126 # it's a context manager. 127 assert TestRunner.shared_self is None 128 TestRunner.shared_self = self 129 130 with Pool(jobs) as p: 131 results = p.starmap(self.proc_run_test, 132 zip(tests, [test_field_width] * len(tests))) 133 134 TestRunner.shared_self = None 135 136 return results 137 138 def __init__(self, env: TestEnv, tap: bool = False, 139 color: str = 'auto') -> None: 140 self.env = env 141 self.tap = tap 142 self.last_elapsed = LastElapsedTime('.last-elapsed-cache', env) 143 144 assert color in ('auto', 'on', 'off') 145 self.color = (color == 'on') or (color == 'auto' and 146 sys.stdout.isatty()) 147 148 self._stack: contextlib.ExitStack 149 150 def __enter__(self) -> 'TestRunner': 151 self._stack = contextlib.ExitStack() 152 self._stack.enter_context(self.env) 153 self._stack.enter_context(self.last_elapsed) 154 return self 155 156 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 157 self._stack.close() 158 159 def test_print_one_line(self, test: str, 160 test_field_width: int, 161 starttime: str, 162 endtime: Optional[str] = None, status: str = '...', 163 lasttime: Optional[float] = None, 164 thistime: Optional[float] = None, 165 description: str = '', 166 end: str = '\n') -> None: 167 """ Print short test info before/after test run """ 168 test = os.path.basename(test) 169 170 if test_field_width is None: 171 test_field_width = 8 172 173 if self.tap: 174 if status == 'pass': 175 print(f'ok {self.env.imgfmt} {test}') 176 elif status == 'fail': 177 print(f'not ok {self.env.imgfmt} {test}') 178 elif status == 'not run': 179 print(f'ok {self.env.imgfmt} {test} # SKIP') 180 return 181 182 if lasttime: 183 lasttime_s = f' (last: {lasttime:.1f}s)' 184 else: 185 lasttime_s = '' 186 if thistime: 187 thistime_s = f'{thistime:.1f}s' 188 else: 189 thistime_s = '...' 190 191 if endtime: 192 endtime = f'[{endtime}]' 193 else: 194 endtime = '' 195 196 if self.color: 197 if status == 'pass': 198 col = '\033[32m' 199 elif status == 'fail': 200 col = '\033[1m\033[31m' 201 elif status == 'not run': 202 col = '\033[33m' 203 else: 204 col = '' 205 206 col_end = '\033[0m' 207 else: 208 col = '' 209 col_end = '' 210 211 print(f'{test:{test_field_width}} {col}{status:10}{col_end} ' 212 f'[{starttime}] {endtime:13}{thistime_s:5} {lasttime_s:14} ' 213 f'{description}', end=end) 214 215 def find_reference(self, test: str) -> str: 216 if self.env.cachemode == 'none': 217 ref = f'{test}.out.nocache' 218 if os.path.isfile(ref): 219 return ref 220 221 ref = f'{test}.out.{self.env.imgfmt}' 222 if os.path.isfile(ref): 223 return ref 224 225 ref = f'{test}.{self.env.qemu_default_machine}.out' 226 if os.path.isfile(ref): 227 return ref 228 229 return f'{test}.out' 230 231 def do_run_test(self, test: str) -> TestResult: 232 """ 233 Run one test 234 235 :param test: test file path 236 237 Note: this method may be called from subprocess, so it does not 238 change ``self`` object in any way! 239 """ 240 241 f_test = Path(test) 242 f_reference = Path(self.find_reference(test)) 243 244 if not f_test.exists(): 245 return TestResult(status='fail', 246 description=f'No such test file: {f_test}') 247 248 if not os.access(str(f_test), os.X_OK): 249 sys.exit(f'Not executable: {f_test}') 250 251 if not f_reference.exists(): 252 return TestResult(status='not run', 253 description='No qualified output ' 254 f'(expected {f_reference})') 255 256 args = [str(f_test.resolve())] 257 env = self.env.prepare_subprocess(args) 258 259 # Split test directories, so that tests running in parallel don't 260 # break each other. 261 for d in ['TEST_DIR', 'SOCK_DIR']: 262 env[d] = os.path.join( 263 env[d], 264 f"{self.env.imgfmt}-{self.env.imgproto}-{f_test.name}") 265 Path(env[d]).mkdir(parents=True, exist_ok=True) 266 267 test_dir = env['TEST_DIR'] 268 f_bad = Path(test_dir, f_test.name + '.out.bad') 269 f_notrun = Path(test_dir, f_test.name + '.notrun') 270 f_casenotrun = Path(test_dir, f_test.name + '.casenotrun') 271 272 for p in (f_notrun, f_casenotrun): 273 silent_unlink(p) 274 275 t0 = time.time() 276 with f_bad.open('w', encoding="utf-8") as f: 277 with subprocess.Popen(args, cwd=str(f_test.parent), env=env, 278 stdin=subprocess.DEVNULL, 279 stdout=f, stderr=subprocess.STDOUT) as proc: 280 try: 281 proc.wait() 282 except KeyboardInterrupt: 283 proc.terminate() 284 proc.wait() 285 return TestResult(status='not run', 286 description='Interrupted by user', 287 interrupted=True) 288 ret = proc.returncode 289 290 elapsed = round(time.time() - t0, 1) 291 292 if ret != 0: 293 return TestResult(status='fail', elapsed=elapsed, 294 description=f'failed, exit status {ret}', 295 diff=file_diff(str(f_reference), str(f_bad))) 296 297 if f_notrun.exists(): 298 return TestResult( 299 status='not run', 300 description=f_notrun.read_text(encoding='utf-8').strip()) 301 302 casenotrun = '' 303 if f_casenotrun.exists(): 304 casenotrun = f_casenotrun.read_text(encoding='utf-8') 305 306 diff = file_diff(str(f_reference), str(f_bad)) 307 if diff: 308 if os.environ.get("QEMU_IOTESTS_REGEN", None) is not None: 309 shutil.copyfile(str(f_bad), str(f_reference)) 310 print("########################################") 311 print("##### REFERENCE FILE UPDATED #####") 312 print("########################################") 313 return TestResult(status='fail', elapsed=elapsed, 314 description=f'output mismatch (see {f_bad})', 315 diff=diff, casenotrun=casenotrun) 316 else: 317 f_bad.unlink() 318 return TestResult(status='pass', elapsed=elapsed, 319 casenotrun=casenotrun) 320 321 def run_test(self, test: str, 322 test_field_width: int, 323 mp: bool = False) -> TestResult: 324 """ 325 Run one test and print short status 326 327 :param test: test file path 328 :param test_field_width: width for first field of status format 329 :param mp: if true, we are in a multiprocessing environment, don't try 330 to rewrite things in stdout 331 332 Note: this method may be called from subprocess, so it does not 333 change ``self`` object in any way! 334 """ 335 336 last_el = self.last_elapsed.get(test) 337 start = datetime.datetime.now().strftime('%H:%M:%S') 338 339 if not self.tap: 340 self.test_print_one_line(test=test, 341 test_field_width=test_field_width, 342 status = 'started' if mp else '...', 343 starttime=start, 344 lasttime=last_el, 345 end = '\n' if mp else '\r') 346 else: 347 testname = os.path.basename(test) 348 print(f'# running {self.env.imgfmt} {testname}') 349 350 res = self.do_run_test(test) 351 352 end = datetime.datetime.now().strftime('%H:%M:%S') 353 self.test_print_one_line(test=test, 354 test_field_width=test_field_width, 355 status=res.status, 356 starttime=start, endtime=end, 357 lasttime=last_el, thistime=res.elapsed, 358 description=res.description) 359 360 if res.casenotrun: 361 if self.tap: 362 print('#' + res.casenotrun.replace('\n', '\n#')) 363 else: 364 print(res.casenotrun) 365 366 sys.stdout.flush() 367 return res 368 369 def run_tests(self, tests: List[str], jobs: int = 1) -> bool: 370 n_run = 0 371 failed = [] 372 notrun = [] 373 casenotrun = [] 374 375 if self.tap: 376 print('TAP version 13') 377 self.env.print_env('# ') 378 print('1..%d' % len(tests)) 379 else: 380 self.env.print_env() 381 382 test_field_width = max(len(os.path.basename(t)) for t in tests) + 2 383 384 if jobs > 1: 385 results = self.run_tests_pool(tests, test_field_width, jobs) 386 387 for i, t in enumerate(tests): 388 name = os.path.basename(t) 389 390 if jobs > 1: 391 res = results[i] 392 else: 393 res = self.run_test(t, test_field_width) 394 395 assert res.status in ('pass', 'fail', 'not run') 396 397 if res.casenotrun: 398 casenotrun.append(t) 399 400 if res.status != 'not run': 401 n_run += 1 402 403 if res.status == 'fail': 404 failed.append(name) 405 if res.diff: 406 if self.tap: 407 print('\n'.join(res.diff), file=sys.stderr) 408 else: 409 print('\n'.join(res.diff)) 410 elif res.status == 'not run': 411 notrun.append(name) 412 elif res.status == 'pass': 413 assert res.elapsed is not None 414 self.last_elapsed.update(t, res.elapsed) 415 416 sys.stdout.flush() 417 if res.interrupted: 418 break 419 420 if not self.tap: 421 if notrun: 422 print('Not run:', ' '.join(notrun)) 423 424 if casenotrun: 425 print('Some cases not run in:', ' '.join(casenotrun)) 426 427 if failed: 428 print('Failures:', ' '.join(failed)) 429 print(f'Failed {len(failed)} of {n_run} iotests') 430 else: 431 print(f'Passed all {n_run} iotests') 432 return not failed 433