1# Class for actually running tests. 2# 3# Copyright (c) 2020-2021 Virtuozzo International GmbH 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program. If not, see <http://www.gnu.org/licenses/>. 17# 18 19import os 20from pathlib import Path 21import datetime 22import time 23import difflib 24import subprocess 25import contextlib 26import json 27import termios 28import sys 29from multiprocessing import Pool 30from contextlib import contextmanager 31from typing import List, Optional, Iterator, Any, Sequence, Dict, \ 32 ContextManager 33 34from testenv import TestEnv 35 36 37def silent_unlink(path: Path) -> None: 38 try: 39 path.unlink() 40 except OSError: 41 pass 42 43 44def file_diff(file1: str, file2: str) -> List[str]: 45 with open(file1, encoding="utf-8") as f1, \ 46 open(file2, encoding="utf-8") as f2: 47 # We want to ignore spaces at line ends. There are a lot of mess about 48 # it in iotests. 49 # TODO: fix all tests to not produce extra spaces, fix all .out files 50 # and use strict diff here! 51 seq1 = [line.rstrip() for line in f1] 52 seq2 = [line.rstrip() for line in f2] 53 res = [line.rstrip() 54 for line in difflib.unified_diff(seq1, seq2, file1, file2)] 55 return res 56 57 58# We want to save current tty settings during test run, 59# since an aborting qemu call may leave things screwed up. 60@contextmanager 61def savetty() -> Iterator[None]: 62 isterm = sys.stdin.isatty() 63 if isterm: 64 fd = sys.stdin.fileno() 65 attr = termios.tcgetattr(fd) 66 67 try: 68 yield 69 finally: 70 if isterm: 71 termios.tcsetattr(fd, termios.TCSADRAIN, attr) 72 73 74class LastElapsedTime(ContextManager['LastElapsedTime']): 75 """ Cache for elapsed time for tests, to show it during new test run 76 77 It is safe to use get() at any time. To use update(), you must either 78 use it inside with-block or use save() after update(). 79 """ 80 def __init__(self, cache_file: str, env: TestEnv) -> None: 81 self.env = env 82 self.cache_file = cache_file 83 self.cache: Dict[str, Dict[str, Dict[str, float]]] 84 85 try: 86 with open(cache_file, encoding="utf-8") as f: 87 self.cache = json.load(f) 88 except (OSError, ValueError): 89 self.cache = {} 90 91 def get(self, test: str, 92 default: Optional[float] = None) -> Optional[float]: 93 if test not in self.cache: 94 return default 95 96 if self.env.imgproto not in self.cache[test]: 97 return default 98 99 return self.cache[test][self.env.imgproto].get(self.env.imgfmt, 100 default) 101 102 def update(self, test: str, elapsed: float) -> None: 103 d = self.cache.setdefault(test, {}) 104 d.setdefault(self.env.imgproto, {})[self.env.imgfmt] = elapsed 105 106 def save(self) -> None: 107 with open(self.cache_file, 'w', encoding="utf-8") as f: 108 json.dump(self.cache, f) 109 110 def __enter__(self) -> 'LastElapsedTime': 111 return self 112 113 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 114 self.save() 115 116 117class TestResult: 118 def __init__(self, status: str, description: str = '', 119 elapsed: Optional[float] = None, diff: Sequence[str] = (), 120 casenotrun: str = '', interrupted: bool = False) -> None: 121 self.status = status 122 self.description = description 123 self.elapsed = elapsed 124 self.diff = diff 125 self.casenotrun = casenotrun 126 self.interrupted = interrupted 127 128 129class TestRunner(ContextManager['TestRunner']): 130 shared_self = None 131 132 @staticmethod 133 def proc_run_test(test: str, test_field_width: int) -> TestResult: 134 # We are in a subprocess, we can't change the runner object! 135 runner = TestRunner.shared_self 136 assert runner is not None 137 return runner.run_test(test, test_field_width, mp=True) 138 139 def run_tests_pool(self, tests: List[str], 140 test_field_width: int, jobs: int) -> List[TestResult]: 141 142 # passing self directly to Pool.starmap() just doesn't work, because 143 # it's a context manager. 144 assert TestRunner.shared_self is None 145 TestRunner.shared_self = self 146 147 with Pool(jobs) as p: 148 results = p.starmap(self.proc_run_test, 149 zip(tests, [test_field_width] * len(tests))) 150 151 TestRunner.shared_self = None 152 153 return results 154 155 def __init__(self, env: TestEnv, makecheck: bool = False, 156 color: str = 'auto') -> None: 157 self.env = env 158 self.makecheck = makecheck 159 self.last_elapsed = LastElapsedTime('.last-elapsed-cache', env) 160 161 assert color in ('auto', 'on', 'off') 162 self.color = (color == 'on') or (color == 'auto' and 163 sys.stdout.isatty()) 164 165 self._stack: contextlib.ExitStack 166 167 def __enter__(self) -> 'TestRunner': 168 self._stack = contextlib.ExitStack() 169 self._stack.enter_context(self.env) 170 self._stack.enter_context(self.last_elapsed) 171 self._stack.enter_context(savetty()) 172 return self 173 174 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 175 self._stack.close() 176 177 def test_print_one_line(self, test: str, starttime: str, 178 endtime: Optional[str] = None, status: str = '...', 179 lasttime: Optional[float] = None, 180 thistime: Optional[float] = None, 181 description: str = '', 182 test_field_width: Optional[int] = None, 183 end: str = '\n') -> None: 184 """ Print short test info before/after test run """ 185 test = os.path.basename(test) 186 187 if test_field_width is None: 188 test_field_width = 8 189 190 if self.makecheck and status != '...': 191 if status and status != 'pass': 192 status = f' [{status}]' 193 else: 194 status = '' 195 196 print(f' TEST iotest-{self.env.imgfmt}: {test}{status}') 197 return 198 199 if lasttime: 200 lasttime_s = f' (last: {lasttime:.1f}s)' 201 else: 202 lasttime_s = '' 203 if thistime: 204 thistime_s = f'{thistime:.1f}s' 205 else: 206 thistime_s = '...' 207 208 if endtime: 209 endtime = f'[{endtime}]' 210 else: 211 endtime = '' 212 213 if self.color: 214 if status == 'pass': 215 col = '\033[32m' 216 elif status == 'fail': 217 col = '\033[1m\033[31m' 218 elif status == 'not run': 219 col = '\033[33m' 220 else: 221 col = '' 222 223 col_end = '\033[0m' 224 else: 225 col = '' 226 col_end = '' 227 228 print(f'{test:{test_field_width}} {col}{status:10}{col_end} ' 229 f'[{starttime}] {endtime:13}{thistime_s:5} {lasttime_s:14} ' 230 f'{description}', end=end) 231 232 def find_reference(self, test: str) -> str: 233 if self.env.cachemode == 'none': 234 ref = f'{test}.out.nocache' 235 if os.path.isfile(ref): 236 return ref 237 238 ref = f'{test}.out.{self.env.imgfmt}' 239 if os.path.isfile(ref): 240 return ref 241 242 ref = f'{test}.{self.env.qemu_default_machine}.out' 243 if os.path.isfile(ref): 244 return ref 245 246 return f'{test}.out' 247 248 def do_run_test(self, test: str, mp: bool) -> TestResult: 249 """ 250 Run one test 251 252 :param test: test file path 253 :param mp: if true, we are in a multiprocessing environment, use 254 personal subdirectories for test run 255 256 Note: this method may be called from subprocess, so it does not 257 change ``self`` object in any way! 258 """ 259 260 f_test = Path(test) 261 f_bad = Path(f_test.name + '.out.bad') 262 f_notrun = Path(f_test.name + '.notrun') 263 f_casenotrun = Path(f_test.name + '.casenotrun') 264 f_reference = Path(self.find_reference(test)) 265 266 if not f_test.exists(): 267 return TestResult(status='fail', 268 description=f'No such test file: {f_test}') 269 270 if not os.access(str(f_test), os.X_OK): 271 sys.exit(f'Not executable: {f_test}') 272 273 if not f_reference.exists(): 274 return TestResult(status='not run', 275 description='No qualified output ' 276 f'(expected {f_reference})') 277 278 for p in (f_bad, f_notrun, f_casenotrun): 279 silent_unlink(p) 280 281 args = [str(f_test.resolve())] 282 env = self.env.prepare_subprocess(args) 283 if mp: 284 # Split test directories, so that tests running in parallel don't 285 # break each other. 286 for d in ['TEST_DIR', 'SOCK_DIR']: 287 env[d] = os.path.join(env[d], f_test.name) 288 Path(env[d]).mkdir(parents=True, exist_ok=True) 289 290 t0 = time.time() 291 with f_bad.open('w', encoding="utf-8") as f: 292 with subprocess.Popen(args, cwd=str(f_test.parent), env=env, 293 stdout=f, stderr=subprocess.STDOUT) as proc: 294 try: 295 proc.wait() 296 except KeyboardInterrupt: 297 proc.terminate() 298 proc.wait() 299 return TestResult(status='not run', 300 description='Interrupted by user', 301 interrupted=True) 302 ret = proc.returncode 303 304 elapsed = round(time.time() - t0, 1) 305 306 if ret != 0: 307 return TestResult(status='fail', elapsed=elapsed, 308 description=f'failed, exit status {ret}', 309 diff=file_diff(str(f_reference), str(f_bad))) 310 311 if f_notrun.exists(): 312 return TestResult( 313 status='not run', 314 description=f_notrun.read_text(encoding='utf-8').strip()) 315 316 casenotrun = '' 317 if f_casenotrun.exists(): 318 casenotrun = f_casenotrun.read_text(encoding='utf-8') 319 320 diff = file_diff(str(f_reference), str(f_bad)) 321 if diff: 322 return TestResult(status='fail', elapsed=elapsed, 323 description=f'output mismatch (see {f_bad})', 324 diff=diff, casenotrun=casenotrun) 325 else: 326 f_bad.unlink() 327 return TestResult(status='pass', elapsed=elapsed, 328 casenotrun=casenotrun) 329 330 def run_test(self, test: str, 331 test_field_width: Optional[int] = None, 332 mp: bool = False) -> TestResult: 333 """ 334 Run one test and print short status 335 336 :param test: test file path 337 :param test_field_width: width for first field of status format 338 :param mp: if true, we are in a multiprocessing environment, don't try 339 to rewrite things in stdout 340 341 Note: this method may be called from subprocess, so it does not 342 change ``self`` object in any way! 343 """ 344 345 last_el = self.last_elapsed.get(test) 346 start = datetime.datetime.now().strftime('%H:%M:%S') 347 348 if not self.makecheck: 349 self.test_print_one_line(test=test, 350 status = 'started' if mp else '...', 351 starttime=start, 352 lasttime=last_el, 353 end = '\n' if mp else '\r', 354 test_field_width=test_field_width) 355 356 res = self.do_run_test(test, mp) 357 358 end = datetime.datetime.now().strftime('%H:%M:%S') 359 self.test_print_one_line(test=test, status=res.status, 360 starttime=start, endtime=end, 361 lasttime=last_el, thistime=res.elapsed, 362 description=res.description, 363 test_field_width=test_field_width) 364 365 if res.casenotrun: 366 print(res.casenotrun) 367 368 return res 369 370 def run_tests(self, tests: List[str], jobs: int = 1) -> bool: 371 n_run = 0 372 failed = [] 373 notrun = [] 374 casenotrun = [] 375 376 if not self.makecheck: 377 self.env.print_env() 378 379 test_field_width = max(len(os.path.basename(t)) for t in tests) + 2 380 381 if jobs > 1: 382 results = self.run_tests_pool(tests, test_field_width, jobs) 383 384 for i, t in enumerate(tests): 385 name = os.path.basename(t) 386 387 if jobs > 1: 388 res = results[i] 389 else: 390 res = self.run_test(t, test_field_width) 391 392 assert res.status in ('pass', 'fail', 'not run') 393 394 if res.casenotrun: 395 casenotrun.append(t) 396 397 if res.status != 'not run': 398 n_run += 1 399 400 if res.status == 'fail': 401 failed.append(name) 402 if self.makecheck: 403 self.env.print_env() 404 if res.diff: 405 print('\n'.join(res.diff)) 406 elif res.status == 'not run': 407 notrun.append(name) 408 elif res.status == 'pass': 409 assert res.elapsed is not None 410 self.last_elapsed.update(t, res.elapsed) 411 412 sys.stdout.flush() 413 if res.interrupted: 414 break 415 416 if notrun: 417 print('Not run:', ' '.join(notrun)) 418 419 if casenotrun: 420 print('Some cases not run in:', ' '.join(casenotrun)) 421 422 if failed: 423 print('Failures:', ' '.join(failed)) 424 print(f'Failed {len(failed)} of {n_run} iotests') 425 return False 426 else: 427 print(f'Passed all {n_run} iotests') 428 return True 429