1from __future__ import print_function 2# Common utilities and Python wrappers for qemu-iotests 3# 4# Copyright (C) 2012 IBM Corp. 5# 6# This program is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program. If not, see <http://www.gnu.org/licenses/>. 18# 19 20import errno 21import os 22import re 23import subprocess 24import string 25import unittest 26import sys 27import struct 28import json 29import signal 30import logging 31import atexit 32import io 33from collections import OrderedDict 34 35sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 36from qemu import qtest 37 38assert sys.version_info >= (3,6) 39 40# This will not work if arguments contain spaces but is necessary if we 41# want to support the override options that ./check supports. 42qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] 43if os.environ.get('QEMU_IMG_OPTIONS'): 44 qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') 45 46qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 47if os.environ.get('QEMU_IO_OPTIONS'): 48 qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') 49 50qemu_io_args_no_fmt = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 51if os.environ.get('QEMU_IO_OPTIONS_NO_FMT'): 52 qemu_io_args_no_fmt += \ 53 os.environ['QEMU_IO_OPTIONS_NO_FMT'].strip().split(' ') 54 55qemu_nbd_args = [os.environ.get('QEMU_NBD_PROG', 'qemu-nbd')] 56if os.environ.get('QEMU_NBD_OPTIONS'): 57 qemu_nbd_args += os.environ['QEMU_NBD_OPTIONS'].strip().split(' ') 58 59qemu_prog = os.environ.get('QEMU_PROG', 'qemu') 60qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') 61 62imgfmt = os.environ.get('IMGFMT', 'raw') 63imgproto = os.environ.get('IMGPROTO', 'file') 64test_dir = os.environ.get('TEST_DIR') 65sock_dir = os.environ.get('SOCK_DIR') 66output_dir = os.environ.get('OUTPUT_DIR', '.') 67cachemode = os.environ.get('CACHEMODE') 68qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') 69 70socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') 71 72luks_default_secret_object = 'secret,id=keysec0,data=' + \ 73 os.environ.get('IMGKEYSECRET', '') 74luks_default_key_secret_opt = 'key-secret=keysec0' 75 76 77def qemu_img(*args): 78 '''Run qemu-img and return the exit code''' 79 devnull = open('/dev/null', 'r+') 80 exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) 81 if exitcode < 0: 82 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 83 return exitcode 84 85def ordered_qmp(qmsg, conv_keys=True): 86 # Dictionaries are not ordered prior to 3.6, therefore: 87 if isinstance(qmsg, list): 88 return [ordered_qmp(atom) for atom in qmsg] 89 if isinstance(qmsg, dict): 90 od = OrderedDict() 91 for k, v in sorted(qmsg.items()): 92 if conv_keys: 93 k = k.replace('_', '-') 94 od[k] = ordered_qmp(v, conv_keys=False) 95 return od 96 return qmsg 97 98def qemu_img_create(*args): 99 args = list(args) 100 101 # default luks support 102 if '-f' in args and args[args.index('-f') + 1] == 'luks': 103 if '-o' in args: 104 i = args.index('-o') 105 if 'key-secret' not in args[i + 1]: 106 args[i + 1].append(luks_default_key_secret_opt) 107 args.insert(i + 2, '--object') 108 args.insert(i + 3, luks_default_secret_object) 109 else: 110 args = ['-o', luks_default_key_secret_opt, 111 '--object', luks_default_secret_object] + args 112 113 args.insert(0, 'create') 114 115 return qemu_img(*args) 116 117def qemu_img_verbose(*args): 118 '''Run qemu-img without suppressing its output and return the exit code''' 119 exitcode = subprocess.call(qemu_img_args + list(args)) 120 if exitcode < 0: 121 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 122 return exitcode 123 124def qemu_img_pipe(*args): 125 '''Run qemu-img and return its output''' 126 subp = subprocess.Popen(qemu_img_args + list(args), 127 stdout=subprocess.PIPE, 128 stderr=subprocess.STDOUT, 129 universal_newlines=True) 130 exitcode = subp.wait() 131 if exitcode < 0: 132 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 133 return subp.communicate()[0] 134 135def qemu_img_log(*args): 136 result = qemu_img_pipe(*args) 137 log(result, filters=[filter_testfiles]) 138 return result 139 140def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): 141 args = [ 'info' ] 142 if imgopts: 143 args.append('--image-opts') 144 else: 145 args += [ '-f', imgfmt ] 146 args += extra_args 147 args.append(filename) 148 149 output = qemu_img_pipe(*args) 150 if not filter_path: 151 filter_path = filename 152 log(filter_img_info(output, filter_path)) 153 154def qemu_io(*args): 155 '''Run qemu-io and return the stdout data''' 156 args = qemu_io_args + list(args) 157 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 158 stderr=subprocess.STDOUT, 159 universal_newlines=True) 160 exitcode = subp.wait() 161 if exitcode < 0: 162 sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) 163 return subp.communicate()[0] 164 165def qemu_io_silent(*args): 166 '''Run qemu-io and return the exit code, suppressing stdout''' 167 args = qemu_io_args + list(args) 168 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w')) 169 if exitcode < 0: 170 sys.stderr.write('qemu-io received signal %i: %s\n' % 171 (-exitcode, ' '.join(args))) 172 return exitcode 173 174def qemu_io_silent_check(*args): 175 '''Run qemu-io and return the true if subprocess returned 0''' 176 args = qemu_io_args + list(args) 177 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'), 178 stderr=subprocess.STDOUT) 179 return exitcode == 0 180 181def get_virtio_scsi_device(): 182 if qemu_default_machine == 's390-ccw-virtio': 183 return 'virtio-scsi-ccw' 184 return 'virtio-scsi-pci' 185 186class QemuIoInteractive: 187 def __init__(self, *args): 188 self.args = qemu_io_args + list(args) 189 self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE, 190 stdout=subprocess.PIPE, 191 stderr=subprocess.STDOUT, 192 universal_newlines=True) 193 assert self._p.stdout.read(9) == 'qemu-io> ' 194 195 def close(self): 196 self._p.communicate('q\n') 197 198 def _read_output(self): 199 pattern = 'qemu-io> ' 200 n = len(pattern) 201 pos = 0 202 s = [] 203 while pos != n: 204 c = self._p.stdout.read(1) 205 # check unexpected EOF 206 assert c != '' 207 s.append(c) 208 if c == pattern[pos]: 209 pos += 1 210 else: 211 pos = 0 212 213 return ''.join(s[:-n]) 214 215 def cmd(self, cmd): 216 # quit command is in close(), '\n' is added automatically 217 assert '\n' not in cmd 218 cmd = cmd.strip() 219 assert cmd != 'q' and cmd != 'quit' 220 self._p.stdin.write(cmd + '\n') 221 self._p.stdin.flush() 222 return self._read_output() 223 224 225def qemu_nbd(*args): 226 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 227 return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) 228 229def qemu_nbd_early_pipe(*args): 230 '''Run qemu-nbd in daemon mode and return both the parent's exit code 231 and its output in case of an error''' 232 subp = subprocess.Popen(qemu_nbd_args + ['--fork'] + list(args), 233 stdout=subprocess.PIPE, 234 stderr=subprocess.STDOUT, 235 universal_newlines=True) 236 exitcode = subp.wait() 237 if exitcode < 0: 238 sys.stderr.write('qemu-nbd received signal %i: %s\n' % 239 (-exitcode, 240 ' '.join(qemu_nbd_args + ['--fork'] + list(args)))) 241 if exitcode == 0: 242 return exitcode, '' 243 else: 244 return exitcode, subp.communicate()[0] 245 246def qemu_nbd_popen(*args): 247 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 248 return subprocess.Popen(qemu_nbd_args + ['--persistent'] + list(args)) 249 250def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt): 251 '''Return True if two image files are identical''' 252 return qemu_img('compare', '-f', fmt1, 253 '-F', fmt2, img1, img2) == 0 254 255def create_image(name, size): 256 '''Create a fully-allocated raw image with sector markers''' 257 file = open(name, 'wb') 258 i = 0 259 while i < size: 260 sector = struct.pack('>l504xl', i // 512, i // 512) 261 file.write(sector) 262 i = i + 512 263 file.close() 264 265def image_size(img): 266 '''Return image's virtual size''' 267 r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img) 268 return json.loads(r)['virtual-size'] 269 270def is_str(val): 271 return isinstance(val, str) 272 273test_dir_re = re.compile(r"%s" % test_dir) 274def filter_test_dir(msg): 275 return test_dir_re.sub("TEST_DIR", msg) 276 277win32_re = re.compile(r"\r") 278def filter_win32(msg): 279 return win32_re.sub("", msg) 280 281qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)") 282def filter_qemu_io(msg): 283 msg = filter_win32(msg) 284 return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg) 285 286chown_re = re.compile(r"chown [0-9]+:[0-9]+") 287def filter_chown(msg): 288 return chown_re.sub("chown UID:GID", msg) 289 290def filter_qmp_event(event): 291 '''Filter a QMP event dict''' 292 event = dict(event) 293 if 'timestamp' in event: 294 event['timestamp']['seconds'] = 'SECS' 295 event['timestamp']['microseconds'] = 'USECS' 296 return event 297 298def filter_qmp(qmsg, filter_fn): 299 '''Given a string filter, filter a QMP object's values. 300 filter_fn takes a (key, value) pair.''' 301 # Iterate through either lists or dicts; 302 if isinstance(qmsg, list): 303 items = enumerate(qmsg) 304 else: 305 items = qmsg.items() 306 307 for k, v in items: 308 if isinstance(v, list) or isinstance(v, dict): 309 qmsg[k] = filter_qmp(v, filter_fn) 310 else: 311 qmsg[k] = filter_fn(k, v) 312 return qmsg 313 314def filter_testfiles(msg): 315 prefix = os.path.join(test_dir, "%s-" % (os.getpid())) 316 return msg.replace(prefix, 'TEST_DIR/PID-') 317 318def filter_qmp_testfiles(qmsg): 319 def _filter(key, value): 320 if is_str(value): 321 return filter_testfiles(value) 322 return value 323 return filter_qmp(qmsg, _filter) 324 325def filter_generated_node_ids(msg): 326 return re.sub("#block[0-9]+", "NODE_NAME", msg) 327 328def filter_img_info(output, filename): 329 lines = [] 330 for line in output.split('\n'): 331 if 'disk size' in line or 'actual-size' in line: 332 continue 333 line = line.replace(filename, 'TEST_IMG') \ 334 .replace(imgfmt, 'IMGFMT') 335 line = re.sub('iters: [0-9]+', 'iters: XXX', line) 336 line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line) 337 line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) 338 lines.append(line) 339 return '\n'.join(lines) 340 341def filter_imgfmt(msg): 342 return msg.replace(imgfmt, 'IMGFMT') 343 344def filter_qmp_imgfmt(qmsg): 345 def _filter(key, value): 346 if is_str(value): 347 return filter_imgfmt(value) 348 return value 349 return filter_qmp(qmsg, _filter) 350 351def log(msg, filters=[], indent=None): 352 '''Logs either a string message or a JSON serializable message (like QMP). 353 If indent is provided, JSON serializable messages are pretty-printed.''' 354 for flt in filters: 355 msg = flt(msg) 356 if isinstance(msg, dict) or isinstance(msg, list): 357 # Python < 3.4 needs to know not to add whitespace when pretty-printing: 358 separators = (', ', ': ') if indent is None else (',', ': ') 359 # Don't sort if it's already sorted 360 do_sort = not isinstance(msg, OrderedDict) 361 print(json.dumps(msg, sort_keys=do_sort, 362 indent=indent, separators=separators)) 363 else: 364 print(msg) 365 366class Timeout: 367 def __init__(self, seconds, errmsg = "Timeout"): 368 self.seconds = seconds 369 self.errmsg = errmsg 370 def __enter__(self): 371 signal.signal(signal.SIGALRM, self.timeout) 372 signal.setitimer(signal.ITIMER_REAL, self.seconds) 373 return self 374 def __exit__(self, type, value, traceback): 375 signal.setitimer(signal.ITIMER_REAL, 0) 376 return False 377 def timeout(self, signum, frame): 378 raise Exception(self.errmsg) 379 380def file_pattern(name): 381 return "{0}-{1}".format(os.getpid(), name) 382 383class FilePaths(object): 384 """ 385 FilePaths is an auto-generated filename that cleans itself up. 386 387 Use this context manager to generate filenames and ensure that the file 388 gets deleted:: 389 390 with FilePaths(['test.img']) as img_path: 391 qemu_img('create', img_path, '1G') 392 # migration_sock_path is automatically deleted 393 """ 394 def __init__(self, names, base_dir=test_dir): 395 self.paths = [] 396 for name in names: 397 self.paths.append(os.path.join(base_dir, file_pattern(name))) 398 399 def __enter__(self): 400 return self.paths 401 402 def __exit__(self, exc_type, exc_val, exc_tb): 403 try: 404 for path in self.paths: 405 os.remove(path) 406 except OSError: 407 pass 408 return False 409 410class FilePath(FilePaths): 411 """ 412 FilePath is a specialization of FilePaths that takes a single filename. 413 """ 414 def __init__(self, name, base_dir=test_dir): 415 super(FilePath, self).__init__([name], base_dir) 416 417 def __enter__(self): 418 return self.paths[0] 419 420def file_path_remover(): 421 for path in reversed(file_path_remover.paths): 422 try: 423 os.remove(path) 424 except OSError: 425 pass 426 427 428def file_path(*names, base_dir=test_dir): 429 ''' Another way to get auto-generated filename that cleans itself up. 430 431 Use is as simple as: 432 433 img_a, img_b = file_path('a.img', 'b.img') 434 sock = file_path('socket') 435 ''' 436 437 if not hasattr(file_path_remover, 'paths'): 438 file_path_remover.paths = [] 439 atexit.register(file_path_remover) 440 441 paths = [] 442 for name in names: 443 filename = file_pattern(name) 444 path = os.path.join(base_dir, filename) 445 file_path_remover.paths.append(path) 446 paths.append(path) 447 448 return paths[0] if len(paths) == 1 else paths 449 450def remote_filename(path): 451 if imgproto == 'file': 452 return path 453 elif imgproto == 'ssh': 454 return "ssh://%s@127.0.0.1:22%s" % (os.environ.get('USER'), path) 455 else: 456 raise Exception("Protocol %s not supported" % (imgproto)) 457 458class VM(qtest.QEMUQtestMachine): 459 '''A QEMU VM''' 460 461 def __init__(self, path_suffix=''): 462 name = "qemu%s-%d" % (path_suffix, os.getpid()) 463 super(VM, self).__init__(qemu_prog, qemu_opts, name=name, 464 test_dir=test_dir, 465 socket_scm_helper=socket_scm_helper, 466 sock_dir=sock_dir) 467 self._num_drives = 0 468 469 def add_object(self, opts): 470 self._args.append('-object') 471 self._args.append(opts) 472 return self 473 474 def add_device(self, opts): 475 self._args.append('-device') 476 self._args.append(opts) 477 return self 478 479 def add_drive_raw(self, opts): 480 self._args.append('-drive') 481 self._args.append(opts) 482 return self 483 484 def add_drive(self, path, opts='', interface='virtio', format=imgfmt): 485 '''Add a virtio-blk drive to the VM''' 486 options = ['if=%s' % interface, 487 'id=drive%d' % self._num_drives] 488 489 if path is not None: 490 options.append('file=%s' % path) 491 options.append('format=%s' % format) 492 options.append('cache=%s' % cachemode) 493 494 if opts: 495 options.append(opts) 496 497 if format == 'luks' and 'key-secret' not in opts: 498 # default luks support 499 if luks_default_secret_object not in self._args: 500 self.add_object(luks_default_secret_object) 501 502 options.append(luks_default_key_secret_opt) 503 504 self._args.append('-drive') 505 self._args.append(','.join(options)) 506 self._num_drives += 1 507 return self 508 509 def add_blockdev(self, opts): 510 self._args.append('-blockdev') 511 if isinstance(opts, str): 512 self._args.append(opts) 513 else: 514 self._args.append(','.join(opts)) 515 return self 516 517 def add_incoming(self, addr): 518 self._args.append('-incoming') 519 self._args.append(addr) 520 return self 521 522 def pause_drive(self, drive, event=None): 523 '''Pause drive r/w operations''' 524 if not event: 525 self.pause_drive(drive, "read_aio") 526 self.pause_drive(drive, "write_aio") 527 return 528 self.qmp('human-monitor-command', 529 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) 530 531 def resume_drive(self, drive): 532 self.qmp('human-monitor-command', 533 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) 534 535 def hmp_qemu_io(self, drive, cmd): 536 '''Write to a given drive using an HMP command''' 537 return self.qmp('human-monitor-command', 538 command_line='qemu-io %s "%s"' % (drive, cmd)) 539 540 def flatten_qmp_object(self, obj, output=None, basestr=''): 541 if output is None: 542 output = dict() 543 if isinstance(obj, list): 544 for i in range(len(obj)): 545 self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.') 546 elif isinstance(obj, dict): 547 for key in obj: 548 self.flatten_qmp_object(obj[key], output, basestr + key + '.') 549 else: 550 output[basestr[:-1]] = obj # Strip trailing '.' 551 return output 552 553 def qmp_to_opts(self, obj): 554 obj = self.flatten_qmp_object(obj) 555 output_list = list() 556 for key in obj: 557 output_list += [key + '=' + obj[key]] 558 return ','.join(output_list) 559 560 def get_qmp_events_filtered(self, wait=60.0): 561 result = [] 562 for ev in self.get_qmp_events(wait=wait): 563 result.append(filter_qmp_event(ev)) 564 return result 565 566 def qmp_log(self, cmd, filters=[], indent=None, **kwargs): 567 full_cmd = OrderedDict(( 568 ("execute", cmd), 569 ("arguments", ordered_qmp(kwargs)) 570 )) 571 log(full_cmd, filters, indent=indent) 572 result = self.qmp(cmd, **kwargs) 573 log(result, filters, indent=indent) 574 return result 575 576 # Returns None on success, and an error string on failure 577 def run_job(self, job, auto_finalize=True, auto_dismiss=False, 578 pre_finalize=None, cancel=False, use_log=True, wait=60.0): 579 """ 580 run_job moves a job from creation through to dismissal. 581 582 :param job: String. ID of recently-launched job 583 :param auto_finalize: Bool. True if the job was launched with 584 auto_finalize. Defaults to True. 585 :param auto_dismiss: Bool. True if the job was launched with 586 auto_dismiss=True. Defaults to False. 587 :param pre_finalize: Callback. A callable that takes no arguments to be 588 invoked prior to issuing job-finalize, if any. 589 :param cancel: Bool. When true, cancels the job after the pre_finalize 590 callback. 591 :param use_log: Bool. When false, does not log QMP messages. 592 :param wait: Float. Timeout value specifying how long to wait for any 593 event, in seconds. Defaults to 60.0. 594 """ 595 match_device = {'data': {'device': job}} 596 match_id = {'data': {'id': job}} 597 events = [ 598 ('BLOCK_JOB_COMPLETED', match_device), 599 ('BLOCK_JOB_CANCELLED', match_device), 600 ('BLOCK_JOB_ERROR', match_device), 601 ('BLOCK_JOB_READY', match_device), 602 ('BLOCK_JOB_PENDING', match_id), 603 ('JOB_STATUS_CHANGE', match_id) 604 ] 605 error = None 606 while True: 607 ev = filter_qmp_event(self.events_wait(events)) 608 if ev['event'] != 'JOB_STATUS_CHANGE': 609 if use_log: 610 log(ev) 611 continue 612 status = ev['data']['status'] 613 if status == 'aborting': 614 result = self.qmp('query-jobs') 615 for j in result['return']: 616 if j['id'] == job: 617 error = j['error'] 618 if use_log: 619 log('Job failed: %s' % (j['error'])) 620 elif status == 'pending' and not auto_finalize: 621 if pre_finalize: 622 pre_finalize() 623 if cancel and use_log: 624 self.qmp_log('job-cancel', id=job) 625 elif cancel: 626 self.qmp('job-cancel', id=job) 627 elif use_log: 628 self.qmp_log('job-finalize', id=job) 629 else: 630 self.qmp('job-finalize', id=job) 631 elif status == 'concluded' and not auto_dismiss: 632 if use_log: 633 self.qmp_log('job-dismiss', id=job) 634 else: 635 self.qmp('job-dismiss', id=job) 636 elif status == 'null': 637 return error 638 639 def enable_migration_events(self, name): 640 log('Enabling migration QMP events on %s...' % name) 641 log(self.qmp('migrate-set-capabilities', capabilities=[ 642 { 643 'capability': 'events', 644 'state': True 645 } 646 ])) 647 648 def wait_migration(self): 649 while True: 650 event = self.event_wait('MIGRATION') 651 log(event, filters=[filter_qmp_event]) 652 if event['data']['status'] == 'completed': 653 break 654 655 def node_info(self, node_name): 656 nodes = self.qmp('query-named-block-nodes') 657 for x in nodes['return']: 658 if x['node-name'] == node_name: 659 return x 660 return None 661 662 def query_bitmaps(self): 663 res = self.qmp("query-named-block-nodes") 664 return {device['node-name']: device['dirty-bitmaps'] 665 for device in res['return'] if 'dirty-bitmaps' in device} 666 667 def get_bitmap(self, node_name, bitmap_name, recording=None, bitmaps=None): 668 """ 669 get a specific bitmap from the object returned by query_bitmaps. 670 :param recording: If specified, filter results by the specified value. 671 :param bitmaps: If specified, use it instead of call query_bitmaps() 672 """ 673 if bitmaps is None: 674 bitmaps = self.query_bitmaps() 675 676 for bitmap in bitmaps[node_name]: 677 if bitmap.get('name', '') == bitmap_name: 678 if recording is None: 679 return bitmap 680 elif bitmap.get('recording') == recording: 681 return bitmap 682 return None 683 684 def check_bitmap_status(self, node_name, bitmap_name, fields): 685 ret = self.get_bitmap(node_name, bitmap_name) 686 687 return fields.items() <= ret.items() 688 689 690index_re = re.compile(r'([^\[]+)\[([^\]]+)\]') 691 692class QMPTestCase(unittest.TestCase): 693 '''Abstract base class for QMP test cases''' 694 695 def dictpath(self, d, path): 696 '''Traverse a path in a nested dict''' 697 for component in path.split('/'): 698 m = index_re.match(component) 699 if m: 700 component, idx = m.groups() 701 idx = int(idx) 702 703 if not isinstance(d, dict) or component not in d: 704 self.fail('failed path traversal for "%s" in "%s"' % (path, str(d))) 705 d = d[component] 706 707 if m: 708 if not isinstance(d, list): 709 self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d))) 710 try: 711 d = d[idx] 712 except IndexError: 713 self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d))) 714 return d 715 716 def assert_qmp_absent(self, d, path): 717 try: 718 result = self.dictpath(d, path) 719 except AssertionError: 720 return 721 self.fail('path "%s" has value "%s"' % (path, str(result))) 722 723 def assert_qmp(self, d, path, value): 724 '''Assert that the value for a specific path in a QMP dict 725 matches. When given a list of values, assert that any of 726 them matches.''' 727 728 result = self.dictpath(d, path) 729 730 # [] makes no sense as a list of valid values, so treat it as 731 # an actual single value. 732 if isinstance(value, list) and value != []: 733 for v in value: 734 if result == v: 735 return 736 self.fail('no match for "%s" in %s' % (str(result), str(value))) 737 else: 738 self.assertEqual(result, value, 739 '"%s" is "%s", expected "%s"' 740 % (path, str(result), str(value))) 741 742 def assert_no_active_block_jobs(self): 743 result = self.vm.qmp('query-block-jobs') 744 self.assert_qmp(result, 'return', []) 745 746 def assert_has_block_node(self, node_name=None, file_name=None): 747 """Issue a query-named-block-nodes and assert node_name and/or 748 file_name is present in the result""" 749 def check_equal_or_none(a, b): 750 return a == None or b == None or a == b 751 assert node_name or file_name 752 result = self.vm.qmp('query-named-block-nodes') 753 for x in result["return"]: 754 if check_equal_or_none(x.get("node-name"), node_name) and \ 755 check_equal_or_none(x.get("file"), file_name): 756 return 757 self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \ 758 (node_name, file_name, result)) 759 760 def assert_json_filename_equal(self, json_filename, reference): 761 '''Asserts that the given filename is a json: filename and that its 762 content is equal to the given reference object''' 763 self.assertEqual(json_filename[:5], 'json:') 764 self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])), 765 self.vm.flatten_qmp_object(reference)) 766 767 def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0): 768 '''Cancel a block job and wait for it to finish, returning the event''' 769 result = self.vm.qmp('block-job-cancel', device=drive, force=force) 770 self.assert_qmp(result, 'return', {}) 771 772 if resume: 773 self.vm.resume_drive(drive) 774 775 cancelled = False 776 result = None 777 while not cancelled: 778 for event in self.vm.get_qmp_events(wait=wait): 779 if event['event'] == 'BLOCK_JOB_COMPLETED' or \ 780 event['event'] == 'BLOCK_JOB_CANCELLED': 781 self.assert_qmp(event, 'data/device', drive) 782 result = event 783 cancelled = True 784 elif event['event'] == 'JOB_STATUS_CHANGE': 785 self.assert_qmp(event, 'data/id', drive) 786 787 788 self.assert_no_active_block_jobs() 789 return result 790 791 def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0): 792 '''Wait for a block job to finish, returning the event''' 793 while True: 794 for event in self.vm.get_qmp_events(wait=wait): 795 if event['event'] == 'BLOCK_JOB_COMPLETED': 796 self.assert_qmp(event, 'data/device', drive) 797 self.assert_qmp_absent(event, 'data/error') 798 if check_offset: 799 self.assert_qmp(event, 'data/offset', event['data']['len']) 800 self.assert_no_active_block_jobs() 801 return event 802 elif event['event'] == 'JOB_STATUS_CHANGE': 803 self.assert_qmp(event, 'data/id', drive) 804 805 def wait_ready(self, drive='drive0'): 806 '''Wait until a block job BLOCK_JOB_READY event''' 807 f = {'data': {'type': 'mirror', 'device': drive } } 808 event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f) 809 810 def wait_ready_and_cancel(self, drive='drive0'): 811 self.wait_ready(drive=drive) 812 event = self.cancel_and_wait(drive=drive) 813 self.assertEqual(event['event'], 'BLOCK_JOB_COMPLETED') 814 self.assert_qmp(event, 'data/type', 'mirror') 815 self.assert_qmp(event, 'data/offset', event['data']['len']) 816 817 def complete_and_wait(self, drive='drive0', wait_ready=True): 818 '''Complete a block job and wait for it to finish''' 819 if wait_ready: 820 self.wait_ready(drive=drive) 821 822 result = self.vm.qmp('block-job-complete', device=drive) 823 self.assert_qmp(result, 'return', {}) 824 825 event = self.wait_until_completed(drive=drive) 826 self.assert_qmp(event, 'data/type', 'mirror') 827 828 def pause_wait(self, job_id='job0'): 829 with Timeout(1, "Timeout waiting for job to pause"): 830 while True: 831 result = self.vm.qmp('query-block-jobs') 832 found = False 833 for job in result['return']: 834 if job['device'] == job_id: 835 found = True 836 if job['paused'] == True and job['busy'] == False: 837 return job 838 break 839 assert found 840 841 def pause_job(self, job_id='job0', wait=True): 842 result = self.vm.qmp('block-job-pause', device=job_id) 843 self.assert_qmp(result, 'return', {}) 844 if wait: 845 return self.pause_wait(job_id) 846 return result 847 848 def case_skip(self, reason): 849 '''Skip this test case''' 850 case_notrun(reason) 851 self.skipTest(reason) 852 853 854def notrun(reason): 855 '''Skip this test suite''' 856 # Each test in qemu-iotests has a number ("seq") 857 seq = os.path.basename(sys.argv[0]) 858 859 open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n') 860 print('%s not run: %s' % (seq, reason)) 861 sys.exit(0) 862 863def case_notrun(reason): 864 '''Mark this test case as not having been run (without actually 865 skipping it, that is left to the caller). See 866 QMPTestCase.case_skip() for a variant that actually skips the 867 current test case.''' 868 869 # Each test in qemu-iotests has a number ("seq") 870 seq = os.path.basename(sys.argv[0]) 871 872 open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( 873 ' [case not run] ' + reason + '\n') 874 875def verify_image_format(supported_fmts=[], unsupported_fmts=[]): 876 assert not (supported_fmts and unsupported_fmts) 877 878 if 'generic' in supported_fmts and \ 879 os.environ.get('IMGFMT_GENERIC', 'true') == 'true': 880 # similar to 881 # _supported_fmt generic 882 # for bash tests 883 return 884 885 not_sup = supported_fmts and (imgfmt not in supported_fmts) 886 if not_sup or (imgfmt in unsupported_fmts): 887 notrun('not suitable for this image format: %s' % imgfmt) 888 889def verify_protocol(supported=[], unsupported=[]): 890 assert not (supported and unsupported) 891 892 if 'generic' in supported: 893 return 894 895 not_sup = supported and (imgproto not in supported) 896 if not_sup or (imgproto in unsupported): 897 notrun('not suitable for this protocol: %s' % imgproto) 898 899def verify_platform(supported_oses=['linux']): 900 if True not in [sys.platform.startswith(x) for x in supported_oses]: 901 notrun('not suitable for this OS: %s' % sys.platform) 902 903def verify_cache_mode(supported_cache_modes=[]): 904 if supported_cache_modes and (cachemode not in supported_cache_modes): 905 notrun('not suitable for this cache mode: %s' % cachemode) 906 907def supports_quorum(): 908 return 'quorum' in qemu_img_pipe('--help') 909 910def verify_quorum(): 911 '''Skip test suite if quorum support is not available''' 912 if not supports_quorum(): 913 notrun('quorum support missing') 914 915def qemu_pipe(*args): 916 '''Run qemu with an option to print something and exit (e.g. a help option), 917 and return its output''' 918 args = [qemu_prog] + qemu_opts + list(args) 919 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 920 stderr=subprocess.STDOUT, 921 universal_newlines=True) 922 exitcode = subp.wait() 923 if exitcode < 0: 924 sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, 925 ' '.join(args))) 926 return subp.communicate()[0] 927 928def supported_formats(read_only=False): 929 '''Set 'read_only' to True to check ro-whitelist 930 Otherwise, rw-whitelist is checked''' 931 932 if not hasattr(supported_formats, "formats"): 933 supported_formats.formats = {} 934 935 if read_only not in supported_formats.formats: 936 format_message = qemu_pipe("-drive", "format=help") 937 line = 1 if read_only else 0 938 supported_formats.formats[read_only] = \ 939 format_message.splitlines()[line].split(":")[1].split() 940 941 return supported_formats.formats[read_only] 942 943def skip_if_unsupported(required_formats=[], read_only=False): 944 '''Skip Test Decorator 945 Runs the test if all the required formats are whitelisted''' 946 def skip_test_decorator(func): 947 def func_wrapper(test_case: QMPTestCase, *args, **kwargs): 948 if callable(required_formats): 949 fmts = required_formats(test_case) 950 else: 951 fmts = required_formats 952 953 usf_list = list(set(fmts) - set(supported_formats(read_only))) 954 if usf_list: 955 test_case.case_skip('{}: formats {} are not whitelisted'.format( 956 test_case, usf_list)) 957 else: 958 return func(test_case, *args, **kwargs) 959 return func_wrapper 960 return skip_test_decorator 961 962def skip_if_user_is_root(func): 963 '''Skip Test Decorator 964 Runs the test only without root permissions''' 965 def func_wrapper(*args, **kwargs): 966 if os.getuid() == 0: 967 case_notrun('{}: cannot be run as root'.format(args[0])) 968 else: 969 return func(*args, **kwargs) 970 return func_wrapper 971 972def execute_unittest(output, verbosity, debug): 973 runner = unittest.TextTestRunner(stream=output, descriptions=True, 974 verbosity=verbosity) 975 try: 976 # unittest.main() will use sys.exit(); so expect a SystemExit 977 # exception 978 unittest.main(testRunner=runner) 979 finally: 980 if not debug: 981 out = output.getvalue() 982 out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out) 983 984 # Hide skipped tests from the reference output 985 out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) 986 out_first_line, out_rest = out.split('\n', 1) 987 out = out_first_line.replace('s', '.') + '\n' + out_rest 988 989 sys.stderr.write(out) 990 991def execute_test(test_function=None, 992 supported_fmts=[], supported_oses=['linux'], 993 supported_cache_modes=[], unsupported_fmts=[], 994 supported_protocols=[], unsupported_protocols=[]): 995 """Run either unittest or script-style tests.""" 996 997 # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to 998 # indicate that we're not being run via "check". There may be 999 # other things set up by "check" that individual test cases rely 1000 # on. 1001 if test_dir is None or qemu_default_machine is None: 1002 sys.stderr.write('Please run this test via the "check" script\n') 1003 sys.exit(os.EX_USAGE) 1004 1005 debug = '-d' in sys.argv 1006 verbosity = 1 1007 verify_image_format(supported_fmts, unsupported_fmts) 1008 verify_protocol(supported_protocols, unsupported_protocols) 1009 verify_platform(supported_oses) 1010 verify_cache_mode(supported_cache_modes) 1011 1012 if debug: 1013 output = sys.stdout 1014 verbosity = 2 1015 sys.argv.remove('-d') 1016 else: 1017 # We need to filter out the time taken from the output so that 1018 # qemu-iotest can reliably diff the results against master output. 1019 output = io.StringIO() 1020 1021 logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) 1022 1023 if not test_function: 1024 execute_unittest(output, verbosity, debug) 1025 else: 1026 test_function() 1027 1028def script_main(test_function, *args, **kwargs): 1029 """Run script-style tests outside of the unittest framework""" 1030 execute_test(test_function, *args, **kwargs) 1031 1032def main(*args, **kwargs): 1033 """Run tests using the unittest framework""" 1034 execute_test(None, *args, **kwargs) 1035