1from __future__ import print_function 2# Common utilities and Python wrappers for qemu-iotests 3# 4# Copyright (C) 2012 IBM Corp. 5# 6# This program is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program. If not, see <http://www.gnu.org/licenses/>. 18# 19 20import errno 21import os 22import re 23import subprocess 24import string 25import unittest 26import sys 27import struct 28import json 29import signal 30import logging 31import atexit 32import io 33from collections import OrderedDict 34 35sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 36from qemu import qtest 37 38assert sys.version_info >= (3,6) 39 40# This will not work if arguments contain spaces but is necessary if we 41# want to support the override options that ./check supports. 42qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] 43if os.environ.get('QEMU_IMG_OPTIONS'): 44 qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') 45 46qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 47if os.environ.get('QEMU_IO_OPTIONS'): 48 qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') 49 50qemu_io_args_no_fmt = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 51if os.environ.get('QEMU_IO_OPTIONS_NO_FMT'): 52 qemu_io_args_no_fmt += \ 53 os.environ['QEMU_IO_OPTIONS_NO_FMT'].strip().split(' ') 54 55qemu_nbd_args = [os.environ.get('QEMU_NBD_PROG', 'qemu-nbd')] 56if os.environ.get('QEMU_NBD_OPTIONS'): 57 qemu_nbd_args += os.environ['QEMU_NBD_OPTIONS'].strip().split(' ') 58 59qemu_prog = os.environ.get('QEMU_PROG', 'qemu') 60qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') 61 62imgfmt = os.environ.get('IMGFMT', 'raw') 63imgproto = os.environ.get('IMGPROTO', 'file') 64test_dir = os.environ.get('TEST_DIR') 65sock_dir = os.environ.get('SOCK_DIR') 66output_dir = os.environ.get('OUTPUT_DIR', '.') 67cachemode = os.environ.get('CACHEMODE') 68qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') 69 70socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') 71 72luks_default_secret_object = 'secret,id=keysec0,data=' + \ 73 os.environ.get('IMGKEYSECRET', '') 74luks_default_key_secret_opt = 'key-secret=keysec0' 75 76 77def qemu_img(*args): 78 '''Run qemu-img and return the exit code''' 79 devnull = open('/dev/null', 'r+') 80 exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) 81 if exitcode < 0: 82 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 83 return exitcode 84 85def ordered_qmp(qmsg, conv_keys=True): 86 # Dictionaries are not ordered prior to 3.6, therefore: 87 if isinstance(qmsg, list): 88 return [ordered_qmp(atom) for atom in qmsg] 89 if isinstance(qmsg, dict): 90 od = OrderedDict() 91 for k, v in sorted(qmsg.items()): 92 if conv_keys: 93 k = k.replace('_', '-') 94 od[k] = ordered_qmp(v, conv_keys=False) 95 return od 96 return qmsg 97 98def qemu_img_create(*args): 99 args = list(args) 100 101 # default luks support 102 if '-f' in args and args[args.index('-f') + 1] == 'luks': 103 if '-o' in args: 104 i = args.index('-o') 105 if 'key-secret' not in args[i + 1]: 106 args[i + 1].append(luks_default_key_secret_opt) 107 args.insert(i + 2, '--object') 108 args.insert(i + 3, luks_default_secret_object) 109 else: 110 args = ['-o', luks_default_key_secret_opt, 111 '--object', luks_default_secret_object] + args 112 113 args.insert(0, 'create') 114 115 return qemu_img(*args) 116 117def qemu_img_verbose(*args): 118 '''Run qemu-img without suppressing its output and return the exit code''' 119 exitcode = subprocess.call(qemu_img_args + list(args)) 120 if exitcode < 0: 121 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 122 return exitcode 123 124def qemu_img_pipe(*args): 125 '''Run qemu-img and return its output''' 126 subp = subprocess.Popen(qemu_img_args + list(args), 127 stdout=subprocess.PIPE, 128 stderr=subprocess.STDOUT, 129 universal_newlines=True) 130 exitcode = subp.wait() 131 if exitcode < 0: 132 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 133 return subp.communicate()[0] 134 135def qemu_img_log(*args): 136 result = qemu_img_pipe(*args) 137 log(result, filters=[filter_testfiles]) 138 return result 139 140def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): 141 args = [ 'info' ] 142 if imgopts: 143 args.append('--image-opts') 144 else: 145 args += [ '-f', imgfmt ] 146 args += extra_args 147 args.append(filename) 148 149 output = qemu_img_pipe(*args) 150 if not filter_path: 151 filter_path = filename 152 log(filter_img_info(output, filter_path)) 153 154def qemu_io(*args): 155 '''Run qemu-io and return the stdout data''' 156 args = qemu_io_args + list(args) 157 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 158 stderr=subprocess.STDOUT, 159 universal_newlines=True) 160 exitcode = subp.wait() 161 if exitcode < 0: 162 sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) 163 return subp.communicate()[0] 164 165def qemu_io_log(*args): 166 result = qemu_io(*args) 167 log(result, filters=[filter_testfiles, filter_qemu_io]) 168 return result 169 170def qemu_io_silent(*args): 171 '''Run qemu-io and return the exit code, suppressing stdout''' 172 args = qemu_io_args + list(args) 173 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w')) 174 if exitcode < 0: 175 sys.stderr.write('qemu-io received signal %i: %s\n' % 176 (-exitcode, ' '.join(args))) 177 return exitcode 178 179def qemu_io_silent_check(*args): 180 '''Run qemu-io and return the true if subprocess returned 0''' 181 args = qemu_io_args + list(args) 182 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'), 183 stderr=subprocess.STDOUT) 184 return exitcode == 0 185 186def get_virtio_scsi_device(): 187 if qemu_default_machine == 's390-ccw-virtio': 188 return 'virtio-scsi-ccw' 189 return 'virtio-scsi-pci' 190 191class QemuIoInteractive: 192 def __init__(self, *args): 193 self.args = qemu_io_args + list(args) 194 self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE, 195 stdout=subprocess.PIPE, 196 stderr=subprocess.STDOUT, 197 universal_newlines=True) 198 assert self._p.stdout.read(9) == 'qemu-io> ' 199 200 def close(self): 201 self._p.communicate('q\n') 202 203 def _read_output(self): 204 pattern = 'qemu-io> ' 205 n = len(pattern) 206 pos = 0 207 s = [] 208 while pos != n: 209 c = self._p.stdout.read(1) 210 # check unexpected EOF 211 assert c != '' 212 s.append(c) 213 if c == pattern[pos]: 214 pos += 1 215 else: 216 pos = 0 217 218 return ''.join(s[:-n]) 219 220 def cmd(self, cmd): 221 # quit command is in close(), '\n' is added automatically 222 assert '\n' not in cmd 223 cmd = cmd.strip() 224 assert cmd != 'q' and cmd != 'quit' 225 self._p.stdin.write(cmd + '\n') 226 self._p.stdin.flush() 227 return self._read_output() 228 229 230def qemu_nbd(*args): 231 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 232 return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) 233 234def qemu_nbd_early_pipe(*args): 235 '''Run qemu-nbd in daemon mode and return both the parent's exit code 236 and its output in case of an error''' 237 subp = subprocess.Popen(qemu_nbd_args + ['--fork'] + list(args), 238 stdout=subprocess.PIPE, 239 stderr=subprocess.STDOUT, 240 universal_newlines=True) 241 exitcode = subp.wait() 242 if exitcode < 0: 243 sys.stderr.write('qemu-nbd received signal %i: %s\n' % 244 (-exitcode, 245 ' '.join(qemu_nbd_args + ['--fork'] + list(args)))) 246 if exitcode == 0: 247 return exitcode, '' 248 else: 249 return exitcode, subp.communicate()[0] 250 251def qemu_nbd_popen(*args): 252 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 253 return subprocess.Popen(qemu_nbd_args + ['--persistent'] + list(args)) 254 255def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt): 256 '''Return True if two image files are identical''' 257 return qemu_img('compare', '-f', fmt1, 258 '-F', fmt2, img1, img2) == 0 259 260def create_image(name, size): 261 '''Create a fully-allocated raw image with sector markers''' 262 file = open(name, 'wb') 263 i = 0 264 while i < size: 265 sector = struct.pack('>l504xl', i // 512, i // 512) 266 file.write(sector) 267 i = i + 512 268 file.close() 269 270def image_size(img): 271 '''Return image's virtual size''' 272 r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img) 273 return json.loads(r)['virtual-size'] 274 275def is_str(val): 276 return isinstance(val, str) 277 278test_dir_re = re.compile(r"%s" % test_dir) 279def filter_test_dir(msg): 280 return test_dir_re.sub("TEST_DIR", msg) 281 282win32_re = re.compile(r"\r") 283def filter_win32(msg): 284 return win32_re.sub("", msg) 285 286qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)") 287def filter_qemu_io(msg): 288 msg = filter_win32(msg) 289 return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg) 290 291chown_re = re.compile(r"chown [0-9]+:[0-9]+") 292def filter_chown(msg): 293 return chown_re.sub("chown UID:GID", msg) 294 295def filter_qmp_event(event): 296 '''Filter a QMP event dict''' 297 event = dict(event) 298 if 'timestamp' in event: 299 event['timestamp']['seconds'] = 'SECS' 300 event['timestamp']['microseconds'] = 'USECS' 301 return event 302 303def filter_qmp(qmsg, filter_fn): 304 '''Given a string filter, filter a QMP object's values. 305 filter_fn takes a (key, value) pair.''' 306 # Iterate through either lists or dicts; 307 if isinstance(qmsg, list): 308 items = enumerate(qmsg) 309 else: 310 items = qmsg.items() 311 312 for k, v in items: 313 if isinstance(v, list) or isinstance(v, dict): 314 qmsg[k] = filter_qmp(v, filter_fn) 315 else: 316 qmsg[k] = filter_fn(k, v) 317 return qmsg 318 319def filter_testfiles(msg): 320 prefix = os.path.join(test_dir, "%s-" % (os.getpid())) 321 return msg.replace(prefix, 'TEST_DIR/PID-') 322 323def filter_qmp_testfiles(qmsg): 324 def _filter(key, value): 325 if is_str(value): 326 return filter_testfiles(value) 327 return value 328 return filter_qmp(qmsg, _filter) 329 330def filter_generated_node_ids(msg): 331 return re.sub("#block[0-9]+", "NODE_NAME", msg) 332 333def filter_img_info(output, filename): 334 lines = [] 335 for line in output.split('\n'): 336 if 'disk size' in line or 'actual-size' in line: 337 continue 338 line = line.replace(filename, 'TEST_IMG') \ 339 .replace(imgfmt, 'IMGFMT') 340 line = re.sub('iters: [0-9]+', 'iters: XXX', line) 341 line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line) 342 line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) 343 lines.append(line) 344 return '\n'.join(lines) 345 346def filter_imgfmt(msg): 347 return msg.replace(imgfmt, 'IMGFMT') 348 349def filter_qmp_imgfmt(qmsg): 350 def _filter(key, value): 351 if is_str(value): 352 return filter_imgfmt(value) 353 return value 354 return filter_qmp(qmsg, _filter) 355 356def log(msg, filters=[], indent=None): 357 '''Logs either a string message or a JSON serializable message (like QMP). 358 If indent is provided, JSON serializable messages are pretty-printed.''' 359 for flt in filters: 360 msg = flt(msg) 361 if isinstance(msg, dict) or isinstance(msg, list): 362 # Python < 3.4 needs to know not to add whitespace when pretty-printing: 363 separators = (', ', ': ') if indent is None else (',', ': ') 364 # Don't sort if it's already sorted 365 do_sort = not isinstance(msg, OrderedDict) 366 print(json.dumps(msg, sort_keys=do_sort, 367 indent=indent, separators=separators)) 368 else: 369 print(msg) 370 371class Timeout: 372 def __init__(self, seconds, errmsg = "Timeout"): 373 self.seconds = seconds 374 self.errmsg = errmsg 375 def __enter__(self): 376 signal.signal(signal.SIGALRM, self.timeout) 377 signal.setitimer(signal.ITIMER_REAL, self.seconds) 378 return self 379 def __exit__(self, type, value, traceback): 380 signal.setitimer(signal.ITIMER_REAL, 0) 381 return False 382 def timeout(self, signum, frame): 383 raise Exception(self.errmsg) 384 385def file_pattern(name): 386 return "{0}-{1}".format(os.getpid(), name) 387 388class FilePaths(object): 389 """ 390 FilePaths is an auto-generated filename that cleans itself up. 391 392 Use this context manager to generate filenames and ensure that the file 393 gets deleted:: 394 395 with FilePaths(['test.img']) as img_path: 396 qemu_img('create', img_path, '1G') 397 # migration_sock_path is automatically deleted 398 """ 399 def __init__(self, names, base_dir=test_dir): 400 self.paths = [] 401 for name in names: 402 self.paths.append(os.path.join(base_dir, file_pattern(name))) 403 404 def __enter__(self): 405 return self.paths 406 407 def __exit__(self, exc_type, exc_val, exc_tb): 408 try: 409 for path in self.paths: 410 os.remove(path) 411 except OSError: 412 pass 413 return False 414 415class FilePath(FilePaths): 416 """ 417 FilePath is a specialization of FilePaths that takes a single filename. 418 """ 419 def __init__(self, name, base_dir=test_dir): 420 super(FilePath, self).__init__([name], base_dir) 421 422 def __enter__(self): 423 return self.paths[0] 424 425def file_path_remover(): 426 for path in reversed(file_path_remover.paths): 427 try: 428 os.remove(path) 429 except OSError: 430 pass 431 432 433def file_path(*names, base_dir=test_dir): 434 ''' Another way to get auto-generated filename that cleans itself up. 435 436 Use is as simple as: 437 438 img_a, img_b = file_path('a.img', 'b.img') 439 sock = file_path('socket') 440 ''' 441 442 if not hasattr(file_path_remover, 'paths'): 443 file_path_remover.paths = [] 444 atexit.register(file_path_remover) 445 446 paths = [] 447 for name in names: 448 filename = file_pattern(name) 449 path = os.path.join(base_dir, filename) 450 file_path_remover.paths.append(path) 451 paths.append(path) 452 453 return paths[0] if len(paths) == 1 else paths 454 455def remote_filename(path): 456 if imgproto == 'file': 457 return path 458 elif imgproto == 'ssh': 459 return "ssh://%s@127.0.0.1:22%s" % (os.environ.get('USER'), path) 460 else: 461 raise Exception("Protocol %s not supported" % (imgproto)) 462 463class VM(qtest.QEMUQtestMachine): 464 '''A QEMU VM''' 465 466 def __init__(self, path_suffix=''): 467 name = "qemu%s-%d" % (path_suffix, os.getpid()) 468 super(VM, self).__init__(qemu_prog, qemu_opts, name=name, 469 test_dir=test_dir, 470 socket_scm_helper=socket_scm_helper, 471 sock_dir=sock_dir) 472 self._num_drives = 0 473 474 def add_object(self, opts): 475 self._args.append('-object') 476 self._args.append(opts) 477 return self 478 479 def add_device(self, opts): 480 self._args.append('-device') 481 self._args.append(opts) 482 return self 483 484 def add_drive_raw(self, opts): 485 self._args.append('-drive') 486 self._args.append(opts) 487 return self 488 489 def add_drive(self, path, opts='', interface='virtio', format=imgfmt): 490 '''Add a virtio-blk drive to the VM''' 491 options = ['if=%s' % interface, 492 'id=drive%d' % self._num_drives] 493 494 if path is not None: 495 options.append('file=%s' % path) 496 options.append('format=%s' % format) 497 options.append('cache=%s' % cachemode) 498 499 if opts: 500 options.append(opts) 501 502 if format == 'luks' and 'key-secret' not in opts: 503 # default luks support 504 if luks_default_secret_object not in self._args: 505 self.add_object(luks_default_secret_object) 506 507 options.append(luks_default_key_secret_opt) 508 509 self._args.append('-drive') 510 self._args.append(','.join(options)) 511 self._num_drives += 1 512 return self 513 514 def add_blockdev(self, opts): 515 self._args.append('-blockdev') 516 if isinstance(opts, str): 517 self._args.append(opts) 518 else: 519 self._args.append(','.join(opts)) 520 return self 521 522 def add_incoming(self, addr): 523 self._args.append('-incoming') 524 self._args.append(addr) 525 return self 526 527 def pause_drive(self, drive, event=None): 528 '''Pause drive r/w operations''' 529 if not event: 530 self.pause_drive(drive, "read_aio") 531 self.pause_drive(drive, "write_aio") 532 return 533 self.qmp('human-monitor-command', 534 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) 535 536 def resume_drive(self, drive): 537 self.qmp('human-monitor-command', 538 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) 539 540 def hmp_qemu_io(self, drive, cmd): 541 '''Write to a given drive using an HMP command''' 542 return self.qmp('human-monitor-command', 543 command_line='qemu-io %s "%s"' % (drive, cmd)) 544 545 def flatten_qmp_object(self, obj, output=None, basestr=''): 546 if output is None: 547 output = dict() 548 if isinstance(obj, list): 549 for i in range(len(obj)): 550 self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.') 551 elif isinstance(obj, dict): 552 for key in obj: 553 self.flatten_qmp_object(obj[key], output, basestr + key + '.') 554 else: 555 output[basestr[:-1]] = obj # Strip trailing '.' 556 return output 557 558 def qmp_to_opts(self, obj): 559 obj = self.flatten_qmp_object(obj) 560 output_list = list() 561 for key in obj: 562 output_list += [key + '=' + obj[key]] 563 return ','.join(output_list) 564 565 def get_qmp_events_filtered(self, wait=60.0): 566 result = [] 567 for ev in self.get_qmp_events(wait=wait): 568 result.append(filter_qmp_event(ev)) 569 return result 570 571 def qmp_log(self, cmd, filters=[], indent=None, **kwargs): 572 full_cmd = OrderedDict(( 573 ("execute", cmd), 574 ("arguments", ordered_qmp(kwargs)) 575 )) 576 log(full_cmd, filters, indent=indent) 577 result = self.qmp(cmd, **kwargs) 578 log(result, filters, indent=indent) 579 return result 580 581 # Returns None on success, and an error string on failure 582 def run_job(self, job, auto_finalize=True, auto_dismiss=False, 583 pre_finalize=None, cancel=False, use_log=True, wait=60.0): 584 """ 585 run_job moves a job from creation through to dismissal. 586 587 :param job: String. ID of recently-launched job 588 :param auto_finalize: Bool. True if the job was launched with 589 auto_finalize. Defaults to True. 590 :param auto_dismiss: Bool. True if the job was launched with 591 auto_dismiss=True. Defaults to False. 592 :param pre_finalize: Callback. A callable that takes no arguments to be 593 invoked prior to issuing job-finalize, if any. 594 :param cancel: Bool. When true, cancels the job after the pre_finalize 595 callback. 596 :param use_log: Bool. When false, does not log QMP messages. 597 :param wait: Float. Timeout value specifying how long to wait for any 598 event, in seconds. Defaults to 60.0. 599 """ 600 match_device = {'data': {'device': job}} 601 match_id = {'data': {'id': job}} 602 events = [ 603 ('BLOCK_JOB_COMPLETED', match_device), 604 ('BLOCK_JOB_CANCELLED', match_device), 605 ('BLOCK_JOB_ERROR', match_device), 606 ('BLOCK_JOB_READY', match_device), 607 ('BLOCK_JOB_PENDING', match_id), 608 ('JOB_STATUS_CHANGE', match_id) 609 ] 610 error = None 611 while True: 612 ev = filter_qmp_event(self.events_wait(events, timeout=wait)) 613 if ev['event'] != 'JOB_STATUS_CHANGE': 614 if use_log: 615 log(ev) 616 continue 617 status = ev['data']['status'] 618 if status == 'aborting': 619 result = self.qmp('query-jobs') 620 for j in result['return']: 621 if j['id'] == job: 622 error = j['error'] 623 if use_log: 624 log('Job failed: %s' % (j['error'])) 625 elif status == 'ready': 626 self.qmp_log('job-complete', id=job) 627 elif status == 'pending' and not auto_finalize: 628 if pre_finalize: 629 pre_finalize() 630 if cancel and use_log: 631 self.qmp_log('job-cancel', id=job) 632 elif cancel: 633 self.qmp('job-cancel', id=job) 634 elif use_log: 635 self.qmp_log('job-finalize', id=job) 636 else: 637 self.qmp('job-finalize', id=job) 638 elif status == 'concluded' and not auto_dismiss: 639 if use_log: 640 self.qmp_log('job-dismiss', id=job) 641 else: 642 self.qmp('job-dismiss', id=job) 643 elif status == 'null': 644 return error 645 646 # Returns None on success, and an error string on failure 647 def blockdev_create(self, options, job_id='job0', filters=None): 648 if filters is None: 649 filters = [filter_qmp_testfiles] 650 result = self.qmp_log('blockdev-create', filters=filters, 651 job_id=job_id, options=options) 652 653 if 'return' in result: 654 assert result['return'] == {} 655 job_result = self.run_job(job_id) 656 else: 657 job_result = result['error'] 658 659 log("") 660 return job_result 661 662 def enable_migration_events(self, name): 663 log('Enabling migration QMP events on %s...' % name) 664 log(self.qmp('migrate-set-capabilities', capabilities=[ 665 { 666 'capability': 'events', 667 'state': True 668 } 669 ])) 670 671 def wait_migration(self): 672 while True: 673 event = self.event_wait('MIGRATION') 674 log(event, filters=[filter_qmp_event]) 675 if event['data']['status'] == 'completed': 676 break 677 678 def node_info(self, node_name): 679 nodes = self.qmp('query-named-block-nodes') 680 for x in nodes['return']: 681 if x['node-name'] == node_name: 682 return x 683 return None 684 685 def query_bitmaps(self): 686 res = self.qmp("query-named-block-nodes") 687 return {device['node-name']: device['dirty-bitmaps'] 688 for device in res['return'] if 'dirty-bitmaps' in device} 689 690 def get_bitmap(self, node_name, bitmap_name, recording=None, bitmaps=None): 691 """ 692 get a specific bitmap from the object returned by query_bitmaps. 693 :param recording: If specified, filter results by the specified value. 694 :param bitmaps: If specified, use it instead of call query_bitmaps() 695 """ 696 if bitmaps is None: 697 bitmaps = self.query_bitmaps() 698 699 for bitmap in bitmaps[node_name]: 700 if bitmap.get('name', '') == bitmap_name: 701 if recording is None: 702 return bitmap 703 elif bitmap.get('recording') == recording: 704 return bitmap 705 return None 706 707 def check_bitmap_status(self, node_name, bitmap_name, fields): 708 ret = self.get_bitmap(node_name, bitmap_name) 709 710 return fields.items() <= ret.items() 711 712 713index_re = re.compile(r'([^\[]+)\[([^\]]+)\]') 714 715class QMPTestCase(unittest.TestCase): 716 '''Abstract base class for QMP test cases''' 717 718 def dictpath(self, d, path): 719 '''Traverse a path in a nested dict''' 720 for component in path.split('/'): 721 m = index_re.match(component) 722 if m: 723 component, idx = m.groups() 724 idx = int(idx) 725 726 if not isinstance(d, dict) or component not in d: 727 self.fail('failed path traversal for "%s" in "%s"' % (path, str(d))) 728 d = d[component] 729 730 if m: 731 if not isinstance(d, list): 732 self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d))) 733 try: 734 d = d[idx] 735 except IndexError: 736 self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d))) 737 return d 738 739 def assert_qmp_absent(self, d, path): 740 try: 741 result = self.dictpath(d, path) 742 except AssertionError: 743 return 744 self.fail('path "%s" has value "%s"' % (path, str(result))) 745 746 def assert_qmp(self, d, path, value): 747 '''Assert that the value for a specific path in a QMP dict 748 matches. When given a list of values, assert that any of 749 them matches.''' 750 751 result = self.dictpath(d, path) 752 753 # [] makes no sense as a list of valid values, so treat it as 754 # an actual single value. 755 if isinstance(value, list) and value != []: 756 for v in value: 757 if result == v: 758 return 759 self.fail('no match for "%s" in %s' % (str(result), str(value))) 760 else: 761 self.assertEqual(result, value, 762 '"%s" is "%s", expected "%s"' 763 % (path, str(result), str(value))) 764 765 def assert_no_active_block_jobs(self): 766 result = self.vm.qmp('query-block-jobs') 767 self.assert_qmp(result, 'return', []) 768 769 def assert_has_block_node(self, node_name=None, file_name=None): 770 """Issue a query-named-block-nodes and assert node_name and/or 771 file_name is present in the result""" 772 def check_equal_or_none(a, b): 773 return a == None or b == None or a == b 774 assert node_name or file_name 775 result = self.vm.qmp('query-named-block-nodes') 776 for x in result["return"]: 777 if check_equal_or_none(x.get("node-name"), node_name) and \ 778 check_equal_or_none(x.get("file"), file_name): 779 return 780 self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \ 781 (node_name, file_name, result)) 782 783 def assert_json_filename_equal(self, json_filename, reference): 784 '''Asserts that the given filename is a json: filename and that its 785 content is equal to the given reference object''' 786 self.assertEqual(json_filename[:5], 'json:') 787 self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])), 788 self.vm.flatten_qmp_object(reference)) 789 790 def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0): 791 '''Cancel a block job and wait for it to finish, returning the event''' 792 result = self.vm.qmp('block-job-cancel', device=drive, force=force) 793 self.assert_qmp(result, 'return', {}) 794 795 if resume: 796 self.vm.resume_drive(drive) 797 798 cancelled = False 799 result = None 800 while not cancelled: 801 for event in self.vm.get_qmp_events(wait=wait): 802 if event['event'] == 'BLOCK_JOB_COMPLETED' or \ 803 event['event'] == 'BLOCK_JOB_CANCELLED': 804 self.assert_qmp(event, 'data/device', drive) 805 result = event 806 cancelled = True 807 elif event['event'] == 'JOB_STATUS_CHANGE': 808 self.assert_qmp(event, 'data/id', drive) 809 810 811 self.assert_no_active_block_jobs() 812 return result 813 814 def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0): 815 '''Wait for a block job to finish, returning the event''' 816 while True: 817 for event in self.vm.get_qmp_events(wait=wait): 818 if event['event'] == 'BLOCK_JOB_COMPLETED': 819 self.assert_qmp(event, 'data/device', drive) 820 self.assert_qmp_absent(event, 'data/error') 821 if check_offset: 822 self.assert_qmp(event, 'data/offset', event['data']['len']) 823 self.assert_no_active_block_jobs() 824 return event 825 elif event['event'] == 'JOB_STATUS_CHANGE': 826 self.assert_qmp(event, 'data/id', drive) 827 828 def wait_ready(self, drive='drive0'): 829 '''Wait until a block job BLOCK_JOB_READY event''' 830 f = {'data': {'type': 'mirror', 'device': drive } } 831 event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f) 832 833 def wait_ready_and_cancel(self, drive='drive0'): 834 self.wait_ready(drive=drive) 835 event = self.cancel_and_wait(drive=drive) 836 self.assertEqual(event['event'], 'BLOCK_JOB_COMPLETED') 837 self.assert_qmp(event, 'data/type', 'mirror') 838 self.assert_qmp(event, 'data/offset', event['data']['len']) 839 840 def complete_and_wait(self, drive='drive0', wait_ready=True): 841 '''Complete a block job and wait for it to finish''' 842 if wait_ready: 843 self.wait_ready(drive=drive) 844 845 result = self.vm.qmp('block-job-complete', device=drive) 846 self.assert_qmp(result, 'return', {}) 847 848 event = self.wait_until_completed(drive=drive) 849 self.assert_qmp(event, 'data/type', 'mirror') 850 851 def pause_wait(self, job_id='job0'): 852 with Timeout(1, "Timeout waiting for job to pause"): 853 while True: 854 result = self.vm.qmp('query-block-jobs') 855 found = False 856 for job in result['return']: 857 if job['device'] == job_id: 858 found = True 859 if job['paused'] == True and job['busy'] == False: 860 return job 861 break 862 assert found 863 864 def pause_job(self, job_id='job0', wait=True): 865 result = self.vm.qmp('block-job-pause', device=job_id) 866 self.assert_qmp(result, 'return', {}) 867 if wait: 868 return self.pause_wait(job_id) 869 return result 870 871 def case_skip(self, reason): 872 '''Skip this test case''' 873 case_notrun(reason) 874 self.skipTest(reason) 875 876 877def notrun(reason): 878 '''Skip this test suite''' 879 # Each test in qemu-iotests has a number ("seq") 880 seq = os.path.basename(sys.argv[0]) 881 882 open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n') 883 print('%s not run: %s' % (seq, reason)) 884 sys.exit(0) 885 886def case_notrun(reason): 887 '''Mark this test case as not having been run (without actually 888 skipping it, that is left to the caller). See 889 QMPTestCase.case_skip() for a variant that actually skips the 890 current test case.''' 891 892 # Each test in qemu-iotests has a number ("seq") 893 seq = os.path.basename(sys.argv[0]) 894 895 open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( 896 ' [case not run] ' + reason + '\n') 897 898def verify_image_format(supported_fmts=[], unsupported_fmts=[]): 899 assert not (supported_fmts and unsupported_fmts) 900 901 if 'generic' in supported_fmts and \ 902 os.environ.get('IMGFMT_GENERIC', 'true') == 'true': 903 # similar to 904 # _supported_fmt generic 905 # for bash tests 906 return 907 908 not_sup = supported_fmts and (imgfmt not in supported_fmts) 909 if not_sup or (imgfmt in unsupported_fmts): 910 notrun('not suitable for this image format: %s' % imgfmt) 911 912def verify_protocol(supported=[], unsupported=[]): 913 assert not (supported and unsupported) 914 915 if 'generic' in supported: 916 return 917 918 not_sup = supported and (imgproto not in supported) 919 if not_sup or (imgproto in unsupported): 920 notrun('not suitable for this protocol: %s' % imgproto) 921 922def verify_platform(supported_oses=['linux']): 923 if True not in [sys.platform.startswith(x) for x in supported_oses]: 924 notrun('not suitable for this OS: %s' % sys.platform) 925 926def verify_cache_mode(supported_cache_modes=[]): 927 if supported_cache_modes and (cachemode not in supported_cache_modes): 928 notrun('not suitable for this cache mode: %s' % cachemode) 929 930def supports_quorum(): 931 return 'quorum' in qemu_img_pipe('--help') 932 933def verify_quorum(): 934 '''Skip test suite if quorum support is not available''' 935 if not supports_quorum(): 936 notrun('quorum support missing') 937 938def qemu_pipe(*args): 939 '''Run qemu with an option to print something and exit (e.g. a help option), 940 and return its output''' 941 args = [qemu_prog] + qemu_opts + list(args) 942 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 943 stderr=subprocess.STDOUT, 944 universal_newlines=True) 945 exitcode = subp.wait() 946 if exitcode < 0: 947 sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, 948 ' '.join(args))) 949 return subp.communicate()[0] 950 951def supported_formats(read_only=False): 952 '''Set 'read_only' to True to check ro-whitelist 953 Otherwise, rw-whitelist is checked''' 954 955 if not hasattr(supported_formats, "formats"): 956 supported_formats.formats = {} 957 958 if read_only not in supported_formats.formats: 959 format_message = qemu_pipe("-drive", "format=help") 960 line = 1 if read_only else 0 961 supported_formats.formats[read_only] = \ 962 format_message.splitlines()[line].split(":")[1].split() 963 964 return supported_formats.formats[read_only] 965 966def skip_if_unsupported(required_formats=[], read_only=False): 967 '''Skip Test Decorator 968 Runs the test if all the required formats are whitelisted''' 969 def skip_test_decorator(func): 970 def func_wrapper(test_case: QMPTestCase, *args, **kwargs): 971 if callable(required_formats): 972 fmts = required_formats(test_case) 973 else: 974 fmts = required_formats 975 976 usf_list = list(set(fmts) - set(supported_formats(read_only))) 977 if usf_list: 978 test_case.case_skip('{}: formats {} are not whitelisted'.format( 979 test_case, usf_list)) 980 else: 981 return func(test_case, *args, **kwargs) 982 return func_wrapper 983 return skip_test_decorator 984 985def skip_if_user_is_root(func): 986 '''Skip Test Decorator 987 Runs the test only without root permissions''' 988 def func_wrapper(*args, **kwargs): 989 if os.getuid() == 0: 990 case_notrun('{}: cannot be run as root'.format(args[0])) 991 else: 992 return func(*args, **kwargs) 993 return func_wrapper 994 995def execute_unittest(output, verbosity, debug): 996 runner = unittest.TextTestRunner(stream=output, descriptions=True, 997 verbosity=verbosity) 998 try: 999 # unittest.main() will use sys.exit(); so expect a SystemExit 1000 # exception 1001 unittest.main(testRunner=runner) 1002 finally: 1003 if not debug: 1004 out = output.getvalue() 1005 out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out) 1006 1007 # Hide skipped tests from the reference output 1008 out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) 1009 out_first_line, out_rest = out.split('\n', 1) 1010 out = out_first_line.replace('s', '.') + '\n' + out_rest 1011 1012 sys.stderr.write(out) 1013 1014def execute_test(test_function=None, 1015 supported_fmts=[], supported_oses=['linux'], 1016 supported_cache_modes=[], unsupported_fmts=[], 1017 supported_protocols=[], unsupported_protocols=[]): 1018 """Run either unittest or script-style tests.""" 1019 1020 # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to 1021 # indicate that we're not being run via "check". There may be 1022 # other things set up by "check" that individual test cases rely 1023 # on. 1024 if test_dir is None or qemu_default_machine is None: 1025 sys.stderr.write('Please run this test via the "check" script\n') 1026 sys.exit(os.EX_USAGE) 1027 1028 debug = '-d' in sys.argv 1029 verbosity = 1 1030 verify_image_format(supported_fmts, unsupported_fmts) 1031 verify_protocol(supported_protocols, unsupported_protocols) 1032 verify_platform(supported_oses) 1033 verify_cache_mode(supported_cache_modes) 1034 1035 if debug: 1036 output = sys.stdout 1037 verbosity = 2 1038 sys.argv.remove('-d') 1039 else: 1040 # We need to filter out the time taken from the output so that 1041 # qemu-iotest can reliably diff the results against master output. 1042 output = io.StringIO() 1043 1044 logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) 1045 1046 if not test_function: 1047 execute_unittest(output, verbosity, debug) 1048 else: 1049 test_function() 1050 1051def script_main(test_function, *args, **kwargs): 1052 """Run script-style tests outside of the unittest framework""" 1053 execute_test(test_function, *args, **kwargs) 1054 1055def main(*args, **kwargs): 1056 """Run tests using the unittest framework""" 1057 execute_test(None, *args, **kwargs) 1058