1from __future__ import print_function 2# Common utilities and Python wrappers for qemu-iotests 3# 4# Copyright (C) 2012 IBM Corp. 5# 6# This program is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program. If not, see <http://www.gnu.org/licenses/>. 18# 19 20import errno 21import os 22import re 23import subprocess 24import string 25import unittest 26import sys 27import struct 28import json 29import signal 30import logging 31import atexit 32import io 33from collections import OrderedDict 34 35sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 36from qemu import qtest 37 38assert sys.version_info >= (3,6) 39 40# This will not work if arguments contain spaces but is necessary if we 41# want to support the override options that ./check supports. 42qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] 43if os.environ.get('QEMU_IMG_OPTIONS'): 44 qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') 45 46qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 47if os.environ.get('QEMU_IO_OPTIONS'): 48 qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') 49 50qemu_nbd_args = [os.environ.get('QEMU_NBD_PROG', 'qemu-nbd')] 51if os.environ.get('QEMU_NBD_OPTIONS'): 52 qemu_nbd_args += os.environ['QEMU_NBD_OPTIONS'].strip().split(' ') 53 54qemu_prog = os.environ.get('QEMU_PROG', 'qemu') 55qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') 56 57imgfmt = os.environ.get('IMGFMT', 'raw') 58imgproto = os.environ.get('IMGPROTO', 'file') 59test_dir = os.environ.get('TEST_DIR') 60sock_dir = os.environ.get('SOCK_DIR') 61output_dir = os.environ.get('OUTPUT_DIR', '.') 62cachemode = os.environ.get('CACHEMODE') 63qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') 64 65socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') 66 67luks_default_secret_object = 'secret,id=keysec0,data=' + \ 68 os.environ.get('IMGKEYSECRET', '') 69luks_default_key_secret_opt = 'key-secret=keysec0' 70 71 72def qemu_img(*args): 73 '''Run qemu-img and return the exit code''' 74 devnull = open('/dev/null', 'r+') 75 exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) 76 if exitcode < 0: 77 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 78 return exitcode 79 80def ordered_qmp(qmsg, conv_keys=True): 81 # Dictionaries are not ordered prior to 3.6, therefore: 82 if isinstance(qmsg, list): 83 return [ordered_qmp(atom) for atom in qmsg] 84 if isinstance(qmsg, dict): 85 od = OrderedDict() 86 for k, v in sorted(qmsg.items()): 87 if conv_keys: 88 k = k.replace('_', '-') 89 od[k] = ordered_qmp(v, conv_keys=False) 90 return od 91 return qmsg 92 93def qemu_img_create(*args): 94 args = list(args) 95 96 # default luks support 97 if '-f' in args and args[args.index('-f') + 1] == 'luks': 98 if '-o' in args: 99 i = args.index('-o') 100 if 'key-secret' not in args[i + 1]: 101 args[i + 1].append(luks_default_key_secret_opt) 102 args.insert(i + 2, '--object') 103 args.insert(i + 3, luks_default_secret_object) 104 else: 105 args = ['-o', luks_default_key_secret_opt, 106 '--object', luks_default_secret_object] + args 107 108 args.insert(0, 'create') 109 110 return qemu_img(*args) 111 112def qemu_img_verbose(*args): 113 '''Run qemu-img without suppressing its output and return the exit code''' 114 exitcode = subprocess.call(qemu_img_args + list(args)) 115 if exitcode < 0: 116 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 117 return exitcode 118 119def qemu_img_pipe(*args): 120 '''Run qemu-img and return its output''' 121 subp = subprocess.Popen(qemu_img_args + list(args), 122 stdout=subprocess.PIPE, 123 stderr=subprocess.STDOUT, 124 universal_newlines=True) 125 exitcode = subp.wait() 126 if exitcode < 0: 127 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 128 return subp.communicate()[0] 129 130def qemu_img_log(*args): 131 result = qemu_img_pipe(*args) 132 log(result, filters=[filter_testfiles]) 133 return result 134 135def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): 136 args = [ 'info' ] 137 if imgopts: 138 args.append('--image-opts') 139 else: 140 args += [ '-f', imgfmt ] 141 args += extra_args 142 args.append(filename) 143 144 output = qemu_img_pipe(*args) 145 if not filter_path: 146 filter_path = filename 147 log(filter_img_info(output, filter_path)) 148 149def qemu_io(*args): 150 '''Run qemu-io and return the stdout data''' 151 args = qemu_io_args + list(args) 152 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 153 stderr=subprocess.STDOUT, 154 universal_newlines=True) 155 exitcode = subp.wait() 156 if exitcode < 0: 157 sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) 158 return subp.communicate()[0] 159 160def qemu_io_silent(*args): 161 '''Run qemu-io and return the exit code, suppressing stdout''' 162 args = qemu_io_args + list(args) 163 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w')) 164 if exitcode < 0: 165 sys.stderr.write('qemu-io received signal %i: %s\n' % 166 (-exitcode, ' '.join(args))) 167 return exitcode 168 169def qemu_io_silent_check(*args): 170 '''Run qemu-io and return the true if subprocess returned 0''' 171 args = qemu_io_args + list(args) 172 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'), 173 stderr=subprocess.STDOUT) 174 return exitcode == 0 175 176def get_virtio_scsi_device(): 177 if qemu_default_machine == 's390-ccw-virtio': 178 return 'virtio-scsi-ccw' 179 return 'virtio-scsi-pci' 180 181class QemuIoInteractive: 182 def __init__(self, *args): 183 self.args = qemu_io_args + list(args) 184 self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE, 185 stdout=subprocess.PIPE, 186 stderr=subprocess.STDOUT, 187 universal_newlines=True) 188 assert self._p.stdout.read(9) == 'qemu-io> ' 189 190 def close(self): 191 self._p.communicate('q\n') 192 193 def _read_output(self): 194 pattern = 'qemu-io> ' 195 n = len(pattern) 196 pos = 0 197 s = [] 198 while pos != n: 199 c = self._p.stdout.read(1) 200 # check unexpected EOF 201 assert c != '' 202 s.append(c) 203 if c == pattern[pos]: 204 pos += 1 205 else: 206 pos = 0 207 208 return ''.join(s[:-n]) 209 210 def cmd(self, cmd): 211 # quit command is in close(), '\n' is added automatically 212 assert '\n' not in cmd 213 cmd = cmd.strip() 214 assert cmd != 'q' and cmd != 'quit' 215 self._p.stdin.write(cmd + '\n') 216 self._p.stdin.flush() 217 return self._read_output() 218 219 220def qemu_nbd(*args): 221 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 222 return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) 223 224def qemu_nbd_early_pipe(*args): 225 '''Run qemu-nbd in daemon mode and return both the parent's exit code 226 and its output in case of an error''' 227 subp = subprocess.Popen(qemu_nbd_args + ['--fork'] + list(args), 228 stdout=subprocess.PIPE, 229 stderr=subprocess.STDOUT, 230 universal_newlines=True) 231 exitcode = subp.wait() 232 if exitcode < 0: 233 sys.stderr.write('qemu-nbd received signal %i: %s\n' % 234 (-exitcode, 235 ' '.join(qemu_nbd_args + ['--fork'] + list(args)))) 236 if exitcode == 0: 237 return exitcode, '' 238 else: 239 return exitcode, subp.communicate()[0] 240 241def qemu_nbd_popen(*args): 242 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 243 return subprocess.Popen(qemu_nbd_args + ['--persistent'] + list(args)) 244 245def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt): 246 '''Return True if two image files are identical''' 247 return qemu_img('compare', '-f', fmt1, 248 '-F', fmt2, img1, img2) == 0 249 250def create_image(name, size): 251 '''Create a fully-allocated raw image with sector markers''' 252 file = open(name, 'wb') 253 i = 0 254 while i < size: 255 sector = struct.pack('>l504xl', i // 512, i // 512) 256 file.write(sector) 257 i = i + 512 258 file.close() 259 260def image_size(img): 261 '''Return image's virtual size''' 262 r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img) 263 return json.loads(r)['virtual-size'] 264 265def is_str(val): 266 return isinstance(val, str) 267 268test_dir_re = re.compile(r"%s" % test_dir) 269def filter_test_dir(msg): 270 return test_dir_re.sub("TEST_DIR", msg) 271 272win32_re = re.compile(r"\r") 273def filter_win32(msg): 274 return win32_re.sub("", msg) 275 276qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)") 277def filter_qemu_io(msg): 278 msg = filter_win32(msg) 279 return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg) 280 281chown_re = re.compile(r"chown [0-9]+:[0-9]+") 282def filter_chown(msg): 283 return chown_re.sub("chown UID:GID", msg) 284 285def filter_qmp_event(event): 286 '''Filter a QMP event dict''' 287 event = dict(event) 288 if 'timestamp' in event: 289 event['timestamp']['seconds'] = 'SECS' 290 event['timestamp']['microseconds'] = 'USECS' 291 return event 292 293def filter_qmp(qmsg, filter_fn): 294 '''Given a string filter, filter a QMP object's values. 295 filter_fn takes a (key, value) pair.''' 296 # Iterate through either lists or dicts; 297 if isinstance(qmsg, list): 298 items = enumerate(qmsg) 299 else: 300 items = qmsg.items() 301 302 for k, v in items: 303 if isinstance(v, list) or isinstance(v, dict): 304 qmsg[k] = filter_qmp(v, filter_fn) 305 else: 306 qmsg[k] = filter_fn(k, v) 307 return qmsg 308 309def filter_testfiles(msg): 310 prefix = os.path.join(test_dir, "%s-" % (os.getpid())) 311 return msg.replace(prefix, 'TEST_DIR/PID-') 312 313def filter_qmp_testfiles(qmsg): 314 def _filter(key, value): 315 if is_str(value): 316 return filter_testfiles(value) 317 return value 318 return filter_qmp(qmsg, _filter) 319 320def filter_generated_node_ids(msg): 321 return re.sub("#block[0-9]+", "NODE_NAME", msg) 322 323def filter_img_info(output, filename): 324 lines = [] 325 for line in output.split('\n'): 326 if 'disk size' in line or 'actual-size' in line: 327 continue 328 line = line.replace(filename, 'TEST_IMG') \ 329 .replace(imgfmt, 'IMGFMT') 330 line = re.sub('iters: [0-9]+', 'iters: XXX', line) 331 line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line) 332 line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) 333 lines.append(line) 334 return '\n'.join(lines) 335 336def filter_imgfmt(msg): 337 return msg.replace(imgfmt, 'IMGFMT') 338 339def filter_qmp_imgfmt(qmsg): 340 def _filter(key, value): 341 if is_str(value): 342 return filter_imgfmt(value) 343 return value 344 return filter_qmp(qmsg, _filter) 345 346def log(msg, filters=[], indent=None): 347 '''Logs either a string message or a JSON serializable message (like QMP). 348 If indent is provided, JSON serializable messages are pretty-printed.''' 349 for flt in filters: 350 msg = flt(msg) 351 if isinstance(msg, dict) or isinstance(msg, list): 352 # Python < 3.4 needs to know not to add whitespace when pretty-printing: 353 separators = (', ', ': ') if indent is None else (',', ': ') 354 # Don't sort if it's already sorted 355 do_sort = not isinstance(msg, OrderedDict) 356 print(json.dumps(msg, sort_keys=do_sort, 357 indent=indent, separators=separators)) 358 else: 359 print(msg) 360 361class Timeout: 362 def __init__(self, seconds, errmsg = "Timeout"): 363 self.seconds = seconds 364 self.errmsg = errmsg 365 def __enter__(self): 366 signal.signal(signal.SIGALRM, self.timeout) 367 signal.setitimer(signal.ITIMER_REAL, self.seconds) 368 return self 369 def __exit__(self, type, value, traceback): 370 signal.setitimer(signal.ITIMER_REAL, 0) 371 return False 372 def timeout(self, signum, frame): 373 raise Exception(self.errmsg) 374 375def file_pattern(name): 376 return "{0}-{1}".format(os.getpid(), name) 377 378class FilePaths(object): 379 """ 380 FilePaths is an auto-generated filename that cleans itself up. 381 382 Use this context manager to generate filenames and ensure that the file 383 gets deleted:: 384 385 with FilePaths(['test.img']) as img_path: 386 qemu_img('create', img_path, '1G') 387 # migration_sock_path is automatically deleted 388 """ 389 def __init__(self, names, base_dir=test_dir): 390 self.paths = [] 391 for name in names: 392 self.paths.append(os.path.join(base_dir, file_pattern(name))) 393 394 def __enter__(self): 395 return self.paths 396 397 def __exit__(self, exc_type, exc_val, exc_tb): 398 try: 399 for path in self.paths: 400 os.remove(path) 401 except OSError: 402 pass 403 return False 404 405class FilePath(FilePaths): 406 """ 407 FilePath is a specialization of FilePaths that takes a single filename. 408 """ 409 def __init__(self, name, base_dir=test_dir): 410 super(FilePath, self).__init__([name], base_dir) 411 412 def __enter__(self): 413 return self.paths[0] 414 415def file_path_remover(): 416 for path in reversed(file_path_remover.paths): 417 try: 418 os.remove(path) 419 except OSError: 420 pass 421 422 423def file_path(*names, base_dir=test_dir): 424 ''' Another way to get auto-generated filename that cleans itself up. 425 426 Use is as simple as: 427 428 img_a, img_b = file_path('a.img', 'b.img') 429 sock = file_path('socket') 430 ''' 431 432 if not hasattr(file_path_remover, 'paths'): 433 file_path_remover.paths = [] 434 atexit.register(file_path_remover) 435 436 paths = [] 437 for name in names: 438 filename = file_pattern(name) 439 path = os.path.join(base_dir, filename) 440 file_path_remover.paths.append(path) 441 paths.append(path) 442 443 return paths[0] if len(paths) == 1 else paths 444 445def remote_filename(path): 446 if imgproto == 'file': 447 return path 448 elif imgproto == 'ssh': 449 return "ssh://%s@127.0.0.1:22%s" % (os.environ.get('USER'), path) 450 else: 451 raise Exception("Protocol %s not supported" % (imgproto)) 452 453class VM(qtest.QEMUQtestMachine): 454 '''A QEMU VM''' 455 456 def __init__(self, path_suffix=''): 457 name = "qemu%s-%d" % (path_suffix, os.getpid()) 458 super(VM, self).__init__(qemu_prog, qemu_opts, name=name, 459 test_dir=test_dir, 460 socket_scm_helper=socket_scm_helper, 461 sock_dir=sock_dir) 462 self._num_drives = 0 463 464 def add_object(self, opts): 465 self._args.append('-object') 466 self._args.append(opts) 467 return self 468 469 def add_device(self, opts): 470 self._args.append('-device') 471 self._args.append(opts) 472 return self 473 474 def add_drive_raw(self, opts): 475 self._args.append('-drive') 476 self._args.append(opts) 477 return self 478 479 def add_drive(self, path, opts='', interface='virtio', format=imgfmt): 480 '''Add a virtio-blk drive to the VM''' 481 options = ['if=%s' % interface, 482 'id=drive%d' % self._num_drives] 483 484 if path is not None: 485 options.append('file=%s' % path) 486 options.append('format=%s' % format) 487 options.append('cache=%s' % cachemode) 488 489 if opts: 490 options.append(opts) 491 492 if format == 'luks' and 'key-secret' not in opts: 493 # default luks support 494 if luks_default_secret_object not in self._args: 495 self.add_object(luks_default_secret_object) 496 497 options.append(luks_default_key_secret_opt) 498 499 self._args.append('-drive') 500 self._args.append(','.join(options)) 501 self._num_drives += 1 502 return self 503 504 def add_blockdev(self, opts): 505 self._args.append('-blockdev') 506 if isinstance(opts, str): 507 self._args.append(opts) 508 else: 509 self._args.append(','.join(opts)) 510 return self 511 512 def add_incoming(self, addr): 513 self._args.append('-incoming') 514 self._args.append(addr) 515 return self 516 517 def pause_drive(self, drive, event=None): 518 '''Pause drive r/w operations''' 519 if not event: 520 self.pause_drive(drive, "read_aio") 521 self.pause_drive(drive, "write_aio") 522 return 523 self.qmp('human-monitor-command', 524 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) 525 526 def resume_drive(self, drive): 527 self.qmp('human-monitor-command', 528 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) 529 530 def hmp_qemu_io(self, drive, cmd): 531 '''Write to a given drive using an HMP command''' 532 return self.qmp('human-monitor-command', 533 command_line='qemu-io %s "%s"' % (drive, cmd)) 534 535 def flatten_qmp_object(self, obj, output=None, basestr=''): 536 if output is None: 537 output = dict() 538 if isinstance(obj, list): 539 for i in range(len(obj)): 540 self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.') 541 elif isinstance(obj, dict): 542 for key in obj: 543 self.flatten_qmp_object(obj[key], output, basestr + key + '.') 544 else: 545 output[basestr[:-1]] = obj # Strip trailing '.' 546 return output 547 548 def qmp_to_opts(self, obj): 549 obj = self.flatten_qmp_object(obj) 550 output_list = list() 551 for key in obj: 552 output_list += [key + '=' + obj[key]] 553 return ','.join(output_list) 554 555 def get_qmp_events_filtered(self, wait=60.0): 556 result = [] 557 for ev in self.get_qmp_events(wait=wait): 558 result.append(filter_qmp_event(ev)) 559 return result 560 561 def qmp_log(self, cmd, filters=[], indent=None, **kwargs): 562 full_cmd = OrderedDict(( 563 ("execute", cmd), 564 ("arguments", ordered_qmp(kwargs)) 565 )) 566 log(full_cmd, filters, indent=indent) 567 result = self.qmp(cmd, **kwargs) 568 log(result, filters, indent=indent) 569 return result 570 571 # Returns None on success, and an error string on failure 572 def run_job(self, job, auto_finalize=True, auto_dismiss=False, 573 pre_finalize=None, cancel=False, use_log=True, wait=60.0): 574 """ 575 run_job moves a job from creation through to dismissal. 576 577 :param job: String. ID of recently-launched job 578 :param auto_finalize: Bool. True if the job was launched with 579 auto_finalize. Defaults to True. 580 :param auto_dismiss: Bool. True if the job was launched with 581 auto_dismiss=True. Defaults to False. 582 :param pre_finalize: Callback. A callable that takes no arguments to be 583 invoked prior to issuing job-finalize, if any. 584 :param cancel: Bool. When true, cancels the job after the pre_finalize 585 callback. 586 :param use_log: Bool. When false, does not log QMP messages. 587 :param wait: Float. Timeout value specifying how long to wait for any 588 event, in seconds. Defaults to 60.0. 589 """ 590 match_device = {'data': {'device': job}} 591 match_id = {'data': {'id': job}} 592 events = [ 593 ('BLOCK_JOB_COMPLETED', match_device), 594 ('BLOCK_JOB_CANCELLED', match_device), 595 ('BLOCK_JOB_ERROR', match_device), 596 ('BLOCK_JOB_READY', match_device), 597 ('BLOCK_JOB_PENDING', match_id), 598 ('JOB_STATUS_CHANGE', match_id) 599 ] 600 error = None 601 while True: 602 ev = filter_qmp_event(self.events_wait(events)) 603 if ev['event'] != 'JOB_STATUS_CHANGE': 604 if use_log: 605 log(ev) 606 continue 607 status = ev['data']['status'] 608 if status == 'aborting': 609 result = self.qmp('query-jobs') 610 for j in result['return']: 611 if j['id'] == job: 612 error = j['error'] 613 if use_log: 614 log('Job failed: %s' % (j['error'])) 615 elif status == 'pending' and not auto_finalize: 616 if pre_finalize: 617 pre_finalize() 618 if cancel and use_log: 619 self.qmp_log('job-cancel', id=job) 620 elif cancel: 621 self.qmp('job-cancel', id=job) 622 elif use_log: 623 self.qmp_log('job-finalize', id=job) 624 else: 625 self.qmp('job-finalize', id=job) 626 elif status == 'concluded' and not auto_dismiss: 627 if use_log: 628 self.qmp_log('job-dismiss', id=job) 629 else: 630 self.qmp('job-dismiss', id=job) 631 elif status == 'null': 632 return error 633 634 def enable_migration_events(self, name): 635 log('Enabling migration QMP events on %s...' % name) 636 log(self.qmp('migrate-set-capabilities', capabilities=[ 637 { 638 'capability': 'events', 639 'state': True 640 } 641 ])) 642 643 def wait_migration(self): 644 while True: 645 event = self.event_wait('MIGRATION') 646 log(event, filters=[filter_qmp_event]) 647 if event['data']['status'] == 'completed': 648 break 649 650 def node_info(self, node_name): 651 nodes = self.qmp('query-named-block-nodes') 652 for x in nodes['return']: 653 if x['node-name'] == node_name: 654 return x 655 return None 656 657 def query_bitmaps(self): 658 res = self.qmp("query-named-block-nodes") 659 return {device['node-name']: device['dirty-bitmaps'] 660 for device in res['return'] if 'dirty-bitmaps' in device} 661 662 def get_bitmap(self, node_name, bitmap_name, recording=None, bitmaps=None): 663 """ 664 get a specific bitmap from the object returned by query_bitmaps. 665 :param recording: If specified, filter results by the specified value. 666 :param bitmaps: If specified, use it instead of call query_bitmaps() 667 """ 668 if bitmaps is None: 669 bitmaps = self.query_bitmaps() 670 671 for bitmap in bitmaps[node_name]: 672 if bitmap.get('name', '') == bitmap_name: 673 if recording is None: 674 return bitmap 675 elif bitmap.get('recording') == recording: 676 return bitmap 677 return None 678 679 def check_bitmap_status(self, node_name, bitmap_name, fields): 680 ret = self.get_bitmap(node_name, bitmap_name) 681 682 return fields.items() <= ret.items() 683 684 685index_re = re.compile(r'([^\[]+)\[([^\]]+)\]') 686 687class QMPTestCase(unittest.TestCase): 688 '''Abstract base class for QMP test cases''' 689 690 def dictpath(self, d, path): 691 '''Traverse a path in a nested dict''' 692 for component in path.split('/'): 693 m = index_re.match(component) 694 if m: 695 component, idx = m.groups() 696 idx = int(idx) 697 698 if not isinstance(d, dict) or component not in d: 699 self.fail('failed path traversal for "%s" in "%s"' % (path, str(d))) 700 d = d[component] 701 702 if m: 703 if not isinstance(d, list): 704 self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d))) 705 try: 706 d = d[idx] 707 except IndexError: 708 self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d))) 709 return d 710 711 def assert_qmp_absent(self, d, path): 712 try: 713 result = self.dictpath(d, path) 714 except AssertionError: 715 return 716 self.fail('path "%s" has value "%s"' % (path, str(result))) 717 718 def assert_qmp(self, d, path, value): 719 '''Assert that the value for a specific path in a QMP dict 720 matches. When given a list of values, assert that any of 721 them matches.''' 722 723 result = self.dictpath(d, path) 724 725 # [] makes no sense as a list of valid values, so treat it as 726 # an actual single value. 727 if isinstance(value, list) and value != []: 728 for v in value: 729 if result == v: 730 return 731 self.fail('no match for "%s" in %s' % (str(result), str(value))) 732 else: 733 self.assertEqual(result, value, 734 'values not equal "%s" and "%s"' 735 % (str(result), str(value))) 736 737 def assert_no_active_block_jobs(self): 738 result = self.vm.qmp('query-block-jobs') 739 self.assert_qmp(result, 'return', []) 740 741 def assert_has_block_node(self, node_name=None, file_name=None): 742 """Issue a query-named-block-nodes and assert node_name and/or 743 file_name is present in the result""" 744 def check_equal_or_none(a, b): 745 return a == None or b == None or a == b 746 assert node_name or file_name 747 result = self.vm.qmp('query-named-block-nodes') 748 for x in result["return"]: 749 if check_equal_or_none(x.get("node-name"), node_name) and \ 750 check_equal_or_none(x.get("file"), file_name): 751 return 752 self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \ 753 (node_name, file_name, result)) 754 755 def assert_json_filename_equal(self, json_filename, reference): 756 '''Asserts that the given filename is a json: filename and that its 757 content is equal to the given reference object''' 758 self.assertEqual(json_filename[:5], 'json:') 759 self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])), 760 self.vm.flatten_qmp_object(reference)) 761 762 def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0): 763 '''Cancel a block job and wait for it to finish, returning the event''' 764 result = self.vm.qmp('block-job-cancel', device=drive, force=force) 765 self.assert_qmp(result, 'return', {}) 766 767 if resume: 768 self.vm.resume_drive(drive) 769 770 cancelled = False 771 result = None 772 while not cancelled: 773 for event in self.vm.get_qmp_events(wait=wait): 774 if event['event'] == 'BLOCK_JOB_COMPLETED' or \ 775 event['event'] == 'BLOCK_JOB_CANCELLED': 776 self.assert_qmp(event, 'data/device', drive) 777 result = event 778 cancelled = True 779 elif event['event'] == 'JOB_STATUS_CHANGE': 780 self.assert_qmp(event, 'data/id', drive) 781 782 783 self.assert_no_active_block_jobs() 784 return result 785 786 def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0): 787 '''Wait for a block job to finish, returning the event''' 788 while True: 789 for event in self.vm.get_qmp_events(wait=wait): 790 if event['event'] == 'BLOCK_JOB_COMPLETED': 791 self.assert_qmp(event, 'data/device', drive) 792 self.assert_qmp_absent(event, 'data/error') 793 if check_offset: 794 self.assert_qmp(event, 'data/offset', event['data']['len']) 795 self.assert_no_active_block_jobs() 796 return event 797 elif event['event'] == 'JOB_STATUS_CHANGE': 798 self.assert_qmp(event, 'data/id', drive) 799 800 def wait_ready(self, drive='drive0'): 801 '''Wait until a block job BLOCK_JOB_READY event''' 802 f = {'data': {'type': 'mirror', 'device': drive } } 803 event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f) 804 805 def wait_ready_and_cancel(self, drive='drive0'): 806 self.wait_ready(drive=drive) 807 event = self.cancel_and_wait(drive=drive) 808 self.assertEqual(event['event'], 'BLOCK_JOB_COMPLETED') 809 self.assert_qmp(event, 'data/type', 'mirror') 810 self.assert_qmp(event, 'data/offset', event['data']['len']) 811 812 def complete_and_wait(self, drive='drive0', wait_ready=True): 813 '''Complete a block job and wait for it to finish''' 814 if wait_ready: 815 self.wait_ready(drive=drive) 816 817 result = self.vm.qmp('block-job-complete', device=drive) 818 self.assert_qmp(result, 'return', {}) 819 820 event = self.wait_until_completed(drive=drive) 821 self.assert_qmp(event, 'data/type', 'mirror') 822 823 def pause_wait(self, job_id='job0'): 824 with Timeout(1, "Timeout waiting for job to pause"): 825 while True: 826 result = self.vm.qmp('query-block-jobs') 827 found = False 828 for job in result['return']: 829 if job['device'] == job_id: 830 found = True 831 if job['paused'] == True and job['busy'] == False: 832 return job 833 break 834 assert found 835 836 def pause_job(self, job_id='job0', wait=True): 837 result = self.vm.qmp('block-job-pause', device=job_id) 838 self.assert_qmp(result, 'return', {}) 839 if wait: 840 return self.pause_wait(job_id) 841 return result 842 843 def case_skip(self, reason): 844 '''Skip this test case''' 845 case_notrun(reason) 846 self.skipTest(reason) 847 848 849def notrun(reason): 850 '''Skip this test suite''' 851 # Each test in qemu-iotests has a number ("seq") 852 seq = os.path.basename(sys.argv[0]) 853 854 open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n') 855 print('%s not run: %s' % (seq, reason)) 856 sys.exit(0) 857 858def case_notrun(reason): 859 '''Mark this test case as not having been run (without actually 860 skipping it, that is left to the caller). See 861 QMPTestCase.case_skip() for a variant that actually skips the 862 current test case.''' 863 864 # Each test in qemu-iotests has a number ("seq") 865 seq = os.path.basename(sys.argv[0]) 866 867 open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( 868 ' [case not run] ' + reason + '\n') 869 870def verify_image_format(supported_fmts=[], unsupported_fmts=[]): 871 assert not (supported_fmts and unsupported_fmts) 872 873 if 'generic' in supported_fmts and \ 874 os.environ.get('IMGFMT_GENERIC', 'true') == 'true': 875 # similar to 876 # _supported_fmt generic 877 # for bash tests 878 return 879 880 not_sup = supported_fmts and (imgfmt not in supported_fmts) 881 if not_sup or (imgfmt in unsupported_fmts): 882 notrun('not suitable for this image format: %s' % imgfmt) 883 884def verify_protocol(supported=[], unsupported=[]): 885 assert not (supported and unsupported) 886 887 if 'generic' in supported: 888 return 889 890 not_sup = supported and (imgproto not in supported) 891 if not_sup or (imgproto in unsupported): 892 notrun('not suitable for this protocol: %s' % imgproto) 893 894def verify_platform(supported_oses=['linux']): 895 if True not in [sys.platform.startswith(x) for x in supported_oses]: 896 notrun('not suitable for this OS: %s' % sys.platform) 897 898def verify_cache_mode(supported_cache_modes=[]): 899 if supported_cache_modes and (cachemode not in supported_cache_modes): 900 notrun('not suitable for this cache mode: %s' % cachemode) 901 902def supports_quorum(): 903 return 'quorum' in qemu_img_pipe('--help') 904 905def verify_quorum(): 906 '''Skip test suite if quorum support is not available''' 907 if not supports_quorum(): 908 notrun('quorum support missing') 909 910def qemu_pipe(*args): 911 '''Run qemu with an option to print something and exit (e.g. a help option), 912 and return its output''' 913 args = [qemu_prog] + qemu_opts + list(args) 914 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 915 stderr=subprocess.STDOUT, 916 universal_newlines=True) 917 exitcode = subp.wait() 918 if exitcode < 0: 919 sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, 920 ' '.join(args))) 921 return subp.communicate()[0] 922 923def supported_formats(read_only=False): 924 '''Set 'read_only' to True to check ro-whitelist 925 Otherwise, rw-whitelist is checked''' 926 927 if not hasattr(supported_formats, "formats"): 928 supported_formats.formats = {} 929 930 if read_only not in supported_formats.formats: 931 format_message = qemu_pipe("-drive", "format=help") 932 line = 1 if read_only else 0 933 supported_formats.formats[read_only] = \ 934 format_message.splitlines()[line].split(":")[1].split() 935 936 return supported_formats.formats[read_only] 937 938def skip_if_unsupported(required_formats=[], read_only=False): 939 '''Skip Test Decorator 940 Runs the test if all the required formats are whitelisted''' 941 def skip_test_decorator(func): 942 def func_wrapper(test_case: QMPTestCase, *args, **kwargs): 943 if callable(required_formats): 944 fmts = required_formats(test_case) 945 else: 946 fmts = required_formats 947 948 usf_list = list(set(fmts) - set(supported_formats(read_only))) 949 if usf_list: 950 test_case.case_skip('{}: formats {} are not whitelisted'.format( 951 test_case, usf_list)) 952 else: 953 return func(test_case, *args, **kwargs) 954 return func_wrapper 955 return skip_test_decorator 956 957def skip_if_user_is_root(func): 958 '''Skip Test Decorator 959 Runs the test only without root permissions''' 960 def func_wrapper(*args, **kwargs): 961 if os.getuid() == 0: 962 case_notrun('{}: cannot be run as root'.format(args[0])) 963 else: 964 return func(*args, **kwargs) 965 return func_wrapper 966 967def execute_unittest(output, verbosity, debug): 968 runner = unittest.TextTestRunner(stream=output, descriptions=True, 969 verbosity=verbosity) 970 try: 971 # unittest.main() will use sys.exit(); so expect a SystemExit 972 # exception 973 unittest.main(testRunner=runner) 974 finally: 975 if not debug: 976 out = output.getvalue() 977 out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out) 978 979 # Hide skipped tests from the reference output 980 out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) 981 out_first_line, out_rest = out.split('\n', 1) 982 out = out_first_line.replace('s', '.') + '\n' + out_rest 983 984 sys.stderr.write(out) 985 986def execute_test(test_function=None, 987 supported_fmts=[], supported_oses=['linux'], 988 supported_cache_modes=[], unsupported_fmts=[], 989 supported_protocols=[], unsupported_protocols=[]): 990 """Run either unittest or script-style tests.""" 991 992 # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to 993 # indicate that we're not being run via "check". There may be 994 # other things set up by "check" that individual test cases rely 995 # on. 996 if test_dir is None or qemu_default_machine is None: 997 sys.stderr.write('Please run this test via the "check" script\n') 998 sys.exit(os.EX_USAGE) 999 1000 debug = '-d' in sys.argv 1001 verbosity = 1 1002 verify_image_format(supported_fmts, unsupported_fmts) 1003 verify_protocol(supported_protocols, unsupported_protocols) 1004 verify_platform(supported_oses) 1005 verify_cache_mode(supported_cache_modes) 1006 1007 if debug: 1008 output = sys.stdout 1009 verbosity = 2 1010 sys.argv.remove('-d') 1011 else: 1012 # We need to filter out the time taken from the output so that 1013 # qemu-iotest can reliably diff the results against master output. 1014 output = io.StringIO() 1015 1016 logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) 1017 1018 if not test_function: 1019 execute_unittest(output, verbosity, debug) 1020 else: 1021 test_function() 1022 1023def script_main(test_function, *args, **kwargs): 1024 """Run script-style tests outside of the unittest framework""" 1025 execute_test(test_function, *args, **kwargs) 1026 1027def main(*args, **kwargs): 1028 """Run tests using the unittest framework""" 1029 execute_test(None, *args, **kwargs) 1030