1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Collection, 37 Dict, 38 Iterable, 39 List, 40 Optional, 41 Union, 42 cast, 43) 44 45from .common import c_name 46from .error import QAPISemError 47from .parser import QAPIDoc 48from .source import QAPISourceInfo 49 50 51# Deserialized JSON objects as returned by the parser. 52# The values of this mapping are not necessary to exhaustively type 53# here (and also not practical as long as mypy lacks recursive 54# types), because the purpose of this module is to interrogate that 55# type. 56_JSONObject = Dict[str, object] 57 58 59# See check_name_str(), below. 60valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 61 r'(x-)?' 62 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 63 64 65def check_name_is_str(name: object, 66 info: QAPISourceInfo, 67 source: str) -> None: 68 """ 69 Ensure that ``name`` is a ``str``. 70 71 :raise QAPISemError: When ``name`` fails validation. 72 """ 73 if not isinstance(name, str): 74 raise QAPISemError(info, "%s requires a string name" % source) 75 76 77def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 78 """ 79 Ensure that ``name`` is a valid QAPI name. 80 81 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 82 starting with a letter. It may be prefixed by a downstream prefix 83 of the form __RFQDN_, or the experimental prefix ``x-``. If both 84 prefixes are present, the __RFDQN_ prefix goes first. 85 86 A valid name cannot start with ``q_``, which is reserved. 87 88 :param name: Name to check. 89 :param info: QAPI schema source file information. 90 :param source: Error string describing what ``name`` belongs to. 91 92 :raise QAPISemError: When ``name`` fails validation. 93 :return: The stem of the valid name, with no prefixes. 94 """ 95 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 96 # and 'q_obj_*' implicit type names. 97 match = valid_name.match(name) 98 if not match or c_name(name, False).startswith('q_'): 99 raise QAPISemError(info, "%s has an invalid name" % source) 100 return match.group(3) 101 102 103def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 104 """ 105 Ensure that ``name`` is a valid event name. 106 107 This means it must be a valid QAPI name as checked by 108 `check_name_str()`, but where the stem prohibits lowercase 109 characters and ``-``. 110 111 :param name: Name to check. 112 :param info: QAPI schema source file information. 113 :param source: Error string describing what ``name`` belongs to. 114 115 :raise QAPISemError: When ``name`` fails validation. 116 """ 117 stem = check_name_str(name, info, source) 118 if re.search(r'[a-z-]', stem): 119 raise QAPISemError( 120 info, "name of %s must not use lowercase or '-'" % source) 121 122 123def check_name_lower(name: str, info: QAPISourceInfo, source: str, 124 permit_upper: bool = False, 125 permit_underscore: bool = False) -> None: 126 """ 127 Ensure that ``name`` is a valid command or member name. 128 129 This means it must be a valid QAPI name as checked by 130 `check_name_str()`, but where the stem prohibits uppercase 131 characters and ``_``. 132 133 :param name: Name to check. 134 :param info: QAPI schema source file information. 135 :param source: Error string describing what ``name`` belongs to. 136 :param permit_upper: Additionally permit uppercase. 137 :param permit_underscore: Additionally permit ``_``. 138 139 :raise QAPISemError: When ``name`` fails validation. 140 """ 141 stem = check_name_str(name, info, source) 142 if ((not permit_upper and re.search(r'[A-Z]', stem)) 143 or (not permit_underscore and '_' in stem)): 144 raise QAPISemError( 145 info, "name of %s must not use uppercase or '_'" % source) 146 147 148def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 149 """ 150 Ensure that ``name`` is a valid user-defined type name. 151 152 This means it must be a valid QAPI name as checked by 153 `check_name_str()`, but where the stem must be in CamelCase. 154 155 :param name: Name to check. 156 :param info: QAPI schema source file information. 157 :param source: Error string describing what ``name`` belongs to. 158 159 :raise QAPISemError: When ``name`` fails validation. 160 """ 161 stem = check_name_str(name, info, source) 162 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 163 raise QAPISemError(info, "name of %s must use CamelCase" % source) 164 165 166def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 167 """ 168 Ensure that ``name`` is a valid definition name. 169 170 Based on the value of ``meta``, this means that: 171 - 'event' names adhere to `check_name_upper()`. 172 - 'command' names adhere to `check_name_lower()`. 173 - Else, meta is a type, and must pass `check_name_camel()`. 174 These names must not end with ``Kind`` nor ``List``. 175 176 :param name: Name to check. 177 :param info: QAPI schema source file information. 178 :param meta: Meta-type name of the QAPI expression. 179 180 :raise QAPISemError: When ``name`` fails validation. 181 """ 182 if meta == 'event': 183 check_name_upper(name, info, meta) 184 elif meta == 'command': 185 check_name_lower( 186 name, info, meta, 187 permit_underscore=name in info.pragma.command_name_exceptions) 188 else: 189 check_name_camel(name, info, meta) 190 if name.endswith('Kind') or name.endswith('List'): 191 raise QAPISemError( 192 info, "%s name should not end in '%s'" % (meta, name[-4:])) 193 194 195def check_keys(value: _JSONObject, 196 info: QAPISourceInfo, 197 source: str, 198 required: Collection[str], 199 optional: Collection[str]) -> None: 200 """ 201 Ensure that a dict has a specific set of keys. 202 203 :param value: The dict to check. 204 :param info: QAPI schema source file information. 205 :param source: Error string describing this ``value``. 206 :param required: Keys that *must* be present. 207 :param optional: Keys that *may* be present. 208 209 :raise QAPISemError: When unknown keys are present. 210 """ 211 212 def pprint(elems: Iterable[str]) -> str: 213 return ', '.join("'" + e + "'" for e in sorted(elems)) 214 215 missing = set(required) - set(value) 216 if missing: 217 raise QAPISemError( 218 info, 219 "%s misses key%s %s" 220 % (source, 's' if len(missing) > 1 else '', 221 pprint(missing))) 222 allowed = set(required) | set(optional) 223 unknown = set(value) - allowed 224 if unknown: 225 raise QAPISemError( 226 info, 227 "%s has unknown key%s %s\nValid keys are %s." 228 % (source, 's' if len(unknown) > 1 else '', 229 pprint(unknown), pprint(allowed))) 230 231 232def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 233 """ 234 Ensure flag members (if present) have valid values. 235 236 :param expr: The expression to validate. 237 :param info: QAPI schema source file information. 238 239 :raise QAPISemError: 240 When certain flags have an invalid value, or when 241 incompatible flags are present. 242 """ 243 for key in ('gen', 'success-response'): 244 if key in expr and expr[key] is not False: 245 raise QAPISemError( 246 info, "flag '%s' may only use false value" % key) 247 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 248 if key in expr and expr[key] is not True: 249 raise QAPISemError( 250 info, "flag '%s' may only use true value" % key) 251 if 'allow-oob' in expr and 'coroutine' in expr: 252 # This is not necessarily a fundamental incompatibility, but 253 # we don't have a use case and the desired semantics isn't 254 # obvious. The simplest solution is to forbid it until we get 255 # a use case for it. 256 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 257 "are incompatible") 258 259 260def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 261 """ 262 Validate the ``if`` member of an object. 263 264 The ``if`` member may be either a ``str`` or a dict. 265 266 :param expr: The expression containing the ``if`` member to validate. 267 :param info: QAPI schema source file information. 268 :param source: Error string describing ``expr``. 269 270 :raise QAPISemError: 271 When the "if" member fails validation, or when there are no 272 non-empty conditions. 273 :return: None 274 """ 275 276 def _check_if(cond: Union[str, object]) -> None: 277 if isinstance(cond, str): 278 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 279 raise QAPISemError( 280 info, 281 "'if' condition '%s' of %s is not a valid identifier" 282 % (cond, source)) 283 return 284 285 if not isinstance(cond, dict): 286 raise QAPISemError( 287 info, 288 "'if' condition of %s must be a string or an object" % source) 289 check_keys(cond, info, "'if' condition of %s" % source, [], 290 ["all", "any", "not"]) 291 if len(cond) != 1: 292 raise QAPISemError( 293 info, 294 "'if' condition of %s has conflicting keys" % source) 295 296 oper, operands = next(iter(cond.items())) 297 if not operands: 298 raise QAPISemError( 299 info, "'if' condition [] of %s is useless" % source) 300 301 if oper == "not": 302 _check_if(operands) 303 return 304 if oper in ("all", "any") and not isinstance(operands, list): 305 raise QAPISemError( 306 info, "'%s' condition of %s must be an array" % (oper, source)) 307 for operand in operands: 308 _check_if(operand) 309 310 ifcond = expr.get('if') 311 if ifcond is None: 312 return 313 314 _check_if(ifcond) 315 316 317def normalize_members(members: object) -> None: 318 """ 319 Normalize a "members" value. 320 321 If ``members`` is a dict, for every value in that dict, if that 322 value is not itself already a dict, normalize it to 323 ``{'type': value}``. 324 325 :forms: 326 :sugared: ``Dict[str, Union[str, TypeRef]]`` 327 :canonical: ``Dict[str, TypeRef]`` 328 329 :param members: The members value to normalize. 330 331 :return: None, ``members`` is normalized in-place as needed. 332 """ 333 if isinstance(members, dict): 334 for key, arg in members.items(): 335 if isinstance(arg, dict): 336 continue 337 members[key] = {'type': arg} 338 339 340def check_type(value: Optional[object], 341 info: QAPISourceInfo, 342 source: str, 343 allow_array: bool = False, 344 allow_dict: Union[bool, str] = False) -> None: 345 """ 346 Normalize and validate the QAPI type of ``value``. 347 348 Python types of ``str`` or ``None`` are always allowed. 349 350 :param value: The value to check. 351 :param info: QAPI schema source file information. 352 :param source: Error string describing this ``value``. 353 :param allow_array: 354 Allow a ``List[str]`` of length 1, which indicates an array of 355 the type named by the list element. 356 :param allow_dict: 357 Allow a dict. Its members can be struct type members or union 358 branches. When the value of ``allow_dict`` is in pragma 359 ``member-name-exceptions``, the dict's keys may violate the 360 member naming rules. The dict members are normalized in place. 361 362 :raise QAPISemError: When ``value`` fails validation. 363 :return: None, ``value`` is normalized in-place as needed. 364 """ 365 if value is None: 366 return 367 368 # Type name 369 if isinstance(value, str): 370 return 371 372 # Array type 373 if isinstance(value, list): 374 if not allow_array: 375 raise QAPISemError(info, "%s cannot be an array" % source) 376 if len(value) != 1 or not isinstance(value[0], str): 377 raise QAPISemError(info, 378 "%s: array type must contain single type name" % 379 source) 380 return 381 382 # Anonymous type 383 384 if not allow_dict: 385 raise QAPISemError(info, "%s should be a type name" % source) 386 387 if not isinstance(value, dict): 388 raise QAPISemError(info, 389 "%s should be an object or type name" % source) 390 391 permissive = False 392 if isinstance(allow_dict, str): 393 permissive = allow_dict in info.pragma.member_name_exceptions 394 395 # value is a dictionary, check that each member is okay 396 for (key, arg) in value.items(): 397 key_source = "%s member '%s'" % (source, key) 398 if key.startswith('*'): 399 key = key[1:] 400 check_name_lower(key, info, key_source, 401 permit_upper=permissive, 402 permit_underscore=permissive) 403 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 404 raise QAPISemError(info, "%s uses reserved name" % key_source) 405 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 406 check_if(arg, info, key_source) 407 check_features(arg.get('features'), info) 408 check_type(arg['type'], info, key_source, allow_array=True) 409 410 411def check_features(features: Optional[object], 412 info: QAPISourceInfo) -> None: 413 """ 414 Normalize and validate the ``features`` member. 415 416 ``features`` may be a ``list`` of either ``str`` or ``dict``. 417 Any ``str`` element will be normalized to ``{'name': element}``. 418 419 :forms: 420 :sugared: ``List[Union[str, Feature]]`` 421 :canonical: ``List[Feature]`` 422 423 :param features: The features member value to validate. 424 :param info: QAPI schema source file information. 425 426 :raise QAPISemError: When ``features`` fails validation. 427 :return: None, ``features`` is normalized in-place as needed. 428 """ 429 if features is None: 430 return 431 if not isinstance(features, list): 432 raise QAPISemError(info, "'features' must be an array") 433 features[:] = [f if isinstance(f, dict) else {'name': f} 434 for f in features] 435 for feat in features: 436 source = "'features' member" 437 assert isinstance(feat, dict) 438 check_keys(feat, info, source, ['name'], ['if']) 439 check_name_is_str(feat['name'], info, source) 440 source = "%s '%s'" % (source, feat['name']) 441 check_name_str(feat['name'], info, source) 442 check_if(feat, info, source) 443 444 445def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 446 """ 447 Normalize and validate this expression as an ``enum`` definition. 448 449 :param expr: The expression to validate. 450 :param info: QAPI schema source file information. 451 452 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 453 :return: None, ``expr`` is normalized in-place as needed. 454 """ 455 name = expr['enum'] 456 members = expr['data'] 457 prefix = expr.get('prefix') 458 459 if not isinstance(members, list): 460 raise QAPISemError(info, "'data' must be an array") 461 if prefix is not None and not isinstance(prefix, str): 462 raise QAPISemError(info, "'prefix' must be a string") 463 464 permissive = name in info.pragma.member_name_exceptions 465 466 members[:] = [m if isinstance(m, dict) else {'name': m} 467 for m in members] 468 for member in members: 469 source = "'data' member" 470 check_keys(member, info, source, ['name'], ['if']) 471 member_name = member['name'] 472 check_name_is_str(member_name, info, source) 473 source = "%s '%s'" % (source, member_name) 474 # Enum members may start with a digit 475 if member_name[0].isdigit(): 476 member_name = 'd' + member_name # Hack: hide the digit 477 check_name_lower(member_name, info, source, 478 permit_upper=permissive, 479 permit_underscore=permissive) 480 check_if(member, info, source) 481 482 483def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 484 """ 485 Normalize and validate this expression as a ``struct`` definition. 486 487 :param expr: The expression to validate. 488 :param info: QAPI schema source file information. 489 490 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 491 :return: None, ``expr`` is normalized in-place as needed. 492 """ 493 name = cast(str, expr['struct']) # Checked in check_exprs 494 members = expr['data'] 495 496 check_type(members, info, "'data'", allow_dict=name) 497 check_type(expr.get('base'), info, "'base'") 498 499 500def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 501 """ 502 Normalize and validate this expression as a ``union`` definition. 503 504 :param expr: The expression to validate. 505 :param info: QAPI schema source file information. 506 507 :raise QAPISemError: when ``expr`` is not a valid ``union``. 508 :return: None, ``expr`` is normalized in-place as needed. 509 """ 510 name = cast(str, expr['union']) # Checked in check_exprs 511 base = expr.get('base') 512 discriminator = expr.get('discriminator') 513 members = expr['data'] 514 515 if discriminator is None: # simple union 516 if base is not None: 517 raise QAPISemError(info, "'base' requires 'discriminator'") 518 else: # flat union 519 check_type(base, info, "'base'", allow_dict=name) 520 if not base: 521 raise QAPISemError(info, "'discriminator' requires 'base'") 522 check_name_is_str(discriminator, info, "'discriminator'") 523 524 if not isinstance(members, dict): 525 raise QAPISemError(info, "'data' must be an object") 526 527 for (key, value) in members.items(): 528 source = "'data' member '%s'" % key 529 if discriminator is None: 530 check_name_lower(key, info, source) 531 # else: name is in discriminator enum, which gets checked 532 check_keys(value, info, source, ['type'], ['if']) 533 check_if(value, info, source) 534 check_type(value['type'], info, source, allow_array=not base) 535 536 537def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 538 """ 539 Normalize and validate this expression as an ``alternate`` definition. 540 541 :param expr: The expression to validate. 542 :param info: QAPI schema source file information. 543 544 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 545 :return: None, ``expr`` is normalized in-place as needed. 546 """ 547 members = expr['data'] 548 549 if not members: 550 raise QAPISemError(info, "'data' must not be empty") 551 552 if not isinstance(members, dict): 553 raise QAPISemError(info, "'data' must be an object") 554 555 for (key, value) in members.items(): 556 source = "'data' member '%s'" % key 557 check_name_lower(key, info, source) 558 check_keys(value, info, source, ['type'], ['if']) 559 check_if(value, info, source) 560 check_type(value['type'], info, source) 561 562 563def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 564 """ 565 Normalize and validate this expression as a ``command`` definition. 566 567 :param expr: The expression to validate. 568 :param info: QAPI schema source file information. 569 570 :raise QAPISemError: When ``expr`` is not a valid ``command``. 571 :return: None, ``expr`` is normalized in-place as needed. 572 """ 573 args = expr.get('data') 574 rets = expr.get('returns') 575 boxed = expr.get('boxed', False) 576 577 if boxed and args is None: 578 raise QAPISemError(info, "'boxed': true requires 'data'") 579 check_type(args, info, "'data'", allow_dict=not boxed) 580 check_type(rets, info, "'returns'", allow_array=True) 581 582 583def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 584 """ 585 Normalize and validate this expression as an ``event`` definition. 586 587 :param expr: The expression to validate. 588 :param info: QAPI schema source file information. 589 590 :raise QAPISemError: When ``expr`` is not a valid ``event``. 591 :return: None, ``expr`` is normalized in-place as needed. 592 """ 593 args = expr.get('data') 594 boxed = expr.get('boxed', False) 595 596 if boxed and args is None: 597 raise QAPISemError(info, "'boxed': true requires 'data'") 598 check_type(args, info, "'data'", allow_dict=not boxed) 599 600 601def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 602 """ 603 Validate and normalize a list of parsed QAPI schema expressions. 604 605 This function accepts a list of expressions and metadata as returned 606 by the parser. It destructively normalizes the expressions in-place. 607 608 :param exprs: The list of expressions to normalize and validate. 609 610 :raise QAPISemError: When any expression fails validation. 611 :return: The same list of expressions (now modified). 612 """ 613 for expr_elem in exprs: 614 # Expression 615 assert isinstance(expr_elem['expr'], dict) 616 for key in expr_elem['expr'].keys(): 617 assert isinstance(key, str) 618 expr: _JSONObject = expr_elem['expr'] 619 620 # QAPISourceInfo 621 assert isinstance(expr_elem['info'], QAPISourceInfo) 622 info: QAPISourceInfo = expr_elem['info'] 623 624 # Optional[QAPIDoc] 625 tmp = expr_elem.get('doc') 626 assert tmp is None or isinstance(tmp, QAPIDoc) 627 doc: Optional[QAPIDoc] = tmp 628 629 if 'include' in expr: 630 continue 631 632 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 633 'command', 'event'} 634 if len(metas) != 1: 635 raise QAPISemError( 636 info, 637 "expression must have exactly one key" 638 " 'enum', 'struct', 'union', 'alternate'," 639 " 'command', 'event'") 640 meta = metas.pop() 641 642 check_name_is_str(expr[meta], info, "'%s'" % meta) 643 name = cast(str, expr[meta]) 644 info.set_defn(meta, name) 645 check_defn_name_str(name, info, meta) 646 647 if doc: 648 if doc.symbol != name: 649 raise QAPISemError( 650 info, "documentation comment is for '%s'" % doc.symbol) 651 doc.check_expr(expr) 652 elif info.pragma.doc_required: 653 raise QAPISemError(info, 654 "documentation comment required") 655 656 if meta == 'enum': 657 check_keys(expr, info, meta, 658 ['enum', 'data'], ['if', 'features', 'prefix']) 659 check_enum(expr, info) 660 elif meta == 'union': 661 check_keys(expr, info, meta, 662 ['union', 'data'], 663 ['base', 'discriminator', 'if', 'features']) 664 normalize_members(expr.get('base')) 665 normalize_members(expr['data']) 666 check_union(expr, info) 667 elif meta == 'alternate': 668 check_keys(expr, info, meta, 669 ['alternate', 'data'], ['if', 'features']) 670 normalize_members(expr['data']) 671 check_alternate(expr, info) 672 elif meta == 'struct': 673 check_keys(expr, info, meta, 674 ['struct', 'data'], ['base', 'if', 'features']) 675 normalize_members(expr['data']) 676 check_struct(expr, info) 677 elif meta == 'command': 678 check_keys(expr, info, meta, 679 ['command'], 680 ['data', 'returns', 'boxed', 'if', 'features', 681 'gen', 'success-response', 'allow-oob', 682 'allow-preconfig', 'coroutine']) 683 normalize_members(expr.get('data')) 684 check_command(expr, info) 685 elif meta == 'event': 686 check_keys(expr, info, meta, 687 ['event'], ['data', 'boxed', 'if', 'features']) 688 normalize_members(expr.get('data')) 689 check_event(expr, info) 690 else: 691 assert False, 'unexpected meta type' 692 693 check_if(expr, info, meta) 694 check_features(expr.get('features'), info) 695 check_flags(expr, info) 696 697 return exprs 698