1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Collection, 37 Dict, 38 Iterable, 39 List, 40 Optional, 41 Union, 42 cast, 43) 44 45from .common import c_name 46from .error import QAPISemError 47from .parser import QAPIDoc 48from .source import QAPISourceInfo 49 50 51# Deserialized JSON objects as returned by the parser. 52# The values of this mapping are not necessary to exhaustively type 53# here (and also not practical as long as mypy lacks recursive 54# types), because the purpose of this module is to interrogate that 55# type. 56_JSONObject = Dict[str, object] 57 58 59# See check_name_str(), below. 60valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 61 r'(x-)?' 62 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 63 64 65def check_name_is_str(name: object, 66 info: QAPISourceInfo, 67 source: str) -> None: 68 """ 69 Ensure that ``name`` is a ``str``. 70 71 :raise QAPISemError: When ``name`` fails validation. 72 """ 73 if not isinstance(name, str): 74 raise QAPISemError(info, "%s requires a string name" % source) 75 76 77def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 78 """ 79 Ensure that ``name`` is a valid QAPI name. 80 81 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 82 starting with a letter. It may be prefixed by a downstream prefix 83 of the form __RFQDN_, or the experimental prefix ``x-``. If both 84 prefixes are present, the __RFDQN_ prefix goes first. 85 86 A valid name cannot start with ``q_``, which is reserved. 87 88 :param name: Name to check. 89 :param info: QAPI schema source file information. 90 :param source: Error string describing what ``name`` belongs to. 91 92 :raise QAPISemError: When ``name`` fails validation. 93 :return: The stem of the valid name, with no prefixes. 94 """ 95 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 96 # and 'q_obj_*' implicit type names. 97 match = valid_name.match(name) 98 if not match or c_name(name, False).startswith('q_'): 99 raise QAPISemError(info, "%s has an invalid name" % source) 100 return match.group(3) 101 102 103def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 104 """ 105 Ensure that ``name`` is a valid event name. 106 107 This means it must be a valid QAPI name as checked by 108 `check_name_str()`, but where the stem prohibits lowercase 109 characters and ``-``. 110 111 :param name: Name to check. 112 :param info: QAPI schema source file information. 113 :param source: Error string describing what ``name`` belongs to. 114 115 :raise QAPISemError: When ``name`` fails validation. 116 """ 117 stem = check_name_str(name, info, source) 118 if re.search(r'[a-z-]', stem): 119 raise QAPISemError( 120 info, "name of %s must not use lowercase or '-'" % source) 121 122 123def check_name_lower(name: str, info: QAPISourceInfo, source: str, 124 permit_upper: bool = False, 125 permit_underscore: bool = False) -> None: 126 """ 127 Ensure that ``name`` is a valid command or member name. 128 129 This means it must be a valid QAPI name as checked by 130 `check_name_str()`, but where the stem prohibits uppercase 131 characters and ``_``. 132 133 :param name: Name to check. 134 :param info: QAPI schema source file information. 135 :param source: Error string describing what ``name`` belongs to. 136 :param permit_upper: Additionally permit uppercase. 137 :param permit_underscore: Additionally permit ``_``. 138 139 :raise QAPISemError: When ``name`` fails validation. 140 """ 141 stem = check_name_str(name, info, source) 142 if ((not permit_upper and re.search(r'[A-Z]', stem)) 143 or (not permit_underscore and '_' in stem)): 144 raise QAPISemError( 145 info, "name of %s must not use uppercase or '_'" % source) 146 147 148def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 149 """ 150 Ensure that ``name`` is a valid user-defined type name. 151 152 This means it must be a valid QAPI name as checked by 153 `check_name_str()`, but where the stem must be in CamelCase. 154 155 :param name: Name to check. 156 :param info: QAPI schema source file information. 157 :param source: Error string describing what ``name`` belongs to. 158 159 :raise QAPISemError: When ``name`` fails validation. 160 """ 161 stem = check_name_str(name, info, source) 162 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 163 raise QAPISemError(info, "name of %s must use CamelCase" % source) 164 165 166def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 167 """ 168 Ensure that ``name`` is a valid definition name. 169 170 Based on the value of ``meta``, this means that: 171 - 'event' names adhere to `check_name_upper()`. 172 - 'command' names adhere to `check_name_lower()`. 173 - Else, meta is a type, and must pass `check_name_camel()`. 174 These names must not end with ``Kind`` nor ``List``. 175 176 :param name: Name to check. 177 :param info: QAPI schema source file information. 178 :param meta: Meta-type name of the QAPI expression. 179 180 :raise QAPISemError: When ``name`` fails validation. 181 """ 182 if meta == 'event': 183 check_name_upper(name, info, meta) 184 elif meta == 'command': 185 check_name_lower( 186 name, info, meta, 187 permit_underscore=name in info.pragma.command_name_exceptions) 188 else: 189 check_name_camel(name, info, meta) 190 if name.endswith('Kind') or name.endswith('List'): 191 raise QAPISemError( 192 info, "%s name should not end in '%s'" % (meta, name[-4:])) 193 194 195def check_keys(value: _JSONObject, 196 info: QAPISourceInfo, 197 source: str, 198 required: Collection[str], 199 optional: Collection[str]) -> None: 200 """ 201 Ensure that a dict has a specific set of keys. 202 203 :param value: The dict to check. 204 :param info: QAPI schema source file information. 205 :param source: Error string describing this ``value``. 206 :param required: Keys that *must* be present. 207 :param optional: Keys that *may* be present. 208 209 :raise QAPISemError: When unknown keys are present. 210 """ 211 212 def pprint(elems: Iterable[str]) -> str: 213 return ', '.join("'" + e + "'" for e in sorted(elems)) 214 215 missing = set(required) - set(value) 216 if missing: 217 raise QAPISemError( 218 info, 219 "%s misses key%s %s" 220 % (source, 's' if len(missing) > 1 else '', 221 pprint(missing))) 222 allowed = set(required) | set(optional) 223 unknown = set(value) - allowed 224 if unknown: 225 raise QAPISemError( 226 info, 227 "%s has unknown key%s %s\nValid keys are %s." 228 % (source, 's' if len(unknown) > 1 else '', 229 pprint(unknown), pprint(allowed))) 230 231 232def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 233 """ 234 Ensure flag members (if present) have valid values. 235 236 :param expr: The expression to validate. 237 :param info: QAPI schema source file information. 238 239 :raise QAPISemError: 240 When certain flags have an invalid value, or when 241 incompatible flags are present. 242 """ 243 for key in ('gen', 'success-response'): 244 if key in expr and expr[key] is not False: 245 raise QAPISemError( 246 info, "flag '%s' may only use false value" % key) 247 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 248 if key in expr and expr[key] is not True: 249 raise QAPISemError( 250 info, "flag '%s' may only use true value" % key) 251 if 'allow-oob' in expr and 'coroutine' in expr: 252 # This is not necessarily a fundamental incompatibility, but 253 # we don't have a use case and the desired semantics isn't 254 # obvious. The simplest solution is to forbid it until we get 255 # a use case for it. 256 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 257 "are incompatible") 258 259 260def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 261 """ 262 Validate the ``if`` member of an object. 263 264 The ``if`` member may be either a ``str`` or a dict. 265 266 :param expr: The expression containing the ``if`` member to validate. 267 :param info: QAPI schema source file information. 268 :param source: Error string describing ``expr``. 269 270 :raise QAPISemError: 271 When the "if" member fails validation, or when there are no 272 non-empty conditions. 273 :return: None 274 """ 275 276 def _check_if(cond: Union[str, object]) -> None: 277 if isinstance(cond, str): 278 if not re.match(r'^[A-Z][A-Z0-9_]*$', cond): 279 raise QAPISemError( 280 info, 281 "'if' condition '%s' of %s is not a valid identifier" 282 % (cond, source)) 283 return 284 285 if not isinstance(cond, dict): 286 raise QAPISemError( 287 info, 288 "'if' condition of %s must be a string or an object" % source) 289 if len(cond) != 1: 290 raise QAPISemError( 291 info, 292 "'if' condition dict of %s must have one key: " 293 "'all', 'any' or 'not'" % source) 294 check_keys(cond, info, "'if' condition", [], 295 ["all", "any", "not"]) 296 297 oper, operands = next(iter(cond.items())) 298 if not operands: 299 raise QAPISemError( 300 info, "'if' condition [] of %s is useless" % source) 301 302 if oper == "not": 303 _check_if(operands) 304 return 305 if oper in ("all", "any") and not isinstance(operands, list): 306 raise QAPISemError( 307 info, "'%s' condition of %s must be an array" % (oper, source)) 308 for operand in operands: 309 _check_if(operand) 310 311 ifcond = expr.get('if') 312 if ifcond is None: 313 return 314 315 _check_if(ifcond) 316 317 318def normalize_members(members: object) -> None: 319 """ 320 Normalize a "members" value. 321 322 If ``members`` is a dict, for every value in that dict, if that 323 value is not itself already a dict, normalize it to 324 ``{'type': value}``. 325 326 :forms: 327 :sugared: ``Dict[str, Union[str, TypeRef]]`` 328 :canonical: ``Dict[str, TypeRef]`` 329 330 :param members: The members value to normalize. 331 332 :return: None, ``members`` is normalized in-place as needed. 333 """ 334 if isinstance(members, dict): 335 for key, arg in members.items(): 336 if isinstance(arg, dict): 337 continue 338 members[key] = {'type': arg} 339 340 341def check_type(value: Optional[object], 342 info: QAPISourceInfo, 343 source: str, 344 allow_array: bool = False, 345 allow_dict: Union[bool, str] = False) -> None: 346 """ 347 Normalize and validate the QAPI type of ``value``. 348 349 Python types of ``str`` or ``None`` are always allowed. 350 351 :param value: The value to check. 352 :param info: QAPI schema source file information. 353 :param source: Error string describing this ``value``. 354 :param allow_array: 355 Allow a ``List[str]`` of length 1, which indicates an array of 356 the type named by the list element. 357 :param allow_dict: 358 Allow a dict. Its members can be struct type members or union 359 branches. When the value of ``allow_dict`` is in pragma 360 ``member-name-exceptions``, the dict's keys may violate the 361 member naming rules. The dict members are normalized in place. 362 363 :raise QAPISemError: When ``value`` fails validation. 364 :return: None, ``value`` is normalized in-place as needed. 365 """ 366 if value is None: 367 return 368 369 # Type name 370 if isinstance(value, str): 371 return 372 373 # Array type 374 if isinstance(value, list): 375 if not allow_array: 376 raise QAPISemError(info, "%s cannot be an array" % source) 377 if len(value) != 1 or not isinstance(value[0], str): 378 raise QAPISemError(info, 379 "%s: array type must contain single type name" % 380 source) 381 return 382 383 # Anonymous type 384 385 if not allow_dict: 386 raise QAPISemError(info, "%s should be a type name" % source) 387 388 if not isinstance(value, dict): 389 raise QAPISemError(info, 390 "%s should be an object or type name" % source) 391 392 permissive = False 393 if isinstance(allow_dict, str): 394 permissive = allow_dict in info.pragma.member_name_exceptions 395 396 # value is a dictionary, check that each member is okay 397 for (key, arg) in value.items(): 398 key_source = "%s member '%s'" % (source, key) 399 if key.startswith('*'): 400 key = key[1:] 401 check_name_lower(key, info, key_source, 402 permit_upper=permissive, 403 permit_underscore=permissive) 404 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 405 raise QAPISemError(info, "%s uses reserved name" % key_source) 406 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 407 check_if(arg, info, key_source) 408 check_features(arg.get('features'), info) 409 check_type(arg['type'], info, key_source, allow_array=True) 410 411 412def check_features(features: Optional[object], 413 info: QAPISourceInfo) -> None: 414 """ 415 Normalize and validate the ``features`` member. 416 417 ``features`` may be a ``list`` of either ``str`` or ``dict``. 418 Any ``str`` element will be normalized to ``{'name': element}``. 419 420 :forms: 421 :sugared: ``List[Union[str, Feature]]`` 422 :canonical: ``List[Feature]`` 423 424 :param features: The features member value to validate. 425 :param info: QAPI schema source file information. 426 427 :raise QAPISemError: When ``features`` fails validation. 428 :return: None, ``features`` is normalized in-place as needed. 429 """ 430 if features is None: 431 return 432 if not isinstance(features, list): 433 raise QAPISemError(info, "'features' must be an array") 434 features[:] = [f if isinstance(f, dict) else {'name': f} 435 for f in features] 436 for feat in features: 437 source = "'features' member" 438 assert isinstance(feat, dict) 439 check_keys(feat, info, source, ['name'], ['if']) 440 check_name_is_str(feat['name'], info, source) 441 source = "%s '%s'" % (source, feat['name']) 442 check_name_str(feat['name'], info, source) 443 check_if(feat, info, source) 444 445 446def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 447 """ 448 Normalize and validate this expression as an ``enum`` definition. 449 450 :param expr: The expression to validate. 451 :param info: QAPI schema source file information. 452 453 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 454 :return: None, ``expr`` is normalized in-place as needed. 455 """ 456 name = expr['enum'] 457 members = expr['data'] 458 prefix = expr.get('prefix') 459 460 if not isinstance(members, list): 461 raise QAPISemError(info, "'data' must be an array") 462 if prefix is not None and not isinstance(prefix, str): 463 raise QAPISemError(info, "'prefix' must be a string") 464 465 permissive = name in info.pragma.member_name_exceptions 466 467 members[:] = [m if isinstance(m, dict) else {'name': m} 468 for m in members] 469 for member in members: 470 source = "'data' member" 471 check_keys(member, info, source, ['name'], ['if']) 472 member_name = member['name'] 473 check_name_is_str(member_name, info, source) 474 source = "%s '%s'" % (source, member_name) 475 # Enum members may start with a digit 476 if member_name[0].isdigit(): 477 member_name = 'd' + member_name # Hack: hide the digit 478 check_name_lower(member_name, info, source, 479 permit_upper=permissive, 480 permit_underscore=permissive) 481 check_if(member, info, source) 482 483 484def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 485 """ 486 Normalize and validate this expression as a ``struct`` definition. 487 488 :param expr: The expression to validate. 489 :param info: QAPI schema source file information. 490 491 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 492 :return: None, ``expr`` is normalized in-place as needed. 493 """ 494 name = cast(str, expr['struct']) # Checked in check_exprs 495 members = expr['data'] 496 497 check_type(members, info, "'data'", allow_dict=name) 498 check_type(expr.get('base'), info, "'base'") 499 500 501def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 502 """ 503 Normalize and validate this expression as a ``union`` definition. 504 505 :param expr: The expression to validate. 506 :param info: QAPI schema source file information. 507 508 :raise QAPISemError: when ``expr`` is not a valid ``union``. 509 :return: None, ``expr`` is normalized in-place as needed. 510 """ 511 name = cast(str, expr['union']) # Checked in check_exprs 512 base = expr.get('base') 513 discriminator = expr.get('discriminator') 514 members = expr['data'] 515 516 if discriminator is None: # simple union 517 if base is not None: 518 raise QAPISemError(info, "'base' requires 'discriminator'") 519 else: # flat union 520 check_type(base, info, "'base'", allow_dict=name) 521 if not base: 522 raise QAPISemError(info, "'discriminator' requires 'base'") 523 check_name_is_str(discriminator, info, "'discriminator'") 524 525 if not isinstance(members, dict): 526 raise QAPISemError(info, "'data' must be an object") 527 528 for (key, value) in members.items(): 529 source = "'data' member '%s'" % key 530 if discriminator is None: 531 check_name_lower(key, info, source) 532 # else: name is in discriminator enum, which gets checked 533 check_keys(value, info, source, ['type'], ['if']) 534 check_if(value, info, source) 535 check_type(value['type'], info, source, allow_array=not base) 536 537 538def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 539 """ 540 Normalize and validate this expression as an ``alternate`` definition. 541 542 :param expr: The expression to validate. 543 :param info: QAPI schema source file information. 544 545 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 546 :return: None, ``expr`` is normalized in-place as needed. 547 """ 548 members = expr['data'] 549 550 if not members: 551 raise QAPISemError(info, "'data' must not be empty") 552 553 if not isinstance(members, dict): 554 raise QAPISemError(info, "'data' must be an object") 555 556 for (key, value) in members.items(): 557 source = "'data' member '%s'" % key 558 check_name_lower(key, info, source) 559 check_keys(value, info, source, ['type'], ['if']) 560 check_if(value, info, source) 561 check_type(value['type'], info, source) 562 563 564def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 565 """ 566 Normalize and validate this expression as a ``command`` definition. 567 568 :param expr: The expression to validate. 569 :param info: QAPI schema source file information. 570 571 :raise QAPISemError: When ``expr`` is not a valid ``command``. 572 :return: None, ``expr`` is normalized in-place as needed. 573 """ 574 args = expr.get('data') 575 rets = expr.get('returns') 576 boxed = expr.get('boxed', False) 577 578 if boxed and args is None: 579 raise QAPISemError(info, "'boxed': true requires 'data'") 580 check_type(args, info, "'data'", allow_dict=not boxed) 581 check_type(rets, info, "'returns'", allow_array=True) 582 583 584def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 585 """ 586 Normalize and validate this expression as an ``event`` definition. 587 588 :param expr: The expression to validate. 589 :param info: QAPI schema source file information. 590 591 :raise QAPISemError: When ``expr`` is not a valid ``event``. 592 :return: None, ``expr`` is normalized in-place as needed. 593 """ 594 args = expr.get('data') 595 boxed = expr.get('boxed', False) 596 597 if boxed and args is None: 598 raise QAPISemError(info, "'boxed': true requires 'data'") 599 check_type(args, info, "'data'", allow_dict=not boxed) 600 601 602def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 603 """ 604 Validate and normalize a list of parsed QAPI schema expressions. 605 606 This function accepts a list of expressions and metadata as returned 607 by the parser. It destructively normalizes the expressions in-place. 608 609 :param exprs: The list of expressions to normalize and validate. 610 611 :raise QAPISemError: When any expression fails validation. 612 :return: The same list of expressions (now modified). 613 """ 614 for expr_elem in exprs: 615 # Expression 616 assert isinstance(expr_elem['expr'], dict) 617 for key in expr_elem['expr'].keys(): 618 assert isinstance(key, str) 619 expr: _JSONObject = expr_elem['expr'] 620 621 # QAPISourceInfo 622 assert isinstance(expr_elem['info'], QAPISourceInfo) 623 info: QAPISourceInfo = expr_elem['info'] 624 625 # Optional[QAPIDoc] 626 tmp = expr_elem.get('doc') 627 assert tmp is None or isinstance(tmp, QAPIDoc) 628 doc: Optional[QAPIDoc] = tmp 629 630 if 'include' in expr: 631 continue 632 633 if 'enum' in expr: 634 meta = 'enum' 635 elif 'union' in expr: 636 meta = 'union' 637 elif 'alternate' in expr: 638 meta = 'alternate' 639 elif 'struct' in expr: 640 meta = 'struct' 641 elif 'command' in expr: 642 meta = 'command' 643 elif 'event' in expr: 644 meta = 'event' 645 else: 646 raise QAPISemError(info, "expression is missing metatype") 647 648 check_name_is_str(expr[meta], info, "'%s'" % meta) 649 name = cast(str, expr[meta]) 650 info.set_defn(meta, name) 651 check_defn_name_str(name, info, meta) 652 653 if doc: 654 if doc.symbol != name: 655 raise QAPISemError( 656 info, "documentation comment is for '%s'" % doc.symbol) 657 doc.check_expr(expr) 658 elif info.pragma.doc_required: 659 raise QAPISemError(info, 660 "documentation comment required") 661 662 if meta == 'enum': 663 check_keys(expr, info, meta, 664 ['enum', 'data'], ['if', 'features', 'prefix']) 665 check_enum(expr, info) 666 elif meta == 'union': 667 check_keys(expr, info, meta, 668 ['union', 'data'], 669 ['base', 'discriminator', 'if', 'features']) 670 normalize_members(expr.get('base')) 671 normalize_members(expr['data']) 672 check_union(expr, info) 673 elif meta == 'alternate': 674 check_keys(expr, info, meta, 675 ['alternate', 'data'], ['if', 'features']) 676 normalize_members(expr['data']) 677 check_alternate(expr, info) 678 elif meta == 'struct': 679 check_keys(expr, info, meta, 680 ['struct', 'data'], ['base', 'if', 'features']) 681 normalize_members(expr['data']) 682 check_struct(expr, info) 683 elif meta == 'command': 684 check_keys(expr, info, meta, 685 ['command'], 686 ['data', 'returns', 'boxed', 'if', 'features', 687 'gen', 'success-response', 'allow-oob', 688 'allow-preconfig', 'coroutine']) 689 normalize_members(expr.get('data')) 690 check_command(expr, info) 691 elif meta == 'event': 692 check_keys(expr, info, meta, 693 ['event'], ['data', 'boxed', 'if', 'features']) 694 normalize_members(expr.get('data')) 695 check_event(expr, info) 696 else: 697 assert False, 'unexpected meta type' 698 699 check_if(expr, info, meta) 700 check_features(expr.get('features'), info) 701 check_flags(expr, info) 702 703 return exprs 704