1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Collection, 37 Dict, 38 Iterable, 39 List, 40 Optional, 41 Union, 42 cast, 43) 44 45from .common import c_name 46from .error import QAPISemError 47from .parser import QAPIDoc 48from .source import QAPISourceInfo 49 50 51# Deserialized JSON objects as returned by the parser. 52# The values of this mapping are not necessary to exhaustively type 53# here (and also not practical as long as mypy lacks recursive 54# types), because the purpose of this module is to interrogate that 55# type. 56_JSONObject = Dict[str, object] 57 58 59# See check_name_str(), below. 60valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 61 r'(x-)?' 62 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 63 64 65def check_name_is_str(name: object, 66 info: QAPISourceInfo, 67 source: str) -> None: 68 """ 69 Ensure that ``name`` is a ``str``. 70 71 :raise QAPISemError: When ``name`` fails validation. 72 """ 73 if not isinstance(name, str): 74 raise QAPISemError(info, "%s requires a string name" % source) 75 76 77def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 78 """ 79 Ensure that ``name`` is a valid QAPI name. 80 81 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 82 starting with a letter. It may be prefixed by a downstream prefix 83 of the form __RFQDN_, or the experimental prefix ``x-``. If both 84 prefixes are present, the __RFDQN_ prefix goes first. 85 86 A valid name cannot start with ``q_``, which is reserved. 87 88 :param name: Name to check. 89 :param info: QAPI schema source file information. 90 :param source: Error string describing what ``name`` belongs to. 91 92 :raise QAPISemError: When ``name`` fails validation. 93 :return: The stem of the valid name, with no prefixes. 94 """ 95 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 96 # and 'q_obj_*' implicit type names. 97 match = valid_name.match(name) 98 if not match or c_name(name, False).startswith('q_'): 99 raise QAPISemError(info, "%s has an invalid name" % source) 100 return match.group(3) 101 102 103def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 104 """ 105 Ensure that ``name`` is a valid event name. 106 107 This means it must be a valid QAPI name as checked by 108 `check_name_str()`, but where the stem prohibits lowercase 109 characters and ``-``. 110 111 :param name: Name to check. 112 :param info: QAPI schema source file information. 113 :param source: Error string describing what ``name`` belongs to. 114 115 :raise QAPISemError: When ``name`` fails validation. 116 """ 117 stem = check_name_str(name, info, source) 118 if re.search(r'[a-z-]', stem): 119 raise QAPISemError( 120 info, "name of %s must not use lowercase or '-'" % source) 121 122 123def check_name_lower(name: str, info: QAPISourceInfo, source: str, 124 permit_upper: bool = False, 125 permit_underscore: bool = False) -> None: 126 """ 127 Ensure that ``name`` is a valid command or member name. 128 129 This means it must be a valid QAPI name as checked by 130 `check_name_str()`, but where the stem prohibits uppercase 131 characters and ``_``. 132 133 :param name: Name to check. 134 :param info: QAPI schema source file information. 135 :param source: Error string describing what ``name`` belongs to. 136 :param permit_upper: Additionally permit uppercase. 137 :param permit_underscore: Additionally permit ``_``. 138 139 :raise QAPISemError: When ``name`` fails validation. 140 """ 141 stem = check_name_str(name, info, source) 142 if ((not permit_upper and re.search(r'[A-Z]', stem)) 143 or (not permit_underscore and '_' in stem)): 144 raise QAPISemError( 145 info, "name of %s must not use uppercase or '_'" % source) 146 147 148def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 149 """ 150 Ensure that ``name`` is a valid user-defined type name. 151 152 This means it must be a valid QAPI name as checked by 153 `check_name_str()`, but where the stem must be in CamelCase. 154 155 :param name: Name to check. 156 :param info: QAPI schema source file information. 157 :param source: Error string describing what ``name`` belongs to. 158 159 :raise QAPISemError: When ``name`` fails validation. 160 """ 161 stem = check_name_str(name, info, source) 162 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 163 raise QAPISemError(info, "name of %s must use CamelCase" % source) 164 165 166def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 167 """ 168 Ensure that ``name`` is a valid definition name. 169 170 Based on the value of ``meta``, this means that: 171 - 'event' names adhere to `check_name_upper()`. 172 - 'command' names adhere to `check_name_lower()`. 173 - Else, meta is a type, and must pass `check_name_camel()`. 174 These names must not end with ``Kind`` nor ``List``. 175 176 :param name: Name to check. 177 :param info: QAPI schema source file information. 178 :param meta: Meta-type name of the QAPI expression. 179 180 :raise QAPISemError: When ``name`` fails validation. 181 """ 182 if meta == 'event': 183 check_name_upper(name, info, meta) 184 elif meta == 'command': 185 check_name_lower( 186 name, info, meta, 187 permit_underscore=name in info.pragma.command_name_exceptions) 188 else: 189 check_name_camel(name, info, meta) 190 if name.endswith('Kind') or name.endswith('List'): 191 raise QAPISemError( 192 info, "%s name should not end in '%s'" % (meta, name[-4:])) 193 194 195def check_keys(value: _JSONObject, 196 info: QAPISourceInfo, 197 source: str, 198 required: Collection[str], 199 optional: Collection[str]) -> None: 200 """ 201 Ensure that a dict has a specific set of keys. 202 203 :param value: The dict to check. 204 :param info: QAPI schema source file information. 205 :param source: Error string describing this ``value``. 206 :param required: Keys that *must* be present. 207 :param optional: Keys that *may* be present. 208 209 :raise QAPISemError: When unknown keys are present. 210 """ 211 212 def pprint(elems: Iterable[str]) -> str: 213 return ', '.join("'" + e + "'" for e in sorted(elems)) 214 215 missing = set(required) - set(value) 216 if missing: 217 raise QAPISemError( 218 info, 219 "%s misses key%s %s" 220 % (source, 's' if len(missing) > 1 else '', 221 pprint(missing))) 222 allowed = set(required) | set(optional) 223 unknown = set(value) - allowed 224 if unknown: 225 raise QAPISemError( 226 info, 227 "%s has unknown key%s %s\nValid keys are %s." 228 % (source, 's' if len(unknown) > 1 else '', 229 pprint(unknown), pprint(allowed))) 230 231 232def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 233 """ 234 Ensure flag members (if present) have valid values. 235 236 :param expr: The expression to validate. 237 :param info: QAPI schema source file information. 238 239 :raise QAPISemError: 240 When certain flags have an invalid value, or when 241 incompatible flags are present. 242 """ 243 for key in ('gen', 'success-response'): 244 if key in expr and expr[key] is not False: 245 raise QAPISemError( 246 info, "flag '%s' may only use false value" % key) 247 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 248 if key in expr and expr[key] is not True: 249 raise QAPISemError( 250 info, "flag '%s' may only use true value" % key) 251 if 'allow-oob' in expr and 'coroutine' in expr: 252 # This is not necessarily a fundamental incompatibility, but 253 # we don't have a use case and the desired semantics isn't 254 # obvious. The simplest solution is to forbid it until we get 255 # a use case for it. 256 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 257 "are incompatible") 258 259 260def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 261 """ 262 Normalize and validate the ``if`` member of an object. 263 264 The ``if`` member may be either a ``str`` or a ``List[str]``. 265 A ``str`` value will be normalized to ``List[str]``. 266 267 :forms: 268 :sugared: ``Union[str, List[str]]`` 269 :canonical: ``List[str]`` 270 271 :param expr: The expression containing the ``if`` member to validate. 272 :param info: QAPI schema source file information. 273 :param source: Error string describing ``expr``. 274 275 :raise QAPISemError: 276 When the "if" member fails validation, or when there are no 277 non-empty conditions. 278 :return: None, ``expr`` is normalized in-place as needed. 279 """ 280 ifcond = expr.get('if') 281 if ifcond is None: 282 return 283 284 if isinstance(ifcond, list): 285 if not ifcond: 286 raise QAPISemError( 287 info, "'if' condition [] of %s is useless" % source) 288 else: 289 # Normalize to a list 290 ifcond = expr['if'] = [ifcond] 291 292 for elt in ifcond: 293 if not isinstance(elt, str): 294 raise QAPISemError( 295 info, 296 "'if' condition of %s must be a string or a list of strings" 297 % source) 298 if not elt.strip(): 299 raise QAPISemError( 300 info, 301 "'if' condition '%s' of %s makes no sense" 302 % (elt, source)) 303 304 305def normalize_members(members: object) -> None: 306 """ 307 Normalize a "members" value. 308 309 If ``members`` is a dict, for every value in that dict, if that 310 value is not itself already a dict, normalize it to 311 ``{'type': value}``. 312 313 :forms: 314 :sugared: ``Dict[str, Union[str, TypeRef]]`` 315 :canonical: ``Dict[str, TypeRef]`` 316 317 :param members: The members value to normalize. 318 319 :return: None, ``members`` is normalized in-place as needed. 320 """ 321 if isinstance(members, dict): 322 for key, arg in members.items(): 323 if isinstance(arg, dict): 324 continue 325 members[key] = {'type': arg} 326 327 328def check_type(value: Optional[object], 329 info: QAPISourceInfo, 330 source: str, 331 allow_array: bool = False, 332 allow_dict: Union[bool, str] = False) -> None: 333 """ 334 Normalize and validate the QAPI type of ``value``. 335 336 Python types of ``str`` or ``None`` are always allowed. 337 338 :param value: The value to check. 339 :param info: QAPI schema source file information. 340 :param source: Error string describing this ``value``. 341 :param allow_array: 342 Allow a ``List[str]`` of length 1, which indicates an array of 343 the type named by the list element. 344 :param allow_dict: 345 Allow a dict. Its members can be struct type members or union 346 branches. When the value of ``allow_dict`` is in pragma 347 ``member-name-exceptions``, the dict's keys may violate the 348 member naming rules. The dict members are normalized in place. 349 350 :raise QAPISemError: When ``value`` fails validation. 351 :return: None, ``value`` is normalized in-place as needed. 352 """ 353 if value is None: 354 return 355 356 # Type name 357 if isinstance(value, str): 358 return 359 360 # Array type 361 if isinstance(value, list): 362 if not allow_array: 363 raise QAPISemError(info, "%s cannot be an array" % source) 364 if len(value) != 1 or not isinstance(value[0], str): 365 raise QAPISemError(info, 366 "%s: array type must contain single type name" % 367 source) 368 return 369 370 # Anonymous type 371 372 if not allow_dict: 373 raise QAPISemError(info, "%s should be a type name" % source) 374 375 if not isinstance(value, dict): 376 raise QAPISemError(info, 377 "%s should be an object or type name" % source) 378 379 permissive = False 380 if isinstance(allow_dict, str): 381 permissive = allow_dict in info.pragma.member_name_exceptions 382 383 # value is a dictionary, check that each member is okay 384 for (key, arg) in value.items(): 385 key_source = "%s member '%s'" % (source, key) 386 if key.startswith('*'): 387 key = key[1:] 388 check_name_lower(key, info, key_source, 389 permit_upper=permissive, 390 permit_underscore=permissive) 391 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 392 raise QAPISemError(info, "%s uses reserved name" % key_source) 393 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 394 check_if(arg, info, key_source) 395 check_features(arg.get('features'), info) 396 check_type(arg['type'], info, key_source, allow_array=True) 397 398 399def check_features(features: Optional[object], 400 info: QAPISourceInfo) -> None: 401 """ 402 Normalize and validate the ``features`` member. 403 404 ``features`` may be a ``list`` of either ``str`` or ``dict``. 405 Any ``str`` element will be normalized to ``{'name': element}``. 406 407 :forms: 408 :sugared: ``List[Union[str, Feature]]`` 409 :canonical: ``List[Feature]`` 410 411 :param features: The features member value to validate. 412 :param info: QAPI schema source file information. 413 414 :raise QAPISemError: When ``features`` fails validation. 415 :return: None, ``features`` is normalized in-place as needed. 416 """ 417 if features is None: 418 return 419 if not isinstance(features, list): 420 raise QAPISemError(info, "'features' must be an array") 421 features[:] = [f if isinstance(f, dict) else {'name': f} 422 for f in features] 423 for feat in features: 424 source = "'features' member" 425 assert isinstance(feat, dict) 426 check_keys(feat, info, source, ['name'], ['if']) 427 check_name_is_str(feat['name'], info, source) 428 source = "%s '%s'" % (source, feat['name']) 429 check_name_str(feat['name'], info, source) 430 check_if(feat, info, source) 431 432 433def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 434 """ 435 Normalize and validate this expression as an ``enum`` definition. 436 437 :param expr: The expression to validate. 438 :param info: QAPI schema source file information. 439 440 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 441 :return: None, ``expr`` is normalized in-place as needed. 442 """ 443 name = expr['enum'] 444 members = expr['data'] 445 prefix = expr.get('prefix') 446 447 if not isinstance(members, list): 448 raise QAPISemError(info, "'data' must be an array") 449 if prefix is not None and not isinstance(prefix, str): 450 raise QAPISemError(info, "'prefix' must be a string") 451 452 permissive = name in info.pragma.member_name_exceptions 453 454 members[:] = [m if isinstance(m, dict) else {'name': m} 455 for m in members] 456 for member in members: 457 source = "'data' member" 458 member_name = member['name'] 459 check_keys(member, info, source, ['name'], ['if']) 460 check_name_is_str(member_name, info, source) 461 source = "%s '%s'" % (source, member_name) 462 # Enum members may start with a digit 463 if member_name[0].isdigit(): 464 member_name = 'd' + member_name # Hack: hide the digit 465 check_name_lower(member_name, info, source, 466 permit_upper=permissive, 467 permit_underscore=permissive) 468 check_if(member, info, source) 469 470 471def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 472 """ 473 Normalize and validate this expression as a ``struct`` definition. 474 475 :param expr: The expression to validate. 476 :param info: QAPI schema source file information. 477 478 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 479 :return: None, ``expr`` is normalized in-place as needed. 480 """ 481 name = cast(str, expr['struct']) # Checked in check_exprs 482 members = expr['data'] 483 484 check_type(members, info, "'data'", allow_dict=name) 485 check_type(expr.get('base'), info, "'base'") 486 487 488def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 489 """ 490 Normalize and validate this expression as a ``union`` definition. 491 492 :param expr: The expression to validate. 493 :param info: QAPI schema source file information. 494 495 :raise QAPISemError: when ``expr`` is not a valid ``union``. 496 :return: None, ``expr`` is normalized in-place as needed. 497 """ 498 name = cast(str, expr['union']) # Checked in check_exprs 499 base = expr.get('base') 500 discriminator = expr.get('discriminator') 501 members = expr['data'] 502 503 if discriminator is None: # simple union 504 if base is not None: 505 raise QAPISemError(info, "'base' requires 'discriminator'") 506 else: # flat union 507 check_type(base, info, "'base'", allow_dict=name) 508 if not base: 509 raise QAPISemError(info, "'discriminator' requires 'base'") 510 check_name_is_str(discriminator, info, "'discriminator'") 511 512 if not isinstance(members, dict): 513 raise QAPISemError(info, "'data' must be an object") 514 515 for (key, value) in members.items(): 516 source = "'data' member '%s'" % key 517 if discriminator is None: 518 check_name_lower(key, info, source) 519 # else: name is in discriminator enum, which gets checked 520 check_keys(value, info, source, ['type'], ['if']) 521 check_if(value, info, source) 522 check_type(value['type'], info, source, allow_array=not base) 523 524 525def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 526 """ 527 Normalize and validate this expression as an ``alternate`` definition. 528 529 :param expr: The expression to validate. 530 :param info: QAPI schema source file information. 531 532 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 533 :return: None, ``expr`` is normalized in-place as needed. 534 """ 535 members = expr['data'] 536 537 if not members: 538 raise QAPISemError(info, "'data' must not be empty") 539 540 if not isinstance(members, dict): 541 raise QAPISemError(info, "'data' must be an object") 542 543 for (key, value) in members.items(): 544 source = "'data' member '%s'" % key 545 check_name_lower(key, info, source) 546 check_keys(value, info, source, ['type'], ['if']) 547 check_if(value, info, source) 548 check_type(value['type'], info, source) 549 550 551def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 552 """ 553 Normalize and validate this expression as a ``command`` definition. 554 555 :param expr: The expression to validate. 556 :param info: QAPI schema source file information. 557 558 :raise QAPISemError: When ``expr`` is not a valid ``command``. 559 :return: None, ``expr`` is normalized in-place as needed. 560 """ 561 args = expr.get('data') 562 rets = expr.get('returns') 563 boxed = expr.get('boxed', False) 564 565 if boxed and args is None: 566 raise QAPISemError(info, "'boxed': true requires 'data'") 567 check_type(args, info, "'data'", allow_dict=not boxed) 568 check_type(rets, info, "'returns'", allow_array=True) 569 570 571def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 572 """ 573 Normalize and validate this expression as an ``event`` definition. 574 575 :param expr: The expression to validate. 576 :param info: QAPI schema source file information. 577 578 :raise QAPISemError: When ``expr`` is not a valid ``event``. 579 :return: None, ``expr`` is normalized in-place as needed. 580 """ 581 args = expr.get('data') 582 boxed = expr.get('boxed', False) 583 584 if boxed and args is None: 585 raise QAPISemError(info, "'boxed': true requires 'data'") 586 check_type(args, info, "'data'", allow_dict=not boxed) 587 588 589def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 590 """ 591 Validate and normalize a list of parsed QAPI schema expressions. 592 593 This function accepts a list of expressions and metadata as returned 594 by the parser. It destructively normalizes the expressions in-place. 595 596 :param exprs: The list of expressions to normalize and validate. 597 598 :raise QAPISemError: When any expression fails validation. 599 :return: The same list of expressions (now modified). 600 """ 601 for expr_elem in exprs: 602 # Expression 603 assert isinstance(expr_elem['expr'], dict) 604 for key in expr_elem['expr'].keys(): 605 assert isinstance(key, str) 606 expr: _JSONObject = expr_elem['expr'] 607 608 # QAPISourceInfo 609 assert isinstance(expr_elem['info'], QAPISourceInfo) 610 info: QAPISourceInfo = expr_elem['info'] 611 612 # Optional[QAPIDoc] 613 tmp = expr_elem.get('doc') 614 assert tmp is None or isinstance(tmp, QAPIDoc) 615 doc: Optional[QAPIDoc] = tmp 616 617 if 'include' in expr: 618 continue 619 620 if 'enum' in expr: 621 meta = 'enum' 622 elif 'union' in expr: 623 meta = 'union' 624 elif 'alternate' in expr: 625 meta = 'alternate' 626 elif 'struct' in expr: 627 meta = 'struct' 628 elif 'command' in expr: 629 meta = 'command' 630 elif 'event' in expr: 631 meta = 'event' 632 else: 633 raise QAPISemError(info, "expression is missing metatype") 634 635 check_name_is_str(expr[meta], info, "'%s'" % meta) 636 name = cast(str, expr[meta]) 637 info.set_defn(meta, name) 638 check_defn_name_str(name, info, meta) 639 640 if doc: 641 if doc.symbol != name: 642 raise QAPISemError( 643 info, "documentation comment is for '%s'" % doc.symbol) 644 doc.check_expr(expr) 645 elif info.pragma.doc_required: 646 raise QAPISemError(info, 647 "documentation comment required") 648 649 if meta == 'enum': 650 check_keys(expr, info, meta, 651 ['enum', 'data'], ['if', 'features', 'prefix']) 652 check_enum(expr, info) 653 elif meta == 'union': 654 check_keys(expr, info, meta, 655 ['union', 'data'], 656 ['base', 'discriminator', 'if', 'features']) 657 normalize_members(expr.get('base')) 658 normalize_members(expr['data']) 659 check_union(expr, info) 660 elif meta == 'alternate': 661 check_keys(expr, info, meta, 662 ['alternate', 'data'], ['if', 'features']) 663 normalize_members(expr['data']) 664 check_alternate(expr, info) 665 elif meta == 'struct': 666 check_keys(expr, info, meta, 667 ['struct', 'data'], ['base', 'if', 'features']) 668 normalize_members(expr['data']) 669 check_struct(expr, info) 670 elif meta == 'command': 671 check_keys(expr, info, meta, 672 ['command'], 673 ['data', 'returns', 'boxed', 'if', 'features', 674 'gen', 'success-response', 'allow-oob', 675 'allow-preconfig', 'coroutine']) 676 normalize_members(expr.get('data')) 677 check_command(expr, info) 678 elif meta == 'event': 679 check_keys(expr, info, meta, 680 ['event'], ['data', 'boxed', 'if', 'features']) 681 normalize_members(expr.get('data')) 682 check_event(expr, info) 683 else: 684 assert False, 'unexpected meta type' 685 686 check_if(expr, info, meta) 687 check_features(expr.get('features'), info) 688 check_flags(expr, info) 689 690 return exprs 691