1# vim:fileencoding=utf-8:noet 2from __future__ import (unicode_literals, division, absolute_import, print_function) 3 4import itertools 5import re 6 7from copy import copy 8 9from powerline.lib.unicode import unicode 10from powerline.lint.markedjson.error import echoerr, DelayedEchoErr, NON_PRINTABLE_STR 11from powerline.lint.selfcheck import havemarks 12 13 14NON_PRINTABLE_RE = re.compile( 15 NON_PRINTABLE_STR.translate({ 16 ord('\t'): None, 17 ord('\n'): None, 18 0x0085: None, 19 }) 20) 21 22 23class Spec(object): 24 '''Class that describes some JSON value 25 26 In powerline it is only used to describe JSON values stored in powerline 27 configuration. 28 29 :param dict keys: 30 Dictionary that maps keys that may be present in the given JSON 31 dictionary to their descriptions. If this parameter is not empty it 32 implies that described value has dictionary type. Non-dictionary types 33 must be described using ``Spec()``: without arguments. 34 35 .. note:: 36 Methods that create the specifications return ``self``, so calls to them 37 may be chained: ``Spec().type(unicode).re('^\w+$')``. This does not 38 apply to functions that *apply* specification like :py:meth`Spec.match`. 39 40 .. note:: 41 Methods starting with ``check_`` return two values: first determines 42 whether caller should proceed on running other checks, second 43 determines whether there were any problems (i.e. whether error was 44 reported). One should not call these methods directly: there is 45 :py:meth:`Spec.match` method for checking values. 46 47 .. note:: 48 In ``check_`` and ``match`` methods specifications are identified by 49 their indexes for the purpose of simplyfying :py:meth:`Spec.copy` 50 method. 51 52 Some common parameters: 53 54 ``data``: 55 Whatever data supplied by the first caller for checker functions. Is not 56 processed by :py:class:`Spec` methods in any fashion. 57 ``context``: 58 :py:class:`powerline.lint.context.Context` instance, describes context 59 of the value. :py:class:`Spec` methods only use its ``.key`` methods for 60 error messages. 61 ``echoerr``: 62 Callable that should be used to echo errors. Is supposed to take four 63 optional keyword arguments: ``problem``, ``problem_mark``, ``context``, 64 ``context_mark``. 65 ``value``: 66 Checked value. 67 ''' 68 69 def __init__(self, **keys): 70 self.specs = [] 71 self.keys = {} 72 self.checks = [] 73 self.cmsg = '' 74 self.isoptional = False 75 self.uspecs = [] 76 self.ufailmsg = lambda key: 'found unknown key: {0}'.format(key) 77 self.did_type = False 78 self.update(**keys) 79 80 def update(self, **keys): 81 '''Describe additional keys that may be present in given JSON value 82 83 If called with some keyword arguments implies that described value is 84 a dictionary. If called without keyword parameters it is no-op. 85 86 :return: self. 87 ''' 88 for k, v in keys.items(): 89 self.keys[k] = len(self.specs) 90 self.specs.append(v) 91 if self.keys and not self.did_type: 92 self.type(dict) 93 self.did_type = True 94 return self 95 96 def copy(self, copied=None): 97 '''Deep copy the spec 98 99 :param dict copied: 100 Internal dictionary used for storing already copied values. This 101 parameter should not be used. 102 103 :return: New :py:class:`Spec` object that is a deep copy of ``self``. 104 ''' 105 copied = copied or {} 106 try: 107 return copied[id(self)] 108 except KeyError: 109 instance = self.__class__() 110 copied[id(self)] = instance 111 return self.__class__()._update(self.__dict__, copied) 112 113 def _update(self, d, copied): 114 '''Helper for the :py:meth:`Spec.copy` function 115 116 Populates new instance with values taken from the old one. 117 118 :param dict d: 119 ``__dict__`` of the old instance. 120 :param dict copied: 121 Storage for already copied values. 122 ''' 123 self.__dict__.update(d) 124 self.keys = copy(self.keys) 125 self.checks = copy(self.checks) 126 self.uspecs = copy(self.uspecs) 127 self.specs = [spec.copy(copied) for spec in self.specs] 128 return self 129 130 def unknown_spec(self, keyfunc, spec): 131 '''Define specification for non-static keys 132 133 This method should be used if key names cannot be determined at runtime 134 or if a number of keys share identical spec (in order to not repeat it). 135 :py:meth:`Spec.match` method processes dictionary in the given order: 136 137 * First it tries to use specifications provided at the initialization or 138 by the :py:meth:`Spec.update` method. 139 * If no specification for given key was provided it processes 140 specifications from ``keyfunc`` argument in order they were supplied. 141 Once some key matches specification supplied second ``spec`` argument 142 is used to determine correctness of the value. 143 144 :param Spec keyfunc: 145 :py:class:`Spec` instance or a regular function that returns two 146 values (the same :py:meth:`Spec.match` returns). This argument is 147 used to match keys that were not provided at initialization or via 148 :py:meth:`Spec.update`. 149 :param Spec spec: 150 :py:class:`Spec` instance that will be used to check keys matched by 151 ``keyfunc``. 152 153 :return: self. 154 ''' 155 if isinstance(keyfunc, Spec): 156 self.specs.append(keyfunc) 157 keyfunc = len(self.specs) - 1 158 self.specs.append(spec) 159 self.uspecs.append((keyfunc, len(self.specs) - 1)) 160 return self 161 162 def unknown_msg(self, msgfunc): 163 '''Define message which will be used when unknown key was found 164 165 “Unknown” is a key that was not provided at the initialization and via 166 :py:meth:`Spec.update` and did not match any ``keyfunc`` provided via 167 :py:meth:`Spec.unknown_spec`. 168 169 :param msgfunc: 170 Function that takes that unknown key as an argument and returns the 171 message text. Text will appear at the top (start of the sentence). 172 173 :return: self. 174 ''' 175 self.ufailmsg = msgfunc 176 return self 177 178 def context_message(self, msg): 179 '''Define message that describes context 180 181 :param str msg: 182 Message that describes context. Is written using the 183 :py:meth:`str.format` syntax and is expected to display keyword 184 parameter ``key``. 185 186 :return: self. 187 ''' 188 self.cmsg = msg 189 for spec in self.specs: 190 if not spec.cmsg: 191 spec.context_message(msg) 192 return self 193 194 def check_type(self, value, context_mark, data, context, echoerr, types): 195 '''Check that given value matches given type(s) 196 197 :param tuple types: 198 List of accepted types. Since :py:class:`Spec` is supposed to 199 describe JSON values only ``dict``, ``list``, ``unicode``, ``bool``, 200 ``float`` and ``NoneType`` types make any sense. 201 202 :return: proceed, hadproblem. 203 ''' 204 havemarks(value) 205 if type(value.value) not in types: 206 echoerr( 207 context=self.cmsg.format(key=context.key), 208 context_mark=context_mark, 209 problem='{0!r} must be a {1} instance, not {2}'.format( 210 value, 211 ', '.join((t.__name__ for t in types)), 212 type(value.value).__name__ 213 ), 214 problem_mark=value.mark 215 ) 216 return False, True 217 return True, False 218 219 def check_func(self, value, context_mark, data, context, echoerr, func, msg_func): 220 '''Check value using given function 221 222 :param function func: 223 Callable that should accept four positional parameters: 224 225 #. checked value, 226 #. ``data`` parameter with arbitrary data (supplied by top-level 227 caller), 228 #. current context and 229 #. function used for echoing errors. 230 231 This callable should return three values: 232 233 #. determines whether ``check_func`` caller should proceed 234 calling other checks, 235 #. determines whether ``check_func`` should echo error on its own 236 (it should be set to False if ``func`` echoes error itself) and 237 #. determines whether function has found some errors in the checked 238 value. 239 240 :param function msg_func: 241 Callable that takes checked value as the only positional parameter 242 and returns a string that describes the problem. Only useful for 243 small checker functions since it is ignored when second returned 244 value is false. 245 246 :return: proceed, hadproblem. 247 ''' 248 havemarks(value) 249 proceed, echo, hadproblem = func(value, data, context, echoerr) 250 if echo and hadproblem: 251 echoerr(context=self.cmsg.format(key=context.key), 252 context_mark=context_mark, 253 problem=msg_func(value), 254 problem_mark=value.mark) 255 return proceed, hadproblem 256 257 def check_list(self, value, context_mark, data, context, echoerr, item_func, msg_func): 258 '''Check that each value in the list matches given specification 259 260 :param function item_func: 261 Callable like ``func`` from :py:meth:`Spec.check_func`. Unlike 262 ``func`` this callable is called for each value in the list and may 263 be a :py:class:`Spec` object index. 264 :param func msg_func: 265 Callable like ``msg_func`` from :py:meth:`Spec.check_func`. Should 266 accept one problematic item and is not used for :py:class:`Spec` 267 object indices in ``item_func`` method. 268 269 :return: proceed, hadproblem. 270 ''' 271 havemarks(value) 272 i = 0 273 hadproblem = False 274 for item in value: 275 havemarks(item) 276 if isinstance(item_func, int): 277 spec = self.specs[item_func] 278 proceed, fhadproblem = spec.match( 279 item, 280 value.mark, 281 data, 282 context.enter_item('list item ' + unicode(i), item), 283 echoerr 284 ) 285 else: 286 proceed, echo, fhadproblem = item_func(item, data, context, echoerr) 287 if echo and fhadproblem: 288 echoerr(context=self.cmsg.format(key=context.key + '/list item ' + unicode(i)), 289 context_mark=value.mark, 290 problem=msg_func(item), 291 problem_mark=item.mark) 292 if fhadproblem: 293 hadproblem = True 294 if not proceed: 295 return proceed, hadproblem 296 i += 1 297 return True, hadproblem 298 299 def check_either(self, value, context_mark, data, context, echoerr, start, end): 300 '''Check that given value matches one of the given specifications 301 302 :param int start: 303 First specification index. 304 :param int end: 305 Specification index that is greater by 1 then last specification 306 index. 307 308 This method does not give an error if any specification from 309 ``self.specs[start:end]`` is matched by the given value. 310 ''' 311 havemarks(value) 312 new_echoerr = DelayedEchoErr( 313 echoerr, 314 'One of the either variants failed. Messages from the first variant:', 315 'messages from the next variant:' 316 ) 317 318 hadproblem = False 319 for spec in self.specs[start:end]: 320 proceed, hadproblem = spec.match(value, value.mark, data, context, new_echoerr) 321 new_echoerr.next_variant() 322 if not proceed: 323 break 324 if not hadproblem: 325 return True, False 326 327 new_echoerr.echo_all() 328 329 return False, hadproblem 330 331 def check_tuple(self, value, context_mark, data, context, echoerr, start, end): 332 '''Check that given value is a list with items matching specifications 333 334 :param int start: 335 First specification index. 336 :param int end: 337 Specification index that is greater by 1 then last specification 338 index. 339 340 This method checks that each item in the value list matches 341 specification with index ``start + item_number``. 342 ''' 343 havemarks(value) 344 hadproblem = False 345 for (i, item, spec) in zip(itertools.count(), value, self.specs[start:end]): 346 proceed, ihadproblem = spec.match( 347 item, 348 value.mark, 349 data, 350 context.enter_item('tuple item ' + unicode(i), item), 351 echoerr 352 ) 353 if ihadproblem: 354 hadproblem = True 355 if not proceed: 356 return False, hadproblem 357 return True, hadproblem 358 359 def check_printable(self, value, context_mark, data, context, echoerr, _): 360 '''Check that given unicode string contains only printable characters 361 ''' 362 hadproblem = False 363 for match in NON_PRINTABLE_RE.finditer(value): 364 hadproblem = True 365 echoerr( 366 context=self.cmsg.format(key=context.key), 367 context_mark=value.mark, 368 problem='found not printable character U+{0:04x} in a configuration string'.format( 369 ord(match.group(0))), 370 problem_mark=value.mark.advance_string(match.start() + 1) 371 ) 372 return True, hadproblem 373 374 def printable(self, *args): 375 self.type(unicode) 376 self.checks.append(('check_printable', args)) 377 return self 378 379 def type(self, *args): 380 '''Describe value that has one of the types given in arguments 381 382 :param args: 383 List of accepted types. Since :py:class:`Spec` is supposed to 384 describe JSON values only ``dict``, ``list``, ``unicode``, ``bool``, 385 ``float`` and ``NoneType`` types make any sense. 386 387 :return: self. 388 ''' 389 self.checks.append(('check_type', args)) 390 return self 391 392 cmp_funcs = { 393 'le': lambda x, y: x <= y, 394 'lt': lambda x, y: x < y, 395 'ge': lambda x, y: x >= y, 396 'gt': lambda x, y: x > y, 397 'eq': lambda x, y: x == y, 398 } 399 400 cmp_msgs = { 401 'le': 'lesser or equal to', 402 'lt': 'lesser then', 403 'ge': 'greater or equal to', 404 'gt': 'greater then', 405 'eq': 'equal to', 406 } 407 408 def len(self, comparison, cint, msg_func=None): 409 '''Describe value that has given length 410 411 :param str comparison: 412 Type of the comparison. Valid values: ``le``, ``lt``, ``ge``, 413 ``gt``, ``eq``. 414 :param int cint: 415 Integer with which length is compared. 416 :param function msg_func: 417 Function that should accept checked value and return message that 418 describes the problem with this value. Default value will emit 419 something like “length of ['foo', 'bar'] is not greater then 10”. 420 421 :return: self. 422 ''' 423 cmp_func = self.cmp_funcs[comparison] 424 msg_func = ( 425 msg_func 426 or (lambda value: 'length of {0!r} is not {1} {2}'.format( 427 value, self.cmp_msgs[comparison], cint)) 428 ) 429 self.checks.append(( 430 'check_func', 431 (lambda value, *args: (True, True, not cmp_func(len(value), cint))), 432 msg_func 433 )) 434 return self 435 436 def cmp(self, comparison, cint, msg_func=None): 437 '''Describe value that is a number or string that has given property 438 439 :param str comparison: 440 Type of the comparison. Valid values: ``le``, ``lt``, ``ge``, 441 ``gt``, ``eq``. This argument will restrict the number or string to 442 emit True on the given comparison. 443 :param cint: 444 Number or string with which value is compared. Type of this 445 parameter affects required type of the checked value: ``str`` and 446 ``unicode`` types imply ``unicode`` values, ``float`` type implies 447 that value can be either ``int`` or ``float``, ``int`` type implies 448 ``int`` value and for any other type the behavior is undefined. 449 :param function msg_func: 450 Function that should accept checked value and return message that 451 describes the problem with this value. Default value will emit 452 something like “10 is not greater then 10”. 453 454 :return: self. 455 ''' 456 if type(cint) is str: 457 self.type(unicode) 458 elif type(cint) is float: 459 self.type(int, float) 460 else: 461 self.type(type(cint)) 462 cmp_func = self.cmp_funcs[comparison] 463 msg_func = msg_func or (lambda value: '{0} is not {1} {2}'.format(value, self.cmp_msgs[comparison], cint)) 464 self.checks.append(( 465 'check_func', 466 (lambda value, *args: (True, True, not cmp_func(value.value, cint))), 467 msg_func 468 )) 469 return self 470 471 def unsigned(self, msg_func=None): 472 '''Describe unsigned integer value 473 474 :param function msg_func: 475 Function that should accept checked value and return message that 476 describes the problem with this value. 477 478 :return: self. 479 ''' 480 self.type(int) 481 self.checks.append(( 482 'check_func', 483 (lambda value, *args: (True, True, value < 0)), 484 (lambda value: '{0} must be greater then zero'.format(value)) 485 )) 486 return self 487 488 def list(self, item_func, msg_func=None): 489 '''Describe list with any number of elements, each matching given spec 490 491 :param item_func: 492 :py:class:`Spec` instance or a callable. Check out 493 :py:meth:`Spec.check_list` documentation for more details. Note that 494 in :py:meth:`Spec.check_list` description :py:class:`Spec` instance 495 is replaced with its index in ``self.specs``. 496 :param function msg_func: 497 Function that should accept checked value and return message that 498 describes the problem with this value. Default value will emit just 499 “failed check”, which is rather indescriptive. 500 501 :return: self. 502 ''' 503 self.type(list) 504 if isinstance(item_func, Spec): 505 self.specs.append(item_func) 506 item_func = len(self.specs) - 1 507 self.checks.append(('check_list', item_func, msg_func or (lambda item: 'failed check'))) 508 return self 509 510 def tuple(self, *specs): 511 '''Describe list with the given number of elements, each matching corresponding spec 512 513 :param (Spec,) specs: 514 List of specifications. Last element(s) in this list may be 515 optional. Each element in this list describes element with the same 516 index in the checked value. Check out :py:meth:`Spec.check_tuple` 517 for more details, but note that there list of specifications is 518 replaced with start and end indices in ``self.specs``. 519 520 :return: self. 521 ''' 522 self.type(list) 523 524 max_len = len(specs) 525 min_len = max_len 526 for spec in reversed(specs): 527 if spec.isoptional: 528 min_len -= 1 529 else: 530 break 531 if max_len == min_len: 532 self.len('eq', len(specs)) 533 else: 534 if min_len > 0: 535 self.len('ge', min_len) 536 self.len('le', max_len) 537 538 start = len(self.specs) 539 for i, spec in zip(itertools.count(), specs): 540 self.specs.append(spec) 541 self.checks.append(('check_tuple', start, len(self.specs))) 542 return self 543 544 def func(self, func, msg_func=None): 545 '''Describe value that is checked by the given function 546 547 Check out :py:meth:`Spec.check_func` documentation for more details. 548 ''' 549 self.checks.append(('check_func', func, msg_func or (lambda value: 'failed check'))) 550 return self 551 552 def re(self, regex, msg_func=None): 553 '''Describe value that is a string that matches given regular expression 554 555 :param str regex: 556 Regular expression that should be matched by the value. 557 :param function msg_func: 558 Function that should accept checked value and return message that 559 describes the problem with this value. Default value will emit 560 something like “String "xyz" does not match "[a-f]+"”. 561 562 :return: self. 563 ''' 564 self.type(unicode) 565 compiled = re.compile(regex) 566 msg_func = msg_func or (lambda value: 'String "{0}" does not match "{1}"'.format(value, regex)) 567 self.checks.append(( 568 'check_func', 569 (lambda value, *args: (True, True, not compiled.match(value.value))), 570 msg_func 571 )) 572 return self 573 574 def ident(self, msg_func=None): 575 '''Describe value that is an identifier like ``foo:bar`` or ``foo`` 576 577 :param function msg_func: 578 Function that should accept checked value and return message that 579 describes the problem with this value. Default value will emit 580 something like “String "xyz" is not an … identifier”. 581 582 :return: self. 583 ''' 584 msg_func = ( 585 msg_func 586 or (lambda value: 'String "{0}" is not an alphanumeric/underscore colon-separated identifier'.format(value)) 587 ) 588 return self.re('^\w+(?::\w+)?$', msg_func) 589 590 def oneof(self, collection, msg_func=None): 591 '''Describe value that is equal to one of the value in the collection 592 593 :param set collection: 594 A collection of possible values. 595 :param function msg_func: 596 Function that should accept checked value and return message that 597 describes the problem with this value. Default value will emit 598 something like “"xyz" must be one of {'abc', 'def', 'ghi'}”. 599 600 :return: self. 601 ''' 602 msg_func = msg_func or (lambda value: '"{0}" must be one of {1!r}'.format(value, list(collection))) 603 self.checks.append(( 604 'check_func', 605 (lambda value, *args: (True, True, value not in collection)), 606 msg_func 607 )) 608 return self 609 610 def error(self, msg): 611 '''Describe value that must not be there 612 613 Useful for giving more descriptive errors for some specific keys then 614 just “found unknown key: shutdown_event” or for forbidding certain 615 values when :py:meth:`Spec.unknown_spec` was used. 616 617 :param str msg: 618 Message given for the offending value. It is formatted using 619 :py:meth:`str.format` with the only positional parameter which is 620 the value itself. 621 622 :return: self. 623 ''' 624 self.checks.append(( 625 'check_func', 626 (lambda *args: (True, True, True)), 627 (lambda value: msg.format(value)) 628 )) 629 return self 630 631 def either(self, *specs): 632 '''Describes value that matches one of the given specs 633 634 Check out :py:meth:`Spec.check_either` method documentation for more 635 details, but note that there a list of specs was replaced by start and 636 end indices in ``self.specs``. 637 638 :return: self. 639 ''' 640 start = len(self.specs) 641 self.specs.extend(specs) 642 self.checks.append(('check_either', start, len(self.specs))) 643 return self 644 645 def optional(self): 646 '''Mark value as optional 647 648 Only useful for key specs in :py:meth:`Spec.__init__` and 649 :py:meth:`Spec.update` and some last supplied to :py:meth:`Spec.tuple`. 650 651 :return: self. 652 ''' 653 self.isoptional = True 654 return self 655 656 def required(self): 657 '''Mark value as required 658 659 Only useful for key specs in :py:meth:`Spec.__init__` and 660 :py:meth:`Spec.update` and some last supplied to :py:meth:`Spec.tuple`. 661 662 .. note:: 663 Value is required by default. This method is only useful for 664 altering existing specification (or rather its copy). 665 666 :return: self. 667 ''' 668 self.isoptional = False 669 return self 670 671 def match_checks(self, *args): 672 '''Process checks registered for the given value 673 674 Processes only “top-level” checks: key specifications given using at the 675 initialization or via :py:meth:`Spec.unknown_spec` are processed by 676 :py:meth:`Spec.match`. 677 678 :return: proceed, hadproblem. 679 ''' 680 hadproblem = False 681 for check in self.checks: 682 proceed, chadproblem = getattr(self, check[0])(*(args + check[1:])) 683 if chadproblem: 684 hadproblem = True 685 if not proceed: 686 return False, hadproblem 687 return True, hadproblem 688 689 def match(self, value, context_mark=None, data=None, context=(), echoerr=echoerr): 690 '''Check that given value matches this specification 691 692 :return: proceed, hadproblem. 693 ''' 694 havemarks(value) 695 proceed, hadproblem = self.match_checks(value, context_mark, data, context, echoerr) 696 if proceed: 697 if self.keys or self.uspecs: 698 for key, vali in self.keys.items(): 699 valspec = self.specs[vali] 700 if key in value: 701 proceed, mhadproblem = valspec.match( 702 value[key], 703 value.mark, 704 data, 705 context.enter_key(value, key), 706 echoerr 707 ) 708 if mhadproblem: 709 hadproblem = True 710 if not proceed: 711 return False, hadproblem 712 else: 713 if not valspec.isoptional: 714 hadproblem = True 715 echoerr(context=self.cmsg.format(key=context.key), 716 context_mark=None, 717 problem='required key is missing: {0}'.format(key), 718 problem_mark=value.mark) 719 for key in value.keys(): 720 havemarks(key) 721 if key not in self.keys: 722 for keyfunc, vali in self.uspecs: 723 valspec = self.specs[vali] 724 if isinstance(keyfunc, int): 725 spec = self.specs[keyfunc] 726 proceed, khadproblem = spec.match(key, context_mark, data, context, echoerr) 727 else: 728 proceed, khadproblem = keyfunc(key, data, context, echoerr) 729 if khadproblem: 730 hadproblem = True 731 if proceed: 732 proceed, vhadproblem = valspec.match( 733 value[key], 734 value.mark, 735 data, 736 context.enter_key(value, key), 737 echoerr 738 ) 739 if vhadproblem: 740 hadproblem = True 741 break 742 else: 743 hadproblem = True 744 if self.ufailmsg: 745 echoerr(context=self.cmsg.format(key=context.key), 746 context_mark=None, 747 problem=self.ufailmsg(key), 748 problem_mark=key.mark) 749 return True, hadproblem 750 751 def __getitem__(self, key): 752 '''Get specification for the given key 753 ''' 754 return self.specs[self.keys[key]] 755 756 def __setitem__(self, key, value): 757 '''Set specification for the given key 758 ''' 759 self.update(**{key: value}) 760