1#!/usr/bin/env python3 2# 3# Argument Clinic 4# Copyright 2012-2013 by Larry Hastings. 5# Licensed to the PSF under a contributor agreement. 6# 7 8import abc 9import ast 10import collections 11import contextlib 12import copy 13import cpp 14import functools 15import hashlib 16import inspect 17import io 18import itertools 19import os 20import pprint 21import re 22import shlex 23import string 24import sys 25import tempfile 26import textwrap 27import traceback 28import types 29 30from types import * 31NoneType = type(None) 32 33# TODO: 34# 35# soon: 36# 37# * allow mixing any two of {positional-only, positional-or-keyword, 38# keyword-only} 39# * dict constructor uses positional-only and keyword-only 40# * max and min use positional only with an optional group 41# and keyword-only 42# 43 44version = '1' 45 46NoneType = type(None) 47NO_VARARG = "PY_SSIZE_T_MAX" 48CLINIC_PREFIX = "__clinic_" 49CLINIC_PREFIXED_ARGS = {"args"} 50 51class Unspecified: 52 def __repr__(self): 53 return '<Unspecified>' 54 55unspecified = Unspecified() 56 57 58class Null: 59 def __repr__(self): 60 return '<Null>' 61 62NULL = Null() 63 64 65class Unknown: 66 def __repr__(self): 67 return '<Unknown>' 68 69unknown = Unknown() 70 71sig_end_marker = '--' 72 73 74_text_accumulator_nt = collections.namedtuple("_text_accumulator", "text append output") 75 76def _text_accumulator(): 77 text = [] 78 def output(): 79 s = ''.join(text) 80 text.clear() 81 return s 82 return _text_accumulator_nt(text, text.append, output) 83 84 85text_accumulator_nt = collections.namedtuple("text_accumulator", "text append") 86 87def text_accumulator(): 88 """ 89 Creates a simple text accumulator / joiner. 90 91 Returns a pair of callables: 92 append, output 93 "append" appends a string to the accumulator. 94 "output" returns the contents of the accumulator 95 joined together (''.join(accumulator)) and 96 empties the accumulator. 97 """ 98 text, append, output = _text_accumulator() 99 return text_accumulator_nt(append, output) 100 101 102def warn_or_fail(fail=False, *args, filename=None, line_number=None): 103 joined = " ".join([str(a) for a in args]) 104 add, output = text_accumulator() 105 if fail: 106 add("Error") 107 else: 108 add("Warning") 109 if clinic: 110 if filename is None: 111 filename = clinic.filename 112 if getattr(clinic, 'block_parser', None) and (line_number is None): 113 line_number = clinic.block_parser.line_number 114 if filename is not None: 115 add(' in file "' + filename + '"') 116 if line_number is not None: 117 add(" on line " + str(line_number)) 118 add(':\n') 119 add(joined) 120 print(output()) 121 if fail: 122 sys.exit(-1) 123 124 125def warn(*args, filename=None, line_number=None): 126 return warn_or_fail(False, *args, filename=filename, line_number=line_number) 127 128def fail(*args, filename=None, line_number=None): 129 return warn_or_fail(True, *args, filename=filename, line_number=line_number) 130 131 132def quoted_for_c_string(s): 133 for old, new in ( 134 ('\\', '\\\\'), # must be first! 135 ('"', '\\"'), 136 ("'", "\\'"), 137 ): 138 s = s.replace(old, new) 139 return s 140 141def c_repr(s): 142 return '"' + s + '"' 143 144 145is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match 146 147def is_legal_py_identifier(s): 148 return all(is_legal_c_identifier(field) for field in s.split('.')) 149 150# identifiers that are okay in Python but aren't a good idea in C. 151# so if they're used Argument Clinic will add "_value" to the end 152# of the name in C. 153c_keywords = set(""" 154asm auto break case char const continue default do double 155else enum extern float for goto if inline int long 156register return short signed sizeof static struct switch 157typedef typeof union unsigned void volatile while 158""".strip().split()) 159 160def ensure_legal_c_identifier(s): 161 # for now, just complain if what we're given isn't legal 162 if not is_legal_c_identifier(s): 163 fail("Illegal C identifier: {}".format(s)) 164 # but if we picked a C keyword, pick something else 165 if s in c_keywords: 166 return s + "_value" 167 return s 168 169def rstrip_lines(s): 170 text, add, output = _text_accumulator() 171 for line in s.split('\n'): 172 add(line.rstrip()) 173 add('\n') 174 text.pop() 175 return output() 176 177def format_escape(s): 178 # double up curly-braces, this string will be used 179 # as part of a format_map() template later 180 s = s.replace('{', '{{') 181 s = s.replace('}', '}}') 182 return s 183 184def linear_format(s, **kwargs): 185 """ 186 Perform str.format-like substitution, except: 187 * The strings substituted must be on lines by 188 themselves. (This line is the "source line".) 189 * If the substitution text is empty, the source line 190 is removed in the output. 191 * If the field is not recognized, the original line 192 is passed unmodified through to the output. 193 * If the substitution text is not empty: 194 * Each line of the substituted text is indented 195 by the indent of the source line. 196 * A newline will be added to the end. 197 """ 198 199 add, output = text_accumulator() 200 for line in s.split('\n'): 201 indent, curly, trailing = line.partition('{') 202 if not curly: 203 add(line) 204 add('\n') 205 continue 206 207 name, curly, trailing = trailing.partition('}') 208 if not curly or name not in kwargs: 209 add(line) 210 add('\n') 211 continue 212 213 if trailing: 214 fail("Text found after {" + name + "} block marker! It must be on a line by itself.") 215 if indent.strip(): 216 fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.") 217 218 value = kwargs[name] 219 if not value: 220 continue 221 222 value = textwrap.indent(rstrip_lines(value), indent) 223 add(value) 224 add('\n') 225 226 return output()[:-1] 227 228def indent_all_lines(s, prefix): 229 """ 230 Returns 's', with 'prefix' prepended to all lines. 231 232 If the last line is empty, prefix is not prepended 233 to it. (If s is blank, returns s unchanged.) 234 235 (textwrap.indent only adds to non-blank lines.) 236 """ 237 split = s.split('\n') 238 last = split.pop() 239 final = [] 240 for line in split: 241 final.append(prefix) 242 final.append(line) 243 final.append('\n') 244 if last: 245 final.append(prefix) 246 final.append(last) 247 return ''.join(final) 248 249def suffix_all_lines(s, suffix): 250 """ 251 Returns 's', with 'suffix' appended to all lines. 252 253 If the last line is empty, suffix is not appended 254 to it. (If s is blank, returns s unchanged.) 255 """ 256 split = s.split('\n') 257 last = split.pop() 258 final = [] 259 for line in split: 260 final.append(line) 261 final.append(suffix) 262 final.append('\n') 263 if last: 264 final.append(last) 265 final.append(suffix) 266 return ''.join(final) 267 268 269def version_splitter(s): 270 """Splits a version string into a tuple of integers. 271 272 The following ASCII characters are allowed, and employ 273 the following conversions: 274 a -> -3 275 b -> -2 276 c -> -1 277 (This permits Python-style version strings such as "1.4b3".) 278 """ 279 version = [] 280 accumulator = [] 281 def flush(): 282 if not accumulator: 283 raise ValueError('Unsupported version string: ' + repr(s)) 284 version.append(int(''.join(accumulator))) 285 accumulator.clear() 286 287 for c in s: 288 if c.isdigit(): 289 accumulator.append(c) 290 elif c == '.': 291 flush() 292 elif c in 'abc': 293 flush() 294 version.append('abc'.index(c) - 3) 295 else: 296 raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) 297 flush() 298 return tuple(version) 299 300def version_comparitor(version1, version2): 301 iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) 302 for i, (a, b) in enumerate(iterator): 303 if a < b: 304 return -1 305 if a > b: 306 return 1 307 return 0 308 309 310class CRenderData: 311 def __init__(self): 312 313 # The C statements to declare variables. 314 # Should be full lines with \n eol characters. 315 self.declarations = [] 316 317 # The C statements required to initialize the variables before the parse call. 318 # Should be full lines with \n eol characters. 319 self.initializers = [] 320 321 # The C statements needed to dynamically modify the values 322 # parsed by the parse call, before calling the impl. 323 self.modifications = [] 324 325 # The entries for the "keywords" array for PyArg_ParseTuple. 326 # Should be individual strings representing the names. 327 self.keywords = [] 328 329 # The "format units" for PyArg_ParseTuple. 330 # Should be individual strings that will get 331 self.format_units = [] 332 333 # The varargs arguments for PyArg_ParseTuple. 334 self.parse_arguments = [] 335 336 # The parameter declarations for the impl function. 337 self.impl_parameters = [] 338 339 # The arguments to the impl function at the time it's called. 340 self.impl_arguments = [] 341 342 # For return converters: the name of the variable that 343 # should receive the value returned by the impl. 344 self.return_value = "return_value" 345 346 # For return converters: the code to convert the return 347 # value from the parse function. This is also where 348 # you should check the _return_value for errors, and 349 # "goto exit" if there are any. 350 self.return_conversion = [] 351 352 # The C statements required to clean up after the impl call. 353 self.cleanup = [] 354 355 356class FormatCounterFormatter(string.Formatter): 357 """ 358 This counts how many instances of each formatter 359 "replacement string" appear in the format string. 360 361 e.g. after evaluating "string {a}, {b}, {c}, {a}" 362 the counts dict would now look like 363 {'a': 2, 'b': 1, 'c': 1} 364 """ 365 def __init__(self): 366 self.counts = collections.Counter() 367 368 def get_value(self, key, args, kwargs): 369 self.counts[key] += 1 370 return '' 371 372class Language(metaclass=abc.ABCMeta): 373 374 start_line = "" 375 body_prefix = "" 376 stop_line = "" 377 checksum_line = "" 378 379 def __init__(self, filename): 380 pass 381 382 @abc.abstractmethod 383 def render(self, clinic, signatures): 384 pass 385 386 def parse_line(self, line): 387 pass 388 389 def validate(self): 390 def assert_only_one(attr, *additional_fields): 391 """ 392 Ensures that the string found at getattr(self, attr) 393 contains exactly one formatter replacement string for 394 each valid field. The list of valid fields is 395 ['dsl_name'] extended by additional_fields. 396 397 e.g. 398 self.fmt = "{dsl_name} {a} {b}" 399 400 # this passes 401 self.assert_only_one('fmt', 'a', 'b') 402 403 # this fails, the format string has a {b} in it 404 self.assert_only_one('fmt', 'a') 405 406 # this fails, the format string doesn't have a {c} in it 407 self.assert_only_one('fmt', 'a', 'b', 'c') 408 409 # this fails, the format string has two {a}s in it, 410 # it must contain exactly one 411 self.fmt2 = '{dsl_name} {a} {a}' 412 self.assert_only_one('fmt2', 'a') 413 414 """ 415 fields = ['dsl_name'] 416 fields.extend(additional_fields) 417 line = getattr(self, attr) 418 fcf = FormatCounterFormatter() 419 fcf.format(line) 420 def local_fail(should_be_there_but_isnt): 421 if should_be_there_but_isnt: 422 fail("{} {} must contain {{{}}} exactly once!".format( 423 self.__class__.__name__, attr, name)) 424 else: 425 fail("{} {} must not contain {{{}}}!".format( 426 self.__class__.__name__, attr, name)) 427 428 for name, count in fcf.counts.items(): 429 if name in fields: 430 if count > 1: 431 local_fail(True) 432 else: 433 local_fail(False) 434 for name in fields: 435 if fcf.counts.get(name) != 1: 436 local_fail(True) 437 438 assert_only_one('start_line') 439 assert_only_one('stop_line') 440 441 field = "arguments" if "{arguments}" in self.checksum_line else "checksum" 442 assert_only_one('checksum_line', field) 443 444 445 446class PythonLanguage(Language): 447 448 language = 'Python' 449 start_line = "#/*[{dsl_name} input]" 450 body_prefix = "#" 451 stop_line = "#[{dsl_name} start generated code]*/" 452 checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" 453 454 455def permute_left_option_groups(l): 456 """ 457 Given [1, 2, 3], should yield: 458 () 459 (3,) 460 (2, 3) 461 (1, 2, 3) 462 """ 463 yield tuple() 464 accumulator = [] 465 for group in reversed(l): 466 accumulator = list(group) + accumulator 467 yield tuple(accumulator) 468 469 470def permute_right_option_groups(l): 471 """ 472 Given [1, 2, 3], should yield: 473 () 474 (1,) 475 (1, 2) 476 (1, 2, 3) 477 """ 478 yield tuple() 479 accumulator = [] 480 for group in l: 481 accumulator.extend(group) 482 yield tuple(accumulator) 483 484 485def permute_optional_groups(left, required, right): 486 """ 487 Generator function that computes the set of acceptable 488 argument lists for the provided iterables of 489 argument groups. (Actually it generates a tuple of tuples.) 490 491 Algorithm: prefer left options over right options. 492 493 If required is empty, left must also be empty. 494 """ 495 required = tuple(required) 496 result = [] 497 498 if not required: 499 assert not left 500 501 accumulator = [] 502 counts = set() 503 for r in permute_right_option_groups(right): 504 for l in permute_left_option_groups(left): 505 t = l + required + r 506 if len(t) in counts: 507 continue 508 counts.add(len(t)) 509 accumulator.append(t) 510 511 accumulator.sort(key=len) 512 return tuple(accumulator) 513 514 515def strip_leading_and_trailing_blank_lines(s): 516 lines = s.rstrip().split('\n') 517 while lines: 518 line = lines[0] 519 if line.strip(): 520 break 521 del lines[0] 522 return '\n'.join(lines) 523 524@functools.lru_cache() 525def normalize_snippet(s, *, indent=0): 526 """ 527 Reformats s: 528 * removes leading and trailing blank lines 529 * ensures that it does not end with a newline 530 * dedents so the first nonwhite character on any line is at column "indent" 531 """ 532 s = strip_leading_and_trailing_blank_lines(s) 533 s = textwrap.dedent(s) 534 if indent: 535 s = textwrap.indent(s, ' ' * indent) 536 return s 537 538 539def wrap_declarations(text, length=78): 540 """ 541 A simple-minded text wrapper for C function declarations. 542 543 It views a declaration line as looking like this: 544 xxxxxxxx(xxxxxxxxx,xxxxxxxxx) 545 If called with length=30, it would wrap that line into 546 xxxxxxxx(xxxxxxxxx, 547 xxxxxxxxx) 548 (If the declaration has zero or one parameters, this 549 function won't wrap it.) 550 551 If this doesn't work properly, it's probably better to 552 start from scratch with a more sophisticated algorithm, 553 rather than try and improve/debug this dumb little function. 554 """ 555 lines = [] 556 for line in text.split('\n'): 557 prefix, _, after_l_paren = line.partition('(') 558 if not after_l_paren: 559 lines.append(line) 560 continue 561 parameters, _, after_r_paren = after_l_paren.partition(')') 562 if not _: 563 lines.append(line) 564 continue 565 if ',' not in parameters: 566 lines.append(line) 567 continue 568 parameters = [x.strip() + ", " for x in parameters.split(',')] 569 prefix += "(" 570 if len(prefix) < length: 571 spaces = " " * len(prefix) 572 else: 573 spaces = " " * 4 574 575 while parameters: 576 line = prefix 577 first = True 578 while parameters: 579 if (not first and 580 (len(line) + len(parameters[0]) > length)): 581 break 582 line += parameters.pop(0) 583 first = False 584 if not parameters: 585 line = line.rstrip(", ") + ")" + after_r_paren 586 lines.append(line.rstrip()) 587 prefix = spaces 588 return "\n".join(lines) 589 590 591class CLanguage(Language): 592 593 body_prefix = "#" 594 language = 'C' 595 start_line = "/*[{dsl_name} input]" 596 body_prefix = "" 597 stop_line = "[{dsl_name} start generated code]*/" 598 checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" 599 600 def __init__(self, filename): 601 super().__init__(filename) 602 self.cpp = cpp.Monitor(filename) 603 self.cpp.fail = fail 604 605 def parse_line(self, line): 606 self.cpp.writeline(line) 607 608 def render(self, clinic, signatures): 609 function = None 610 for o in signatures: 611 if isinstance(o, Function): 612 if function: 613 fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) 614 function = o 615 return self.render_function(clinic, function) 616 617 def docstring_for_c_string(self, f): 618 if re.search(r'[^\x00-\x7F]', f.docstring): 619 warn("Non-ascii character appear in docstring.") 620 621 text, add, output = _text_accumulator() 622 # turn docstring into a properly quoted C string 623 for line in f.docstring.split('\n'): 624 add('"') 625 add(quoted_for_c_string(line)) 626 add('\\n"\n') 627 628 if text[-2] == sig_end_marker: 629 # If we only have a signature, add the blank line that the 630 # __text_signature__ getter expects to be there. 631 add('"\\n"') 632 else: 633 text.pop() 634 add('"') 635 return ''.join(text) 636 637 def output_templates(self, f): 638 parameters = list(f.parameters.values()) 639 assert parameters 640 assert isinstance(parameters[0].converter, self_converter) 641 del parameters[0] 642 converters = [p.converter for p in parameters] 643 644 has_option_groups = parameters and (parameters[0].group or parameters[-1].group) 645 default_return_converter = (not f.return_converter or 646 f.return_converter.type == 'PyObject *') 647 648 new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) 649 650 vararg = NO_VARARG 651 pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 652 for i, p in enumerate(parameters, 1): 653 if p.is_keyword_only() or vararg != NO_VARARG: 654 assert not p.is_positional_only() 655 if not p.is_optional(): 656 min_kw_only = i - max_pos 657 elif p.is_vararg(): 658 if vararg != NO_VARARG: 659 fail("Too many var args") 660 pseudo_args += 1 661 vararg = i - 1 662 else: 663 if vararg == NO_VARARG: 664 max_pos = i 665 if p.is_positional_only(): 666 pos_only = i 667 if not p.is_optional(): 668 min_pos = i 669 670 requires_defining_class = any( 671 isinstance(p.converter, defining_class_converter) 672 for p in parameters) 673 674 meth_o = (len(parameters) == 1 and 675 parameters[0].is_positional_only() and 676 not converters[0].is_optional() and 677 not requires_defining_class and 678 not new_or_init) 679 680 # we have to set these things before we're done: 681 # 682 # docstring_prototype 683 # docstring_definition 684 # impl_prototype 685 # methoddef_define 686 # parser_prototype 687 # parser_definition 688 # impl_definition 689 # cpp_if 690 # cpp_endif 691 # methoddef_ifndef 692 693 return_value_declaration = "PyObject *return_value = NULL;" 694 695 methoddef_define = normalize_snippet(""" 696 #define {methoddef_name} \\ 697 {{"{name}", {methoddef_cast}{c_basename}, {methoddef_flags}, {c_basename}__doc__}}, 698 """) 699 if new_or_init and not f.docstring: 700 docstring_prototype = docstring_definition = '' 701 else: 702 docstring_prototype = normalize_snippet(""" 703 PyDoc_VAR({c_basename}__doc__); 704 """) 705 docstring_definition = normalize_snippet(""" 706 PyDoc_STRVAR({c_basename}__doc__, 707 {docstring}); 708 """) 709 impl_definition = normalize_snippet(""" 710 static {impl_return_type} 711 {c_basename}_impl({impl_parameters}) 712 """) 713 impl_prototype = parser_prototype = parser_definition = None 714 715 parser_prototype_keyword = normalize_snippet(""" 716 static PyObject * 717 {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) 718 """) 719 720 parser_prototype_varargs = normalize_snippet(""" 721 static PyObject * 722 {c_basename}({self_type}{self_name}, PyObject *args) 723 """) 724 725 parser_prototype_fastcall = normalize_snippet(""" 726 static PyObject * 727 {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) 728 """) 729 730 parser_prototype_fastcall_keywords = normalize_snippet(""" 731 static PyObject * 732 {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) 733 """) 734 735 parser_prototype_def_class = normalize_snippet(""" 736 static PyObject * 737 {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) 738 """) 739 740 # parser_body_fields remembers the fields passed in to the 741 # previous call to parser_body. this is used for an awful hack. 742 parser_body_fields = () 743 parser_body_declarations = '' 744 def parser_body(prototype, *fields, declarations=''): 745 nonlocal parser_body_fields, parser_body_declarations 746 add, output = text_accumulator() 747 add(prototype) 748 parser_body_fields = fields 749 parser_body_declarations = declarations 750 751 fields = list(fields) 752 fields.insert(0, normalize_snippet(""" 753 {{ 754 {return_value_declaration} 755 {parser_declarations} 756 {declarations} 757 {initializers} 758 """) + "\n") 759 # just imagine--your code is here in the middle 760 fields.append(normalize_snippet(""" 761 {modifications} 762 {return_value} = {c_basename}_impl({impl_arguments}); 763 {return_conversion} 764 765 {exit_label} 766 {cleanup} 767 return return_value; 768 }} 769 """)) 770 for field in fields: 771 add('\n') 772 add(field) 773 return linear_format(output(), parser_declarations=declarations) 774 775 if not parameters: 776 # no parameters, METH_NOARGS 777 778 flags = "METH_NOARGS" 779 780 parser_prototype = normalize_snippet(""" 781 static PyObject * 782 {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) 783 """) 784 parser_definition = parser_prototype 785 786 if default_return_converter: 787 parser_definition = parser_prototype + '\n' + normalize_snippet(""" 788 {{ 789 return {c_basename}_impl({impl_arguments}); 790 }} 791 """) 792 else: 793 parser_definition = parser_body(parser_prototype) 794 795 elif meth_o: 796 flags = "METH_O" 797 798 if (isinstance(converters[0], object_converter) and 799 converters[0].format_unit == 'O'): 800 meth_o_prototype = normalize_snippet(""" 801 static PyObject * 802 {c_basename}({impl_parameters}) 803 """) 804 805 if default_return_converter: 806 # maps perfectly to METH_O, doesn't need a return converter. 807 # so we skip making a parse function 808 # and call directly into the impl function. 809 impl_prototype = parser_prototype = parser_definition = '' 810 impl_definition = meth_o_prototype 811 else: 812 # SLIGHT HACK 813 # use impl_parameters for the parser here! 814 parser_prototype = meth_o_prototype 815 parser_definition = parser_body(parser_prototype) 816 817 else: 818 argname = 'arg' 819 if parameters[0].name == argname: 820 argname += '_' 821 parser_prototype = normalize_snippet(""" 822 static PyObject * 823 {c_basename}({self_type}{self_name}, PyObject *%s) 824 """ % argname) 825 826 displayname = parameters[0].get_displayname(0) 827 parsearg = converters[0].parse_arg(argname, displayname) 828 if parsearg is None: 829 parsearg = """ 830 if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ 831 goto exit; 832 }} 833 """ % argname 834 parser_definition = parser_body(parser_prototype, 835 normalize_snippet(parsearg, indent=4)) 836 837 elif has_option_groups: 838 # positional parameters with option groups 839 # (we have to generate lots of PyArg_ParseTuple calls 840 # in a big switch statement) 841 842 flags = "METH_VARARGS" 843 parser_prototype = parser_prototype_varargs 844 845 parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') 846 847 elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: 848 if not new_or_init: 849 # positional-only, but no option groups 850 # we only need one call to _PyArg_ParseStack 851 852 flags = "METH_FASTCALL" 853 parser_prototype = parser_prototype_fastcall 854 nargs = 'nargs' 855 argname_fmt = 'args[%d]' 856 else: 857 # positional-only, but no option groups 858 # we only need one call to PyArg_ParseTuple 859 860 flags = "METH_VARARGS" 861 parser_prototype = parser_prototype_varargs 862 nargs = 'PyTuple_GET_SIZE(args)' 863 argname_fmt = 'PyTuple_GET_ITEM(args, %d)' 864 865 866 left_args = "{} - {}".format(nargs, max_pos) 867 max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos 868 parser_code = [normalize_snippet(""" 869 if (!_PyArg_CheckPositional("{name}", %s, %d, %s)) {{ 870 goto exit; 871 }} 872 """ % (nargs, min_pos, max_args), indent=4)] 873 874 has_optional = False 875 for i, p in enumerate(parameters): 876 displayname = p.get_displayname(i+1) 877 argname = argname_fmt % i 878 879 if p.is_vararg(): 880 if not new_or_init: 881 parser_code.append(normalize_snippet(""" 882 %s = PyTuple_New(%s); 883 for (Py_ssize_t i = 0; i < %s; ++i) {{ 884 PyTuple_SET_ITEM(%s, i, args[%d + i]); 885 }} 886 """ % ( 887 p.converter.parser_name, 888 left_args, 889 left_args, 890 p.converter.parser_name, 891 max_pos 892 ), indent=4)) 893 else: 894 parser_code.append(normalize_snippet(""" 895 %s = PyTuple_GetSlice(%d, -1); 896 """ % ( 897 p.converter.parser_name, 898 max_pos 899 ), indent=4)) 900 continue 901 902 parsearg = p.converter.parse_arg(argname, displayname) 903 if parsearg is None: 904 #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) 905 parser_code = None 906 break 907 if has_optional or p.is_optional(): 908 has_optional = True 909 parser_code.append(normalize_snippet(""" 910 if (%s < %d) {{ 911 goto skip_optional; 912 }} 913 """, indent=4) % (nargs, i + 1)) 914 parser_code.append(normalize_snippet(parsearg, indent=4)) 915 916 if parser_code is not None: 917 if has_optional: 918 parser_code.append("skip_optional:") 919 else: 920 if not new_or_init: 921 parser_code = [normalize_snippet(""" 922 if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", 923 {parse_arguments})) {{ 924 goto exit; 925 }} 926 """, indent=4)] 927 else: 928 parser_code = [normalize_snippet(""" 929 if (!PyArg_ParseTuple(args, "{format_units}:{name}", 930 {parse_arguments})) {{ 931 goto exit; 932 }} 933 """, indent=4)] 934 parser_definition = parser_body(parser_prototype, *parser_code) 935 936 else: 937 has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters)) 938 if vararg == NO_VARARG: 939 args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( 940 min_pos, 941 max_pos, 942 min_kw_only 943 ) 944 else: 945 args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( 946 min_pos, 947 max_pos, 948 min_kw_only, 949 vararg 950 ) 951 if not new_or_init: 952 flags = "METH_FASTCALL|METH_KEYWORDS" 953 parser_prototype = parser_prototype_fastcall_keywords 954 argname_fmt = 'args[%d]' 955 declarations = normalize_snippet(""" 956 static const char * const _keywords[] = {{{keywords} NULL}}; 957 static _PyArg_Parser _parser = {{NULL, _keywords, "{name}", 0}}; 958 PyObject *argsbuf[%s]; 959 """ % len(converters)) 960 if has_optional_kw: 961 pre_buffer = "0" if vararg != NO_VARARG else "nargs" 962 declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (pre_buffer, min_pos + min_kw_only) 963 parser_code = [normalize_snippet(""" 964 args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); 965 if (!args) {{ 966 goto exit; 967 }} 968 """ % args_declaration, indent=4)] 969 else: 970 # positional-or-keyword arguments 971 flags = "METH_VARARGS|METH_KEYWORDS" 972 parser_prototype = parser_prototype_keyword 973 argname_fmt = 'fastargs[%d]' 974 declarations = normalize_snippet(""" 975 static const char * const _keywords[] = {{{keywords} NULL}}; 976 static _PyArg_Parser _parser = {{NULL, _keywords, "{name}", 0}}; 977 PyObject *argsbuf[%s]; 978 PyObject * const *fastargs; 979 Py_ssize_t nargs = PyTuple_GET_SIZE(args); 980 """ % len(converters)) 981 if has_optional_kw: 982 declarations += "\nPy_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (min_pos + min_kw_only) 983 parser_code = [normalize_snippet(""" 984 fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); 985 if (!fastargs) {{ 986 goto exit; 987 }} 988 """ % args_declaration, indent=4)] 989 990 if requires_defining_class: 991 flags = 'METH_METHOD|' + flags 992 parser_prototype = parser_prototype_def_class 993 994 add_label = None 995 for i, p in enumerate(parameters): 996 displayname = p.get_displayname(i+1) 997 parsearg = p.converter.parse_arg(argname_fmt % i, displayname) 998 if parsearg is None: 999 #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) 1000 parser_code = None 1001 break 1002 if add_label and (i == pos_only or i == max_pos): 1003 parser_code.append("%s:" % add_label) 1004 add_label = None 1005 if not p.is_optional(): 1006 parser_code.append(normalize_snippet(parsearg, indent=4)) 1007 elif i < pos_only: 1008 add_label = 'skip_optional_posonly' 1009 parser_code.append(normalize_snippet(""" 1010 if (nargs < %d) {{ 1011 goto %s; 1012 }} 1013 """ % (i + 1, add_label), indent=4)) 1014 if has_optional_kw: 1015 parser_code.append(normalize_snippet(""" 1016 noptargs--; 1017 """, indent=4)) 1018 parser_code.append(normalize_snippet(parsearg, indent=4)) 1019 else: 1020 if i < max_pos: 1021 label = 'skip_optional_pos' 1022 first_opt = max(min_pos, pos_only) 1023 else: 1024 label = 'skip_optional_kwonly' 1025 first_opt = max_pos + min_kw_only 1026 if vararg != NO_VARARG: 1027 first_opt += 1 1028 if i == first_opt: 1029 add_label = label 1030 parser_code.append(normalize_snippet(""" 1031 if (!noptargs) {{ 1032 goto %s; 1033 }} 1034 """ % add_label, indent=4)) 1035 if i + 1 == len(parameters): 1036 parser_code.append(normalize_snippet(parsearg, indent=4)) 1037 else: 1038 add_label = label 1039 parser_code.append(normalize_snippet(""" 1040 if (%s) {{ 1041 """ % (argname_fmt % i), indent=4)) 1042 parser_code.append(normalize_snippet(parsearg, indent=8)) 1043 parser_code.append(normalize_snippet(""" 1044 if (!--noptargs) {{ 1045 goto %s; 1046 }} 1047 }} 1048 """ % add_label, indent=4)) 1049 1050 if parser_code is not None: 1051 if add_label: 1052 parser_code.append("%s:" % add_label) 1053 else: 1054 declarations = ( 1055 'static const char * const _keywords[] = {{{keywords} NULL}};\n' 1056 'static _PyArg_Parser _parser = {{"{format_units}:{name}", _keywords, 0}};') 1057 if not new_or_init: 1058 parser_code = [normalize_snippet(""" 1059 if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} 1060 {parse_arguments})) {{ 1061 goto exit; 1062 }} 1063 """, indent=4)] 1064 else: 1065 parser_code = [normalize_snippet(""" 1066 if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, 1067 {parse_arguments})) {{ 1068 goto exit; 1069 }} 1070 """, indent=4)] 1071 parser_definition = parser_body(parser_prototype, *parser_code, 1072 declarations=declarations) 1073 1074 1075 if new_or_init: 1076 methoddef_define = '' 1077 1078 if f.kind == METHOD_NEW: 1079 parser_prototype = parser_prototype_keyword 1080 else: 1081 return_value_declaration = "int return_value = -1;" 1082 parser_prototype = normalize_snippet(""" 1083 static int 1084 {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) 1085 """) 1086 1087 fields = list(parser_body_fields) 1088 parses_positional = 'METH_NOARGS' not in flags 1089 parses_keywords = 'METH_KEYWORDS' in flags 1090 if parses_keywords: 1091 assert parses_positional 1092 1093 if requires_defining_class: 1094 raise ValueError("Slot methods cannot access their defining class.") 1095 1096 if not parses_keywords: 1097 fields.insert(0, normalize_snippet(""" 1098 if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ 1099 goto exit; 1100 }} 1101 """, indent=4)) 1102 if not parses_positional: 1103 fields.insert(0, normalize_snippet(""" 1104 if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ 1105 goto exit; 1106 }} 1107 """, indent=4)) 1108 1109 parser_definition = parser_body(parser_prototype, *fields, 1110 declarations=parser_body_declarations) 1111 1112 1113 if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): 1114 methoddef_cast = "(PyCFunction)" 1115 else: 1116 methoddef_cast = "(PyCFunction)(void(*)(void))" 1117 1118 if f.methoddef_flags: 1119 flags += '|' + f.methoddef_flags 1120 1121 methoddef_define = methoddef_define.replace('{methoddef_flags}', flags) 1122 methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast) 1123 1124 methoddef_ifndef = '' 1125 conditional = self.cpp.condition() 1126 if not conditional: 1127 cpp_if = cpp_endif = '' 1128 else: 1129 cpp_if = "#if " + conditional 1130 cpp_endif = "#endif /* " + conditional + " */" 1131 1132 if methoddef_define and f.full_name not in clinic.ifndef_symbols: 1133 clinic.ifndef_symbols.add(f.full_name) 1134 methoddef_ifndef = normalize_snippet(""" 1135 #ifndef {methoddef_name} 1136 #define {methoddef_name} 1137 #endif /* !defined({methoddef_name}) */ 1138 """) 1139 1140 1141 # add ';' to the end of parser_prototype and impl_prototype 1142 # (they mustn't be None, but they could be an empty string.) 1143 assert parser_prototype is not None 1144 if parser_prototype: 1145 assert not parser_prototype.endswith(';') 1146 parser_prototype += ';' 1147 1148 if impl_prototype is None: 1149 impl_prototype = impl_definition 1150 if impl_prototype: 1151 impl_prototype += ";" 1152 1153 parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) 1154 1155 d = { 1156 "docstring_prototype" : docstring_prototype, 1157 "docstring_definition" : docstring_definition, 1158 "impl_prototype" : impl_prototype, 1159 "methoddef_define" : methoddef_define, 1160 "parser_prototype" : parser_prototype, 1161 "parser_definition" : parser_definition, 1162 "impl_definition" : impl_definition, 1163 "cpp_if" : cpp_if, 1164 "cpp_endif" : cpp_endif, 1165 "methoddef_ifndef" : methoddef_ifndef, 1166 } 1167 1168 # make sure we didn't forget to assign something, 1169 # and wrap each non-empty value in \n's 1170 d2 = {} 1171 for name, value in d.items(): 1172 assert value is not None, "got a None value for template " + repr(name) 1173 if value: 1174 value = '\n' + value + '\n' 1175 d2[name] = value 1176 return d2 1177 1178 @staticmethod 1179 def group_to_variable_name(group): 1180 adjective = "left_" if group < 0 else "right_" 1181 return "group_" + adjective + str(abs(group)) 1182 1183 def render_option_group_parsing(self, f, template_dict): 1184 # positional only, grouped, optional arguments! 1185 # can be optional on the left or right. 1186 # here's an example: 1187 # 1188 # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] 1189 # 1190 # Here group D are required, and all other groups are optional. 1191 # (Group D's "group" is actually None.) 1192 # We can figure out which sets of arguments we have based on 1193 # how many arguments are in the tuple. 1194 # 1195 # Note that you need to count up on both sides. For example, 1196 # you could have groups C+D, or C+D+E, or C+D+E+F. 1197 # 1198 # What if the number of arguments leads us to an ambiguous result? 1199 # Clinic prefers groups on the left. So in the above example, 1200 # five arguments would map to B+C, not C+D. 1201 1202 add, output = text_accumulator() 1203 parameters = list(f.parameters.values()) 1204 if isinstance(parameters[0].converter, self_converter): 1205 del parameters[0] 1206 1207 groups = [] 1208 group = None 1209 left = [] 1210 right = [] 1211 required = [] 1212 last = unspecified 1213 1214 for p in parameters: 1215 group_id = p.group 1216 if group_id != last: 1217 last = group_id 1218 group = [] 1219 if group_id < 0: 1220 left.append(group) 1221 elif group_id == 0: 1222 group = required 1223 else: 1224 right.append(group) 1225 group.append(p) 1226 1227 count_min = sys.maxsize 1228 count_max = -1 1229 1230 add("switch (PyTuple_GET_SIZE(args)) {\n") 1231 for subset in permute_optional_groups(left, required, right): 1232 count = len(subset) 1233 count_min = min(count_min, count) 1234 count_max = max(count_max, count) 1235 1236 if count == 0: 1237 add(""" case 0: 1238 break; 1239""") 1240 continue 1241 1242 group_ids = {p.group for p in subset} # eliminate duplicates 1243 d = {} 1244 d['count'] = count 1245 d['name'] = f.name 1246 d['format_units'] = "".join(p.converter.format_unit for p in subset) 1247 1248 parse_arguments = [] 1249 for p in subset: 1250 p.converter.parse_argument(parse_arguments) 1251 d['parse_arguments'] = ", ".join(parse_arguments) 1252 1253 group_ids.discard(0) 1254 lines = [self.group_to_variable_name(g) + " = 1;" for g in group_ids] 1255 lines = "\n".join(lines) 1256 1257 s = """\ 1258 case {count}: 1259 if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ 1260 goto exit; 1261 }} 1262 {group_booleans} 1263 break; 1264""" 1265 s = linear_format(s, group_booleans=lines) 1266 s = s.format_map(d) 1267 add(s) 1268 1269 add(" default:\n") 1270 s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' 1271 add(s.format(f.full_name, count_min, count_max)) 1272 add(' goto exit;\n') 1273 add("}") 1274 template_dict['option_group_parsing'] = format_escape(output()) 1275 1276 def render_function(self, clinic, f): 1277 if not f: 1278 return "" 1279 1280 add, output = text_accumulator() 1281 data = CRenderData() 1282 1283 assert f.parameters, "We should always have a 'self' at this point!" 1284 parameters = f.render_parameters 1285 converters = [p.converter for p in parameters] 1286 1287 templates = self.output_templates(f) 1288 1289 f_self = parameters[0] 1290 selfless = parameters[1:] 1291 assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" 1292 1293 last_group = 0 1294 first_optional = len(selfless) 1295 positional = selfless and selfless[-1].is_positional_only() 1296 new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) 1297 default_return_converter = (not f.return_converter or 1298 f.return_converter.type == 'PyObject *') 1299 has_option_groups = False 1300 1301 # offset i by -1 because first_optional needs to ignore self 1302 for i, p in enumerate(parameters, -1): 1303 c = p.converter 1304 1305 if (i != -1) and (p.default is not unspecified): 1306 first_optional = min(first_optional, i) 1307 1308 if p.is_vararg(): 1309 data.cleanup.append("Py_XDECREF({});".format(c.parser_name)) 1310 1311 # insert group variable 1312 group = p.group 1313 if last_group != group: 1314 last_group = group 1315 if group: 1316 group_name = self.group_to_variable_name(group) 1317 data.impl_arguments.append(group_name) 1318 data.declarations.append("int " + group_name + " = 0;") 1319 data.impl_parameters.append("int " + group_name) 1320 has_option_groups = True 1321 1322 c.render(p, data) 1323 1324 if has_option_groups and (not positional): 1325 fail("You cannot use optional groups ('[' and ']')\nunless all parameters are positional-only ('/').") 1326 1327 # HACK 1328 # when we're METH_O, but have a custom return converter, 1329 # we use "impl_parameters" for the parsing function 1330 # because that works better. but that means we must 1331 # suppress actually declaring the impl's parameters 1332 # as variables in the parsing function. but since it's 1333 # METH_O, we have exactly one anyway, so we know exactly 1334 # where it is. 1335 if ("METH_O" in templates['methoddef_define'] and 1336 '{impl_parameters}' in templates['parser_prototype']): 1337 data.declarations.pop(0) 1338 1339 template_dict = {} 1340 1341 full_name = f.full_name 1342 template_dict['full_name'] = full_name 1343 1344 if new_or_init: 1345 name = f.cls.name 1346 else: 1347 name = f.name 1348 1349 template_dict['name'] = name 1350 1351 if f.c_basename: 1352 c_basename = f.c_basename 1353 else: 1354 fields = full_name.split(".") 1355 if fields[-1] == '__new__': 1356 fields.pop() 1357 c_basename = "_".join(fields) 1358 1359 template_dict['c_basename'] = c_basename 1360 1361 methoddef_name = "{}_METHODDEF".format(c_basename.upper()) 1362 template_dict['methoddef_name'] = methoddef_name 1363 1364 template_dict['docstring'] = self.docstring_for_c_string(f) 1365 1366 template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' 1367 for converter in converters: 1368 converter.set_template_dict(template_dict) 1369 1370 f.return_converter.render(f, data) 1371 template_dict['impl_return_type'] = f.return_converter.type 1372 1373 template_dict['declarations'] = format_escape("\n".join(data.declarations)) 1374 template_dict['initializers'] = "\n\n".join(data.initializers) 1375 template_dict['modifications'] = '\n\n'.join(data.modifications) 1376 template_dict['keywords'] = ' '.join('"' + k + '",' for k in data.keywords) 1377 template_dict['format_units'] = ''.join(data.format_units) 1378 template_dict['parse_arguments'] = ', '.join(data.parse_arguments) 1379 if data.parse_arguments: 1380 template_dict['parse_arguments_comma'] = ','; 1381 else: 1382 template_dict['parse_arguments_comma'] = ''; 1383 template_dict['impl_parameters'] = ", ".join(data.impl_parameters) 1384 template_dict['impl_arguments'] = ", ".join(data.impl_arguments) 1385 template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) 1386 template_dict['cleanup'] = format_escape("".join(data.cleanup)) 1387 template_dict['return_value'] = data.return_value 1388 1389 # used by unpack tuple code generator 1390 ignore_self = -1 if isinstance(converters[0], self_converter) else 0 1391 unpack_min = first_optional 1392 unpack_max = len(selfless) 1393 template_dict['unpack_min'] = str(unpack_min) 1394 template_dict['unpack_max'] = str(unpack_max) 1395 1396 if has_option_groups: 1397 self.render_option_group_parsing(f, template_dict) 1398 1399 # buffers, not destination 1400 for name, destination in clinic.destination_buffers.items(): 1401 template = templates[name] 1402 if has_option_groups: 1403 template = linear_format(template, 1404 option_group_parsing=template_dict['option_group_parsing']) 1405 template = linear_format(template, 1406 declarations=template_dict['declarations'], 1407 return_conversion=template_dict['return_conversion'], 1408 initializers=template_dict['initializers'], 1409 modifications=template_dict['modifications'], 1410 cleanup=template_dict['cleanup'], 1411 ) 1412 1413 # Only generate the "exit:" label 1414 # if we have any gotos 1415 need_exit_label = "goto exit;" in template 1416 template = linear_format(template, 1417 exit_label="exit:" if need_exit_label else '' 1418 ) 1419 1420 s = template.format_map(template_dict) 1421 1422 # mild hack: 1423 # reflow long impl declarations 1424 if name in {"impl_prototype", "impl_definition"}: 1425 s = wrap_declarations(s) 1426 1427 if clinic.line_prefix: 1428 s = indent_all_lines(s, clinic.line_prefix) 1429 if clinic.line_suffix: 1430 s = suffix_all_lines(s, clinic.line_suffix) 1431 1432 destination.append(s) 1433 1434 return clinic.get_destination('block').dump() 1435 1436 1437 1438 1439@contextlib.contextmanager 1440def OverrideStdioWith(stdout): 1441 saved_stdout = sys.stdout 1442 sys.stdout = stdout 1443 try: 1444 yield 1445 finally: 1446 assert sys.stdout is stdout 1447 sys.stdout = saved_stdout 1448 1449 1450def create_regex(before, after, word=True, whole_line=True): 1451 """Create an re object for matching marker lines.""" 1452 group_re = r"\w+" if word else ".+" 1453 pattern = r'{}({}){}' 1454 if whole_line: 1455 pattern = '^' + pattern + '$' 1456 pattern = pattern.format(re.escape(before), group_re, re.escape(after)) 1457 return re.compile(pattern) 1458 1459 1460class Block: 1461 r""" 1462 Represents a single block of text embedded in 1463 another file. If dsl_name is None, the block represents 1464 verbatim text, raw original text from the file, in 1465 which case "input" will be the only non-false member. 1466 If dsl_name is not None, the block represents a Clinic 1467 block. 1468 1469 input is always str, with embedded \n characters. 1470 input represents the original text from the file; 1471 if it's a Clinic block, it is the original text with 1472 the body_prefix and redundant leading whitespace removed. 1473 1474 dsl_name is either str or None. If str, it's the text 1475 found on the start line of the block between the square 1476 brackets. 1477 1478 signatures is either list or None. If it's a list, 1479 it may only contain clinic.Module, clinic.Class, and 1480 clinic.Function objects. At the moment it should 1481 contain at most one of each. 1482 1483 output is either str or None. If str, it's the output 1484 from this block, with embedded '\n' characters. 1485 1486 indent is either str or None. It's the leading whitespace 1487 that was found on every line of input. (If body_prefix is 1488 not empty, this is the indent *after* removing the 1489 body_prefix.) 1490 1491 preindent is either str or None. It's the whitespace that 1492 was found in front of every line of input *before* the 1493 "body_prefix" (see the Language object). If body_prefix 1494 is empty, preindent must always be empty too. 1495 1496 To illustrate indent and preindent: Assume that '_' 1497 represents whitespace. If the block processed was in a 1498 Python file, and looked like this: 1499 ____#/*[python] 1500 ____#__for a in range(20): 1501 ____#____print(a) 1502 ____#[python]*/ 1503 "preindent" would be "____" and "indent" would be "__". 1504 1505 """ 1506 def __init__(self, input, dsl_name=None, signatures=None, output=None, indent='', preindent=''): 1507 assert isinstance(input, str) 1508 self.input = input 1509 self.dsl_name = dsl_name 1510 self.signatures = signatures or [] 1511 self.output = output 1512 self.indent = indent 1513 self.preindent = preindent 1514 1515 def __repr__(self): 1516 dsl_name = self.dsl_name or "text" 1517 def summarize(s): 1518 s = repr(s) 1519 if len(s) > 30: 1520 return s[:26] + "..." + s[0] 1521 return s 1522 return "".join(( 1523 "<Block ", dsl_name, " input=", summarize(self.input), " output=", summarize(self.output), ">")) 1524 1525 1526class BlockParser: 1527 """ 1528 Block-oriented parser for Argument Clinic. 1529 Iterator, yields Block objects. 1530 """ 1531 1532 def __init__(self, input, language, *, verify=True): 1533 """ 1534 "input" should be a str object 1535 with embedded \n characters. 1536 1537 "language" should be a Language object. 1538 """ 1539 language.validate() 1540 1541 self.input = collections.deque(reversed(input.splitlines(keepends=True))) 1542 self.block_start_line_number = self.line_number = 0 1543 1544 self.language = language 1545 before, _, after = language.start_line.partition('{dsl_name}') 1546 assert _ == '{dsl_name}' 1547 self.find_start_re = create_regex(before, after, whole_line=False) 1548 self.start_re = create_regex(before, after) 1549 self.verify = verify 1550 self.last_checksum_re = None 1551 self.last_dsl_name = None 1552 self.dsl_name = None 1553 self.first_block = True 1554 1555 def __iter__(self): 1556 return self 1557 1558 def __next__(self): 1559 while True: 1560 if not self.input: 1561 raise StopIteration 1562 1563 if self.dsl_name: 1564 return_value = self.parse_clinic_block(self.dsl_name) 1565 self.dsl_name = None 1566 self.first_block = False 1567 return return_value 1568 block = self.parse_verbatim_block() 1569 if self.first_block and not block.input: 1570 continue 1571 self.first_block = False 1572 return block 1573 1574 1575 def is_start_line(self, line): 1576 match = self.start_re.match(line.lstrip()) 1577 return match.group(1) if match else None 1578 1579 def _line(self, lookahead=False): 1580 self.line_number += 1 1581 line = self.input.pop() 1582 if not lookahead: 1583 self.language.parse_line(line) 1584 return line 1585 1586 def parse_verbatim_block(self): 1587 add, output = text_accumulator() 1588 self.block_start_line_number = self.line_number 1589 1590 while self.input: 1591 line = self._line() 1592 dsl_name = self.is_start_line(line) 1593 if dsl_name: 1594 self.dsl_name = dsl_name 1595 break 1596 add(line) 1597 1598 return Block(output()) 1599 1600 def parse_clinic_block(self, dsl_name): 1601 input_add, input_output = text_accumulator() 1602 self.block_start_line_number = self.line_number + 1 1603 stop_line = self.language.stop_line.format(dsl_name=dsl_name) 1604 body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) 1605 1606 def is_stop_line(line): 1607 # make sure to recognize stop line even if it 1608 # doesn't end with EOL (it could be the very end of the file) 1609 if not line.startswith(stop_line): 1610 return False 1611 remainder = line[len(stop_line):] 1612 return (not remainder) or remainder.isspace() 1613 1614 # consume body of program 1615 while self.input: 1616 line = self._line() 1617 if is_stop_line(line) or self.is_start_line(line): 1618 break 1619 if body_prefix: 1620 line = line.lstrip() 1621 assert line.startswith(body_prefix) 1622 line = line[len(body_prefix):] 1623 input_add(line) 1624 1625 # consume output and checksum line, if present. 1626 if self.last_dsl_name == dsl_name: 1627 checksum_re = self.last_checksum_re 1628 else: 1629 before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') 1630 assert _ == '{arguments}' 1631 checksum_re = create_regex(before, after, word=False) 1632 self.last_dsl_name = dsl_name 1633 self.last_checksum_re = checksum_re 1634 1635 # scan forward for checksum line 1636 output_add, output_output = text_accumulator() 1637 arguments = None 1638 while self.input: 1639 line = self._line(lookahead=True) 1640 match = checksum_re.match(line.lstrip()) 1641 arguments = match.group(1) if match else None 1642 if arguments: 1643 break 1644 output_add(line) 1645 if self.is_start_line(line): 1646 break 1647 1648 output = output_output() 1649 if arguments: 1650 d = {} 1651 for field in shlex.split(arguments): 1652 name, equals, value = field.partition('=') 1653 if not equals: 1654 fail("Mangled Argument Clinic marker line: {!r}".format(line)) 1655 d[name.strip()] = value.strip() 1656 1657 if self.verify: 1658 if 'input' in d: 1659 checksum = d['output'] 1660 input_checksum = d['input'] 1661 else: 1662 checksum = d['checksum'] 1663 input_checksum = None 1664 1665 computed = compute_checksum(output, len(checksum)) 1666 if checksum != computed: 1667 fail("Checksum mismatch!\nExpected: {}\nComputed: {}\n" 1668 "Suggested fix: remove all generated code including " 1669 "the end marker,\n" 1670 "or use the '-f' option." 1671 .format(checksum, computed)) 1672 else: 1673 # put back output 1674 output_lines = output.splitlines(keepends=True) 1675 self.line_number -= len(output_lines) 1676 self.input.extend(reversed(output_lines)) 1677 output = None 1678 1679 return Block(input_output(), dsl_name, output=output) 1680 1681 1682class BlockPrinter: 1683 1684 def __init__(self, language, f=None): 1685 self.language = language 1686 self.f = f or io.StringIO() 1687 1688 def print_block(self, block): 1689 input = block.input 1690 output = block.output 1691 dsl_name = block.dsl_name 1692 write = self.f.write 1693 1694 assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) 1695 1696 if not dsl_name: 1697 write(input) 1698 return 1699 1700 write(self.language.start_line.format(dsl_name=dsl_name)) 1701 write("\n") 1702 1703 body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) 1704 if not body_prefix: 1705 write(input) 1706 else: 1707 for line in input.split('\n'): 1708 write(body_prefix) 1709 write(line) 1710 write("\n") 1711 1712 write(self.language.stop_line.format(dsl_name=dsl_name)) 1713 write("\n") 1714 1715 input = ''.join(block.input) 1716 output = ''.join(block.output) 1717 if output: 1718 if not output.endswith('\n'): 1719 output += '\n' 1720 write(output) 1721 1722 arguments="output={} input={}".format(compute_checksum(output, 16), compute_checksum(input, 16)) 1723 write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) 1724 write("\n") 1725 1726 def write(self, text): 1727 self.f.write(text) 1728 1729 1730class BufferSeries: 1731 """ 1732 Behaves like a "defaultlist". 1733 When you ask for an index that doesn't exist yet, 1734 the object grows the list until that item exists. 1735 So o[n] will always work. 1736 1737 Supports negative indices for actual items. 1738 e.g. o[-1] is an element immediately preceding o[0]. 1739 """ 1740 1741 def __init__(self): 1742 self._start = 0 1743 self._array = [] 1744 self._constructor = _text_accumulator 1745 1746 def __getitem__(self, i): 1747 i -= self._start 1748 if i < 0: 1749 self._start += i 1750 prefix = [self._constructor() for x in range(-i)] 1751 self._array = prefix + self._array 1752 i = 0 1753 while i >= len(self._array): 1754 self._array.append(self._constructor()) 1755 return self._array[i] 1756 1757 def clear(self): 1758 for ta in self._array: 1759 ta._text.clear() 1760 1761 def dump(self): 1762 texts = [ta.output() for ta in self._array] 1763 return "".join(texts) 1764 1765 1766class Destination: 1767 def __init__(self, name, type, clinic, *args): 1768 self.name = name 1769 self.type = type 1770 self.clinic = clinic 1771 valid_types = ('buffer', 'file', 'suppress') 1772 if type not in valid_types: 1773 fail("Invalid destination type " + repr(type) + " for " + name + " , must be " + ', '.join(valid_types)) 1774 extra_arguments = 1 if type == "file" else 0 1775 if len(args) < extra_arguments: 1776 fail("Not enough arguments for destination " + name + " new " + type) 1777 if len(args) > extra_arguments: 1778 fail("Too many arguments for destination " + name + " new " + type) 1779 if type =='file': 1780 d = {} 1781 filename = clinic.filename 1782 d['path'] = filename 1783 dirname, basename = os.path.split(filename) 1784 if not dirname: 1785 dirname = '.' 1786 d['dirname'] = dirname 1787 d['basename'] = basename 1788 d['basename_root'], d['basename_extension'] = os.path.splitext(filename) 1789 self.filename = args[0].format_map(d) 1790 1791 self.buffers = BufferSeries() 1792 1793 def __repr__(self): 1794 if self.type == 'file': 1795 file_repr = " " + repr(self.filename) 1796 else: 1797 file_repr = '' 1798 return "".join(("<Destination ", self.name, " ", self.type, file_repr, ">")) 1799 1800 def clear(self): 1801 if self.type != 'buffer': 1802 fail("Can't clear destination" + self.name + " , it's not of type buffer") 1803 self.buffers.clear() 1804 1805 def dump(self): 1806 return self.buffers.dump() 1807 1808 1809# maps strings to Language objects. 1810# "languages" maps the name of the language ("C", "Python"). 1811# "extensions" maps the file extension ("c", "py"). 1812languages = { 'C': CLanguage, 'Python': PythonLanguage } 1813extensions = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } 1814extensions['py'] = PythonLanguage 1815 1816 1817# maps strings to callables. 1818# these callables must be of the form: 1819# def foo(name, default, *, ...) 1820# The callable may have any number of keyword-only parameters. 1821# The callable must return a CConverter object. 1822# The callable should not call builtins.print. 1823converters = {} 1824 1825# maps strings to callables. 1826# these callables follow the same rules as those for "converters" above. 1827# note however that they will never be called with keyword-only parameters. 1828legacy_converters = {} 1829 1830 1831# maps strings to callables. 1832# these callables must be of the form: 1833# def foo(*, ...) 1834# The callable may have any number of keyword-only parameters. 1835# The callable must return a CConverter object. 1836# The callable should not call builtins.print. 1837return_converters = {} 1838 1839 1840def write_file(filename, new_contents): 1841 try: 1842 with open(filename, 'r', encoding="utf-8") as fp: 1843 old_contents = fp.read() 1844 1845 if old_contents == new_contents: 1846 # no change: avoid modifying the file modification time 1847 return 1848 except FileNotFoundError: 1849 pass 1850 1851 # Atomic write using a temporary file and os.replace() 1852 filename_new = f"{filename}.new" 1853 with open(filename_new, "w", encoding="utf-8") as fp: 1854 fp.write(new_contents) 1855 1856 try: 1857 os.replace(filename_new, filename) 1858 except: 1859 os.unlink(filename_new) 1860 raise 1861 1862 1863clinic = None 1864class Clinic: 1865 1866 presets_text = """ 1867preset block 1868everything block 1869methoddef_ifndef buffer 1 1870docstring_prototype suppress 1871parser_prototype suppress 1872cpp_if suppress 1873cpp_endif suppress 1874 1875preset original 1876everything block 1877methoddef_ifndef buffer 1 1878docstring_prototype suppress 1879parser_prototype suppress 1880cpp_if suppress 1881cpp_endif suppress 1882 1883preset file 1884everything file 1885methoddef_ifndef file 1 1886docstring_prototype suppress 1887parser_prototype suppress 1888impl_definition block 1889 1890preset buffer 1891everything buffer 1892methoddef_ifndef buffer 1 1893impl_definition block 1894docstring_prototype suppress 1895impl_prototype suppress 1896parser_prototype suppress 1897 1898preset partial-buffer 1899everything buffer 1900methoddef_ifndef buffer 1 1901docstring_prototype block 1902impl_prototype suppress 1903methoddef_define block 1904parser_prototype block 1905impl_definition block 1906 1907""" 1908 1909 def __init__(self, language, printer=None, *, verify=True, filename=None): 1910 # maps strings to Parser objects. 1911 # (instantiated from the "parsers" global.) 1912 self.parsers = {} 1913 self.language = language 1914 if printer: 1915 fail("Custom printers are broken right now") 1916 self.printer = printer or BlockPrinter(language) 1917 self.verify = verify 1918 self.filename = filename 1919 self.modules = collections.OrderedDict() 1920 self.classes = collections.OrderedDict() 1921 self.functions = [] 1922 1923 self.line_prefix = self.line_suffix = '' 1924 1925 self.destinations = {} 1926 self.add_destination("block", "buffer") 1927 self.add_destination("suppress", "suppress") 1928 self.add_destination("buffer", "buffer") 1929 if filename: 1930 self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") 1931 1932 d = self.get_destination_buffer 1933 self.destination_buffers = collections.OrderedDict(( 1934 ('cpp_if', d('file')), 1935 ('docstring_prototype', d('suppress')), 1936 ('docstring_definition', d('file')), 1937 ('methoddef_define', d('file')), 1938 ('impl_prototype', d('file')), 1939 ('parser_prototype', d('suppress')), 1940 ('parser_definition', d('file')), 1941 ('cpp_endif', d('file')), 1942 ('methoddef_ifndef', d('file', 1)), 1943 ('impl_definition', d('block')), 1944 )) 1945 1946 self.destination_buffers_stack = [] 1947 self.ifndef_symbols = set() 1948 1949 self.presets = {} 1950 preset = None 1951 for line in self.presets_text.strip().split('\n'): 1952 line = line.strip() 1953 if not line: 1954 continue 1955 name, value, *options = line.split() 1956 if name == 'preset': 1957 self.presets[value] = preset = collections.OrderedDict() 1958 continue 1959 1960 if len(options): 1961 index = int(options[0]) 1962 else: 1963 index = 0 1964 buffer = self.get_destination_buffer(value, index) 1965 1966 if name == 'everything': 1967 for name in self.destination_buffers: 1968 preset[name] = buffer 1969 continue 1970 1971 assert name in self.destination_buffers 1972 preset[name] = buffer 1973 1974 global clinic 1975 clinic = self 1976 1977 def add_destination(self, name, type, *args): 1978 if name in self.destinations: 1979 fail("Destination already exists: " + repr(name)) 1980 self.destinations[name] = Destination(name, type, self, *args) 1981 1982 def get_destination(self, name): 1983 d = self.destinations.get(name) 1984 if not d: 1985 fail("Destination does not exist: " + repr(name)) 1986 return d 1987 1988 def get_destination_buffer(self, name, item=0): 1989 d = self.get_destination(name) 1990 return d.buffers[item] 1991 1992 def parse(self, input): 1993 printer = self.printer 1994 self.block_parser = BlockParser(input, self.language, verify=self.verify) 1995 for block in self.block_parser: 1996 dsl_name = block.dsl_name 1997 if dsl_name: 1998 if dsl_name not in self.parsers: 1999 assert dsl_name in parsers, "No parser to handle {!r} block.".format(dsl_name) 2000 self.parsers[dsl_name] = parsers[dsl_name](self) 2001 parser = self.parsers[dsl_name] 2002 try: 2003 parser.parse(block) 2004 except Exception: 2005 fail('Exception raised during parsing:\n' + 2006 traceback.format_exc().rstrip()) 2007 printer.print_block(block) 2008 2009 second_pass_replacements = {} 2010 2011 # these are destinations not buffers 2012 for name, destination in self.destinations.items(): 2013 if destination.type == 'suppress': 2014 continue 2015 output = destination.dump() 2016 2017 if output: 2018 2019 block = Block("", dsl_name="clinic", output=output) 2020 2021 if destination.type == 'buffer': 2022 block.input = "dump " + name + "\n" 2023 warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") 2024 printer.write("\n") 2025 printer.print_block(block) 2026 continue 2027 2028 if destination.type == 'file': 2029 try: 2030 dirname = os.path.dirname(destination.filename) 2031 try: 2032 os.makedirs(dirname) 2033 except FileExistsError: 2034 if not os.path.isdir(dirname): 2035 fail("Can't write to destination {}, " 2036 "can't make directory {}!".format( 2037 destination.filename, dirname)) 2038 if self.verify: 2039 with open(destination.filename, "rt") as f: 2040 parser_2 = BlockParser(f.read(), language=self.language) 2041 blocks = list(parser_2) 2042 if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): 2043 fail("Modified destination file " + repr(destination.filename) + ", not overwriting!") 2044 except FileNotFoundError: 2045 pass 2046 2047 block.input = 'preserve\n' 2048 printer_2 = BlockPrinter(self.language) 2049 printer_2.print_block(block) 2050 write_file(destination.filename, printer_2.f.getvalue()) 2051 continue 2052 text = printer.f.getvalue() 2053 2054 if second_pass_replacements: 2055 printer_2 = BlockPrinter(self.language) 2056 parser_2 = BlockParser(text, self.language) 2057 changed = False 2058 for block in parser_2: 2059 if block.dsl_name: 2060 for id, replacement in second_pass_replacements.items(): 2061 if id in block.output: 2062 changed = True 2063 block.output = block.output.replace(id, replacement) 2064 printer_2.print_block(block) 2065 if changed: 2066 text = printer_2.f.getvalue() 2067 2068 return text 2069 2070 2071 def _module_and_class(self, fields): 2072 """ 2073 fields should be an iterable of field names. 2074 returns a tuple of (module, class). 2075 the module object could actually be self (a clinic object). 2076 this function is only ever used to find the parent of where 2077 a new class/module should go. 2078 """ 2079 in_classes = False 2080 parent = module = self 2081 cls = None 2082 so_far = [] 2083 2084 for field in fields: 2085 so_far.append(field) 2086 if not in_classes: 2087 child = parent.modules.get(field) 2088 if child: 2089 parent = module = child 2090 continue 2091 in_classes = True 2092 if not hasattr(parent, 'classes'): 2093 return module, cls 2094 child = parent.classes.get(field) 2095 if not child: 2096 fail('Parent class or module ' + '.'.join(so_far) + " does not exist.") 2097 cls = parent = child 2098 2099 return module, cls 2100 2101 2102def parse_file(filename, *, verify=True, output=None): 2103 if not output: 2104 output = filename 2105 2106 extension = os.path.splitext(filename)[1][1:] 2107 if not extension: 2108 fail("Can't extract file type for file " + repr(filename)) 2109 2110 try: 2111 language = extensions[extension](filename) 2112 except KeyError: 2113 fail("Can't identify file type for file " + repr(filename)) 2114 2115 with open(filename, 'r', encoding="utf-8") as f: 2116 raw = f.read() 2117 2118 # exit quickly if there are no clinic markers in the file 2119 find_start_re = BlockParser("", language).find_start_re 2120 if not find_start_re.search(raw): 2121 return 2122 2123 clinic = Clinic(language, verify=verify, filename=filename) 2124 cooked = clinic.parse(raw) 2125 2126 write_file(output, cooked) 2127 2128 2129def compute_checksum(input, length=None): 2130 input = input or '' 2131 s = hashlib.sha1(input.encode('utf-8')).hexdigest() 2132 if length: 2133 s = s[:length] 2134 return s 2135 2136 2137 2138 2139class PythonParser: 2140 def __init__(self, clinic): 2141 pass 2142 2143 def parse(self, block): 2144 s = io.StringIO() 2145 with OverrideStdioWith(s): 2146 exec(block.input) 2147 block.output = s.getvalue() 2148 2149 2150class Module: 2151 def __init__(self, name, module=None): 2152 self.name = name 2153 self.module = self.parent = module 2154 2155 self.modules = collections.OrderedDict() 2156 self.classes = collections.OrderedDict() 2157 self.functions = [] 2158 2159 def __repr__(self): 2160 return "<clinic.Module " + repr(self.name) + " at " + str(id(self)) + ">" 2161 2162class Class: 2163 def __init__(self, name, module=None, cls=None, typedef=None, type_object=None): 2164 self.name = name 2165 self.module = module 2166 self.cls = cls 2167 self.typedef = typedef 2168 self.type_object = type_object 2169 self.parent = cls or module 2170 2171 self.classes = collections.OrderedDict() 2172 self.functions = [] 2173 2174 def __repr__(self): 2175 return "<clinic.Class " + repr(self.name) + " at " + str(id(self)) + ">" 2176 2177unsupported_special_methods = set(""" 2178 2179__abs__ 2180__add__ 2181__and__ 2182__call__ 2183__delitem__ 2184__divmod__ 2185__eq__ 2186__float__ 2187__floordiv__ 2188__ge__ 2189__getattr__ 2190__getattribute__ 2191__getitem__ 2192__gt__ 2193__hash__ 2194__iadd__ 2195__iand__ 2196__ifloordiv__ 2197__ilshift__ 2198__imatmul__ 2199__imod__ 2200__imul__ 2201__index__ 2202__int__ 2203__invert__ 2204__ior__ 2205__ipow__ 2206__irshift__ 2207__isub__ 2208__iter__ 2209__itruediv__ 2210__ixor__ 2211__le__ 2212__len__ 2213__lshift__ 2214__lt__ 2215__matmul__ 2216__mod__ 2217__mul__ 2218__neg__ 2219__next__ 2220__or__ 2221__pos__ 2222__pow__ 2223__radd__ 2224__rand__ 2225__rdivmod__ 2226__repr__ 2227__rfloordiv__ 2228__rlshift__ 2229__rmatmul__ 2230__rmod__ 2231__rmul__ 2232__ror__ 2233__rpow__ 2234__rrshift__ 2235__rshift__ 2236__rsub__ 2237__rtruediv__ 2238__rxor__ 2239__setattr__ 2240__setitem__ 2241__str__ 2242__sub__ 2243__truediv__ 2244__xor__ 2245 2246""".strip().split()) 2247 2248 2249INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW = """ 2250INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW 2251""".replace(",", "").strip().split() 2252 2253class Function: 2254 """ 2255 Mutable duck type for inspect.Function. 2256 2257 docstring - a str containing 2258 * embedded line breaks 2259 * text outdented to the left margin 2260 * no trailing whitespace. 2261 It will always be true that 2262 (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) 2263 """ 2264 2265 def __init__(self, parameters=None, *, name, 2266 module, cls=None, c_basename=None, 2267 full_name=None, 2268 return_converter, return_annotation=inspect.Signature.empty, 2269 docstring=None, kind=CALLABLE, coexist=False, 2270 docstring_only=False): 2271 self.parameters = parameters or collections.OrderedDict() 2272 self.return_annotation = return_annotation 2273 self.name = name 2274 self.full_name = full_name 2275 self.module = module 2276 self.cls = cls 2277 self.parent = cls or module 2278 self.c_basename = c_basename 2279 self.return_converter = return_converter 2280 self.docstring = docstring or '' 2281 self.kind = kind 2282 self.coexist = coexist 2283 self.self_converter = None 2284 # docstring_only means "don't generate a machine-readable 2285 # signature, just a normal docstring". it's True for 2286 # functions with optional groups because we can't represent 2287 # those accurately with inspect.Signature in 3.4. 2288 self.docstring_only = docstring_only 2289 2290 self.rendered_parameters = None 2291 2292 __render_parameters__ = None 2293 @property 2294 def render_parameters(self): 2295 if not self.__render_parameters__: 2296 self.__render_parameters__ = l = [] 2297 for p in self.parameters.values(): 2298 p = p.copy() 2299 p.converter.pre_render() 2300 l.append(p) 2301 return self.__render_parameters__ 2302 2303 @property 2304 def methoddef_flags(self): 2305 if self.kind in (METHOD_INIT, METHOD_NEW): 2306 return None 2307 flags = [] 2308 if self.kind == CLASS_METHOD: 2309 flags.append('METH_CLASS') 2310 elif self.kind == STATIC_METHOD: 2311 flags.append('METH_STATIC') 2312 else: 2313 assert self.kind == CALLABLE, "unknown kind: " + repr(self.kind) 2314 if self.coexist: 2315 flags.append('METH_COEXIST') 2316 return '|'.join(flags) 2317 2318 def __repr__(self): 2319 return '<clinic.Function ' + self.name + '>' 2320 2321 def copy(self, **overrides): 2322 kwargs = { 2323 'name': self.name, 'module': self.module, 'parameters': self.parameters, 2324 'cls': self.cls, 'c_basename': self.c_basename, 2325 'full_name': self.full_name, 2326 'return_converter': self.return_converter, 'return_annotation': self.return_annotation, 2327 'docstring': self.docstring, 'kind': self.kind, 'coexist': self.coexist, 2328 'docstring_only': self.docstring_only, 2329 } 2330 kwargs.update(overrides) 2331 f = Function(**kwargs) 2332 2333 parameters = collections.OrderedDict() 2334 for name, value in f.parameters.items(): 2335 value = value.copy(function=f) 2336 parameters[name] = value 2337 f.parameters = parameters 2338 return f 2339 2340 2341class Parameter: 2342 """ 2343 Mutable duck type of inspect.Parameter. 2344 """ 2345 2346 def __init__(self, name, kind, *, default=inspect.Parameter.empty, 2347 function, converter, annotation=inspect.Parameter.empty, 2348 docstring=None, group=0): 2349 self.name = name 2350 self.kind = kind 2351 self.default = default 2352 self.function = function 2353 self.converter = converter 2354 self.annotation = annotation 2355 self.docstring = docstring or '' 2356 self.group = group 2357 2358 def __repr__(self): 2359 return '<clinic.Parameter ' + self.name + '>' 2360 2361 def is_keyword_only(self): 2362 return self.kind == inspect.Parameter.KEYWORD_ONLY 2363 2364 def is_positional_only(self): 2365 return self.kind == inspect.Parameter.POSITIONAL_ONLY 2366 2367 def is_vararg(self): 2368 return self.kind == inspect.Parameter.VAR_POSITIONAL 2369 2370 def is_optional(self): 2371 return not self.is_vararg() and (self.default is not unspecified) 2372 2373 def copy(self, **overrides): 2374 kwargs = { 2375 'name': self.name, 'kind': self.kind, 'default':self.default, 2376 'function': self.function, 'converter': self.converter, 'annotation': self.annotation, 2377 'docstring': self.docstring, 'group': self.group, 2378 } 2379 kwargs.update(overrides) 2380 if 'converter' not in overrides: 2381 converter = copy.copy(self.converter) 2382 converter.function = kwargs['function'] 2383 kwargs['converter'] = converter 2384 return Parameter(**kwargs) 2385 2386 def get_displayname(self, i): 2387 if i == 0: 2388 return '"argument"' 2389 if not self.is_positional_only(): 2390 return '''"argument '{}'"'''.format(self.name) 2391 else: 2392 return '"argument {}"'.format(i) 2393 2394 2395class LandMine: 2396 # try to access any 2397 def __init__(self, message): 2398 self.__message__ = message 2399 2400 def __repr__(self): 2401 return '<LandMine ' + repr(self.__message__) + ">" 2402 2403 def __getattribute__(self, name): 2404 if name in ('__repr__', '__message__'): 2405 return super().__getattribute__(name) 2406 # raise RuntimeError(repr(name)) 2407 fail("Stepped on a land mine, trying to access attribute " + repr(name) + ":\n" + self.__message__) 2408 2409 2410def add_c_converter(f, name=None): 2411 if not name: 2412 name = f.__name__ 2413 if not name.endswith('_converter'): 2414 return f 2415 name = name[:-len('_converter')] 2416 converters[name] = f 2417 return f 2418 2419def add_default_legacy_c_converter(cls): 2420 # automatically add converter for default format unit 2421 # (but without stomping on the existing one if it's already 2422 # set, in case you subclass) 2423 if ((cls.format_unit not in ('O&', '')) and 2424 (cls.format_unit not in legacy_converters)): 2425 legacy_converters[cls.format_unit] = cls 2426 return cls 2427 2428def add_legacy_c_converter(format_unit, **kwargs): 2429 """ 2430 Adds a legacy converter. 2431 """ 2432 def closure(f): 2433 if not kwargs: 2434 added_f = f 2435 else: 2436 added_f = functools.partial(f, **kwargs) 2437 if format_unit: 2438 legacy_converters[format_unit] = added_f 2439 return f 2440 return closure 2441 2442class CConverterAutoRegister(type): 2443 def __init__(cls, name, bases, classdict): 2444 add_c_converter(cls) 2445 add_default_legacy_c_converter(cls) 2446 2447class CConverter(metaclass=CConverterAutoRegister): 2448 """ 2449 For the init function, self, name, function, and default 2450 must be keyword-or-positional parameters. All other 2451 parameters must be keyword-only. 2452 """ 2453 2454 # The C name to use for this variable. 2455 name = None 2456 2457 # The Python name to use for this variable. 2458 py_name = None 2459 2460 # The C type to use for this variable. 2461 # 'type' should be a Python string specifying the type, e.g. "int". 2462 # If this is a pointer type, the type string should end with ' *'. 2463 type = None 2464 2465 # The Python default value for this parameter, as a Python value. 2466 # Or the magic value "unspecified" if there is no default. 2467 # Or the magic value "unknown" if this value is a cannot be evaluated 2468 # at Argument-Clinic-preprocessing time (but is presumed to be valid 2469 # at runtime). 2470 default = unspecified 2471 2472 # If not None, default must be isinstance() of this type. 2473 # (You can also specify a tuple of types.) 2474 default_type = None 2475 2476 # "default" converted into a C value, as a string. 2477 # Or None if there is no default. 2478 c_default = None 2479 2480 # "default" converted into a Python value, as a string. 2481 # Or None if there is no default. 2482 py_default = None 2483 2484 # The default value used to initialize the C variable when 2485 # there is no default, but not specifying a default may 2486 # result in an "uninitialized variable" warning. This can 2487 # easily happen when using option groups--although 2488 # properly-written code won't actually use the variable, 2489 # the variable does get passed in to the _impl. (Ah, if 2490 # only dataflow analysis could inline the static function!) 2491 # 2492 # This value is specified as a string. 2493 # Every non-abstract subclass should supply a valid value. 2494 c_ignored_default = 'NULL' 2495 2496 # The C converter *function* to be used, if any. 2497 # (If this is not None, format_unit must be 'O&'.) 2498 converter = None 2499 2500 # Should Argument Clinic add a '&' before the name of 2501 # the variable when passing it into the _impl function? 2502 impl_by_reference = False 2503 2504 # Should Argument Clinic add a '&' before the name of 2505 # the variable when passing it into PyArg_ParseTuple (AndKeywords)? 2506 parse_by_reference = True 2507 2508 ############################################################# 2509 ############################################################# 2510 ## You shouldn't need to read anything below this point to ## 2511 ## write your own converter functions. ## 2512 ############################################################# 2513 ############################################################# 2514 2515 # The "format unit" to specify for this variable when 2516 # parsing arguments using PyArg_ParseTuple (AndKeywords). 2517 # Custom converters should always use the default value of 'O&'. 2518 format_unit = 'O&' 2519 2520 # What encoding do we want for this variable? Only used 2521 # by format units starting with 'e'. 2522 encoding = None 2523 2524 # Should this object be required to be a subclass of a specific type? 2525 # If not None, should be a string representing a pointer to a 2526 # PyTypeObject (e.g. "&PyUnicode_Type"). 2527 # Only used by the 'O!' format unit (and the "object" converter). 2528 subclass_of = None 2529 2530 # Do we want an adjacent '_length' variable for this variable? 2531 # Only used by format units ending with '#'. 2532 length = False 2533 2534 # Should we show this parameter in the generated 2535 # __text_signature__? This is *almost* always True. 2536 # (It's only False for __new__, __init__, and METH_STATIC functions.) 2537 show_in_signature = True 2538 2539 # Overrides the name used in a text signature. 2540 # The name used for a "self" parameter must be one of 2541 # self, type, or module; however users can set their own. 2542 # This lets the self_converter overrule the user-settable 2543 # name, *just* for the text signature. 2544 # Only set by self_converter. 2545 signature_name = None 2546 2547 # keep in sync with self_converter.__init__! 2548 def __init__(self, name, py_name, function, default=unspecified, *, c_default=None, py_default=None, annotation=unspecified, **kwargs): 2549 self.name = ensure_legal_c_identifier(name) 2550 self.py_name = py_name 2551 2552 if default is not unspecified: 2553 if self.default_type and not isinstance(default, (self.default_type, Unknown)): 2554 if isinstance(self.default_type, type): 2555 types_str = self.default_type.__name__ 2556 else: 2557 types_str = ', '.join((cls.__name__ for cls in self.default_type)) 2558 fail("{}: default value {!r} for field {} is not of type {}".format( 2559 self.__class__.__name__, default, name, types_str)) 2560 self.default = default 2561 2562 if c_default: 2563 self.c_default = c_default 2564 if py_default: 2565 self.py_default = py_default 2566 2567 if annotation != unspecified: 2568 fail("The 'annotation' parameter is not currently permitted.") 2569 2570 # this is deliberate, to prevent you from caching information 2571 # about the function in the init. 2572 # (that breaks if we get cloned.) 2573 # so after this change we will noisily fail. 2574 self.function = LandMine("Don't access members of self.function inside converter_init!") 2575 self.converter_init(**kwargs) 2576 self.function = function 2577 2578 def converter_init(self): 2579 pass 2580 2581 def is_optional(self): 2582 return (self.default is not unspecified) 2583 2584 def _render_self(self, parameter, data): 2585 self.parameter = parameter 2586 name = self.parser_name 2587 2588 # impl_arguments 2589 s = ("&" if self.impl_by_reference else "") + name 2590 data.impl_arguments.append(s) 2591 if self.length: 2592 data.impl_arguments.append(self.length_name()) 2593 2594 # impl_parameters 2595 data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) 2596 if self.length: 2597 data.impl_parameters.append("Py_ssize_t " + self.length_name()) 2598 2599 def _render_non_self(self, parameter, data): 2600 self.parameter = parameter 2601 name = self.name 2602 2603 # declarations 2604 d = self.declaration(in_parser=True) 2605 data.declarations.append(d) 2606 2607 # initializers 2608 initializers = self.initialize() 2609 if initializers: 2610 data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) 2611 2612 # modifications 2613 modifications = self.modify() 2614 if modifications: 2615 data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) 2616 2617 # keywords 2618 if parameter.is_vararg(): 2619 pass 2620 elif parameter.is_positional_only(): 2621 data.keywords.append('') 2622 else: 2623 data.keywords.append(parameter.name) 2624 2625 # format_units 2626 if self.is_optional() and '|' not in data.format_units: 2627 data.format_units.append('|') 2628 if parameter.is_keyword_only() and '$' not in data.format_units: 2629 data.format_units.append('$') 2630 data.format_units.append(self.format_unit) 2631 2632 # parse_arguments 2633 self.parse_argument(data.parse_arguments) 2634 2635 # cleanup 2636 cleanup = self.cleanup() 2637 if cleanup: 2638 data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") 2639 2640 def render(self, parameter, data): 2641 """ 2642 parameter is a clinic.Parameter instance. 2643 data is a CRenderData instance. 2644 """ 2645 self._render_self(parameter, data) 2646 self._render_non_self(parameter, data) 2647 2648 def length_name(self): 2649 """Computes the name of the associated "length" variable.""" 2650 if not self.length: 2651 return None 2652 return self.parser_name + "_length" 2653 2654 # Why is this one broken out separately? 2655 # For "positional-only" function parsing, 2656 # which generates a bunch of PyArg_ParseTuple calls. 2657 def parse_argument(self, list): 2658 assert not (self.converter and self.encoding) 2659 if self.format_unit == 'O&': 2660 assert self.converter 2661 list.append(self.converter) 2662 2663 if self.encoding: 2664 list.append(c_repr(self.encoding)) 2665 elif self.subclass_of: 2666 list.append(self.subclass_of) 2667 2668 s = ("&" if self.parse_by_reference else "") + self.name 2669 list.append(s) 2670 2671 if self.length: 2672 list.append("&" + self.length_name()) 2673 2674 # 2675 # All the functions after here are intended as extension points. 2676 # 2677 2678 def simple_declaration(self, by_reference=False, *, in_parser=False): 2679 """ 2680 Computes the basic declaration of the variable. 2681 Used in computing the prototype declaration and the 2682 variable declaration. 2683 """ 2684 prototype = [self.type] 2685 if by_reference or not self.type.endswith('*'): 2686 prototype.append(" ") 2687 if by_reference: 2688 prototype.append('*') 2689 if in_parser: 2690 name = self.parser_name 2691 else: 2692 name = self.name 2693 prototype.append(name) 2694 return "".join(prototype) 2695 2696 def declaration(self, *, in_parser=False): 2697 """ 2698 The C statement to declare this variable. 2699 """ 2700 declaration = [self.simple_declaration(in_parser=True)] 2701 default = self.c_default 2702 if not default and self.parameter.group: 2703 default = self.c_ignored_default 2704 if default: 2705 declaration.append(" = ") 2706 declaration.append(default) 2707 declaration.append(";") 2708 if self.length: 2709 declaration.append('\nPy_ssize_t ') 2710 declaration.append(self.length_name()) 2711 declaration.append(';') 2712 return "".join(declaration) 2713 2714 def initialize(self): 2715 """ 2716 The C statements required to set up this variable before parsing. 2717 Returns a string containing this code indented at column 0. 2718 If no initialization is necessary, returns an empty string. 2719 """ 2720 return "" 2721 2722 def modify(self): 2723 """ 2724 The C statements required to modify this variable after parsing. 2725 Returns a string containing this code indented at column 0. 2726 If no initialization is necessary, returns an empty string. 2727 """ 2728 return "" 2729 2730 def cleanup(self): 2731 """ 2732 The C statements required to clean up after this variable. 2733 Returns a string containing this code indented at column 0. 2734 If no cleanup is necessary, returns an empty string. 2735 """ 2736 return "" 2737 2738 def pre_render(self): 2739 """ 2740 A second initialization function, like converter_init, 2741 called just before rendering. 2742 You are permitted to examine self.function here. 2743 """ 2744 pass 2745 2746 def parse_arg(self, argname, displayname): 2747 if self.format_unit == 'O&': 2748 return """ 2749 if (!{converter}({argname}, &{paramname})) {{{{ 2750 goto exit; 2751 }}}} 2752 """.format(argname=argname, paramname=self.parser_name, 2753 converter=self.converter) 2754 if self.format_unit == 'O!': 2755 cast = '(%s)' % self.type if self.type != 'PyObject *' else '' 2756 if self.subclass_of in type_checks: 2757 typecheck, typename = type_checks[self.subclass_of] 2758 return """ 2759 if (!{typecheck}({argname})) {{{{ 2760 _PyArg_BadArgument("{{name}}", {displayname}, "{typename}", {argname}); 2761 goto exit; 2762 }}}} 2763 {paramname} = {cast}{argname}; 2764 """.format(argname=argname, paramname=self.parser_name, 2765 displayname=displayname, typecheck=typecheck, 2766 typename=typename, cast=cast) 2767 return """ 2768 if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ 2769 _PyArg_BadArgument("{{name}}", {displayname}, ({subclass_of})->tp_name, {argname}); 2770 goto exit; 2771 }}}} 2772 {paramname} = {cast}{argname}; 2773 """.format(argname=argname, paramname=self.parser_name, 2774 subclass_of=self.subclass_of, cast=cast, 2775 displayname=displayname) 2776 if self.format_unit == 'O': 2777 cast = '(%s)' % self.type if self.type != 'PyObject *' else '' 2778 return """ 2779 {paramname} = {cast}{argname}; 2780 """.format(argname=argname, paramname=self.parser_name, cast=cast) 2781 return None 2782 2783 def set_template_dict(self, template_dict): 2784 pass 2785 2786 @property 2787 def parser_name(self): 2788 if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741 2789 return CLINIC_PREFIX + self.name 2790 else: 2791 return self.name 2792 2793type_checks = { 2794 '&PyLong_Type': ('PyLong_Check', 'int'), 2795 '&PyTuple_Type': ('PyTuple_Check', 'tuple'), 2796 '&PyList_Type': ('PyList_Check', 'list'), 2797 '&PySet_Type': ('PySet_Check', 'set'), 2798 '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), 2799 '&PyDict_Type': ('PyDict_Check', 'dict'), 2800 '&PyUnicode_Type': ('PyUnicode_Check', 'str'), 2801 '&PyBytes_Type': ('PyBytes_Check', 'bytes'), 2802 '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), 2803} 2804 2805 2806class bool_converter(CConverter): 2807 type = 'int' 2808 default_type = bool 2809 format_unit = 'p' 2810 c_ignored_default = '0' 2811 2812 def converter_init(self, *, accept={object}): 2813 if accept == {int}: 2814 self.format_unit = 'i' 2815 elif accept != {object}: 2816 fail("bool_converter: illegal 'accept' argument " + repr(accept)) 2817 if self.default is not unspecified: 2818 self.default = bool(self.default) 2819 self.c_default = str(int(self.default)) 2820 2821 def parse_arg(self, argname, displayname): 2822 if self.format_unit == 'i': 2823 # XXX PyFloat_Check can be removed after the end of the 2824 # deprecation in _PyLong_FromNbIndexOrNbInt. 2825 return """ 2826 {paramname} = _PyLong_AsInt({argname}); 2827 if ({paramname} == -1 && PyErr_Occurred()) {{{{ 2828 goto exit; 2829 }}}} 2830 """.format(argname=argname, paramname=self.parser_name) 2831 elif self.format_unit == 'p': 2832 return """ 2833 {paramname} = PyObject_IsTrue({argname}); 2834 if ({paramname} < 0) {{{{ 2835 goto exit; 2836 }}}} 2837 """.format(argname=argname, paramname=self.parser_name) 2838 return super().parse_arg(argname, displayname) 2839 2840class defining_class_converter(CConverter): 2841 """ 2842 A special-case converter: 2843 this is the default converter used for the defining class. 2844 """ 2845 type = 'PyTypeObject *' 2846 format_unit = '' 2847 show_in_signature = False 2848 2849 def converter_init(self, *, type=None): 2850 self.specified_type = type 2851 2852 def render(self, parameter, data): 2853 self._render_self(parameter, data) 2854 2855 def set_template_dict(self, template_dict): 2856 template_dict['defining_class_name'] = self.name 2857 2858 2859class char_converter(CConverter): 2860 type = 'char' 2861 default_type = (bytes, bytearray) 2862 format_unit = 'c' 2863 c_ignored_default = "'\0'" 2864 2865 def converter_init(self): 2866 if isinstance(self.default, self.default_type): 2867 if len(self.default) != 1: 2868 fail("char_converter: illegal default value " + repr(self.default)) 2869 2870 self.c_default = repr(bytes(self.default))[1:] 2871 if self.c_default == '"\'"': 2872 self.c_default = r"'\''" 2873 2874 def parse_arg(self, argname, displayname): 2875 if self.format_unit == 'c': 2876 return """ 2877 if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ 2878 {paramname} = PyBytes_AS_STRING({argname})[0]; 2879 }}}} 2880 else if (PyByteArray_Check({argname}) && PyByteArray_GET_SIZE({argname}) == 1) {{{{ 2881 {paramname} = PyByteArray_AS_STRING({argname})[0]; 2882 }}}} 2883 else {{{{ 2884 _PyArg_BadArgument("{{name}}", {displayname}, "a byte string of length 1", {argname}); 2885 goto exit; 2886 }}}} 2887 """.format(argname=argname, paramname=self.parser_name, 2888 displayname=displayname) 2889 return super().parse_arg(argname, displayname) 2890 2891 2892@add_legacy_c_converter('B', bitwise=True) 2893class unsigned_char_converter(CConverter): 2894 type = 'unsigned char' 2895 default_type = int 2896 format_unit = 'b' 2897 c_ignored_default = "'\0'" 2898 2899 def converter_init(self, *, bitwise=False): 2900 if bitwise: 2901 self.format_unit = 'B' 2902 2903 def parse_arg(self, argname, displayname): 2904 if self.format_unit == 'b': 2905 return """ 2906 {{{{ 2907 long ival = PyLong_AsLong({argname}); 2908 if (ival == -1 && PyErr_Occurred()) {{{{ 2909 goto exit; 2910 }}}} 2911 else if (ival < 0) {{{{ 2912 PyErr_SetString(PyExc_OverflowError, 2913 "unsigned byte integer is less than minimum"); 2914 goto exit; 2915 }}}} 2916 else if (ival > UCHAR_MAX) {{{{ 2917 PyErr_SetString(PyExc_OverflowError, 2918 "unsigned byte integer is greater than maximum"); 2919 goto exit; 2920 }}}} 2921 else {{{{ 2922 {paramname} = (unsigned char) ival; 2923 }}}} 2924 }}}} 2925 """.format(argname=argname, paramname=self.parser_name) 2926 elif self.format_unit == 'B': 2927 return """ 2928 {{{{ 2929 unsigned long ival = PyLong_AsUnsignedLongMask({argname}); 2930 if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ 2931 goto exit; 2932 }}}} 2933 else {{{{ 2934 {paramname} = (unsigned char) ival; 2935 }}}} 2936 }}}} 2937 """.format(argname=argname, paramname=self.parser_name) 2938 return super().parse_arg(argname, displayname) 2939 2940class byte_converter(unsigned_char_converter): pass 2941 2942class short_converter(CConverter): 2943 type = 'short' 2944 default_type = int 2945 format_unit = 'h' 2946 c_ignored_default = "0" 2947 2948 def parse_arg(self, argname, displayname): 2949 if self.format_unit == 'h': 2950 return """ 2951 {{{{ 2952 long ival = PyLong_AsLong({argname}); 2953 if (ival == -1 && PyErr_Occurred()) {{{{ 2954 goto exit; 2955 }}}} 2956 else if (ival < SHRT_MIN) {{{{ 2957 PyErr_SetString(PyExc_OverflowError, 2958 "signed short integer is less than minimum"); 2959 goto exit; 2960 }}}} 2961 else if (ival > SHRT_MAX) {{{{ 2962 PyErr_SetString(PyExc_OverflowError, 2963 "signed short integer is greater than maximum"); 2964 goto exit; 2965 }}}} 2966 else {{{{ 2967 {paramname} = (short) ival; 2968 }}}} 2969 }}}} 2970 """.format(argname=argname, paramname=self.parser_name) 2971 return super().parse_arg(argname, displayname) 2972 2973class unsigned_short_converter(CConverter): 2974 type = 'unsigned short' 2975 default_type = int 2976 c_ignored_default = "0" 2977 2978 def converter_init(self, *, bitwise=False): 2979 if bitwise: 2980 self.format_unit = 'H' 2981 else: 2982 self.converter = '_PyLong_UnsignedShort_Converter' 2983 2984 def parse_arg(self, argname, displayname): 2985 if self.format_unit == 'H': 2986 return """ 2987 {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); 2988 if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ 2989 goto exit; 2990 }}}} 2991 """.format(argname=argname, paramname=self.parser_name) 2992 return super().parse_arg(argname, displayname) 2993 2994@add_legacy_c_converter('C', accept={str}) 2995class int_converter(CConverter): 2996 type = 'int' 2997 default_type = int 2998 format_unit = 'i' 2999 c_ignored_default = "0" 3000 3001 def converter_init(self, *, accept={int}, type=None): 3002 if accept == {str}: 3003 self.format_unit = 'C' 3004 elif accept != {int}: 3005 fail("int_converter: illegal 'accept' argument " + repr(accept)) 3006 if type is not None: 3007 self.type = type 3008 3009 def parse_arg(self, argname, displayname): 3010 if self.format_unit == 'i': 3011 return """ 3012 {paramname} = _PyLong_AsInt({argname}); 3013 if ({paramname} == -1 && PyErr_Occurred()) {{{{ 3014 goto exit; 3015 }}}} 3016 """.format(argname=argname, paramname=self.parser_name) 3017 elif self.format_unit == 'C': 3018 return """ 3019 if (!PyUnicode_Check({argname})) {{{{ 3020 _PyArg_BadArgument("{{name}}", {displayname}, "a unicode character", {argname}); 3021 goto exit; 3022 }}}} 3023 if (PyUnicode_READY({argname})) {{{{ 3024 goto exit; 3025 }}}} 3026 if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ 3027 _PyArg_BadArgument("{{name}}", {displayname}, "a unicode character", {argname}); 3028 goto exit; 3029 }}}} 3030 {paramname} = PyUnicode_READ_CHAR({argname}, 0); 3031 """.format(argname=argname, paramname=self.parser_name, 3032 displayname=displayname) 3033 return super().parse_arg(argname, displayname) 3034 3035class unsigned_int_converter(CConverter): 3036 type = 'unsigned int' 3037 default_type = int 3038 c_ignored_default = "0" 3039 3040 def converter_init(self, *, bitwise=False): 3041 if bitwise: 3042 self.format_unit = 'I' 3043 else: 3044 self.converter = '_PyLong_UnsignedInt_Converter' 3045 3046 def parse_arg(self, argname, displayname): 3047 if self.format_unit == 'I': 3048 return """ 3049 {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); 3050 if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ 3051 goto exit; 3052 }}}} 3053 """.format(argname=argname, paramname=self.parser_name) 3054 return super().parse_arg(argname, displayname) 3055 3056class long_converter(CConverter): 3057 type = 'long' 3058 default_type = int 3059 format_unit = 'l' 3060 c_ignored_default = "0" 3061 3062 def parse_arg(self, argname, displayname): 3063 if self.format_unit == 'l': 3064 return """ 3065 {paramname} = PyLong_AsLong({argname}); 3066 if ({paramname} == -1 && PyErr_Occurred()) {{{{ 3067 goto exit; 3068 }}}} 3069 """.format(argname=argname, paramname=self.parser_name) 3070 return super().parse_arg(argname, displayname) 3071 3072class unsigned_long_converter(CConverter): 3073 type = 'unsigned long' 3074 default_type = int 3075 c_ignored_default = "0" 3076 3077 def converter_init(self, *, bitwise=False): 3078 if bitwise: 3079 self.format_unit = 'k' 3080 else: 3081 self.converter = '_PyLong_UnsignedLong_Converter' 3082 3083 def parse_arg(self, argname, displayname): 3084 if self.format_unit == 'k': 3085 return """ 3086 if (!PyLong_Check({argname})) {{{{ 3087 _PyArg_BadArgument("{{name}}", {displayname}, "int", {argname}); 3088 goto exit; 3089 }}}} 3090 {paramname} = PyLong_AsUnsignedLongMask({argname}); 3091 """.format(argname=argname, paramname=self.parser_name, 3092 displayname=displayname) 3093 return super().parse_arg(argname, displayname) 3094 3095class long_long_converter(CConverter): 3096 type = 'long long' 3097 default_type = int 3098 format_unit = 'L' 3099 c_ignored_default = "0" 3100 3101 def parse_arg(self, argname, displayname): 3102 if self.format_unit == 'L': 3103 return """ 3104 {paramname} = PyLong_AsLongLong({argname}); 3105 if ({paramname} == -1 && PyErr_Occurred()) {{{{ 3106 goto exit; 3107 }}}} 3108 """.format(argname=argname, paramname=self.parser_name) 3109 return super().parse_arg(argname, displayname) 3110 3111class unsigned_long_long_converter(CConverter): 3112 type = 'unsigned long long' 3113 default_type = int 3114 c_ignored_default = "0" 3115 3116 def converter_init(self, *, bitwise=False): 3117 if bitwise: 3118 self.format_unit = 'K' 3119 else: 3120 self.converter = '_PyLong_UnsignedLongLong_Converter' 3121 3122 def parse_arg(self, argname, displayname): 3123 if self.format_unit == 'K': 3124 return """ 3125 if (!PyLong_Check({argname})) {{{{ 3126 _PyArg_BadArgument("{{name}}", {displayname}, "int", {argname}); 3127 goto exit; 3128 }}}} 3129 {paramname} = PyLong_AsUnsignedLongLongMask({argname}); 3130 """.format(argname=argname, paramname=self.parser_name, 3131 displayname=displayname) 3132 return super().parse_arg(argname, displayname) 3133 3134class Py_ssize_t_converter(CConverter): 3135 type = 'Py_ssize_t' 3136 c_ignored_default = "0" 3137 3138 def converter_init(self, *, accept={int}): 3139 if accept == {int}: 3140 self.format_unit = 'n' 3141 self.default_type = int 3142 elif accept == {int, NoneType}: 3143 self.converter = '_Py_convert_optional_to_ssize_t' 3144 else: 3145 fail("Py_ssize_t_converter: illegal 'accept' argument " + repr(accept)) 3146 3147 def parse_arg(self, argname, displayname): 3148 if self.format_unit == 'n': 3149 return """ 3150 {{{{ 3151 Py_ssize_t ival = -1; 3152 PyObject *iobj = _PyNumber_Index({argname}); 3153 if (iobj != NULL) {{{{ 3154 ival = PyLong_AsSsize_t(iobj); 3155 Py_DECREF(iobj); 3156 }}}} 3157 if (ival == -1 && PyErr_Occurred()) {{{{ 3158 goto exit; 3159 }}}} 3160 {paramname} = ival; 3161 }}}} 3162 """.format(argname=argname, paramname=self.parser_name) 3163 return super().parse_arg(argname, displayname) 3164 3165 3166class slice_index_converter(CConverter): 3167 type = 'Py_ssize_t' 3168 3169 def converter_init(self, *, accept={int, NoneType}): 3170 if accept == {int}: 3171 self.converter = '_PyEval_SliceIndexNotNone' 3172 elif accept == {int, NoneType}: 3173 self.converter = '_PyEval_SliceIndex' 3174 else: 3175 fail("slice_index_converter: illegal 'accept' argument " + repr(accept)) 3176 3177class size_t_converter(CConverter): 3178 type = 'size_t' 3179 converter = '_PyLong_Size_t_Converter' 3180 c_ignored_default = "0" 3181 3182 def parse_arg(self, argname, displayname): 3183 if self.format_unit == 'n': 3184 return """ 3185 {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); 3186 if ({paramname} == -1 && PyErr_Occurred()) {{{{ 3187 goto exit; 3188 }}}} 3189 """.format(argname=argname, paramname=self.parser_name) 3190 return super().parse_arg(argname, displayname) 3191 3192 3193class fildes_converter(CConverter): 3194 type = 'int' 3195 converter = '_PyLong_FileDescriptor_Converter' 3196 3197 def _parse_arg(self, argname, displayname): 3198 return """ 3199 {paramname} = PyObject_AsFileDescriptor({argname}); 3200 if ({paramname} == -1) {{{{ 3201 goto exit; 3202 }}}} 3203 """.format(argname=argname, paramname=self.name) 3204 3205 3206class float_converter(CConverter): 3207 type = 'float' 3208 default_type = float 3209 format_unit = 'f' 3210 c_ignored_default = "0.0" 3211 3212 def parse_arg(self, argname, displayname): 3213 if self.format_unit == 'f': 3214 return """ 3215 if (PyFloat_CheckExact({argname})) {{{{ 3216 {paramname} = (float) (PyFloat_AS_DOUBLE({argname})); 3217 }}}} 3218 else 3219 {{{{ 3220 {paramname} = (float) PyFloat_AsDouble({argname}); 3221 if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ 3222 goto exit; 3223 }}}} 3224 }}}} 3225 """.format(argname=argname, paramname=self.parser_name) 3226 return super().parse_arg(argname, displayname) 3227 3228class double_converter(CConverter): 3229 type = 'double' 3230 default_type = float 3231 format_unit = 'd' 3232 c_ignored_default = "0.0" 3233 3234 def parse_arg(self, argname, displayname): 3235 if self.format_unit == 'd': 3236 return """ 3237 if (PyFloat_CheckExact({argname})) {{{{ 3238 {paramname} = PyFloat_AS_DOUBLE({argname}); 3239 }}}} 3240 else 3241 {{{{ 3242 {paramname} = PyFloat_AsDouble({argname}); 3243 if ({paramname} == -1.0 && PyErr_Occurred()) {{{{ 3244 goto exit; 3245 }}}} 3246 }}}} 3247 """.format(argname=argname, paramname=self.parser_name) 3248 return super().parse_arg(argname, displayname) 3249 3250 3251class Py_complex_converter(CConverter): 3252 type = 'Py_complex' 3253 default_type = complex 3254 format_unit = 'D' 3255 c_ignored_default = "{0.0, 0.0}" 3256 3257 def parse_arg(self, argname, displayname): 3258 if self.format_unit == 'D': 3259 return """ 3260 {paramname} = PyComplex_AsCComplex({argname}); 3261 if (PyErr_Occurred()) {{{{ 3262 goto exit; 3263 }}}} 3264 """.format(argname=argname, paramname=self.parser_name) 3265 return super().parse_arg(argname, displayname) 3266 3267 3268class object_converter(CConverter): 3269 type = 'PyObject *' 3270 format_unit = 'O' 3271 3272 def converter_init(self, *, converter=None, type=None, subclass_of=None): 3273 if converter: 3274 if subclass_of: 3275 fail("object: Cannot pass in both 'converter' and 'subclass_of'") 3276 self.format_unit = 'O&' 3277 self.converter = converter 3278 elif subclass_of: 3279 self.format_unit = 'O!' 3280 self.subclass_of = subclass_of 3281 3282 if type is not None: 3283 self.type = type 3284 3285 3286# 3287# We define three conventions for buffer types in the 'accept' argument: 3288# 3289# buffer : any object supporting the buffer interface 3290# rwbuffer: any object supporting the buffer interface, but must be writeable 3291# robuffer: any object supporting the buffer interface, but must not be writeable 3292# 3293 3294class buffer: pass 3295class rwbuffer: pass 3296class robuffer: pass 3297 3298def str_converter_key(types, encoding, zeroes): 3299 return (frozenset(types), bool(encoding), bool(zeroes)) 3300 3301str_converter_argument_map = {} 3302 3303class str_converter(CConverter): 3304 type = 'const char *' 3305 default_type = (str, Null, NoneType) 3306 format_unit = 's' 3307 3308 def converter_init(self, *, accept={str}, encoding=None, zeroes=False): 3309 3310 key = str_converter_key(accept, encoding, zeroes) 3311 format_unit = str_converter_argument_map.get(key) 3312 if not format_unit: 3313 fail("str_converter: illegal combination of arguments", key) 3314 3315 self.format_unit = format_unit 3316 self.length = bool(zeroes) 3317 if encoding: 3318 if self.default not in (Null, None, unspecified): 3319 fail("str_converter: Argument Clinic doesn't support default values for encoded strings") 3320 self.encoding = encoding 3321 self.type = 'char *' 3322 # sorry, clinic can't support preallocated buffers 3323 # for es# and et# 3324 self.c_default = "NULL" 3325 if NoneType in accept and self.c_default == "Py_None": 3326 self.c_default = "NULL" 3327 3328 def cleanup(self): 3329 if self.encoding: 3330 name = self.name 3331 return "".join(["if (", name, ") {\n PyMem_FREE(", name, ");\n}\n"]) 3332 3333 def parse_arg(self, argname, displayname): 3334 if self.format_unit == 's': 3335 return """ 3336 if (!PyUnicode_Check({argname})) {{{{ 3337 _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); 3338 goto exit; 3339 }}}} 3340 Py_ssize_t {paramname}_length; 3341 {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); 3342 if ({paramname} == NULL) {{{{ 3343 goto exit; 3344 }}}} 3345 if (strlen({paramname}) != (size_t){paramname}_length) {{{{ 3346 PyErr_SetString(PyExc_ValueError, "embedded null character"); 3347 goto exit; 3348 }}}} 3349 """.format(argname=argname, paramname=self.parser_name, 3350 displayname=displayname) 3351 if self.format_unit == 'z': 3352 return """ 3353 if ({argname} == Py_None) {{{{ 3354 {paramname} = NULL; 3355 }}}} 3356 else if (PyUnicode_Check({argname})) {{{{ 3357 Py_ssize_t {paramname}_length; 3358 {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); 3359 if ({paramname} == NULL) {{{{ 3360 goto exit; 3361 }}}} 3362 if (strlen({paramname}) != (size_t){paramname}_length) {{{{ 3363 PyErr_SetString(PyExc_ValueError, "embedded null character"); 3364 goto exit; 3365 }}}} 3366 }}}} 3367 else {{{{ 3368 _PyArg_BadArgument("{{name}}", {displayname}, "str or None", {argname}); 3369 goto exit; 3370 }}}} 3371 """.format(argname=argname, paramname=self.parser_name, 3372 displayname=displayname) 3373 return super().parse_arg(argname, displayname) 3374 3375# 3376# This is the fourth or fifth rewrite of registering all the 3377# string converter format units. Previous approaches hid 3378# bugs--generally mismatches between the semantics of the format 3379# unit and the arguments necessary to represent those semantics 3380# properly. Hopefully with this approach we'll get it 100% right. 3381# 3382# The r() function (short for "register") both registers the 3383# mapping from arguments to format unit *and* registers the 3384# legacy C converter for that format unit. 3385# 3386def r(format_unit, *, accept, encoding=False, zeroes=False): 3387 if not encoding and format_unit != 's': 3388 # add the legacy c converters here too. 3389 # 3390 # note: add_legacy_c_converter can't work for 3391 # es, es#, et, or et# 3392 # because of their extra encoding argument 3393 # 3394 # also don't add the converter for 's' because 3395 # the metaclass for CConverter adds it for us. 3396 kwargs = {} 3397 if accept != {str}: 3398 kwargs['accept'] = accept 3399 if zeroes: 3400 kwargs['zeroes'] = True 3401 added_f = functools.partial(str_converter, **kwargs) 3402 legacy_converters[format_unit] = added_f 3403 3404 d = str_converter_argument_map 3405 key = str_converter_key(accept, encoding, zeroes) 3406 if key in d: 3407 sys.exit("Duplicate keys specified for str_converter_argument_map!") 3408 d[key] = format_unit 3409 3410r('es', encoding=True, accept={str}) 3411r('es#', encoding=True, zeroes=True, accept={str}) 3412r('et', encoding=True, accept={bytes, bytearray, str}) 3413r('et#', encoding=True, zeroes=True, accept={bytes, bytearray, str}) 3414r('s', accept={str}) 3415r('s#', zeroes=True, accept={robuffer, str}) 3416r('y', accept={robuffer}) 3417r('y#', zeroes=True, accept={robuffer}) 3418r('z', accept={str, NoneType}) 3419r('z#', zeroes=True, accept={robuffer, str, NoneType}) 3420del r 3421 3422 3423class PyBytesObject_converter(CConverter): 3424 type = 'PyBytesObject *' 3425 format_unit = 'S' 3426 # accept = {bytes} 3427 3428 def parse_arg(self, argname, displayname): 3429 if self.format_unit == 'S': 3430 return """ 3431 if (!PyBytes_Check({argname})) {{{{ 3432 _PyArg_BadArgument("{{name}}", {displayname}, "bytes", {argname}); 3433 goto exit; 3434 }}}} 3435 {paramname} = ({type}){argname}; 3436 """.format(argname=argname, paramname=self.parser_name, 3437 type=self.type, displayname=displayname) 3438 return super().parse_arg(argname, displayname) 3439 3440class PyByteArrayObject_converter(CConverter): 3441 type = 'PyByteArrayObject *' 3442 format_unit = 'Y' 3443 # accept = {bytearray} 3444 3445 def parse_arg(self, argname, displayname): 3446 if self.format_unit == 'Y': 3447 return """ 3448 if (!PyByteArray_Check({argname})) {{{{ 3449 _PyArg_BadArgument("{{name}}", {displayname}, "bytearray", {argname}); 3450 goto exit; 3451 }}}} 3452 {paramname} = ({type}){argname}; 3453 """.format(argname=argname, paramname=self.parser_name, 3454 type=self.type, displayname=displayname) 3455 return super().parse_arg(argname, displayname) 3456 3457class unicode_converter(CConverter): 3458 type = 'PyObject *' 3459 default_type = (str, Null, NoneType) 3460 format_unit = 'U' 3461 3462 def parse_arg(self, argname, displayname): 3463 if self.format_unit == 'U': 3464 return """ 3465 if (!PyUnicode_Check({argname})) {{{{ 3466 _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); 3467 goto exit; 3468 }}}} 3469 if (PyUnicode_READY({argname}) == -1) {{{{ 3470 goto exit; 3471 }}}} 3472 {paramname} = {argname}; 3473 """.format(argname=argname, paramname=self.parser_name, 3474 displayname=displayname) 3475 return super().parse_arg(argname, displayname) 3476 3477@add_legacy_c_converter('u') 3478@add_legacy_c_converter('u#', zeroes=True) 3479@add_legacy_c_converter('Z', accept={str, NoneType}) 3480@add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) 3481class Py_UNICODE_converter(CConverter): 3482 type = 'const Py_UNICODE *' 3483 default_type = (str, Null, NoneType) 3484 3485 def converter_init(self, *, accept={str}, zeroes=False): 3486 format_unit = 'Z' if accept=={str, NoneType} else 'u' 3487 if zeroes: 3488 format_unit += '#' 3489 self.length = True 3490 self.format_unit = format_unit 3491 else: 3492 self.accept = accept 3493 if accept == {str}: 3494 self.converter = '_PyUnicode_WideCharString_Converter' 3495 elif accept == {str, NoneType}: 3496 self.converter = '_PyUnicode_WideCharString_Opt_Converter' 3497 else: 3498 fail("Py_UNICODE_converter: illegal 'accept' argument " + repr(accept)) 3499 3500 def cleanup(self): 3501 if not self.length: 3502 return """\ 3503#if !USE_UNICODE_WCHAR_CACHE 3504PyMem_Free((void *){name}); 3505#endif /* USE_UNICODE_WCHAR_CACHE */ 3506""".format(name=self.name) 3507 3508 def parse_arg(self, argname, argnum): 3509 if not self.length: 3510 if self.accept == {str}: 3511 return """ 3512 if (!PyUnicode_Check({argname})) {{{{ 3513 _PyArg_BadArgument("{{name}}", {argnum}, "str", {argname}); 3514 goto exit; 3515 }}}} 3516 #if USE_UNICODE_WCHAR_CACHE 3517 {paramname} = _PyUnicode_AsUnicode({argname}); 3518 #else /* USE_UNICODE_WCHAR_CACHE */ 3519 {paramname} = PyUnicode_AsWideCharString({argname}, NULL); 3520 #endif /* USE_UNICODE_WCHAR_CACHE */ 3521 if ({paramname} == NULL) {{{{ 3522 goto exit; 3523 }}}} 3524 """.format(argname=argname, paramname=self.name, argnum=argnum) 3525 elif self.accept == {str, NoneType}: 3526 return """ 3527 if ({argname} == Py_None) {{{{ 3528 {paramname} = NULL; 3529 }}}} 3530 else if (PyUnicode_Check({argname})) {{{{ 3531 #if USE_UNICODE_WCHAR_CACHE 3532 {paramname} = _PyUnicode_AsUnicode({argname}); 3533 #else /* USE_UNICODE_WCHAR_CACHE */ 3534 {paramname} = PyUnicode_AsWideCharString({argname}, NULL); 3535 #endif /* USE_UNICODE_WCHAR_CACHE */ 3536 if ({paramname} == NULL) {{{{ 3537 goto exit; 3538 }}}} 3539 }}}} 3540 else {{{{ 3541 _PyArg_BadArgument("{{name}}", {argnum}, "str or None", {argname}); 3542 goto exit; 3543 }}}} 3544 """.format(argname=argname, paramname=self.name, argnum=argnum) 3545 return super().parse_arg(argname, argnum) 3546 3547@add_legacy_c_converter('s*', accept={str, buffer}) 3548@add_legacy_c_converter('z*', accept={str, buffer, NoneType}) 3549@add_legacy_c_converter('w*', accept={rwbuffer}) 3550class Py_buffer_converter(CConverter): 3551 type = 'Py_buffer' 3552 format_unit = 'y*' 3553 impl_by_reference = True 3554 c_ignored_default = "{NULL, NULL}" 3555 3556 def converter_init(self, *, accept={buffer}): 3557 if self.default not in (unspecified, None): 3558 fail("The only legal default value for Py_buffer is None.") 3559 3560 self.c_default = self.c_ignored_default 3561 3562 if accept == {str, buffer, NoneType}: 3563 format_unit = 'z*' 3564 elif accept == {str, buffer}: 3565 format_unit = 's*' 3566 elif accept == {buffer}: 3567 format_unit = 'y*' 3568 elif accept == {rwbuffer}: 3569 format_unit = 'w*' 3570 else: 3571 fail("Py_buffer_converter: illegal combination of arguments") 3572 3573 self.format_unit = format_unit 3574 3575 def cleanup(self): 3576 name = self.name 3577 return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) 3578 3579 def parse_arg(self, argname, displayname): 3580 if self.format_unit == 'y*': 3581 return """ 3582 if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ 3583 goto exit; 3584 }}}} 3585 if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ 3586 _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); 3587 goto exit; 3588 }}}} 3589 """.format(argname=argname, paramname=self.parser_name, 3590 displayname=displayname) 3591 elif self.format_unit == 's*': 3592 return """ 3593 if (PyUnicode_Check({argname})) {{{{ 3594 Py_ssize_t len; 3595 const char *ptr = PyUnicode_AsUTF8AndSize({argname}, &len); 3596 if (ptr == NULL) {{{{ 3597 goto exit; 3598 }}}} 3599 PyBuffer_FillInfo(&{paramname}, {argname}, (void *)ptr, len, 1, 0); 3600 }}}} 3601 else {{{{ /* any bytes-like object */ 3602 if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ 3603 goto exit; 3604 }}}} 3605 if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ 3606 _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); 3607 goto exit; 3608 }}}} 3609 }}}} 3610 """.format(argname=argname, paramname=self.parser_name, 3611 displayname=displayname) 3612 elif self.format_unit == 'w*': 3613 return """ 3614 if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_WRITABLE) < 0) {{{{ 3615 PyErr_Clear(); 3616 _PyArg_BadArgument("{{name}}", {displayname}, "read-write bytes-like object", {argname}); 3617 goto exit; 3618 }}}} 3619 if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ 3620 _PyArg_BadArgument("{{name}}", {displayname}, "contiguous buffer", {argname}); 3621 goto exit; 3622 }}}} 3623 """.format(argname=argname, paramname=self.parser_name, 3624 displayname=displayname) 3625 return super().parse_arg(argname, displayname) 3626 3627 3628def correct_name_for_self(f): 3629 if f.kind in (CALLABLE, METHOD_INIT): 3630 if f.cls: 3631 return "PyObject *", "self" 3632 return "PyObject *", "module" 3633 if f.kind == STATIC_METHOD: 3634 return "void *", "null" 3635 if f.kind in (CLASS_METHOD, METHOD_NEW): 3636 return "PyTypeObject *", "type" 3637 raise RuntimeError("Unhandled type of function f: " + repr(f.kind)) 3638 3639def required_type_for_self_for_parser(f): 3640 type, _ = correct_name_for_self(f) 3641 if f.kind in (METHOD_INIT, METHOD_NEW, STATIC_METHOD, CLASS_METHOD): 3642 return type 3643 return None 3644 3645 3646class self_converter(CConverter): 3647 """ 3648 A special-case converter: 3649 this is the default converter used for "self". 3650 """ 3651 type = None 3652 format_unit = '' 3653 3654 def converter_init(self, *, type=None): 3655 self.specified_type = type 3656 3657 def pre_render(self): 3658 f = self.function 3659 default_type, default_name = correct_name_for_self(f) 3660 self.signature_name = default_name 3661 self.type = self.specified_type or self.type or default_type 3662 3663 kind = self.function.kind 3664 new_or_init = kind in (METHOD_NEW, METHOD_INIT) 3665 3666 if (kind == STATIC_METHOD) or new_or_init: 3667 self.show_in_signature = False 3668 3669 # tp_new (METHOD_NEW) functions are of type newfunc: 3670 # typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *); 3671 # PyTypeObject is a typedef for struct _typeobject. 3672 # 3673 # tp_init (METHOD_INIT) functions are of type initproc: 3674 # typedef int (*initproc)(PyObject *, PyObject *, PyObject *); 3675 # 3676 # All other functions generated by Argument Clinic are stored in 3677 # PyMethodDef structures, in the ml_meth slot, which is of type PyCFunction: 3678 # typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); 3679 # However! We habitually cast these functions to PyCFunction, 3680 # since functions that accept keyword arguments don't fit this signature 3681 # but are stored there anyway. So strict type equality isn't important 3682 # for these functions. 3683 # 3684 # So: 3685 # 3686 # * The name of the first parameter to the impl and the parsing function will always 3687 # be self.name. 3688 # 3689 # * The type of the first parameter to the impl will always be of self.type. 3690 # 3691 # * If the function is neither tp_new (METHOD_NEW) nor tp_init (METHOD_INIT): 3692 # * The type of the first parameter to the parsing function is also self.type. 3693 # This means that if you step into the parsing function, your "self" parameter 3694 # is of the correct type, which may make debugging more pleasant. 3695 # 3696 # * Else if the function is tp_new (METHOD_NEW): 3697 # * The type of the first parameter to the parsing function is "PyTypeObject *", 3698 # so the type signature of the function call is an exact match. 3699 # * If self.type != "PyTypeObject *", we cast the first parameter to self.type 3700 # in the impl call. 3701 # 3702 # * Else if the function is tp_init (METHOD_INIT): 3703 # * The type of the first parameter to the parsing function is "PyObject *", 3704 # so the type signature of the function call is an exact match. 3705 # * If self.type != "PyObject *", we cast the first parameter to self.type 3706 # in the impl call. 3707 3708 @property 3709 def parser_type(self): 3710 return required_type_for_self_for_parser(self.function) or self.type 3711 3712 def render(self, parameter, data): 3713 """ 3714 parameter is a clinic.Parameter instance. 3715 data is a CRenderData instance. 3716 """ 3717 if self.function.kind == STATIC_METHOD: 3718 return 3719 3720 self._render_self(parameter, data) 3721 3722 if self.type != self.parser_type: 3723 # insert cast to impl_argument[0], aka self. 3724 # we know we're in the first slot in all the CRenderData lists, 3725 # because we render parameters in order, and self is always first. 3726 assert len(data.impl_arguments) == 1 3727 assert data.impl_arguments[0] == self.name 3728 data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] 3729 3730 def set_template_dict(self, template_dict): 3731 template_dict['self_name'] = self.name 3732 template_dict['self_type'] = self.parser_type 3733 kind = self.function.kind 3734 cls = self.function.cls 3735 3736 if ((kind in (METHOD_NEW, METHOD_INIT)) and cls and cls.typedef): 3737 type_object = self.function.cls.type_object 3738 prefix = (type_object[1:] + '.' if type_object[0] == '&' else 3739 type_object + '->') 3740 if kind == METHOD_NEW: 3741 type_check = ('({0} == {1} ||\n ' 3742 ' {0}->tp_init == {2}tp_init)' 3743 ).format(self.name, type_object, prefix) 3744 else: 3745 type_check = ('(Py_IS_TYPE({0}, {1}) ||\n ' 3746 ' Py_TYPE({0})->tp_new == {2}tp_new)' 3747 ).format(self.name, type_object, prefix) 3748 3749 line = '{} &&\n '.format(type_check) 3750 template_dict['self_type_check'] = line 3751 3752 3753 3754def add_c_return_converter(f, name=None): 3755 if not name: 3756 name = f.__name__ 3757 if not name.endswith('_return_converter'): 3758 return f 3759 name = name[:-len('_return_converter')] 3760 return_converters[name] = f 3761 return f 3762 3763 3764class CReturnConverterAutoRegister(type): 3765 def __init__(cls, name, bases, classdict): 3766 add_c_return_converter(cls) 3767 3768class CReturnConverter(metaclass=CReturnConverterAutoRegister): 3769 3770 # The C type to use for this variable. 3771 # 'type' should be a Python string specifying the type, e.g. "int". 3772 # If this is a pointer type, the type string should end with ' *'. 3773 type = 'PyObject *' 3774 3775 # The Python default value for this parameter, as a Python value. 3776 # Or the magic value "unspecified" if there is no default. 3777 default = None 3778 3779 def __init__(self, *, py_default=None, **kwargs): 3780 self.py_default = py_default 3781 try: 3782 self.return_converter_init(**kwargs) 3783 except TypeError as e: 3784 s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) 3785 sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) 3786 3787 def return_converter_init(self): 3788 pass 3789 3790 def declare(self, data, name="_return_value"): 3791 line = [] 3792 add = line.append 3793 add(self.type) 3794 if not self.type.endswith('*'): 3795 add(' ') 3796 add(name + ';') 3797 data.declarations.append(''.join(line)) 3798 data.return_value = name 3799 3800 def err_occurred_if(self, expr, data): 3801 data.return_conversion.append('if (({}) && PyErr_Occurred()) {{\n goto exit;\n}}\n'.format(expr)) 3802 3803 def err_occurred_if_null_pointer(self, variable, data): 3804 data.return_conversion.append('if ({} == NULL) {{\n goto exit;\n}}\n'.format(variable)) 3805 3806 def render(self, function, data): 3807 """ 3808 function is a clinic.Function instance. 3809 data is a CRenderData instance. 3810 """ 3811 pass 3812 3813add_c_return_converter(CReturnConverter, 'object') 3814 3815class NoneType_return_converter(CReturnConverter): 3816 def render(self, function, data): 3817 self.declare(data) 3818 data.return_conversion.append(''' 3819if (_return_value != Py_None) { 3820 goto exit; 3821} 3822return_value = Py_None; 3823Py_INCREF(Py_None); 3824'''.strip()) 3825 3826class bool_return_converter(CReturnConverter): 3827 type = 'int' 3828 3829 def render(self, function, data): 3830 self.declare(data) 3831 self.err_occurred_if("_return_value == -1", data) 3832 data.return_conversion.append('return_value = PyBool_FromLong((long)_return_value);\n') 3833 3834class long_return_converter(CReturnConverter): 3835 type = 'long' 3836 conversion_fn = 'PyLong_FromLong' 3837 cast = '' 3838 unsigned_cast = '' 3839 3840 def render(self, function, data): 3841 self.declare(data) 3842 self.err_occurred_if("_return_value == {}-1".format(self.unsigned_cast), data) 3843 data.return_conversion.append( 3844 ''.join(('return_value = ', self.conversion_fn, '(', self.cast, '_return_value);\n'))) 3845 3846class int_return_converter(long_return_converter): 3847 type = 'int' 3848 cast = '(long)' 3849 3850class init_return_converter(long_return_converter): 3851 """ 3852 Special return converter for __init__ functions. 3853 """ 3854 type = 'int' 3855 cast = '(long)' 3856 3857 def render(self, function, data): 3858 pass 3859 3860class unsigned_long_return_converter(long_return_converter): 3861 type = 'unsigned long' 3862 conversion_fn = 'PyLong_FromUnsignedLong' 3863 unsigned_cast = '(unsigned long)' 3864 3865class unsigned_int_return_converter(unsigned_long_return_converter): 3866 type = 'unsigned int' 3867 cast = '(unsigned long)' 3868 unsigned_cast = '(unsigned int)' 3869 3870class Py_ssize_t_return_converter(long_return_converter): 3871 type = 'Py_ssize_t' 3872 conversion_fn = 'PyLong_FromSsize_t' 3873 3874class size_t_return_converter(long_return_converter): 3875 type = 'size_t' 3876 conversion_fn = 'PyLong_FromSize_t' 3877 unsigned_cast = '(size_t)' 3878 3879 3880class double_return_converter(CReturnConverter): 3881 type = 'double' 3882 cast = '' 3883 3884 def render(self, function, data): 3885 self.declare(data) 3886 self.err_occurred_if("_return_value == -1.0", data) 3887 data.return_conversion.append( 3888 'return_value = PyFloat_FromDouble(' + self.cast + '_return_value);\n') 3889 3890class float_return_converter(double_return_converter): 3891 type = 'float' 3892 cast = '(double)' 3893 3894 3895def eval_ast_expr(node, globals, *, filename='-'): 3896 """ 3897 Takes an ast.Expr node. Compiles and evaluates it. 3898 Returns the result of the expression. 3899 3900 globals represents the globals dict the expression 3901 should see. (There's no equivalent for "locals" here.) 3902 """ 3903 3904 if isinstance(node, ast.Expr): 3905 node = node.value 3906 3907 node = ast.Expression(node) 3908 co = compile(node, filename, 'eval') 3909 fn = types.FunctionType(co, globals) 3910 return fn() 3911 3912 3913class IndentStack: 3914 def __init__(self): 3915 self.indents = [] 3916 self.margin = None 3917 3918 def _ensure(self): 3919 if not self.indents: 3920 fail('IndentStack expected indents, but none are defined.') 3921 3922 def measure(self, line): 3923 """ 3924 Returns the length of the line's margin. 3925 """ 3926 if '\t' in line: 3927 fail('Tab characters are illegal in the Argument Clinic DSL.') 3928 stripped = line.lstrip() 3929 if not len(stripped): 3930 # we can't tell anything from an empty line 3931 # so just pretend it's indented like our current indent 3932 self._ensure() 3933 return self.indents[-1] 3934 return len(line) - len(stripped) 3935 3936 def infer(self, line): 3937 """ 3938 Infer what is now the current margin based on this line. 3939 Returns: 3940 1 if we have indented (or this is the first margin) 3941 0 if the margin has not changed 3942 -N if we have dedented N times 3943 """ 3944 indent = self.measure(line) 3945 margin = ' ' * indent 3946 if not self.indents: 3947 self.indents.append(indent) 3948 self.margin = margin 3949 return 1 3950 current = self.indents[-1] 3951 if indent == current: 3952 return 0 3953 if indent > current: 3954 self.indents.append(indent) 3955 self.margin = margin 3956 return 1 3957 # indent < current 3958 if indent not in self.indents: 3959 fail("Illegal outdent.") 3960 outdent_count = 0 3961 while indent != current: 3962 self.indents.pop() 3963 current = self.indents[-1] 3964 outdent_count -= 1 3965 self.margin = margin 3966 return outdent_count 3967 3968 @property 3969 def depth(self): 3970 """ 3971 Returns how many margins are currently defined. 3972 """ 3973 return len(self.indents) 3974 3975 def indent(self, line): 3976 """ 3977 Indents a line by the currently defined margin. 3978 """ 3979 return self.margin + line 3980 3981 def dedent(self, line): 3982 """ 3983 Dedents a line by the currently defined margin. 3984 (The inverse of 'indent'.) 3985 """ 3986 margin = self.margin 3987 indent = self.indents[-1] 3988 if not line.startswith(margin): 3989 fail('Cannot dedent, line does not start with the previous margin:') 3990 return line[indent:] 3991 3992 3993class DSLParser: 3994 def __init__(self, clinic): 3995 self.clinic = clinic 3996 3997 self.directives = {} 3998 for name in dir(self): 3999 # functions that start with directive_ are added to directives 4000 _, s, key = name.partition("directive_") 4001 if s: 4002 self.directives[key] = getattr(self, name) 4003 4004 # functions that start with at_ are too, with an @ in front 4005 _, s, key = name.partition("at_") 4006 if s: 4007 self.directives['@' + key] = getattr(self, name) 4008 4009 self.reset() 4010 4011 def reset(self): 4012 self.function = None 4013 self.state = self.state_dsl_start 4014 self.parameter_indent = None 4015 self.keyword_only = False 4016 self.positional_only = False 4017 self.group = 0 4018 self.parameter_state = self.ps_start 4019 self.seen_positional_with_default = False 4020 self.indent = IndentStack() 4021 self.kind = CALLABLE 4022 self.coexist = False 4023 self.parameter_continuation = '' 4024 self.preserve_output = False 4025 4026 def directive_version(self, required): 4027 global version 4028 if version_comparitor(version, required) < 0: 4029 fail("Insufficient Clinic version!\n Version: " + version + "\n Required: " + required) 4030 4031 def directive_module(self, name): 4032 fields = name.split('.') 4033 new = fields.pop() 4034 module, cls = self.clinic._module_and_class(fields) 4035 if cls: 4036 fail("Can't nest a module inside a class!") 4037 4038 if name in module.classes: 4039 fail("Already defined module " + repr(name) + "!") 4040 4041 m = Module(name, module) 4042 module.modules[name] = m 4043 self.block.signatures.append(m) 4044 4045 def directive_class(self, name, typedef, type_object): 4046 fields = name.split('.') 4047 in_classes = False 4048 parent = self 4049 name = fields.pop() 4050 so_far = [] 4051 module, cls = self.clinic._module_and_class(fields) 4052 4053 parent = cls or module 4054 if name in parent.classes: 4055 fail("Already defined class " + repr(name) + "!") 4056 4057 c = Class(name, module, cls, typedef, type_object) 4058 parent.classes[name] = c 4059 self.block.signatures.append(c) 4060 4061 def directive_set(self, name, value): 4062 if name not in ("line_prefix", "line_suffix"): 4063 fail("unknown variable", repr(name)) 4064 4065 value = value.format_map({ 4066 'block comment start': '/*', 4067 'block comment end': '*/', 4068 }) 4069 4070 self.clinic.__dict__[name] = value 4071 4072 def directive_destination(self, name, command, *args): 4073 if command == 'new': 4074 self.clinic.add_destination(name, *args) 4075 return 4076 4077 if command == 'clear': 4078 self.clinic.get_destination(name).clear() 4079 fail("unknown destination command", repr(command)) 4080 4081 4082 def directive_output(self, command_or_name, destination=''): 4083 fd = self.clinic.destination_buffers 4084 4085 if command_or_name == "preset": 4086 preset = self.clinic.presets.get(destination) 4087 if not preset: 4088 fail("Unknown preset " + repr(destination) + "!") 4089 fd.update(preset) 4090 return 4091 4092 if command_or_name == "push": 4093 self.clinic.destination_buffers_stack.append(fd.copy()) 4094 return 4095 4096 if command_or_name == "pop": 4097 if not self.clinic.destination_buffers_stack: 4098 fail("Can't 'output pop', stack is empty!") 4099 previous_fd = self.clinic.destination_buffers_stack.pop() 4100 fd.update(previous_fd) 4101 return 4102 4103 # secret command for debugging! 4104 if command_or_name == "print": 4105 self.block.output.append(pprint.pformat(fd)) 4106 self.block.output.append('\n') 4107 return 4108 4109 d = self.clinic.get_destination(destination) 4110 4111 if command_or_name == "everything": 4112 for name in list(fd): 4113 fd[name] = d 4114 return 4115 4116 if command_or_name not in fd: 4117 fail("Invalid command / destination name " + repr(command_or_name) + ", must be one of:\n preset push pop print everything " + " ".join(fd)) 4118 fd[command_or_name] = d 4119 4120 def directive_dump(self, name): 4121 self.block.output.append(self.clinic.get_destination(name).dump()) 4122 4123 def directive_printout(self, *args): 4124 self.block.output.append(' '.join(args)) 4125 self.block.output.append('\n') 4126 4127 def directive_preserve(self): 4128 if self.preserve_output: 4129 fail("Can't have preserve twice in one block!") 4130 self.preserve_output = True 4131 4132 def at_classmethod(self): 4133 if self.kind is not CALLABLE: 4134 fail("Can't set @classmethod, function is not a normal callable") 4135 self.kind = CLASS_METHOD 4136 4137 def at_staticmethod(self): 4138 if self.kind is not CALLABLE: 4139 fail("Can't set @staticmethod, function is not a normal callable") 4140 self.kind = STATIC_METHOD 4141 4142 def at_coexist(self): 4143 if self.coexist: 4144 fail("Called @coexist twice!") 4145 self.coexist = True 4146 4147 def parse(self, block): 4148 self.reset() 4149 self.block = block 4150 self.saved_output = self.block.output 4151 block.output = [] 4152 block_start = self.clinic.block_parser.line_number 4153 lines = block.input.split('\n') 4154 for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): 4155 if '\t' in line: 4156 fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) 4157 self.state(line) 4158 4159 self.next(self.state_terminal) 4160 self.state(None) 4161 4162 block.output.extend(self.clinic.language.render(clinic, block.signatures)) 4163 4164 if self.preserve_output: 4165 if block.output: 4166 fail("'preserve' only works for blocks that don't produce any output!") 4167 block.output = self.saved_output 4168 4169 @staticmethod 4170 def ignore_line(line): 4171 # ignore comment-only lines 4172 if line.lstrip().startswith('#'): 4173 return True 4174 4175 # Ignore empty lines too 4176 # (but not in docstring sections!) 4177 if not line.strip(): 4178 return True 4179 4180 return False 4181 4182 @staticmethod 4183 def calculate_indent(line): 4184 return len(line) - len(line.strip()) 4185 4186 def next(self, state, line=None): 4187 # real_print(self.state.__name__, "->", state.__name__, ", line=", line) 4188 self.state = state 4189 if line is not None: 4190 self.state(line) 4191 4192 def state_dsl_start(self, line): 4193 # self.block = self.ClinicOutputBlock(self) 4194 if self.ignore_line(line): 4195 return 4196 4197 # is it a directive? 4198 fields = shlex.split(line) 4199 directive_name = fields[0] 4200 directive = self.directives.get(directive_name, None) 4201 if directive: 4202 try: 4203 directive(*fields[1:]) 4204 except TypeError as e: 4205 fail(str(e)) 4206 return 4207 4208 self.next(self.state_modulename_name, line) 4209 4210 def state_modulename_name(self, line): 4211 # looking for declaration, which establishes the leftmost column 4212 # line should be 4213 # modulename.fnname [as c_basename] [-> return annotation] 4214 # square brackets denote optional syntax. 4215 # 4216 # alternatively: 4217 # modulename.fnname [as c_basename] = modulename.existing_fn_name 4218 # clones the parameters and return converter from that 4219 # function. you can't modify them. you must enter a 4220 # new docstring. 4221 # 4222 # (but we might find a directive first!) 4223 # 4224 # this line is permitted to start with whitespace. 4225 # we'll call this number of spaces F (for "function"). 4226 4227 if not line.strip(): 4228 return 4229 4230 self.indent.infer(line) 4231 4232 # are we cloning? 4233 before, equals, existing = line.rpartition('=') 4234 if equals: 4235 full_name, _, c_basename = before.partition(' as ') 4236 full_name = full_name.strip() 4237 c_basename = c_basename.strip() 4238 existing = existing.strip() 4239 if (is_legal_py_identifier(full_name) and 4240 (not c_basename or is_legal_c_identifier(c_basename)) and 4241 is_legal_py_identifier(existing)): 4242 # we're cloning! 4243 fields = [x.strip() for x in existing.split('.')] 4244 function_name = fields.pop() 4245 module, cls = self.clinic._module_and_class(fields) 4246 4247 for existing_function in (cls or module).functions: 4248 if existing_function.name == function_name: 4249 break 4250 else: 4251 existing_function = None 4252 if not existing_function: 4253 print("class", cls, "module", module, "existing", existing) 4254 print("cls. functions", cls.functions) 4255 fail("Couldn't find existing function " + repr(existing) + "!") 4256 4257 fields = [x.strip() for x in full_name.split('.')] 4258 function_name = fields.pop() 4259 module, cls = self.clinic._module_and_class(fields) 4260 4261 if not (existing_function.kind == self.kind and existing_function.coexist == self.coexist): 4262 fail("'kind' of function and cloned function don't match! (@classmethod/@staticmethod/@coexist)") 4263 self.function = existing_function.copy(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, docstring='') 4264 4265 self.block.signatures.append(self.function) 4266 (cls or module).functions.append(self.function) 4267 self.next(self.state_function_docstring) 4268 return 4269 4270 line, _, returns = line.partition('->') 4271 4272 full_name, _, c_basename = line.partition(' as ') 4273 full_name = full_name.strip() 4274 c_basename = c_basename.strip() or None 4275 4276 if not is_legal_py_identifier(full_name): 4277 fail("Illegal function name: {}".format(full_name)) 4278 if c_basename and not is_legal_c_identifier(c_basename): 4279 fail("Illegal C basename: {}".format(c_basename)) 4280 4281 return_converter = None 4282 if returns: 4283 ast_input = "def x() -> {}: pass".format(returns) 4284 module = None 4285 try: 4286 module = ast.parse(ast_input) 4287 except SyntaxError: 4288 pass 4289 if not module: 4290 fail("Badly-formed annotation for " + full_name + ": " + returns) 4291 try: 4292 name, legacy, kwargs = self.parse_converter(module.body[0].returns) 4293 if legacy: 4294 fail("Legacy converter {!r} not allowed as a return converter" 4295 .format(name)) 4296 if name not in return_converters: 4297 fail("No available return converter called " + repr(name)) 4298 return_converter = return_converters[name](**kwargs) 4299 except ValueError: 4300 fail("Badly-formed annotation for " + full_name + ": " + returns) 4301 4302 fields = [x.strip() for x in full_name.split('.')] 4303 function_name = fields.pop() 4304 module, cls = self.clinic._module_and_class(fields) 4305 4306 fields = full_name.split('.') 4307 if fields[-1] in unsupported_special_methods: 4308 fail(f"{fields[-1]} is a special method and cannot be converted to Argument Clinic! (Yet.)") 4309 4310 if fields[-1] == '__new__': 4311 if (self.kind != CLASS_METHOD) or (not cls): 4312 fail("__new__ must be a class method!") 4313 self.kind = METHOD_NEW 4314 elif fields[-1] == '__init__': 4315 if (self.kind != CALLABLE) or (not cls): 4316 fail("__init__ must be a normal method, not a class or static method!") 4317 self.kind = METHOD_INIT 4318 if not return_converter: 4319 return_converter = init_return_converter() 4320 4321 if not return_converter: 4322 return_converter = CReturnConverter() 4323 4324 if not module: 4325 fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".") 4326 self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, 4327 return_converter=return_converter, kind=self.kind, coexist=self.coexist) 4328 self.block.signatures.append(self.function) 4329 4330 # insert a self converter automatically 4331 type, name = correct_name_for_self(self.function) 4332 kwargs = {} 4333 if cls and type == "PyObject *": 4334 kwargs['type'] = cls.typedef 4335 sc = self.function.self_converter = self_converter(name, name, self.function, **kwargs) 4336 p_self = Parameter(sc.name, inspect.Parameter.POSITIONAL_ONLY, function=self.function, converter=sc) 4337 self.function.parameters[sc.name] = p_self 4338 4339 (cls or module).functions.append(self.function) 4340 self.next(self.state_parameters_start) 4341 4342 # Now entering the parameters section. The rules, formally stated: 4343 # 4344 # * All lines must be indented with spaces only. 4345 # * The first line must be a parameter declaration. 4346 # * The first line must be indented. 4347 # * This first line establishes the indent for parameters. 4348 # * We'll call this number of spaces P (for "parameter"). 4349 # * Thenceforth: 4350 # * Lines indented with P spaces specify a parameter. 4351 # * Lines indented with > P spaces are docstrings for the previous 4352 # parameter. 4353 # * We'll call this number of spaces D (for "docstring"). 4354 # * All subsequent lines indented with >= D spaces are stored as 4355 # part of the per-parameter docstring. 4356 # * All lines will have the first D spaces of the indent stripped 4357 # before they are stored. 4358 # * It's illegal to have a line starting with a number of spaces X 4359 # such that P < X < D. 4360 # * A line with < P spaces is the first line of the function 4361 # docstring, which ends processing for parameters and per-parameter 4362 # docstrings. 4363 # * The first line of the function docstring must be at the same 4364 # indent as the function declaration. 4365 # * It's illegal to have any line in the parameters section starting 4366 # with X spaces such that F < X < P. (As before, F is the indent 4367 # of the function declaration.) 4368 # 4369 # Also, currently Argument Clinic places the following restrictions on groups: 4370 # * Each group must contain at least one parameter. 4371 # * Each group may contain at most one group, which must be the furthest 4372 # thing in the group from the required parameters. (The nested group 4373 # must be the first in the group when it's before the required 4374 # parameters, and the last thing in the group when after the required 4375 # parameters.) 4376 # * There may be at most one (top-level) group to the left or right of 4377 # the required parameters. 4378 # * You must specify a slash, and it must be after all parameters. 4379 # (In other words: either all parameters are positional-only, 4380 # or none are.) 4381 # 4382 # Said another way: 4383 # * Each group must contain at least one parameter. 4384 # * All left square brackets before the required parameters must be 4385 # consecutive. (You can't have a left square bracket followed 4386 # by a parameter, then another left square bracket. You can't 4387 # have a left square bracket, a parameter, a right square bracket, 4388 # and then a left square bracket.) 4389 # * All right square brackets after the required parameters must be 4390 # consecutive. 4391 # 4392 # These rules are enforced with a single state variable: 4393 # "parameter_state". (Previously the code was a miasma of ifs and 4394 # separate boolean state variables.) The states are: 4395 # 4396 # [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line 4397 # 01 2 3 4 5 6 <- state transitions 4398 # 4399 # 0: ps_start. before we've seen anything. legal transitions are to 1 or 3. 4400 # 1: ps_left_square_before. left square brackets before required parameters. 4401 # 2: ps_group_before. in a group, before required parameters. 4402 # 3: ps_required. required parameters, positional-or-keyword or positional-only 4403 # (we don't know yet). (renumber left groups!) 4404 # 4: ps_optional. positional-or-keyword or positional-only parameters that 4405 # now must have default values. 4406 # 5: ps_group_after. in a group, after required parameters. 4407 # 6: ps_right_square_after. right square brackets after required parameters. 4408 ps_start, ps_left_square_before, ps_group_before, ps_required, \ 4409 ps_optional, ps_group_after, ps_right_square_after = range(7) 4410 4411 def state_parameters_start(self, line): 4412 if self.ignore_line(line): 4413 return 4414 4415 # if this line is not indented, we have no parameters 4416 if not self.indent.infer(line): 4417 return self.next(self.state_function_docstring, line) 4418 4419 self.parameter_continuation = '' 4420 return self.next(self.state_parameter, line) 4421 4422 4423 def to_required(self): 4424 """ 4425 Transition to the "required" parameter state. 4426 """ 4427 if self.parameter_state != self.ps_required: 4428 self.parameter_state = self.ps_required 4429 for p in self.function.parameters.values(): 4430 p.group = -p.group 4431 4432 def state_parameter(self, line): 4433 if self.parameter_continuation: 4434 line = self.parameter_continuation + ' ' + line.lstrip() 4435 self.parameter_continuation = '' 4436 4437 if self.ignore_line(line): 4438 return 4439 4440 assert self.indent.depth == 2 4441 indent = self.indent.infer(line) 4442 if indent == -1: 4443 # we outdented, must be to definition column 4444 return self.next(self.state_function_docstring, line) 4445 4446 if indent == 1: 4447 # we indented, must be to new parameter docstring column 4448 return self.next(self.state_parameter_docstring_start, line) 4449 4450 line = line.rstrip() 4451 if line.endswith('\\'): 4452 self.parameter_continuation = line[:-1] 4453 return 4454 4455 line = line.lstrip() 4456 4457 if line in ('*', '/', '[', ']'): 4458 self.parse_special_symbol(line) 4459 return 4460 4461 if self.parameter_state in (self.ps_start, self.ps_required): 4462 self.to_required() 4463 elif self.parameter_state == self.ps_left_square_before: 4464 self.parameter_state = self.ps_group_before 4465 elif self.parameter_state == self.ps_group_before: 4466 if not self.group: 4467 self.to_required() 4468 elif self.parameter_state in (self.ps_group_after, self.ps_optional): 4469 pass 4470 else: 4471 fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".a)") 4472 4473 # handle "as" for parameters too 4474 c_name = None 4475 name, have_as_token, trailing = line.partition(' as ') 4476 if have_as_token: 4477 name = name.strip() 4478 if ' ' not in name: 4479 fields = trailing.strip().split(' ') 4480 if not fields: 4481 fail("Invalid 'as' clause!") 4482 c_name = fields[0] 4483 if c_name.endswith(':'): 4484 name += ':' 4485 c_name = c_name[:-1] 4486 fields[0] = name 4487 line = ' '.join(fields) 4488 4489 base, equals, default = line.rpartition('=') 4490 if not equals: 4491 base = default 4492 default = None 4493 4494 module = None 4495 try: 4496 ast_input = "def x({}): pass".format(base) 4497 module = ast.parse(ast_input) 4498 except SyntaxError: 4499 try: 4500 # the last = was probably inside a function call, like 4501 # c: int(accept={str}) 4502 # so assume there was no actual default value. 4503 default = None 4504 ast_input = "def x({}): pass".format(line) 4505 module = ast.parse(ast_input) 4506 except SyntaxError: 4507 pass 4508 if not module: 4509 fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) 4510 4511 function_args = module.body[0].args 4512 4513 if len(function_args.args) > 1: 4514 fail("Function " + self.function.name + " has an invalid parameter declaration (comma?):\n\t" + line) 4515 if function_args.defaults or function_args.kw_defaults: 4516 fail("Function " + self.function.name + " has an invalid parameter declaration (default value?):\n\t" + line) 4517 if function_args.kwarg: 4518 fail("Function " + self.function.name + " has an invalid parameter declaration (**kwargs?):\n\t" + line) 4519 4520 if function_args.vararg: 4521 is_vararg = True 4522 parameter = function_args.vararg 4523 else: 4524 is_vararg = False 4525 parameter = function_args.args[0] 4526 4527 parameter_name = parameter.arg 4528 name, legacy, kwargs = self.parse_converter(parameter.annotation) 4529 4530 if not default: 4531 if self.parameter_state == self.ps_optional: 4532 fail("Can't have a parameter without a default (" + repr(parameter_name) + ")\nafter a parameter with a default!") 4533 if is_vararg: 4534 value = NULL 4535 kwargs.setdefault('c_default', "NULL") 4536 else: 4537 value = unspecified 4538 if 'py_default' in kwargs: 4539 fail("You can't specify py_default without specifying a default value!") 4540 else: 4541 if is_vararg: 4542 fail("Vararg can't take a default value!") 4543 4544 if self.parameter_state == self.ps_required: 4545 self.parameter_state = self.ps_optional 4546 default = default.strip() 4547 bad = False 4548 ast_input = "x = {}".format(default) 4549 bad = False 4550 try: 4551 module = ast.parse(ast_input) 4552 4553 if 'c_default' not in kwargs: 4554 # we can only represent very simple data values in C. 4555 # detect whether default is okay, via a denylist 4556 # of disallowed ast nodes. 4557 class DetectBadNodes(ast.NodeVisitor): 4558 bad = False 4559 def bad_node(self, node): 4560 self.bad = True 4561 4562 # inline function call 4563 visit_Call = bad_node 4564 # inline if statement ("x = 3 if y else z") 4565 visit_IfExp = bad_node 4566 4567 # comprehensions and generator expressions 4568 visit_ListComp = visit_SetComp = bad_node 4569 visit_DictComp = visit_GeneratorExp = bad_node 4570 4571 # literals for advanced types 4572 visit_Dict = visit_Set = bad_node 4573 visit_List = visit_Tuple = bad_node 4574 4575 # "starred": "a = [1, 2, 3]; *a" 4576 visit_Starred = bad_node 4577 4578 denylist = DetectBadNodes() 4579 denylist.visit(module) 4580 bad = denylist.bad 4581 else: 4582 # if they specify a c_default, we can be more lenient about the default value. 4583 # but at least make an attempt at ensuring it's a valid expression. 4584 try: 4585 value = eval(default) 4586 if value == unspecified: 4587 fail("'unspecified' is not a legal default value!") 4588 except NameError: 4589 pass # probably a named constant 4590 except Exception as e: 4591 fail("Malformed expression given as default value\n" 4592 "{!r} caused {!r}".format(default, e)) 4593 if bad: 4594 fail("Unsupported expression as default value: " + repr(default)) 4595 4596 expr = module.body[0].value 4597 # mild hack: explicitly support NULL as a default value 4598 if isinstance(expr, ast.Name) and expr.id == 'NULL': 4599 value = NULL 4600 py_default = '<unrepresentable>' 4601 c_default = "NULL" 4602 elif (isinstance(expr, ast.BinOp) or 4603 (isinstance(expr, ast.UnaryOp) and 4604 not (isinstance(expr.operand, ast.Num) or 4605 (hasattr(ast, 'Constant') and 4606 isinstance(expr.operand, ast.Constant) and 4607 type(expr.operand.value) in (int, float, complex))) 4608 )): 4609 c_default = kwargs.get("c_default") 4610 if not (isinstance(c_default, str) and c_default): 4611 fail("When you specify an expression (" + repr(default) + ") as your default value,\nyou MUST specify a valid c_default." + ast.dump(expr)) 4612 py_default = default 4613 value = unknown 4614 elif isinstance(expr, ast.Attribute): 4615 a = [] 4616 n = expr 4617 while isinstance(n, ast.Attribute): 4618 a.append(n.attr) 4619 n = n.value 4620 if not isinstance(n, ast.Name): 4621 fail("Unsupported default value " + repr(default) + " (looked like a Python constant)") 4622 a.append(n.id) 4623 py_default = ".".join(reversed(a)) 4624 4625 c_default = kwargs.get("c_default") 4626 if not (isinstance(c_default, str) and c_default): 4627 fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") 4628 4629 try: 4630 value = eval(py_default) 4631 except NameError: 4632 value = unknown 4633 else: 4634 value = ast.literal_eval(expr) 4635 py_default = repr(value) 4636 if isinstance(value, (bool, None.__class__)): 4637 c_default = "Py_" + py_default 4638 elif isinstance(value, str): 4639 c_default = c_repr(value) 4640 else: 4641 c_default = py_default 4642 4643 except SyntaxError as e: 4644 fail("Syntax error: " + repr(e.text)) 4645 except (ValueError, AttributeError): 4646 value = unknown 4647 c_default = kwargs.get("c_default") 4648 py_default = default 4649 if not (isinstance(c_default, str) and c_default): 4650 fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") 4651 4652 kwargs.setdefault('c_default', c_default) 4653 kwargs.setdefault('py_default', py_default) 4654 4655 dict = legacy_converters if legacy else converters 4656 legacy_str = "legacy " if legacy else "" 4657 if name not in dict: 4658 fail('{} is not a valid {}converter'.format(name, legacy_str)) 4659 # if you use a c_name for the parameter, we just give that name to the converter 4660 # but the parameter object gets the python name 4661 converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) 4662 4663 if is_vararg: 4664 kind = inspect.Parameter.VAR_POSITIONAL 4665 elif self.keyword_only: 4666 kind = inspect.Parameter.KEYWORD_ONLY 4667 else: 4668 kind = inspect.Parameter.POSITIONAL_OR_KEYWORD 4669 4670 if isinstance(converter, self_converter): 4671 if len(self.function.parameters) == 1: 4672 if (self.parameter_state != self.ps_required): 4673 fail("A 'self' parameter cannot be marked optional.") 4674 if value is not unspecified: 4675 fail("A 'self' parameter cannot have a default value.") 4676 if self.group: 4677 fail("A 'self' parameter cannot be in an optional group.") 4678 kind = inspect.Parameter.POSITIONAL_ONLY 4679 self.parameter_state = self.ps_start 4680 self.function.parameters.clear() 4681 else: 4682 fail("A 'self' parameter, if specified, must be the very first thing in the parameter block.") 4683 4684 if isinstance(converter, defining_class_converter): 4685 _lp = len(self.function.parameters) 4686 if _lp == 1: 4687 if (self.parameter_state != self.ps_required): 4688 fail("A 'defining_class' parameter cannot be marked optional.") 4689 if value is not unspecified: 4690 fail("A 'defining_class' parameter cannot have a default value.") 4691 if self.group: 4692 fail("A 'defining_class' parameter cannot be in an optional group.") 4693 else: 4694 fail("A 'defining_class' parameter, if specified, must either be the first thing in the parameter block, or come just after 'self'.") 4695 4696 4697 p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) 4698 4699 if parameter_name in self.function.parameters: 4700 fail("You can't have two parameters named " + repr(parameter_name) + "!") 4701 self.function.parameters[parameter_name] = p 4702 4703 def parse_converter(self, annotation): 4704 if (hasattr(ast, 'Constant') and 4705 isinstance(annotation, ast.Constant) and 4706 type(annotation.value) is str): 4707 return annotation.value, True, {} 4708 4709 if isinstance(annotation, ast.Str): 4710 return annotation.s, True, {} 4711 4712 if isinstance(annotation, ast.Name): 4713 return annotation.id, False, {} 4714 4715 if not isinstance(annotation, ast.Call): 4716 fail("Annotations must be either a name, a function call, or a string.") 4717 4718 name = annotation.func.id 4719 symbols = globals() 4720 4721 kwargs = {node.arg: eval_ast_expr(node.value, symbols) for node in annotation.keywords} 4722 return name, False, kwargs 4723 4724 def parse_special_symbol(self, symbol): 4725 if symbol == '*': 4726 if self.keyword_only: 4727 fail("Function " + self.function.name + " uses '*' more than once.") 4728 self.keyword_only = True 4729 elif symbol == '[': 4730 if self.parameter_state in (self.ps_start, self.ps_left_square_before): 4731 self.parameter_state = self.ps_left_square_before 4732 elif self.parameter_state in (self.ps_required, self.ps_group_after): 4733 self.parameter_state = self.ps_group_after 4734 else: 4735 fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".b)") 4736 self.group += 1 4737 self.function.docstring_only = True 4738 elif symbol == ']': 4739 if not self.group: 4740 fail("Function " + self.function.name + " has a ] without a matching [.") 4741 if not any(p.group == self.group for p in self.function.parameters.values()): 4742 fail("Function " + self.function.name + " has an empty group.\nAll groups must contain at least one parameter.") 4743 self.group -= 1 4744 if self.parameter_state in (self.ps_left_square_before, self.ps_group_before): 4745 self.parameter_state = self.ps_group_before 4746 elif self.parameter_state in (self.ps_group_after, self.ps_right_square_after): 4747 self.parameter_state = self.ps_right_square_after 4748 else: 4749 fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".c)") 4750 elif symbol == '/': 4751 if self.positional_only: 4752 fail("Function " + self.function.name + " uses '/' more than once.") 4753 self.positional_only = True 4754 # ps_required and ps_optional are allowed here, that allows positional-only without option groups 4755 # to work (and have default values!) 4756 if (self.parameter_state not in (self.ps_required, self.ps_optional, self.ps_right_square_after, self.ps_group_before)) or self.group: 4757 fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ".d)") 4758 if self.keyword_only: 4759 fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") 4760 # fixup preceding parameters 4761 for p in self.function.parameters.values(): 4762 if p.is_vararg(): 4763 continue 4764 if (p.kind != inspect.Parameter.POSITIONAL_OR_KEYWORD and not isinstance(p.converter, self_converter)): 4765 fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") 4766 p.kind = inspect.Parameter.POSITIONAL_ONLY 4767 4768 def state_parameter_docstring_start(self, line): 4769 self.parameter_docstring_indent = len(self.indent.margin) 4770 assert self.indent.depth == 3 4771 return self.next(self.state_parameter_docstring, line) 4772 4773 # every line of the docstring must start with at least F spaces, 4774 # where F > P. 4775 # these F spaces will be stripped. 4776 def state_parameter_docstring(self, line): 4777 stripped = line.strip() 4778 if stripped.startswith('#'): 4779 return 4780 4781 indent = self.indent.measure(line) 4782 if indent < self.parameter_docstring_indent: 4783 self.indent.infer(line) 4784 assert self.indent.depth < 3 4785 if self.indent.depth == 2: 4786 # back to a parameter 4787 return self.next(self.state_parameter, line) 4788 assert self.indent.depth == 1 4789 return self.next(self.state_function_docstring, line) 4790 4791 assert self.function.parameters 4792 last_parameter = next(reversed(list(self.function.parameters.values()))) 4793 4794 new_docstring = last_parameter.docstring 4795 4796 if new_docstring: 4797 new_docstring += '\n' 4798 if stripped: 4799 new_docstring += self.indent.dedent(line) 4800 4801 last_parameter.docstring = new_docstring 4802 4803 # the final stanza of the DSL is the docstring. 4804 def state_function_docstring(self, line): 4805 if self.group: 4806 fail("Function " + self.function.name + " has a ] without a matching [.") 4807 4808 stripped = line.strip() 4809 if stripped.startswith('#'): 4810 return 4811 4812 new_docstring = self.function.docstring 4813 if new_docstring: 4814 new_docstring += "\n" 4815 if stripped: 4816 line = self.indent.dedent(line).rstrip() 4817 else: 4818 line = '' 4819 new_docstring += line 4820 self.function.docstring = new_docstring 4821 4822 def format_docstring(self): 4823 f = self.function 4824 4825 new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) 4826 if new_or_init and not f.docstring: 4827 # don't render a docstring at all, no signature, nothing. 4828 return f.docstring 4829 4830 text, add, output = _text_accumulator() 4831 parameters = f.render_parameters 4832 4833 ## 4834 ## docstring first line 4835 ## 4836 4837 if new_or_init: 4838 # classes get *just* the name of the class 4839 # not __new__, not __init__, and not module.classname 4840 assert f.cls 4841 add(f.cls.name) 4842 else: 4843 add(f.name) 4844 add('(') 4845 4846 # populate "right_bracket_count" field for every parameter 4847 assert parameters, "We should always have a self parameter. " + repr(f) 4848 assert isinstance(parameters[0].converter, self_converter) 4849 # self is always positional-only. 4850 assert parameters[0].is_positional_only() 4851 parameters[0].right_bracket_count = 0 4852 positional_only = True 4853 for p in parameters[1:]: 4854 if not p.is_positional_only(): 4855 positional_only = False 4856 else: 4857 assert positional_only 4858 if positional_only: 4859 p.right_bracket_count = abs(p.group) 4860 else: 4861 # don't put any right brackets around non-positional-only parameters, ever. 4862 p.right_bracket_count = 0 4863 4864 right_bracket_count = 0 4865 4866 def fix_right_bracket_count(desired): 4867 nonlocal right_bracket_count 4868 s = '' 4869 while right_bracket_count < desired: 4870 s += '[' 4871 right_bracket_count += 1 4872 while right_bracket_count > desired: 4873 s += ']' 4874 right_bracket_count -= 1 4875 return s 4876 4877 need_slash = False 4878 added_slash = False 4879 need_a_trailing_slash = False 4880 4881 # we only need a trailing slash: 4882 # * if this is not a "docstring_only" signature 4883 # * and if the last *shown* parameter is 4884 # positional only 4885 if not f.docstring_only: 4886 for p in reversed(parameters): 4887 if not p.converter.show_in_signature: 4888 continue 4889 if p.is_positional_only(): 4890 need_a_trailing_slash = True 4891 break 4892 4893 4894 added_star = False 4895 4896 first_parameter = True 4897 last_p = parameters[-1] 4898 line_length = len(''.join(text)) 4899 indent = " " * line_length 4900 def add_parameter(text): 4901 nonlocal line_length 4902 nonlocal first_parameter 4903 if first_parameter: 4904 s = text 4905 first_parameter = False 4906 else: 4907 s = ' ' + text 4908 if line_length + len(s) >= 72: 4909 add('\n') 4910 add(indent) 4911 line_length = len(indent) 4912 s = text 4913 line_length += len(s) 4914 add(s) 4915 4916 for p in parameters: 4917 if not p.converter.show_in_signature: 4918 continue 4919 assert p.name 4920 4921 is_self = isinstance(p.converter, self_converter) 4922 if is_self and f.docstring_only: 4923 # this isn't a real machine-parsable signature, 4924 # so let's not print the "self" parameter 4925 continue 4926 4927 if p.is_positional_only(): 4928 need_slash = not f.docstring_only 4929 elif need_slash and not (added_slash or p.is_positional_only()): 4930 added_slash = True 4931 add_parameter('/,') 4932 4933 if p.is_keyword_only() and not added_star: 4934 added_star = True 4935 add_parameter('*,') 4936 4937 p_add, p_output = text_accumulator() 4938 p_add(fix_right_bracket_count(p.right_bracket_count)) 4939 4940 if isinstance(p.converter, self_converter): 4941 # annotate first parameter as being a "self". 4942 # 4943 # if inspect.Signature gets this function, 4944 # and it's already bound, the self parameter 4945 # will be stripped off. 4946 # 4947 # if it's not bound, it should be marked 4948 # as positional-only. 4949 # 4950 # note: we don't print "self" for __init__, 4951 # because this isn't actually the signature 4952 # for __init__. (it can't be, __init__ doesn't 4953 # have a docstring.) if this is an __init__ 4954 # (or __new__), then this signature is for 4955 # calling the class to construct a new instance. 4956 p_add('$') 4957 4958 if p.is_vararg(): 4959 p_add("*") 4960 4961 name = p.converter.signature_name or p.name 4962 p_add(name) 4963 4964 if not p.is_vararg() and p.converter.is_optional(): 4965 p_add('=') 4966 value = p.converter.py_default 4967 if not value: 4968 value = repr(p.converter.default) 4969 p_add(value) 4970 4971 if (p != last_p) or need_a_trailing_slash: 4972 p_add(',') 4973 4974 add_parameter(p_output()) 4975 4976 add(fix_right_bracket_count(0)) 4977 if need_a_trailing_slash: 4978 add_parameter('/') 4979 add(')') 4980 4981 # PEP 8 says: 4982 # 4983 # The Python standard library will not use function annotations 4984 # as that would result in a premature commitment to a particular 4985 # annotation style. Instead, the annotations are left for users 4986 # to discover and experiment with useful annotation styles. 4987 # 4988 # therefore this is commented out: 4989 # 4990 # if f.return_converter.py_default: 4991 # add(' -> ') 4992 # add(f.return_converter.py_default) 4993 4994 if not f.docstring_only: 4995 add("\n" + sig_end_marker + "\n") 4996 4997 docstring_first_line = output() 4998 4999 # now fix up the places where the brackets look wrong 5000 docstring_first_line = docstring_first_line.replace(', ]', ',] ') 5001 5002 # okay. now we're officially building the "parameters" section. 5003 # create substitution text for {parameters} 5004 spacer_line = False 5005 for p in parameters: 5006 if not p.docstring.strip(): 5007 continue 5008 if spacer_line: 5009 add('\n') 5010 else: 5011 spacer_line = True 5012 add(" ") 5013 add(p.name) 5014 add('\n') 5015 add(textwrap.indent(rstrip_lines(p.docstring.rstrip()), " ")) 5016 parameters = output() 5017 if parameters: 5018 parameters += '\n' 5019 5020 ## 5021 ## docstring body 5022 ## 5023 5024 docstring = f.docstring.rstrip() 5025 lines = [line.rstrip() for line in docstring.split('\n')] 5026 5027 # Enforce the summary line! 5028 # The first line of a docstring should be a summary of the function. 5029 # It should fit on one line (80 columns? 79 maybe?) and be a paragraph 5030 # by itself. 5031 # 5032 # Argument Clinic enforces the following rule: 5033 # * either the docstring is empty, 5034 # * or it must have a summary line. 5035 # 5036 # Guido said Clinic should enforce this: 5037 # http://mail.python.org/pipermail/python-dev/2013-June/127110.html 5038 5039 if len(lines) >= 2: 5040 if lines[1]: 5041 fail("Docstring for " + f.full_name + " does not have a summary line!\n" + 5042 "Every non-blank function docstring must start with\n" + 5043 "a single line summary followed by an empty line.") 5044 elif len(lines) == 1: 5045 # the docstring is only one line right now--the summary line. 5046 # add an empty line after the summary line so we have space 5047 # between it and the {parameters} we're about to add. 5048 lines.append('') 5049 5050 parameters_marker_count = len(docstring.split('{parameters}')) - 1 5051 if parameters_marker_count > 1: 5052 fail('You may not specify {parameters} more than once in a docstring!') 5053 5054 if not parameters_marker_count: 5055 # insert after summary line 5056 lines.insert(2, '{parameters}') 5057 5058 # insert at front of docstring 5059 lines.insert(0, docstring_first_line) 5060 5061 docstring = "\n".join(lines) 5062 5063 add(docstring) 5064 docstring = output() 5065 5066 docstring = linear_format(docstring, parameters=parameters) 5067 docstring = docstring.rstrip() 5068 5069 return docstring 5070 5071 def state_terminal(self, line): 5072 """ 5073 Called when processing the block is done. 5074 """ 5075 assert not line 5076 5077 if not self.function: 5078 return 5079 5080 if self.keyword_only: 5081 values = self.function.parameters.values() 5082 if not values: 5083 no_parameter_after_star = True 5084 else: 5085 last_parameter = next(reversed(list(values))) 5086 no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY 5087 if no_parameter_after_star: 5088 fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") 5089 5090 # remove trailing whitespace from all parameter docstrings 5091 for name, value in self.function.parameters.items(): 5092 if not value: 5093 continue 5094 value.docstring = value.docstring.rstrip() 5095 5096 self.function.docstring = self.format_docstring() 5097 5098 5099 5100 5101# maps strings to callables. 5102# the callable should return an object 5103# that implements the clinic parser 5104# interface (__init__ and parse). 5105# 5106# example parsers: 5107# "clinic", handles the Clinic DSL 5108# "python", handles running Python code 5109# 5110parsers = {'clinic' : DSLParser, 'python': PythonParser} 5111 5112 5113clinic = None 5114 5115 5116def main(argv): 5117 import sys 5118 5119 if sys.version_info.major < 3 or sys.version_info.minor < 3: 5120 sys.exit("Error: clinic.py requires Python 3.3 or greater.") 5121 5122 import argparse 5123 cmdline = argparse.ArgumentParser( 5124 description="""Preprocessor for CPython C files. 5125 5126The purpose of the Argument Clinic is automating all the boilerplate involved 5127with writing argument parsing code for builtins and providing introspection 5128signatures ("docstrings") for CPython builtins. 5129 5130For more information see https://docs.python.org/3/howto/clinic.html""") 5131 cmdline.add_argument("-f", "--force", action='store_true') 5132 cmdline.add_argument("-o", "--output", type=str) 5133 cmdline.add_argument("-v", "--verbose", action='store_true') 5134 cmdline.add_argument("--converters", action='store_true') 5135 cmdline.add_argument("--make", action='store_true', 5136 help="Walk --srcdir to run over all relevant files.") 5137 cmdline.add_argument("--srcdir", type=str, default=os.curdir, 5138 help="The directory tree to walk in --make mode.") 5139 cmdline.add_argument("filename", type=str, nargs="*") 5140 ns = cmdline.parse_args(argv) 5141 5142 if ns.converters: 5143 if ns.filename: 5144 print("Usage error: can't specify --converters and a filename at the same time.") 5145 print() 5146 cmdline.print_usage() 5147 sys.exit(-1) 5148 converters = [] 5149 return_converters = [] 5150 ignored = set(""" 5151 add_c_converter 5152 add_c_return_converter 5153 add_default_legacy_c_converter 5154 add_legacy_c_converter 5155 """.strip().split()) 5156 module = globals() 5157 for name in module: 5158 for suffix, ids in ( 5159 ("_return_converter", return_converters), 5160 ("_converter", converters), 5161 ): 5162 if name in ignored: 5163 continue 5164 if name.endswith(suffix): 5165 ids.append((name, name[:-len(suffix)])) 5166 break 5167 print() 5168 5169 print("Legacy converters:") 5170 legacy = sorted(legacy_converters) 5171 print(' ' + ' '.join(c for c in legacy if c[0].isupper())) 5172 print(' ' + ' '.join(c for c in legacy if c[0].islower())) 5173 print() 5174 5175 for title, attribute, ids in ( 5176 ("Converters", 'converter_init', converters), 5177 ("Return converters", 'return_converter_init', return_converters), 5178 ): 5179 print(title + ":") 5180 longest = -1 5181 for name, short_name in ids: 5182 longest = max(longest, len(short_name)) 5183 for name, short_name in sorted(ids, key=lambda x: x[1].lower()): 5184 cls = module[name] 5185 callable = getattr(cls, attribute, None) 5186 if not callable: 5187 continue 5188 signature = inspect.signature(callable) 5189 parameters = [] 5190 for parameter_name, parameter in signature.parameters.items(): 5191 if parameter.kind == inspect.Parameter.KEYWORD_ONLY: 5192 if parameter.default != inspect.Parameter.empty: 5193 s = '{}={!r}'.format(parameter_name, parameter.default) 5194 else: 5195 s = parameter_name 5196 parameters.append(s) 5197 print(' {}({})'.format(short_name, ', '.join(parameters))) 5198 print() 5199 print("All converters also accept (c_default=None, py_default=None, annotation=None).") 5200 print("All return converters also accept (py_default=None).") 5201 sys.exit(0) 5202 5203 if ns.make: 5204 if ns.output or ns.filename: 5205 print("Usage error: can't use -o or filenames with --make.") 5206 print() 5207 cmdline.print_usage() 5208 sys.exit(-1) 5209 if not ns.srcdir: 5210 print("Usage error: --srcdir must not be empty with --make.") 5211 print() 5212 cmdline.print_usage() 5213 sys.exit(-1) 5214 for root, dirs, files in os.walk(ns.srcdir): 5215 for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): 5216 if rcs_dir in dirs: 5217 dirs.remove(rcs_dir) 5218 for filename in files: 5219 if not (filename.endswith('.c') or filename.endswith('.h')): 5220 continue 5221 path = os.path.join(root, filename) 5222 if ns.verbose: 5223 print(path) 5224 parse_file(path, verify=not ns.force) 5225 return 5226 5227 if not ns.filename: 5228 cmdline.print_usage() 5229 sys.exit(-1) 5230 5231 if ns.output and len(ns.filename) > 1: 5232 print("Usage error: can't use -o with multiple filenames.") 5233 print() 5234 cmdline.print_usage() 5235 sys.exit(-1) 5236 5237 for filename in ns.filename: 5238 if ns.verbose: 5239 print(filename) 5240 parse_file(filename, output=ns.output, verify=not ns.force) 5241 5242 5243if __name__ == "__main__": 5244 sys.exit(main(sys.argv[1:])) 5245