1# Copyright 2021, Kay Hayen, mailto:kay.hayen@gmail.com 2# 3# Part of "Nuitka", an optimizing Python compiler that is compatible and 4# integrates with CPython, but also works on its own. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18""" Optimizations of built-ins to built-in calls. 19 20""" 21from __future__ import print_function 22 23import math 24 25from nuitka.__past__ import builtins 26from nuitka.PythonVersions import python_version 27from nuitka.Tracing import optimization_logger 28 29from .ParameterSpecs import ParameterSpec, TooManyArguments, matchCall 30 31 32class BuiltinParameterSpec(ParameterSpec): 33 __slots__ = ("builtin",) 34 35 def __init__( 36 self, 37 name, 38 arg_names, 39 default_count, 40 list_star_arg=None, 41 dict_star_arg=None, 42 pos_only_args=(), 43 kw_only_args=(), 44 ): 45 ParameterSpec.__init__( 46 self, 47 ps_name=name, 48 ps_normal_args=arg_names, 49 ps_list_star_arg=list_star_arg, 50 ps_dict_star_arg=dict_star_arg, 51 ps_default_count=default_count, 52 ps_pos_only_args=pos_only_args, 53 ps_kw_only_args=kw_only_args, 54 ) 55 56 self.builtin = getattr(builtins, name, None) 57 58 assert default_count <= len(arg_names) + len(kw_only_args) + len(pos_only_args) 59 60 def __repr__(self): 61 return "<BuiltinParameterSpec %s>" % self.name 62 63 def getName(self): 64 return self.name 65 66 def isCompileTimeComputable(self, values): 67 # By default, we make this dependent on the ability to compute the 68 # arguments, which is of course a good start for most cases, so this 69 # is for overloads, pylint: disable=no-self-use 70 71 for value in values: 72 if value is not None and not value.isCompileTimeConstant(): 73 return False 74 75 return True 76 77 def simulateCall(self, given_values): 78 # Using star dict call for simulation and catch any exception as really 79 # fatal, pylint: disable=broad-except,too-many-branches 80 81 try: 82 given_normal_args = given_values[: self.getArgumentCount()] 83 84 if self.list_star_arg: 85 given_list_star_args = given_values[self.getArgumentCount()] 86 else: 87 given_list_star_args = None 88 89 if self.dict_star_arg: 90 given_dict_star_args = given_values[-1] 91 else: 92 given_dict_star_args = None 93 94 arg_dict = {} 95 96 for arg_name, given_value in zip( 97 self.getArgumentNames(), given_normal_args 98 ): 99 assert type(given_value) not in ( 100 tuple, 101 list, 102 ), "do not like a tuple %s" % (given_value,) 103 104 if given_value is not None: 105 arg_dict[arg_name] = given_value.getCompileTimeConstant() 106 107 if given_dict_star_args: 108 for given_dict_star_arg in reversed(given_dict_star_args): 109 arg_name = given_dict_star_arg.subnode_key.getCompileTimeConstant() 110 arg_value = ( 111 given_dict_star_arg.subnode_value.getCompileTimeConstant() 112 ) 113 114 arg_dict[arg_name] = arg_value 115 116 arg_list = [] 117 118 for arg_name in self.getArgumentNames(): 119 if arg_name not in arg_dict: 120 break 121 122 arg_list.append(arg_dict[arg_name]) 123 del arg_dict[arg_name] 124 125 except Exception as e: 126 optimization_logger.sysexit_exception("Fatal optimization problem", e) 127 128 if given_list_star_args: 129 return self.builtin( 130 *( 131 arg_list 132 + list( 133 value.getCompileTimeConstant() for value in given_list_star_args 134 ) 135 ), 136 **arg_dict 137 ) 138 else: 139 return self.builtin(*arg_list, **arg_dict) 140 141 142class BuiltinParameterSpecNoKeywords(BuiltinParameterSpec): 143 __slots__ = () 144 145 def allowsKeywords(self): 146 return False 147 148 def simulateCall(self, given_values): 149 # Using star dict call for simulation and catch any exception as really fatal, 150 # pylint: disable=broad-except 151 152 try: 153 if self.list_star_arg: 154 given_list_star_arg = given_values[self.getArgumentCount()] 155 else: 156 given_list_star_arg = None 157 158 arg_list = [] 159 refuse_more = False 160 161 for _arg_name, given_value in zip(self.getArgumentNames(), given_values): 162 assert type(given_value) not in ( 163 tuple, 164 list, 165 ), "do not like tuple %s" % (given_value,) 166 167 if given_value is not None: 168 if not refuse_more: 169 arg_list.append(given_value.getCompileTimeConstant()) 170 else: 171 assert False 172 else: 173 refuse_more = True 174 175 if given_list_star_arg is not None: 176 arg_list += [ 177 value.getCompileTimeConstant() for value in given_list_star_arg 178 ] 179 except Exception as e: 180 optimization_logger.sysexit_exception("matching call", e) 181 182 return self.builtin(*arg_list) 183 184 185class BuiltinParameterSpecExceptionsKwOnly(BuiltinParameterSpec): 186 def __init__(self, exception_name, kw_only_args): 187 BuiltinParameterSpec.__init__( 188 self, 189 name=exception_name, 190 arg_names=(), 191 default_count=len(kw_only_args), # For exceptions, they will be required. 192 list_star_arg="args", 193 kw_only_args=kw_only_args, 194 ) 195 196 197class BuiltinParameterSpecExceptions(BuiltinParameterSpec): 198 def __init__(self, exception_name): 199 BuiltinParameterSpec.__init__( 200 self, 201 name=exception_name, 202 arg_names=(), 203 default_count=0, 204 list_star_arg="args", 205 ) 206 207 def allowsKeywords(self): 208 return False 209 210 def getKeywordRefusalText(self): 211 return "exceptions.%s does not take keyword arguments" % self.name 212 213 def getCallableName(self): 214 return "exceptions." + self.getName() 215 216 217def makeBuiltinExceptionParameterSpec(exception_name): 218 """Factory function to create parameter spec for an exception from its name. 219 220 Args: 221 exception_name - (str) name of the built-in exception 222 223 Returns: 224 ParameterSpec that can be used to evaluate calls of these exceptions. 225 """ 226 if exception_name == "ImportError" and python_version >= 0x300: 227 # This is currently the only known built-in exception that does it, but let's 228 # be general, as surely that list is going to expand only. 229 230 return BuiltinParameterSpecExceptionsKwOnly( 231 exception_name=exception_name, kw_only_args=("name", "path") 232 ) 233 else: 234 return BuiltinParameterSpecExceptions(exception_name=exception_name) 235 236 237class BuiltinParameterSpecPosArgs(BuiltinParameterSpec): 238 def __init__( 239 self, 240 name, 241 pos_only_args, 242 arg_names, 243 default_count, 244 list_star_arg=None, 245 dict_star_arg=None, 246 ): 247 BuiltinParameterSpec.__init__( 248 self, 249 name=name, 250 arg_names=arg_names, 251 default_count=default_count, 252 pos_only_args=pos_only_args, 253 list_star_arg=list_star_arg, 254 dict_star_arg=dict_star_arg, 255 ) 256 257 258if python_version < 0x370: 259 builtin_int_spec = BuiltinParameterSpec("int", ("x", "base"), default_count=2) 260else: 261 builtin_int_spec = BuiltinParameterSpecPosArgs( 262 "int", ("x",), ("base",), default_count=2 263 ) 264 265 266# These builtins are only available for Python2 267if python_version < 0x300: 268 builtin_long_spec = BuiltinParameterSpec("long", ("x", "base"), default_count=2) 269 builtin_execfile_spec = BuiltinParameterSpecNoKeywords( 270 "execfile", ("filename", "globals", "locals"), default_count=2 271 ) 272 273builtin_unicode_p2_spec = BuiltinParameterSpec( 274 "unicode", ("string", "encoding", "errors"), default_count=3 275) 276 277builtin_xrange_spec = BuiltinParameterSpecNoKeywords( 278 "xrange" if python_version < 0x300 else "range", 279 ("start", "stop", "step"), 280 default_count=2, 281) 282 283 284if python_version < 0x370: 285 builtin_bool_spec = BuiltinParameterSpec("bool", ("x",), default_count=1) 286else: 287 builtin_bool_spec = BuiltinParameterSpecNoKeywords("bool", ("x",), default_count=1) 288 289if python_version < 0x370: 290 builtin_float_spec = BuiltinParameterSpec("float", ("x",), default_count=1) 291else: 292 builtin_float_spec = BuiltinParameterSpecNoKeywords( 293 "float", ("x",), default_count=1 294 ) 295 296builtin_complex_spec = BuiltinParameterSpec( 297 "complex", ("real", "imag"), default_count=2 298) 299 300# This built-in have variable parameters for Python2/3 301if python_version < 0x300: 302 builtin_str_spec = BuiltinParameterSpec("str", ("object",), default_count=1) 303else: 304 builtin_str_spec = BuiltinParameterSpec( 305 "str", ("object", "encoding", "errors"), default_count=3 306 ) 307 308builtin_len_spec = BuiltinParameterSpecNoKeywords("len", ("object",), default_count=0) 309builtin_dict_spec = BuiltinParameterSpec( 310 "dict", (), default_count=0, list_star_arg="list_args", dict_star_arg="dict_args" 311) 312builtin_any_spec = BuiltinParameterSpecNoKeywords("any", ("object",), default_count=0) 313builtin_abs_spec = BuiltinParameterSpecNoKeywords("abs", ("object",), default_count=0) 314builtin_all_spec = BuiltinParameterSpecNoKeywords("all", ("object",), default_count=0) 315 316if python_version < 0x370: 317 builtin_tuple_spec = BuiltinParameterSpec("tuple", ("sequence",), default_count=1) 318 builtin_list_spec = BuiltinParameterSpec("list", ("sequence",), default_count=1) 319else: 320 builtin_tuple_spec = BuiltinParameterSpecNoKeywords( 321 "tuple", ("sequence",), default_count=1 322 ) 323 builtin_list_spec = BuiltinParameterSpecNoKeywords( 324 "list", ("sequence",), default_count=1 325 ) 326 327builtin_set_spec = BuiltinParameterSpecNoKeywords("set", ("iterable",), default_count=1) 328builtin_frozenset_spec = BuiltinParameterSpecNoKeywords( 329 "frozenset", ("iterable",), default_count=1 330) 331 332builtin_import_spec = BuiltinParameterSpec( 333 "__import__", ("name", "globals", "locals", "fromlist", "level"), default_count=4 334) 335 336if python_version < 0x300: 337 builtin_open_spec = BuiltinParameterSpec( 338 "open", ("name", "mode", "buffering"), default_count=3 339 ) 340else: 341 builtin_open_spec = BuiltinParameterSpec( 342 "open", 343 ( 344 "file", 345 "mode", 346 "buffering", 347 "encoding", 348 "errors", 349 "newline", 350 "closefd", 351 "opener", 352 ), 353 default_count=7, 354 ) 355 356builtin_chr_spec = BuiltinParameterSpecNoKeywords("chr", ("i",), default_count=0) 357builtin_ord_spec = BuiltinParameterSpecNoKeywords("ord", ("c",), default_count=0) 358builtin_bin_spec = BuiltinParameterSpecNoKeywords("bin", ("number",), default_count=0) 359builtin_oct_spec = BuiltinParameterSpecNoKeywords("oct", ("number",), default_count=0) 360builtin_hex_spec = BuiltinParameterSpecNoKeywords("hex", ("number",), default_count=0) 361builtin_id_spec = BuiltinParameterSpecNoKeywords("id", ("object",), default_count=0) 362builtin_repr_spec = BuiltinParameterSpecNoKeywords("repr", ("object",), default_count=0) 363 364builtin_dir_spec = BuiltinParameterSpecNoKeywords("dir", ("object",), default_count=1) 365builtin_vars_spec = BuiltinParameterSpecNoKeywords("vars", ("object",), default_count=1) 366 367builtin_locals_spec = BuiltinParameterSpecNoKeywords("locals", (), default_count=0) 368builtin_globals_spec = BuiltinParameterSpecNoKeywords("globals", (), default_count=0) 369builtin_eval_spec = BuiltinParameterSpecNoKeywords( 370 "eval", ("source", "globals", "locals"), 2 371) 372if python_version < 0x300: 373 builtin_compile_spec = BuiltinParameterSpec( 374 "compile", 375 ("source", "filename", "mode", "flags", "dont_inherit"), 376 default_count=2, 377 ) 378else: 379 builtin_compile_spec = BuiltinParameterSpec( 380 "compile", 381 ("source", "filename", "mode", "flags", "dont_inherit", "optimize"), 382 default_count=3, 383 ) 384if python_version >= 0x300: 385 builtin_exec_spec = BuiltinParameterSpecNoKeywords( 386 "exec", ("source", "globals", "locals"), default_count=2 387 ) 388 389# Note: Iter in fact names its first argument if the default applies 390# "collection", fixed up in a wrapper. 391builtin_iter_spec = BuiltinParameterSpecNoKeywords( 392 "iter", ("callable", "sentinel"), default_count=1 393) 394builtin_next_spec = BuiltinParameterSpecNoKeywords( 395 "next", ("iterator", "default"), default_count=1 396) 397 398# Note: type with 1 and type with 3 arguments are too different. 399builtin_type1_spec = BuiltinParameterSpecNoKeywords( 400 "type", ("object",), default_count=0 401) 402builtin_type3_spec = BuiltinParameterSpecNoKeywords( 403 "type", ("name", "bases", "dict"), default_count=0 404) 405 406builtin_super_spec = BuiltinParameterSpecNoKeywords( 407 "super", ("type", "object"), default_count=1 if python_version < 0x300 else 2 408) 409 410builtin_hasattr_spec = BuiltinParameterSpecNoKeywords( 411 "hasattr", ("object", "name"), default_count=0 412) 413builtin_getattr_spec = BuiltinParameterSpecNoKeywords( 414 "getattr", ("object", "name", "default"), default_count=1 415) 416builtin_setattr_spec = BuiltinParameterSpecNoKeywords( 417 "setattr", ("object", "name", "value"), default_count=0 418) 419 420builtin_isinstance_spec = BuiltinParameterSpecNoKeywords( 421 "isinstance", ("instance", "classes"), default_count=0 422) 423 424builtin_issubclass_spec = BuiltinParameterSpecNoKeywords( 425 "issubclass", ("cls", "classes"), default_count=0 426) 427 428 429class BuiltinBytearraySpec(BuiltinParameterSpecPosArgs): 430 def isCompileTimeComputable(self, values): 431 # For bytearrays, we need to avoid the case of large bytearray 432 # construction from an integer at compile time. 433 434 result = BuiltinParameterSpec.isCompileTimeComputable(self, values=values) 435 436 if result and len(values) == 1: 437 index_value = values[0].getIndexValue() 438 439 if index_value is None: 440 return result 441 442 return index_value < 256 443 else: 444 return result 445 446 447builtin_bytearray_spec = BuiltinBytearraySpec( 448 "bytearray", ("string",), ("encoding", "errors"), default_count=2 449) 450 451builtin_bytes_p3_spec = BuiltinBytearraySpec( 452 "bytes", ("string",), ("encoding", "errors"), default_count=3 453) 454 455 456# Beware: One argument version defines "stop", not "start". 457builtin_slice_spec = BuiltinParameterSpecNoKeywords( 458 "slice", ("start", "stop", "step"), default_count=2 459) 460 461builtin_hash_spec = BuiltinParameterSpecNoKeywords("hash", ("object",), default_count=0) 462 463builtin_format_spec = BuiltinParameterSpecNoKeywords( 464 "format", ("value", "format_spec"), default_count=1 465) 466 467if python_version < 0x380: 468 builtin_sum_spec = BuiltinParameterSpecNoKeywords( 469 "sum", ("sequence", "start"), default_count=1 470 ) 471else: 472 builtin_sum_spec = BuiltinParameterSpecPosArgs( 473 "sum", ("sequence",), ("start",), default_count=1 474 ) 475 476builtin_staticmethod_spec = BuiltinParameterSpecNoKeywords( 477 "staticmethod", ("function",), default_count=0 478) 479builtin_classmethod_spec = BuiltinParameterSpecNoKeywords( 480 "classmethod", ("function",), default_count=0 481) 482 483if python_version < 0x300: 484 builtin_sorted_spec = BuiltinParameterSpecNoKeywords( 485 "sorted", ("iterable", "cmp", "key", "reverse"), default_count=2 486 ) 487else: 488 builtin_sorted_spec = BuiltinParameterSpecNoKeywords( 489 "sorted", ("iterable", "key", "reverse"), default_count=2 490 ) 491 492builtin_reversed_spec = BuiltinParameterSpecNoKeywords( 493 "reversed", ("object",), default_count=0 494) 495 496builtin_reversed_spec = BuiltinParameterSpecNoKeywords( 497 "reversed", ("object",), default_count=0 498) 499 500if python_version < 0x300: 501 builtin_enumerate_spec = BuiltinParameterSpec( 502 "enumerate", ("sequence", "start"), default_count=1 503 ) 504else: 505 builtin_enumerate_spec = BuiltinParameterSpec( 506 "enumerate", ("iterable", "start"), default_count=1 507 ) 508 509 510class BuiltinRangeSpec(BuiltinParameterSpecNoKeywords): 511 def isCompileTimeComputable(self, values): 512 # For ranges, we need have many cases that can prevent the ability 513 # to pre-compute, pylint: disable=too-many-branches,too-many-return-statements 514 515 result = BuiltinParameterSpecNoKeywords.isCompileTimeComputable( 516 self, values=values 517 ) 518 519 if result: 520 arg_count = len(values) 521 522 if arg_count == 1: 523 low = values[0] 524 525 # If it's not a number constant, we can compute the exception 526 # that will be raised. 527 if not low.isNumberConstant(): 528 return True 529 530 return low.getCompileTimeConstant() < 256 531 elif arg_count == 2: 532 low, high = values 533 534 # If it's not a number constant, we can compute the exception 535 # that will be raised. 536 if not low.isNumberConstant() or not high.isNumberConstant(): 537 return True 538 539 return ( 540 high.getCompileTimeConstant() - low.getCompileTimeConstant() < 256 541 ) 542 elif arg_count == 3: 543 low, high, step = values 544 545 if ( 546 not low.isNumberConstant() 547 or not high.isNumberConstant() 548 or not step.isNumberConstant() 549 ): 550 return True 551 552 low = low.getCompileTimeConstant() 553 high = high.getCompileTimeConstant() 554 step = step.getCompileTimeConstant() 555 556 # It's going to give a ZeroDivisionError in this case. 557 if step == 0: 558 return True 559 560 if low < high: 561 if step < 0: 562 return True 563 else: 564 return math.ceil(float(high - low) / step) < 256 565 else: 566 if step > 0: 567 return True 568 else: 569 return math.ceil(float(high - low) / step) < 256 570 else: 571 assert False 572 else: 573 return False 574 575 576builtin_range_spec = BuiltinRangeSpec( 577 "range", ("start", "stop", "step"), default_count=2 578) 579 580if python_version >= 0x300: 581 builtin_ascii_spec = BuiltinParameterSpecNoKeywords( 582 "ascii", ("object",), default_count=0 583 ) 584 585 586builtin_divmod_spec = BuiltinParameterSpecNoKeywords( 587 "divmod", ("left", "right"), default_count=0 588) 589 590 591def extractBuiltinArgs(node, builtin_spec, builtin_class, empty_special_class=None): 592 # Many cases to deal with, pylint: disable=too-many-branches 593 594 try: 595 kw = node.subnode_kwargs 596 597 # TODO: Could check for too many / too few, even if they are unknown, we 598 # might raise that error, but that need not be optimized immediately. 599 if kw is not None: 600 if not kw.isMappingWithConstantStringKeys(): 601 return None 602 603 pairs = kw.getMappingStringKeyPairs() 604 605 if pairs and not builtin_spec.allowsKeywords(): 606 raise TooManyArguments(TypeError(builtin_spec.getKeywordRefusalText())) 607 else: 608 pairs = () 609 610 args = node.subnode_args 611 612 if args: 613 if not args.canPredictIterationValues(): 614 return None 615 616 positional = args.getIterationValues() 617 else: 618 positional = () 619 620 if not positional and not pairs and empty_special_class is not None: 621 return empty_special_class(source_ref=node.getSourceReference()) 622 623 args_dict = matchCall( 624 func_name=builtin_spec.getName(), 625 args=builtin_spec.getArgumentNames(), 626 kw_only_args=builtin_spec.getKwOnlyParameterNames(), 627 star_list_arg=builtin_spec.getStarListArgumentName(), 628 star_dict_arg=builtin_spec.getStarDictArgumentName(), 629 num_defaults=builtin_spec.getDefaultCount(), 630 num_posonly=builtin_spec.getPosOnlyParameterCount(), 631 positional=positional, 632 pairs=pairs, 633 ) 634 except TooManyArguments as e: 635 from nuitka.nodes.NodeMakingHelpers import ( 636 makeRaiseExceptionReplacementExpressionFromInstance, 637 wrapExpressionWithSideEffects, 638 ) 639 640 return wrapExpressionWithSideEffects( 641 new_node=makeRaiseExceptionReplacementExpressionFromInstance( 642 expression=node, exception=e.getRealException() 643 ), 644 old_node=node, 645 side_effects=node.extractSideEffectsPreCall(), 646 ) 647 648 # Using list reference for passing the arguments without names where it 649 # it possible, otherwise dictionary to make those distinuishable. 650 args_list = [] 651 652 for argument_name in builtin_spec.getArgumentNames(): 653 args_list.append(args_dict[argument_name]) 654 655 if builtin_spec.getStarListArgumentName() is not None: 656 args_list.append(args_dict[builtin_spec.getStarListArgumentName()]) 657 658 if builtin_spec.getStarDictArgumentName() is not None: 659 args_list.append(args_dict[builtin_spec.getStarDictArgumentName()]) 660 661 for argument_name in builtin_spec.getKwOnlyParameterNames(): 662 args_list.append(args_dict[argument_name]) 663 664 # Using list reference for passing the arguments without names, 665 result = builtin_class(*args_list, source_ref=node.getSourceReference()) 666 667 if python_version < 0x380: 668 result.setCompatibleSourceReference(node.getCompatibleSourceReference()) 669 670 return result 671