1# cython: language_level = 2 2# cython: auto_pickle=False 3# 4# Code output module 5# 6 7from __future__ import absolute_import 8 9import cython 10cython.declare(os=object, re=object, operator=object, textwrap=object, 11 Template=object, Naming=object, Options=object, StringEncoding=object, 12 Utils=object, SourceDescriptor=object, StringIOTree=object, 13 DebugFlags=object, basestring=object, defaultdict=object, 14 closing=object, partial=object) 15 16import os 17import re 18import shutil 19import sys 20import operator 21import textwrap 22from string import Template 23from functools import partial 24from contextlib import closing 25from collections import defaultdict 26 27try: 28 import hashlib 29except ImportError: 30 import md5 as hashlib 31 32from . import Naming 33from . import Options 34from . import DebugFlags 35from . import StringEncoding 36from . import Version 37from .. import Utils 38from .Scanning import SourceDescriptor 39from ..StringIOTree import StringIOTree 40 41try: 42 from __builtin__ import basestring 43except ImportError: 44 from builtins import str as basestring 45 46KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7) 47 48 49non_portable_builtins_map = { 50 # builtins that have different names in different Python versions 51 'bytes' : ('PY_MAJOR_VERSION < 3', 'str'), 52 'unicode' : ('PY_MAJOR_VERSION >= 3', 'str'), 53 'basestring' : ('PY_MAJOR_VERSION >= 3', 'str'), 54 'xrange' : ('PY_MAJOR_VERSION >= 3', 'range'), 55 'raw_input' : ('PY_MAJOR_VERSION >= 3', 'input'), 56} 57 58ctypedef_builtins_map = { 59 # types of builtins in "ctypedef class" statements which we don't 60 # import either because the names conflict with C types or because 61 # the type simply is not exposed. 62 'py_int' : '&PyInt_Type', 63 'py_long' : '&PyLong_Type', 64 'py_float' : '&PyFloat_Type', 65 'wrapper_descriptor' : '&PyWrapperDescr_Type', 66} 67 68basicsize_builtins_map = { 69 # builtins whose type has a different tp_basicsize than sizeof(...) 70 'PyTypeObject': 'PyHeapTypeObject', 71} 72 73uncachable_builtins = [ 74 # Global/builtin names that cannot be cached because they may or may not 75 # be available at import time, for various reasons: 76 ## - Py3.7+ 77 'breakpoint', # might deserve an implementation in Cython 78 ## - Py3.4+ 79 '__loader__', 80 '__spec__', 81 ## - Py3+ 82 'BlockingIOError', 83 'BrokenPipeError', 84 'ChildProcessError', 85 'ConnectionAbortedError', 86 'ConnectionError', 87 'ConnectionRefusedError', 88 'ConnectionResetError', 89 'FileExistsError', 90 'FileNotFoundError', 91 'InterruptedError', 92 'IsADirectoryError', 93 'ModuleNotFoundError', 94 'NotADirectoryError', 95 'PermissionError', 96 'ProcessLookupError', 97 'RecursionError', 98 'ResourceWarning', 99 #'StopAsyncIteration', # backported 100 'TimeoutError', 101 '__build_class__', 102 'ascii', # might deserve an implementation in Cython 103 #'exec', # implemented in Cython 104 ## - Py2.7+ 105 'memoryview', 106 ## - platform specific 107 'WindowsError', 108 ## - others 109 '_', # e.g. used by gettext 110] 111 112special_py_methods = set([ 113 '__cinit__', '__dealloc__', '__richcmp__', '__next__', 114 '__await__', '__aiter__', '__anext__', 115 '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', 116 '__getcharbuffer__', '__getbuffer__', '__releasebuffer__' 117]) 118 119modifier_output_mapper = { 120 'inline': 'CYTHON_INLINE' 121}.get 122 123 124class IncludeCode(object): 125 """ 126 An include file and/or verbatim C code to be included in the 127 generated sources. 128 """ 129 # attributes: 130 # 131 # pieces {order: unicode}: pieces of C code to be generated. 132 # For the included file, the key "order" is zero. 133 # For verbatim include code, the "order" is the "order" 134 # attribute of the original IncludeCode where this piece 135 # of C code was first added. This is needed to prevent 136 # duplication if the same include code is found through 137 # multiple cimports. 138 # location int: where to put this include in the C sources, one 139 # of the constants INITIAL, EARLY, LATE 140 # order int: sorting order (automatically set by increasing counter) 141 142 # Constants for location. If the same include occurs with different 143 # locations, the earliest one takes precedense. 144 INITIAL = 0 145 EARLY = 1 146 LATE = 2 147 148 counter = 1 # Counter for "order" 149 150 def __init__(self, include=None, verbatim=None, late=True, initial=False): 151 self.order = self.counter 152 type(self).counter += 1 153 self.pieces = {} 154 155 if include: 156 if include[0] == '<' and include[-1] == '>': 157 self.pieces[0] = u'#include {0}'.format(include) 158 late = False # system include is never late 159 else: 160 self.pieces[0] = u'#include "{0}"'.format(include) 161 162 if verbatim: 163 self.pieces[self.order] = verbatim 164 165 if initial: 166 self.location = self.INITIAL 167 elif late: 168 self.location = self.LATE 169 else: 170 self.location = self.EARLY 171 172 def dict_update(self, d, key): 173 """ 174 Insert `self` in dict `d` with key `key`. If that key already 175 exists, update the attributes of the existing value with `self`. 176 """ 177 if key in d: 178 other = d[key] 179 other.location = min(self.location, other.location) 180 other.pieces.update(self.pieces) 181 else: 182 d[key] = self 183 184 def sortkey(self): 185 return self.order 186 187 def mainpiece(self): 188 """ 189 Return the main piece of C code, corresponding to the include 190 file. If there was no include file, return None. 191 """ 192 return self.pieces.get(0) 193 194 def write(self, code): 195 # Write values of self.pieces dict, sorted by the keys 196 for k in sorted(self.pieces): 197 code.putln(self.pieces[k]) 198 199 200def get_utility_dir(): 201 # make this a function and not global variables: 202 # http://trac.cython.org/cython_trac/ticket/475 203 Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 204 return os.path.join(Cython_dir, "Utility") 205 206 207class UtilityCodeBase(object): 208 """ 209 Support for loading utility code from a file. 210 211 Code sections in the file can be specified as follows: 212 213 ##### MyUtility.proto ##### 214 215 [proto declarations] 216 217 ##### MyUtility.init ##### 218 219 [code run at module initialization] 220 221 ##### MyUtility ##### 222 #@requires: MyOtherUtility 223 #@substitute: naming 224 225 [definitions] 226 227 for prototypes and implementation respectively. For non-python or 228 -cython files backslashes should be used instead. 5 to 30 comment 229 characters may be used on either side. 230 231 If the @cname decorator is not used and this is a CythonUtilityCode, 232 one should pass in the 'name' keyword argument to be used for name 233 mangling of such entries. 234 """ 235 236 is_cython_utility = False 237 _utility_cache = {} 238 239 @classmethod 240 def _add_utility(cls, utility, type, lines, begin_lineno, tags=None): 241 if utility is None: 242 return 243 244 code = '\n'.join(lines) 245 if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): 246 del tags['substitute'] 247 try: 248 code = Template(code).substitute(vars(Naming)) 249 except (KeyError, ValueError) as e: 250 raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % ( 251 type, begin_lineno, e)) 252 253 # remember correct line numbers at least until after templating 254 code = '\n' * begin_lineno + code 255 256 if type == 'proto': 257 utility[0] = code 258 elif type == 'impl': 259 utility[1] = code 260 else: 261 all_tags = utility[2] 262 if KEYWORDS_MUST_BE_BYTES: 263 type = type.encode('ASCII') 264 all_tags[type] = code 265 266 if tags: 267 all_tags = utility[2] 268 for name, values in tags.items(): 269 if KEYWORDS_MUST_BE_BYTES: 270 name = name.encode('ASCII') 271 all_tags.setdefault(name, set()).update(values) 272 273 @classmethod 274 def load_utilities_from_file(cls, path): 275 utilities = cls._utility_cache.get(path) 276 if utilities: 277 return utilities 278 279 filename = os.path.join(get_utility_dir(), path) 280 _, ext = os.path.splitext(path) 281 if ext in ('.pyx', '.py', '.pxd', '.pxi'): 282 comment = '#' 283 strip_comments = partial(re.compile(r'^\s*#(?!\s*cython\s*:).*').sub, '') 284 rstrip = StringEncoding._unicode.rstrip 285 else: 286 comment = '/' 287 strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '') 288 rstrip = partial(re.compile(r'\s+(\\?)$').sub, r'\1') 289 match_special = re.compile( 290 (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|' 291 r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') % 292 {'C': comment}).match 293 match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match 294 295 with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: 296 all_lines = f.readlines() 297 298 utilities = defaultdict(lambda: [None, None, {}]) 299 lines = [] 300 tags = defaultdict(set) 301 utility = type = None 302 begin_lineno = 0 303 304 for lineno, line in enumerate(all_lines): 305 m = match_special(line) 306 if m: 307 if m.group('name'): 308 cls._add_utility(utility, type, lines, begin_lineno, tags) 309 310 begin_lineno = lineno + 1 311 del lines[:] 312 tags.clear() 313 314 name = m.group('name') 315 mtype = match_type(name) 316 if mtype: 317 name, type = mtype.groups() 318 else: 319 type = 'impl' 320 utility = utilities[name] 321 else: 322 tags[m.group('tag')].add(m.group('value')) 323 lines.append('') # keep line number correct 324 else: 325 lines.append(rstrip(strip_comments(line))) 326 327 if utility is None: 328 raise ValueError("Empty utility code file") 329 330 # Don't forget to add the last utility code 331 cls._add_utility(utility, type, lines, begin_lineno, tags) 332 333 utilities = dict(utilities) # un-defaultdict-ify 334 cls._utility_cache[path] = utilities 335 return utilities 336 337 @classmethod 338 def load(cls, util_code_name, from_file=None, **kwargs): 339 """ 340 Load utility code from a file specified by from_file (relative to 341 Cython/Utility) and name util_code_name. If from_file is not given, 342 load it from the file util_code_name.*. There should be only one 343 file matched by this pattern. 344 """ 345 if '::' in util_code_name: 346 from_file, util_code_name = util_code_name.rsplit('::', 1) 347 if not from_file: 348 utility_dir = get_utility_dir() 349 prefix = util_code_name + '.' 350 try: 351 listing = os.listdir(utility_dir) 352 except OSError: 353 # XXX the code below assumes as 'zipimport.zipimporter' instance 354 # XXX should be easy to generalize, but too lazy right now to write it 355 import zipfile 356 global __loader__ 357 loader = __loader__ 358 archive = loader.archive 359 with closing(zipfile.ZipFile(archive)) as fileobj: 360 listing = [os.path.basename(name) 361 for name in fileobj.namelist() 362 if os.path.join(archive, name).startswith(utility_dir)] 363 files = [filename for filename in listing 364 if filename.startswith(prefix)] 365 if not files: 366 raise ValueError("No match found for utility code " + util_code_name) 367 if len(files) > 1: 368 raise ValueError("More than one filename match found for utility code " + util_code_name) 369 from_file = files[0] 370 371 utilities = cls.load_utilities_from_file(from_file) 372 proto, impl, tags = utilities[util_code_name] 373 374 if tags: 375 orig_kwargs = kwargs.copy() 376 for name, values in tags.items(): 377 if name in kwargs: 378 continue 379 # only pass lists when we have to: most argument expect one value or None 380 if name == 'requires': 381 if orig_kwargs: 382 values = [cls.load(dep, from_file, **orig_kwargs) 383 for dep in sorted(values)] 384 else: 385 # dependencies are rarely unique, so use load_cached() when we can 386 values = [cls.load_cached(dep, from_file) 387 for dep in sorted(values)] 388 elif not values: 389 values = None 390 elif len(values) == 1: 391 values = list(values)[0] 392 kwargs[name] = values 393 394 if proto is not None: 395 kwargs['proto'] = proto 396 if impl is not None: 397 kwargs['impl'] = impl 398 399 if 'name' not in kwargs: 400 kwargs['name'] = util_code_name 401 402 if 'file' not in kwargs and from_file: 403 kwargs['file'] = from_file 404 return cls(**kwargs) 405 406 @classmethod 407 def load_cached(cls, utility_code_name, from_file=None, __cache={}): 408 """ 409 Calls .load(), but using a per-type cache based on utility name and file name. 410 """ 411 key = (cls, from_file, utility_code_name) 412 try: 413 return __cache[key] 414 except KeyError: 415 pass 416 code = __cache[key] = cls.load(utility_code_name, from_file) 417 return code 418 419 @classmethod 420 def load_as_string(cls, util_code_name, from_file=None, **kwargs): 421 """ 422 Load a utility code as a string. Returns (proto, implementation) 423 """ 424 util = cls.load(util_code_name, from_file, **kwargs) 425 proto, impl = util.proto, util.impl 426 return util.format_code(proto), util.format_code(impl) 427 428 def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub): 429 """ 430 Format a code section for output. 431 """ 432 if code_string: 433 code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n' 434 return code_string 435 436 def __str__(self): 437 return "<%s(%s)>" % (type(self).__name__, self.name) 438 439 def get_tree(self, **kwargs): 440 pass 441 442 def __deepcopy__(self, memodict=None): 443 # No need to deep-copy utility code since it's essentially immutable. 444 return self 445 446 447class UtilityCode(UtilityCodeBase): 448 """ 449 Stores utility code to add during code generation. 450 451 See GlobalState.put_utility_code. 452 453 hashes/equals by instance 454 455 proto C prototypes 456 impl implementation code 457 init code to call on module initialization 458 requires utility code dependencies 459 proto_block the place in the resulting file where the prototype should 460 end up 461 name name of the utility code (or None) 462 file filename of the utility code file this utility was loaded 463 from (or None) 464 """ 465 466 def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None, 467 proto_block='utility_code_proto', name=None, file=None): 468 # proto_block: Which code block to dump prototype in. See GlobalState. 469 self.proto = proto 470 self.impl = impl 471 self.init = init 472 self.cleanup = cleanup 473 self.requires = requires 474 self._cache = {} 475 self.specialize_list = [] 476 self.proto_block = proto_block 477 self.name = name 478 self.file = file 479 480 def __hash__(self): 481 return hash((self.proto, self.impl)) 482 483 def __eq__(self, other): 484 if self is other: 485 return True 486 self_type, other_type = type(self), type(other) 487 if self_type is not other_type and not (isinstance(other, self_type) or isinstance(self, other_type)): 488 return False 489 490 self_proto = getattr(self, 'proto', None) 491 other_proto = getattr(other, 'proto', None) 492 return (self_proto, self.impl) == (other_proto, other.impl) 493 494 def none_or_sub(self, s, context): 495 """ 496 Format a string in this utility code with context. If None, do nothing. 497 """ 498 if s is None: 499 return None 500 return s % context 501 502 def specialize(self, pyrex_type=None, **data): 503 # Dicts aren't hashable... 504 if pyrex_type is not None: 505 data['type'] = pyrex_type.empty_declaration_code() 506 data['type_name'] = pyrex_type.specialization_name() 507 key = tuple(sorted(data.items())) 508 try: 509 return self._cache[key] 510 except KeyError: 511 if self.requires is None: 512 requires = None 513 else: 514 requires = [r.specialize(data) for r in self.requires] 515 516 s = self._cache[key] = UtilityCode( 517 self.none_or_sub(self.proto, data), 518 self.none_or_sub(self.impl, data), 519 self.none_or_sub(self.init, data), 520 self.none_or_sub(self.cleanup, data), 521 requires, 522 self.proto_block) 523 524 self.specialize_list.append(s) 525 return s 526 527 def inject_string_constants(self, impl, output): 528 """Replace 'PYIDENT("xyz")' by a constant Python identifier cname. 529 """ 530 if 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl: 531 return False, impl 532 533 replacements = {} 534 def externalise(matchobj): 535 key = matchobj.groups() 536 try: 537 cname = replacements[key] 538 except KeyError: 539 str_type, name = key 540 cname = replacements[key] = output.get_py_string_const( 541 StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname 542 return cname 543 544 impl = re.sub(r'PY(IDENT|UNICODE)\("([^"]+)"\)', externalise, impl) 545 assert 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl 546 return True, impl 547 548 def inject_unbound_methods(self, impl, output): 549 """Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname. 550 """ 551 if 'CALL_UNBOUND_METHOD(' not in impl: 552 return False, impl 553 554 def externalise(matchobj): 555 type_cname, method_name, obj_cname, args = matchobj.groups() 556 args = [arg.strip() for arg in args[1:].split(',')] if args else [] 557 assert len(args) < 3, "CALL_UNBOUND_METHOD() does not support %d call arguments" % len(args) 558 return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args) 559 560 impl = re.sub( 561 r'CALL_UNBOUND_METHOD\(' 562 r'([a-zA-Z_]+),' # type cname 563 r'\s*"([^"]+)",' # method name 564 r'\s*([^),]+)' # object cname 565 r'((?:,\s*[^),]+)*)' # args* 566 r'\)', externalise, impl) 567 assert 'CALL_UNBOUND_METHOD(' not in impl 568 569 return True, impl 570 571 def wrap_c_strings(self, impl): 572 """Replace CSTRING('''xyz''') by a C compatible string 573 """ 574 if 'CSTRING(' not in impl: 575 return impl 576 577 def split_string(matchobj): 578 content = matchobj.group(1).replace('"', '\042') 579 return ''.join( 580 '"%s\\n"\n' % line if not line.endswith('\\') or line.endswith('\\\\') else '"%s"\n' % line[:-1] 581 for line in content.splitlines()) 582 583 impl = re.sub(r'CSTRING\(\s*"""([^"]*(?:"[^"]+)*)"""\s*\)', split_string, impl) 584 assert 'CSTRING(' not in impl 585 return impl 586 587 def put_code(self, output): 588 if self.requires: 589 for dependency in self.requires: 590 output.use_utility_code(dependency) 591 if self.proto: 592 writer = output[self.proto_block] 593 writer.putln("/* %s.proto */" % self.name) 594 writer.put_or_include( 595 self.format_code(self.proto), '%s_proto' % self.name) 596 if self.impl: 597 impl = self.format_code(self.wrap_c_strings(self.impl)) 598 is_specialised1, impl = self.inject_string_constants(impl, output) 599 is_specialised2, impl = self.inject_unbound_methods(impl, output) 600 writer = output['utility_code_def'] 601 writer.putln("/* %s */" % self.name) 602 if not (is_specialised1 or is_specialised2): 603 # no module specific adaptations => can be reused 604 writer.put_or_include(impl, '%s_impl' % self.name) 605 else: 606 writer.put(impl) 607 if self.init: 608 writer = output['init_globals'] 609 writer.putln("/* %s.init */" % self.name) 610 if isinstance(self.init, basestring): 611 writer.put(self.format_code(self.init)) 612 else: 613 self.init(writer, output.module_pos) 614 writer.putln(writer.error_goto_if_PyErr(output.module_pos)) 615 writer.putln() 616 if self.cleanup and Options.generate_cleanup_code: 617 writer = output['cleanup_globals'] 618 writer.putln("/* %s.cleanup */" % self.name) 619 if isinstance(self.cleanup, basestring): 620 writer.put_or_include( 621 self.format_code(self.cleanup), 622 '%s_cleanup' % self.name) 623 else: 624 self.cleanup(writer, output.module_pos) 625 626 627def sub_tempita(s, context, file=None, name=None): 628 "Run tempita on string s with given context." 629 if not s: 630 return None 631 632 if file: 633 context['__name'] = "%s:%s" % (file, name) 634 elif name: 635 context['__name'] = name 636 637 from ..Tempita import sub 638 return sub(s, **context) 639 640 641class TempitaUtilityCode(UtilityCode): 642 def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs): 643 if context is None: 644 context = {} 645 proto = sub_tempita(proto, context, file, name) 646 impl = sub_tempita(impl, context, file, name) 647 init = sub_tempita(init, context, file, name) 648 super(TempitaUtilityCode, self).__init__( 649 proto, impl, init=init, name=name, file=file, **kwargs) 650 651 @classmethod 652 def load_cached(cls, utility_code_name, from_file=None, context=None, __cache={}): 653 context_key = tuple(sorted(context.items())) if context else None 654 assert hash(context_key) is not None # raise TypeError if not hashable 655 key = (cls, from_file, utility_code_name, context_key) 656 try: 657 return __cache[key] 658 except KeyError: 659 pass 660 code = __cache[key] = cls.load(utility_code_name, from_file, context=context) 661 return code 662 663 def none_or_sub(self, s, context): 664 """ 665 Format a string in this utility code with context. If None, do nothing. 666 """ 667 if s is None: 668 return None 669 return sub_tempita(s, context, self.file, self.name) 670 671 672class LazyUtilityCode(UtilityCodeBase): 673 """ 674 Utility code that calls a callback with the root code writer when 675 available. Useful when you only have 'env' but not 'code'. 676 """ 677 __name__ = '<lazy>' 678 requires = None 679 680 def __init__(self, callback): 681 self.callback = callback 682 683 def put_code(self, globalstate): 684 utility = self.callback(globalstate.rootwriter) 685 globalstate.use_utility_code(utility) 686 687 688class FunctionState(object): 689 # return_label string function return point label 690 # error_label string error catch point label 691 # continue_label string loop continue point label 692 # break_label string loop break point label 693 # return_from_error_cleanup_label string 694 # label_counter integer counter for naming labels 695 # in_try_finally boolean inside try of try...finally 696 # exc_vars (string * 3) exception variables for reraise, or None 697 # can_trace boolean line tracing is supported in the current context 698 # scope Scope the scope object of the current function 699 700 # Not used for now, perhaps later 701 def __init__(self, owner, names_taken=set(), scope=None): 702 self.names_taken = names_taken 703 self.owner = owner 704 self.scope = scope 705 706 self.error_label = None 707 self.label_counter = 0 708 self.labels_used = set() 709 self.return_label = self.new_label() 710 self.new_error_label() 711 self.continue_label = None 712 self.break_label = None 713 self.yield_labels = [] 714 715 self.in_try_finally = 0 716 self.exc_vars = None 717 self.current_except = None 718 self.can_trace = False 719 self.gil_owned = True 720 721 self.temps_allocated = [] # of (name, type, manage_ref, static) 722 self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status 723 self.temps_used_type = {} # name -> (type, manage_ref) 724 self.zombie_temps = set() # temps that must not be reused after release 725 self.temp_counter = 0 726 self.closure_temps = None 727 728 # This is used to collect temporaries, useful to find out which temps 729 # need to be privatized in parallel sections 730 self.collect_temps_stack = [] 731 732 # This is used for the error indicator, which needs to be local to the 733 # function. It used to be global, which relies on the GIL being held. 734 # However, exceptions may need to be propagated through 'nogil' 735 # sections, in which case we introduce a race condition. 736 self.should_declare_error_indicator = False 737 self.uses_error_indicator = False 738 739 # safety checks 740 741 def validate_exit(self): 742 # validate that all allocated temps have been freed 743 if self.temps_allocated: 744 leftovers = self.temps_in_use() 745 if leftovers: 746 msg = "TEMPGUARD: Temps left over at end of '%s': %s" % (self.scope.name, ', '.join([ 747 '%s [%s]' % (name, ctype) 748 for name, ctype, is_pytemp in sorted(leftovers)]), 749 ) 750 #print(msg) 751 raise RuntimeError(msg) 752 753 # labels 754 755 def new_label(self, name=None): 756 n = self.label_counter 757 self.label_counter = n + 1 758 label = "%s%d" % (Naming.label_prefix, n) 759 if name is not None: 760 label += '_' + name 761 return label 762 763 def new_yield_label(self, expr_type='yield'): 764 label = self.new_label('resume_from_%s' % expr_type) 765 num_and_label = (len(self.yield_labels) + 1, label) 766 self.yield_labels.append(num_and_label) 767 return num_and_label 768 769 def new_error_label(self): 770 old_err_lbl = self.error_label 771 self.error_label = self.new_label('error') 772 return old_err_lbl 773 774 def get_loop_labels(self): 775 return ( 776 self.continue_label, 777 self.break_label) 778 779 def set_loop_labels(self, labels): 780 (self.continue_label, 781 self.break_label) = labels 782 783 def new_loop_labels(self): 784 old_labels = self.get_loop_labels() 785 self.set_loop_labels( 786 (self.new_label("continue"), 787 self.new_label("break"))) 788 return old_labels 789 790 def get_all_labels(self): 791 return ( 792 self.continue_label, 793 self.break_label, 794 self.return_label, 795 self.error_label) 796 797 def set_all_labels(self, labels): 798 (self.continue_label, 799 self.break_label, 800 self.return_label, 801 self.error_label) = labels 802 803 def all_new_labels(self): 804 old_labels = self.get_all_labels() 805 new_labels = [] 806 for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']): 807 if old_label: 808 new_labels.append(self.new_label(name)) 809 else: 810 new_labels.append(old_label) 811 self.set_all_labels(new_labels) 812 return old_labels 813 814 def use_label(self, lbl): 815 self.labels_used.add(lbl) 816 817 def label_used(self, lbl): 818 return lbl in self.labels_used 819 820 # temp handling 821 822 def allocate_temp(self, type, manage_ref, static=False, reusable=True): 823 """ 824 Allocates a temporary (which may create a new one or get a previously 825 allocated and released one of the same type). Type is simply registered 826 and handed back, but will usually be a PyrexType. 827 828 If type.is_pyobject, manage_ref comes into play. If manage_ref is set to 829 True, the temp will be decref-ed on return statements and in exception 830 handling clauses. Otherwise the caller has to deal with any reference 831 counting of the variable. 832 833 If not type.is_pyobject, then manage_ref will be ignored, but it 834 still has to be passed. It is recommended to pass False by convention 835 if it is known that type will never be a Python object. 836 837 static=True marks the temporary declaration with "static". 838 This is only used when allocating backing store for a module-level 839 C array literals. 840 841 if reusable=False, the temp will not be reused after release. 842 843 A C string referring to the variable is returned. 844 """ 845 if type.is_const and not type.is_reference: 846 type = type.const_base_type 847 elif type.is_reference and not type.is_fake_reference: 848 type = type.ref_base_type 849 elif type.is_cfunction: 850 from . import PyrexTypes 851 type = PyrexTypes.c_ptr_type(type) # A function itself isn't an l-value 852 if not type.is_pyobject and not type.is_memoryviewslice: 853 # Make manage_ref canonical, so that manage_ref will always mean 854 # a decref is needed. 855 manage_ref = False 856 857 freelist = self.temps_free.get((type, manage_ref)) 858 if reusable and freelist is not None and freelist[0]: 859 result = freelist[0].pop() 860 freelist[1].remove(result) 861 else: 862 while True: 863 self.temp_counter += 1 864 result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter) 865 if result not in self.names_taken: break 866 self.temps_allocated.append((result, type, manage_ref, static)) 867 if not reusable: 868 self.zombie_temps.add(result) 869 self.temps_used_type[result] = (type, manage_ref) 870 if DebugFlags.debug_temp_code_comments: 871 self.owner.putln("/* %s allocated (%s)%s */" % (result, type, "" if reusable else " - zombie")) 872 873 if self.collect_temps_stack: 874 self.collect_temps_stack[-1].add((result, type)) 875 876 return result 877 878 def release_temp(self, name): 879 """ 880 Releases a temporary so that it can be reused by other code needing 881 a temp of the same type. 882 """ 883 type, manage_ref = self.temps_used_type[name] 884 freelist = self.temps_free.get((type, manage_ref)) 885 if freelist is None: 886 freelist = ([], set()) # keep order in list and make lookups in set fast 887 self.temps_free[(type, manage_ref)] = freelist 888 if name in freelist[1]: 889 raise RuntimeError("Temp %s freed twice!" % name) 890 if name not in self.zombie_temps: 891 freelist[0].append(name) 892 freelist[1].add(name) 893 if DebugFlags.debug_temp_code_comments: 894 self.owner.putln("/* %s released %s*/" % ( 895 name, " - zombie" if name in self.zombie_temps else "")) 896 897 def temps_in_use(self): 898 """Return a list of (cname,type,manage_ref) tuples of temp names and their type 899 that are currently in use. 900 """ 901 used = [] 902 for name, type, manage_ref, static in self.temps_allocated: 903 freelist = self.temps_free.get((type, manage_ref)) 904 if freelist is None or name not in freelist[1]: 905 used.append((name, type, manage_ref and type.is_pyobject)) 906 return used 907 908 def temps_holding_reference(self): 909 """Return a list of (cname,type) tuples of temp names and their type 910 that are currently in use. This includes only temps of a 911 Python object type which owns its reference. 912 """ 913 return [(name, type) 914 for name, type, manage_ref in self.temps_in_use() 915 if manage_ref and type.is_pyobject] 916 917 def all_managed_temps(self): 918 """Return a list of (cname, type) tuples of refcount-managed Python objects. 919 """ 920 return [(cname, type) 921 for cname, type, manage_ref, static in self.temps_allocated 922 if manage_ref] 923 924 def all_free_managed_temps(self): 925 """Return a list of (cname, type) tuples of refcount-managed Python 926 objects that are not currently in use. This is used by 927 try-except and try-finally blocks to clean up temps in the 928 error case. 929 """ 930 return sorted([ # Enforce deterministic order. 931 (cname, type) 932 for (type, manage_ref), freelist in self.temps_free.items() if manage_ref 933 for cname in freelist[0] 934 ]) 935 936 def start_collecting_temps(self): 937 """ 938 Useful to find out which temps were used in a code block 939 """ 940 self.collect_temps_stack.append(set()) 941 942 def stop_collecting_temps(self): 943 return self.collect_temps_stack.pop() 944 945 def init_closure_temps(self, scope): 946 self.closure_temps = ClosureTempAllocator(scope) 947 948 949class NumConst(object): 950 """Global info about a Python number constant held by GlobalState. 951 952 cname string 953 value string 954 py_type string int, long, float 955 value_code string evaluation code if different from value 956 """ 957 958 def __init__(self, cname, value, py_type, value_code=None): 959 self.cname = cname 960 self.value = value 961 self.py_type = py_type 962 self.value_code = value_code or value 963 964 965class PyObjectConst(object): 966 """Global info about a generic constant held by GlobalState. 967 """ 968 # cname string 969 # type PyrexType 970 971 def __init__(self, cname, type): 972 self.cname = cname 973 self.type = type 974 975 976cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object, 977 replace_identifier=object, find_alphanums=object) 978possible_unicode_identifier = re.compile(br"(?![0-9])\w+$".decode('ascii'), re.U).match 979possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match 980replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub 981find_alphanums = re.compile('([a-zA-Z0-9]+)').findall 982 983class StringConst(object): 984 """Global info about a C string constant held by GlobalState. 985 """ 986 # cname string 987 # text EncodedString or BytesLiteral 988 # py_strings {(identifier, encoding) : PyStringConst} 989 990 def __init__(self, cname, text, byte_string): 991 self.cname = cname 992 self.text = text 993 self.escaped_value = StringEncoding.escape_byte_string(byte_string) 994 self.py_strings = None 995 self.py_versions = [] 996 997 def add_py_version(self, version): 998 if not version: 999 self.py_versions = [2, 3] 1000 elif version not in self.py_versions: 1001 self.py_versions.append(version) 1002 1003 def get_py_string_const(self, encoding, identifier=None, 1004 is_str=False, py3str_cstring=None): 1005 py_strings = self.py_strings 1006 text = self.text 1007 1008 is_str = bool(identifier or is_str) 1009 is_unicode = encoding is None and not is_str 1010 1011 if encoding is None: 1012 # unicode string 1013 encoding_key = None 1014 else: 1015 # bytes or str 1016 encoding = encoding.lower() 1017 if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'): 1018 encoding = None 1019 encoding_key = None 1020 else: 1021 encoding_key = ''.join(find_alphanums(encoding)) 1022 1023 key = (is_str, is_unicode, encoding_key, py3str_cstring) 1024 if py_strings is not None: 1025 try: 1026 return py_strings[key] 1027 except KeyError: 1028 pass 1029 else: 1030 self.py_strings = {} 1031 1032 if identifier: 1033 intern = True 1034 elif identifier is None: 1035 if isinstance(text, bytes): 1036 intern = bool(possible_bytes_identifier(text)) 1037 else: 1038 intern = bool(possible_unicode_identifier(text)) 1039 else: 1040 intern = False 1041 if intern: 1042 prefix = Naming.interned_prefixes['str'] 1043 else: 1044 prefix = Naming.py_const_prefix 1045 1046 if encoding_key: 1047 encoding_prefix = '_%s' % encoding_key 1048 else: 1049 encoding_prefix = '' 1050 1051 pystring_cname = "%s%s%s_%s" % ( 1052 prefix, 1053 (is_str and 's') or (is_unicode and 'u') or 'b', 1054 encoding_prefix, 1055 self.cname[len(Naming.const_prefix):]) 1056 1057 py_string = PyStringConst( 1058 pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern) 1059 self.py_strings[key] = py_string 1060 return py_string 1061 1062class PyStringConst(object): 1063 """Global info about a Python string constant held by GlobalState. 1064 """ 1065 # cname string 1066 # py3str_cstring string 1067 # encoding string 1068 # intern boolean 1069 # is_unicode boolean 1070 # is_str boolean 1071 1072 def __init__(self, cname, encoding, is_unicode, is_str=False, 1073 py3str_cstring=None, intern=False): 1074 self.cname = cname 1075 self.py3str_cstring = py3str_cstring 1076 self.encoding = encoding 1077 self.is_str = is_str 1078 self.is_unicode = is_unicode 1079 self.intern = intern 1080 1081 def __lt__(self, other): 1082 return self.cname < other.cname 1083 1084 1085class GlobalState(object): 1086 # filename_table {string : int} for finding filename table indexes 1087 # filename_list [string] filenames in filename table order 1088 # input_file_contents dict contents (=list of lines) of any file that was used as input 1089 # to create this output C code. This is 1090 # used to annotate the comments. 1091 # 1092 # utility_codes set IDs of used utility code (to avoid reinsertion) 1093 # 1094 # declared_cnames {string:Entry} used in a transition phase to merge pxd-declared 1095 # constants etc. into the pyx-declared ones (i.e, 1096 # check if constants are already added). 1097 # In time, hopefully the literals etc. will be 1098 # supplied directly instead. 1099 # 1100 # const_cnames_used dict global counter for unique constant identifiers 1101 # 1102 1103 # parts {string:CCodeWriter} 1104 1105 1106 # interned_strings 1107 # consts 1108 # interned_nums 1109 1110 # directives set Temporary variable used to track 1111 # the current set of directives in the code generation 1112 # process. 1113 1114 directives = {} 1115 1116 code_layout = [ 1117 'h_code', 1118 'filename_table', 1119 'utility_code_proto_before_types', 1120 'numeric_typedefs', # Let these detailed individual parts stay!, 1121 'complex_type_declarations', # as the proper solution is to make a full DAG... 1122 'type_declarations', # More coarse-grained blocks would simply hide 1123 'utility_code_proto', # the ugliness, not fix it 1124 'module_declarations', 1125 'typeinfo', 1126 'before_global_var', 1127 'global_var', 1128 'string_decls', 1129 'decls', 1130 'late_includes', 1131 'all_the_rest', 1132 'pystring_table', 1133 'cached_builtins', 1134 'cached_constants', 1135 'init_globals', 1136 'init_module', 1137 'cleanup_globals', 1138 'cleanup_module', 1139 'main_method', 1140 'utility_code_def', 1141 'end' 1142 ] 1143 1144 1145 def __init__(self, writer, module_node, code_config, common_utility_include_dir=None): 1146 self.filename_table = {} 1147 self.filename_list = [] 1148 self.input_file_contents = {} 1149 self.utility_codes = set() 1150 self.declared_cnames = {} 1151 self.in_utility_code_generation = False 1152 self.code_config = code_config 1153 self.common_utility_include_dir = common_utility_include_dir 1154 self.parts = {} 1155 self.module_node = module_node # because some utility code generation needs it 1156 # (generating backwards-compatible Get/ReleaseBuffer 1157 1158 self.const_cnames_used = {} 1159 self.string_const_index = {} 1160 self.dedup_const_index = {} 1161 self.pyunicode_ptr_const_index = {} 1162 self.num_const_index = {} 1163 self.py_constants = [] 1164 self.cached_cmethods = {} 1165 self.initialised_constants = set() 1166 1167 writer.set_global_state(self) 1168 self.rootwriter = writer 1169 1170 def initialize_main_c_code(self): 1171 rootwriter = self.rootwriter 1172 for part in self.code_layout: 1173 self.parts[part] = rootwriter.insertion_point() 1174 1175 if not Options.cache_builtins: 1176 del self.parts['cached_builtins'] 1177 else: 1178 w = self.parts['cached_builtins'] 1179 w.enter_cfunc_scope() 1180 w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) {") 1181 1182 w = self.parts['cached_constants'] 1183 w.enter_cfunc_scope() 1184 w.putln("") 1185 w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {") 1186 w.put_declare_refcount_context() 1187 w.put_setup_refcount_context("__Pyx_InitCachedConstants") 1188 1189 w = self.parts['init_globals'] 1190 w.enter_cfunc_scope() 1191 w.putln("") 1192 w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {") 1193 1194 if not Options.generate_cleanup_code: 1195 del self.parts['cleanup_globals'] 1196 else: 1197 w = self.parts['cleanup_globals'] 1198 w.enter_cfunc_scope() 1199 w.putln("") 1200 w.putln("static CYTHON_SMALL_CODE void __Pyx_CleanupGlobals(void) {") 1201 1202 code = self.parts['utility_code_proto'] 1203 code.putln("") 1204 code.putln("/* --- Runtime support code (head) --- */") 1205 1206 code = self.parts['utility_code_def'] 1207 if self.code_config.emit_linenums: 1208 code.write('\n#line 1 "cython_utility"\n') 1209 code.putln("") 1210 code.putln("/* --- Runtime support code --- */") 1211 1212 def finalize_main_c_code(self): 1213 self.close_global_decls() 1214 1215 # 1216 # utility_code_def 1217 # 1218 code = self.parts['utility_code_def'] 1219 util = TempitaUtilityCode.load_cached("TypeConversions", "TypeConversion.c") 1220 code.put(util.format_code(util.impl)) 1221 code.putln("") 1222 1223 def __getitem__(self, key): 1224 return self.parts[key] 1225 1226 # 1227 # Global constants, interned objects, etc. 1228 # 1229 def close_global_decls(self): 1230 # This is called when it is known that no more global declarations will 1231 # declared. 1232 self.generate_const_declarations() 1233 if Options.cache_builtins: 1234 w = self.parts['cached_builtins'] 1235 w.putln("return 0;") 1236 if w.label_used(w.error_label): 1237 w.put_label(w.error_label) 1238 w.putln("return -1;") 1239 w.putln("}") 1240 w.exit_cfunc_scope() 1241 1242 w = self.parts['cached_constants'] 1243 w.put_finish_refcount_context() 1244 w.putln("return 0;") 1245 if w.label_used(w.error_label): 1246 w.put_label(w.error_label) 1247 w.put_finish_refcount_context() 1248 w.putln("return -1;") 1249 w.putln("}") 1250 w.exit_cfunc_scope() 1251 1252 w = self.parts['init_globals'] 1253 w.putln("return 0;") 1254 if w.label_used(w.error_label): 1255 w.put_label(w.error_label) 1256 w.putln("return -1;") 1257 w.putln("}") 1258 w.exit_cfunc_scope() 1259 1260 if Options.generate_cleanup_code: 1261 w = self.parts['cleanup_globals'] 1262 w.putln("}") 1263 w.exit_cfunc_scope() 1264 1265 if Options.generate_cleanup_code: 1266 w = self.parts['cleanup_module'] 1267 w.putln("}") 1268 w.exit_cfunc_scope() 1269 1270 def put_pyobject_decl(self, entry): 1271 self['global_var'].putln("static PyObject *%s;" % entry.cname) 1272 1273 # constant handling at code generation time 1274 1275 def get_cached_constants_writer(self, target=None): 1276 if target is not None: 1277 if target in self.initialised_constants: 1278 # Return None on second/later calls to prevent duplicate creation code. 1279 return None 1280 self.initialised_constants.add(target) 1281 return self.parts['cached_constants'] 1282 1283 def get_int_const(self, str_value, longness=False): 1284 py_type = longness and 'long' or 'int' 1285 try: 1286 c = self.num_const_index[(str_value, py_type)] 1287 except KeyError: 1288 c = self.new_num_const(str_value, py_type) 1289 return c 1290 1291 def get_float_const(self, str_value, value_code): 1292 try: 1293 c = self.num_const_index[(str_value, 'float')] 1294 except KeyError: 1295 c = self.new_num_const(str_value, 'float', value_code) 1296 return c 1297 1298 def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None): 1299 if dedup_key is not None: 1300 const = self.dedup_const_index.get(dedup_key) 1301 if const is not None: 1302 return const 1303 # create a new Python object constant 1304 const = self.new_py_const(type, prefix) 1305 if cleanup_level is not None \ 1306 and cleanup_level <= Options.generate_cleanup_code: 1307 cleanup_writer = self.parts['cleanup_globals'] 1308 cleanup_writer.putln('Py_CLEAR(%s);' % const.cname) 1309 if dedup_key is not None: 1310 self.dedup_const_index[dedup_key] = const 1311 return const 1312 1313 def get_string_const(self, text, py_version=None): 1314 # return a C string constant, creating a new one if necessary 1315 if text.is_unicode: 1316 byte_string = text.utf8encode() 1317 else: 1318 byte_string = text.byteencode() 1319 try: 1320 c = self.string_const_index[byte_string] 1321 except KeyError: 1322 c = self.new_string_const(text, byte_string) 1323 c.add_py_version(py_version) 1324 return c 1325 1326 def get_pyunicode_ptr_const(self, text): 1327 # return a Py_UNICODE[] constant, creating a new one if necessary 1328 assert text.is_unicode 1329 try: 1330 c = self.pyunicode_ptr_const_index[text] 1331 except KeyError: 1332 c = self.pyunicode_ptr_const_index[text] = self.new_const_cname() 1333 return c 1334 1335 def get_py_string_const(self, text, identifier=None, 1336 is_str=False, unicode_value=None): 1337 # return a Python string constant, creating a new one if necessary 1338 py3str_cstring = None 1339 if is_str and unicode_value is not None \ 1340 and unicode_value.utf8encode() != text.byteencode(): 1341 py3str_cstring = self.get_string_const(unicode_value, py_version=3) 1342 c_string = self.get_string_const(text, py_version=2) 1343 else: 1344 c_string = self.get_string_const(text) 1345 py_string = c_string.get_py_string_const( 1346 text.encoding, identifier, is_str, py3str_cstring) 1347 return py_string 1348 1349 def get_interned_identifier(self, text): 1350 return self.get_py_string_const(text, identifier=True) 1351 1352 def new_string_const(self, text, byte_string): 1353 cname = self.new_string_const_cname(byte_string) 1354 c = StringConst(cname, text, byte_string) 1355 self.string_const_index[byte_string] = c 1356 return c 1357 1358 def new_num_const(self, value, py_type, value_code=None): 1359 cname = self.new_num_const_cname(value, py_type) 1360 c = NumConst(cname, value, py_type, value_code) 1361 self.num_const_index[(value, py_type)] = c 1362 return c 1363 1364 def new_py_const(self, type, prefix=''): 1365 cname = self.new_const_cname(prefix) 1366 c = PyObjectConst(cname, type) 1367 self.py_constants.append(c) 1368 return c 1369 1370 def new_string_const_cname(self, bytes_value): 1371 # Create a new globally-unique nice name for a C string constant. 1372 value = bytes_value.decode('ASCII', 'ignore') 1373 return self.new_const_cname(value=value) 1374 1375 def new_num_const_cname(self, value, py_type): 1376 if py_type == 'long': 1377 value += 'L' 1378 py_type = 'int' 1379 prefix = Naming.interned_prefixes[py_type] 1380 cname = "%s%s" % (prefix, value) 1381 cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_') 1382 return cname 1383 1384 def new_const_cname(self, prefix='', value=''): 1385 value = replace_identifier('_', value)[:32].strip('_') 1386 used = self.const_cnames_used 1387 name_suffix = value 1388 while name_suffix in used: 1389 counter = used[value] = used[value] + 1 1390 name_suffix = '%s_%d' % (value, counter) 1391 used[name_suffix] = 1 1392 if prefix: 1393 prefix = Naming.interned_prefixes[prefix] 1394 else: 1395 prefix = Naming.const_prefix 1396 return "%s%s" % (prefix, name_suffix) 1397 1398 def get_cached_unbound_method(self, type_cname, method_name): 1399 key = (type_cname, method_name) 1400 try: 1401 cname = self.cached_cmethods[key] 1402 except KeyError: 1403 cname = self.cached_cmethods[key] = self.new_const_cname( 1404 'umethod', '%s_%s' % (type_cname, method_name)) 1405 return cname 1406 1407 def cached_unbound_method_call_code(self, obj_cname, type_cname, method_name, arg_cnames): 1408 # admittedly, not the best place to put this method, but it is reused by UtilityCode and ExprNodes ... 1409 utility_code_name = "CallUnboundCMethod%d" % len(arg_cnames) 1410 self.use_utility_code(UtilityCode.load_cached(utility_code_name, "ObjectHandling.c")) 1411 cache_cname = self.get_cached_unbound_method(type_cname, method_name) 1412 args = [obj_cname] + arg_cnames 1413 return "__Pyx_%s(&%s, %s)" % ( 1414 utility_code_name, 1415 cache_cname, 1416 ', '.join(args), 1417 ) 1418 1419 def add_cached_builtin_decl(self, entry): 1420 if entry.is_builtin and entry.is_const: 1421 if self.should_declare(entry.cname, entry): 1422 self.put_pyobject_decl(entry) 1423 w = self.parts['cached_builtins'] 1424 condition = None 1425 if entry.name in non_portable_builtins_map: 1426 condition, replacement = non_portable_builtins_map[entry.name] 1427 w.putln('#if %s' % condition) 1428 self.put_cached_builtin_init( 1429 entry.pos, StringEncoding.EncodedString(replacement), 1430 entry.cname) 1431 w.putln('#else') 1432 self.put_cached_builtin_init( 1433 entry.pos, StringEncoding.EncodedString(entry.name), 1434 entry.cname) 1435 if condition: 1436 w.putln('#endif') 1437 1438 def put_cached_builtin_init(self, pos, name, cname): 1439 w = self.parts['cached_builtins'] 1440 interned_cname = self.get_interned_identifier(name).cname 1441 self.use_utility_code( 1442 UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c")) 1443 w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % ( 1444 cname, 1445 interned_cname, 1446 cname, 1447 w.error_goto(pos))) 1448 1449 def generate_const_declarations(self): 1450 self.generate_cached_methods_decls() 1451 self.generate_string_constants() 1452 self.generate_num_constants() 1453 self.generate_object_constant_decls() 1454 1455 def generate_object_constant_decls(self): 1456 consts = [(len(c.cname), c.cname, c) 1457 for c in self.py_constants] 1458 consts.sort() 1459 decls_writer = self.parts['decls'] 1460 for _, cname, c in consts: 1461 decls_writer.putln( 1462 "static %s;" % c.type.declaration_code(cname)) 1463 1464 def generate_cached_methods_decls(self): 1465 if not self.cached_cmethods: 1466 return 1467 1468 decl = self.parts['decls'] 1469 init = self.parts['init_globals'] 1470 cnames = [] 1471 for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): 1472 cnames.append(cname) 1473 method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname 1474 decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( 1475 cname, method_name_cname)) 1476 # split type reference storage as it might not be static 1477 init.putln('%s.type = (PyObject*)&%s;' % ( 1478 cname, type_cname)) 1479 1480 if Options.generate_cleanup_code: 1481 cleanup = self.parts['cleanup_globals'] 1482 for cname in cnames: 1483 cleanup.putln("Py_CLEAR(%s.method);" % cname) 1484 1485 def generate_string_constants(self): 1486 c_consts = [(len(c.cname), c.cname, c) for c in self.string_const_index.values()] 1487 c_consts.sort() 1488 py_strings = [] 1489 1490 decls_writer = self.parts['string_decls'] 1491 for _, cname, c in c_consts: 1492 conditional = False 1493 if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions): 1494 conditional = True 1495 decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % ( 1496 (2 in c.py_versions) and '<' or '>=')) 1497 decls_writer.putln('static const char %s[] = "%s";' % ( 1498 cname, StringEncoding.split_string_literal(c.escaped_value))) 1499 if conditional: 1500 decls_writer.putln("#endif") 1501 if c.py_strings is not None: 1502 for py_string in c.py_strings.values(): 1503 py_strings.append((c.cname, len(py_string.cname), py_string)) 1504 1505 for c, cname in sorted(self.pyunicode_ptr_const_index.items()): 1506 utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c) 1507 if utf16_array: 1508 # Narrow and wide representations differ 1509 decls_writer.putln("#ifdef Py_UNICODE_WIDE") 1510 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array)) 1511 if utf16_array: 1512 decls_writer.putln("#else") 1513 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) 1514 decls_writer.putln("#endif") 1515 1516 if py_strings: 1517 self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) 1518 py_strings.sort() 1519 w = self.parts['pystring_table'] 1520 w.putln("") 1521 w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) 1522 for c_cname, _, py_string in py_strings: 1523 if not py_string.is_str or not py_string.encoding or \ 1524 py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 1525 'UTF8', 'UTF-8'): 1526 encoding = '0' 1527 else: 1528 encoding = '"%s"' % py_string.encoding.lower() 1529 1530 decls_writer.putln( 1531 "static PyObject *%s;" % py_string.cname) 1532 if py_string.py3str_cstring: 1533 w.putln("#if PY_MAJOR_VERSION >= 3") 1534 w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( 1535 py_string.cname, 1536 py_string.py3str_cstring.cname, 1537 py_string.py3str_cstring.cname, 1538 '0', 1, 0, 1539 py_string.intern 1540 )) 1541 w.putln("#else") 1542 w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( 1543 py_string.cname, 1544 c_cname, 1545 c_cname, 1546 encoding, 1547 py_string.is_unicode, 1548 py_string.is_str, 1549 py_string.intern 1550 )) 1551 if py_string.py3str_cstring: 1552 w.putln("#endif") 1553 w.putln("{0, 0, 0, 0, 0, 0, 0}") 1554 w.putln("};") 1555 1556 init_globals = self.parts['init_globals'] 1557 init_globals.putln( 1558 "if (__Pyx_InitStrings(%s) < 0) %s;" % ( 1559 Naming.stringtab_cname, 1560 init_globals.error_goto(self.module_pos))) 1561 1562 def generate_num_constants(self): 1563 consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) 1564 for c in self.num_const_index.values()] 1565 consts.sort() 1566 decls_writer = self.parts['decls'] 1567 init_globals = self.parts['init_globals'] 1568 for py_type, _, _, value, value_code, c in consts: 1569 cname = c.cname 1570 decls_writer.putln("static PyObject *%s;" % cname) 1571 if py_type == 'float': 1572 function = 'PyFloat_FromDouble(%s)' 1573 elif py_type == 'long': 1574 function = 'PyLong_FromString((char *)"%s", 0, 0)' 1575 elif Utils.long_literal(value): 1576 function = 'PyInt_FromString((char *)"%s", 0, 0)' 1577 elif len(value.lstrip('-')) > 4: 1578 function = "PyInt_FromLong(%sL)" 1579 else: 1580 function = "PyInt_FromLong(%s)" 1581 init_globals.putln('%s = %s; %s' % ( 1582 cname, function % value_code, 1583 init_globals.error_goto_if_null(cname, self.module_pos))) 1584 1585 # The functions below are there in a transition phase only 1586 # and will be deprecated. They are called from Nodes.BlockNode. 1587 # The copy&paste duplication is intentional in order to be able 1588 # to see quickly how BlockNode worked, until this is replaced. 1589 1590 def should_declare(self, cname, entry): 1591 if cname in self.declared_cnames: 1592 other = self.declared_cnames[cname] 1593 assert str(entry.type) == str(other.type) 1594 assert entry.init == other.init 1595 return False 1596 else: 1597 self.declared_cnames[cname] = entry 1598 return True 1599 1600 # 1601 # File name state 1602 # 1603 1604 def lookup_filename(self, source_desc): 1605 entry = source_desc.get_filenametable_entry() 1606 try: 1607 index = self.filename_table[entry] 1608 except KeyError: 1609 index = len(self.filename_list) 1610 self.filename_list.append(source_desc) 1611 self.filename_table[entry] = index 1612 return index 1613 1614 def commented_file_contents(self, source_desc): 1615 try: 1616 return self.input_file_contents[source_desc] 1617 except KeyError: 1618 pass 1619 source_file = source_desc.get_lines(encoding='ASCII', 1620 error_handling='ignore') 1621 try: 1622 F = [u' * ' + line.rstrip().replace( 1623 u'*/', u'*[inserted by cython to avoid comment closer]/' 1624 ).replace( 1625 u'/*', u'/[inserted by cython to avoid comment start]*' 1626 ) 1627 for line in source_file] 1628 finally: 1629 if hasattr(source_file, 'close'): 1630 source_file.close() 1631 if not F: F.append(u'') 1632 self.input_file_contents[source_desc] = F 1633 return F 1634 1635 # 1636 # Utility code state 1637 # 1638 1639 def use_utility_code(self, utility_code): 1640 """ 1641 Adds code to the C file. utility_code should 1642 a) implement __eq__/__hash__ for the purpose of knowing whether the same 1643 code has already been included 1644 b) implement put_code, which takes a globalstate instance 1645 1646 See UtilityCode. 1647 """ 1648 if utility_code and utility_code not in self.utility_codes: 1649 self.utility_codes.add(utility_code) 1650 utility_code.put_code(self) 1651 1652 def use_entry_utility_code(self, entry): 1653 if entry is None: 1654 return 1655 if entry.utility_code: 1656 self.use_utility_code(entry.utility_code) 1657 if entry.utility_code_definition: 1658 self.use_utility_code(entry.utility_code_definition) 1659 1660 1661def funccontext_property(func): 1662 name = func.__name__ 1663 attribute_of = operator.attrgetter(name) 1664 def get(self): 1665 return attribute_of(self.funcstate) 1666 def set(self, value): 1667 setattr(self.funcstate, name, value) 1668 return property(get, set) 1669 1670 1671class CCodeConfig(object): 1672 # emit_linenums boolean write #line pragmas? 1673 # emit_code_comments boolean copy the original code into C comments? 1674 # c_line_in_traceback boolean append the c file and line number to the traceback for exceptions? 1675 1676 def __init__(self, emit_linenums=True, emit_code_comments=True, c_line_in_traceback=True): 1677 self.emit_code_comments = emit_code_comments 1678 self.emit_linenums = emit_linenums 1679 self.c_line_in_traceback = c_line_in_traceback 1680 1681 1682class CCodeWriter(object): 1683 """ 1684 Utility class to output C code. 1685 1686 When creating an insertion point one must care about the state that is 1687 kept: 1688 - formatting state (level, bol) is cloned and used in insertion points 1689 as well 1690 - labels, temps, exc_vars: One must construct a scope in which these can 1691 exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for 1692 sanity checking and forward compatibility). Created insertion points 1693 looses this scope and cannot access it. 1694 - marker: Not copied to insertion point 1695 - filename_table, filename_list, input_file_contents: All codewriters 1696 coming from the same root share the same instances simultaneously. 1697 """ 1698 1699 # f file output file 1700 # buffer StringIOTree 1701 1702 # level int indentation level 1703 # bol bool beginning of line? 1704 # marker string comment to emit before next line 1705 # funcstate FunctionState contains state local to a C function used for code 1706 # generation (labels and temps state etc.) 1707 # globalstate GlobalState contains state global for a C file (input file info, 1708 # utility code, declared constants etc.) 1709 # pyclass_stack list used during recursive code generation to pass information 1710 # about the current class one is in 1711 # code_config CCodeConfig configuration options for the C code writer 1712 1713 @cython.locals(create_from='CCodeWriter') 1714 def __init__(self, create_from=None, buffer=None, copy_formatting=False): 1715 if buffer is None: buffer = StringIOTree() 1716 self.buffer = buffer 1717 self.last_pos = None 1718 self.last_marked_pos = None 1719 self.pyclass_stack = [] 1720 1721 self.funcstate = None 1722 self.globalstate = None 1723 self.code_config = None 1724 self.level = 0 1725 self.call_level = 0 1726 self.bol = 1 1727 1728 if create_from is not None: 1729 # Use same global state 1730 self.set_global_state(create_from.globalstate) 1731 self.funcstate = create_from.funcstate 1732 # Clone formatting state 1733 if copy_formatting: 1734 self.level = create_from.level 1735 self.bol = create_from.bol 1736 self.call_level = create_from.call_level 1737 self.last_pos = create_from.last_pos 1738 self.last_marked_pos = create_from.last_marked_pos 1739 1740 def create_new(self, create_from, buffer, copy_formatting): 1741 # polymorphic constructor -- very slightly more versatile 1742 # than using __class__ 1743 result = CCodeWriter(create_from, buffer, copy_formatting) 1744 return result 1745 1746 def set_global_state(self, global_state): 1747 assert self.globalstate is None # prevent overwriting once it's set 1748 self.globalstate = global_state 1749 self.code_config = global_state.code_config 1750 1751 def copyto(self, f): 1752 self.buffer.copyto(f) 1753 1754 def getvalue(self): 1755 return self.buffer.getvalue() 1756 1757 def write(self, s): 1758 # also put invalid markers (lineno 0), to indicate that those lines 1759 # have no Cython source code correspondence 1760 cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0 1761 self.buffer.markers.extend([cython_lineno] * s.count('\n')) 1762 self.buffer.write(s) 1763 1764 def insertion_point(self): 1765 other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True) 1766 return other 1767 1768 def new_writer(self): 1769 """ 1770 Creates a new CCodeWriter connected to the same global state, which 1771 can later be inserted using insert. 1772 """ 1773 return CCodeWriter(create_from=self) 1774 1775 def insert(self, writer): 1776 """ 1777 Inserts the contents of another code writer (created with 1778 the same global state) in the current location. 1779 1780 It is ok to write to the inserted writer also after insertion. 1781 """ 1782 assert writer.globalstate is self.globalstate 1783 self.buffer.insert(writer.buffer) 1784 1785 # Properties delegated to function scope 1786 @funccontext_property 1787 def label_counter(self): pass 1788 @funccontext_property 1789 def return_label(self): pass 1790 @funccontext_property 1791 def error_label(self): pass 1792 @funccontext_property 1793 def labels_used(self): pass 1794 @funccontext_property 1795 def continue_label(self): pass 1796 @funccontext_property 1797 def break_label(self): pass 1798 @funccontext_property 1799 def return_from_error_cleanup_label(self): pass 1800 @funccontext_property 1801 def yield_labels(self): pass 1802 1803 # Functions delegated to function scope 1804 def new_label(self, name=None): return self.funcstate.new_label(name) 1805 def new_error_label(self): return self.funcstate.new_error_label() 1806 def new_yield_label(self, *args): return self.funcstate.new_yield_label(*args) 1807 def get_loop_labels(self): return self.funcstate.get_loop_labels() 1808 def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels) 1809 def new_loop_labels(self): return self.funcstate.new_loop_labels() 1810 def get_all_labels(self): return self.funcstate.get_all_labels() 1811 def set_all_labels(self, labels): return self.funcstate.set_all_labels(labels) 1812 def all_new_labels(self): return self.funcstate.all_new_labels() 1813 def use_label(self, lbl): return self.funcstate.use_label(lbl) 1814 def label_used(self, lbl): return self.funcstate.label_used(lbl) 1815 1816 1817 def enter_cfunc_scope(self, scope=None): 1818 self.funcstate = FunctionState(self, scope=scope) 1819 1820 def exit_cfunc_scope(self): 1821 self.funcstate = None 1822 1823 # constant handling 1824 1825 def get_py_int(self, str_value, longness): 1826 return self.globalstate.get_int_const(str_value, longness).cname 1827 1828 def get_py_float(self, str_value, value_code): 1829 return self.globalstate.get_float_const(str_value, value_code).cname 1830 1831 def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None): 1832 return self.globalstate.get_py_const(type, prefix, cleanup_level, dedup_key).cname 1833 1834 def get_string_const(self, text): 1835 return self.globalstate.get_string_const(text).cname 1836 1837 def get_pyunicode_ptr_const(self, text): 1838 return self.globalstate.get_pyunicode_ptr_const(text) 1839 1840 def get_py_string_const(self, text, identifier=None, 1841 is_str=False, unicode_value=None): 1842 return self.globalstate.get_py_string_const( 1843 text, identifier, is_str, unicode_value).cname 1844 1845 def get_argument_default_const(self, type): 1846 return self.globalstate.get_py_const(type).cname 1847 1848 def intern(self, text): 1849 return self.get_py_string_const(text) 1850 1851 def intern_identifier(self, text): 1852 return self.get_py_string_const(text, identifier=True) 1853 1854 def get_cached_constants_writer(self, target=None): 1855 return self.globalstate.get_cached_constants_writer(target) 1856 1857 # code generation 1858 1859 def putln(self, code="", safe=False): 1860 if self.last_pos and self.bol: 1861 self.emit_marker() 1862 if self.code_config.emit_linenums and self.last_marked_pos: 1863 source_desc, line, _ = self.last_marked_pos 1864 self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description())) 1865 if code: 1866 if safe: 1867 self.put_safe(code) 1868 else: 1869 self.put(code) 1870 self.write("\n") 1871 self.bol = 1 1872 1873 def mark_pos(self, pos, trace=True): 1874 if pos is None: 1875 return 1876 if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]: 1877 return 1878 self.last_pos = (pos, trace) 1879 1880 def emit_marker(self): 1881 pos, trace = self.last_pos 1882 self.last_marked_pos = pos 1883 self.last_pos = None 1884 self.write("\n") 1885 if self.code_config.emit_code_comments: 1886 self.indent() 1887 self.write("/* %s */\n" % self._build_marker(pos)) 1888 if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']: 1889 self.indent() 1890 self.write('__Pyx_TraceLine(%d,%d,%s)\n' % ( 1891 pos[1], not self.funcstate.gil_owned, self.error_goto(pos))) 1892 1893 def _build_marker(self, pos): 1894 source_desc, line, col = pos 1895 assert isinstance(source_desc, SourceDescriptor) 1896 contents = self.globalstate.commented_file_contents(source_desc) 1897 lines = contents[max(0, line-3):line] # line numbers start at 1 1898 lines[-1] += u' # <<<<<<<<<<<<<<' 1899 lines += contents[line:line+2] 1900 return u'"%s":%d\n%s\n' % (source_desc.get_escaped_description(), line, u'\n'.join(lines)) 1901 1902 def put_safe(self, code): 1903 # put code, but ignore {} 1904 self.write(code) 1905 self.bol = 0 1906 1907 def put_or_include(self, code, name): 1908 include_dir = self.globalstate.common_utility_include_dir 1909 if include_dir and len(code) > 1024: 1910 include_file = "%s_%s.h" % ( 1911 name, hashlib.md5(code.encode('utf8')).hexdigest()) 1912 path = os.path.join(include_dir, include_file) 1913 if not os.path.exists(path): 1914 tmp_path = '%s.tmp%s' % (path, os.getpid()) 1915 with closing(Utils.open_new_file(tmp_path)) as f: 1916 f.write(code) 1917 shutil.move(tmp_path, path) 1918 code = '#include "%s"\n' % path 1919 self.put(code) 1920 1921 def put(self, code): 1922 fix_indent = False 1923 if "{" in code: 1924 dl = code.count("{") 1925 else: 1926 dl = 0 1927 if "}" in code: 1928 dl -= code.count("}") 1929 if dl < 0: 1930 self.level += dl 1931 elif dl == 0 and code[0] == "}": 1932 # special cases like "} else {" need a temporary dedent 1933 fix_indent = True 1934 self.level -= 1 1935 if self.bol: 1936 self.indent() 1937 self.write(code) 1938 self.bol = 0 1939 if dl > 0: 1940 self.level += dl 1941 elif fix_indent: 1942 self.level += 1 1943 1944 def putln_tempita(self, code, **context): 1945 from ..Tempita import sub 1946 self.putln(sub(code, **context)) 1947 1948 def put_tempita(self, code, **context): 1949 from ..Tempita import sub 1950 self.put(sub(code, **context)) 1951 1952 def increase_indent(self): 1953 self.level += 1 1954 1955 def decrease_indent(self): 1956 self.level -= 1 1957 1958 def begin_block(self): 1959 self.putln("{") 1960 self.increase_indent() 1961 1962 def end_block(self): 1963 self.decrease_indent() 1964 self.putln("}") 1965 1966 def indent(self): 1967 self.write(" " * self.level) 1968 1969 def get_py_version_hex(self, pyversion): 1970 return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4] 1971 1972 def put_label(self, lbl): 1973 if lbl in self.funcstate.labels_used: 1974 self.putln("%s:;" % lbl) 1975 1976 def put_goto(self, lbl): 1977 self.funcstate.use_label(lbl) 1978 self.putln("goto %s;" % lbl) 1979 1980 def put_var_declaration(self, entry, storage_class="", 1981 dll_linkage=None, definition=True): 1982 #print "Code.put_var_declaration:", entry.name, "definition =", definition ### 1983 if entry.visibility == 'private' and not (definition or entry.defined_in_pxd): 1984 #print "...private and not definition, skipping", entry.cname ### 1985 return 1986 if entry.visibility == "private" and not entry.used: 1987 #print "...private and not used, skipping", entry.cname ### 1988 return 1989 if storage_class: 1990 self.put("%s " % storage_class) 1991 if not entry.cf_used: 1992 self.put('CYTHON_UNUSED ') 1993 self.put(entry.type.declaration_code( 1994 entry.cname, dll_linkage=dll_linkage)) 1995 if entry.init is not None: 1996 self.put_safe(" = %s" % entry.type.literal_code(entry.init)) 1997 elif entry.type.is_pyobject: 1998 self.put(" = NULL") 1999 self.putln(";") 2000 2001 def put_temp_declarations(self, func_context): 2002 for name, type, manage_ref, static in func_context.temps_allocated: 2003 decl = type.declaration_code(name) 2004 if type.is_pyobject: 2005 self.putln("%s = NULL;" % decl) 2006 elif type.is_memoryviewslice: 2007 from . import MemoryView 2008 self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) 2009 else: 2010 self.putln("%s%s;" % (static and "static " or "", decl)) 2011 2012 if func_context.should_declare_error_indicator: 2013 if self.funcstate.uses_error_indicator: 2014 unused = '' 2015 else: 2016 unused = 'CYTHON_UNUSED ' 2017 # Initialize these variables to silence compiler warnings 2018 self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname)) 2019 self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname)) 2020 self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname)) 2021 2022 def put_generated_by(self): 2023 self.putln("/* Generated by Cython %s */" % Version.watermark) 2024 self.putln("") 2025 2026 def put_h_guard(self, guard): 2027 self.putln("#ifndef %s" % guard) 2028 self.putln("#define %s" % guard) 2029 2030 def unlikely(self, cond): 2031 if Options.gcc_branch_hints: 2032 return 'unlikely(%s)' % cond 2033 else: 2034 return cond 2035 2036 def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper): 2037 if not modifiers: 2038 return '' 2039 return '%s ' % ' '.join([mapper(m,m) for m in modifiers]) 2040 2041 # Python objects and reference counting 2042 2043 def entry_as_pyobject(self, entry): 2044 type = entry.type 2045 if (not entry.is_self_arg and not entry.type.is_complete() 2046 or entry.type.is_extension_type): 2047 return "(PyObject *)" + entry.cname 2048 else: 2049 return entry.cname 2050 2051 def as_pyobject(self, cname, type): 2052 from .PyrexTypes import py_object_type, typecast 2053 return typecast(py_object_type, type, cname) 2054 2055 def put_gotref(self, cname): 2056 self.putln("__Pyx_GOTREF(%s);" % cname) 2057 2058 def put_giveref(self, cname): 2059 self.putln("__Pyx_GIVEREF(%s);" % cname) 2060 2061 def put_xgiveref(self, cname): 2062 self.putln("__Pyx_XGIVEREF(%s);" % cname) 2063 2064 def put_xgotref(self, cname): 2065 self.putln("__Pyx_XGOTREF(%s);" % cname) 2066 2067 def put_incref(self, cname, type, nanny=True): 2068 if nanny: 2069 self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) 2070 else: 2071 self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) 2072 2073 def put_decref(self, cname, type, nanny=True): 2074 self._put_decref(cname, type, nanny, null_check=False, clear=False) 2075 2076 def put_var_gotref(self, entry): 2077 if entry.type.is_pyobject: 2078 self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) 2079 2080 def put_var_giveref(self, entry): 2081 if entry.type.is_pyobject: 2082 self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) 2083 2084 def put_var_xgotref(self, entry): 2085 if entry.type.is_pyobject: 2086 self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) 2087 2088 def put_var_xgiveref(self, entry): 2089 if entry.type.is_pyobject: 2090 self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) 2091 2092 def put_var_incref(self, entry, nanny=True): 2093 if entry.type.is_pyobject: 2094 if nanny: 2095 self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) 2096 else: 2097 self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) 2098 2099 def put_var_xincref(self, entry): 2100 if entry.type.is_pyobject: 2101 self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry)) 2102 2103 def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): 2104 self._put_decref(cname, type, nanny, null_check=False, 2105 clear=True, clear_before_decref=clear_before_decref) 2106 2107 def put_xdecref(self, cname, type, nanny=True, have_gil=True): 2108 self._put_decref(cname, type, nanny, null_check=True, 2109 have_gil=have_gil, clear=False) 2110 2111 def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): 2112 self._put_decref(cname, type, nanny, null_check=True, 2113 clear=True, clear_before_decref=clear_before_decref) 2114 2115 def _put_decref(self, cname, type, nanny=True, null_check=False, 2116 have_gil=True, clear=False, clear_before_decref=False): 2117 if type.is_memoryviewslice: 2118 self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) 2119 return 2120 2121 prefix = '__Pyx' if nanny else 'Py' 2122 X = 'X' if null_check else '' 2123 2124 if clear: 2125 if clear_before_decref: 2126 if not nanny: 2127 X = '' # CPython doesn't have a Py_XCLEAR() 2128 self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) 2129 else: 2130 self.putln("%s_%sDECREF(%s); %s = 0;" % ( 2131 prefix, X, self.as_pyobject(cname, type), cname)) 2132 else: 2133 self.putln("%s_%sDECREF(%s);" % ( 2134 prefix, X, self.as_pyobject(cname, type))) 2135 2136 def put_decref_set(self, cname, rhs_cname): 2137 self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) 2138 2139 def put_xdecref_set(self, cname, rhs_cname): 2140 self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) 2141 2142 def put_var_decref(self, entry): 2143 if entry.type.is_pyobject: 2144 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) 2145 2146 def put_var_xdecref(self, entry, nanny=True): 2147 if entry.type.is_pyobject: 2148 if nanny: 2149 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) 2150 else: 2151 self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) 2152 2153 def put_var_decref_clear(self, entry): 2154 self._put_var_decref_clear(entry, null_check=False) 2155 2156 def put_var_xdecref_clear(self, entry): 2157 self._put_var_decref_clear(entry, null_check=True) 2158 2159 def _put_var_decref_clear(self, entry, null_check): 2160 if entry.type.is_pyobject: 2161 if entry.in_closure: 2162 # reset before DECREF to make sure closure state is 2163 # consistent during call to DECREF() 2164 self.putln("__Pyx_%sCLEAR(%s);" % ( 2165 null_check and 'X' or '', 2166 entry.cname)) 2167 else: 2168 self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( 2169 null_check and 'X' or '', 2170 self.entry_as_pyobject(entry), 2171 entry.cname)) 2172 2173 def put_var_decrefs(self, entries, used_only = 0): 2174 for entry in entries: 2175 if not used_only or entry.used: 2176 if entry.xdecref_cleanup: 2177 self.put_var_xdecref(entry) 2178 else: 2179 self.put_var_decref(entry) 2180 2181 def put_var_xdecrefs(self, entries): 2182 for entry in entries: 2183 self.put_var_xdecref(entry) 2184 2185 def put_var_xdecrefs_clear(self, entries): 2186 for entry in entries: 2187 self.put_var_xdecref_clear(entry) 2188 2189 def put_incref_memoryviewslice(self, slice_cname, have_gil=False): 2190 from . import MemoryView 2191 self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) 2192 self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) 2193 2194 def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): 2195 from . import MemoryView 2196 self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) 2197 self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) 2198 2199 def put_xgiveref_memoryviewslice(self, slice_cname): 2200 self.put_xgiveref("%s.memview" % slice_cname) 2201 2202 def put_init_to_py_none(self, cname, type, nanny=True): 2203 from .PyrexTypes import py_object_type, typecast 2204 py_none = typecast(type, py_object_type, "Py_None") 2205 if nanny: 2206 self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none)) 2207 else: 2208 self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none)) 2209 2210 def put_init_var_to_py_none(self, entry, template = "%s", nanny=True): 2211 code = template % entry.cname 2212 #if entry.type.is_extension_type: 2213 # code = "((PyObject*)%s)" % code 2214 self.put_init_to_py_none(code, entry.type, nanny) 2215 if entry.in_closure: 2216 self.put_giveref('Py_None') 2217 2218 def put_pymethoddef(self, entry, term, allow_skip=True, wrapper_code_writer=None): 2219 if entry.is_special or entry.name == '__getattribute__': 2220 if entry.name not in special_py_methods: 2221 if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']: 2222 pass 2223 # Python's typeobject.c will automatically fill in our slot 2224 # in add_operators() (called by PyType_Ready) with a value 2225 # that's better than ours. 2226 elif allow_skip: 2227 return 2228 2229 method_flags = entry.signature.method_flags() 2230 if not method_flags: 2231 return 2232 if entry.is_special: 2233 from . import TypeSlots 2234 method_flags += [TypeSlots.method_coexist] 2235 func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname 2236 # Add required casts, but try not to shadow real warnings. 2237 cast = '__Pyx_PyCFunctionFast' if 'METH_FASTCALL' in method_flags else 'PyCFunction' 2238 if 'METH_KEYWORDS' in method_flags: 2239 cast += 'WithKeywords' 2240 if cast != 'PyCFunction': 2241 func_ptr = '(void*)(%s)%s' % (cast, func_ptr) 2242 self.putln( 2243 '{"%s", (PyCFunction)%s, %s, %s}%s' % ( 2244 entry.name, 2245 func_ptr, 2246 "|".join(method_flags), 2247 entry.doc_cname if entry.doc else '0', 2248 term)) 2249 2250 def put_pymethoddef_wrapper(self, entry): 2251 func_cname = entry.func_cname 2252 if entry.is_special: 2253 method_flags = entry.signature.method_flags() 2254 if method_flags and 'METH_NOARGS' in method_flags: 2255 # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one. 2256 func_cname = Naming.method_wrapper_prefix + func_cname 2257 self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % ( 2258 func_cname, entry.func_cname)) 2259 return func_cname 2260 2261 # GIL methods 2262 2263 def put_ensure_gil(self, declare_gilstate=True, variable=None): 2264 """ 2265 Acquire the GIL. The generated code is safe even when no PyThreadState 2266 has been allocated for this thread (for threads not initialized by 2267 using the Python API). Additionally, the code generated by this method 2268 may be called recursively. 2269 """ 2270 self.globalstate.use_utility_code( 2271 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) 2272 if self.globalstate.directives['fast_gil']: 2273 self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) 2274 else: 2275 self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) 2276 self.putln("#ifdef WITH_THREAD") 2277 if not variable: 2278 variable = '__pyx_gilstate_save' 2279 if declare_gilstate: 2280 self.put("PyGILState_STATE ") 2281 self.putln("%s = __Pyx_PyGILState_Ensure();" % variable) 2282 self.putln("#endif") 2283 2284 def put_release_ensured_gil(self, variable=None): 2285 """ 2286 Releases the GIL, corresponds to `put_ensure_gil`. 2287 """ 2288 if self.globalstate.directives['fast_gil']: 2289 self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) 2290 else: 2291 self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) 2292 if not variable: 2293 variable = '__pyx_gilstate_save' 2294 self.putln("#ifdef WITH_THREAD") 2295 self.putln("__Pyx_PyGILState_Release(%s);" % variable) 2296 self.putln("#endif") 2297 2298 def put_acquire_gil(self, variable=None): 2299 """ 2300 Acquire the GIL. The thread's thread state must have been initialized 2301 by a previous `put_release_gil` 2302 """ 2303 if self.globalstate.directives['fast_gil']: 2304 self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) 2305 else: 2306 self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) 2307 self.putln("#ifdef WITH_THREAD") 2308 self.putln("__Pyx_FastGIL_Forget();") 2309 if variable: 2310 self.putln('_save = %s;' % variable) 2311 self.putln("Py_BLOCK_THREADS") 2312 self.putln("#endif") 2313 2314 def put_release_gil(self, variable=None): 2315 "Release the GIL, corresponds to `put_acquire_gil`." 2316 if self.globalstate.directives['fast_gil']: 2317 self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) 2318 else: 2319 self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) 2320 self.putln("#ifdef WITH_THREAD") 2321 self.putln("PyThreadState *_save;") 2322 self.putln("Py_UNBLOCK_THREADS") 2323 if variable: 2324 self.putln('%s = _save;' % variable) 2325 self.putln("__Pyx_FastGIL_Remember();") 2326 self.putln("#endif") 2327 2328 def declare_gilstate(self): 2329 self.putln("#ifdef WITH_THREAD") 2330 self.putln("PyGILState_STATE __pyx_gilstate_save;") 2331 self.putln("#endif") 2332 2333 # error handling 2334 2335 def put_error_if_neg(self, pos, value): 2336 # TODO this path is almost _never_ taken, yet this macro makes is slower! 2337 # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) 2338 return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) 2339 2340 def put_error_if_unbound(self, pos, entry, in_nogil_context=False): 2341 from . import ExprNodes 2342 if entry.from_closure: 2343 func = '__Pyx_RaiseClosureNameError' 2344 self.globalstate.use_utility_code( 2345 ExprNodes.raise_closure_name_error_utility_code) 2346 elif entry.type.is_memoryviewslice and in_nogil_context: 2347 func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' 2348 self.globalstate.use_utility_code( 2349 ExprNodes.raise_unbound_memoryview_utility_code_nogil) 2350 else: 2351 func = '__Pyx_RaiseUnboundLocalError' 2352 self.globalstate.use_utility_code( 2353 ExprNodes.raise_unbound_local_error_utility_code) 2354 2355 self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( 2356 entry.type.check_for_null_code(entry.cname), 2357 func, 2358 entry.name, 2359 self.error_goto(pos))) 2360 2361 def set_error_info(self, pos, used=False): 2362 self.funcstate.should_declare_error_indicator = True 2363 if used: 2364 self.funcstate.uses_error_indicator = True 2365 return "__PYX_MARK_ERR_POS(%s, %s)" % ( 2366 self.lookup_filename(pos[0]), 2367 pos[1]) 2368 2369 def error_goto(self, pos, used=True): 2370 lbl = self.funcstate.error_label 2371 self.funcstate.use_label(lbl) 2372 if pos is None: 2373 return 'goto %s;' % lbl 2374 self.funcstate.should_declare_error_indicator = True 2375 if used: 2376 self.funcstate.uses_error_indicator = True 2377 return "__PYX_ERR(%s, %s, %s)" % ( 2378 self.lookup_filename(pos[0]), 2379 pos[1], 2380 lbl) 2381 2382 def error_goto_if(self, cond, pos): 2383 return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos)) 2384 2385 def error_goto_if_null(self, cname, pos): 2386 return self.error_goto_if("!%s" % cname, pos) 2387 2388 def error_goto_if_neg(self, cname, pos): 2389 return self.error_goto_if("%s < 0" % cname, pos) 2390 2391 def error_goto_if_PyErr(self, pos): 2392 return self.error_goto_if("PyErr_Occurred()", pos) 2393 2394 def lookup_filename(self, filename): 2395 return self.globalstate.lookup_filename(filename) 2396 2397 def put_declare_refcount_context(self): 2398 self.putln('__Pyx_RefNannyDeclarations') 2399 2400 def put_setup_refcount_context(self, name, acquire_gil=False): 2401 if acquire_gil: 2402 self.globalstate.use_utility_code( 2403 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) 2404 self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) 2405 2406 def put_finish_refcount_context(self): 2407 self.putln("__Pyx_RefNannyFinishContext();") 2408 2409 def put_add_traceback(self, qualified_name, include_cline=True): 2410 """ 2411 Build a Python traceback for propagating exceptions. 2412 2413 qualified_name should be the qualified name of the function. 2414 """ 2415 format_tuple = ( 2416 qualified_name, 2417 Naming.clineno_cname if include_cline else 0, 2418 Naming.lineno_cname, 2419 Naming.filename_cname, 2420 ) 2421 self.funcstate.uses_error_indicator = True 2422 self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) 2423 2424 def put_unraisable(self, qualified_name, nogil=False): 2425 """ 2426 Generate code to print a Python warning for an unraisable exception. 2427 2428 qualified_name should be the qualified name of the function. 2429 """ 2430 format_tuple = ( 2431 qualified_name, 2432 Naming.clineno_cname, 2433 Naming.lineno_cname, 2434 Naming.filename_cname, 2435 self.globalstate.directives['unraisable_tracebacks'], 2436 nogil, 2437 ) 2438 self.funcstate.uses_error_indicator = True 2439 self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %d, %d);' % format_tuple) 2440 self.globalstate.use_utility_code( 2441 UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c")) 2442 2443 def put_trace_declarations(self): 2444 self.putln('__Pyx_TraceDeclarations') 2445 2446 def put_trace_frame_init(self, codeobj=None): 2447 if codeobj: 2448 self.putln('__Pyx_TraceFrameInit(%s)' % codeobj) 2449 2450 def put_trace_call(self, name, pos, nogil=False): 2451 self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d, %s);' % ( 2452 name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil, self.error_goto(pos))) 2453 2454 def put_trace_exception(self): 2455 self.putln("__Pyx_TraceException();") 2456 2457 def put_trace_return(self, retvalue_cname, nogil=False): 2458 self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil)) 2459 2460 def putln_openmp(self, string): 2461 self.putln("#ifdef _OPENMP") 2462 self.putln(string) 2463 self.putln("#endif /* _OPENMP */") 2464 2465 def undef_builtin_expect(self, cond): 2466 """ 2467 Redefine the macros likely() and unlikely to no-ops, depending on 2468 condition 'cond' 2469 """ 2470 self.putln("#if %s" % cond) 2471 self.putln(" #undef likely") 2472 self.putln(" #undef unlikely") 2473 self.putln(" #define likely(x) (x)") 2474 self.putln(" #define unlikely(x) (x)") 2475 self.putln("#endif") 2476 2477 def redef_builtin_expect(self, cond): 2478 self.putln("#if %s" % cond) 2479 self.putln(" #undef likely") 2480 self.putln(" #undef unlikely") 2481 self.putln(" #define likely(x) __builtin_expect(!!(x), 1)") 2482 self.putln(" #define unlikely(x) __builtin_expect(!!(x), 0)") 2483 self.putln("#endif") 2484 2485 2486class PyrexCodeWriter(object): 2487 # f file output file 2488 # level int indentation level 2489 2490 def __init__(self, outfile_name): 2491 self.f = Utils.open_new_file(outfile_name) 2492 self.level = 0 2493 2494 def putln(self, code): 2495 self.f.write("%s%s\n" % (" " * self.level, code)) 2496 2497 def indent(self): 2498 self.level += 1 2499 2500 def dedent(self): 2501 self.level -= 1 2502 2503class PyxCodeWriter(object): 2504 """ 2505 Can be used for writing out some Cython code. To use the indenter 2506 functionality, the Cython.Compiler.Importer module will have to be used 2507 to load the code to support python 2.4 2508 """ 2509 2510 def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'): 2511 self.buffer = buffer or StringIOTree() 2512 self.level = indent_level 2513 self.context = context 2514 self.encoding = encoding 2515 2516 def indent(self, levels=1): 2517 self.level += levels 2518 return True 2519 2520 def dedent(self, levels=1): 2521 self.level -= levels 2522 2523 def indenter(self, line): 2524 """ 2525 Instead of 2526 2527 with pyx_code.indenter("for i in range(10):"): 2528 pyx_code.putln("print i") 2529 2530 write 2531 2532 if pyx_code.indenter("for i in range(10);"): 2533 pyx_code.putln("print i") 2534 pyx_code.dedent() 2535 """ 2536 self.putln(line) 2537 self.indent() 2538 return True 2539 2540 def getvalue(self): 2541 result = self.buffer.getvalue() 2542 if isinstance(result, bytes): 2543 result = result.decode(self.encoding) 2544 return result 2545 2546 def putln(self, line, context=None): 2547 context = context or self.context 2548 if context: 2549 line = sub_tempita(line, context) 2550 self._putln(line) 2551 2552 def _putln(self, line): 2553 self.buffer.write("%s%s\n" % (self.level * " ", line)) 2554 2555 def put_chunk(self, chunk, context=None): 2556 context = context or self.context 2557 if context: 2558 chunk = sub_tempita(chunk, context) 2559 2560 chunk = textwrap.dedent(chunk) 2561 for line in chunk.splitlines(): 2562 self._putln(line) 2563 2564 def insertion_point(self): 2565 return PyxCodeWriter(self.buffer.insertion_point(), self.level, 2566 self.context) 2567 2568 def named_insertion_point(self, name): 2569 setattr(self, name, self.insertion_point()) 2570 2571 2572class ClosureTempAllocator(object): 2573 def __init__(self, klass): 2574 self.klass = klass 2575 self.temps_allocated = {} 2576 self.temps_free = {} 2577 self.temps_count = 0 2578 2579 def reset(self): 2580 for type, cnames in self.temps_allocated.items(): 2581 self.temps_free[type] = list(cnames) 2582 2583 def allocate_temp(self, type): 2584 if type not in self.temps_allocated: 2585 self.temps_allocated[type] = [] 2586 self.temps_free[type] = [] 2587 elif self.temps_free[type]: 2588 return self.temps_free[type].pop(0) 2589 cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count) 2590 self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True) 2591 self.temps_allocated[type].append(cname) 2592 self.temps_count += 1 2593 return cname 2594