1#
2#   Parse tree nodes
3#
4
5from __future__ import absolute_import
6
7import cython
8
9cython.declare(sys=object, os=object, copy=object,
10               Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
11               py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
12               StructOrUnionScope=object, PyClassScope=object,
13               CppClassScope=object, UtilityCode=object, EncodedString=object,
14               error_type=object, _py_int_types=object)
15
16import sys, os, copy
17from itertools import chain
18
19from . import Builtin
20from .Errors import error, warning, InternalError, CompileError, CannotSpecialize
21from . import Naming
22from . import PyrexTypes
23from . import TypeSlots
24from .PyrexTypes import py_object_type, error_type
25from .Symtab import (ModuleScope, LocalScope, ClosureScope, PropertyScope,
26                     StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope,
27                     CppScopedEnumScope, punycodify_name)
28from .Code import UtilityCode
29from .StringEncoding import EncodedString
30from . import Future
31from . import Options
32from . import DebugFlags
33from .Pythran import has_np_pythran, pythran_type, is_pythran_buffer
34from ..Utils import add_metaclass
35
36
37if sys.version_info[0] >= 3:
38    _py_int_types = int
39else:
40    _py_int_types = (int, long)
41
42
43IMPLICIT_CLASSMETHODS = {"__init_subclass__", "__class_getitem__"}
44
45
46def relative_position(pos):
47    return (pos[0].get_filenametable_entry(), pos[1])
48
49
50def embed_position(pos, docstring):
51    if not Options.embed_pos_in_docstring:
52        return docstring
53    pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
54    if docstring is None:
55        # unicode string
56        return EncodedString(pos_line)
57
58    # make sure we can encode the filename in the docstring encoding
59    # otherwise make the docstring a unicode string
60    encoding = docstring.encoding
61    if encoding is not None:
62        try:
63            pos_line.encode(encoding)
64        except UnicodeEncodeError:
65            encoding = None
66
67    if not docstring:
68        # reuse the string encoding of the original docstring
69        doc = EncodedString(pos_line)
70    else:
71        doc = EncodedString(pos_line + u'\n' + docstring)
72    doc.encoding = encoding
73    return doc
74
75
76def write_func_call(func, codewriter_class):
77    def f(*args, **kwds):
78        if len(args) > 1 and isinstance(args[1], codewriter_class):
79            # here we annotate the code with this function call
80            # but only if new code is generated
81            node, code = args[:2]
82            marker = '                    /* %s -> %s.%s %s */' % (
83                ' ' * code.call_level,
84                node.__class__.__name__,
85                func.__name__,
86                node.pos[1:],
87            )
88            insertion_point = code.insertion_point()
89            start = code.buffer.stream.tell()
90            code.call_level += 4
91            res = func(*args, **kwds)
92            code.call_level -= 4
93            if start != code.buffer.stream.tell():
94                code.putln(marker.replace('->', '<-', 1))
95                insertion_point.putln(marker)
96            return res
97        else:
98            return func(*args, **kwds)
99    return f
100
101
102class VerboseCodeWriter(type):
103    # Set this as a metaclass to trace function calls in code.
104    # This slows down code generation and makes much larger files.
105    def __new__(cls, name, bases, attrs):
106        from types import FunctionType
107        from .Code import CCodeWriter
108        attrs = dict(attrs)
109        for mname, m in attrs.items():
110            if isinstance(m, FunctionType):
111                attrs[mname] = write_func_call(m, CCodeWriter)
112        return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)
113
114
115class CheckAnalysers(type):
116    """Metaclass to check that type analysis functions return a node.
117    """
118    methods = frozenset({
119        'analyse_types',
120        'analyse_expressions',
121        'analyse_target_types',
122    })
123
124    def __new__(cls, name, bases, attrs):
125        from types import FunctionType
126        def check(name, func):
127            def call(*args, **kwargs):
128                retval = func(*args, **kwargs)
129                if retval is None:
130                    print('%s %s %s' % (name, args, kwargs))
131                return retval
132            return call
133
134        attrs = dict(attrs)
135        for mname, m in attrs.items():
136            if isinstance(m, FunctionType) and mname in cls.methods:
137                attrs[mname] = check(mname, m)
138        return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)
139
140
141def _with_metaclass(cls):
142    if DebugFlags.debug_trace_code_generation:
143        return add_metaclass(VerboseCodeWriter)(cls)
144    #return add_metaclass(CheckAnalysers)(cls)
145    return cls
146
147
148@_with_metaclass
149class Node(object):
150    #  pos         (string, int, int)   Source file position
151    #  is_name     boolean              Is a NameNode
152    #  is_literal  boolean              Is a ConstNode
153
154    is_name = 0
155    is_none = 0
156    is_nonecheck = 0
157    is_literal = 0
158    is_terminator = 0
159    is_wrapper = False  # is a DefNode wrapper for a C function
160    is_cproperty = False
161    temps = None
162
163    # All descendants should set child_attrs to a list of the attributes
164    # containing nodes considered "children" in the tree. Each such attribute
165    # can either contain a single node or a list of nodes. See Visitor.py.
166    child_attrs = None
167
168    # Subset of attributes that are evaluated in the outer scope (e.g. function default arguments).
169    outer_attrs = None
170
171    cf_state = None
172
173    # This may be an additional (or 'actual') type that will be checked when
174    # this node is coerced to another type. This could be useful to set when
175    # the actual type to which it can coerce is known, but you want to leave
176    # the type a py_object_type
177    coercion_type = None
178
179    def __init__(self, pos, **kw):
180        self.pos = pos
181        self.__dict__.update(kw)
182
183    gil_message = "Operation"
184
185    nogil_check = None
186    in_nogil_context = False  # For use only during code generation.
187
188    def gil_error(self, env=None):
189        error(self.pos, "%s not allowed without gil" % self.gil_message)
190
191    cpp_message = "Operation"
192
193    def cpp_check(self, env):
194        if not env.is_cpp():
195            self.cpp_error()
196
197    def cpp_error(self):
198        error(self.pos, "%s only allowed in c++" % self.cpp_message)
199
200    def clone_node(self):
201        """Clone the node. This is defined as a shallow copy, except for member lists
202           amongst the child attributes (from get_child_accessors) which are also
203           copied. Lists containing child nodes are thus seen as a way for the node
204           to hold multiple children directly; the list is not treated as a separate
205           level in the tree."""
206        result = copy.copy(self)
207        for attrname in result.child_attrs:
208            value = getattr(result, attrname)
209            if isinstance(value, list):
210                setattr(result, attrname, [x for x in value])
211        return result
212
213
214    #
215    #  There are 3 main phases of parse tree processing, applied in order to
216    #  all the statements in a given scope-block:
217    #
218    #  (0) analyse_declarations
219    #        Make symbol table entries for all declarations at the current
220    #        level, both explicit (def, cdef, etc.) and implicit (assignment
221    #        to an otherwise undeclared name).
222    #
223    #  (1) analyse_expressions
224    #         Determine the result types of expressions and fill in the
225    #         'type' attribute of each ExprNode. Insert coercion nodes into the
226    #         tree where needed to convert to and from Python objects.
227    #         Replace tree nodes with more appropriate implementations found by
228    #         the type analysis.
229    #
230    #  (2) generate_code
231    #         Emit C code for all declarations, statements and expressions.
232    #
233    #  These phases are triggered by tree transformations.
234    #  See the full pipeline in Pipeline.py.
235    #
236
237    def analyse_declarations(self, env):
238        pass
239
240    def analyse_expressions(self, env):
241        raise InternalError("analyse_expressions not implemented for %s" %
242            self.__class__.__name__)
243
244    def generate_code(self, code):
245        raise InternalError("generate_code not implemented for %s" %
246            self.__class__.__name__)
247
248    def annotate(self, code):
249        # mro does the wrong thing
250        if isinstance(self, BlockNode):
251            self.body.annotate(code)
252
253    def end_pos(self):
254        try:
255            return self._end_pos
256        except AttributeError:
257            pos = self.pos
258            if not self.child_attrs:
259                self._end_pos = pos
260                return pos
261            for attr in self.child_attrs:
262                child = getattr(self, attr)
263                # Sometimes lists, sometimes nodes
264                if child is None:
265                    pass
266                elif isinstance(child, list):
267                    for c in child:
268                        pos = max(pos, c.end_pos())
269                else:
270                    pos = max(pos, child.end_pos())
271            self._end_pos = pos
272            return pos
273
274    def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
275        """Debug helper method that returns a recursive string representation of this node.
276        """
277        if cutoff == 0:
278            return "<...nesting level cutoff...>"
279        if encountered is None:
280            encountered = set()
281        if id(self) in encountered:
282            return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
283        encountered.add(id(self))
284
285        def dump_child(x, level):
286            if isinstance(x, Node):
287                return x.dump(level, filter_out, cutoff-1, encountered)
288            elif isinstance(x, list):
289                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
290            else:
291                return repr(x)
292
293        attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
294        if len(attrs) == 0:
295            return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
296        else:
297            indent = "  " * level
298            res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
299            for key, value in attrs:
300                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
301            res += "%s>" % indent
302            return res
303
304    def dump_pos(self, mark_column=False, marker='(#)'):
305        """Debug helper method that returns the source code context of this node as a string.
306        """
307        if not self.pos:
308            return u''
309        source_desc, line, col = self.pos
310        contents = source_desc.get_lines(encoding='ASCII', error_handling='ignore')
311        # line numbers start at 1
312        lines = contents[max(0, line-3):line]
313        current = lines[-1]
314        if mark_column:
315            current = current[:col] + marker + current[col:]
316        lines[-1] = current.rstrip() + u'             # <<<<<<<<<<<<<<\n'
317        lines += contents[line:line+2]
318        return u'"%s":%d:%d\n%s\n' % (
319            source_desc.get_escaped_description(), line, col, u''.join(lines))
320
321
322class CompilerDirectivesNode(Node):
323    """
324    Sets compiler directives for the children nodes
325    """
326    #  directives     {string:value}  A dictionary holding the right value for
327    #                                 *all* possible directives.
328    #  body           Node
329    child_attrs = ["body"]
330
331    def analyse_declarations(self, env):
332        old = env.directives
333        env.directives = self.directives
334        self.body.analyse_declarations(env)
335        env.directives = old
336
337    def analyse_expressions(self, env):
338        old = env.directives
339        env.directives = self.directives
340        self.body = self.body.analyse_expressions(env)
341        env.directives = old
342        return self
343
344    def generate_function_definitions(self, env, code):
345        env_old = env.directives
346        code_old = code.globalstate.directives
347        code.globalstate.directives = self.directives
348        self.body.generate_function_definitions(env, code)
349        env.directives = env_old
350        code.globalstate.directives = code_old
351
352    def generate_execution_code(self, code):
353        old = code.globalstate.directives
354        code.globalstate.directives = self.directives
355        self.body.generate_execution_code(code)
356        code.globalstate.directives = old
357
358    def annotate(self, code):
359        old = code.globalstate.directives
360        code.globalstate.directives = self.directives
361        self.body.annotate(code)
362        code.globalstate.directives = old
363
364
365class BlockNode(object):
366    #  Mixin class for nodes representing a declaration block.
367
368    def generate_cached_builtins_decls(self, env, code):
369        entries = env.global_scope().undeclared_cached_builtins
370        for entry in entries:
371            code.globalstate.add_cached_builtin_decl(entry)
372        del entries[:]
373
374    def generate_lambda_definitions(self, env, code):
375        for node in env.lambda_defs:
376            node.generate_function_definitions(env, code)
377
378
379class StatListNode(Node):
380    # stats     a list of StatNode
381
382    child_attrs = ["stats"]
383
384    @staticmethod
385    def create_analysed(pos, env, *args, **kw):
386        node = StatListNode(pos, *args, **kw)
387        return node  # No node-specific analysis needed
388
389    def analyse_declarations(self, env):
390        #print "StatListNode.analyse_declarations" ###
391        for stat in self.stats:
392            stat.analyse_declarations(env)
393
394    def analyse_expressions(self, env):
395        #print "StatListNode.analyse_expressions" ###
396        self.stats = [stat.analyse_expressions(env)
397                      for stat in self.stats]
398        return self
399
400    def generate_function_definitions(self, env, code):
401        #print "StatListNode.generate_function_definitions" ###
402        for stat in self.stats:
403            stat.generate_function_definitions(env, code)
404
405    def generate_execution_code(self, code):
406        #print "StatListNode.generate_execution_code" ###
407        for stat in self.stats:
408            code.mark_pos(stat.pos)
409            stat.generate_execution_code(code)
410
411    def annotate(self, code):
412        for stat in self.stats:
413            stat.annotate(code)
414
415
416class StatNode(Node):
417    #
418    #  Code generation for statements is split into the following subphases:
419    #
420    #  (1) generate_function_definitions
421    #        Emit C code for the definitions of any structs,
422    #        unions, enums and functions defined in the current
423    #        scope-block.
424    #
425    #  (2) generate_execution_code
426    #        Emit C code for executable statements.
427    #
428
429    def generate_function_definitions(self, env, code):
430        pass
431
432    def generate_execution_code(self, code):
433        raise InternalError("generate_execution_code not implemented for %s" %
434            self.__class__.__name__)
435
436
437class CDefExternNode(StatNode):
438    #  include_file       string or None
439    #  verbatim_include   string or None
440    #  body               StatListNode
441
442    child_attrs = ["body"]
443
444    def analyse_declarations(self, env):
445        old_cinclude_flag = env.in_cinclude
446        env.in_cinclude = 1
447        self.body.analyse_declarations(env)
448        env.in_cinclude = old_cinclude_flag
449
450        if self.include_file or self.verbatim_include:
451            # Determine whether include should be late
452            stats = self.body.stats
453            if not env.directives['preliminary_late_includes_cy28']:
454                late = False
455            elif not stats:
456                # Special case: empty 'cdef extern' blocks are early
457                late = False
458            else:
459                late = all(isinstance(node, CVarDefNode) for node in stats)
460            env.add_include_file(self.include_file, self.verbatim_include, late)
461
462    def analyse_expressions(self, env):
463        # Allow C properties, inline methods, etc. also in external types.
464        self.body = self.body.analyse_expressions(env)
465        return self
466
467    def generate_function_definitions(self, env, code):
468        self.body.generate_function_definitions(env, code)
469
470    def generate_execution_code(self, code):
471        pass
472
473    def annotate(self, code):
474        self.body.annotate(code)
475
476
477class CDeclaratorNode(Node):
478    # Part of a C declaration.
479    #
480    # Processing during analyse_declarations phase:
481    #
482    #   analyse
483    #      Returns (name, type) pair where name is the
484    #      CNameDeclaratorNode of the name being declared
485    #      and type is the type it is being declared as.
486    #
487    #  calling_convention  string   Calling convention of CFuncDeclaratorNode
488    #                               for which this is a base
489
490    child_attrs = []
491
492    calling_convention = ""
493
494    def declared_name(self):
495        return None
496
497    def analyse_templates(self):
498        # Only C++ functions have templates.
499        return None
500
501
502class CNameDeclaratorNode(CDeclaratorNode):
503    #  name    string             The Cython name being declared
504    #  cname   string or None     C name, if specified
505    #  default ExprNode or None   the value assigned on declaration
506
507    child_attrs = ['default']
508
509    default = None
510
511    def declared_name(self):
512        return self.name
513
514    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
515        if nonempty and self.name == '':
516            # May have mistaken the name for the type.
517            if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
518                error(self.pos, "Missing argument name")
519            elif base_type.is_void:
520                error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
521            else:
522                self.name = base_type.declaration_code("", for_display=1, pyrex=1)
523                base_type = py_object_type
524
525        if base_type.is_fused and env.fused_to_specific:
526            try:
527                base_type = base_type.specialize(env.fused_to_specific)
528            except CannotSpecialize:
529                error(self.pos,
530                      "'%s' cannot be specialized since its type is not a fused argument to this function" %
531                      self.name)
532
533        self.type = base_type
534        return self, base_type
535
536
537class CPtrDeclaratorNode(CDeclaratorNode):
538    # base     CDeclaratorNode
539
540    child_attrs = ["base"]
541
542    def declared_name(self):
543        return self.base.declared_name()
544
545    def analyse_templates(self):
546        return self.base.analyse_templates()
547
548    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
549        if base_type.is_pyobject:
550            error(self.pos, "Pointer base type cannot be a Python object")
551        ptr_type = PyrexTypes.c_ptr_type(base_type)
552        return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
553
554
555class _CReferenceDeclaratorBaseNode(CDeclaratorNode):
556    child_attrs = ["base"]
557
558    def declared_name(self):
559        return self.base.declared_name()
560
561    def analyse_templates(self):
562        return self.base.analyse_templates()
563
564
565class CReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode):
566    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
567        if base_type.is_pyobject:
568            error(self.pos, "Reference base type cannot be a Python object")
569        ref_type = PyrexTypes.c_ref_type(base_type)
570        return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
571
572
573class CppRvalueReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode):
574    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
575        if base_type.is_pyobject:
576            error(self.pos, "Rvalue-reference base type cannot be a Python object")
577        ref_type = PyrexTypes.cpp_rvalue_ref_type(base_type)
578        return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
579
580
581class CArrayDeclaratorNode(CDeclaratorNode):
582    # base        CDeclaratorNode
583    # dimension   ExprNode
584
585    child_attrs = ["base", "dimension"]
586
587    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
588        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
589            from .ExprNodes import TupleNode
590            if isinstance(self.dimension, TupleNode):
591                args = self.dimension.args
592            else:
593                args = self.dimension,
594            values = [v.analyse_as_type(env) for v in args]
595            if None in values:
596                ix = values.index(None)
597                error(args[ix].pos, "Template parameter not a type")
598                base_type = error_type
599            else:
600                base_type = base_type.specialize_here(self.pos, values)
601            return self.base.analyse(base_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
602        if self.dimension:
603            self.dimension = self.dimension.analyse_const_expression(env)
604            if not self.dimension.type.is_int:
605                error(self.dimension.pos, "Array dimension not integer")
606            size = self.dimension.get_constant_c_result_code()
607            if size is not None:
608                try:
609                    size = int(size)
610                except ValueError:
611                    # runtime constant?
612                    pass
613        else:
614            size = None
615        if not base_type.is_complete():
616            error(self.pos, "Array element type '%s' is incomplete" % base_type)
617        if base_type.is_pyobject:
618            error(self.pos, "Array element cannot be a Python object")
619        if base_type.is_cfunction:
620            error(self.pos, "Array element cannot be a function")
621        array_type = PyrexTypes.c_array_type(base_type, size)
622        return self.base.analyse(array_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
623
624
625class CFuncDeclaratorNode(CDeclaratorNode):
626    # base             CDeclaratorNode
627    # args             [CArgDeclNode]
628    # templates        [TemplatePlaceholderType]
629    # has_varargs      boolean
630    # exception_value  ConstNode or NameNode    NameNode when the name of a c++ exception conversion function
631    # exception_check  boolean or "+"    True if PyErr_Occurred check needed, "+" for a c++ check
632    # nogil            boolean    Can be called without gil
633    # with_gil         boolean    Acquire gil around function body
634    # is_const_method  boolean    Whether this is a const method
635
636    child_attrs = ["base", "args", "exception_value"]
637
638    overridable = 0
639    optional_arg_count = 0
640    is_const_method = 0
641    templates = None
642
643    def declared_name(self):
644        return self.base.declared_name()
645
646    def analyse_templates(self):
647        if isinstance(self.base, CArrayDeclaratorNode):
648            from .ExprNodes import TupleNode, NameNode
649            template_node = self.base.dimension
650            if isinstance(template_node, TupleNode):
651                template_nodes = template_node.args
652            elif isinstance(template_node, NameNode):
653                template_nodes = [template_node]
654            else:
655                error(template_node.pos, "Template arguments must be a list of names")
656                return None
657            self.templates = []
658            for template in template_nodes:
659                if isinstance(template, NameNode):
660                    self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
661                else:
662                    error(template.pos, "Template arguments must be a list of names")
663            self.base = self.base.base
664            return self.templates
665        else:
666            return None
667
668    def analyse(self, return_type, env, nonempty=0, directive_locals=None, visibility=None, in_pxd=False):
669        if directive_locals is None:
670            directive_locals = {}
671        if nonempty:
672            nonempty -= 1
673        func_type_args = []
674        for i, arg_node in enumerate(self.args):
675            name_declarator, type = arg_node.analyse(
676                env, nonempty=nonempty,
677                is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
678            name = name_declarator.name
679            if name in directive_locals:
680                type_node = directive_locals[name]
681                other_type = type_node.analyse_as_type(env)
682                if other_type is None:
683                    error(type_node.pos, "Not a type")
684                elif (type is not PyrexTypes.py_object_type
685                      and not type.same_as(other_type)):
686                    error(self.base.pos, "Signature does not agree with previous declaration")
687                    error(type_node.pos, "Previous declaration here")
688                else:
689                    type = other_type
690            if name_declarator.cname:
691                error(self.pos, "Function argument cannot have C name specification")
692            if i == 0 and env.is_c_class_scope and type.is_unspecified:
693                # fix the type of self
694                type = env.parent_type
695            # Turn *[] argument into **
696            if type.is_array:
697                type = PyrexTypes.c_ptr_type(type.base_type)
698            # Catch attempted C-style func(void) decl
699            if type.is_void:
700                error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
701            func_type_args.append(
702                PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
703            if arg_node.default:
704                self.optional_arg_count += 1
705            elif self.optional_arg_count:
706                error(self.pos, "Non-default argument follows default argument")
707
708        exc_val = None
709        exc_check = 0
710        if self.exception_check == '+':
711            env.add_include_file('ios')         # for std::ios_base::failure
712            env.add_include_file('new')         # for std::bad_alloc
713            env.add_include_file('stdexcept')
714            env.add_include_file('typeinfo')    # for std::bad_cast
715        if (return_type.is_pyobject
716                and (self.exception_value or self.exception_check)
717                and self.exception_check != '+'):
718            error(self.pos, "Exception clause not allowed for function returning Python object")
719        else:
720            if self.exception_value is None and self.exception_check and self.exception_check != '+':
721                # Use an explicit exception return value to speed up exception checks.
722                # Even if it is not declared, we can use the default exception value of the return type,
723                # unless the function is some kind of external function that we do not control.
724                if return_type.exception_value is not None and (visibility != 'extern' and not in_pxd):
725                    # Extension types are more difficult because the signature must match the base type signature.
726                    if not env.is_c_class_scope:
727                        from .ExprNodes import ConstNode
728                        self.exception_value = ConstNode(
729                            self.pos, value=return_type.exception_value, type=return_type)
730            if self.exception_value:
731                if self.exception_check == '+':
732                    self.exception_value = self.exception_value.analyse_const_expression(env)
733                    exc_val_type = self.exception_value.type
734                    if (not exc_val_type.is_error
735                            and not exc_val_type.is_pyobject
736                            and not (exc_val_type.is_cfunction
737                                     and not exc_val_type.return_type.is_pyobject
738                                     and not exc_val_type.args)
739                            and not (exc_val_type == PyrexTypes.c_char_type
740                                     and self.exception_value.value == '*')):
741                        error(self.exception_value.pos,
742                              "Exception value must be a Python exception or cdef function with no arguments or *.")
743                    exc_val = self.exception_value
744                else:
745                    self.exception_value = self.exception_value.analyse_types(env).coerce_to(
746                        return_type, env).analyse_const_expression(env)
747                    exc_val = self.exception_value.get_constant_c_result_code()
748                    if exc_val is None:
749                        error(self.exception_value.pos, "Exception value must be constant")
750                    if not return_type.assignable_from(self.exception_value.type):
751                        error(self.exception_value.pos,
752                              "Exception value incompatible with function return type")
753                    if (visibility != 'extern'
754                            and (return_type.is_int or return_type.is_float)
755                            and self.exception_value.has_constant_result()):
756                        try:
757                            type_default_value = float(return_type.default_value)
758                        except ValueError:
759                            pass
760                        else:
761                            if self.exception_value.constant_result == type_default_value:
762                                warning(self.pos, "Ambiguous exception value, same as default return value: %r" %
763                                        self.exception_value.constant_result)
764            exc_check = self.exception_check
765        if return_type.is_cfunction:
766            error(self.pos, "Function cannot return a function")
767        func_type = PyrexTypes.CFuncType(
768            return_type, func_type_args, self.has_varargs,
769            optional_arg_count=self.optional_arg_count,
770            exception_value=exc_val, exception_check=exc_check,
771            calling_convention=self.base.calling_convention,
772            nogil=self.nogil, with_gil=self.with_gil, is_overridable=self.overridable,
773            is_const_method=self.is_const_method,
774            templates=self.templates)
775
776        if self.optional_arg_count:
777            if func_type.is_fused:
778                # This is a bit of a hack... When we need to create specialized CFuncTypes
779                # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
780                # struct
781                def declare_opt_arg_struct(func_type, fused_cname):
782                    self.declare_optional_arg_struct(func_type, env, fused_cname)
783
784                func_type.declare_opt_arg_struct = declare_opt_arg_struct
785            else:
786                self.declare_optional_arg_struct(func_type, env)
787
788        callspec = env.directives['callspec']
789        if callspec:
790            current = func_type.calling_convention
791            if current and current != callspec:
792                error(self.pos, "cannot have both '%s' and '%s' "
793                      "calling conventions" % (current, callspec))
794            func_type.calling_convention = callspec
795
796        if func_type.return_type.is_rvalue_reference:
797            warning(self.pos, "Rvalue-reference as function return type not supported", 1)
798        for arg in func_type.args:
799            if arg.type.is_rvalue_reference and not arg.is_forwarding_reference():
800                warning(self.pos, "Rvalue-reference as function argument not supported", 1)
801
802        return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd)
803
804    def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
805        """
806        Declares the optional argument struct (the struct used to hold the
807        values for optional arguments). For fused cdef functions, this is
808        deferred as analyse_declarations is called only once (on the fused
809        cdef function).
810        """
811        scope = StructOrUnionScope()
812        arg_count_member = '%sn' % Naming.pyrex_prefix
813        scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)
814
815        for arg in func_type.args[len(func_type.args) - self.optional_arg_count:]:
816            scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject=True, allow_memoryview=True)
817
818        struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)
819
820        if fused_cname is not None:
821            struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)
822
823        op_args_struct = env.global_scope().declare_struct_or_union(
824            name=struct_cname,
825            kind='struct',
826            scope=scope,
827            typedef_flag=0,
828            pos=self.pos,
829            cname=struct_cname)
830
831        op_args_struct.defined_in_pxd = 1
832        op_args_struct.used = 1
833
834        func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)
835
836
837class CConstDeclaratorNode(CDeclaratorNode):
838    # base     CDeclaratorNode
839
840    child_attrs = ["base"]
841
842    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
843        if base_type.is_pyobject:
844            error(self.pos,
845                  "Const base type cannot be a Python object")
846        const = PyrexTypes.c_const_type(base_type)
847        return self.base.analyse(const, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
848
849
850class CArgDeclNode(Node):
851    # Item in a function declaration argument list.
852    #
853    # base_type      CBaseTypeNode
854    # declarator     CDeclaratorNode
855    # not_none       boolean            Tagged with 'not None'
856    # or_none        boolean            Tagged with 'or None'
857    # accept_none    boolean            Resolved boolean for not_none/or_none
858    # default        ExprNode or None
859    # default_value  PyObjectConst      constant for default value
860    # annotation     ExprNode or None   Py3 function arg annotation
861    # is_self_arg    boolean            Is the "self" arg of an extension type method
862    # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
863    # kw_only        boolean            Is a keyword-only argument
864    # is_dynamic     boolean            Non-literal arg stored inside CyFunction
865    # pos_only       boolean            Is a positional-only argument
866    #
867    # name_cstring                         property that converts the name to a cstring taking care of unicode
868    #                                      and quoting it
869
870    child_attrs = ["base_type", "declarator", "default", "annotation"]
871    outer_attrs = ["default", "annotation"]
872
873    is_self_arg = 0
874    is_type_arg = 0
875    is_generic = 1
876    kw_only = 0
877    pos_only = 0
878    not_none = 0
879    or_none = 0
880    type = None
881    name_declarator = None
882    default_value = None
883    annotation = None
884    is_dynamic = 0
885
886    def declared_name(self):
887        return self.declarator.declared_name()
888
889    @property
890    def name_cstring(self):
891        return self.name.as_c_string_literal()
892
893    @property
894    def hdr_cname(self):
895        # done lazily - needs self.entry to be set to get the class-mangled
896        # name, which means it has to be generated relatively late
897        if self.needs_conversion:
898            return punycodify_name(Naming.arg_prefix + self.entry.name)
899        else:
900            return punycodify_name(Naming.var_prefix + self.entry.name)
901
902
903    def analyse(self, env, nonempty=0, is_self_arg=False):
904        if is_self_arg:
905            self.base_type.is_self_arg = self.is_self_arg = is_self_arg
906        if self.type is not None:
907            return self.name_declarator, self.type
908
909        # The parser may misinterpret names as types. We fix that here.
910        if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
911            if nonempty:
912                if self.base_type.is_basic_c_type:
913                    # char, short, long called "int"
914                    type = self.base_type.analyse(env, could_be_name=True)
915                    arg_name = type.empty_declaration_code()
916                else:
917                    arg_name = self.base_type.name
918                self.declarator.name = EncodedString(arg_name)
919                self.base_type.name = None
920                self.base_type.is_basic_c_type = False
921            could_be_name = True
922        else:
923            could_be_name = False
924        self.base_type.is_arg = True
925        base_type = self.base_type.analyse(env, could_be_name=could_be_name)
926        base_arg_name = getattr(self.base_type, 'arg_name', None)
927        if base_arg_name:
928            self.declarator.name = base_arg_name
929
930        # The parser is unable to resolve the ambiguity of [] as part of the
931        # type (e.g. in buffers) or empty declarator (as with arrays).
932        # This is only arises for empty multi-dimensional arrays.
933        if (base_type.is_array
934                and isinstance(self.base_type, TemplatedTypeNode)
935                and isinstance(self.declarator, CArrayDeclaratorNode)):
936            declarator = self.declarator
937            while isinstance(declarator.base, CArrayDeclaratorNode):
938                declarator = declarator.base
939            declarator.base = self.base_type.array_declarator
940            base_type = base_type.base_type
941
942        # inject type declaration from annotations
943        # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis
944        if (self.annotation and env and env.directives['annotation_typing']
945                # CSimpleBaseTypeNode has a name attribute; CAnalysedBaseTypeNode
946                # (and maybe other options) doesn't
947                and getattr(self.base_type, "name", None) is None):
948            arg_type = self.inject_type_from_annotations(env)
949            if arg_type is not None:
950                base_type = arg_type
951        return self.declarator.analyse(base_type, env, nonempty=nonempty)
952
953    def inject_type_from_annotations(self, env):
954        annotation = self.annotation
955        if not annotation:
956            return None
957        base_type, arg_type = annotation.analyse_type_annotation(env, assigned_value=self.default)
958        if base_type is not None:
959            self.base_type = base_type
960        return arg_type
961
962    def calculate_default_value_code(self, code):
963        if self.default_value is None:
964            if self.default:
965                if self.default.is_literal:
966                    # will not output any code, just assign the result_code
967                    self.default.generate_evaluation_code(code)
968                    return self.type.cast_code(self.default.result())
969                self.default_value = code.get_argument_default_const(self.type)
970        return self.default_value
971
972    def annotate(self, code):
973        if self.default:
974            self.default.annotate(code)
975
976    def generate_assignment_code(self, code, target=None, overloaded_assignment=False):
977        default = self.default
978        if default is None or default.is_literal:
979            return
980        if target is None:
981            target = self.calculate_default_value_code(code)
982        default.generate_evaluation_code(code)
983        default.make_owned_reference(code)
984        result = default.result() if overloaded_assignment else default.result_as(self.type)
985        code.putln("%s = %s;" % (target, result))
986        code.put_giveref(default.result(), self.type)
987        default.generate_post_assignment_code(code)
988        default.free_temps(code)
989
990
991class CBaseTypeNode(Node):
992    # Abstract base class for C base type nodes.
993    #
994    # Processing during analyse_declarations phase:
995    #
996    #   analyse
997    #     Returns the type.
998
999    def analyse_as_type(self, env):
1000        return self.analyse(env)
1001
1002
1003class CAnalysedBaseTypeNode(Node):
1004    # type            type
1005
1006    child_attrs = []
1007
1008    def analyse(self, env, could_be_name=False):
1009        return self.type
1010
1011
1012class CSimpleBaseTypeNode(CBaseTypeNode):
1013    # name             string
1014    # module_path      [string]     Qualifying name components
1015    # is_basic_c_type  boolean
1016    # signed           boolean
1017    # longness         integer
1018    # complex          boolean
1019    # is_self_arg      boolean      Is self argument of C method
1020    # ##is_type_arg      boolean      Is type argument of class method
1021
1022    child_attrs = []
1023    arg_name = None   # in case the argument name was interpreted as a type
1024    module_path = []
1025    is_basic_c_type = False
1026    complex = False
1027    is_self_arg = False
1028
1029    def analyse(self, env, could_be_name=False):
1030        # Return type descriptor.
1031        #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
1032        type = None
1033        if self.is_basic_c_type:
1034            type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
1035            if not type:
1036                error(self.pos, "Unrecognised type modifier combination")
1037        elif self.name == "object" and not self.module_path:
1038            type = py_object_type
1039        elif self.name is None:
1040            if self.is_self_arg and env.is_c_class_scope:
1041                #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
1042                type = env.parent_type
1043            ## elif self.is_type_arg and env.is_c_class_scope:
1044            ##     type = Builtin.type_type
1045            else:
1046                type = py_object_type
1047        else:
1048            if self.module_path:
1049                # Maybe it's a nested C++ class.
1050                scope = env
1051                for item in self.module_path:
1052                    entry = scope.lookup(item)
1053                    if entry is not None and (
1054                        entry.is_cpp_class or
1055                        entry.is_type and entry.type.is_cpp_class
1056                    ):
1057                        scope = entry.type.scope
1058                    else:
1059                        scope = None
1060                        break
1061
1062                if scope is None:
1063                    # Maybe it's a cimport.
1064                    scope = env.find_imported_module(self.module_path, self.pos)
1065            else:
1066                scope = env
1067
1068            if scope:
1069                if scope.is_c_class_scope:
1070                    scope = scope.global_scope()
1071
1072                type = scope.lookup_type(self.name)
1073                if type is not None:
1074                    pass
1075                elif could_be_name:
1076                    if self.is_self_arg and env.is_c_class_scope:
1077                        type = env.parent_type
1078                    ## elif self.is_type_arg and env.is_c_class_scope:
1079                    ##     type = Builtin.type_type
1080                    else:
1081                        type = py_object_type
1082                    self.arg_name = EncodedString(self.name)
1083                else:
1084                    if self.templates:
1085                        if self.name not in self.templates:
1086                            error(self.pos, "'%s' is not a type identifier" % self.name)
1087                        type = PyrexTypes.TemplatePlaceholderType(self.name)
1088                    else:
1089                        error(self.pos, "'%s' is not a type identifier" % self.name)
1090        if type and type.is_fused and env.fused_to_specific:
1091            type = type.specialize(env.fused_to_specific)
1092        if self.complex:
1093            if not type.is_numeric or type.is_complex:
1094                error(self.pos, "can only complexify c numeric types")
1095            type = PyrexTypes.CComplexType(type)
1096            type.create_declaration_utility_code(env)
1097        elif type is Builtin.complex_type:
1098            # Special case: optimise builtin complex type into C's
1099            # double complex.  The parser cannot do this (as for the
1100            # normal scalar types) as the user may have redeclared the
1101            # 'complex' type.  Testing for the exact type here works.
1102            type = PyrexTypes.c_double_complex_type
1103            type.create_declaration_utility_code(env)
1104            self.complex = True
1105        if type:
1106            return type
1107        else:
1108            return PyrexTypes.error_type
1109
1110class MemoryViewSliceTypeNode(CBaseTypeNode):
1111
1112    name = 'memoryview'
1113    child_attrs = ['base_type_node', 'axes']
1114
1115    def analyse(self, env, could_be_name=False):
1116
1117        base_type = self.base_type_node.analyse(env)
1118        if base_type.is_error: return base_type
1119
1120        from . import MemoryView
1121
1122        try:
1123            axes_specs = MemoryView.get_axes_specs(env, self.axes)
1124        except CompileError as e:
1125            error(e.position, e.message_only)
1126            self.type = PyrexTypes.ErrorType()
1127            return self.type
1128
1129        if not MemoryView.validate_axes(self.pos, axes_specs):
1130            self.type = error_type
1131        else:
1132            self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
1133            self.type.validate_memslice_dtype(self.pos)
1134            self.use_memview_utilities(env)
1135
1136        return self.type
1137
1138    def use_memview_utilities(self, env):
1139        from . import MemoryView
1140        env.use_utility_code(MemoryView.view_utility_code)
1141
1142
1143class CNestedBaseTypeNode(CBaseTypeNode):
1144    # For C++ classes that live inside other C++ classes.
1145
1146    # name             string
1147    # base_type        CBaseTypeNode
1148
1149    child_attrs = ['base_type']
1150
1151    def analyse(self, env, could_be_name=None):
1152        base_type = self.base_type.analyse(env)
1153        if base_type is PyrexTypes.error_type:
1154            return PyrexTypes.error_type
1155        if not base_type.is_cpp_class:
1156            error(self.pos, "'%s' is not a valid type scope" % base_type)
1157            return PyrexTypes.error_type
1158        type_entry = base_type.scope.lookup_here(self.name)
1159        if not type_entry or not type_entry.is_type:
1160            error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
1161            return PyrexTypes.error_type
1162        return type_entry.type
1163
1164
1165class TemplatedTypeNode(CBaseTypeNode):
1166    #  After parsing:
1167    #  positional_args  [ExprNode]        List of positional arguments
1168    #  keyword_args     DictNode          Keyword arguments
1169    #  base_type_node   CBaseTypeNode
1170
1171    #  After analysis:
1172    #  type             PyrexTypes.BufferType or PyrexTypes.CppClassType  ...containing the right options
1173
1174    child_attrs = ["base_type_node", "positional_args",
1175                   "keyword_args", "dtype_node"]
1176
1177    dtype_node = None
1178
1179    name = None
1180
1181    def analyse(self, env, could_be_name=False, base_type=None):
1182        if base_type is None:
1183            base_type = self.base_type_node.analyse(env)
1184        if base_type.is_error: return base_type
1185
1186        if base_type.is_cpp_class and base_type.is_template_type():
1187            # Templated class
1188            if self.keyword_args and self.keyword_args.key_value_pairs:
1189                error(self.pos, "c++ templates cannot take keyword arguments")
1190                self.type = PyrexTypes.error_type
1191            else:
1192                template_types = []
1193                for template_node in self.positional_args:
1194                    type = template_node.analyse_as_type(env)
1195                    if type is None:
1196                        error(template_node.pos, "unknown type in template argument")
1197                        type = error_type
1198                    template_types.append(type)
1199                self.type = base_type.specialize_here(self.pos, template_types)
1200
1201        elif base_type.is_pyobject:
1202            # Buffer
1203            from . import Buffer
1204
1205            options = Buffer.analyse_buffer_options(
1206                self.pos,
1207                env,
1208                self.positional_args,
1209                self.keyword_args,
1210                base_type.buffer_defaults)
1211
1212            if sys.version_info[0] < 3:
1213                # Py 2.x enforces byte strings as keyword arguments ...
1214                options = dict([(name.encode('ASCII'), value)
1215                                for name, value in options.items()])
1216
1217            self.type = PyrexTypes.BufferType(base_type, **options)
1218            if has_np_pythran(env) and is_pythran_buffer(self.type):
1219                self.type = PyrexTypes.PythranExpr(pythran_type(self.type), self.type)
1220
1221        else:
1222            # Array
1223            empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
1224            if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
1225                error(self.pos, "invalid array declaration")
1226                self.type = PyrexTypes.error_type
1227            else:
1228                # It would be nice to merge this class with CArrayDeclaratorNode,
1229                # but arrays are part of the declaration, not the type...
1230                if not self.positional_args:
1231                    dimension = None
1232                else:
1233                    dimension = self.positional_args[0]
1234                self.array_declarator = CArrayDeclaratorNode(
1235                    self.pos,
1236                    base=empty_declarator,
1237                    dimension=dimension)
1238                self.type = self.array_declarator.analyse(base_type, env)[1]
1239
1240        if self.type.is_fused and env.fused_to_specific:
1241            try:
1242                self.type = self.type.specialize(env.fused_to_specific)
1243            except CannotSpecialize:
1244                error(self.pos,
1245                      "'%s' cannot be specialized since its type is not a fused argument to this function" %
1246                      self.name)
1247
1248        return self.type
1249
1250
1251class CComplexBaseTypeNode(CBaseTypeNode):
1252    # base_type   CBaseTypeNode
1253    # declarator  CDeclaratorNode
1254
1255    child_attrs = ["base_type", "declarator"]
1256
1257    def analyse(self, env, could_be_name=False):
1258        base = self.base_type.analyse(env, could_be_name)
1259        _, type = self.declarator.analyse(base, env)
1260        return type
1261
1262
1263class CTupleBaseTypeNode(CBaseTypeNode):
1264    # components [CBaseTypeNode]
1265
1266    child_attrs = ["components"]
1267
1268    def analyse(self, env, could_be_name=False):
1269        component_types = []
1270        for c in self.components:
1271            type = c.analyse(env)
1272            if type.is_pyobject:
1273                error(c.pos, "Tuple types can't (yet) contain Python objects.")
1274                return error_type
1275            component_types.append(type)
1276        entry = env.declare_tuple_type(self.pos, component_types)
1277        entry.used = True
1278        return entry.type
1279
1280
1281class FusedTypeNode(CBaseTypeNode):
1282    """
1283    Represents a fused type in a ctypedef statement:
1284
1285        ctypedef cython.fused_type(int, long, long long) integral
1286
1287    name            str                     name of this fused type
1288    types           [CSimpleBaseTypeNode]   is the list of types to be fused
1289    """
1290
1291    child_attrs = []
1292
1293    def analyse_declarations(self, env):
1294        type = self.analyse(env)
1295        entry = env.declare_typedef(self.name, type, self.pos)
1296
1297        # Omit the typedef declaration that self.declarator would produce
1298        entry.in_cinclude = True
1299
1300    def analyse(self, env, could_be_name=False):
1301        types = []
1302        for type_node in self.types:
1303            type = type_node.analyse_as_type(env)
1304
1305            if not type:
1306                error(type_node.pos, "Not a type")
1307                continue
1308
1309            if type in types:
1310                error(type_node.pos, "Type specified multiple times")
1311            else:
1312                types.append(type)
1313
1314        # if len(self.types) == 1:
1315        #     return types[0]
1316
1317        return PyrexTypes.FusedType(types, name=self.name)
1318
1319
1320class CConstOrVolatileTypeNode(CBaseTypeNode):
1321    # base_type     CBaseTypeNode
1322    # is_const      boolean
1323    # is_volatile   boolean
1324
1325    child_attrs = ["base_type"]
1326
1327    def analyse(self, env, could_be_name=False):
1328        base = self.base_type.analyse(env, could_be_name)
1329        if base.is_pyobject:
1330            error(self.pos,
1331                  "Const/volatile base type cannot be a Python object")
1332        return PyrexTypes.c_const_or_volatile_type(base, self.is_const, self.is_volatile)
1333
1334
1335class CVarDefNode(StatNode):
1336    #  C variable definition or forward/extern function declaration.
1337    #
1338    #  visibility    'private' or 'public' or 'extern'
1339    #  base_type     CBaseTypeNode
1340    #  declarators   [CDeclaratorNode]
1341    #  in_pxd        boolean
1342    #  api           boolean
1343    #  overridable   boolean        whether it is a cpdef
1344    #  modifiers     ['inline']
1345
1346    #  decorators    [cython.locals(...)] or None
1347    #  directive_locals { string : NameNode } locals defined by cython.locals(...)
1348
1349    child_attrs = ["base_type", "declarators"]
1350
1351    decorators = None
1352    directive_locals = None
1353
1354    def analyse_declarations(self, env, dest_scope=None):
1355        if self.directive_locals is None:
1356            self.directive_locals = {}
1357        if not dest_scope:
1358            dest_scope = env
1359        self.dest_scope = dest_scope
1360
1361        if self.declarators:
1362            templates = self.declarators[0].analyse_templates()
1363        else:
1364            templates = None
1365        if templates is not None:
1366            if self.visibility != 'extern':
1367                error(self.pos, "Only extern functions allowed")
1368            if len(self.declarators) > 1:
1369                error(self.declarators[1].pos, "Can't multiply declare template types")
1370            env = TemplateScope('func_template', env)
1371            env.directives = env.outer_scope.directives
1372            for template_param in templates:
1373                env.declare_type(template_param.name, template_param, self.pos)
1374
1375        base_type = self.base_type.analyse(env)
1376
1377        if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
1378                                                       env.is_module_scope):
1379            error(self.pos, "Fused types not allowed here")
1380            return error_type
1381
1382        self.entry = None
1383        visibility = self.visibility
1384
1385        for declarator in self.declarators:
1386
1387            if (len(self.declarators) > 1
1388                    and not isinstance(declarator, CNameDeclaratorNode)
1389                    and env.directives['warn.multiple_declarators']):
1390                warning(
1391                    declarator.pos,
1392                    "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). "
1393                    "Each pointer declaration should be on its own line.", 1)
1394
1395            create_extern_wrapper = (self.overridable
1396                                     and self.visibility == 'extern'
1397                                     and env.is_module_scope)
1398            if create_extern_wrapper:
1399                declarator.overridable = False
1400            if isinstance(declarator, CFuncDeclaratorNode):
1401                name_declarator, type = declarator.analyse(
1402                    base_type, env, directive_locals=self.directive_locals, visibility=visibility, in_pxd=self.in_pxd)
1403            else:
1404                name_declarator, type = declarator.analyse(
1405                    base_type, env, visibility=visibility, in_pxd=self.in_pxd)
1406            if not type.is_complete():
1407                if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
1408                    error(declarator.pos, "Variable type '%s' is incomplete" % type)
1409            if self.visibility == 'extern' and type.is_pyobject:
1410                error(declarator.pos, "Python object cannot be declared extern")
1411            name = name_declarator.name
1412            cname = name_declarator.cname
1413            if name == '':
1414                error(declarator.pos, "Missing name in declaration.")
1415                return
1416            if type.is_reference and self.visibility != 'extern':
1417                error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
1418            if type.is_rvalue_reference and self.visibility != 'extern':
1419                error(declarator.pos, "C++ rvalue-references cannot be declared")
1420            if type.is_cfunction:
1421                if 'staticmethod' in env.directives:
1422                    type.is_static_method = True
1423                self.entry = dest_scope.declare_cfunction(
1424                    name, type, declarator.pos,
1425                    cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
1426                    api=self.api, modifiers=self.modifiers, overridable=self.overridable)
1427                if self.entry is not None:
1428                    self.entry.directive_locals = copy.copy(self.directive_locals)
1429                if create_extern_wrapper:
1430                    self.entry.type.create_to_py_utility_code(env)
1431                    self.entry.create_wrapper = True
1432            else:
1433                if self.overridable:
1434                    error(self.pos, "Variables cannot be declared with 'cpdef'. Use 'cdef' instead.")
1435                if self.directive_locals:
1436                    error(self.pos, "Decorators can only be followed by functions")
1437                self.entry = dest_scope.declare_var(
1438                    name, type, declarator.pos,
1439                    cname=cname, visibility=visibility, in_pxd=self.in_pxd,
1440                    api=self.api, is_cdef=1)
1441                if Options.docstrings:
1442                    self.entry.doc = embed_position(self.pos, self.doc)
1443
1444
1445class CStructOrUnionDefNode(StatNode):
1446    #  name          string
1447    #  cname         string or None
1448    #  kind          "struct" or "union"
1449    #  typedef_flag  boolean
1450    #  visibility    "public" or "private"
1451    #  api           boolean
1452    #  in_pxd        boolean
1453    #  attributes    [CVarDefNode] or None
1454    #  entry         Entry
1455    #  packed        boolean
1456
1457    child_attrs = ["attributes"]
1458
1459    def declare(self, env, scope=None):
1460        self.entry = env.declare_struct_or_union(
1461            self.name, self.kind, scope, self.typedef_flag, self.pos,
1462            self.cname, visibility=self.visibility, api=self.api,
1463            packed=self.packed)
1464
1465    def analyse_declarations(self, env):
1466        scope = None
1467        if self.attributes is not None:
1468            scope = StructOrUnionScope(self.name)
1469        self.declare(env, scope)
1470        if self.attributes is not None:
1471            if self.in_pxd and not env.in_cinclude:
1472                self.entry.defined_in_pxd = 1
1473            for attr in self.attributes:
1474                attr.analyse_declarations(env, scope)
1475            if self.visibility != 'extern':
1476                for attr in scope.var_entries:
1477                    type = attr.type
1478                    while type.is_array:
1479                        type = type.base_type
1480                    if type == self.entry.type:
1481                        error(attr.pos, "Struct cannot contain itself as a member.")
1482
1483    def analyse_expressions(self, env):
1484        return self
1485
1486    def generate_execution_code(self, code):
1487        pass
1488
1489
1490class CppClassNode(CStructOrUnionDefNode, BlockNode):
1491
1492    #  name          string
1493    #  cname         string or None
1494    #  visibility    "extern"
1495    #  in_pxd        boolean
1496    #  attributes    [CVarDefNode] or None
1497    #  entry         Entry
1498    #  base_classes  [CBaseTypeNode]
1499    #  templates     [(string, bool)] or None
1500    #  decorators    [DecoratorNode] or None
1501
1502    decorators = None
1503
1504    def declare(self, env):
1505        if self.templates is None:
1506            template_types = None
1507        else:
1508            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
1509                              for template_name, required in self.templates]
1510            num_optional_templates = sum(not required for _, required in self.templates)
1511            if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
1512                error(self.pos, "Required template parameters must precede optional template parameters.")
1513        self.entry = env.declare_cpp_class(
1514            self.name, None, self.pos, self.cname,
1515            base_classes=[], visibility=self.visibility, templates=template_types)
1516
1517    def analyse_declarations(self, env):
1518        if self.templates is None:
1519            template_types = template_names = None
1520        else:
1521            template_names = [template_name for template_name, _ in self.templates]
1522            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
1523                              for template_name, required in self.templates]
1524        scope = None
1525        if self.attributes is not None:
1526            scope = CppClassScope(self.name, env, templates=template_names)
1527        def base_ok(base_class):
1528            if base_class.is_cpp_class or base_class.is_struct:
1529                return True
1530            else:
1531                error(self.pos, "Base class '%s' not a struct or class." % base_class)
1532        base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
1533        self.entry = env.declare_cpp_class(
1534            self.name, scope, self.pos,
1535            self.cname, base_class_types, visibility=self.visibility, templates=template_types)
1536        if self.entry is None:
1537            return
1538        self.entry.is_cpp_class = 1
1539        if scope is not None:
1540            scope.type = self.entry.type
1541        defined_funcs = []
1542        def func_attributes(attributes):
1543            for attr in attributes:
1544                if isinstance(attr, CFuncDefNode):
1545                    yield attr
1546                elif isinstance(attr, CompilerDirectivesNode):
1547                    for sub_attr in func_attributes(attr.body.stats):
1548                        yield sub_attr
1549        if self.attributes is not None:
1550            if self.in_pxd and not env.in_cinclude:
1551                self.entry.defined_in_pxd = 1
1552            for attr in self.attributes:
1553                declare = getattr(attr, 'declare', None)
1554                if declare:
1555                    attr.declare(scope)
1556                attr.analyse_declarations(scope)
1557            for func in func_attributes(self.attributes):
1558                defined_funcs.append(func)
1559                if self.templates is not None:
1560                    func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
1561        self.body = StatListNode(self.pos, stats=defined_funcs)
1562        self.scope = scope
1563
1564    def analyse_expressions(self, env):
1565        self.body = self.body.analyse_expressions(self.entry.type.scope)
1566        return self
1567
1568    def generate_function_definitions(self, env, code):
1569        self.body.generate_function_definitions(self.entry.type.scope, code)
1570
1571    def generate_execution_code(self, code):
1572        self.body.generate_execution_code(code)
1573
1574    def annotate(self, code):
1575        self.body.annotate(code)
1576
1577
1578class CEnumDefNode(StatNode):
1579    #  name               string or None
1580    #  cname              string or None
1581    #  scoped             boolean                Is a C++ scoped enum
1582    #  underlying_type    CSimpleBaseTypeNode    The underlying value type (int or C++ type)
1583    #  items              [CEnumDefItemNode]
1584    #  typedef_flag       boolean
1585    #  visibility         "public" or "private" or "extern"
1586    #  api                boolean
1587    #  in_pxd             boolean
1588    #  create_wrapper     boolean
1589    #  entry              Entry
1590    #  doc                EncodedString or None    Doc string
1591
1592    child_attrs = ["items", "underlying_type"]
1593    doc = None
1594
1595    def declare(self, env):
1596        doc = None
1597        if Options.docstrings:
1598            doc = embed_position(self.pos, self.doc)
1599
1600        self.entry = env.declare_enum(
1601            self.name, self.pos,
1602            cname=self.cname,
1603            scoped=self.scoped,
1604            typedef_flag=self.typedef_flag,
1605            visibility=self.visibility, api=self.api,
1606            create_wrapper=self.create_wrapper, doc=doc)
1607
1608    def analyse_declarations(self, env):
1609        scope = None
1610        underlying_type = self.underlying_type.analyse(env)
1611
1612        if not underlying_type.is_int:
1613            error(self.underlying_type.pos, "underlying type is not an integral type")
1614
1615        self.entry.type.underlying_type = underlying_type
1616
1617        if self.scoped and self.items is not None:
1618            scope = CppScopedEnumScope(self.name, env)
1619            scope.type = self.entry.type
1620        else:
1621            scope = env
1622
1623        if self.items is not None:
1624            if self.in_pxd and not env.in_cinclude:
1625                self.entry.defined_in_pxd = 1
1626            for item in self.items:
1627                item.analyse_declarations(scope, self.entry)
1628
1629    def analyse_expressions(self, env):
1630        return self
1631
1632    def generate_execution_code(self, code):
1633        if self.scoped:
1634            return  # nothing to do here for C++ enums
1635        if self.visibility == 'public' or self.api:
1636            code.mark_pos(self.pos)
1637            temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
1638            for item in self.entry.enum_values:
1639                code.putln("%s = PyInt_FromLong(%s); %s" % (
1640                    temp,
1641                    item.cname,
1642                    code.error_goto_if_null(temp, item.pos)))
1643                code.put_gotref(temp, PyrexTypes.py_object_type)
1644                code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
1645                    Naming.moddict_cname,
1646                    item.name,
1647                    temp,
1648                    code.error_goto(item.pos)))
1649                code.put_decref_clear(temp, PyrexTypes.py_object_type)
1650            code.funcstate.release_temp(temp)
1651
1652
1653class CEnumDefItemNode(StatNode):
1654    #  name     string
1655    #  cname    string or None
1656    #  value    ExprNode or None
1657
1658    child_attrs = ["value"]
1659
1660    def analyse_declarations(self, env, enum_entry):
1661        if self.value:
1662            self.value = self.value.analyse_const_expression(env)
1663            if not self.value.type.is_int:
1664                self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
1665                self.value = self.value.analyse_const_expression(env)
1666
1667        if enum_entry.type.is_cpp_enum:
1668            cname = "%s::%s" % (enum_entry.cname, self.name)
1669        else:
1670            cname = self.cname
1671
1672        entry = env.declare_const(
1673            self.name, enum_entry.type,
1674            self.value, self.pos, cname=cname,
1675            visibility=enum_entry.visibility, api=enum_entry.api,
1676            create_wrapper=enum_entry.create_wrapper and enum_entry.name is None)
1677        enum_entry.enum_values.append(entry)
1678        if enum_entry.name:
1679            enum_entry.type.values.append(entry.name)
1680
1681
1682class CTypeDefNode(StatNode):
1683    #  base_type    CBaseTypeNode
1684    #  declarator   CDeclaratorNode
1685    #  visibility   "public" or "private"
1686    #  api          boolean
1687    #  in_pxd       boolean
1688
1689    child_attrs = ["base_type", "declarator"]
1690
1691    def analyse_declarations(self, env):
1692        base = self.base_type.analyse(env)
1693        name_declarator, type = self.declarator.analyse(
1694            base, env, visibility=self.visibility, in_pxd=self.in_pxd)
1695        name = name_declarator.name
1696        cname = name_declarator.cname
1697
1698        entry = env.declare_typedef(
1699            name, type, self.pos,
1700            cname=cname, visibility=self.visibility, api=self.api)
1701
1702        if type.is_fused:
1703            entry.in_cinclude = True
1704
1705        if self.in_pxd and not env.in_cinclude:
1706            entry.defined_in_pxd = 1
1707
1708    def analyse_expressions(self, env):
1709        return self
1710
1711    def generate_execution_code(self, code):
1712        pass
1713
1714
1715class FuncDefNode(StatNode, BlockNode):
1716    #  Base class for function definition nodes.
1717    #
1718    #  return_type     PyrexType
1719    #  #filename        string        C name of filename string const
1720    #  entry           Symtab.Entry
1721    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
1722    #  needs_outer_scope boolean      Whether or not this function requires outer scope
1723    #  pymethdef_required boolean     Force Python method struct generation
1724    #  directive_locals { string : ExprNode } locals defined by cython.locals(...)
1725    #  directive_returns [ExprNode] type defined by cython.returns(...)
1726    #  star_arg      PyArgDeclNode or None  * argument
1727    #  starstar_arg  PyArgDeclNode or None  ** argument
1728    #
1729    #  is_async_def  boolean          is a Coroutine function
1730    #
1731    #  has_fused_arguments  boolean
1732    #       Whether this cdef function has fused parameters. This is needed
1733    #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
1734    #       with fused argument types with a FusedCFuncDefNode
1735
1736    py_func = None
1737    needs_closure = False
1738    needs_outer_scope = False
1739    pymethdef_required = False
1740    is_generator = False
1741    is_coroutine = False
1742    is_asyncgen = False
1743    is_generator_body = False
1744    is_async_def = False
1745    modifiers = []
1746    has_fused_arguments = False
1747    star_arg = None
1748    starstar_arg = None
1749    is_cyfunction = False
1750    code_object = None
1751    return_type_annotation = None
1752
1753    outer_attrs = None  # overridden by some derived classes - to be visited outside the node's scope
1754
1755    def analyse_default_values(self, env):
1756        default_seen = 0
1757        for arg in self.args:
1758            if arg.default:
1759                default_seen = 1
1760                if arg.is_generic:
1761                    arg.default = arg.default.analyse_types(env)
1762                    arg.default = arg.default.coerce_to(arg.type, env)
1763                else:
1764                    error(arg.pos, "This argument cannot have a default value")
1765                    arg.default = None
1766            elif arg.kw_only:
1767                default_seen = 1
1768            elif default_seen:
1769                error(arg.pos, "Non-default argument following default argument")
1770
1771    def analyse_annotations(self, env):
1772        for arg in self.args:
1773            if arg.annotation:
1774                arg.annotation = arg.annotation.analyse_types(env)
1775        if self.return_type_annotation:
1776            self.return_type_annotation = self.return_type_annotation.analyse_types(env)
1777
1778    def align_argument_type(self, env, arg):
1779        # @cython.locals()
1780        directive_locals = self.directive_locals
1781        orig_type = arg.type
1782        if arg.name in directive_locals:
1783            type_node = directive_locals[arg.name]
1784            other_type = type_node.analyse_as_type(env)
1785        elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
1786            type_node = arg.annotation
1787            other_type = arg.inject_type_from_annotations(env)
1788            if other_type is None:
1789                return arg
1790        else:
1791            return arg
1792        if other_type is None:
1793            error(type_node.pos, "Not a type")
1794        elif orig_type is not py_object_type and not orig_type.same_as(other_type):
1795            error(arg.base_type.pos, "Signature does not agree with previous declaration")
1796            error(type_node.pos, "Previous declaration here")
1797        else:
1798            arg.type = other_type
1799            if arg.type.is_complex:
1800                # utility code for complex types is special-cased and also important to ensure that it's run
1801                arg.type.create_declaration_utility_code(env)
1802        return arg
1803
1804    def need_gil_acquisition(self, lenv):
1805        return 0
1806
1807    def create_local_scope(self, env):
1808        genv = env
1809        while genv.is_py_class_scope or genv.is_c_class_scope:
1810            genv = genv.outer_scope
1811        if self.needs_closure:
1812            lenv = ClosureScope(name=self.entry.name,
1813                                outer_scope=genv,
1814                                parent_scope=env,
1815                                scope_name=self.entry.cname)
1816        else:
1817            lenv = LocalScope(name=self.entry.name,
1818                              outer_scope=genv,
1819                              parent_scope=env)
1820        lenv.return_type = self.return_type
1821        type = self.entry.type
1822        if type.is_cfunction:
1823            lenv.nogil = type.nogil and not type.with_gil
1824        self.local_scope = lenv
1825        lenv.directives = env.directives
1826        return lenv
1827
1828    def generate_function_body(self, env, code):
1829        self.body.generate_execution_code(code)
1830
1831    def generate_function_definitions(self, env, code):
1832        from . import Buffer
1833
1834        lenv = self.local_scope
1835        if lenv.is_closure_scope and not lenv.is_passthrough:
1836            outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
1837                                            Naming.outer_scope_cname)
1838        else:
1839            outer_scope_cname = Naming.outer_scope_cname
1840        lenv.mangle_closure_cnames(outer_scope_cname)
1841        # Generate closure function definitions
1842        self.body.generate_function_definitions(lenv, code)
1843        # generate lambda function definitions
1844        self.generate_lambda_definitions(lenv, code)
1845
1846        is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
1847                             self.entry.scope.is_c_class_scope)
1848        is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
1849                                 self.entry.scope.is_c_class_scope)
1850        is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
1851        if is_buffer_slot:
1852            if 'cython_unused' not in self.modifiers:
1853                self.modifiers = self.modifiers + ['cython_unused']
1854
1855        preprocessor_guard = self.get_preprocessor_guard()
1856
1857        profile = code.globalstate.directives['profile']
1858        linetrace = code.globalstate.directives['linetrace']
1859        if profile or linetrace:
1860            if linetrace:
1861                code.use_fast_gil_utility_code()
1862            code.globalstate.use_utility_code(
1863                UtilityCode.load_cached("Profile", "Profile.c"))
1864
1865        # Generate C code for header and body of function
1866        code.enter_cfunc_scope(lenv)
1867        code.return_from_error_cleanup_label = code.new_label()
1868        code.funcstate.gil_owned = not lenv.nogil
1869
1870        # ----- Top-level constants used by this function
1871        code.mark_pos(self.pos)
1872        self.generate_cached_builtins_decls(lenv, code)
1873        # ----- Function header
1874        code.putln("")
1875
1876        if preprocessor_guard:
1877            code.putln(preprocessor_guard)
1878
1879        with_pymethdef = (self.needs_assignment_synthesis(env, code) or
1880                          self.pymethdef_required)
1881        if self.py_func:
1882            self.py_func.generate_function_header(
1883                code, with_pymethdef=with_pymethdef, proto_only=True)
1884        self.generate_function_header(code, with_pymethdef=with_pymethdef)
1885        # ----- Local variable declarations
1886        # Find function scope
1887        cenv = env
1888        while cenv.is_py_class_scope or cenv.is_c_class_scope:
1889            cenv = cenv.outer_scope
1890        if self.needs_closure:
1891            code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
1892            code.putln(";")
1893        elif self.needs_outer_scope:
1894            if lenv.is_passthrough:
1895                code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
1896                code.putln(";")
1897            code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
1898            code.putln(";")
1899        self.generate_argument_declarations(lenv, code)
1900
1901        for entry in lenv.var_entries:
1902            if not (entry.in_closure or entry.is_arg):
1903                code.put_var_declaration(entry)
1904
1905        # Initialize the return variable __pyx_r
1906        init = ""
1907        return_type = self.return_type
1908        if not return_type.is_void:
1909            if return_type.is_pyobject:
1910                init = " = NULL"
1911            elif return_type.is_memoryviewslice:
1912                init = ' = ' + return_type.literal_code(return_type.default_value)
1913
1914            code.putln("%s%s;" % (
1915                return_type.declaration_code(Naming.retval_cname),
1916                init))
1917
1918        tempvardecl_code = code.insertion_point()
1919        self.generate_keyword_list(code)
1920
1921        # ----- GIL acquisition
1922        acquire_gil = self.acquire_gil
1923
1924        # See if we need to acquire the GIL for variable declarations, or for
1925        # refnanny only
1926
1927        # Closures are not currently possible for cdef nogil functions,
1928        # but check them anyway
1929        have_object_args = self.needs_closure or self.needs_outer_scope
1930        for arg in lenv.arg_entries:
1931            if arg.type.is_pyobject:
1932                have_object_args = True
1933                break
1934
1935        used_buffer_entries = [entry for entry in lenv.buffer_entries if entry.used]
1936
1937        acquire_gil_for_var_decls_only = (
1938            lenv.nogil and lenv.has_with_gil_block and
1939            (have_object_args or used_buffer_entries))
1940
1941        acquire_gil_for_refnanny_only = (
1942            lenv.nogil and lenv.has_with_gil_block and not
1943            acquire_gil_for_var_decls_only)
1944
1945        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
1946
1947        gilstate_decl = None
1948        if acquire_gil or acquire_gil_for_var_decls_only:
1949            code.put_ensure_gil()
1950            code.funcstate.gil_owned = True
1951        else:
1952            gilstate_decl = code.insertion_point()
1953
1954        if profile or linetrace:
1955            if not self.is_generator:
1956                # generators are traced when iterated, not at creation
1957                tempvardecl_code.put_trace_declarations()
1958                code_object = self.code_object.calculate_result_code(code) if self.code_object else None
1959                code.put_trace_frame_init(code_object)
1960
1961        # ----- Special check for getbuffer
1962        if is_getbuffer_slot:
1963            self.getbuffer_check(code)
1964
1965        # ----- set up refnanny
1966        if use_refnanny:
1967            tempvardecl_code.put_declare_refcount_context()
1968            code.put_setup_refcount_context(
1969                self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
1970
1971        # ----- Automatic lead-ins for certain special functions
1972        if is_getbuffer_slot:
1973            self.getbuffer_init(code)
1974        # ----- Create closure scope object
1975        if self.needs_closure:
1976            tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1977            slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
1978            if not slot_func_cname:
1979                slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
1980            code.putln("%s = (%s)%s(%s, %s, NULL);" % (
1981                Naming.cur_scope_cname,
1982                lenv.scope_class.type.empty_declaration_code(),
1983                slot_func_cname,
1984                lenv.scope_class.type.typeptr_cname,
1985                Naming.empty_tuple))
1986            code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
1987            # Scope unconditionally DECREFed on return.
1988            code.putln("%s = %s;" % (
1989                Naming.cur_scope_cname,
1990                lenv.scope_class.type.cast_code("Py_None")))
1991            code.put_incref("Py_None", py_object_type)
1992            code.putln(code.error_goto(self.pos))
1993            code.putln("} else {")
1994            code.put_gotref(Naming.cur_scope_cname, lenv.scope_class.type)
1995            code.putln("}")
1996            # Note that it is unsafe to decref the scope at this point.
1997        if self.needs_outer_scope:
1998            if self.is_cyfunction:
1999                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
2000                    outer_scope_cname,
2001                    cenv.scope_class.type.empty_declaration_code(),
2002                    Naming.self_cname))
2003            else:
2004                code.putln("%s = (%s) %s;" % (
2005                    outer_scope_cname,
2006                    cenv.scope_class.type.empty_declaration_code(),
2007                    Naming.self_cname))
2008            if lenv.is_passthrough:
2009                code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
2010            elif self.needs_closure:
2011                # inner closures own a reference to their outer parent
2012                code.put_incref(outer_scope_cname, cenv.scope_class.type)
2013                code.put_giveref(outer_scope_cname, cenv.scope_class.type)
2014        # ----- Trace function call
2015        if profile or linetrace:
2016            # this looks a bit late, but if we don't get here due to a
2017            # fatal error before hand, it's not really worth tracing
2018            if not self.is_generator:
2019                # generators are traced when iterated, not at creation
2020                if self.is_wrapper:
2021                    trace_name = self.entry.name + " (wrapper)"
2022                else:
2023                    trace_name = self.entry.name
2024                code.put_trace_call(
2025                    trace_name, self.pos, nogil=not code.funcstate.gil_owned)
2026            code.funcstate.can_trace = True
2027        # ----- Fetch arguments
2028        self.generate_argument_parsing_code(env, code)
2029        # If an argument is assigned to in the body, we must
2030        # incref it to properly keep track of refcounts.
2031        is_cdef = isinstance(self, CFuncDefNode)
2032        for entry in lenv.arg_entries:
2033            if not entry.type.is_memoryviewslice:
2034                if (acquire_gil or entry.cf_is_reassigned) and not entry.in_closure:
2035                    code.put_var_incref(entry)
2036            # Note: defaults are always incref-ed. For def functions, we
2037            #       we acquire arguments from object conversion, so we have
2038            #       new references. If we are a cdef function, we need to
2039            #       incref our arguments
2040            elif is_cdef and entry.cf_is_reassigned:
2041                code.put_var_incref_memoryviewslice(entry,
2042                                    have_gil=code.funcstate.gil_owned)
2043        for entry in lenv.var_entries:
2044            if entry.is_arg and entry.cf_is_reassigned and not entry.in_closure:
2045                if entry.xdecref_cleanup:
2046                    code.put_var_xincref(entry)
2047                else:
2048                    code.put_var_incref(entry)
2049
2050        # ----- Initialise local buffer auxiliary variables
2051        for entry in lenv.var_entries + lenv.arg_entries:
2052            if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
2053                Buffer.put_init_vars(entry, code)
2054
2055        # ----- Check and convert arguments
2056        self.generate_argument_type_tests(code)
2057        # ----- Acquire buffer arguments
2058        for entry in lenv.arg_entries:
2059            if entry.type.is_buffer:
2060                Buffer.put_acquire_arg_buffer(entry, code, self.pos)
2061
2062        if acquire_gil_for_var_decls_only:
2063            code.put_release_ensured_gil()
2064            code.funcstate.gil_owned = False
2065
2066        # -------------------------
2067        # ----- Function body -----
2068        # -------------------------
2069        self.generate_function_body(env, code)
2070
2071        code.mark_pos(self.pos, trace=False)
2072        code.putln("")
2073        code.putln("/* function exit code */")
2074
2075        gil_owned = {
2076            'success': code.funcstate.gil_owned,
2077            'error': code.funcstate.gil_owned,
2078            'gil_state_declared': gilstate_decl is None,
2079        }
2080        def assure_gil(code_path, code=code):
2081            if not gil_owned[code_path]:
2082                if not gil_owned['gil_state_declared']:
2083                    gilstate_decl.declare_gilstate()
2084                    gil_owned['gil_state_declared'] = True
2085                code.put_ensure_gil(declare_gilstate=False)
2086                gil_owned[code_path] = True
2087
2088        # ----- Default return value
2089        return_type = self.return_type
2090        if not self.body.is_terminator:
2091            if return_type.is_pyobject:
2092                #if return_type.is_extension_type:
2093                #    lhs = "(PyObject *)%s" % Naming.retval_cname
2094                #else:
2095                lhs = Naming.retval_cname
2096                assure_gil('success')
2097                code.put_init_to_py_none(lhs, return_type)
2098            elif not return_type.is_memoryviewslice:
2099                # memory view structs receive their default value on initialisation
2100                val = return_type.default_value
2101                if val:
2102                    code.putln("%s = %s;" % (Naming.retval_cname, val))
2103                elif not return_type.is_void:
2104                    code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
2105
2106        # ----- Error cleanup
2107        if code.label_used(code.error_label):
2108            if not self.body.is_terminator:
2109                code.put_goto(code.return_label)
2110            code.put_label(code.error_label)
2111            for cname, type in code.funcstate.all_managed_temps():
2112                assure_gil('error')
2113                code.put_xdecref(cname, type, have_gil=gil_owned['error'])
2114
2115            # Clean up buffers -- this calls a Python function
2116            # so need to save and restore error state
2117            buffers_present = len(used_buffer_entries) > 0
2118            #memslice_entries = [e for e in lenv.entries.values() if e.type.is_memoryviewslice]
2119            if buffers_present:
2120                code.globalstate.use_utility_code(restore_exception_utility_code)
2121                code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
2122                code.putln("__Pyx_PyThreadState_declare")
2123                assure_gil('error')
2124                code.putln("__Pyx_PyThreadState_assign")
2125                code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
2126                for entry in used_buffer_entries:
2127                    Buffer.put_release_buffer_code(code, entry)
2128                    #code.putln("%s = 0;" % entry.cname)
2129                code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
2130
2131            if return_type.is_memoryviewslice:
2132                from . import MemoryView
2133                MemoryView.put_init_entry(Naming.retval_cname, code)
2134                err_val = Naming.retval_cname
2135            else:
2136                err_val = self.error_value()
2137
2138            exc_check = self.caller_will_check_exceptions()
2139            if err_val is not None or exc_check:
2140                # TODO: Fix exception tracing (though currently unused by cProfile).
2141                # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
2142                # code.put_trace_exception()
2143
2144                assure_gil('error')
2145                code.put_add_traceback(self.entry.qualified_name)
2146            else:
2147                warning(self.entry.pos,
2148                        "Unraisable exception in function '%s'." %
2149                        self.entry.qualified_name, 0)
2150                assure_gil('error')
2151                code.put_unraisable(self.entry.qualified_name)
2152            default_retval = return_type.default_value
2153            if err_val is None and default_retval:
2154                err_val = default_retval
2155            if err_val is not None:
2156                if err_val != Naming.retval_cname:
2157                    code.putln("%s = %s;" % (Naming.retval_cname, err_val))
2158            elif not return_type.is_void:
2159                code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
2160
2161            if is_getbuffer_slot:
2162                assure_gil('error')
2163                self.getbuffer_error_cleanup(code)
2164
2165            def align_error_path_gil_to_success_path(code=code.insertion_point()):
2166                # align error and success GIL state when both join
2167                if gil_owned['success']:
2168                    assure_gil('error', code=code)
2169                elif gil_owned['error']:
2170                    code.put_release_ensured_gil()
2171                    gil_owned['error'] = False
2172                assert gil_owned['error'] == gil_owned['success'], "%s: error path %s != success path %s" % (
2173                    self.pos, gil_owned['error'], gil_owned['success'])
2174
2175            # If we are using the non-error cleanup section we should
2176            # jump past it if we have an error. The if-test below determine
2177            # whether this section is used.
2178            if buffers_present or is_getbuffer_slot or return_type.is_memoryviewslice:
2179                # In the buffer cases, we already called assure_gil('error') and own the GIL.
2180                assert gil_owned['error'] or return_type.is_memoryviewslice
2181                code.put_goto(code.return_from_error_cleanup_label)
2182            else:
2183                # Adapt the GIL state to the success path right now.
2184                align_error_path_gil_to_success_path()
2185        else:
2186            # No error path, no need to adapt the GIL state.
2187            def align_error_path_gil_to_success_path(): pass
2188
2189        # ----- Non-error return cleanup
2190        if code.label_used(code.return_label) or not code.label_used(code.error_label):
2191            code.put_label(code.return_label)
2192
2193            for entry in used_buffer_entries:
2194                assure_gil('success')
2195                Buffer.put_release_buffer_code(code, entry)
2196            if is_getbuffer_slot:
2197                assure_gil('success')
2198                self.getbuffer_normal_cleanup(code)
2199
2200            if return_type.is_memoryviewslice:
2201                # See if our return value is uninitialized on non-error return
2202                # from . import MemoryView
2203                # MemoryView.err_if_nogil_initialized_check(self.pos, env)
2204                cond = code.unlikely(return_type.error_condition(Naming.retval_cname))
2205                code.putln(
2206                    'if (%s) {' % cond)
2207                if not gil_owned['success']:
2208                    code.put_ensure_gil()
2209                code.putln(
2210                    'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
2211                if not gil_owned['success']:
2212                    code.put_release_ensured_gil()
2213                code.putln(
2214                    '}')
2215
2216        # ----- Return cleanup for both error and no-error return
2217        if code.label_used(code.return_from_error_cleanup_label):
2218            align_error_path_gil_to_success_path()
2219            code.put_label(code.return_from_error_cleanup_label)
2220
2221        for entry in lenv.var_entries:
2222            if not entry.used or entry.in_closure:
2223                continue
2224
2225            if entry.type.is_pyobject:
2226                if entry.is_arg and not entry.cf_is_reassigned:
2227                    continue
2228            if entry.type.needs_refcounting:
2229                assure_gil('success')
2230            # FIXME ideally use entry.xdecref_cleanup but this currently isn't reliable
2231            code.put_var_xdecref(entry, have_gil=gil_owned['success'])
2232
2233        # Decref any increfed args
2234        for entry in lenv.arg_entries:
2235            if entry.type.is_memoryviewslice:
2236                # decref slices of def functions and acquired slices from cdef
2237                # functions, but not borrowed slices from cdef functions.
2238                if is_cdef and not entry.cf_is_reassigned:
2239                    continue
2240            else:
2241                if entry.in_closure:
2242                    continue
2243                if not acquire_gil and not entry.cf_is_reassigned:
2244                    continue
2245                if entry.type.needs_refcounting:
2246                    assure_gil('success')
2247
2248            # FIXME use entry.xdecref_cleanup - del arg seems to be the problem
2249            code.put_var_xdecref(entry, have_gil=gil_owned['success'])
2250        if self.needs_closure:
2251            assure_gil('success')
2252            code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
2253
2254        # ----- Return
2255        # This code is duplicated in ModuleNode.generate_module_init_func
2256        if not lenv.nogil:
2257            default_retval = return_type.default_value
2258            err_val = self.error_value()
2259            if err_val is None and default_retval:
2260                err_val = default_retval  # FIXME: why is err_val not used?
2261            code.put_xgiveref(Naming.retval_cname, return_type)
2262
2263        if self.entry.is_special and self.entry.name == "__hash__":
2264            # Returning -1 for __hash__ is supposed to signal an error
2265            # We do as Python instances and coerce -1 into -2.
2266            assure_gil('success')  # in special methods, the GIL is owned anyway
2267            code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
2268                Naming.retval_cname, Naming.retval_cname))
2269
2270        if profile or linetrace:
2271            code.funcstate.can_trace = False
2272            if not self.is_generator:
2273                # generators are traced when iterated, not at creation
2274                if return_type.is_pyobject:
2275                    code.put_trace_return(
2276                        Naming.retval_cname, nogil=not gil_owned['success'])
2277                else:
2278                    code.put_trace_return(
2279                        "Py_None", nogil=not gil_owned['success'])
2280
2281        if use_refnanny:
2282            code.put_finish_refcount_context(nogil=not gil_owned['success'])
2283
2284        if acquire_gil or (lenv.nogil and gil_owned['success']):
2285            # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
2286            code.put_release_ensured_gil()
2287            code.funcstate.gil_owned = False
2288
2289        if not return_type.is_void:
2290            code.putln("return %s;" % Naming.retval_cname)
2291
2292        code.putln("}")
2293
2294        if preprocessor_guard:
2295            code.putln("#endif /*!(%s)*/" % preprocessor_guard)
2296
2297        # ----- Go back and insert temp variable declarations
2298        tempvardecl_code.put_temp_declarations(code.funcstate)
2299
2300        # ----- Python version
2301        code.exit_cfunc_scope()
2302        if self.py_func:
2303            self.py_func.generate_function_definitions(env, code)
2304        self.generate_wrapper_functions(code)
2305
2306    def declare_argument(self, env, arg):
2307        if arg.type.is_void:
2308            error(arg.pos, "Invalid use of 'void'")
2309        elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
2310            error(arg.pos, "Argument type '%s' is incomplete" % arg.type)
2311        entry = env.declare_arg(arg.name, arg.type, arg.pos)
2312        if arg.annotation:
2313            entry.annotation = arg.annotation
2314        return entry
2315
2316    def generate_arg_type_test(self, arg, code):
2317        # Generate type test for one argument.
2318        if arg.type.typeobj_is_available():
2319            code.globalstate.use_utility_code(
2320                UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
2321            typeptr_cname = arg.type.typeptr_cname
2322            arg_code = "((PyObject *)%s)" % arg.entry.cname
2323            code.putln(
2324                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, %s, %s))) %s' % (
2325                    arg_code,
2326                    typeptr_cname,
2327                    arg.accept_none,
2328                    arg.name_cstring,
2329                    arg.type.is_builtin_type and arg.type.require_exact,
2330                    code.error_goto(arg.pos)))
2331        else:
2332            error(arg.pos, "Cannot test type of extern C class without type object name specification")
2333
2334    def generate_arg_none_check(self, arg, code):
2335        # Generate None check for one argument.
2336        if arg.type.is_memoryviewslice:
2337            cname = "%s.memview" % arg.entry.cname
2338        else:
2339            cname = arg.entry.cname
2340
2341        code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
2342        code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", %s); %s''' % (
2343            max(200, len(arg.name_cstring)), arg.name_cstring,
2344            code.error_goto(arg.pos)))
2345        code.putln('}')
2346
2347    def generate_wrapper_functions(self, code):
2348        pass
2349
2350    def generate_execution_code(self, code):
2351        code.mark_pos(self.pos)
2352        # Evaluate and store argument default values
2353        # skip this for wrappers since it's done by wrapped function
2354        if not self.is_wrapper:
2355            for arg in self.args:
2356                if not arg.is_dynamic:
2357                    arg.generate_assignment_code(code)
2358
2359    #
2360    # Special code for the __getbuffer__ function
2361    #
2362    def _get_py_buffer_info(self):
2363        py_buffer = self.local_scope.arg_entries[1]
2364        try:
2365            # Check builtin definition of struct Py_buffer
2366            obj_type = py_buffer.type.base_type.scope.entries['obj'].type
2367        except (AttributeError, KeyError):
2368            # User code redeclared struct Py_buffer
2369            obj_type = None
2370        return py_buffer, obj_type
2371
2372    # Old Python 3 used to support write-locks on buffer-like objects by
2373    # calling PyObject_GetBuffer() with a view==NULL parameter. This obscure
2374    # feature is obsolete, it was almost never used (only one instance in
2375    # `Modules/posixmodule.c` in Python 3.1) and it is now officially removed
2376    # (see bpo-14203). We add an extra check here to prevent legacy code from
2377    # from trying to use the feature and prevent segmentation faults.
2378    def getbuffer_check(self, code):
2379        py_buffer, _ = self._get_py_buffer_info()
2380        view = py_buffer.cname
2381        code.putln("if (unlikely(%s == NULL)) {" % view)
2382        code.putln("PyErr_SetString(PyExc_BufferError, "
2383                   "\"PyObject_GetBuffer: view==NULL argument is obsolete\");")
2384        code.putln("return -1;")
2385        code.putln("}")
2386
2387    def getbuffer_init(self, code):
2388        py_buffer, obj_type = self._get_py_buffer_info()
2389        view = py_buffer.cname
2390        if obj_type and obj_type.is_pyobject:
2391            code.put_init_to_py_none("%s->obj" % view, obj_type)
2392            code.put_giveref("%s->obj" % view, obj_type)  # Do not refnanny object within structs
2393        else:
2394            code.putln("%s->obj = NULL;" % view)
2395
2396    def getbuffer_error_cleanup(self, code):
2397        py_buffer, obj_type = self._get_py_buffer_info()
2398        view = py_buffer.cname
2399        if obj_type and obj_type.is_pyobject:
2400            code.putln("if (%s->obj != NULL) {" % view)
2401            code.put_gotref("%s->obj" % view, obj_type)
2402            code.put_decref_clear("%s->obj" % view, obj_type)
2403            code.putln("}")
2404        else:
2405            code.putln("Py_CLEAR(%s->obj);" % view)
2406
2407    def getbuffer_normal_cleanup(self, code):
2408        py_buffer, obj_type = self._get_py_buffer_info()
2409        view = py_buffer.cname
2410        if obj_type and obj_type.is_pyobject:
2411            code.putln("if (%s->obj == Py_None) {" % view)
2412            code.put_gotref("%s->obj" % view, obj_type)
2413            code.put_decref_clear("%s->obj" % view, obj_type)
2414            code.putln("}")
2415
2416    def get_preprocessor_guard(self):
2417        if not self.entry.is_special:
2418            return None
2419        name = self.entry.name
2420        slot = TypeSlots.method_name_to_slot.get(name)
2421        if not slot:
2422            return None
2423        if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
2424            return None
2425        if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
2426            return None
2427        return slot.preprocessor_guard_code()
2428
2429
2430class CFuncDefNode(FuncDefNode):
2431    #  C function definition.
2432    #
2433    #  modifiers     ['inline']
2434    #  visibility    'private' or 'public' or 'extern'
2435    #  base_type     CBaseTypeNode
2436    #  declarator    CDeclaratorNode
2437    #  cfunc_declarator  the CFuncDeclarator of this function
2438    #                    (this is also available through declarator or a
2439    #                     base thereof)
2440    #  body          StatListNode
2441    #  api           boolean
2442    #  decorators    [DecoratorNode]        list of decorators
2443    #
2444    #  with_gil      boolean    Acquire GIL around body
2445    #  type          CFuncType
2446    #  py_func       wrapper for calling from Python
2447    #  overridable   whether or not this is a cpdef function
2448    #  inline_in_pxd whether this is an inline function in a pxd file
2449    #  template_declaration  String or None   Used for c++ class methods
2450    #  is_const_method whether this is a const method
2451    #  is_static_method whether this is a static method
2452    #  is_c_class_method whether this is a cclass method
2453
2454    child_attrs = ["base_type", "declarator", "body", "decorators", "py_func_stat"]
2455    outer_attrs = ["decorators", "py_func_stat"]
2456
2457    inline_in_pxd = False
2458    decorators = None
2459    directive_locals = None
2460    directive_returns = None
2461    override = None
2462    template_declaration = None
2463    is_const_method = False
2464    py_func_stat = None
2465
2466    def unqualified_name(self):
2467        return self.entry.name
2468
2469    def declared_name(self):
2470        return self.declarator.declared_name()
2471
2472    @property
2473    def code_object(self):
2474        # share the CodeObject with the cpdef wrapper (if available)
2475        return self.py_func.code_object if self.py_func else None
2476
2477    def analyse_declarations(self, env):
2478        self.is_c_class_method = env.is_c_class_scope
2479        if self.directive_locals is None:
2480            self.directive_locals = {}
2481        self.directive_locals.update(env.directives.get('locals', {}))
2482        if self.directive_returns is not None:
2483            base_type = self.directive_returns.analyse_as_type(env)
2484            if base_type is None:
2485                error(self.directive_returns.pos, "Not a type")
2486                base_type = PyrexTypes.error_type
2487        else:
2488            base_type = self.base_type.analyse(env)
2489        self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
2490        # The 2 here is because we need both function and argument names.
2491        if isinstance(self.declarator, CFuncDeclaratorNode):
2492            name_declarator, typ = self.declarator.analyse(
2493                base_type, env, nonempty=2 * (self.body is not None),
2494                directive_locals=self.directive_locals, visibility=self.visibility)
2495        else:
2496            name_declarator, typ = self.declarator.analyse(
2497                base_type, env, nonempty=2 * (self.body is not None), visibility=self.visibility)
2498        if not typ.is_cfunction:
2499            error(self.pos, "Suite attached to non-function declaration")
2500        # Remember the actual type according to the function header
2501        # written here, because the type in the symbol table entry
2502        # may be different if we're overriding a C method inherited
2503        # from the base type of an extension type.
2504        self.type = typ
2505        typ.is_overridable = self.overridable
2506        declarator = self.declarator
2507        while not hasattr(declarator, 'args'):
2508            declarator = declarator.base
2509
2510        self.cfunc_declarator = declarator
2511        self.args = declarator.args
2512
2513        opt_arg_count = self.cfunc_declarator.optional_arg_count
2514        if (self.visibility == 'public' or self.api) and opt_arg_count:
2515            error(self.cfunc_declarator.pos,
2516                  "Function with optional arguments may not be declared public or api")
2517
2518        if typ.exception_check == '+' and self.visibility != 'extern':
2519            if typ.exception_value and typ.exception_value.is_name:
2520                # it really is impossible to reason about what the user wants to happens
2521                # if they've specified a C++ exception translation function. Therefore,
2522                # raise an error.
2523                error(self.cfunc_declarator.pos,
2524                    "Only extern functions can throw C++ exceptions.")
2525            else:
2526                warning(self.cfunc_declarator.pos,
2527                    "Only extern functions can throw C++ exceptions.", 2)
2528
2529        for formal_arg, type_arg in zip(self.args, typ.args):
2530            self.align_argument_type(env, type_arg)
2531            formal_arg.type = type_arg.type
2532            formal_arg.name = type_arg.name
2533            formal_arg.cname = type_arg.cname
2534
2535            self._validate_type_visibility(type_arg.type, type_arg.pos, env)
2536
2537            if type_arg.type.is_fused:
2538                self.has_fused_arguments = True
2539
2540            if type_arg.type.is_buffer and 'inline' in self.modifiers:
2541                warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2542
2543            if type_arg.type.is_buffer or type_arg.type.is_pythran_expr:
2544                if self.type.nogil:
2545                    error(formal_arg.pos,
2546                          "Buffer may not be acquired without the GIL. Consider using memoryview slices instead.")
2547                elif 'inline' in self.modifiers:
2548                    warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2549
2550        self._validate_type_visibility(typ.return_type, self.pos, env)
2551
2552        name = name_declarator.name
2553        cname = name_declarator.cname
2554
2555        typ.is_const_method = self.is_const_method
2556        typ.is_static_method = self.is_static_method
2557
2558        self.entry = env.declare_cfunction(
2559            name, typ, self.pos,
2560            cname=cname, visibility=self.visibility, api=self.api,
2561            defining=self.body is not None, modifiers=self.modifiers,
2562            overridable=self.overridable)
2563        self.entry.inline_func_in_pxd = self.inline_in_pxd
2564        self.return_type = typ.return_type
2565        if self.return_type.is_array and self.visibility != 'extern':
2566            error(self.pos, "Function cannot return an array")
2567        if self.return_type.is_cpp_class:
2568            self.return_type.check_nullary_constructor(self.pos, "used as a return value")
2569
2570        if self.overridable and not env.is_module_scope and not self.is_static_method:
2571            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
2572                # An error will be produced in the cdef function
2573                self.overridable = False
2574
2575        self.declare_cpdef_wrapper(env)
2576        self.create_local_scope(env)
2577
2578    def declare_cpdef_wrapper(self, env):
2579        if not self.overridable:
2580            return
2581        if self.is_static_method:
2582            # TODO(robertwb): Finish this up, perhaps via more function refactoring.
2583            error(self.pos, "static cpdef methods not yet supported")
2584
2585        name = self.entry.name
2586        py_func_body = self.call_self_node(is_module_scope=env.is_module_scope)
2587        if self.is_static_method:
2588            from .ExprNodes import NameNode
2589            decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name=EncodedString('staticmethod')))]
2590            decorators[0].decorator.analyse_types(env)
2591        else:
2592            decorators = []
2593        self.py_func = DefNode(pos=self.pos,
2594                               name=self.entry.name,
2595                               args=self.args,
2596                               star_arg=None,
2597                               starstar_arg=None,
2598                               doc=self.doc,
2599                               body=py_func_body,
2600                               decorators=decorators,
2601                               is_wrapper=1)
2602        self.py_func.is_module_scope = env.is_module_scope
2603        self.py_func.analyse_declarations(env)
2604        self.py_func.entry.is_overridable = True
2605        self.py_func_stat = StatListNode(self.pos, stats=[self.py_func])
2606        self.py_func.type = PyrexTypes.py_object_type
2607        self.entry.as_variable = self.py_func.entry
2608        self.entry.used = self.entry.as_variable.used = True
2609        # Reset scope entry the above cfunction
2610        env.entries[name] = self.entry
2611        if (not self.entry.is_final_cmethod and
2612                (not env.is_module_scope or Options.lookup_module_cpdef)):
2613            if self.override:
2614                # This is a hack: we shouldn't create the wrapper twice, but we do for fused functions.
2615                assert self.entry.is_fused_specialized  # should not happen for non-fused cpdef functions
2616                self.override.py_func = self.py_func
2617            else:
2618                self.override = OverrideCheckNode(self.pos, py_func=self.py_func)
2619                self.body = StatListNode(self.pos, stats=[self.override, self.body])
2620
2621    def _validate_type_visibility(self, type, pos, env):
2622        """
2623        Ensure that types used in cdef functions are public or api, or
2624        defined in a C header.
2625        """
2626        public_or_api = (self.visibility == 'public' or self.api)
2627        entry = getattr(type, 'entry', None)
2628        if public_or_api and entry and env.is_module_scope:
2629            if not (entry.visibility in ('public', 'extern') or
2630                    entry.api or entry.in_cinclude):
2631                error(pos, "Function declared public or api may not have private types")
2632
2633    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
2634        from . import ExprNodes
2635        args = self.type.args
2636        if omit_optional_args:
2637            args = args[:len(args) - self.type.optional_arg_count]
2638        arg_names = [arg.name for arg in args]
2639        if is_module_scope:
2640            cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
2641            call_arg_names = arg_names
2642            skip_dispatch = Options.lookup_module_cpdef
2643        elif self.type.is_static_method:
2644            class_entry = self.entry.scope.parent_type.entry
2645            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
2646            class_node.entry = class_entry
2647            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
2648            # Calling static c(p)def methods on an instance disallowed.
2649            # TODO(robertwb): Support by passing self to check for override?
2650            skip_dispatch = True
2651        else:
2652            type_entry = self.type.args[0].type.entry
2653            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
2654            type_arg.entry = type_entry
2655            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
2656        skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
2657        c_call = ExprNodes.SimpleCallNode(
2658            self.pos,
2659            function=cfunc,
2660            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
2661            wrapper_call=skip_dispatch)
2662        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)
2663
2664    def declare_arguments(self, env):
2665        for arg in self.type.args:
2666            if not arg.name:
2667                error(arg.pos, "Missing argument name")
2668            self.declare_argument(env, arg)
2669
2670    def need_gil_acquisition(self, lenv):
2671        return self.type.with_gil
2672
2673    def nogil_check(self, env):
2674        type = self.type
2675        with_gil = type.with_gil
2676        if type.nogil and not with_gil:
2677            if type.return_type.is_pyobject:
2678                error(self.pos,
2679                      "Function with Python return type cannot be declared nogil")
2680            for entry in self.local_scope.var_entries:
2681                if entry.type.is_pyobject and not entry.in_with_gil_block:
2682                    error(self.pos, "Function declared nogil has Python locals or temporaries")
2683
2684    def analyse_expressions(self, env):
2685        self.local_scope.directives = env.directives
2686        if self.py_func_stat is not None:
2687            # this will also analyse the default values and the function name assignment
2688            self.py_func_stat = self.py_func_stat.analyse_expressions(env)
2689        elif self.py_func is not None:
2690            # this will also analyse the default values
2691            self.py_func = self.py_func.analyse_expressions(env)
2692        else:
2693            self.analyse_default_values(env)
2694            self.analyse_annotations(env)
2695        self.acquire_gil = self.need_gil_acquisition(self.local_scope)
2696        return self
2697
2698    def needs_assignment_synthesis(self, env, code=None):
2699        return False
2700
2701    def generate_function_header(self, code, with_pymethdef, with_opt_args=1, with_dispatch=1, cname=None):
2702        scope = self.local_scope
2703        arg_decls = []
2704        type = self.type
2705        for arg in type.args[:len(type.args)-type.optional_arg_count]:
2706            arg_decl = arg.declaration_code()
2707            entry = scope.lookup(arg.name)
2708            if not entry.cf_used:
2709                arg_decl = 'CYTHON_UNUSED %s' % arg_decl
2710            arg_decls.append(arg_decl)
2711        if with_dispatch and self.overridable:
2712            dispatch_arg = PyrexTypes.c_int_type.declaration_code(
2713                Naming.skip_dispatch_cname)
2714            if self.override:
2715                arg_decls.append(dispatch_arg)
2716            else:
2717                arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
2718        if type.optional_arg_count and with_opt_args:
2719            arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
2720        if type.has_varargs:
2721            arg_decls.append("...")
2722        if not arg_decls:
2723            arg_decls = ["void"]
2724        if cname is None:
2725            cname = self.entry.func_cname
2726        entity = type.function_header_code(cname, ', '.join(arg_decls))
2727        if self.entry.visibility == 'private' and '::' not in cname:
2728            storage_class = "static "
2729        else:
2730            storage_class = ""
2731        dll_linkage = None
2732        modifiers = code.build_function_modifiers(self.entry.func_modifiers)
2733
2734        header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
2735        #print (storage_class, modifiers, header)
2736        needs_proto = self.is_c_class_method or self.entry.is_cproperty
2737        if self.template_declaration:
2738            if needs_proto:
2739                code.globalstate.parts['module_declarations'].putln(self.template_declaration)
2740            code.putln(self.template_declaration)
2741        if needs_proto:
2742            code.globalstate.parts['module_declarations'].putln(
2743                "%s%s%s; /* proto*/" % (storage_class, modifiers, header))
2744        code.putln("%s%s%s {" % (storage_class, modifiers, header))
2745
2746    def generate_argument_declarations(self, env, code):
2747        scope = self.local_scope
2748        for arg in self.args:
2749            if arg.default:
2750                entry = scope.lookup(arg.name)
2751                if self.override or entry.cf_used:
2752                    result = arg.calculate_default_value_code(code)
2753                    code.putln('%s = %s;' % (
2754                        arg.type.declaration_code(arg.cname), result))
2755
2756    def generate_keyword_list(self, code):
2757        pass
2758
2759    def generate_argument_parsing_code(self, env, code):
2760        i = 0
2761        used = 0
2762        scope = self.local_scope
2763        if self.type.optional_arg_count:
2764            code.putln('if (%s) {' % Naming.optional_args_cname)
2765            for arg in self.args:
2766                if arg.default:
2767                    entry = scope.lookup(arg.name)
2768                    if self.override or entry.cf_used:
2769                        code.putln('if (%s->%sn > %s) {' %
2770                                   (Naming.optional_args_cname,
2771                                    Naming.pyrex_prefix, i))
2772                        declarator = arg.declarator
2773                        while not hasattr(declarator, 'name'):
2774                            declarator = declarator.base
2775                        code.putln('%s = %s->%s;' %
2776                                   (arg.cname, Naming.optional_args_cname,
2777                                    self.type.opt_arg_cname(declarator.name)))
2778                        used += 1
2779                    i += 1
2780            for _ in range(used):
2781                code.putln('}')
2782            code.putln('}')
2783
2784        # Move arguments into closure if required
2785        def put_into_closure(entry):
2786            if entry.in_closure and not arg.default:
2787                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
2788                code.put_var_incref(entry)
2789                code.put_var_giveref(entry)
2790        for arg in self.args:
2791            put_into_closure(scope.lookup_here(arg.name))
2792
2793
2794    def generate_argument_conversion_code(self, code):
2795        pass
2796
2797    def generate_argument_type_tests(self, code):
2798        # Generate type tests for args whose type in a parent
2799        # class is a supertype of the declared type.
2800        for arg in self.type.args:
2801            if arg.needs_type_test:
2802                self.generate_arg_type_test(arg, code)
2803            elif arg.type.is_pyobject and not arg.accept_none:
2804                self.generate_arg_none_check(arg, code)
2805
2806    def generate_execution_code(self, code):
2807        if code.globalstate.directives['linetrace']:
2808            code.mark_pos(self.pos)
2809            code.putln("")  # generate line tracing code
2810        super(CFuncDefNode, self).generate_execution_code(code)
2811        if self.py_func_stat:
2812            self.py_func_stat.generate_execution_code(code)
2813
2814    def error_value(self):
2815        if self.return_type.is_pyobject:
2816            return "0"
2817        else:
2818            return self.entry.type.exception_value
2819
2820    def caller_will_check_exceptions(self):
2821        return self.entry.type.exception_check
2822
2823    def generate_wrapper_functions(self, code):
2824        # If the C signature of a function has changed, we need to generate
2825        # wrappers to put in the slots here.
2826        k = 0
2827        entry = self.entry
2828        func_type = entry.type
2829        while entry.prev_entry is not None:
2830            k += 1
2831            entry = entry.prev_entry
2832            entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
2833            code.putln()
2834            self.generate_function_header(
2835                code, 0,
2836                with_dispatch=entry.type.is_overridable,
2837                with_opt_args=entry.type.optional_arg_count,
2838                cname=entry.func_cname)
2839            if not self.return_type.is_void:
2840                code.put('return ')
2841            args = self.type.args
2842            arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
2843            if entry.type.is_overridable:
2844                arglist.append(Naming.skip_dispatch_cname)
2845            elif func_type.is_overridable:
2846                arglist.append('0')
2847            if entry.type.optional_arg_count:
2848                arglist.append(Naming.optional_args_cname)
2849            elif func_type.optional_arg_count:
2850                arglist.append('NULL')
2851            code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
2852            code.putln('}')
2853
2854
2855class PyArgDeclNode(Node):
2856    # Argument which must be a Python object (used
2857    # for * and ** arguments).
2858    #
2859    # name        string
2860    # entry       Symtab.Entry
2861    # annotation  ExprNode or None   Py3 argument annotation
2862    child_attrs = []
2863    is_self_arg = False
2864    is_type_arg = False
2865
2866    def generate_function_definitions(self, env, code):
2867        self.entry.generate_function_definitions(env, code)
2868
2869
2870class DecoratorNode(Node):
2871    # A decorator
2872    #
2873    # decorator    NameNode or CallNode or AttributeNode
2874    child_attrs = ['decorator']
2875
2876
2877class DefNode(FuncDefNode):
2878    # A Python function definition.
2879    #
2880    # name          string                 the Python name of the function
2881    # lambda_name   string                 the internal name of a lambda 'function'
2882    # decorators    [DecoratorNode]        list of decorators
2883    # args          [CArgDeclNode]         formal arguments
2884    # doc           EncodedString or None
2885    # body          StatListNode
2886    # return_type_annotation
2887    #               ExprNode or None       the Py3 return type annotation
2888    #
2889    #  The following subnode is constructed internally
2890    #  when the def statement is inside a Python class definition.
2891    #
2892    #  fused_py_func        DefNode     The original fused cpdef DefNode
2893    #                                   (in case this is a specialization)
2894    #  specialized_cpdefs   [DefNode]   list of specialized cpdef DefNodes
2895    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
2896    #
2897    # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
2898
2899    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
2900    outer_attrs = ["decorators", "return_type_annotation"]
2901
2902    is_staticmethod = False
2903    is_classmethod = False
2904
2905    lambda_name = None
2906    reqd_kw_flags_cname = "0"
2907    is_wrapper = 0
2908    no_assignment_synthesis = 0
2909    decorators = None
2910    return_type_annotation = None
2911    entry = None
2912    acquire_gil = 0
2913    self_in_stararg = 0
2914    py_cfunc_node = None
2915    requires_classobj = False
2916    defaults_struct = None  # Dynamic kwrds structure name
2917    doc = None
2918
2919    fused_py_func = False
2920    specialized_cpdefs = None
2921    py_wrapper = None
2922    py_wrapper_required = True
2923    func_cname = None
2924
2925    defaults_getter = None
2926
2927    def __init__(self, pos, **kwds):
2928        FuncDefNode.__init__(self, pos, **kwds)
2929        p = k = rk = r = 0
2930        for arg in self.args:
2931            if arg.pos_only:
2932                p += 1
2933            if arg.kw_only:
2934                k += 1
2935                if not arg.default:
2936                    rk += 1
2937            if not arg.default:
2938                r += 1
2939        self.num_posonly_args = p
2940        self.num_kwonly_args = k
2941        self.num_required_kw_args = rk
2942        self.num_required_args = r
2943
2944    def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, except_val=None, modifiers=None,
2945                     nogil=False, with_gil=False):
2946        if self.star_arg:
2947            error(self.star_arg.pos, "cdef function cannot have star argument")
2948        if self.starstar_arg:
2949            error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
2950        exception_value, exception_check = except_val or (None, False)
2951
2952        if cfunc is None:
2953            cfunc_args = []
2954            for formal_arg in self.args:
2955                name_declarator, type = formal_arg.analyse(scope, nonempty=1)
2956                cfunc_args.append(PyrexTypes.CFuncTypeArg(name=name_declarator.name,
2957                                                          cname=None,
2958                                                          annotation=formal_arg.annotation,
2959                                                          type=py_object_type,
2960                                                          pos=formal_arg.pos))
2961            cfunc_type = PyrexTypes.CFuncType(return_type=py_object_type,
2962                                              args=cfunc_args,
2963                                              has_varargs=False,
2964                                              exception_value=None,
2965                                              exception_check=exception_check,
2966                                              nogil=nogil,
2967                                              with_gil=with_gil,
2968                                              is_overridable=overridable)
2969            cfunc = CVarDefNode(self.pos, type=cfunc_type)
2970        else:
2971            if scope is None:
2972                scope = cfunc.scope
2973            cfunc_type = cfunc.type
2974            if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
2975                error(self.pos, "wrong number of arguments")
2976                error(cfunc.pos, "previous declaration here")
2977            for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
2978                name_declarator, type = formal_arg.analyse(scope, nonempty=1,
2979                                                           is_self_arg=(i == 0 and scope.is_c_class_scope))
2980                if type is None or type is PyrexTypes.py_object_type:
2981                    formal_arg.type = type_arg.type
2982                    formal_arg.name_declarator = name_declarator
2983
2984        if exception_value is None and cfunc_type.exception_value is not None:
2985            from .ExprNodes import ConstNode
2986            exception_value = ConstNode(
2987                self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
2988        declarator = CFuncDeclaratorNode(self.pos,
2989                                         base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
2990                                         args=self.args,
2991                                         has_varargs=False,
2992                                         exception_check=cfunc_type.exception_check,
2993                                         exception_value=exception_value,
2994                                         with_gil=cfunc_type.with_gil,
2995                                         nogil=cfunc_type.nogil)
2996        return CFuncDefNode(self.pos,
2997                            modifiers=modifiers or [],
2998                            base_type=CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
2999                            declarator=declarator,
3000                            body=self.body,
3001                            doc=self.doc,
3002                            overridable=cfunc_type.is_overridable,
3003                            type=cfunc_type,
3004                            with_gil=cfunc_type.with_gil,
3005                            nogil=cfunc_type.nogil,
3006                            visibility='private',
3007                            api=False,
3008                            directive_locals=getattr(cfunc, 'directive_locals', {}),
3009                            directive_returns=returns)
3010
3011    def is_cdef_func_compatible(self):
3012        """Determines if the function's signature is compatible with a
3013        cdef function.  This can be used before calling
3014        .as_cfunction() to see if that will be successful.
3015        """
3016        if self.needs_closure:
3017            return False
3018        if self.star_arg or self.starstar_arg:
3019            return False
3020        return True
3021
3022    def analyse_declarations(self, env):
3023        if self.decorators:
3024            for decorator in self.decorators:
3025                func = decorator.decorator
3026                if func.is_name:
3027                    self.is_classmethod |= func.name == 'classmethod'
3028                    self.is_staticmethod |= func.name == 'staticmethod'
3029
3030        if self.is_classmethod and env.lookup_here('classmethod'):
3031            # classmethod() was overridden - not much we can do here ...
3032            self.is_classmethod = False
3033        if self.is_staticmethod and env.lookup_here('staticmethod'):
3034            # staticmethod() was overridden - not much we can do here ...
3035            self.is_staticmethod = False
3036
3037        if env.is_py_class_scope or env.is_c_class_scope:
3038            if self.name == '__new__' and env.is_py_class_scope:
3039                self.is_staticmethod = True
3040            elif self.name == '__init_subclass__' and env.is_c_class_scope:
3041                error(self.pos, "'__init_subclass__' is not supported by extension class")
3042            elif self.name in IMPLICIT_CLASSMETHODS and not self.is_classmethod:
3043                self.is_classmethod = True
3044                # TODO: remove the need to generate a real decorator here, is_classmethod=True should suffice.
3045                from .ExprNodes import NameNode
3046                self.decorators = self.decorators or []
3047                self.decorators.insert(0, DecoratorNode(
3048                    self.pos, decorator=NameNode(self.pos, name=EncodedString('classmethod'))))
3049
3050        self.analyse_argument_types(env)
3051        if self.name == '<lambda>':
3052            self.declare_lambda_function(env)
3053        else:
3054            self.declare_pyfunction(env)
3055
3056        self.analyse_signature(env)
3057        self.return_type = self.entry.signature.return_type()
3058        # if a signature annotation provides a more specific return object type, use it
3059        if self.return_type is py_object_type and self.return_type_annotation:
3060            if env.directives['annotation_typing'] and not self.entry.is_special:
3061                _, return_type = self.return_type_annotation.analyse_type_annotation(env)
3062                if return_type and return_type.is_pyobject:
3063                    self.return_type = return_type
3064
3065        self.create_local_scope(env)
3066
3067        self.py_wrapper = DefNodeWrapper(
3068            self.pos,
3069            target=self,
3070            name=self.entry.name,
3071            args=self.args,
3072            star_arg=self.star_arg,
3073            starstar_arg=self.starstar_arg,
3074            return_type=self.return_type)
3075        self.py_wrapper.analyse_declarations(env)
3076
3077    def analyse_argument_types(self, env):
3078        self.directive_locals = env.directives.get('locals', {})
3079        allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
3080
3081        f2s = env.fused_to_specific
3082        env.fused_to_specific = None
3083
3084        for arg in self.args:
3085            if hasattr(arg, 'name'):
3086                name_declarator = None
3087            else:
3088                base_type = arg.base_type.analyse(env)
3089                # If we hare in pythran mode and we got a buffer supported by
3090                # Pythran, we change this node to a fused type
3091                if has_np_pythran(env) and base_type.is_pythran_expr:
3092                    base_type = PyrexTypes.FusedType([
3093                        base_type,
3094                        #PyrexTypes.PythranExpr(pythran_type(self.type, "numpy_texpr")),
3095                        base_type.org_buffer])
3096                name_declarator, type = \
3097                    arg.declarator.analyse(base_type, env)
3098                arg.name = name_declarator.name
3099                arg.type = type
3100
3101            self.align_argument_type(env, arg)
3102            if name_declarator and name_declarator.cname:
3103                error(self.pos, "Python function argument cannot have C name specification")
3104            arg.type = arg.type.as_argument_type()
3105            arg.hdr_type = None
3106            arg.needs_conversion = 0
3107            arg.needs_type_test = 0
3108            arg.is_generic = 1
3109            if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
3110                if arg.or_none:
3111                    arg.accept_none = True
3112                elif arg.not_none:
3113                    arg.accept_none = False
3114                elif (arg.type.is_extension_type or arg.type.is_builtin_type
3115                        or arg.type.is_buffer or arg.type.is_memoryviewslice):
3116                    if arg.default and arg.default.constant_result is None:
3117                        # special case: def func(MyType obj = None)
3118                        arg.accept_none = True
3119                    else:
3120                        # default depends on compiler directive
3121                        arg.accept_none = allow_none_for_extension_args
3122                else:
3123                    # probably just a plain 'object'
3124                    arg.accept_none = True
3125            else:
3126                arg.accept_none = True  # won't be used, but must be there
3127                if arg.not_none:
3128                    error(arg.pos, "Only Python type arguments can have 'not None'")
3129                if arg.or_none:
3130                    error(arg.pos, "Only Python type arguments can have 'or None'")
3131
3132            if arg.type.is_fused:
3133                self.has_fused_arguments = True
3134        env.fused_to_specific = f2s
3135
3136        if has_np_pythran(env):
3137            self.np_args_idx = [i for i,a in enumerate(self.args) if a.type.is_numpy_buffer]
3138        else:
3139            self.np_args_idx = []
3140
3141    def analyse_signature(self, env):
3142        if self.entry.is_special:
3143            if self.decorators:
3144                error(self.pos, "special functions of cdef classes cannot have decorators")
3145            self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
3146        elif not (self.star_arg or self.starstar_arg) and (
3147                not env.directives['always_allow_keywords']
3148                or all([arg.pos_only for arg in self.args])):
3149            # Use the simpler calling signature for zero- and one-argument pos-only functions.
3150            if self.entry.signature is TypeSlots.pyfunction_signature:
3151                if len(self.args) == 0:
3152                    self.entry.signature = TypeSlots.pyfunction_noargs
3153                elif len(self.args) == 1:
3154                    if self.args[0].default is None and not self.args[0].kw_only:
3155                        self.entry.signature = TypeSlots.pyfunction_onearg
3156            elif self.entry.signature is TypeSlots.pymethod_signature:
3157                if len(self.args) == 1:
3158                    self.entry.signature = TypeSlots.unaryfunc
3159                elif len(self.args) == 2:
3160                    if self.args[1].default is None and not self.args[1].kw_only:
3161                        self.entry.signature = TypeSlots.ibinaryfunc
3162
3163        sig = self.entry.signature
3164        nfixed = sig.num_fixed_args()
3165        if (sig is TypeSlots.pymethod_signature and nfixed == 1
3166               and len(self.args) == 0 and self.star_arg):
3167            # this is the only case where a diverging number of
3168            # arguments is not an error - when we have no explicit
3169            # 'self' parameter as in method(*args)
3170            sig = self.entry.signature = TypeSlots.pyfunction_signature  # self is not 'really' used
3171            self.self_in_stararg = 1
3172            nfixed = 0
3173
3174        if self.is_staticmethod and env.is_c_class_scope:
3175            nfixed = 0
3176            self.self_in_stararg = True  # FIXME: why for staticmethods?
3177
3178            self.entry.signature = sig = copy.copy(sig)
3179            sig.fixed_arg_format = "*"
3180            sig.is_staticmethod = True
3181            sig.has_generic_args = True
3182
3183        if ((self.is_classmethod or self.is_staticmethod) and
3184                self.has_fused_arguments and env.is_c_class_scope):
3185            del self.decorator_indirection.stats[:]
3186
3187        for i in range(min(nfixed, len(self.args))):
3188            arg = self.args[i]
3189            arg.is_generic = 0
3190            if sig.is_self_arg(i) and not self.is_staticmethod:
3191                if self.is_classmethod:
3192                    arg.is_type_arg = 1
3193                    arg.hdr_type = arg.type = Builtin.type_type
3194                else:
3195                    arg.is_self_arg = 1
3196                    arg.hdr_type = arg.type = env.parent_type
3197                arg.needs_conversion = 0
3198            else:
3199                arg.hdr_type = sig.fixed_arg_type(i)
3200                if not arg.type.same_as(arg.hdr_type):
3201                    if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
3202                        arg.needs_type_test = 1
3203                    else:
3204                        arg.needs_conversion = 1
3205
3206        if nfixed > len(self.args):
3207            self.bad_signature()
3208            return
3209        elif nfixed < len(self.args):
3210            if not sig.has_generic_args:
3211                self.bad_signature()
3212            for arg in self.args:
3213                if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type):
3214                    arg.needs_type_test = 1
3215
3216        # Decide whether to use METH_FASTCALL
3217        # 1. If we use METH_NOARGS or METH_O, keep that. We can only change
3218        #    METH_VARARGS to METH_FASTCALL
3219        # 2. Special methods like __call__ always use the METH_VARGARGS
3220        #    calling convention
3221        mf = sig.method_flags()
3222        if mf and TypeSlots.method_varargs in mf and not self.entry.is_special:
3223            # 3. If the function uses the full args tuple, it's more
3224            #    efficient to use METH_VARARGS. This happens when the function
3225            #    takes *args but no other positional arguments (apart from
3226            #    possibly self). We don't do the analogous check for keyword
3227            #    arguments since the kwargs dict is copied anyway.
3228            if self.star_arg:
3229                uses_args_tuple = True
3230                for arg in self.args:
3231                    if (arg.is_generic and not arg.kw_only and
3232                            not arg.is_self_arg and not arg.is_type_arg):
3233                        # Other positional argument
3234                        uses_args_tuple = False
3235            else:
3236                uses_args_tuple = False
3237
3238            if not uses_args_tuple:
3239                sig = self.entry.signature = sig.with_fastcall()
3240
3241    def bad_signature(self):
3242        sig = self.entry.signature
3243        expected_str = "%d" % sig.num_fixed_args()
3244        if sig.has_generic_args:
3245            expected_str += " or more"
3246        name = self.name
3247        if name.startswith("__") and name.endswith("__"):
3248            desc = "Special method"
3249        else:
3250            desc = "Method"
3251        error(self.pos, "%s %s has wrong number of arguments (%d declared, %s expected)" % (
3252            desc, self.name, len(self.args), expected_str))
3253
3254    def declare_pyfunction(self, env):
3255        #print "DefNode.declare_pyfunction:", self.name, "in", env ###
3256        name = self.name
3257        entry = env.lookup_here(name)
3258        if entry:
3259            if entry.is_final_cmethod and not env.parent_type.is_final_type:
3260                error(self.pos, "Only final types can have final Python (def/cpdef) methods")
3261            if entry.type.is_cfunction and not entry.is_builtin_cmethod and not self.is_wrapper:
3262                warning(self.pos, "Overriding cdef method with def method.", 5)
3263        entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
3264        self.entry = entry
3265        prefix = env.next_id(env.scope_prefix)
3266        self.entry.pyfunc_cname = punycodify_name(Naming.pyfunc_prefix + prefix + name)
3267        if Options.docstrings:
3268            entry.doc = embed_position(self.pos, self.doc)
3269            entry.doc_cname = punycodify_name(Naming.funcdoc_prefix + prefix + name)
3270            if entry.is_special:
3271                if entry.name in TypeSlots.invisible or not entry.doc or (
3272                        entry.name in '__getattr__' and env.directives['fast_getattr']):
3273                    entry.wrapperbase_cname = None
3274                else:
3275                    entry.wrapperbase_cname = punycodify_name(Naming.wrapperbase_prefix + prefix + name)
3276        else:
3277            entry.doc = None
3278
3279    def declare_lambda_function(self, env):
3280        entry = env.declare_lambda_function(self.lambda_name, self.pos)
3281        entry.doc = None
3282        self.entry = entry
3283        self.entry.pyfunc_cname = entry.cname
3284
3285    def declare_arguments(self, env):
3286        for arg in self.args:
3287            if not arg.name:
3288                error(arg.pos, "Missing argument name")
3289            if arg.needs_conversion:
3290                arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
3291                if arg.type.is_pyobject:
3292                    arg.entry.init = "0"
3293            else:
3294                arg.entry = self.declare_argument(env, arg)
3295            arg.entry.is_arg = 1
3296            arg.entry.used = 1
3297            arg.entry.is_self_arg = arg.is_self_arg
3298        self.declare_python_arg(env, self.star_arg)
3299        self.declare_python_arg(env, self.starstar_arg)
3300
3301    def declare_python_arg(self, env, arg):
3302        if arg:
3303            if env.directives['infer_types'] != False:
3304                type = PyrexTypes.unspecified_type
3305            else:
3306                type = py_object_type
3307            entry = env.declare_var(arg.name, type, arg.pos)
3308            entry.is_arg = 1
3309            entry.used = 1
3310            entry.init = "0"
3311            entry.xdecref_cleanup = 1
3312            arg.entry = entry
3313
3314    def analyse_expressions(self, env):
3315        self.local_scope.directives = env.directives
3316        self.analyse_default_values(env)
3317        self.analyse_annotations(env)
3318
3319        if not self.needs_assignment_synthesis(env) and self.decorators:
3320            for decorator in self.decorators[::-1]:
3321                decorator.decorator = decorator.decorator.analyse_expressions(env)
3322
3323        self.py_wrapper.prepare_argument_coercion(env)
3324        return self
3325
3326    def needs_assignment_synthesis(self, env, code=None):
3327        if self.is_staticmethod:
3328            return True
3329        if self.specialized_cpdefs or self.entry.is_fused_specialized:
3330            return False
3331        if self.no_assignment_synthesis:
3332            return False
3333        if self.entry.is_special:
3334            return False
3335        if self.entry.is_anonymous:
3336            return True
3337        if env.is_module_scope or env.is_c_class_scope:
3338            if code is None:
3339                return self.local_scope.directives['binding']
3340            else:
3341                return code.globalstate.directives['binding']
3342        return env.is_py_class_scope or env.is_closure_scope
3343
3344    def error_value(self):
3345        return self.entry.signature.error_value
3346
3347    def caller_will_check_exceptions(self):
3348        return self.entry.signature.exception_check
3349
3350    def generate_function_definitions(self, env, code):
3351        if self.defaults_getter:
3352            # defaults getter must never live in class scopes, it's always a module function
3353            self.defaults_getter.generate_function_definitions(env.global_scope(), code)
3354
3355        # Before closure cnames are mangled
3356        if self.py_wrapper_required:
3357            # func_cname might be modified by @cname
3358            self.py_wrapper.func_cname = self.entry.func_cname
3359            self.py_wrapper.generate_function_definitions(env, code)
3360        FuncDefNode.generate_function_definitions(self, env, code)
3361
3362    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3363        if proto_only:
3364            if self.py_wrapper_required:
3365                self.py_wrapper.generate_function_header(
3366                    code, with_pymethdef, True)
3367            return
3368        arg_code_list = []
3369        if self.entry.signature.has_dummy_arg:
3370            self_arg = 'PyObject *%s' % Naming.self_cname
3371            if not self.needs_outer_scope:
3372                self_arg = 'CYTHON_UNUSED ' + self_arg
3373            arg_code_list.append(self_arg)
3374
3375        def arg_decl_code(arg):
3376            entry = arg.entry
3377            if entry.in_closure:
3378                cname = entry.original_cname
3379            else:
3380                cname = entry.cname
3381            decl = entry.type.declaration_code(cname)
3382            if not entry.cf_used:
3383                decl = 'CYTHON_UNUSED ' + decl
3384            return decl
3385
3386        for arg in self.args:
3387            arg_code_list.append(arg_decl_code(arg))
3388        if self.star_arg:
3389            arg_code_list.append(arg_decl_code(self.star_arg))
3390        if self.starstar_arg:
3391            arg_code_list.append(arg_decl_code(self.starstar_arg))
3392        if arg_code_list:
3393            arg_code = ', '.join(arg_code_list)
3394        else:
3395            arg_code = 'void'  # No arguments
3396        dc = self.return_type.declaration_code(self.entry.pyfunc_cname)
3397
3398        decls_code = code.globalstate['decls']
3399        preprocessor_guard = self.get_preprocessor_guard()
3400        if preprocessor_guard:
3401            decls_code.putln(preprocessor_guard)
3402        decls_code.putln(
3403            "static %s(%s); /* proto */" % (dc, arg_code))
3404        if preprocessor_guard:
3405            decls_code.putln("#endif")
3406        code.putln("static %s(%s) {" % (dc, arg_code))
3407
3408    def generate_argument_declarations(self, env, code):
3409        pass
3410
3411    def generate_keyword_list(self, code):
3412        pass
3413
3414    def generate_argument_parsing_code(self, env, code):
3415        # Move arguments into closure if required
3416        def put_into_closure(entry):
3417            if entry.in_closure:
3418                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
3419                if entry.xdecref_cleanup:
3420                    # mostly applies to the starstar arg - this can sometimes be NULL
3421                    # so must be xincrefed instead
3422                    code.put_var_xincref(entry)
3423                    code.put_var_xgiveref(entry)
3424                else:
3425                    code.put_var_incref(entry)
3426                    code.put_var_giveref(entry)
3427        for arg in self.args:
3428            put_into_closure(arg.entry)
3429        for arg in self.star_arg, self.starstar_arg:
3430            if arg:
3431                put_into_closure(arg.entry)
3432
3433    def generate_argument_type_tests(self, code):
3434        pass
3435
3436
3437class DefNodeWrapper(FuncDefNode):
3438    # DefNode python wrapper code generator
3439
3440    defnode = None
3441    target = None  # Target DefNode
3442
3443    def __init__(self, *args, **kwargs):
3444        FuncDefNode.__init__(self, *args, **kwargs)
3445        self.num_posonly_args = self.target.num_posonly_args
3446        self.num_kwonly_args = self.target.num_kwonly_args
3447        self.num_required_kw_args = self.target.num_required_kw_args
3448        self.num_required_args = self.target.num_required_args
3449        self.self_in_stararg = self.target.self_in_stararg
3450        self.signature = None
3451
3452    def analyse_declarations(self, env):
3453        target_entry = self.target.entry
3454        name = self.name
3455        prefix = env.next_id(env.scope_prefix)
3456        target_entry.func_cname = punycodify_name(Naming.pywrap_prefix + prefix + name)
3457        target_entry.pymethdef_cname = punycodify_name(Naming.pymethdef_prefix + prefix + name)
3458
3459        self.signature = target_entry.signature
3460
3461        self.np_args_idx = self.target.np_args_idx
3462
3463    def prepare_argument_coercion(self, env):
3464        # This is only really required for Cython utility code at this time,
3465        # everything else can be done during code generation.  But we expand
3466        # all utility code here, simply because we cannot easily distinguish
3467        # different code types.
3468        for arg in self.args:
3469            if not arg.type.is_pyobject:
3470                if not arg.type.create_from_py_utility_code(env):
3471                    pass  # will fail later
3472            elif arg.hdr_type and not arg.hdr_type.is_pyobject:
3473                if not arg.hdr_type.create_to_py_utility_code(env):
3474                    pass  # will fail later
3475
3476        if self.starstar_arg and not self.starstar_arg.entry.cf_used:
3477            # we will set the kwargs argument to NULL instead of a new dict
3478            # and must therefore correct the control flow state
3479            entry = self.starstar_arg.entry
3480            entry.xdecref_cleanup = 1
3481            for ass in entry.cf_assignments:
3482                if not ass.is_arg and ass.lhs.is_name:
3483                    ass.lhs.cf_maybe_null = True
3484
3485    def signature_has_nongeneric_args(self):
3486        argcount = len(self.args)
3487        if argcount == 0 or (
3488                argcount == 1 and (self.args[0].is_self_arg or
3489                                   self.args[0].is_type_arg)):
3490            return 0
3491        return 1
3492
3493    def signature_has_generic_args(self):
3494        return self.signature.has_generic_args
3495
3496    def generate_function_body(self, code):
3497        args = []
3498        if self.signature.has_dummy_arg:
3499            args.append(Naming.self_cname)
3500        for arg in self.args:
3501            if arg.type.is_cpp_class:
3502                # it's safe to move converted C++ types because they aren't
3503                # used again afterwards
3504                code.globalstate.use_utility_code(
3505                    UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp"))
3506                args.append("__PYX_STD_MOVE_IF_SUPPORTED(%s)" % arg.entry.cname)
3507            elif arg.hdr_type and not (arg.type.is_memoryviewslice or
3508                                     arg.type.is_struct or
3509                                     arg.type.is_complex):
3510                args.append(arg.type.cast_code(arg.entry.cname))
3511            else:
3512                args.append(arg.entry.cname)
3513        if self.star_arg:
3514            args.append(self.star_arg.entry.cname)
3515        if self.starstar_arg:
3516            args.append(self.starstar_arg.entry.cname)
3517        args = ', '.join(args)
3518        if not self.return_type.is_void:
3519            code.put('%s = ' % Naming.retval_cname)
3520        code.putln('%s(%s);' % (
3521            self.target.entry.pyfunc_cname, args))
3522
3523    def generate_function_definitions(self, env, code):
3524        lenv = self.target.local_scope
3525        # Generate C code for header and body of function
3526        code.mark_pos(self.pos)
3527        code.putln("")
3528        code.putln("/* Python wrapper */")
3529        preprocessor_guard = self.target.get_preprocessor_guard()
3530        if preprocessor_guard:
3531            code.putln(preprocessor_guard)
3532
3533        code.enter_cfunc_scope(lenv)
3534        code.return_from_error_cleanup_label = code.new_label()
3535
3536        with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
3537                          self.target.pymethdef_required)
3538        self.generate_function_header(code, with_pymethdef)
3539        self.generate_argument_declarations(lenv, code)
3540        tempvardecl_code = code.insertion_point()
3541
3542        if self.return_type.is_pyobject:
3543            retval_init = ' = 0'
3544        else:
3545            retval_init = ''
3546        if not self.return_type.is_void:
3547            code.putln('%s%s;' % (
3548                self.return_type.declaration_code(Naming.retval_cname),
3549                retval_init))
3550        code.put_declare_refcount_context()
3551        code.put_setup_refcount_context(EncodedString('%s (wrapper)' % self.name))
3552
3553        self.generate_argument_parsing_code(lenv, code)
3554        self.generate_argument_type_tests(code)
3555        self.generate_function_body(code)
3556
3557        # ----- Go back and insert temp variable declarations
3558        tempvardecl_code.put_temp_declarations(code.funcstate)
3559
3560        code.mark_pos(self.pos)
3561        code.putln("")
3562        code.putln("/* function exit code */")
3563
3564        # ----- Error cleanup
3565        if code.error_label in code.labels_used:
3566            code.put_goto(code.return_label)
3567            code.put_label(code.error_label)
3568            for cname, type in code.funcstate.all_managed_temps():
3569                code.put_xdecref(cname, type)
3570            err_val = self.error_value()
3571            if err_val is not None:
3572                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
3573
3574        # ----- Non-error return cleanup
3575        code.put_label(code.return_label)
3576        for entry in lenv.var_entries:
3577            if entry.is_arg and entry.type.is_pyobject:
3578                if entry.xdecref_cleanup:
3579                    code.put_var_xdecref(entry)
3580                else:
3581                    code.put_var_decref(entry)
3582
3583        code.put_finish_refcount_context()
3584        if not self.return_type.is_void:
3585            code.putln("return %s;" % Naming.retval_cname)
3586        code.putln('}')
3587        code.exit_cfunc_scope()
3588        if preprocessor_guard:
3589            code.putln("#endif /*!(%s)*/" % preprocessor_guard)
3590
3591    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3592        arg_code_list = []
3593        sig = self.signature
3594
3595        if sig.has_dummy_arg or self.self_in_stararg:
3596            arg_code = "PyObject *%s" % Naming.self_cname
3597            if not sig.has_dummy_arg:
3598                arg_code = 'CYTHON_UNUSED ' + arg_code
3599            arg_code_list.append(arg_code)
3600
3601        for arg in self.args:
3602            if not arg.is_generic:
3603                if arg.is_self_arg or arg.is_type_arg:
3604                    arg_code_list.append("PyObject *%s" % arg.hdr_cname)
3605                else:
3606                    arg_code_list.append(
3607                        arg.hdr_type.declaration_code(arg.hdr_cname))
3608        entry = self.target.entry
3609        if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
3610            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3611        if entry.scope.is_c_class_scope and entry.name == "__ipow__":
3612            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3613        if sig.has_generic_args:
3614            varargs_args = "PyObject *%s, PyObject *%s" % (
3615                    Naming.args_cname, Naming.kwds_cname)
3616            if sig.use_fastcall:
3617                fastcall_args = "PyObject *const *%s, Py_ssize_t %s, PyObject *%s" % (
3618                        Naming.args_cname, Naming.nargs_cname, Naming.kwds_cname)
3619                arg_code_list.append(
3620                    "\n#if CYTHON_METH_FASTCALL\n%s\n#else\n%s\n#endif\n" % (
3621                        fastcall_args, varargs_args))
3622            else:
3623                arg_code_list.append(varargs_args)
3624        arg_code = ", ".join(arg_code_list)
3625
3626        # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
3627        mf = ""
3628        if (entry.name in ("__getbuffer__", "__releasebuffer__")
3629                and entry.scope.is_c_class_scope):
3630            mf = "CYTHON_UNUSED "
3631            with_pymethdef = False
3632
3633        dc = self.return_type.declaration_code(entry.func_cname)
3634        header = "static %s%s(%s)" % (mf, dc, arg_code)
3635        code.putln("%s; /*proto*/" % header)
3636
3637        if proto_only:
3638            if self.target.fused_py_func:
3639                # If we are the specialized version of the cpdef, we still
3640                # want the prototype for the "fused cpdef", in case we're
3641                # checking to see if our method was overridden in Python
3642                self.target.fused_py_func.generate_function_header(
3643                    code, with_pymethdef, proto_only=True)
3644            return
3645
3646        if (Options.docstrings and entry.doc and
3647                not self.target.fused_py_func and
3648                not entry.scope.is_property_scope and
3649                (not entry.is_special or entry.wrapperbase_cname)):
3650            # h_code = code.globalstate['h_code']
3651            docstr = entry.doc
3652
3653            if docstr.is_unicode:
3654                docstr = docstr.as_utf8_string()
3655
3656            if not (entry.is_special and entry.name in ('__getbuffer__', '__releasebuffer__')):
3657                code.putln('PyDoc_STRVAR(%s, %s);' % (
3658                    entry.doc_cname,
3659                    docstr.as_c_string_literal()))
3660
3661            if entry.is_special:
3662                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
3663                code.putln(
3664                    "struct wrapperbase %s;" % entry.wrapperbase_cname)
3665                code.putln('#endif')
3666
3667        if with_pymethdef or self.target.fused_py_func:
3668            code.put(
3669                "static PyMethodDef %s = " % entry.pymethdef_cname)
3670            code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
3671        code.putln("%s {" % header)
3672
3673    def generate_argument_declarations(self, env, code):
3674        for arg in self.args:
3675            if arg.is_generic:
3676                if arg.needs_conversion:
3677                    code.putln("PyObject *%s = 0;" % arg.hdr_cname)
3678                else:
3679                    code.put_var_declaration(arg.entry)
3680        for entry in env.var_entries:
3681            if entry.is_arg:
3682                code.put_var_declaration(entry)
3683
3684        # Assign nargs variable as len(args), but avoid an "unused" warning in the few cases where we don't need it.
3685        if self.signature_has_generic_args():
3686            nargs_code = "CYTHON_UNUSED const Py_ssize_t %s = PyTuple_GET_SIZE(%s);" % (
3687                        Naming.nargs_cname, Naming.args_cname)
3688            if self.signature.use_fastcall:
3689                code.putln("#if !CYTHON_METH_FASTCALL")
3690                code.putln(nargs_code)
3691                code.putln("#endif")
3692            else:
3693                code.putln(nargs_code)
3694
3695        # Array containing the values of keyword arguments when using METH_FASTCALL.
3696        code.globalstate.use_utility_code(
3697            UtilityCode.load_cached("fastcall", "FunctionArguments.c"))
3698        code.putln('CYTHON_UNUSED PyObject *const *%s = __Pyx_KwValues_%s(%s, %s);' % (
3699            Naming.kwvalues_cname, self.signature.fastvar, Naming.args_cname, Naming.nargs_cname))
3700
3701    def generate_argument_parsing_code(self, env, code):
3702        # Generate fast equivalent of PyArg_ParseTuple call for
3703        # generic arguments, if any, including args/kwargs
3704        old_error_label = code.new_error_label()
3705        our_error_label = code.error_label
3706        end_label = code.new_label("argument_unpacking_done")
3707
3708        has_kwonly_args = self.num_kwonly_args > 0
3709        has_star_or_kw_args = self.star_arg is not None \
3710            or self.starstar_arg is not None or has_kwonly_args
3711
3712        for arg in self.args:
3713            if not arg.type.is_pyobject:
3714                if not arg.type.create_from_py_utility_code(env):
3715                    pass  # will fail later
3716
3717        if not self.signature_has_generic_args():
3718            if has_star_or_kw_args:
3719                error(self.pos, "This method cannot have * or keyword arguments")
3720            self.generate_argument_conversion_code(code)
3721
3722        elif not self.signature_has_nongeneric_args():
3723            # func(*args) or func(**kw) or func(*args, **kw)
3724            # possibly with a "self" argument but no other non-star
3725            # arguments
3726            self.generate_stararg_copy_code(code)
3727
3728        else:
3729            self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
3730
3731        code.error_label = old_error_label
3732        if code.label_used(our_error_label):
3733            if not code.label_used(end_label):
3734                code.put_goto(end_label)
3735            code.put_label(our_error_label)
3736            if has_star_or_kw_args:
3737                self.generate_arg_decref(self.star_arg, code)
3738                if self.starstar_arg:
3739                    if self.starstar_arg.entry.xdecref_cleanup:
3740                        code.put_var_xdecref_clear(self.starstar_arg.entry)
3741                    else:
3742                        code.put_var_decref_clear(self.starstar_arg.entry)
3743            code.put_add_traceback(self.target.entry.qualified_name)
3744            code.put_finish_refcount_context()
3745            code.putln("return %s;" % self.error_value())
3746        if code.label_used(end_label):
3747            code.put_label(end_label)
3748
3749    def generate_arg_xdecref(self, arg, code):
3750        if arg:
3751            code.put_var_xdecref_clear(arg.entry)
3752
3753    def generate_arg_decref(self, arg, code):
3754        if arg:
3755            code.put_var_decref_clear(arg.entry)
3756
3757    def generate_stararg_copy_code(self, code):
3758        if not self.star_arg:
3759            code.globalstate.use_utility_code(
3760                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3761            code.putln("if (unlikely(%s > 0)) {" % Naming.nargs_cname)
3762            code.put('__Pyx_RaiseArgtupleInvalid(%s, 1, 0, 0, %s); return %s;' % (
3763                self.name.as_c_string_literal(), Naming.nargs_cname, self.error_value()))
3764            code.putln("}")
3765
3766        if self.starstar_arg:
3767            if self.star_arg or not self.starstar_arg.entry.cf_used:
3768                kwarg_check = "unlikely(%s)" % Naming.kwds_cname
3769            else:
3770                kwarg_check = "%s" % Naming.kwds_cname
3771        else:
3772            kwarg_check = "unlikely(%s) && __Pyx_NumKwargs_%s(%s)" % (
3773                Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname)
3774        code.globalstate.use_utility_code(
3775            UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
3776        code.putln(
3777            "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, %s, %d))) return %s;" % (
3778                kwarg_check, Naming.kwds_cname, self.name.as_c_string_literal(),
3779                bool(self.starstar_arg), self.error_value()))
3780
3781        if self.starstar_arg and self.starstar_arg.entry.cf_used:
3782            code.putln("if (%s) {" % kwarg_check)
3783            code.putln("%s = __Pyx_KwargsAsDict_%s(%s, %s);" % (
3784                self.starstar_arg.entry.cname,
3785                self.signature.fastvar,
3786                Naming.kwds_cname,
3787                Naming.kwvalues_cname))
3788            code.putln("if (unlikely(!%s)) return %s;" % (
3789                self.starstar_arg.entry.cname, self.error_value()))
3790            code.put_gotref(self.starstar_arg.entry.cname, py_object_type)
3791            code.putln("} else {")
3792            allow_null = all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references)
3793            if allow_null:
3794                code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,))
3795            else:
3796                code.putln("%s = PyDict_New();" % (self.starstar_arg.entry.cname,))
3797                code.putln("if (unlikely(!%s)) return %s;" % (
3798                    self.starstar_arg.entry.cname, self.error_value()))
3799                code.put_var_gotref(self.starstar_arg.entry)
3800            self.starstar_arg.entry.xdecref_cleanup = allow_null
3801            code.putln("}")
3802
3803        if self.self_in_stararg and not self.target.is_staticmethod:
3804            assert not self.signature.use_fastcall
3805            # need to create a new tuple with 'self' inserted as first item
3806            code.put("%s = PyTuple_New(%s + 1); if (unlikely(!%s)) " % (
3807                self.star_arg.entry.cname,
3808                Naming.nargs_cname,
3809                self.star_arg.entry.cname))
3810            if self.starstar_arg and self.starstar_arg.entry.cf_used:
3811                code.putln("{")
3812                code.put_var_xdecref_clear(self.starstar_arg.entry)
3813                code.putln("return %s;" % self.error_value())
3814                code.putln("}")
3815            else:
3816                code.putln("return %s;" % self.error_value())
3817            code.put_var_gotref(self.star_arg.entry)
3818            code.put_incref(Naming.self_cname, py_object_type)
3819            code.put_giveref(Naming.self_cname, py_object_type)
3820            code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
3821                self.star_arg.entry.cname, Naming.self_cname))
3822            temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
3823            code.putln("for (%s=0; %s < %s; %s++) {" % (
3824                temp, temp, Naming.nargs_cname, temp))
3825            code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
3826                Naming.args_cname, temp))
3827            code.put_incref("item", py_object_type)
3828            code.put_giveref("item", py_object_type)
3829            code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
3830                self.star_arg.entry.cname, temp))
3831            code.putln("}")
3832            code.funcstate.release_temp(temp)
3833            self.star_arg.entry.xdecref_cleanup = 0
3834        elif self.star_arg:
3835            assert not self.signature.use_fastcall
3836            code.put_incref(Naming.args_cname, py_object_type)
3837            code.putln("%s = %s;" % (
3838                self.star_arg.entry.cname,
3839                Naming.args_cname))
3840            self.star_arg.entry.xdecref_cleanup = 0
3841
3842    def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
3843        code.globalstate.use_utility_code(
3844            UtilityCode.load_cached("fastcall", "FunctionArguments.c"))
3845
3846        self_name_csafe = self.name.as_c_string_literal()
3847
3848        argtuple_error_label = code.new_label("argtuple_error")
3849
3850        positional_args = []
3851        required_kw_only_args = []
3852        optional_kw_only_args = []
3853        num_pos_only_args = 0
3854        for arg in args:
3855            if arg.is_generic:
3856                if arg.default:
3857                    if not arg.is_self_arg and not arg.is_type_arg:
3858                        if arg.kw_only:
3859                            optional_kw_only_args.append(arg)
3860                        else:
3861                            positional_args.append(arg)
3862                elif arg.kw_only:
3863                    required_kw_only_args.append(arg)
3864                elif not arg.is_self_arg and not arg.is_type_arg:
3865                    positional_args.append(arg)
3866                if arg.pos_only:
3867                    num_pos_only_args += 1
3868
3869        # sort required kw-only args before optional ones to avoid special
3870        # cases in the unpacking code
3871        kw_only_args = required_kw_only_args + optional_kw_only_args
3872
3873        min_positional_args = self.num_required_args - self.num_required_kw_args
3874        if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
3875            min_positional_args -= 1
3876        max_positional_args = len(positional_args)
3877        has_fixed_positional_count = not self.star_arg and \
3878            min_positional_args == max_positional_args
3879        has_kw_only_args = bool(kw_only_args)
3880
3881        if self.starstar_arg or self.star_arg:
3882            self.generate_stararg_init_code(max_positional_args, code)
3883
3884        code.putln('{')
3885        all_args = tuple(positional_args) + tuple(kw_only_args)
3886        non_posonly_args = [arg for arg in all_args if not arg.pos_only]
3887        non_pos_args_id = ','.join(
3888            ['&%s' % code.intern_identifier(arg.entry.name) for arg in non_posonly_args] + ['0'])
3889        code.putln("#if CYTHON_USE_MODULE_STATE")
3890        code.putln("PyObject **%s[] = {%s};" % (
3891            Naming.pykwdlist_cname,
3892            non_pos_args_id))
3893        code.putln("#else")
3894        code.putln("static PyObject **%s[] = {%s};" % (
3895            Naming.pykwdlist_cname,
3896            non_pos_args_id))
3897        code.putln("#endif")
3898
3899        # Before being converted and assigned to the target variables,
3900        # borrowed references to all unpacked argument values are
3901        # collected into a local PyObject* array called "values",
3902        # regardless if they were taken from default arguments,
3903        # positional arguments or keyword arguments.  Note that
3904        # C-typed default arguments are handled at conversion time,
3905        # so their array value is NULL in the end if no argument
3906        # was passed for them.
3907        self.generate_argument_values_setup_code(all_args, code)
3908
3909        # If all args are positional-only, we can raise an error
3910        # straight away if we receive a non-empty kw-dict.
3911        # This requires a PyDict_Size call.  This call is wasteful
3912        # for functions which do accept kw-args, so we do not generate
3913        # the PyDict_Size call unless all args are positional-only.
3914        accept_kwd_args = non_posonly_args or self.starstar_arg
3915        if accept_kwd_args:
3916            kw_unpacking_condition = Naming.kwds_cname
3917        else:
3918            kw_unpacking_condition = "%s && __Pyx_NumKwargs_%s(%s) > 0" % (
3919                Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname)
3920
3921        if self.num_required_kw_args > 0:
3922            kw_unpacking_condition = "likely(%s)" % kw_unpacking_condition
3923
3924        # --- optimised code when we receive keyword arguments
3925        code.putln("if (%s) {" % kw_unpacking_condition)
3926
3927        if accept_kwd_args:
3928            self.generate_keyword_unpacking_code(
3929                min_positional_args, max_positional_args,
3930                has_fixed_positional_count, has_kw_only_args, all_args, argtuple_error_label, code)
3931        else:
3932            # Here we do not accept kw-args but we are passed a non-empty kw-dict.
3933            # We call ParseOptionalKeywords which will raise an appropriate error if
3934            # the kw-args dict passed is non-empty (which it will be, since kw_unpacking_condition is true)
3935            code.globalstate.use_utility_code(
3936                UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
3937            code.putln('if (likely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % (
3938                Naming.kwds_cname,
3939                Naming.kwvalues_cname,
3940                Naming.pykwdlist_cname,
3941                self.starstar_arg.entry.cname if self.starstar_arg else 0,
3942                'values',
3943                0,
3944                self_name_csafe,
3945                code.error_goto(self.pos)))
3946
3947        # --- optimised code when we do not receive any keyword arguments
3948        if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
3949            # Python raises arg tuple related errors first, so we must
3950            # check the length here
3951            if min_positional_args == max_positional_args and not self.star_arg:
3952                compare = '!='
3953            else:
3954                compare = '<'
3955            code.putln('} else if (unlikely(%s %s %d)) {' % (
3956                Naming.nargs_cname, compare, min_positional_args))
3957            code.put_goto(argtuple_error_label)
3958
3959        if self.num_required_kw_args:
3960            # pure error case: keywords required but not passed
3961            if max_positional_args > min_positional_args and not self.star_arg:
3962                code.putln('} else if (unlikely(%s > %d)) {' % (
3963                    Naming.nargs_cname, max_positional_args))
3964                code.put_goto(argtuple_error_label)
3965            code.putln('} else {')
3966            for i, arg in enumerate(kw_only_args):
3967                if not arg.default:
3968                    pystring_cname = code.intern_identifier(arg.entry.name)
3969                    # required keyword-only argument missing
3970                    code.globalstate.use_utility_code(
3971                        UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
3972                    code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
3973                        self.name,
3974                        pystring_cname))
3975                    code.putln(code.error_goto(self.pos))
3976                    break
3977
3978        else:
3979            # optimised tuple unpacking code
3980            code.putln('} else {')
3981            if min_positional_args == max_positional_args:
3982                # parse the exact number of positional arguments from
3983                # the args tuple
3984                for i, arg in enumerate(positional_args):
3985                    code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % (
3986                            i, self.signature.fastvar, Naming.args_cname, i))
3987            else:
3988                # parse the positional arguments from the variable length
3989                # args tuple and reject illegal argument tuple sizes
3990                code.putln('switch (%s) {' % Naming.nargs_cname)
3991                if self.star_arg:
3992                    code.putln('default:')
3993                reversed_args = list(enumerate(positional_args))[::-1]
3994                for i, arg in reversed_args:
3995                    if i >= min_positional_args-1:
3996                        if i != reversed_args[0][0]:
3997                            code.putln('CYTHON_FALLTHROUGH;')
3998                        code.put('case %2d: ' % (i+1))
3999                    code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % (
4000                            i, self.signature.fastvar, Naming.args_cname, i))
4001                if min_positional_args == 0:
4002                    code.putln('CYTHON_FALLTHROUGH;')
4003                    code.put('case  0: ')
4004                code.putln('break;')
4005                if self.star_arg:
4006                    if min_positional_args:
4007                        for i in range(min_positional_args-1, -1, -1):
4008                            code.putln('case %2d:' % i)
4009                        code.put_goto(argtuple_error_label)
4010                else:
4011                    code.put('default: ')
4012                    code.put_goto(argtuple_error_label)
4013                code.putln('}')
4014
4015        code.putln('}')  # end of the conditional unpacking blocks
4016
4017        # Convert arg values to their final type and assign them.
4018        # Also inject non-Python default arguments, which do cannot
4019        # live in the values[] array.
4020        for i, arg in enumerate(all_args):
4021            self.generate_arg_assignment(arg, "values[%d]" % i, code)
4022
4023        code.putln('}')  # end of the whole argument unpacking block
4024
4025        if code.label_used(argtuple_error_label):
4026            code.put_goto(success_label)
4027            code.put_label(argtuple_error_label)
4028            code.globalstate.use_utility_code(
4029                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
4030            code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %s); ' % (
4031                self_name_csafe, has_fixed_positional_count,
4032                min_positional_args, max_positional_args,
4033                Naming.nargs_cname))
4034            code.putln(code.error_goto(self.pos))
4035
4036    def generate_arg_assignment(self, arg, item, code):
4037        if arg.type.is_pyobject:
4038            # Python default arguments were already stored in 'item' at the very beginning
4039            if arg.is_generic:
4040                item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
4041            entry = arg.entry
4042            code.putln("%s = %s;" % (entry.cname, item))
4043        else:
4044            if arg.type.from_py_function:
4045                if arg.default:
4046                    # C-typed default arguments must be handled here
4047                    code.putln('if (%s) {' % item)
4048                code.putln(arg.type.from_py_call_code(
4049                    item, arg.entry.cname, arg.pos, code))
4050                if arg.default:
4051                    code.putln('} else {')
4052                    code.putln("%s = %s;" % (
4053                        arg.entry.cname,
4054                        arg.calculate_default_value_code(code)))
4055                    if arg.type.is_memoryviewslice:
4056                        code.put_var_incref_memoryviewslice(arg.entry, have_gil=True)
4057                    code.putln('}')
4058            else:
4059                error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
4060
4061    def generate_stararg_init_code(self, max_positional_args, code):
4062        if self.starstar_arg:
4063            self.starstar_arg.entry.xdecref_cleanup = 0
4064            code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
4065                self.starstar_arg.entry.cname,
4066                self.starstar_arg.entry.cname,
4067                self.error_value()))
4068            code.put_var_gotref(self.starstar_arg.entry)
4069        if self.star_arg:
4070            self.star_arg.entry.xdecref_cleanup = 0
4071            if max_positional_args == 0:
4072                # If there are no positional arguments, use the args tuple
4073                # directly
4074                assert not self.signature.use_fastcall
4075                code.put_incref(Naming.args_cname, py_object_type)
4076                code.putln("%s = %s;" % (self.star_arg.entry.cname, Naming.args_cname))
4077            else:
4078                # It is possible that this is a slice of "negative" length,
4079                # as in args[5:3]. That's not a problem, the function below
4080                # handles that efficiently and returns the empty tuple.
4081                code.putln('%s = __Pyx_ArgsSlice_%s(%s, %d, %s);' % (
4082                    self.star_arg.entry.cname, self.signature.fastvar,
4083                    Naming.args_cname, max_positional_args, Naming.nargs_cname))
4084                code.putln("if (unlikely(!%s)) {" %
4085                           self.star_arg.entry.type.nullcheck_string(self.star_arg.entry.cname))
4086                if self.starstar_arg:
4087                    code.put_var_decref_clear(self.starstar_arg.entry)
4088                code.put_finish_refcount_context()
4089                code.putln('return %s;' % self.error_value())
4090                code.putln('}')
4091                code.put_var_gotref(self.star_arg.entry)
4092
4093    def generate_argument_values_setup_code(self, args, code):
4094        max_args = len(args)
4095        # the 'values' array collects borrowed references to arguments
4096        # before doing any type coercion etc.
4097        code.putln("PyObject* values[%d] = {%s};" % (
4098            max_args, ','.join('0'*max_args)))
4099
4100        if self.target.defaults_struct:
4101            code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
4102                self.target.defaults_struct, Naming.dynamic_args_cname,
4103                self.target.defaults_struct, Naming.self_cname))
4104
4105        # assign borrowed Python default values to the values array,
4106        # so that they can be overwritten by received arguments below
4107        for i, arg in enumerate(args):
4108            if arg.default and arg.type.is_pyobject:
4109                default_value = arg.calculate_default_value_code(code)
4110                code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))
4111
4112    def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
4113                                        has_fixed_positional_count,
4114                                        has_kw_only_args, all_args, argtuple_error_label, code):
4115        # First we count how many arguments must be passed as positional
4116        num_required_posonly_args = num_pos_only_args = 0
4117        for i, arg in enumerate(all_args):
4118            if arg.pos_only:
4119                num_pos_only_args += 1
4120                if not arg.default:
4121                    num_required_posonly_args += 1
4122
4123        code.putln('Py_ssize_t kw_args;')
4124        # copy the values from the args tuple and check that it's not too long
4125        code.putln('switch (%s) {' % Naming.nargs_cname)
4126        if self.star_arg:
4127            code.putln('default:')
4128
4129        for i in range(max_positional_args-1, num_required_posonly_args-1, -1):
4130            code.put('case %2d: ' % (i+1))
4131            code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % (
4132                i, self.signature.fastvar, Naming.args_cname, i))
4133            code.putln('CYTHON_FALLTHROUGH;')
4134        if num_required_posonly_args > 0:
4135            code.put('case %2d: ' % num_required_posonly_args)
4136            for i in range(num_required_posonly_args-1, -1, -1):
4137                code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % (
4138                    i, self.signature.fastvar, Naming.args_cname, i))
4139            code.putln('break;')
4140        for i in range(num_required_posonly_args-2, -1, -1):
4141            code.put('case %2d: ' % (i+1))
4142            code.putln('CYTHON_FALLTHROUGH;')
4143
4144        code.put('case  0: ')
4145        if num_required_posonly_args == 0:
4146            code.putln('break;')
4147        else:
4148            # catch-all for not enough pos-only args passed
4149            code.put_goto(argtuple_error_label)
4150        if not self.star_arg:
4151            code.put('default: ')  # more arguments than allowed
4152            code.put_goto(argtuple_error_label)
4153        code.putln('}')
4154
4155        # The code above is very often (but not always) the same as
4156        # the optimised non-kwargs tuple unpacking code, so we keep
4157        # the code block above at the very top, before the following
4158        # 'external' PyDict_Size() call, to make it easy for the C
4159        # compiler to merge the two separate tuple unpacking
4160        # implementations into one when they turn out to be identical.
4161
4162        # If we received kwargs, fill up the positional/required
4163        # arguments with values from the kw dict
4164        self_name_csafe = self.name.as_c_string_literal()
4165
4166        code.putln('kw_args = __Pyx_NumKwargs_%s(%s);' % (
4167                self.signature.fastvar, Naming.kwds_cname))
4168        if self.num_required_args or max_positional_args > 0:
4169            last_required_arg = -1
4170            for i, arg in enumerate(all_args):
4171                if not arg.default:
4172                    last_required_arg = i
4173            if last_required_arg < max_positional_args:
4174                last_required_arg = max_positional_args-1
4175            if max_positional_args > num_pos_only_args:
4176                code.putln('switch (%s) {' % Naming.nargs_cname)
4177            for i, arg in enumerate(all_args[num_pos_only_args:last_required_arg+1], num_pos_only_args):
4178                if max_positional_args > num_pos_only_args and i <= max_positional_args:
4179                    if i != num_pos_only_args:
4180                        code.putln('CYTHON_FALLTHROUGH;')
4181                    if self.star_arg and i == max_positional_args:
4182                        code.putln('default:')
4183                    else:
4184                        code.putln('case %2d:' % i)
4185                pystring_cname = code.intern_identifier(arg.entry.name)
4186                if arg.default:
4187                    if arg.kw_only:
4188                        # optional kw-only args are handled separately below
4189                        continue
4190                    code.putln('if (kw_args > 0) {')
4191                    # don't overwrite default argument
4192                    code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, %s);' % (
4193                        self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname))
4194                    code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
4195                    code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos))
4196                    code.putln('}')
4197                else:
4198                    code.putln('if (likely((values[%d] = __Pyx_GetKwValue_%s(%s, %s, %s)) != 0)) kw_args--;' % (
4199                        i, self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname))
4200                    code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos))
4201                    if i < min_positional_args:
4202                        if i == 0:
4203                            # special case: we know arg 0 is missing
4204                            code.put('else ')
4205                            code.put_goto(argtuple_error_label)
4206                        else:
4207                            # print the correct number of values (args or
4208                            # kwargs) that were passed into positional
4209                            # arguments up to this point
4210                            code.putln('else {')
4211                            code.globalstate.use_utility_code(
4212                                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
4213                            code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %d); ' % (
4214                                self_name_csafe, has_fixed_positional_count,
4215                                min_positional_args, max_positional_args, i))
4216                            code.putln(code.error_goto(self.pos))
4217                            code.putln('}')
4218                    elif arg.kw_only:
4219                        code.putln('else {')
4220                        code.globalstate.use_utility_code(
4221                            UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
4222                        code.put('__Pyx_RaiseKeywordRequired(%s, %s); ' % (
4223                            self_name_csafe, pystring_cname))
4224                        code.putln(code.error_goto(self.pos))
4225                        code.putln('}')
4226            if max_positional_args > num_pos_only_args:
4227                code.putln('}')
4228
4229        if has_kw_only_args:
4230            # unpack optional keyword-only arguments separately because
4231            # checking for interned strings in a dict is faster than iterating
4232            self.generate_optional_kwonly_args_unpacking_code(all_args, code)
4233
4234        code.putln('if (unlikely(kw_args > 0)) {')
4235        # non-positional/-required kw args left in dict: default args,
4236        # kw-only args, **kwargs or error
4237        #
4238        # This is sort of a catch-all: except for checking required
4239        # arguments, this will always do the right thing for unpacking
4240        # keyword arguments, so that we can concentrate on optimising
4241        # common cases above.
4242        #
4243        # ParseOptionalKeywords() needs to know how many of the arguments
4244        # that could be passed as keywords have in fact been passed as
4245        # positional args.
4246        if num_pos_only_args > 0:
4247            # There are positional-only arguments which we don't want to count,
4248            # since they cannot be keyword arguments.  Subtract the number of
4249            # pos-only arguments from the number of positional arguments we got.
4250            # If we get a negative number then none of the keyword arguments were
4251            # passed as positional args.
4252            code.putln('const Py_ssize_t kwd_pos_args = (unlikely(%s < %d)) ? 0 : %s - %d;' % (
4253                Naming.nargs_cname, num_pos_only_args,
4254                Naming.nargs_cname, num_pos_only_args,
4255            ))
4256        elif max_positional_args > 0:
4257            code.putln('const Py_ssize_t kwd_pos_args = %s;' % Naming.nargs_cname)
4258
4259        if max_positional_args == 0:
4260            pos_arg_count = "0"
4261        elif self.star_arg:
4262            # If there is a *arg, the number of used positional args could be larger than
4263            # the number of possible keyword arguments.  But ParseOptionalKeywords() uses the
4264            # number of positional args as an index into the keyword argument name array,
4265            # if this is larger than the number of kwd args we get a segfault.  So round
4266            # this down to max_positional_args - num_pos_only_args (= num possible kwd args).
4267            code.putln("const Py_ssize_t used_pos_args = (kwd_pos_args < %d) ? kwd_pos_args : %d;" % (
4268                max_positional_args - num_pos_only_args, max_positional_args - num_pos_only_args))
4269            pos_arg_count = "used_pos_args"
4270        else:
4271            pos_arg_count = "kwd_pos_args"
4272        if num_pos_only_args < len(all_args):
4273            values_array = 'values + %d' % num_pos_only_args
4274        else:
4275            values_array = 'values'
4276        code.globalstate.use_utility_code(
4277            UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
4278        code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % (
4279            Naming.kwds_cname,
4280            Naming.kwvalues_cname,
4281            Naming.pykwdlist_cname,
4282            self.starstar_arg and self.starstar_arg.entry.cname or '0',
4283            values_array,
4284            pos_arg_count,
4285            self_name_csafe,
4286            code.error_goto(self.pos)))
4287        code.putln('}')
4288
4289    def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
4290        optional_args = []
4291        first_optional_arg = -1
4292        num_posonly_args = 0
4293        for i, arg in enumerate(all_args):
4294            if arg.pos_only:
4295                num_posonly_args += 1
4296            if not arg.kw_only or not arg.default:
4297                continue
4298            if not optional_args:
4299                first_optional_arg = i
4300            optional_args.append(arg.name)
4301        if num_posonly_args > 0:
4302            posonly_correction = '-%d' % num_posonly_args
4303        else:
4304            posonly_correction = ''
4305        if optional_args:
4306            if len(optional_args) > 1:
4307                # if we receive more than the named kwargs, we either have **kwargs
4308                # (in which case we must iterate anyway) or it's an error (which we
4309                # also handle during iteration) => skip this part if there are more
4310                code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
4311                    not self.starstar_arg and 'likely' or '',
4312                    len(optional_args)))
4313                code.putln('Py_ssize_t index;')
4314                # not unrolling the loop here reduces the C code overhead
4315                code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
4316                    first_optional_arg, first_optional_arg + len(optional_args)))
4317            else:
4318                code.putln('if (kw_args == 1) {')
4319                code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
4320            code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, *%s[index%s]);' % (
4321                self.signature.fastvar,
4322                Naming.kwds_cname,
4323                Naming.kwvalues_cname,
4324                Naming.pykwdlist_cname,
4325                posonly_correction))
4326            code.putln('if (value) { values[index] = value; kw_args--; }')
4327            code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos))
4328            if len(optional_args) > 1:
4329                code.putln('}')
4330            code.putln('}')
4331
4332    def generate_argument_conversion_code(self, code):
4333        # Generate code to convert arguments from signature type to
4334        # declared type, if needed.  Also copies signature arguments
4335        # into closure fields.
4336        for arg in self.args:
4337            if arg.needs_conversion:
4338                self.generate_arg_conversion(arg, code)
4339
4340    def generate_arg_conversion(self, arg, code):
4341        # Generate conversion code for one argument.
4342        old_type = arg.hdr_type
4343        new_type = arg.type
4344        if old_type.is_pyobject:
4345            if arg.default:
4346                code.putln("if (%s) {" % arg.hdr_cname)
4347            else:
4348                code.putln("assert(%s); {" % arg.hdr_cname)
4349            self.generate_arg_conversion_from_pyobject(arg, code)
4350            code.putln("}")
4351        elif new_type.is_pyobject:
4352            self.generate_arg_conversion_to_pyobject(arg, code)
4353        else:
4354            if new_type.assignable_from(old_type):
4355                code.putln("%s = %s;" % (arg.entry.cname, arg.hdr_cname))
4356            else:
4357                error(arg.pos, "Cannot convert 1 argument from '%s' to '%s'" % (old_type, new_type))
4358
4359    def generate_arg_conversion_from_pyobject(self, arg, code):
4360        new_type = arg.type
4361        # copied from CoerceFromPyTypeNode
4362        if new_type.from_py_function:
4363            code.putln(new_type.from_py_call_code(
4364                arg.hdr_cname,
4365                arg.entry.cname,
4366                arg.pos,
4367                code,
4368            ))
4369        else:
4370            error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
4371
4372    def generate_arg_conversion_to_pyobject(self, arg, code):
4373        old_type = arg.hdr_type
4374        func = old_type.to_py_function
4375        if func:
4376            code.putln("%s = %s(%s); %s" % (
4377                arg.entry.cname,
4378                func,
4379                arg.hdr_cname,
4380                code.error_goto_if_null(arg.entry.cname, arg.pos)))
4381            code.put_var_gotref(arg.entry)
4382        else:
4383            error(arg.pos, "Cannot convert argument of type '%s' to Python object" % old_type)
4384
4385    def generate_argument_type_tests(self, code):
4386        # Generate type tests for args whose signature
4387        # type is PyObject * and whose declared type is
4388        # a subtype thereof.
4389        for arg in self.args:
4390            if arg.needs_type_test:
4391                self.generate_arg_type_test(arg, code)
4392            elif not arg.accept_none and (arg.type.is_pyobject or
4393                                          arg.type.is_buffer or
4394                                          arg.type.is_memoryviewslice):
4395                self.generate_arg_none_check(arg, code)
4396
4397    def error_value(self):
4398        return self.signature.error_value
4399
4400
4401class GeneratorDefNode(DefNode):
4402    # Generator function node that creates a new generator instance when called.
4403    #
4404    # gbody          GeneratorBodyDefNode   the function implementing the generator
4405    #
4406
4407    is_generator = True
4408    is_iterable_coroutine = False
4409    gen_type_name = 'Generator'
4410    needs_closure = True
4411
4412    child_attrs = DefNode.child_attrs + ["gbody"]
4413
4414    def __init__(self, pos, **kwargs):
4415        # XXX: don't actually needs a body
4416        kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
4417        super(GeneratorDefNode, self).__init__(pos, **kwargs)
4418
4419    def analyse_declarations(self, env):
4420        super(GeneratorDefNode, self).analyse_declarations(env)
4421        self.gbody.local_scope = self.local_scope
4422        self.gbody.analyse_declarations(env)
4423
4424    def generate_function_body(self, env, code):
4425        body_cname = self.gbody.entry.func_cname
4426        name = code.intern_identifier(self.name)
4427        qualname = code.intern_identifier(self.qualname)
4428        module_name = code.intern_identifier(self.module_name)
4429
4430        code.putln('{')
4431        code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
4432                   '(__pyx_coroutine_body_t) %s, %s, (PyObject *) %s, %s, %s, %s); %s' % (
4433                       self.gen_type_name,
4434                       body_cname, self.code_object.calculate_result_code(code) if self.code_object else 'NULL',
4435                       Naming.cur_scope_cname, name, qualname, module_name,
4436                       code.error_goto_if_null('gen', self.pos)))
4437        code.put_decref(Naming.cur_scope_cname, py_object_type)
4438        if self.requires_classobj:
4439            classobj_cname = 'gen->classobj'
4440            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
4441                classobj_cname, Naming.self_cname))
4442            code.put_incref(classobj_cname, py_object_type)
4443            code.put_giveref(classobj_cname, py_object_type)
4444        code.put_finish_refcount_context()
4445        code.putln('return (PyObject *) gen;')
4446        code.putln('}')
4447
4448    def generate_function_definitions(self, env, code):
4449        env.use_utility_code(UtilityCode.load_cached(self.gen_type_name, "Coroutine.c"))
4450        self.gbody.generate_function_header(code, proto=True)
4451        super(GeneratorDefNode, self).generate_function_definitions(env, code)
4452        self.gbody.generate_function_definitions(env, code)
4453
4454
4455class AsyncDefNode(GeneratorDefNode):
4456    gen_type_name = 'Coroutine'
4457    is_coroutine = True
4458
4459
4460class IterableAsyncDefNode(AsyncDefNode):
4461    gen_type_name = 'IterableCoroutine'
4462    is_iterable_coroutine = True
4463
4464
4465class AsyncGenNode(AsyncDefNode):
4466    gen_type_name = 'AsyncGen'
4467    is_asyncgen = True
4468
4469
4470class GeneratorBodyDefNode(DefNode):
4471    # Main code body of a generator implemented as a DefNode.
4472    #
4473
4474    is_generator_body = True
4475    is_inlined = False
4476    is_async_gen_body = False
4477    inlined_comprehension_type = None  # container type for inlined comprehensions
4478
4479    def __init__(self, pos=None, name=None, body=None, is_async_gen_body=False):
4480        super(GeneratorBodyDefNode, self).__init__(
4481            pos=pos, body=body, name=name, is_async_gen_body=is_async_gen_body,
4482            doc=None, args=[], star_arg=None, starstar_arg=None)
4483
4484    def declare_generator_body(self, env):
4485        prefix = env.next_id(env.scope_prefix)
4486        name = env.next_id('generator')
4487        cname = Naming.genbody_prefix + prefix + name
4488        entry = env.declare_var(None, py_object_type, self.pos,
4489                                cname=cname, visibility='private')
4490        entry.func_cname = cname
4491        entry.qualified_name = EncodedString(self.name)
4492        # Work-around for https://github.com/cython/cython/issues/1699
4493        # We don't currently determine whether the generator entry is used or not,
4494        # so mark it as used to avoid false warnings.
4495        entry.used = True
4496        self.entry = entry
4497
4498    def analyse_declarations(self, env):
4499        self.analyse_argument_types(env)
4500        self.declare_generator_body(env)
4501
4502    def generate_function_header(self, code, proto=False):
4503        header = "static PyObject *%s(__pyx_CoroutineObject *%s, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % (
4504            self.entry.func_cname,
4505            Naming.generator_cname,
4506            Naming.local_tstate_cname,
4507            Naming.sent_value_cname)
4508        if proto:
4509            code.putln('%s; /* proto */' % header)
4510        else:
4511            code.putln('%s /* generator body */\n{' % header)
4512
4513    def generate_function_definitions(self, env, code):
4514        lenv = self.local_scope
4515
4516        # Generate closure function definitions
4517        self.body.generate_function_definitions(lenv, code)
4518
4519        # Generate C code for header and body of function
4520        code.enter_cfunc_scope(lenv)
4521        code.return_from_error_cleanup_label = code.new_label()
4522
4523        # ----- Top-level constants used by this function
4524        code.mark_pos(self.pos)
4525        self.generate_cached_builtins_decls(lenv, code)
4526        # ----- Function header
4527        code.putln("")
4528        self.generate_function_header(code)
4529        closure_init_code = code.insertion_point()
4530        # ----- Local variables
4531        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
4532        tempvardecl_code = code.insertion_point()
4533        code.put_declare_refcount_context()
4534        code.put_setup_refcount_context(self.entry.name or self.entry.qualified_name)
4535        profile = code.globalstate.directives['profile']
4536        linetrace = code.globalstate.directives['linetrace']
4537        if profile or linetrace:
4538            tempvardecl_code.put_trace_declarations()
4539            code.funcstate.can_trace = True
4540            code_object = self.code_object.calculate_result_code(code) if self.code_object else None
4541            code.put_trace_frame_init(code_object)
4542
4543        # ----- Resume switch point.
4544        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
4545        resume_code = code.insertion_point()
4546        first_run_label = code.new_label('first_run')
4547        code.use_label(first_run_label)
4548        code.put_label(first_run_label)
4549        code.putln('%s' %
4550                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
4551
4552        # ----- prepare target container for inlined comprehension
4553        if self.is_inlined and self.inlined_comprehension_type is not None:
4554            target_type = self.inlined_comprehension_type
4555            if target_type is Builtin.list_type:
4556                comp_init = 'PyList_New(0)'
4557            elif target_type is Builtin.set_type:
4558                comp_init = 'PySet_New(NULL)'
4559            elif target_type is Builtin.dict_type:
4560                comp_init = 'PyDict_New()'
4561            else:
4562                raise InternalError(
4563                    "invalid type of inlined comprehension: %s" % target_type)
4564            code.putln("%s = %s; %s" % (
4565                Naming.retval_cname, comp_init,
4566                code.error_goto_if_null(Naming.retval_cname, self.pos)))
4567            code.put_gotref(Naming.retval_cname, py_object_type)
4568
4569        # ----- Function body
4570        self.generate_function_body(env, code)
4571        # ----- Closure initialization
4572        if lenv.scope_class.type.scope.var_entries:
4573            closure_init_code.putln('%s = %s;' % (
4574                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
4575                lenv.scope_class.type.cast_code('%s->closure' %
4576                                                Naming.generator_cname)))
4577            # FIXME: this silences a potential "unused" warning => try to avoid unused closures in more cases
4578            code.putln("CYTHON_MAYBE_UNUSED_VAR(%s);" % Naming.cur_scope_cname)
4579
4580        if profile or linetrace:
4581            code.funcstate.can_trace = False
4582
4583        code.mark_pos(self.pos)
4584        code.putln("")
4585        code.putln("/* function exit code */")
4586
4587        # on normal generator termination, we do not take the exception propagation
4588        # path: no traceback info is required and not creating it is much faster
4589        if not self.is_inlined and not self.body.is_terminator:
4590            if self.is_async_gen_body:
4591                code.globalstate.use_utility_code(
4592                    UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
4593            code.putln('PyErr_SetNone(%s);' % (
4594                '__Pyx_PyExc_StopAsyncIteration' if self.is_async_gen_body else 'PyExc_StopIteration'))
4595        # ----- Error cleanup
4596        if code.label_used(code.error_label):
4597            if not self.body.is_terminator:
4598                code.put_goto(code.return_label)
4599            code.put_label(code.error_label)
4600            if self.is_inlined and self.inlined_comprehension_type is not None:
4601                code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4602            if Future.generator_stop in env.global_scope().context.future_directives:
4603                # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
4604                code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
4605                code.putln("__Pyx_Generator_Replace_StopIteration(%d);" % bool(self.is_async_gen_body))
4606            for cname, type in code.funcstate.all_managed_temps():
4607                code.put_xdecref(cname, type)
4608            code.put_add_traceback(self.entry.qualified_name)
4609
4610        # ----- Non-error return cleanup
4611        code.put_label(code.return_label)
4612        if self.is_inlined:
4613            code.put_xgiveref(Naming.retval_cname, py_object_type)
4614        else:
4615            code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4616        # For Py3.7, clearing is already done below.
4617        code.putln("#if !CYTHON_USE_EXC_INFO_STACK")
4618        code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
4619        code.putln("#endif")
4620        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
4621        # clean up as early as possible to help breaking any reference cycles
4622        code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
4623        if profile or linetrace:
4624            code.put_trace_return(Naming.retval_cname,
4625                                  nogil=not code.funcstate.gil_owned)
4626        code.put_finish_refcount_context()
4627        code.putln("return %s;" % Naming.retval_cname)
4628        code.putln("}")
4629
4630        # ----- Go back and insert temp variable declarations
4631        tempvardecl_code.put_temp_declarations(code.funcstate)
4632        # ----- Generator resume code
4633        if profile or linetrace:
4634            resume_code.put_trace_call(self.entry.qualified_name, self.pos,
4635                                       nogil=not code.funcstate.gil_owned)
4636        resume_code.putln("switch (%s->resume_label) {" % (
4637                       Naming.generator_cname))
4638
4639        resume_code.putln("case 0: goto %s;" % first_run_label)
4640
4641        for i, label in code.yield_labels:
4642            resume_code.putln("case %d: goto %s;" % (i, label))
4643        resume_code.putln("default: /* CPython raises the right error here */")
4644        if profile or linetrace:
4645            resume_code.put_trace_return("Py_None",
4646                                         nogil=not code.funcstate.gil_owned)
4647        resume_code.put_finish_refcount_context()
4648        resume_code.putln("return NULL;")
4649        resume_code.putln("}")
4650
4651        code.exit_cfunc_scope()
4652
4653
4654class OverrideCheckNode(StatNode):
4655    # A Node for dispatching to the def method if it
4656    # is overridden.
4657    #
4658    #  py_func
4659    #
4660    #  args
4661    #  func_temp
4662    #  body
4663
4664    child_attrs = ['body']
4665
4666    body = None
4667
4668    def analyse_expressions(self, env):
4669        self.args = env.arg_entries
4670        if self.py_func.is_module_scope:
4671            first_arg = 0
4672        else:
4673            first_arg = 1
4674        from . import ExprNodes
4675        self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
4676        call_node = ExprNodes.SimpleCallNode(
4677            self.pos, function=self.func_node,
4678            args=[ExprNodes.NameNode(self.pos, name=arg.name)
4679                  for arg in self.args[first_arg:]])
4680        if env.return_type.is_void or env.return_type.is_returncode:
4681            self.body = StatListNode(self.pos, stats=[
4682                ExprStatNode(self.pos, expr=call_node),
4683                ReturnStatNode(self.pos, value=None)])
4684        else:
4685            self.body = ReturnStatNode(self.pos, value=call_node)
4686        self.body = self.body.analyse_expressions(env)
4687        return self
4688
4689    def generate_execution_code(self, code):
4690        # For fused functions, look up the dispatch function, not the specialisation.
4691        method_entry = self.py_func.fused_py_func.entry if self.py_func.fused_py_func else self.py_func.entry
4692        interned_attr_cname = code.intern_identifier(method_entry.name)
4693
4694        # Check to see if we are an extension type
4695        if self.py_func.is_module_scope:
4696            self_arg = "((PyObject *)%s)" % Naming.module_cname
4697        else:
4698            self_arg = "((PyObject *)%s)" % self.args[0].cname
4699        code.putln("/* Check if called by wrapper */")
4700        code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
4701        code.putln("/* Check if overridden in Python */")
4702        if self.py_func.is_module_scope:
4703            code.putln("else {")
4704        else:
4705            code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0) || "
4706                       "__Pyx_PyType_HasFeature(Py_TYPE(%s), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % (
4707                self_arg, self_arg))
4708
4709        code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
4710        code.globalstate.use_utility_code(
4711            UtilityCode.load_cached("PyDictVersioning", "ObjectHandling.c"))
4712        # TODO: remove the object dict version check by 'inlining' the getattr implementation for methods.
4713        # This would allow checking the dict versions around _PyType_Lookup() if it returns a descriptor,
4714        # and would (tada!) make this check a pure type based thing instead of supporting only a single
4715        # instance at a time.
4716        code.putln("static PY_UINT64_T %s = __PYX_DICT_VERSION_INIT, %s = __PYX_DICT_VERSION_INIT;" % (
4717            Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
4718        code.putln("if (unlikely(!__Pyx_object_dict_version_matches(%s, %s, %s))) {" % (
4719            self_arg, Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
4720        code.putln("PY_UINT64_T %s = __Pyx_get_tp_dict_version(%s);" % (
4721            Naming.type_dict_guard_temp, self_arg))
4722        code.putln("#endif")
4723
4724        func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
4725        self.func_node.set_cname(func_node_temp)
4726        # need to get attribute manually--scope would return cdef method
4727        code.globalstate.use_utility_code(
4728            UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
4729        err = code.error_goto_if_null(func_node_temp, self.pos)
4730        code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
4731            func_node_temp, self_arg, interned_attr_cname, err))
4732        code.put_gotref(func_node_temp, py_object_type)
4733
4734        is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)(void*)%s)" % (
4735            func_node_temp, method_entry.func_cname)
4736        code.putln("#ifdef __Pyx_CyFunction_USED")
4737        code.putln("if (!__Pyx_IsCyOrPyCFunction(%s)" % func_node_temp)
4738        code.putln("#else")
4739        code.putln("if (!PyCFunction_Check(%s)" % func_node_temp)
4740        code.putln("#endif")
4741        code.putln("        || %s) {" % is_overridden)
4742        self.body.generate_execution_code(code)
4743        code.putln("}")
4744
4745        # NOTE: it's not 100% sure that we catch the exact versions here that were used for the lookup,
4746        # but it is very unlikely that the versions change during lookup, and the type dict safe guard
4747        # should increase the chance of detecting such a case.
4748        code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
4749        code.putln("%s = __Pyx_get_tp_dict_version(%s);" % (
4750            Naming.tp_dict_version_temp, self_arg))
4751        code.putln("%s = __Pyx_get_object_dict_version(%s);" % (
4752            Naming.obj_dict_version_temp, self_arg))
4753        # Safety check that the type dict didn't change during the lookup.  Since CPython looks up the
4754        # attribute (descriptor) first in the type dict and then in the instance dict or through the
4755        # descriptor, the only really far-away lookup when we get here is one in the type dict. So we
4756        # double check the type dict version before and afterwards to guard against later changes of
4757        # the type dict during the lookup process.
4758        code.putln("if (unlikely(%s != %s)) {" % (
4759            Naming.type_dict_guard_temp, Naming.tp_dict_version_temp))
4760        code.putln("%s = %s = __PYX_DICT_VERSION_INIT;" % (
4761            Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
4762        code.putln("}")
4763        code.putln("#endif")
4764
4765        code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
4766        code.funcstate.release_temp(func_node_temp)
4767
4768        code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
4769        code.putln("}")
4770        code.putln("#endif")
4771
4772        code.putln("}")
4773
4774
4775class ClassDefNode(StatNode, BlockNode):
4776    pass
4777
4778
4779class PyClassDefNode(ClassDefNode):
4780    #  A Python class definition.
4781    #
4782    #  name     EncodedString   Name of the class
4783    #  doc      string or None  The class docstring
4784    #  body     StatNode        Attribute definition code
4785    #  entry    Symtab.Entry
4786    #  scope    PyClassScope
4787    #  decorators    [DecoratorNode]        list of decorators or None
4788    #  bases    ExprNode        Expression that evaluates to a tuple of base classes
4789    #
4790    #  The following subnodes are constructed internally:
4791    #
4792    #  doc_node NameNode   '__doc__' name that is made available to the class body
4793    #  dict     DictNode   Class dictionary or Py3 namespace
4794    #  classobj ClassNode  Class object
4795    #  target   NameNode   Variable to assign class object to
4796    #  orig_bases  None or ExprNode  "bases" before transformation by PEP560 __mro_entries__,
4797    #                                used to create the __orig_bases__ attribute
4798
4799    child_attrs = ["doc_node", "body", "dict", "metaclass", "mkw", "bases", "class_result",
4800                   "target", "class_cell", "decorators", "orig_bases"]
4801    decorators = None
4802    class_result = None
4803    is_py3_style_class = False  # Python3 style class (kwargs)
4804    metaclass = None
4805    mkw = None
4806    doc_node = None
4807    orig_bases = None
4808
4809    def __init__(self, pos, name, bases, doc, body, decorators=None,
4810                 keyword_args=None, force_py3_semantics=False):
4811        StatNode.__init__(self, pos)
4812        self.name = name
4813        self.doc = doc
4814        self.body = body
4815        self.decorators = decorators
4816        self.bases = bases
4817        from . import ExprNodes
4818        if self.doc and Options.docstrings:
4819            doc = embed_position(self.pos, self.doc)
4820            doc_node = ExprNodes.StringNode(pos, value=doc)
4821            self.doc_node = ExprNodes.NameNode(name=EncodedString('__doc__'), type=py_object_type, pos=pos)
4822        else:
4823            doc_node = None
4824
4825        allow_py2_metaclass = not force_py3_semantics
4826        if keyword_args:
4827            allow_py2_metaclass = False
4828            self.is_py3_style_class = True
4829            if keyword_args.is_dict_literal:
4830                if keyword_args.key_value_pairs:
4831                    for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
4832                        if item.key.value == 'metaclass':
4833                            if self.metaclass is not None:
4834                                error(item.pos, "keyword argument 'metaclass' passed multiple times")
4835                            # special case: we already know the metaclass,
4836                            # so we don't need to do the "build kwargs,
4837                            # find metaclass" dance at runtime
4838                            self.metaclass = item.value
4839                            del keyword_args.key_value_pairs[i]
4840                    self.mkw = keyword_args
4841                else:
4842                    assert self.metaclass is not None
4843            else:
4844                # MergedDictNode
4845                self.mkw = ExprNodes.ProxyNode(keyword_args)
4846
4847        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
4848            if self.metaclass is None:
4849                if keyword_args and not keyword_args.is_dict_literal:
4850                    # **kwargs may contain 'metaclass' arg
4851                    mkdict = self.mkw
4852                else:
4853                    mkdict = None
4854                if (not mkdict and
4855                        self.bases.is_sequence_constructor and
4856                        not self.bases.args):
4857                    pass  # no base classes => no inherited metaclass
4858                else:
4859                    self.metaclass = ExprNodes.PyClassMetaclassNode(
4860                        pos, class_def_node=self)
4861                needs_metaclass_calculation = False
4862            else:
4863                needs_metaclass_calculation = True
4864
4865            self.dict = ExprNodes.PyClassNamespaceNode(
4866                pos, name=name, doc=doc_node, class_def_node=self)
4867            self.classobj = ExprNodes.Py3ClassNode(
4868                pos, name=name, class_def_node=self, doc=doc_node,
4869                calculate_metaclass=needs_metaclass_calculation,
4870                allow_py2_metaclass=allow_py2_metaclass,
4871                force_type=force_py3_semantics,
4872            )
4873        else:
4874            # no bases, no metaclass => old style class creation
4875            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
4876            self.classobj = ExprNodes.ClassNode(
4877                pos, name=name, class_def_node=self, doc=doc_node)
4878
4879        self.target = ExprNodes.NameNode(pos, name=name)
4880        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
4881
4882    def as_cclass(self):
4883        """
4884        Return this node as if it were declared as an extension class
4885        """
4886        if self.is_py3_style_class:
4887            error(self.classobj.pos, "Python3 style class could not be represented as C class")
4888            return
4889
4890        from . import ExprNodes
4891        return CClassDefNode(self.pos,
4892                             visibility='private',
4893                             module_name=None,
4894                             class_name=self.name,
4895                             bases=self.bases or ExprNodes.TupleNode(self.pos, args=[]),
4896                             decorators=self.decorators,
4897                             body=self.body,
4898                             in_pxd=False,
4899                             doc=self.doc)
4900
4901    def create_scope(self, env):
4902        genv = env
4903        while genv.is_py_class_scope or genv.is_c_class_scope:
4904            genv = genv.outer_scope
4905        cenv = self.scope = PyClassScope(name=self.name, outer_scope=genv)
4906        return cenv
4907
4908    def analyse_declarations(self, env):
4909        unwrapped_class_result = class_result = self.classobj
4910        if self.decorators:
4911            from .ExprNodes import SimpleCallNode
4912            for decorator in self.decorators[::-1]:
4913                class_result = SimpleCallNode(
4914                    decorator.pos,
4915                    function=decorator.decorator,
4916                    args=[class_result])
4917            self.decorators = None
4918        self.class_result = class_result
4919        if self.bases:
4920            self.bases.analyse_declarations(env)
4921        if self.mkw:
4922            self.mkw.analyse_declarations(env)
4923        self.class_result.analyse_declarations(env)
4924        self.target.analyse_target_declaration(env)
4925        cenv = self.create_scope(env)
4926        cenv.directives = env.directives
4927        cenv.class_obj_cname = self.target.entry.cname
4928        if self.doc_node:
4929            self.doc_node.analyse_target_declaration(cenv)
4930        self.body.analyse_declarations(cenv)
4931        unwrapped_class_result.analyse_annotations(cenv)
4932
4933    update_bases_functype = PyrexTypes.CFuncType(
4934        PyrexTypes.py_object_type, [
4935            PyrexTypes.CFuncTypeArg("bases",  PyrexTypes.py_object_type, None)
4936        ])
4937
4938    def analyse_expressions(self, env):
4939        if self.bases and not (self.bases.is_sequence_constructor and len(self.bases.args) == 0):
4940            from .ExprNodes import PythonCapiCallNode, CloneNode
4941            # handle the Python 3.7 __mro_entries__ transformation
4942            orig_bases = self.bases.analyse_expressions(env)
4943            self.bases = PythonCapiCallNode(orig_bases.pos,
4944                function_name="__Pyx_PEP560_update_bases",
4945                func_type=self.update_bases_functype,
4946                utility_code=UtilityCode.load_cached('Py3UpdateBases', 'ObjectHandling.c'),
4947                args=[CloneNode(orig_bases)])
4948            self.orig_bases = orig_bases
4949        if self.bases:
4950            self.bases = self.bases.analyse_expressions(env)
4951        if self.mkw:
4952            self.mkw = self.mkw.analyse_expressions(env)
4953        if self.metaclass:
4954            self.metaclass = self.metaclass.analyse_expressions(env)
4955        self.dict = self.dict.analyse_expressions(env)
4956        self.class_result = self.class_result.analyse_expressions(env)
4957        cenv = self.scope
4958        self.body = self.body.analyse_expressions(cenv)
4959        self.target = self.target.analyse_target_expression(env, self.classobj)
4960        self.class_cell = self.class_cell.analyse_expressions(cenv)
4961        return self
4962
4963    def generate_function_definitions(self, env, code):
4964        self.generate_lambda_definitions(self.scope, code)
4965        self.body.generate_function_definitions(self.scope, code)
4966
4967    def generate_execution_code(self, code):
4968        code.mark_pos(self.pos)
4969        code.pyclass_stack.append(self)
4970        cenv = self.scope
4971        if self.orig_bases:
4972            self.orig_bases.generate_evaluation_code(code)
4973        if self.bases:
4974            self.bases.generate_evaluation_code(code)
4975        if self.mkw:
4976            self.mkw.generate_evaluation_code(code)
4977        if self.metaclass:
4978            self.metaclass.generate_evaluation_code(code)
4979        self.dict.generate_evaluation_code(code)
4980        if self.orig_bases:
4981            # update __orig_bases__ if needed
4982            code.putln("if (%s != %s) {" % (self.bases.result(), self.orig_bases.result()))
4983            code.putln(
4984                code.error_goto_if_neg('PyDict_SetItemString(%s, "__orig_bases__", %s)' % (
4985                    self.dict.result(), self.orig_bases.result()),
4986                    self.pos
4987            ))
4988            code.putln("}")
4989            self.orig_bases.generate_disposal_code(code)
4990            self.orig_bases.free_temps(code)
4991        cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
4992
4993        class_cell = self.class_cell
4994        if class_cell is not None and not class_cell.is_active:
4995            class_cell = None
4996
4997        if class_cell is not None:
4998            class_cell.generate_evaluation_code(code)
4999        self.body.generate_execution_code(code)
5000        self.class_result.generate_evaluation_code(code)
5001        if class_cell is not None:
5002            class_cell.generate_injection_code(
5003                code, self.class_result.result())
5004        if class_cell is not None:
5005            class_cell.generate_disposal_code(code)
5006            class_cell.free_temps(code)
5007
5008        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
5009        self.target.generate_assignment_code(self.class_result, code)
5010        self.dict.generate_disposal_code(code)
5011        self.dict.free_temps(code)
5012        if self.metaclass:
5013            self.metaclass.generate_disposal_code(code)
5014            self.metaclass.free_temps(code)
5015        if self.mkw:
5016            self.mkw.generate_disposal_code(code)
5017            self.mkw.free_temps(code)
5018        if self.bases:
5019            self.bases.generate_disposal_code(code)
5020            self.bases.free_temps(code)
5021        code.pyclass_stack.pop()
5022
5023
5024class CClassDefNode(ClassDefNode):
5025    #  An extension type definition.
5026    #
5027    #  visibility         'private' or 'public' or 'extern'
5028    #  typedef_flag       boolean
5029    #  api                boolean
5030    #  module_name        string or None    For import of extern type objects
5031    #  class_name         string            Unqualified name of class
5032    #  as_name            string or None    Name to declare as in this scope
5033    #  bases              TupleNode         Base class(es)
5034    #  objstruct_name     string or None    Specified C name of object struct
5035    #  typeobj_name       string or None    Specified C name of type object
5036    #  check_size         'warn', 'error', 'ignore'     What to do if tp_basicsize does not match
5037    #  in_pxd             boolean           Is in a .pxd file
5038    #  decorators         [DecoratorNode]   list of decorators or None
5039    #  doc                string or None
5040    #  body               StatNode or None
5041    #  entry              Symtab.Entry
5042    #  base_type          PyExtensionType or None
5043    #  buffer_defaults_node DictNode or None Declares defaults for a buffer
5044    #  buffer_defaults_pos
5045
5046    child_attrs = ["body"]
5047    buffer_defaults_node = None
5048    buffer_defaults_pos = None
5049    typedef_flag = False
5050    api = False
5051    objstruct_name = None
5052    typeobj_name = None
5053    check_size = None
5054    decorators = None
5055    shadow = False
5056
5057    @property
5058    def punycode_class_name(self):
5059        return punycodify_name(self.class_name)
5060
5061    def buffer_defaults(self, env):
5062        if not hasattr(self, '_buffer_defaults'):
5063            from . import Buffer
5064            if self.buffer_defaults_node:
5065                self._buffer_defaults = Buffer.analyse_buffer_options(
5066                    self.buffer_defaults_pos,
5067                    env, [], self.buffer_defaults_node,
5068                    need_complete=False)
5069            else:
5070                self._buffer_defaults = None
5071        return self._buffer_defaults
5072
5073    def declare(self, env):
5074        if self.module_name and self.visibility != 'extern':
5075            module_path = self.module_name.split(".")
5076            home_scope = env.find_imported_module(module_path, self.pos)
5077            if not home_scope:
5078                return None
5079        else:
5080            home_scope = env
5081
5082        self.entry = home_scope.declare_c_class(
5083            name=self.class_name,
5084            pos=self.pos,
5085            defining=0,
5086            implementing=0,
5087            module_name=self.module_name,
5088            base_type=None,
5089            objstruct_cname=self.objstruct_name,
5090            typeobj_cname=self.typeobj_name,
5091            visibility=self.visibility,
5092            typedef_flag=self.typedef_flag,
5093            check_size = self.check_size,
5094            api=self.api,
5095            buffer_defaults=self.buffer_defaults(env),
5096            shadow=self.shadow)
5097
5098    def analyse_declarations(self, env):
5099        #print "CClassDefNode.analyse_declarations:", self.class_name
5100        #print "...visibility =", self.visibility
5101        #print "...module_name =", self.module_name
5102
5103        if env.in_cinclude and not self.objstruct_name:
5104            error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
5105        if self.decorators:
5106            error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
5107        self.base_type = None
5108        # Now that module imports are cached, we need to
5109        # import the modules for extern classes.
5110        if self.module_name:
5111            self.module = None
5112            for module in env.cimported_modules:
5113                if module.name == self.module_name:
5114                    self.module = module
5115            if self.module is None:
5116                self.module = ModuleScope(self.module_name, None, env.context)
5117                self.module.has_extern_class = 1
5118                env.add_imported_module(self.module)
5119
5120        if self.bases.args:
5121            base = self.bases.args[0]
5122            base_type = base.analyse_as_type(env)
5123            if base_type in (PyrexTypes.c_int_type, PyrexTypes.c_long_type, PyrexTypes.c_float_type):
5124                # Use the Python rather than C variant of these types.
5125                base_type = env.lookup(base_type.sign_and_name()).type
5126            if base_type is None:
5127                error(base.pos, "First base of '%s' is not an extension type" % self.class_name)
5128            elif base_type == PyrexTypes.py_object_type:
5129                base_class_scope = None
5130            elif not base_type.is_extension_type and \
5131                     not (base_type.is_builtin_type and base_type.objstruct_cname):
5132                error(base.pos, "'%s' is not an extension type" % base_type)
5133            elif not base_type.is_complete():
5134                error(base.pos, "Base class '%s' of type '%s' is incomplete" % (
5135                    base_type.name, self.class_name))
5136            elif base_type.scope and base_type.scope.directives and \
5137                     base_type.is_final_type:
5138                error(base.pos, "Base class '%s' of type '%s' is final" % (
5139                    base_type, self.class_name))
5140            elif base_type.is_builtin_type and \
5141                     base_type.name in ('tuple', 'str', 'bytes'):
5142                error(base.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
5143                      % base_type.name)
5144            else:
5145                self.base_type = base_type
5146            if env.directives.get('freelist', 0) > 0 and base_type != PyrexTypes.py_object_type:
5147                warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
5148
5149        has_body = self.body is not None
5150        if has_body and self.base_type and not self.base_type.scope:
5151            # To properly initialize inherited attributes, the base type must
5152            # be analysed before this type.
5153            self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
5154            return
5155
5156        if self.module_name and self.visibility != 'extern':
5157            module_path = self.module_name.split(".")
5158            home_scope = env.find_imported_module(module_path, self.pos)
5159            if not home_scope:
5160                return
5161        else:
5162            home_scope = env
5163
5164        if self.visibility == 'extern':
5165            if (self.module_name == '__builtin__' and
5166                    self.class_name in Builtin.builtin_types and
5167                    env.qualified_name[:8] != 'cpython.'):  # allow overloaded names for cimporting from cpython
5168                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
5169
5170        self.entry = home_scope.declare_c_class(
5171            name=self.class_name,
5172            pos=self.pos,
5173            defining=has_body and self.in_pxd,
5174            implementing=has_body and not self.in_pxd,
5175            module_name=self.module_name,
5176            base_type=self.base_type,
5177            objstruct_cname=self.objstruct_name,
5178            typeobj_cname=self.typeobj_name,
5179            check_size=self.check_size,
5180            visibility=self.visibility,
5181            typedef_flag=self.typedef_flag,
5182            api=self.api,
5183            buffer_defaults=self.buffer_defaults(env),
5184            shadow=self.shadow)
5185
5186        if self.shadow:
5187            home_scope.lookup(self.class_name).as_variable = self.entry
5188        if home_scope is not env and self.visibility == 'extern':
5189            env.add_imported_entry(self.class_name, self.entry, self.pos)
5190        self.scope = scope = self.entry.type.scope
5191        if scope is not None:
5192            scope.directives = env.directives
5193
5194        if self.doc and Options.docstrings:
5195            scope.doc = embed_position(self.pos, self.doc)
5196
5197        if has_body:
5198            self.body.analyse_declarations(scope)
5199            dict_entry = self.scope.lookup_here("__dict__")
5200            if dict_entry and dict_entry.is_variable and (not scope.defined and not scope.implemented):
5201                dict_entry.getter_cname = self.scope.mangle_internal("__dict__getter")
5202                self.scope.declare_property("__dict__", dict_entry.doc, dict_entry.pos)
5203            if self.in_pxd:
5204                scope.defined = 1
5205            else:
5206                scope.implemented = 1
5207
5208        if len(self.bases.args) > 1:
5209            if not has_body or self.in_pxd:
5210                error(self.bases.args[1].pos, "Only declare first base in declaration.")
5211            # At runtime, we check that the other bases are heap types
5212            # and that a __dict__ is added if required.
5213            for other_base in self.bases.args[1:]:
5214                if other_base.analyse_as_type(env):
5215                    error(other_base.pos, "Only one extension type base class allowed.")
5216            self.entry.type.early_init = 0
5217            from . import ExprNodes
5218            self.type_init_args = ExprNodes.TupleNode(
5219                self.pos,
5220                args=[ExprNodes.IdentifierStringNode(self.pos, value=self.class_name),
5221                      self.bases,
5222                      ExprNodes.DictNode(self.pos, key_value_pairs=[])])
5223        elif self.base_type:
5224            self.entry.type.early_init = self.base_type.is_external or self.base_type.early_init
5225            self.type_init_args = None
5226        else:
5227            self.entry.type.early_init = 1
5228            self.type_init_args = None
5229
5230        env.allocate_vtable_names(self.entry)
5231
5232        for thunk in self.entry.type.defered_declarations:
5233            thunk()
5234
5235    def analyse_expressions(self, env):
5236        if self.body:
5237            scope = self.entry.type.scope
5238            self.body = self.body.analyse_expressions(scope)
5239        if self.type_init_args:
5240            self.type_init_args.analyse_expressions(env)
5241        return self
5242
5243    def generate_function_definitions(self, env, code):
5244        if self.body:
5245            self.generate_lambda_definitions(self.scope, code)
5246            self.body.generate_function_definitions(self.scope, code)
5247
5248    def generate_execution_code(self, code):
5249        # This is needed to generate evaluation code for
5250        # default values of method arguments.
5251        code.mark_pos(self.pos)
5252        if not self.entry.type.early_init:
5253            bases = None
5254            if self.type_init_args:
5255                # Extract bases tuple and validate 'best base' by actually calling 'type()'.
5256                bases = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
5257
5258                self.type_init_args.generate_evaluation_code(code)
5259                code.putln("%s = PyTuple_GET_ITEM(%s, 1);" % (bases, self.type_init_args.result()))
5260                code.put_incref(bases, PyrexTypes.py_object_type)
5261
5262                first_base = "((PyTypeObject*)PyTuple_GET_ITEM(%s, 0))" % bases
5263                # Let Python do the base types compatibility checking.
5264                trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
5265                code.putln("%s = PyType_Type.tp_new(&PyType_Type, %s, NULL);" % (
5266                    trial_type, self.type_init_args.result()))
5267                code.putln(code.error_goto_if_null(trial_type, self.pos))
5268                code.put_gotref(trial_type, py_object_type)
5269                code.putln("if (((PyTypeObject*) %s)->tp_base != %s) {" % (
5270                    trial_type, first_base))
5271                code.putln("__Pyx_TypeName base_name = __Pyx_PyType_GetName(((PyTypeObject*) %s)->tp_base);" % trial_type)
5272                code.putln("__Pyx_TypeName type_name = __Pyx_PyType_GetName(%s);" % first_base)
5273                code.putln("PyErr_Format(PyExc_TypeError, "
5274                    "\"best base '\" __Pyx_FMT_TYPENAME \"' must be equal to first base '\" __Pyx_FMT_TYPENAME \"'\",")
5275                code.putln("             base_name, type_name);")
5276                code.putln("__Pyx_DECREF_TypeName(base_name);")
5277                code.putln("__Pyx_DECREF_TypeName(type_name);")
5278                code.putln(code.error_goto(self.pos))
5279                code.putln("}")
5280
5281                code.put_decref_clear(trial_type, PyrexTypes.py_object_type)
5282                code.funcstate.release_temp(trial_type)
5283
5284                self.type_init_args.generate_disposal_code(code)
5285                self.type_init_args.free_temps(code)
5286
5287            self.generate_type_ready_code(self.entry, code, bases_tuple_cname=bases, check_heap_type_bases=True)
5288            if bases is not None:
5289                code.put_decref_clear(bases, PyrexTypes.py_object_type)
5290                code.funcstate.release_temp(bases)
5291
5292        if self.body:
5293            self.body.generate_execution_code(code)
5294
5295    # Also called from ModuleNode for early init types.
5296    @staticmethod
5297    def generate_type_ready_code(entry, code, bases_tuple_cname=None, check_heap_type_bases=False):
5298        # Generate a call to PyType_Ready for an extension
5299        # type defined in this module.
5300        type = entry.type
5301        typeptr_cname = type.typeptr_cname
5302        scope = type.scope
5303        if not scope:  # could be None if there was an error
5304            return
5305        if entry.visibility == 'extern':
5306            # Generate code to initialise the typeptr of an external extension
5307            # type defined in this module to point to its type object.
5308            if type.typeobj_cname:
5309                # FIXME: this should not normally be set :-?
5310                assert not type.typeobj_cname
5311                code.putln("%s = &%s;" % (
5312                    type.typeptr_cname,
5313                    type.typeobj_cname,
5314                ))
5315            return
5316        # TODO: remove 'else:' and dedent
5317        else:
5318            assert typeptr_cname
5319            assert type.typeobj_cname
5320            typespec_cname = "%s_spec" % type.typeobj_cname
5321            code.putln("#if CYTHON_USE_TYPE_SPECS")
5322            tuple_temp = None
5323            if not bases_tuple_cname and scope.parent_type.base_type:
5324                tuple_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
5325                code.putln("%s = PyTuple_Pack(1, (PyObject *)%s); %s" % (
5326                    tuple_temp,
5327                    scope.parent_type.base_type.typeptr_cname,
5328                    code.error_goto_if_null(tuple_temp, entry.pos),
5329                ))
5330                code.put_gotref(tuple_temp, py_object_type)
5331
5332            if bases_tuple_cname or tuple_temp:
5333                if check_heap_type_bases:
5334                    code.globalstate.use_utility_code(
5335                        UtilityCode.load_cached('ValidateBasesTuple', 'ExtensionTypes.c'))
5336                    code.put_error_if_neg(entry.pos, "__Pyx_validate_bases_tuple(%s.name, %s, %s)" % (
5337                        typespec_cname,
5338                        TypeSlots.get_slot_by_name("tp_dictoffset").slot_code(scope),
5339                        bases_tuple_cname or tuple_temp,
5340                    ))
5341
5342                code.putln("%s = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(%s, &%s, %s);" % (
5343                    typeptr_cname,
5344                    Naming.module_cname,
5345                    typespec_cname,
5346                    bases_tuple_cname or tuple_temp,
5347                ))
5348                if tuple_temp:
5349                    code.put_xdecref_clear(tuple_temp, type=py_object_type)
5350                    code.funcstate.release_temp(tuple_temp)
5351                code.putln(code.error_goto_if_null(typeptr_cname, entry.pos))
5352            else:
5353                code.putln(
5354                    "%s = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(%s, &%s, NULL); %s" % (
5355                        typeptr_cname,
5356                        Naming.module_cname,
5357                        typespec_cname,
5358                        code.error_goto_if_null(typeptr_cname, entry.pos),
5359                    ))
5360
5361            # The buffer interface is not currently supported by PyType_FromSpec().
5362            buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer")
5363            if not buffer_slot.is_empty(scope):
5364                code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
5365                code.putln("%s->%s = %s;" % (
5366                    typeptr_cname,
5367                    buffer_slot.slot_name,
5368                    buffer_slot.slot_code(scope),
5369                ))
5370                # Still need to inherit buffer methods since PyType_Ready() didn't do it for us.
5371                for buffer_method_name in ("__getbuffer__", "__releasebuffer__"):
5372                    buffer_slot = TypeSlots.get_slot_by_method_name(buffer_method_name)
5373                    if buffer_slot.slot_code(scope) == "0" and not TypeSlots.get_base_slot_function(scope, buffer_slot):
5374                        code.putln("if (!%s->tp_as_buffer->%s &&"
5375                                   " %s->tp_base->tp_as_buffer &&"
5376                                   " %s->tp_base->tp_as_buffer->%s) {" % (
5377                            typeptr_cname, buffer_slot.slot_name,
5378                            typeptr_cname,
5379                            typeptr_cname, buffer_slot.slot_name,
5380                        ))
5381                        code.putln("%s->tp_as_buffer->%s = %s->tp_base->tp_as_buffer->%s;" % (
5382                            typeptr_cname, buffer_slot.slot_name,
5383                            typeptr_cname, buffer_slot.slot_name,
5384                        ))
5385                        code.putln("}")
5386                code.putln("#else")
5387                code.putln("#warning The buffer protocol is not supported in the Limited C-API.")
5388                code.putln("#endif")
5389
5390            code.globalstate.use_utility_code(
5391                UtilityCode.load_cached("FixUpExtensionType", "ExtensionTypes.c"))
5392            code.put_error_if_neg(entry.pos, "__Pyx_fix_up_extension_type_from_spec(&%s, %s)" % (
5393                typespec_cname, typeptr_cname))
5394
5395            code.putln("#else")
5396            if bases_tuple_cname:
5397                code.put_incref(bases_tuple_cname, py_object_type)
5398                code.put_giveref(bases_tuple_cname, py_object_type)
5399                code.putln("%s.tp_bases = %s;" % (type.typeobj_cname, bases_tuple_cname))
5400            code.putln("%s = &%s;" % (
5401                typeptr_cname,
5402                type.typeobj_cname,
5403            ))
5404            code.putln("#endif")  # if CYTHON_USE_TYPE_SPECS
5405
5406            code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
5407            # FIXME: these still need to get initialised even with the limited-API
5408            for slot in TypeSlots.slot_table:
5409                slot.generate_dynamic_init_code(scope, code)
5410            code.putln("#endif")
5411
5412            code.putln("#if !CYTHON_USE_TYPE_SPECS")
5413            code.globalstate.use_utility_code(
5414                UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c'))
5415            code.put_error_if_neg(entry.pos, "__Pyx_PyType_Ready(%s)" % typeptr_cname)
5416            code.putln("#endif")
5417
5418            # Don't inherit tp_print from builtin types in Python 2, restoring the
5419            # behavior of using tp_repr or tp_str instead.
5420            # ("tp_print" was renamed to "tp_vectorcall_offset" in Py3.8b1)
5421            code.putln("#if PY_MAJOR_VERSION < 3")
5422            code.putln("%s->tp_print = 0;" % typeptr_cname)
5423            code.putln("#endif")
5424
5425            # Use specialised attribute lookup for types with generic lookup but no instance dict.
5426            getattr_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_getattro')
5427            dictoffset_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_dictoffset')
5428            if getattr_slot_func == '0' and dictoffset_slot_func == '0':
5429                code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")  # FIXME
5430                if type.is_final_type:
5431                    py_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict"  # grepable
5432                    utility_func = "PyObject_GenericGetAttrNoDict"
5433                else:
5434                    py_cfunc = "__Pyx_PyObject_GenericGetAttr"
5435                    utility_func = "PyObject_GenericGetAttr"
5436                code.globalstate.use_utility_code(UtilityCode.load_cached(utility_func, "ObjectHandling.c"))
5437
5438                code.putln("if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) &&"
5439                           " likely(!%s->tp_dictoffset && %s->tp_getattro == PyObject_GenericGetAttr)) {" % (
5440                    typeptr_cname, typeptr_cname))
5441                code.putln("%s->tp_getattro = %s;" % (
5442                    typeptr_cname, py_cfunc))
5443                code.putln("}")
5444                code.putln("#endif")  # if !CYTHON_COMPILING_IN_LIMITED_API
5445
5446            # Fix special method docstrings. This is a bit of a hack, but
5447            # unless we let PyType_Ready create the slot wrappers we have
5448            # a significant performance hit. (See trac #561.)
5449            for func in entry.type.scope.pyfunc_entries:
5450                is_buffer = func.name in ('__getbuffer__', '__releasebuffer__')
5451                if (func.is_special and Options.docstrings and
5452                        func.wrapperbase_cname and not is_buffer):
5453                    slot = TypeSlots.method_name_to_slot.get(func.name)
5454                    preprocessor_guard = slot.preprocessor_guard_code() if slot else None
5455                    if preprocessor_guard:
5456                        code.putln(preprocessor_guard)
5457                    code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
5458                    code.putln("{")
5459                    code.putln(
5460                        'PyObject *wrapper = PyObject_GetAttrString((PyObject *)%s, "%s"); %s' % (
5461                            typeptr_cname,
5462                            func.name,
5463                            code.error_goto_if_null('wrapper', entry.pos)))
5464                    code.putln(
5465                        "if (__Pyx_IS_TYPE(wrapper, &PyWrapperDescr_Type)) {")
5466                    code.putln(
5467                        "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
5468                            func.wrapperbase_cname))
5469                    code.putln(
5470                        "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
5471                    code.putln(
5472                        "((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
5473                            func.wrapperbase_cname))
5474                    code.putln("}")
5475                    code.putln("}")
5476                    code.putln('#endif')
5477                    if preprocessor_guard:
5478                        code.putln('#endif')
5479
5480            if type.vtable_cname:
5481                code.globalstate.use_utility_code(
5482                    UtilityCode.load_cached('SetVTable', 'ImportExport.c'))
5483                code.put_error_if_neg(entry.pos, "__Pyx_SetVtable(%s, %s)" % (
5484                    typeptr_cname,
5485                    type.vtabptr_cname,
5486                ))
5487                # TODO: find a way to make this work with the Limited API!
5488                code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
5489                code.globalstate.use_utility_code(
5490                    UtilityCode.load_cached('MergeVTables', 'ImportExport.c'))
5491                code.put_error_if_neg(entry.pos, "__Pyx_MergeVtables(%s)" % typeptr_cname)
5492                code.putln("#endif")
5493            if not type.scope.is_internal and not type.scope.directives.get('internal'):
5494                # scope.is_internal is set for types defined by
5495                # Cython (such as closures), the 'internal'
5496                # directive is set by users
5497                code.put_error_if_neg(entry.pos, "PyObject_SetAttr(%s, %s, (PyObject *) %s)" % (
5498                    Naming.module_cname,
5499                    code.intern_identifier(scope.class_name),
5500                    typeptr_cname,
5501                ))
5502
5503            weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
5504            if weakref_entry:
5505                if weakref_entry.type is py_object_type:
5506                    tp_weaklistoffset = "%s->tp_weaklistoffset" % typeptr_cname
5507                    if type.typedef_flag:
5508                        objstruct = type.objstruct_cname
5509                    else:
5510                        objstruct = "struct %s" % type.objstruct_cname
5511                    code.putln("if (%s == 0) %s = offsetof(%s, %s);" % (
5512                        tp_weaklistoffset,
5513                        tp_weaklistoffset,
5514                        objstruct,
5515                        weakref_entry.cname))
5516                else:
5517                    error(weakref_entry.pos, "__weakref__ slot must be of type 'object'")
5518
5519            if scope.lookup_here("__reduce_cython__") if not scope.is_closure_class_scope else None:
5520                # Unfortunately, we cannot reliably detect whether a
5521                # superclass defined __reduce__ at compile time, so we must
5522                # do so at runtime.
5523                code.globalstate.use_utility_code(
5524                    UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c'))
5525                code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")  # FIXME
5526                code.put_error_if_neg(entry.pos, "__Pyx_setup_reduce((PyObject *) %s)" % typeptr_cname)
5527                code.putln("#endif")
5528
5529    def annotate(self, code):
5530        if self.type_init_args:
5531            self.type_init_args.annotate(code)
5532        if self.body:
5533            self.body.annotate(code)
5534
5535
5536class PropertyNode(StatNode):
5537    #  Definition of a property in an extension type.
5538    #
5539    #  name   string
5540    #  doc    EncodedString or None    Doc string
5541    #  entry  Symtab.Entry             The Entry of the property attribute
5542    #  body   StatListNode
5543
5544    child_attrs = ["body"]
5545
5546    def analyse_declarations(self, env):
5547        self.entry = env.declare_property(self.name, self.doc, self.pos)
5548        self.body.analyse_declarations(self.entry.scope)
5549
5550    def analyse_expressions(self, env):
5551        self.body = self.body.analyse_expressions(env)
5552        return self
5553
5554    def generate_function_definitions(self, env, code):
5555        self.body.generate_function_definitions(env, code)
5556
5557    def generate_execution_code(self, code):
5558        pass
5559
5560    def annotate(self, code):
5561        self.body.annotate(code)
5562
5563
5564class CPropertyNode(StatNode):
5565    """Definition of a C property, backed by a CFuncDefNode getter.
5566    """
5567    #  name   string
5568    #  doc    EncodedString or None        Doc string of the property
5569    #  entry  Symtab.Entry                 The Entry of the property attribute
5570    #  body   StatListNode[CFuncDefNode]   (for compatibility with PropertyNode)
5571
5572    child_attrs = ["body"]
5573    is_cproperty = True
5574
5575    @property
5576    def cfunc(self):
5577        stats = self.body.stats
5578        assert stats and isinstance(stats[0], CFuncDefNode), stats
5579        return stats[0]
5580
5581    def analyse_declarations(self, env):
5582        scope = PropertyScope(self.name, class_scope=env)
5583        self.body.analyse_declarations(scope)
5584        entry = self.entry = env.declare_property(
5585            self.name, self.doc, self.pos, ctype=self.cfunc.return_type, property_scope=scope)
5586        entry.getter_cname = self.cfunc.entry.cname
5587
5588    def analyse_expressions(self, env):
5589        self.body = self.body.analyse_expressions(env)
5590        return self
5591
5592    def generate_function_definitions(self, env, code):
5593        self.body.generate_function_definitions(env, code)
5594
5595    def generate_execution_code(self, code):
5596        pass
5597
5598    def annotate(self, code):
5599        self.body.annotate(code)
5600
5601
5602class GlobalNode(StatNode):
5603    # Global variable declaration.
5604    #
5605    # names    [string]
5606
5607    child_attrs = []
5608
5609    def analyse_declarations(self, env):
5610        for name in self.names:
5611            env.declare_global(name, self.pos)
5612
5613    def analyse_expressions(self, env):
5614        return self
5615
5616    def generate_execution_code(self, code):
5617        pass
5618
5619
5620class NonlocalNode(StatNode):
5621    # Nonlocal variable declaration via the 'nonlocal' keyword.
5622    #
5623    # names    [string]
5624
5625    child_attrs = []
5626
5627    def analyse_declarations(self, env):
5628        for name in self.names:
5629            env.declare_nonlocal(name, self.pos)
5630
5631    def analyse_expressions(self, env):
5632        return self
5633
5634    def generate_execution_code(self, code):
5635        pass
5636
5637
5638class ExprStatNode(StatNode):
5639    #  Expression used as a statement.
5640    #
5641    #  expr   ExprNode
5642
5643    child_attrs = ["expr"]
5644
5645    def analyse_declarations(self, env):
5646        from . import ExprNodes
5647        expr = self.expr
5648        if isinstance(expr, ExprNodes.GeneralCallNode):
5649            func = expr.function.as_cython_attribute()
5650            if func == u'declare':
5651                args, kwds = expr.explicit_args_kwds()
5652                if len(args):
5653                    error(expr.pos, "Variable names must be specified.")
5654                for var, type_node in kwds.key_value_pairs:
5655                    type = type_node.analyse_as_type(env)
5656                    if type is None:
5657                        error(type_node.pos, "Unknown type")
5658                    else:
5659                        env.declare_var(var.value, type, var.pos, is_cdef=True)
5660                self.__class__ = PassStatNode
5661        elif getattr(expr, 'annotation', None) is not None:
5662            if expr.is_name:
5663                # non-code variable annotation, e.g. "name: type"
5664                expr.declare_from_annotation(env)
5665                self.__class__ = PassStatNode
5666            elif expr.is_attribute or expr.is_subscript:
5667                # unused expression with annotation, e.g. "a[0]: type" or "a.xyz : type"
5668                self.__class__ = PassStatNode
5669
5670    def analyse_expressions(self, env):
5671        self.expr.result_is_used = False  # hint that .result() may safely be left empty
5672        self.expr = self.expr.analyse_expressions(env)
5673        # Repeat in case of node replacement.
5674        self.expr.result_is_used = False  # hint that .result() may safely be left empty
5675        return self
5676
5677    def nogil_check(self, env):
5678        if self.expr.type.is_pyobject and self.expr.is_temp:
5679            self.gil_error()
5680
5681    gil_message = "Discarding owned Python object"
5682
5683    def generate_execution_code(self, code):
5684        code.mark_pos(self.pos)
5685        self.expr.result_is_used = False  # hint that .result() may safely be left empty
5686        self.expr.generate_evaluation_code(code)
5687        if not self.expr.is_temp and self.expr.result():
5688            result = self.expr.result()
5689            if not self.expr.type.is_void:
5690                result = "(void)(%s)" % result
5691            code.putln("%s;" % result)
5692        self.expr.generate_disposal_code(code)
5693        self.expr.free_temps(code)
5694
5695    def generate_function_definitions(self, env, code):
5696        self.expr.generate_function_definitions(env, code)
5697
5698    def annotate(self, code):
5699        self.expr.annotate(code)
5700
5701
5702class AssignmentNode(StatNode):
5703    #  Abstract base class for assignment nodes.
5704    #
5705    #  The analyse_expressions and generate_execution_code
5706    #  phases of assignments are split into two sub-phases
5707    #  each, to enable all the right hand sides of a
5708    #  parallel assignment to be evaluated before assigning
5709    #  to any of the left hand sides.
5710
5711    def analyse_expressions(self, env):
5712        node = self.analyse_types(env)
5713        if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
5714            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
5715                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
5716        return node
5717
5718#       def analyse_expressions(self, env):
5719#           self.analyse_expressions_1(env)
5720#           self.analyse_expressions_2(env)
5721
5722    def generate_execution_code(self, code):
5723        code.mark_pos(self.pos)
5724        self.generate_rhs_evaluation_code(code)
5725        self.generate_assignment_code(code)
5726
5727
5728class SingleAssignmentNode(AssignmentNode):
5729    #  The simplest case:
5730    #
5731    #    a = b
5732    #
5733    #  lhs                      ExprNode      Left hand side
5734    #  rhs                      ExprNode      Right hand side
5735    #  first                    bool          Is this guaranteed the first assignment to lhs?
5736    #  is_overloaded_assignment bool          Is this assignment done via an overloaded operator=
5737    #  exception_check
5738    #  exception_value
5739
5740    child_attrs = ["lhs", "rhs"]
5741    first = False
5742    is_overloaded_assignment = False
5743    declaration_only = False
5744
5745    def analyse_declarations(self, env):
5746        from . import ExprNodes
5747
5748        # handle declarations of the form x = cython.foo()
5749        if isinstance(self.rhs, ExprNodes.CallNode):
5750            func_name = self.rhs.function.as_cython_attribute()
5751            if func_name:
5752                args, kwds = self.rhs.explicit_args_kwds()
5753                if func_name in ['declare', 'typedef']:
5754                    if len(args) > 2:
5755                        error(args[2].pos, "Invalid positional argument.")
5756                        return
5757                    if kwds is not None:
5758                        kwdict = kwds.compile_time_value(None)
5759                        if func_name == 'typedef' or 'visibility' not in kwdict:
5760                            error(kwds.pos, "Invalid keyword argument.")
5761                            return
5762                        visibility = kwdict['visibility']
5763                    else:
5764                        visibility = 'private'
5765                    type = args[0].analyse_as_type(env)
5766                    if type is None:
5767                        error(args[0].pos, "Unknown type")
5768                        return
5769                    lhs = self.lhs
5770                    if func_name == 'declare':
5771                        if isinstance(lhs, ExprNodes.NameNode):
5772                            vars = [(lhs.name, lhs.pos)]
5773                        elif isinstance(lhs, ExprNodes.TupleNode):
5774                            vars = [(var.name, var.pos) for var in lhs.args]
5775                        else:
5776                            error(lhs.pos, "Invalid declaration")
5777                            return
5778                        for var, pos in vars:
5779                            env.declare_var(var, type, pos, is_cdef=True, visibility=visibility)
5780                        if len(args) == 2:
5781                            # we have a value
5782                            self.rhs = args[1]
5783                        else:
5784                            self.declaration_only = True
5785                    else:
5786                        self.declaration_only = True
5787                        if not isinstance(lhs, ExprNodes.NameNode):
5788                            error(lhs.pos, "Invalid declaration.")
5789                        env.declare_typedef(lhs.name, type, self.pos, visibility='private')
5790
5791                elif func_name in ['struct', 'union']:
5792                    self.declaration_only = True
5793                    if len(args) > 0 or kwds is None:
5794                        error(self.rhs.pos, "Struct or union members must be given by name.")
5795                        return
5796                    members = []
5797                    for member, type_node in kwds.key_value_pairs:
5798                        type = type_node.analyse_as_type(env)
5799                        if type is None:
5800                            error(type_node.pos, "Unknown type")
5801                        else:
5802                            members.append((member.value, type, member.pos))
5803                    if len(members) < len(kwds.key_value_pairs):
5804                        return
5805                    if not isinstance(self.lhs, ExprNodes.NameNode):
5806                        error(self.lhs.pos, "Invalid declaration.")
5807                    name = self.lhs.name
5808                    scope = StructOrUnionScope(name)
5809                    env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
5810                    for member, type, pos in members:
5811                        scope.declare_var(member, type, pos)
5812
5813                elif func_name == 'fused_type':
5814                    # dtype = cython.fused_type(...)
5815                    self.declaration_only = True
5816                    if kwds:
5817                        error(self.rhs.function.pos,
5818                              "fused_type does not take keyword arguments")
5819
5820                    fusednode = FusedTypeNode(self.rhs.pos,
5821                                              name=self.lhs.name, types=args)
5822                    fusednode.analyse_declarations(env)
5823
5824        if self.declaration_only:
5825            return
5826        else:
5827            self.lhs.analyse_target_declaration(env)
5828
5829    def analyse_types(self, env, use_temp=0):
5830        from . import ExprNodes
5831
5832        self.rhs = self.rhs.analyse_types(env)
5833
5834        unrolled_assignment = self.unroll_rhs(env)
5835        if unrolled_assignment:
5836            return unrolled_assignment
5837
5838        self.lhs = self.lhs.analyse_target_types(env)
5839        self.lhs.gil_assignment_check(env)
5840        unrolled_assignment = self.unroll_lhs(env)
5841        if unrolled_assignment:
5842            return unrolled_assignment
5843
5844        if isinstance(self.lhs, ExprNodes.MemoryViewIndexNode):
5845            self.lhs.analyse_broadcast_operation(self.rhs)
5846            self.lhs = self.lhs.analyse_as_memview_scalar_assignment(self.rhs)
5847        elif self.lhs.type.is_array:
5848            if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
5849                # cannot assign to C array, only to its full slice
5850                lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None)
5851                self.lhs = lhs.analyse_target_types(env)
5852
5853        if self.lhs.type.is_cpp_class:
5854            op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type])
5855            if op:
5856                rhs = self.rhs
5857                self.is_overloaded_assignment = True
5858                self.exception_check = op.type.exception_check
5859                self.exception_value = op.type.exception_value
5860                if self.exception_check == '+' and self.exception_value is None:
5861                    env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
5862            else:
5863                rhs = self.rhs.coerce_to(self.lhs.type, env)
5864        else:
5865            rhs = self.rhs.coerce_to(self.lhs.type, env)
5866
5867        if use_temp or rhs.is_attribute or (
5868                not rhs.is_name and not rhs.is_literal and
5869                rhs.type.is_pyobject):
5870            # things like (cdef) attribute access are not safe (traverses pointers)
5871            rhs = rhs.coerce_to_temp(env)
5872        elif rhs.type.is_pyobject:
5873            rhs = rhs.coerce_to_simple(env)
5874        self.rhs = rhs
5875        return self
5876
5877    def unroll(self, node, target_size, env):
5878        from . import ExprNodes, UtilNodes
5879
5880        base = node
5881        start_node = stop_node = step_node = check_node = None
5882
5883        if node.type.is_ctuple:
5884            slice_size = node.type.size
5885
5886        elif node.type.is_ptr or node.type.is_array:
5887            while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
5888                base = node = node.base
5889            if isinstance(node, ExprNodes.SliceIndexNode):
5890                base = node.base
5891                start_node = node.start
5892                if start_node:
5893                    start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5894                stop_node = node.stop
5895                if stop_node:
5896                    stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5897                else:
5898                    if node.type.is_array and node.type.size:
5899                        stop_node = ExprNodes.IntNode(
5900                            self.pos, value=str(node.type.size),
5901                            constant_result=(node.type.size if isinstance(node.type.size, _py_int_types)
5902                                             else ExprNodes.constant_value_not_set))
5903                    else:
5904                        error(self.pos, "C array iteration requires known end index")
5905                        return
5906                step_node = None  #node.step
5907                if step_node:
5908                    step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5909
5910                # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
5911                def get_const(node, none_value):
5912                    if node is None:
5913                        return none_value
5914                    elif node.has_constant_result():
5915                        return node.constant_result
5916                    else:
5917                        raise ValueError("Not a constant.")
5918
5919                try:
5920                    slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
5921                except ValueError:
5922                    error(self.pos, "C array assignment currently requires known endpoints")
5923                    return
5924
5925            elif node.type.is_array:
5926                slice_size = node.type.size
5927                if not isinstance(slice_size, _py_int_types):
5928                    return  # might still work when coercing to Python
5929            else:
5930                return
5931
5932        else:
5933            return
5934
5935        if slice_size != target_size:
5936            error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
5937                slice_size, target_size))
5938            return
5939
5940        items = []
5941        base = UtilNodes.LetRefNode(base)
5942        refs = [base]
5943        if start_node and not start_node.is_literal:
5944            start_node = UtilNodes.LetRefNode(start_node)
5945            refs.append(start_node)
5946        if stop_node and not stop_node.is_literal:
5947            stop_node = UtilNodes.LetRefNode(stop_node)
5948            refs.append(stop_node)
5949        if step_node and not step_node.is_literal:
5950            step_node = UtilNodes.LetRefNode(step_node)
5951            refs.append(step_node)
5952
5953        for ix in range(target_size):
5954            ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
5955            if step_node is not None:
5956                if step_node.has_constant_result():
5957                    step_value = ix_node.constant_result * step_node.constant_result
5958                    ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
5959                else:
5960                    ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
5961            if start_node is not None:
5962                if start_node.has_constant_result() and ix_node.has_constant_result():
5963                    index_value = ix_node.constant_result + start_node.constant_result
5964                    ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
5965                else:
5966                    ix_node = ExprNodes.AddNode(
5967                        self.pos, operator='+', operand1=start_node, operand2=ix_node)
5968            items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
5969        return check_node, refs, items
5970
5971    def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
5972        from . import UtilNodes
5973        assignments = []
5974        for lhs, rhs in zip(lhs_list, rhs_list):
5975            assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
5976        node = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
5977        if check_node:
5978            node = StatListNode(pos=self.pos, stats=[check_node, node])
5979        for ref in refs[::-1]:
5980            node = UtilNodes.LetNode(ref, node)
5981        return node
5982
5983    def unroll_rhs(self, env):
5984        from . import ExprNodes
5985        if not isinstance(self.lhs, ExprNodes.TupleNode):
5986            return
5987        if any(arg.is_starred for arg in self.lhs.args):
5988            return
5989
5990        unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
5991        if not unrolled:
5992            return
5993        check_node, refs, rhs = unrolled
5994        return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)
5995
5996    def unroll_lhs(self, env):
5997        if self.lhs.type.is_ctuple:
5998            # Handled directly.
5999            return
6000        from . import ExprNodes
6001        if not isinstance(self.rhs, ExprNodes.TupleNode):
6002            return
6003
6004        unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
6005        if not unrolled:
6006            return
6007        check_node, refs, lhs = unrolled
6008        return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)
6009
6010    def generate_rhs_evaluation_code(self, code):
6011        self.rhs.generate_evaluation_code(code)
6012
6013    def generate_assignment_code(self, code, overloaded_assignment=False):
6014        if self.is_overloaded_assignment:
6015            self.lhs.generate_assignment_code(
6016                self.rhs,
6017                code,
6018                overloaded_assignment=self.is_overloaded_assignment,
6019                exception_check=self.exception_check,
6020                exception_value=self.exception_value)
6021        else:
6022            self.lhs.generate_assignment_code(self.rhs, code)
6023
6024    def generate_function_definitions(self, env, code):
6025        self.rhs.generate_function_definitions(env, code)
6026
6027    def annotate(self, code):
6028        self.lhs.annotate(code)
6029        self.rhs.annotate(code)
6030
6031
6032class CascadedAssignmentNode(AssignmentNode):
6033    #  An assignment with multiple left hand sides:
6034    #
6035    #    a = b = c
6036    #
6037    #  lhs_list   [ExprNode]   Left hand sides
6038    #  rhs        ExprNode     Right hand sides
6039    #
6040    #  Used internally:
6041    #
6042    #  coerced_values       [ExprNode]   RHS coerced to all distinct LHS types
6043    #  cloned_values        [ExprNode]   cloned RHS value for each LHS
6044    #  assignment_overloads [Bool]       If each assignment uses a C++ operator=
6045
6046    child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
6047    cloned_values = None
6048    coerced_values = None
6049    assignment_overloads = None
6050
6051    def analyse_declarations(self, env):
6052        for lhs in self.lhs_list:
6053            lhs.analyse_target_declaration(env)
6054
6055    def analyse_types(self, env, use_temp=0):
6056        from .ExprNodes import CloneNode, ProxyNode
6057
6058        # collect distinct types used on the LHS
6059        lhs_types = set()
6060        for i, lhs in enumerate(self.lhs_list):
6061            lhs = self.lhs_list[i] = lhs.analyse_target_types(env)
6062            lhs.gil_assignment_check(env)
6063            lhs_types.add(lhs.type)
6064
6065        rhs = self.rhs.analyse_types(env)
6066        # common special case: only one type needed on the LHS => coerce only once
6067        if len(lhs_types) == 1:
6068            # Avoid coercion for overloaded assignment operators.
6069            if next(iter(lhs_types)).is_cpp_class:
6070                op = env.lookup_operator('=', [lhs, self.rhs])
6071                if not op:
6072                    rhs = rhs.coerce_to(lhs_types.pop(), env)
6073            else:
6074                rhs = rhs.coerce_to(lhs_types.pop(), env)
6075
6076        if not rhs.is_name and not rhs.is_literal and (
6077                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
6078            rhs = rhs.coerce_to_temp(env)
6079        else:
6080            rhs = rhs.coerce_to_simple(env)
6081        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs
6082
6083        # clone RHS and coerce it to all distinct LHS types
6084        self.coerced_values = []
6085        coerced_values = {}
6086        self.assignment_overloads = []
6087        for lhs in self.lhs_list:
6088            overloaded = lhs.type.is_cpp_class and env.lookup_operator('=', [lhs, self.rhs])
6089            self.assignment_overloads.append(overloaded)
6090            if lhs.type not in coerced_values and lhs.type != rhs.type:
6091                rhs = CloneNode(self.rhs)
6092                if not overloaded:
6093                    rhs = rhs.coerce_to(lhs.type, env)
6094                self.coerced_values.append(rhs)
6095                coerced_values[lhs.type] = rhs
6096
6097        # clone coerced values for all LHS assignments
6098        self.cloned_values = []
6099        for lhs in self.lhs_list:
6100            rhs = coerced_values.get(lhs.type, self.rhs)
6101            self.cloned_values.append(CloneNode(rhs))
6102        return self
6103
6104    def generate_rhs_evaluation_code(self, code):
6105        self.rhs.generate_evaluation_code(code)
6106
6107    def generate_assignment_code(self, code, overloaded_assignment=False):
6108        # prepare all coercions
6109        for rhs in self.coerced_values:
6110            rhs.generate_evaluation_code(code)
6111        # assign clones to LHS
6112        for lhs, rhs, overload in zip(self.lhs_list, self.cloned_values, self.assignment_overloads):
6113            rhs.generate_evaluation_code(code)
6114            lhs.generate_assignment_code(rhs, code, overloaded_assignment=overload)
6115        # dispose of coerced values and original RHS
6116        for rhs_value in self.coerced_values:
6117            rhs_value.generate_disposal_code(code)
6118            rhs_value.free_temps(code)
6119        self.rhs.generate_disposal_code(code)
6120        self.rhs.free_temps(code)
6121
6122    def generate_function_definitions(self, env, code):
6123        self.rhs.generate_function_definitions(env, code)
6124
6125    def annotate(self, code):
6126        for rhs in self.coerced_values:
6127            rhs.annotate(code)
6128        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
6129            lhs.annotate(code)
6130            rhs.annotate(code)
6131        self.rhs.annotate(code)
6132
6133
6134class ParallelAssignmentNode(AssignmentNode):
6135    #  A combined packing/unpacking assignment:
6136    #
6137    #    a, b, c =  d, e, f
6138    #
6139    #  This has been rearranged by the parser into
6140    #
6141    #    a = d ; b = e ; c = f
6142    #
6143    #  but we must evaluate all the right hand sides
6144    #  before assigning to any of the left hand sides.
6145    #
6146    #  stats     [AssignmentNode]   The constituent assignments
6147
6148    child_attrs = ["stats"]
6149
6150    def analyse_declarations(self, env):
6151        for stat in self.stats:
6152            stat.analyse_declarations(env)
6153
6154    def analyse_expressions(self, env):
6155        self.stats = [stat.analyse_types(env, use_temp=1)
6156                      for stat in self.stats]
6157        return self
6158
6159#    def analyse_expressions(self, env):
6160#        for stat in self.stats:
6161#            stat.analyse_expressions_1(env, use_temp=1)
6162#        for stat in self.stats:
6163#            stat.analyse_expressions_2(env)
6164
6165    def generate_execution_code(self, code):
6166        code.mark_pos(self.pos)
6167        for stat in self.stats:
6168            stat.generate_rhs_evaluation_code(code)
6169        for stat in self.stats:
6170            stat.generate_assignment_code(code)
6171
6172    def generate_function_definitions(self, env, code):
6173        for stat in self.stats:
6174            stat.generate_function_definitions(env, code)
6175
6176    def annotate(self, code):
6177        for stat in self.stats:
6178            stat.annotate(code)
6179
6180
6181class InPlaceAssignmentNode(AssignmentNode):
6182    #  An in place arithmetic operand:
6183    #
6184    #    a += b
6185    #    a -= b
6186    #    ...
6187    #
6188    #  lhs      ExprNode      Left hand side
6189    #  rhs      ExprNode      Right hand side
6190    #  operator char          one of "+-*/%^&|"
6191    #
6192    #  This code is a bit tricky because in order to obey Python
6193    #  semantics the sub-expressions (e.g. indices) of the lhs must
6194    #  not be evaluated twice. So we must re-use the values calculated
6195    #  in evaluation phase for the assignment phase as well.
6196    #  Fortunately, the type of the lhs node is fairly constrained
6197    #  (it must be a NameNode, AttributeNode, or IndexNode).
6198
6199    child_attrs = ["lhs", "rhs"]
6200
6201    def analyse_declarations(self, env):
6202        self.lhs.analyse_target_declaration(env)
6203
6204    def analyse_types(self, env):
6205        self.rhs = self.rhs.analyse_types(env)
6206        self.lhs = self.lhs.analyse_target_types(env)
6207
6208        # When assigning to a fully indexed buffer or memoryview, coerce the rhs
6209        if self.lhs.is_memview_index or self.lhs.is_buffer_access:
6210            self.rhs = self.rhs.coerce_to(self.lhs.type, env)
6211        elif self.lhs.type.is_string and self.operator in '+-':
6212            # use pointer arithmetic for char* LHS instead of string concat
6213            self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
6214        return self
6215
6216    def generate_execution_code(self, code):
6217        code.mark_pos(self.pos)
6218        lhs, rhs = self.lhs, self.rhs
6219        rhs.generate_evaluation_code(code)
6220        lhs.generate_subexpr_evaluation_code(code)
6221        c_op = self.operator
6222        if c_op == "//":
6223            c_op = "/"
6224        elif c_op == "**":
6225            error(self.pos, "No C inplace power operator")
6226        if lhs.is_buffer_access or lhs.is_memview_index:
6227            if lhs.type.is_pyobject:
6228                error(self.pos, "In-place operators not allowed on object buffers in this release.")
6229            if c_op in ('/', '%') and lhs.type.is_int and not code.globalstate.directives['cdivision']:
6230                error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
6231            lhs.generate_buffer_setitem_code(rhs, code, c_op)
6232        elif lhs.is_memview_slice:
6233            error(self.pos, "Inplace operators not supported on memoryview slices")
6234        else:
6235            # C++
6236            # TODO: make sure overload is declared
6237            code.putln("%s %s= %s;" % (lhs.result(), c_op, rhs.result()))
6238        lhs.generate_subexpr_disposal_code(code)
6239        lhs.free_subexpr_temps(code)
6240        rhs.generate_disposal_code(code)
6241        rhs.free_temps(code)
6242
6243    def annotate(self, code):
6244        self.lhs.annotate(code)
6245        self.rhs.annotate(code)
6246
6247    def create_binop_node(self):
6248        from . import ExprNodes
6249        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
6250
6251
6252class PrintStatNode(StatNode):
6253    #  print statement
6254    #
6255    #  arg_tuple         TupleNode
6256    #  stream            ExprNode or None (stdout)
6257    #  append_newline    boolean
6258
6259    child_attrs = ["arg_tuple", "stream"]
6260
6261    def analyse_expressions(self, env):
6262        if self.stream:
6263            stream = self.stream.analyse_expressions(env)
6264            self.stream = stream.coerce_to_pyobject(env)
6265        arg_tuple = self.arg_tuple.analyse_expressions(env)
6266        self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
6267        env.use_utility_code(printing_utility_code)
6268        if len(self.arg_tuple.args) == 1 and self.append_newline:
6269            env.use_utility_code(printing_one_utility_code)
6270        return self
6271
6272    nogil_check = Node.gil_error
6273    gil_message = "Python print statement"
6274
6275    def generate_execution_code(self, code):
6276        code.mark_pos(self.pos)
6277        if self.stream:
6278            self.stream.generate_evaluation_code(code)
6279            stream_result = self.stream.py_result()
6280        else:
6281            stream_result = '0'
6282        if len(self.arg_tuple.args) == 1 and self.append_newline:
6283            arg = self.arg_tuple.args[0]
6284            arg.generate_evaluation_code(code)
6285
6286            code.putln(
6287                "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
6288                    stream_result,
6289                    arg.py_result(),
6290                    code.error_goto(self.pos)))
6291            arg.generate_disposal_code(code)
6292            arg.free_temps(code)
6293        else:
6294            self.arg_tuple.generate_evaluation_code(code)
6295            code.putln(
6296                "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
6297                    stream_result,
6298                    self.arg_tuple.py_result(),
6299                    self.append_newline,
6300                    code.error_goto(self.pos)))
6301            self.arg_tuple.generate_disposal_code(code)
6302            self.arg_tuple.free_temps(code)
6303
6304        if self.stream:
6305            self.stream.generate_disposal_code(code)
6306            self.stream.free_temps(code)
6307
6308    def generate_function_definitions(self, env, code):
6309        if self.stream:
6310            self.stream.generate_function_definitions(env, code)
6311        self.arg_tuple.generate_function_definitions(env, code)
6312
6313    def annotate(self, code):
6314        if self.stream:
6315            self.stream.annotate(code)
6316        self.arg_tuple.annotate(code)
6317
6318
6319class ExecStatNode(StatNode):
6320    #  exec statement
6321    #
6322    #  args     [ExprNode]
6323
6324    child_attrs = ["args"]
6325
6326    def analyse_expressions(self, env):
6327        for i, arg in enumerate(self.args):
6328            arg = arg.analyse_expressions(env)
6329            arg = arg.coerce_to_pyobject(env)
6330            self.args[i] = arg
6331        env.use_utility_code(Builtin.pyexec_utility_code)
6332        return self
6333
6334    nogil_check = Node.gil_error
6335    gil_message = "Python exec statement"
6336
6337    def generate_execution_code(self, code):
6338        code.mark_pos(self.pos)
6339        args = []
6340        for arg in self.args:
6341            arg.generate_evaluation_code(code)
6342            args.append(arg.py_result())
6343        args = tuple(args + ['0', '0'][:3-len(args)])
6344        temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
6345        code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % ((temp_result,) + args))
6346        for arg in self.args:
6347            arg.generate_disposal_code(code)
6348            arg.free_temps(code)
6349        code.putln(
6350            code.error_goto_if_null(temp_result, self.pos))
6351        code.put_gotref(temp_result, py_object_type)
6352        code.put_decref_clear(temp_result, py_object_type)
6353        code.funcstate.release_temp(temp_result)
6354
6355    def annotate(self, code):
6356        for arg in self.args:
6357            arg.annotate(code)
6358
6359
6360class DelStatNode(StatNode):
6361    #  del statement
6362    #
6363    #  args     [ExprNode]
6364
6365    child_attrs = ["args"]
6366    ignore_nonexisting = False
6367
6368    def analyse_declarations(self, env):
6369        for arg in self.args:
6370            arg.analyse_target_declaration(env)
6371
6372    def analyse_expressions(self, env):
6373        for i, arg in enumerate(self.args):
6374            arg = self.args[i] = arg.analyse_target_expression(env, None)
6375            if arg.type.is_pyobject or (arg.is_name and arg.type.is_memoryviewslice):
6376                if arg.is_name and arg.entry.is_cglobal:
6377                    error(arg.pos, "Deletion of global C variable")
6378            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
6379                self.cpp_check(env)
6380            elif arg.type.is_cpp_class:
6381                error(arg.pos, "Deletion of non-heap C++ object")
6382            elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
6383                pass  # del ba[i]
6384            else:
6385                error(arg.pos, "Deletion of non-Python, non-C++ object")
6386            #arg.release_target_temp(env)
6387        return self
6388
6389    def nogil_check(self, env):
6390        for arg in self.args:
6391            if arg.type.is_pyobject:
6392                self.gil_error()
6393
6394    gil_message = "Deleting Python object"
6395
6396    def generate_execution_code(self, code):
6397        code.mark_pos(self.pos)
6398        for arg in self.args:
6399            if (arg.type.is_pyobject or
6400                    arg.type.is_memoryviewslice or
6401                    arg.is_subscript and arg.base.type is Builtin.bytearray_type):
6402                arg.generate_deletion_code(
6403                    code, ignore_nonexisting=self.ignore_nonexisting)
6404            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
6405                arg.generate_evaluation_code(code)
6406                code.putln("delete %s;" % arg.result())
6407                arg.generate_disposal_code(code)
6408                arg.free_temps(code)
6409            # else error reported earlier
6410
6411    def annotate(self, code):
6412        for arg in self.args:
6413            arg.annotate(code)
6414
6415
6416class PassStatNode(StatNode):
6417    #  pass statement
6418
6419    child_attrs = []
6420
6421    def analyse_expressions(self, env):
6422        return self
6423
6424    def generate_execution_code(self, code):
6425        pass
6426
6427
6428class IndirectionNode(StatListNode):
6429    """
6430    This adds an indirection so that the node can be shared and a subtree can
6431    be removed at any time by clearing self.stats.
6432    """
6433
6434    def __init__(self, stats):
6435        super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)
6436
6437
6438class BreakStatNode(StatNode):
6439
6440    child_attrs = []
6441    is_terminator = True
6442
6443    def analyse_expressions(self, env):
6444        return self
6445
6446    def generate_execution_code(self, code):
6447        code.mark_pos(self.pos)
6448        if not code.break_label:
6449            error(self.pos, "break statement not inside loop")
6450        else:
6451            code.put_goto(code.break_label)
6452
6453
6454class ContinueStatNode(StatNode):
6455
6456    child_attrs = []
6457    is_terminator = True
6458
6459    def analyse_expressions(self, env):
6460        return self
6461
6462    def generate_execution_code(self, code):
6463        if not code.continue_label:
6464            error(self.pos, "continue statement not inside loop")
6465            return
6466        code.mark_pos(self.pos)
6467        code.put_goto(code.continue_label)
6468
6469
6470class ReturnStatNode(StatNode):
6471    #  return statement
6472    #
6473    #  value         ExprNode or None
6474    #  return_type   PyrexType
6475    #  in_generator  return inside of generator => raise StopIteration
6476    #  in_async_gen  return inside of async generator
6477
6478    child_attrs = ["value"]
6479    is_terminator = True
6480    in_generator = False
6481    in_async_gen = False
6482
6483    # Whether we are in a parallel section
6484    in_parallel = False
6485
6486    def analyse_expressions(self, env):
6487        return_type = env.return_type
6488        self.return_type = return_type
6489        if not return_type:
6490            error(self.pos, "Return not inside a function body")
6491            return self
6492        if self.value:
6493            if self.in_async_gen:
6494                error(self.pos, "Return with value in async generator")
6495            self.value = self.value.analyse_types(env)
6496            if return_type.is_void or return_type.is_returncode:
6497                error(self.value.pos, "Return with value in void function")
6498            else:
6499                self.value = self.value.coerce_to(env.return_type, env)
6500        else:
6501            if (not return_type.is_void
6502                    and not return_type.is_pyobject
6503                    and not return_type.is_returncode):
6504                error(self.pos, "Return value required")
6505        return self
6506
6507    def nogil_check(self, env):
6508        if self.return_type.is_pyobject:
6509            self.gil_error()
6510
6511    gil_message = "Returning Python object"
6512
6513    def generate_execution_code(self, code):
6514        code.mark_pos(self.pos)
6515        if not self.return_type:
6516            # error reported earlier
6517            return
6518
6519        value = self.value
6520        if self.return_type.is_pyobject:
6521            code.put_xdecref(Naming.retval_cname, self.return_type)
6522            if value and value.is_none:
6523                # Use specialised default handling for "return None".
6524                value = None
6525
6526        if value:
6527            value.generate_evaluation_code(code)
6528            if self.return_type.is_memoryviewslice:
6529                from . import MemoryView
6530                MemoryView.put_acquire_memoryviewslice(
6531                    lhs_cname=Naming.retval_cname,
6532                    lhs_type=self.return_type,
6533                    lhs_pos=value.pos,
6534                    rhs=value,
6535                    code=code,
6536                    have_gil=self.in_nogil_context)
6537                value.generate_post_assignment_code(code)
6538            elif self.in_generator:
6539                # return value == raise StopIteration(value), but uncatchable
6540                code.globalstate.use_utility_code(
6541                    UtilityCode.load_cached("ReturnWithStopIteration", "Coroutine.c"))
6542                code.putln("%s = NULL; __Pyx_ReturnWithStopIteration(%s);" % (
6543                    Naming.retval_cname,
6544                    value.py_result()))
6545                value.generate_disposal_code(code)
6546            else:
6547                value.make_owned_reference(code)
6548                code.putln("%s = %s;" % (
6549                    Naming.retval_cname,
6550                    value.result_as(self.return_type)))
6551                value.generate_post_assignment_code(code)
6552            value.free_temps(code)
6553        else:
6554            if self.return_type.is_pyobject:
6555                if self.in_generator:
6556                    if self.in_async_gen:
6557                        code.globalstate.use_utility_code(
6558                            UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
6559                        code.put("PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); ")
6560                    code.putln("%s = NULL;" % Naming.retval_cname)
6561                else:
6562                    code.put_init_to_py_none(Naming.retval_cname, self.return_type)
6563            elif self.return_type.is_returncode:
6564                self.put_return(code, self.return_type.default_value)
6565
6566        for cname, type in code.funcstate.temps_holding_reference():
6567            code.put_decref_clear(cname, type)
6568
6569        code.put_goto(code.return_label)
6570
6571    def put_return(self, code, value):
6572        if self.in_parallel:
6573            code.putln_openmp("#pragma omp critical(__pyx_returning)")
6574        code.putln("%s = %s;" % (Naming.retval_cname, value))
6575
6576    def generate_function_definitions(self, env, code):
6577        if self.value is not None:
6578            self.value.generate_function_definitions(env, code)
6579
6580    def annotate(self, code):
6581        if self.value:
6582            self.value.annotate(code)
6583
6584
6585class RaiseStatNode(StatNode):
6586    #  raise statement
6587    #
6588    #  exc_type    ExprNode or None
6589    #  exc_value   ExprNode or None
6590    #  exc_tb      ExprNode or None
6591    #  cause       ExprNode or None
6592
6593    child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
6594    is_terminator = True
6595    builtin_exc_name = None
6596    wrap_tuple_value = False
6597
6598    def analyse_expressions(self, env):
6599        if self.exc_type:
6600            exc_type = self.exc_type.analyse_types(env)
6601            self.exc_type = exc_type.coerce_to_pyobject(env)
6602        if self.exc_value:
6603            exc_value = self.exc_value.analyse_types(env)
6604            if self.wrap_tuple_value:
6605                if exc_value.type is Builtin.tuple_type or not exc_value.type.is_builtin_type:
6606                    # prevent tuple values from being interpreted as argument value tuples
6607                    from .ExprNodes import TupleNode
6608                    exc_value = TupleNode(exc_value.pos, args=[exc_value.coerce_to_pyobject(env)], slow=True)
6609                    exc_value = exc_value.analyse_types(env, skip_children=True)
6610            self.exc_value = exc_value.coerce_to_pyobject(env)
6611        if self.exc_tb:
6612            exc_tb = self.exc_tb.analyse_types(env)
6613            self.exc_tb = exc_tb.coerce_to_pyobject(env)
6614        if self.cause:
6615            cause = self.cause.analyse_types(env)
6616            self.cause = cause.coerce_to_pyobject(env)
6617        # special cases for builtin exceptions
6618        if self.exc_type and not self.exc_value and not self.exc_tb:
6619            exc = self.exc_type
6620            from . import ExprNodes
6621            if (isinstance(exc, ExprNodes.SimpleCallNode) and
6622                    not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))):
6623                exc = exc.function  # extract the exception type
6624            if exc.is_name and exc.entry.is_builtin:
6625                self.builtin_exc_name = exc.name
6626                if self.builtin_exc_name == 'MemoryError':
6627                    self.exc_type = None  # has a separate implementation
6628        return self
6629
6630    nogil_check = Node.gil_error
6631    gil_message = "Raising exception"
6632
6633    def generate_execution_code(self, code):
6634        code.mark_pos(self.pos)
6635        if self.builtin_exc_name == 'MemoryError':
6636            code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
6637            return
6638
6639        if self.exc_type:
6640            self.exc_type.generate_evaluation_code(code)
6641            type_code = self.exc_type.py_result()
6642            if self.exc_type.is_name:
6643                code.globalstate.use_entry_utility_code(self.exc_type.entry)
6644        else:
6645            type_code = "0"
6646        if self.exc_value:
6647            self.exc_value.generate_evaluation_code(code)
6648            value_code = self.exc_value.py_result()
6649        else:
6650            value_code = "0"
6651        if self.exc_tb:
6652            self.exc_tb.generate_evaluation_code(code)
6653            tb_code = self.exc_tb.py_result()
6654        else:
6655            tb_code = "0"
6656        if self.cause:
6657            self.cause.generate_evaluation_code(code)
6658            cause_code = self.cause.py_result()
6659        else:
6660            cause_code = "0"
6661        code.globalstate.use_utility_code(raise_utility_code)
6662        code.putln(
6663            "__Pyx_Raise(%s, %s, %s, %s);" % (
6664                type_code,
6665                value_code,
6666                tb_code,
6667                cause_code))
6668        for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
6669            if obj:
6670                obj.generate_disposal_code(code)
6671                obj.free_temps(code)
6672        code.putln(
6673            code.error_goto(self.pos))
6674
6675    def generate_function_definitions(self, env, code):
6676        if self.exc_type is not None:
6677            self.exc_type.generate_function_definitions(env, code)
6678        if self.exc_value is not None:
6679            self.exc_value.generate_function_definitions(env, code)
6680        if self.exc_tb is not None:
6681            self.exc_tb.generate_function_definitions(env, code)
6682        if self.cause is not None:
6683            self.cause.generate_function_definitions(env, code)
6684
6685    def annotate(self, code):
6686        if self.exc_type:
6687            self.exc_type.annotate(code)
6688        if self.exc_value:
6689            self.exc_value.annotate(code)
6690        if self.exc_tb:
6691            self.exc_tb.annotate(code)
6692        if self.cause:
6693            self.cause.annotate(code)
6694
6695
6696class ReraiseStatNode(StatNode):
6697
6698    child_attrs = []
6699    is_terminator = True
6700
6701    def analyse_expressions(self, env):
6702        return self
6703
6704    nogil_check = Node.gil_error
6705    gil_message = "Raising exception"
6706
6707    def generate_execution_code(self, code):
6708        code.mark_pos(self.pos)
6709        vars = code.funcstate.exc_vars
6710        if vars:
6711            code.globalstate.use_utility_code(restore_exception_utility_code)
6712            code.put_giveref(vars[0], py_object_type)
6713            code.put_giveref(vars[1], py_object_type)
6714            # fresh exceptions may not have a traceback yet (-> finally!)
6715            code.put_xgiveref(vars[2], py_object_type)
6716            code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars))
6717            for varname in vars:
6718                code.put("%s = 0; " % varname)
6719            code.putln()
6720            code.putln(code.error_goto(self.pos))
6721        else:
6722            code.globalstate.use_utility_code(
6723                UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
6724            code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
6725
6726
6727class AssertStatNode(StatNode):
6728    #  assert statement
6729    #
6730    #  condition    ExprNode
6731    #  value        ExprNode or None
6732    #  exception    (Raise/GIL)StatNode   created from 'value' in PostParse transform
6733
6734    child_attrs = ["condition", "value", "exception"]
6735    exception = None
6736
6737    def analyse_declarations(self, env):
6738        assert self.value is None, "Message should have been replaced in PostParse()"
6739        assert self.exception is not None, "Message should have been replaced in PostParse()"
6740        self.exception.analyse_declarations(env)
6741
6742    def analyse_expressions(self, env):
6743        self.condition = self.condition.analyse_temp_boolean_expression(env)
6744        self.exception = self.exception.analyse_expressions(env)
6745        return self
6746
6747    def generate_execution_code(self, code):
6748        code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
6749        code.putln("if (unlikely(!Py_OptimizeFlag)) {")
6750        code.mark_pos(self.pos)
6751        self.condition.generate_evaluation_code(code)
6752        code.putln(
6753            "if (unlikely(!%s)) {" % self.condition.result())
6754        self.exception.generate_execution_code(code)
6755        code.putln(
6756            "}")
6757        self.condition.generate_disposal_code(code)
6758        self.condition.free_temps(code)
6759        code.putln(
6760            "}")
6761        code.putln("#else")
6762        # avoid unused labels etc.
6763        code.putln("if ((1)); else %s" % code.error_goto(self.pos, used=False))
6764        code.putln("#endif")
6765
6766    def generate_function_definitions(self, env, code):
6767        self.condition.generate_function_definitions(env, code)
6768        self.exception.generate_function_definitions(env, code)
6769
6770    def annotate(self, code):
6771        self.condition.annotate(code)
6772        self.exception.annotate(code)
6773
6774
6775class IfStatNode(StatNode):
6776    #  if statement
6777    #
6778    #  if_clauses   [IfClauseNode]
6779    #  else_clause  StatNode or None
6780
6781    child_attrs = ["if_clauses", "else_clause"]
6782
6783    def analyse_declarations(self, env):
6784        for if_clause in self.if_clauses:
6785            if_clause.analyse_declarations(env)
6786        if self.else_clause:
6787            self.else_clause.analyse_declarations(env)
6788
6789    def analyse_expressions(self, env):
6790        self.if_clauses = [if_clause.analyse_expressions(env) for if_clause in self.if_clauses]
6791        if self.else_clause:
6792            self.else_clause = self.else_clause.analyse_expressions(env)
6793        return self
6794
6795    def generate_execution_code(self, code):
6796        code.mark_pos(self.pos)
6797        end_label = code.new_label()
6798        last = len(self.if_clauses)
6799        if not self.else_clause:
6800            last -= 1  # avoid redundant goto at end of last if-clause
6801        for i, if_clause in enumerate(self.if_clauses):
6802            if_clause.generate_execution_code(code, end_label, is_last=i == last)
6803        if self.else_clause:
6804            code.mark_pos(self.else_clause.pos)
6805            code.putln("/*else*/ {")
6806            self.else_clause.generate_execution_code(code)
6807            code.putln("}")
6808        code.put_label(end_label)
6809
6810    def generate_function_definitions(self, env, code):
6811        for clause in self.if_clauses:
6812            clause.generate_function_definitions(env, code)
6813        if self.else_clause is not None:
6814            self.else_clause.generate_function_definitions(env, code)
6815
6816    def annotate(self, code):
6817        for if_clause in self.if_clauses:
6818            if_clause.annotate(code)
6819        if self.else_clause:
6820            self.else_clause.annotate(code)
6821
6822
6823class IfClauseNode(Node):
6824    #  if or elif clause in an if statement
6825    #
6826    #  condition   ExprNode
6827    #  body        StatNode
6828
6829    child_attrs = ["condition", "body"]
6830    branch_hint = None
6831
6832    def analyse_declarations(self, env):
6833        self.body.analyse_declarations(env)
6834
6835    def analyse_expressions(self, env):
6836        self.condition = self.condition.analyse_temp_boolean_expression(env)
6837        self.body = self.body.analyse_expressions(env)
6838        return self
6839
6840    def generate_execution_code(self, code, end_label, is_last):
6841        self.condition.generate_evaluation_code(code)
6842        code.mark_pos(self.pos)
6843        condition = self.condition.result()
6844        if self.branch_hint:
6845            condition = '%s(%s)' % (self.branch_hint, condition)
6846        code.putln("if (%s) {" % condition)
6847        self.condition.generate_disposal_code(code)
6848        self.condition.free_temps(code)
6849        self.body.generate_execution_code(code)
6850        code.mark_pos(self.pos, trace=False)
6851        if not (is_last or self.body.is_terminator):
6852            code.put_goto(end_label)
6853        code.putln("}")
6854
6855    def generate_function_definitions(self, env, code):
6856        self.condition.generate_function_definitions(env, code)
6857        self.body.generate_function_definitions(env, code)
6858
6859    def annotate(self, code):
6860        self.condition.annotate(code)
6861        self.body.annotate(code)
6862
6863
6864class SwitchCaseNode(StatNode):
6865    # Generated in the optimization of an if-elif-else node
6866    #
6867    # conditions    [ExprNode]
6868    # body          StatNode
6869
6870    child_attrs = ['conditions', 'body']
6871
6872    def generate_condition_evaluation_code(self, code):
6873        for cond in self.conditions:
6874            cond.generate_evaluation_code(code)
6875
6876    def generate_execution_code(self, code):
6877        num_conditions = len(self.conditions)
6878        line_tracing_enabled = code.globalstate.directives['linetrace']
6879        for i, cond in enumerate(self.conditions, 1):
6880            code.putln("case %s:" % cond.result())
6881            code.mark_pos(cond.pos)  # Tracing code must appear *after* the 'case' statement.
6882            if line_tracing_enabled and i < num_conditions:
6883                # Allow fall-through after the line tracing code.
6884                code.putln('CYTHON_FALLTHROUGH;')
6885        self.body.generate_execution_code(code)
6886        code.mark_pos(self.pos, trace=False)
6887        code.putln("break;")
6888
6889    def generate_function_definitions(self, env, code):
6890        for cond in self.conditions:
6891            cond.generate_function_definitions(env, code)
6892        self.body.generate_function_definitions(env, code)
6893
6894    def annotate(self, code):
6895        for cond in self.conditions:
6896            cond.annotate(code)
6897        self.body.annotate(code)
6898
6899
6900class SwitchStatNode(StatNode):
6901    # Generated in the optimization of an if-elif-else node
6902    #
6903    # test          ExprNode
6904    # cases         [SwitchCaseNode]
6905    # else_clause   StatNode or None
6906
6907    child_attrs = ['test', 'cases', 'else_clause']
6908
6909    def generate_execution_code(self, code):
6910        self.test.generate_evaluation_code(code)
6911        # Make sure all conditions are evaluated before going into the switch() statement.
6912        # This is required in order to prevent any execution code from leaking into the space between the cases.
6913        for case in self.cases:
6914            case.generate_condition_evaluation_code(code)
6915        code.mark_pos(self.pos)
6916        code.putln("switch (%s) {" % self.test.result())
6917        for case in self.cases:
6918            case.generate_execution_code(code)
6919        if self.else_clause is not None:
6920            code.putln("default:")
6921            self.else_clause.generate_execution_code(code)
6922            code.putln("break;")
6923        else:
6924            # Always generate a default clause to prevent C compiler warnings
6925            # about unmatched enum values (it was not the user who decided to
6926            # generate the switch statement, so shouldn't be bothered).
6927            code.putln("default: break;")
6928        code.putln("}")
6929        self.test.generate_disposal_code(code)
6930        self.test.free_temps(code)
6931
6932    def generate_function_definitions(self, env, code):
6933        self.test.generate_function_definitions(env, code)
6934        for case in self.cases:
6935            case.generate_function_definitions(env, code)
6936        if self.else_clause is not None:
6937            self.else_clause.generate_function_definitions(env, code)
6938
6939    def annotate(self, code):
6940        self.test.annotate(code)
6941        for case in self.cases:
6942            case.annotate(code)
6943        if self.else_clause is not None:
6944            self.else_clause.annotate(code)
6945
6946
6947class LoopNode(object):
6948    pass
6949
6950
6951class WhileStatNode(LoopNode, StatNode):
6952    #  while statement
6953    #
6954    #  condition    ExprNode
6955    #  body         StatNode
6956    #  else_clause  StatNode
6957
6958    child_attrs = ["condition", "body", "else_clause"]
6959
6960    def analyse_declarations(self, env):
6961        self.body.analyse_declarations(env)
6962        if self.else_clause:
6963            self.else_clause.analyse_declarations(env)
6964
6965    def analyse_expressions(self, env):
6966        if self.condition:
6967            self.condition = self.condition.analyse_temp_boolean_expression(env)
6968        self.body = self.body.analyse_expressions(env)
6969        if self.else_clause:
6970            self.else_clause = self.else_clause.analyse_expressions(env)
6971        return self
6972
6973    def generate_execution_code(self, code):
6974        code.mark_pos(self.pos)
6975        old_loop_labels = code.new_loop_labels()
6976        code.putln(
6977            "while (1) {")
6978        if self.condition:
6979            self.condition.generate_evaluation_code(code)
6980            self.condition.generate_disposal_code(code)
6981            code.putln(
6982                "if (!%s) break;" % self.condition.result())
6983            self.condition.free_temps(code)
6984        self.body.generate_execution_code(code)
6985        code.put_label(code.continue_label)
6986        code.putln("}")
6987        break_label = code.break_label
6988        code.set_loop_labels(old_loop_labels)
6989        if self.else_clause:
6990            code.mark_pos(self.else_clause.pos)
6991            code.putln("/*else*/ {")
6992            self.else_clause.generate_execution_code(code)
6993            code.putln("}")
6994        code.put_label(break_label)
6995
6996    def generate_function_definitions(self, env, code):
6997        if self.condition:
6998            self.condition.generate_function_definitions(env, code)
6999        self.body.generate_function_definitions(env, code)
7000        if self.else_clause is not None:
7001            self.else_clause.generate_function_definitions(env, code)
7002
7003    def annotate(self, code):
7004        if self.condition:
7005            self.condition.annotate(code)
7006        self.body.annotate(code)
7007        if self.else_clause:
7008            self.else_clause.annotate(code)
7009
7010
7011class DictIterationNextNode(Node):
7012    # Helper node for calling PyDict_Next() inside of a WhileStatNode
7013    # and checking the dictionary size for changes.  Created in
7014    # Optimize.py.
7015    child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
7016                   'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
7017                   'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
7018
7019    coerced_key_var = key_ref = None
7020    coerced_value_var = value_ref = None
7021    coerced_tuple_var = tuple_ref = None
7022
7023    def __init__(self, dict_obj, expected_size, pos_index_var,
7024                 key_target, value_target, tuple_target, is_dict_flag):
7025        Node.__init__(
7026            self, dict_obj.pos,
7027            dict_obj=dict_obj,
7028            expected_size=expected_size,
7029            pos_index_var=pos_index_var,
7030            key_target=key_target,
7031            value_target=value_target,
7032            tuple_target=tuple_target,
7033            is_dict_flag=is_dict_flag,
7034            is_temp=True,
7035            type=PyrexTypes.c_bint_type)
7036
7037    def analyse_expressions(self, env):
7038        from . import ExprNodes
7039        self.dict_obj = self.dict_obj.analyse_types(env)
7040        self.expected_size = self.expected_size.analyse_types(env)
7041        if self.pos_index_var:
7042            self.pos_index_var = self.pos_index_var.analyse_types(env)
7043        if self.key_target:
7044            self.key_target = self.key_target.analyse_target_types(env)
7045            self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
7046            self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
7047        if self.value_target:
7048            self.value_target = self.value_target.analyse_target_types(env)
7049            self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
7050            self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
7051        if self.tuple_target:
7052            self.tuple_target = self.tuple_target.analyse_target_types(env)
7053            self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
7054            self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
7055        self.is_dict_flag = self.is_dict_flag.analyse_types(env)
7056        return self
7057
7058    def generate_function_definitions(self, env, code):
7059        self.dict_obj.generate_function_definitions(env, code)
7060
7061    def generate_execution_code(self, code):
7062        code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
7063        self.dict_obj.generate_evaluation_code(code)
7064
7065        assignments = []
7066        temp_addresses = []
7067        for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
7068                                    (self.value_ref, self.coerced_value_var, self.value_target),
7069                                    (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
7070            if target is None:
7071                addr = 'NULL'
7072            else:
7073                assignments.append((var, result, target))
7074                var.allocate(code)
7075                addr = '&%s' % var.result()
7076            temp_addresses.append(addr)
7077
7078        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
7079        code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
7080            result_temp,
7081            self.dict_obj.py_result(),
7082            self.expected_size.result(),
7083            self.pos_index_var.result(),
7084            temp_addresses[0],
7085            temp_addresses[1],
7086            temp_addresses[2],
7087            self.is_dict_flag.result()
7088        ))
7089        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
7090        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
7091        code.funcstate.release_temp(result_temp)
7092
7093        # evaluate all coercions before the assignments
7094        for var, result, target in assignments:
7095            var.generate_gotref(code)
7096        for var, result, target in assignments:
7097            result.generate_evaluation_code(code)
7098        for var, result, target in assignments:
7099            target.generate_assignment_code(result, code)
7100            var.release(code)
7101
7102
7103class SetIterationNextNode(Node):
7104    # Helper node for calling _PySet_NextEntry() inside of a WhileStatNode
7105    # and checking the set size for changes.  Created in Optimize.py.
7106    child_attrs = ['set_obj', 'expected_size', 'pos_index_var',
7107                   'coerced_value_var', 'value_target', 'is_set_flag']
7108
7109    coerced_value_var = value_ref = None
7110
7111    def __init__(self, set_obj, expected_size, pos_index_var, value_target, is_set_flag):
7112        Node.__init__(
7113            self, set_obj.pos,
7114            set_obj=set_obj,
7115            expected_size=expected_size,
7116            pos_index_var=pos_index_var,
7117            value_target=value_target,
7118            is_set_flag=is_set_flag,
7119            is_temp=True,
7120            type=PyrexTypes.c_bint_type)
7121
7122    def analyse_expressions(self, env):
7123        from . import ExprNodes
7124        self.set_obj = self.set_obj.analyse_types(env)
7125        self.expected_size = self.expected_size.analyse_types(env)
7126        self.pos_index_var = self.pos_index_var.analyse_types(env)
7127        self.value_target = self.value_target.analyse_target_types(env)
7128        self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
7129        self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
7130        self.is_set_flag = self.is_set_flag.analyse_types(env)
7131        return self
7132
7133    def generate_function_definitions(self, env, code):
7134        self.set_obj.generate_function_definitions(env, code)
7135
7136    def generate_execution_code(self, code):
7137        code.globalstate.use_utility_code(UtilityCode.load_cached("set_iter", "Optimize.c"))
7138        self.set_obj.generate_evaluation_code(code)
7139
7140        value_ref = self.value_ref
7141        value_ref.allocate(code)
7142
7143        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
7144        code.putln("%s = __Pyx_set_iter_next(%s, %s, &%s, &%s, %s);" % (
7145            result_temp,
7146            self.set_obj.py_result(),
7147            self.expected_size.result(),
7148            self.pos_index_var.result(),
7149            value_ref.result(),
7150            self.is_set_flag.result()
7151        ))
7152        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
7153        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
7154        code.funcstate.release_temp(result_temp)
7155
7156        # evaluate all coercions before the assignments
7157        value_ref.generate_gotref(code)
7158        self.coerced_value_var.generate_evaluation_code(code)
7159        self.value_target.generate_assignment_code(self.coerced_value_var, code)
7160        value_ref.release(code)
7161
7162
7163def ForStatNode(pos, **kw):
7164    if 'iterator' in kw:
7165        if kw['iterator'].is_async:
7166            return AsyncForStatNode(pos, **kw)
7167        else:
7168            return ForInStatNode(pos, **kw)
7169    else:
7170        return ForFromStatNode(pos, **kw)
7171
7172
7173class _ForInStatNode(LoopNode, StatNode):
7174    #  Base class of 'for-in' statements.
7175    #
7176    #  target        ExprNode
7177    #  iterator      IteratorNode | AIterAwaitExprNode(AsyncIteratorNode)
7178    #  body          StatNode
7179    #  else_clause   StatNode
7180    #  item          NextNode | AwaitExprNode(AsyncNextNode)
7181    #  is_async      boolean        true for 'async for' statements
7182
7183    child_attrs = ["target", "item", "iterator", "body", "else_clause"]
7184    item = None
7185    is_async = False
7186
7187    def _create_item_node(self):
7188        raise NotImplementedError("must be implemented by subclasses")
7189
7190    def analyse_declarations(self, env):
7191        self.target.analyse_target_declaration(env)
7192        self.body.analyse_declarations(env)
7193        if self.else_clause:
7194            self.else_clause.analyse_declarations(env)
7195        self._create_item_node()
7196
7197    def analyse_expressions(self, env):
7198        self.target = self.target.analyse_target_types(env)
7199        self.iterator = self.iterator.analyse_expressions(env)
7200        self._create_item_node()  # must rewrap self.item after analysis
7201        self.item = self.item.analyse_expressions(env)
7202        if (not self.is_async and
7203                (self.iterator.type.is_ptr or self.iterator.type.is_array) and
7204                self.target.type.assignable_from(self.iterator.type)):
7205            # C array slice optimization.
7206            pass
7207        else:
7208            self.item = self.item.coerce_to(self.target.type, env)
7209        self.body = self.body.analyse_expressions(env)
7210        if self.else_clause:
7211            self.else_clause = self.else_clause.analyse_expressions(env)
7212        return self
7213
7214    def generate_execution_code(self, code):
7215        code.mark_pos(self.pos)
7216        old_loop_labels = code.new_loop_labels()
7217        self.iterator.generate_evaluation_code(code)
7218        code.putln("for (;;) {")
7219        self.item.generate_evaluation_code(code)
7220        self.target.generate_assignment_code(self.item, code)
7221        self.body.generate_execution_code(code)
7222        code.mark_pos(self.pos)
7223        code.put_label(code.continue_label)
7224        code.putln("}")
7225        break_label = code.break_label
7226        code.set_loop_labels(old_loop_labels)
7227
7228        if self.else_clause:
7229            # in nested loops, the 'else' block can contain a
7230            # 'continue' statement for the outer loop, but we may need
7231            # to generate cleanup code before taking that path, so we
7232            # intercept it here
7233            orig_continue_label = code.continue_label
7234            code.continue_label = code.new_label('outer_continue')
7235
7236            code.putln("/*else*/ {")
7237            self.else_clause.generate_execution_code(code)
7238            code.putln("}")
7239
7240            if code.label_used(code.continue_label):
7241                code.put_goto(break_label)
7242                code.mark_pos(self.pos)
7243                code.put_label(code.continue_label)
7244                self.iterator.generate_disposal_code(code)
7245                code.put_goto(orig_continue_label)
7246            code.set_loop_labels(old_loop_labels)
7247
7248        code.mark_pos(self.pos)
7249        if code.label_used(break_label):
7250            code.put_label(break_label)
7251        self.iterator.generate_disposal_code(code)
7252        self.iterator.free_temps(code)
7253
7254    def generate_function_definitions(self, env, code):
7255        self.target.generate_function_definitions(env, code)
7256        self.iterator.generate_function_definitions(env, code)
7257        self.body.generate_function_definitions(env, code)
7258        if self.else_clause is not None:
7259            self.else_clause.generate_function_definitions(env, code)
7260
7261    def annotate(self, code):
7262        self.target.annotate(code)
7263        self.iterator.annotate(code)
7264        self.body.annotate(code)
7265        if self.else_clause:
7266            self.else_clause.annotate(code)
7267        self.item.annotate(code)
7268
7269
7270class ForInStatNode(_ForInStatNode):
7271    #  'for' statement
7272
7273    is_async = False
7274
7275    def _create_item_node(self):
7276        from .ExprNodes import NextNode
7277        self.item = NextNode(self.iterator)
7278
7279
7280class AsyncForStatNode(_ForInStatNode):
7281    #  'async for' statement
7282    #
7283    #  iterator      AIterAwaitExprNode(AsyncIteratorNode)
7284    #  item          AwaitIterNextExprNode(AsyncIteratorNode)
7285
7286    is_async = True
7287
7288    def __init__(self, pos, **kw):
7289        assert 'item' not in kw
7290        from . import ExprNodes
7291        # AwaitExprNodes must appear before running MarkClosureVisitor
7292        kw['item'] = ExprNodes.AwaitIterNextExprNode(kw['iterator'].pos, arg=None)
7293        _ForInStatNode.__init__(self, pos, **kw)
7294
7295    def _create_item_node(self):
7296        from . import ExprNodes
7297        self.item.arg = ExprNodes.AsyncNextNode(self.iterator)
7298
7299
7300class ForFromStatNode(LoopNode, StatNode):
7301    #  for name from expr rel name rel expr
7302    #
7303    #  target        NameNode
7304    #  bound1        ExprNode
7305    #  relation1     string
7306    #  relation2     string
7307    #  bound2        ExprNode
7308    #  step          ExprNode or None
7309    #  body          StatNode
7310    #  else_clause   StatNode or None
7311    #
7312    #  Used internally:
7313    #
7314    #  from_range         bool
7315    #  is_py_target       bool
7316    #  loopvar_node       ExprNode (usually a NameNode or temp node)
7317    #  py_loopvar_node    PyTempNode or None
7318    child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
7319
7320    is_py_target = False
7321    loopvar_node = None
7322    py_loopvar_node = None
7323    from_range = False
7324
7325    gil_message = "For-loop using object bounds or target"
7326
7327    def nogil_check(self, env):
7328        for x in (self.target, self.bound1, self.bound2):
7329            if x.type.is_pyobject:
7330                self.gil_error()
7331
7332    def analyse_declarations(self, env):
7333        self.target.analyse_target_declaration(env)
7334        self.body.analyse_declarations(env)
7335        if self.else_clause:
7336            self.else_clause.analyse_declarations(env)
7337
7338    def analyse_expressions(self, env):
7339        from . import ExprNodes
7340        self.target = self.target.analyse_target_types(env)
7341        self.bound1 = self.bound1.analyse_types(env)
7342        self.bound2 = self.bound2.analyse_types(env)
7343        if self.step is not None:
7344            if isinstance(self.step, ExprNodes.UnaryMinusNode):
7345                warning(self.step.pos, "Probable infinite loop in for-from-by statement. "
7346                        "Consider switching the directions of the relations.", 2)
7347            self.step = self.step.analyse_types(env)
7348
7349        self.set_up_loop(env)
7350        target_type = self.target.type
7351        if not (target_type.is_pyobject or target_type.is_numeric):
7352            error(self.target.pos, "for-from loop variable must be c numeric type or Python object")
7353
7354        self.body = self.body.analyse_expressions(env)
7355        if self.else_clause:
7356            self.else_clause = self.else_clause.analyse_expressions(env)
7357        return self
7358
7359    def set_up_loop(self, env):
7360        from . import ExprNodes
7361
7362        target_type = self.target.type
7363        if target_type.is_numeric:
7364            loop_type = target_type
7365        else:
7366            if target_type.is_enum:
7367                warning(self.target.pos,
7368                        "Integer loops over enum values are fragile. Please cast to a safe integer type instead.")
7369            loop_type = PyrexTypes.c_long_type if target_type.is_pyobject else PyrexTypes.c_int_type
7370            if not self.bound1.type.is_pyobject:
7371                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
7372            if not self.bound2.type.is_pyobject:
7373                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
7374            if self.step is not None and not self.step.type.is_pyobject:
7375                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
7376        self.bound1 = self.bound1.coerce_to(loop_type, env)
7377        self.bound2 = self.bound2.coerce_to(loop_type, env)
7378        if not self.bound2.is_literal:
7379            self.bound2 = self.bound2.coerce_to_temp(env)
7380        if self.step is not None:
7381            self.step = self.step.coerce_to(loop_type, env)
7382            if not self.step.is_literal:
7383                self.step = self.step.coerce_to_temp(env)
7384
7385        if target_type.is_numeric or target_type.is_enum:
7386            self.is_py_target = False
7387            if isinstance(self.target, ExprNodes.BufferIndexNode):
7388                raise error(self.pos, "Buffer or memoryview slicing/indexing not allowed as for-loop target.")
7389            self.loopvar_node = self.target
7390            self.py_loopvar_node = None
7391        else:
7392            self.is_py_target = True
7393            c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
7394            self.loopvar_node = c_loopvar_node
7395            self.py_loopvar_node = ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
7396
7397    def generate_execution_code(self, code):
7398        code.mark_pos(self.pos)
7399        old_loop_labels = code.new_loop_labels()
7400        from_range = self.from_range
7401        self.bound1.generate_evaluation_code(code)
7402        self.bound2.generate_evaluation_code(code)
7403        offset, incop = self.relation_table[self.relation1]
7404        if self.step is not None:
7405            self.step.generate_evaluation_code(code)
7406            step = self.step.result()
7407            incop = "%s=%s" % (incop[0], step)  # e.g. '++' => '+= STEP'
7408        else:
7409            step = '1'
7410
7411        from . import ExprNodes
7412        if isinstance(self.loopvar_node, ExprNodes.TempNode):
7413            self.loopvar_node.allocate(code)
7414        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
7415            self.py_loopvar_node.allocate(code)
7416
7417        loopvar_type = PyrexTypes.c_long_type if self.target.type.is_enum else self.target.type
7418
7419        if from_range and not self.is_py_target:
7420            loopvar_name = code.funcstate.allocate_temp(loopvar_type, False)
7421        else:
7422            loopvar_name = self.loopvar_node.result()
7423        if loopvar_type.is_int and not loopvar_type.signed and self.relation2[0] == '>':
7424            # Handle the case where the endpoint of an unsigned int iteration
7425            # is within step of 0.
7426            code.putln("for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
7427                loopvar_name,
7428                self.bound1.result(), offset, step,
7429                loopvar_name, self.relation2, self.bound2.result(), step,
7430                loopvar_name, incop))
7431        else:
7432            code.putln("for (%s = %s%s; %s %s %s; %s%s) {" % (
7433                loopvar_name,
7434                self.bound1.result(), offset,
7435                loopvar_name, self.relation2, self.bound2.result(),
7436                loopvar_name, incop))
7437
7438        coerced_loopvar_node = self.py_loopvar_node
7439        if coerced_loopvar_node is None and from_range:
7440            coerced_loopvar_node = ExprNodes.RawCNameExprNode(self.target.pos, loopvar_type, loopvar_name)
7441        if coerced_loopvar_node is not None:
7442            coerced_loopvar_node.generate_evaluation_code(code)
7443            self.target.generate_assignment_code(coerced_loopvar_node, code)
7444
7445        self.body.generate_execution_code(code)
7446        code.put_label(code.continue_label)
7447
7448        if not from_range and self.py_loopvar_node:
7449            # This mess is to make for..from loops with python targets behave
7450            # exactly like those with C targets with regards to re-assignment
7451            # of the loop variable.
7452            if self.target.entry.is_pyglobal:
7453                # We know target is a NameNode, this is the only ugly case.
7454                target_node = ExprNodes.PyTempNode(self.target.pos, None)
7455                target_node.allocate(code)
7456                interned_cname = code.intern_identifier(self.target.entry.name)
7457                if self.target.entry.scope.is_module_scope:
7458                    code.globalstate.use_utility_code(
7459                        UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
7460                    lookup_func = '__Pyx_GetModuleGlobalName(%s, %s); %s'
7461                else:
7462                    code.globalstate.use_utility_code(
7463                        UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
7464                    lookup_func = '__Pyx_GetNameInClass(%s, {}, %s); %s'.format(
7465                        self.target.entry.scope.namespace_cname)
7466                code.putln(lookup_func % (
7467                    target_node.result(),
7468                    interned_cname,
7469                    code.error_goto_if_null(target_node.result(), self.target.pos)))
7470                target_node.generate_gotref(code)
7471            else:
7472                target_node = self.target
7473            from_py_node = ExprNodes.CoerceFromPyTypeNode(
7474                self.loopvar_node.type, target_node, self.target.entry.scope)
7475            from_py_node.temp_code = loopvar_name
7476            from_py_node.generate_result_code(code)
7477            if self.target.entry.is_pyglobal:
7478                code.put_decref(target_node.result(), target_node.type)
7479                target_node.release(code)
7480
7481        code.putln("}")
7482
7483        if not from_range and self.py_loopvar_node:
7484            # This is potentially wasteful, but we don't want the semantics to
7485            # depend on whether or not the loop is a python type.
7486            self.py_loopvar_node.generate_evaluation_code(code)
7487            self.target.generate_assignment_code(self.py_loopvar_node, code)
7488        if from_range and not self.is_py_target:
7489            code.funcstate.release_temp(loopvar_name)
7490
7491        break_label = code.break_label
7492        code.set_loop_labels(old_loop_labels)
7493        if self.else_clause:
7494            code.putln("/*else*/ {")
7495            self.else_clause.generate_execution_code(code)
7496            code.putln("}")
7497        code.put_label(break_label)
7498        self.bound1.generate_disposal_code(code)
7499        self.bound1.free_temps(code)
7500        self.bound2.generate_disposal_code(code)
7501        self.bound2.free_temps(code)
7502        if isinstance(self.loopvar_node, ExprNodes.TempNode):
7503            self.loopvar_node.release(code)
7504        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
7505            self.py_loopvar_node.release(code)
7506        if self.step is not None:
7507            self.step.generate_disposal_code(code)
7508            self.step.free_temps(code)
7509
7510    relation_table = {
7511        # {relop : (initial offset, increment op)}
7512        '<=': ("",   "++"),
7513        '<' : ("+1", "++"),
7514        '>=': ("",   "--"),
7515        '>' : ("-1", "--"),
7516    }
7517
7518    def generate_function_definitions(self, env, code):
7519        self.target.generate_function_definitions(env, code)
7520        self.bound1.generate_function_definitions(env, code)
7521        self.bound2.generate_function_definitions(env, code)
7522        if self.step is not None:
7523            self.step.generate_function_definitions(env, code)
7524        self.body.generate_function_definitions(env, code)
7525        if self.else_clause is not None:
7526            self.else_clause.generate_function_definitions(env, code)
7527
7528    def annotate(self, code):
7529        self.target.annotate(code)
7530        self.bound1.annotate(code)
7531        self.bound2.annotate(code)
7532        if self.step:
7533            self.step.annotate(code)
7534        self.body.annotate(code)
7535        if self.else_clause:
7536            self.else_clause.annotate(code)
7537
7538
7539class WithStatNode(StatNode):
7540    """
7541    Represents a Python with statement.
7542
7543    Implemented by the WithTransform as follows:
7544
7545        MGR = EXPR
7546        EXIT = MGR.__exit__
7547        VALUE = MGR.__enter__()
7548        EXC = True
7549        try:
7550            try:
7551                TARGET = VALUE  # optional
7552                BODY
7553            except:
7554                EXC = False
7555                if not EXIT(*EXCINFO):
7556                    raise
7557        finally:
7558            if EXC:
7559                EXIT(None, None, None)
7560            MGR = EXIT = VALUE = None
7561    """
7562    #  manager          The with statement manager object
7563    #  target           ExprNode  the target lhs of the __enter__() call
7564    #  body             StatNode
7565    #  enter_call       ExprNode  the call to the __enter__() method
7566    #  exit_var         String    the cname of the __exit__() method reference
7567
7568    child_attrs = ["manager", "enter_call", "target", "body"]
7569
7570    enter_call = None
7571    target_temp = None
7572
7573    def analyse_declarations(self, env):
7574        self.manager.analyse_declarations(env)
7575        self.enter_call.analyse_declarations(env)
7576        self.body.analyse_declarations(env)
7577
7578    def analyse_expressions(self, env):
7579        self.manager = self.manager.analyse_types(env)
7580        self.enter_call = self.enter_call.analyse_types(env)
7581        if self.target:
7582            # set up target_temp before descending into body (which uses it)
7583            from .ExprNodes import TempNode
7584            self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
7585        self.body = self.body.analyse_expressions(env)
7586        return self
7587
7588    def generate_function_definitions(self, env, code):
7589        self.manager.generate_function_definitions(env, code)
7590        self.enter_call.generate_function_definitions(env, code)
7591        self.body.generate_function_definitions(env, code)
7592
7593    def generate_execution_code(self, code):
7594        code.mark_pos(self.pos)
7595        code.putln("/*with:*/ {")
7596        self.manager.generate_evaluation_code(code)
7597        self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
7598        code.globalstate.use_utility_code(
7599            UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
7600        code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
7601            self.exit_var,
7602            self.manager.py_result(),
7603            code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')),
7604            code.error_goto_if_null(self.exit_var, self.pos),
7605            ))
7606        code.put_gotref(self.exit_var, py_object_type)
7607
7608        # need to free exit_var in the face of exceptions during setup
7609        old_error_label = code.new_error_label()
7610        intermediate_error_label = code.error_label
7611
7612        self.enter_call.generate_evaluation_code(code)
7613        if self.target:
7614            # The temp result will be cleaned up by the WithTargetAssignmentStatNode
7615            # after assigning its result to the target of the 'with' statement.
7616            self.target_temp.allocate(code)
7617            self.enter_call.make_owned_reference(code)
7618            code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
7619            self.enter_call.generate_post_assignment_code(code)
7620        else:
7621            self.enter_call.generate_disposal_code(code)
7622        self.enter_call.free_temps(code)
7623
7624        self.manager.generate_disposal_code(code)
7625        self.manager.free_temps(code)
7626
7627        code.error_label = old_error_label
7628        self.body.generate_execution_code(code)
7629
7630        if code.label_used(intermediate_error_label):
7631            step_over_label = code.new_label()
7632            code.put_goto(step_over_label)
7633            code.put_label(intermediate_error_label)
7634            code.put_decref_clear(self.exit_var, py_object_type)
7635            code.put_goto(old_error_label)
7636            code.put_label(step_over_label)
7637
7638        code.funcstate.release_temp(self.exit_var)
7639        code.putln('}')
7640
7641
7642class WithTargetAssignmentStatNode(AssignmentNode):
7643    # The target assignment of the 'with' statement value (return
7644    # value of the __enter__() call).
7645    #
7646    # This is a special cased assignment that properly cleans up the RHS.
7647    #
7648    # lhs       ExprNode      the assignment target
7649    # rhs       ExprNode      a (coerced) TempNode for the rhs (from WithStatNode)
7650    # with_node WithStatNode  the surrounding with-statement
7651
7652    child_attrs = ["rhs", "lhs"]
7653    with_node = None
7654    rhs = None
7655
7656    def analyse_declarations(self, env):
7657        self.lhs.analyse_target_declaration(env)
7658
7659    def analyse_expressions(self, env):
7660        self.lhs = self.lhs.analyse_target_types(env)
7661        self.lhs.gil_assignment_check(env)
7662        self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
7663        return self
7664
7665    def generate_execution_code(self, code):
7666        self.rhs.generate_evaluation_code(code)
7667        self.lhs.generate_assignment_code(self.rhs, code)
7668        self.with_node.target_temp.release(code)
7669
7670    def annotate(self, code):
7671        self.lhs.annotate(code)
7672        self.rhs.annotate(code)
7673
7674
7675class TryExceptStatNode(StatNode):
7676    #  try .. except statement
7677    #
7678    #  body             StatNode
7679    #  except_clauses   [ExceptClauseNode]
7680    #  else_clause      StatNode or None
7681
7682    child_attrs = ["body", "except_clauses", "else_clause"]
7683    in_generator = False
7684
7685    def analyse_declarations(self, env):
7686        self.body.analyse_declarations(env)
7687        for except_clause in self.except_clauses:
7688            except_clause.analyse_declarations(env)
7689        if self.else_clause:
7690            self.else_clause.analyse_declarations(env)
7691
7692    def analyse_expressions(self, env):
7693        self.body = self.body.analyse_expressions(env)
7694        default_clause_seen = 0
7695        for i, except_clause in enumerate(self.except_clauses):
7696            except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
7697            if default_clause_seen:
7698                error(except_clause.pos, "default 'except:' must be last")
7699            if not except_clause.pattern:
7700                default_clause_seen = 1
7701        self.has_default_clause = default_clause_seen
7702        if self.else_clause:
7703            self.else_clause = self.else_clause.analyse_expressions(env)
7704        return self
7705
7706    nogil_check = Node.gil_error
7707    gil_message = "Try-except statement"
7708
7709    def generate_execution_code(self, code):
7710        code.mark_pos(self.pos)  # before changing the error label, in case of tracing errors
7711        code.putln("{")
7712
7713        old_return_label = code.return_label
7714        old_break_label = code.break_label
7715        old_continue_label = code.continue_label
7716        old_error_label = code.new_error_label()
7717        our_error_label = code.error_label
7718        except_end_label = code.new_label('exception_handled')
7719        except_error_label = code.new_label('except_error')
7720        except_return_label = code.new_label('except_return')
7721        try_return_label = code.new_label('try_return')
7722        try_break_label = code.new_label('try_break') if old_break_label else None
7723        try_continue_label = code.new_label('try_continue') if old_continue_label else None
7724        try_end_label = code.new_label('try_end')
7725
7726        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
7727                         for _ in range(3)]
7728        save_exc = code.insertion_point()
7729        code.putln(
7730            "/*try:*/ {")
7731        code.return_label = try_return_label
7732        code.break_label = try_break_label
7733        code.continue_label = try_continue_label
7734        self.body.generate_execution_code(code)
7735        code.mark_pos(self.pos, trace=False)
7736        code.putln(
7737            "}")
7738        temps_to_clean_up = code.funcstate.all_free_managed_temps()
7739        can_raise = code.label_used(our_error_label)
7740
7741        if can_raise:
7742            # inject code before the try block to save away the exception state
7743            code.globalstate.use_utility_code(reset_exception_utility_code)
7744            if not self.in_generator:
7745                save_exc.putln("__Pyx_PyThreadState_declare")
7746                save_exc.putln("__Pyx_PyThreadState_assign")
7747            save_exc.putln("__Pyx_ExceptionSave(%s);" % (
7748                ', '.join(['&%s' % var for var in exc_save_vars])))
7749            for var in exc_save_vars:
7750                save_exc.put_xgotref(var, py_object_type)
7751
7752            def restore_saved_exception():
7753                for name in exc_save_vars:
7754                    code.put_xgiveref(name, py_object_type)
7755                code.putln("__Pyx_ExceptionReset(%s);" %
7756                           ', '.join(exc_save_vars))
7757        else:
7758            # try block cannot raise exceptions, but we had to allocate the temps above,
7759            # so just keep the C compiler from complaining about them being unused
7760            mark_vars_used = ["(void)%s;" % var for var in exc_save_vars]
7761            save_exc.putln("%s /* mark used */" % ' '.join(mark_vars_used))
7762
7763            def restore_saved_exception():
7764                pass
7765
7766        code.error_label = except_error_label
7767        code.return_label = except_return_label
7768        normal_case_terminates = self.body.is_terminator
7769        if self.else_clause:
7770            code.mark_pos(self.else_clause.pos)
7771            code.putln(
7772                "/*else:*/ {")
7773            self.else_clause.generate_execution_code(code)
7774            code.putln(
7775                "}")
7776            if not normal_case_terminates:
7777                normal_case_terminates = self.else_clause.is_terminator
7778
7779        if can_raise:
7780            if not normal_case_terminates:
7781                for var in exc_save_vars:
7782                    code.put_xdecref_clear(var, py_object_type)
7783                code.put_goto(try_end_label)
7784            code.put_label(our_error_label)
7785            for temp_name, temp_type in temps_to_clean_up:
7786                code.put_xdecref_clear(temp_name, temp_type)
7787
7788            outer_except = code.funcstate.current_except
7789            # Currently points to self, but the ExceptClauseNode would also be ok. Change if needed.
7790            code.funcstate.current_except = self
7791            for except_clause in self.except_clauses:
7792                except_clause.generate_handling_code(code, except_end_label)
7793            code.funcstate.current_except = outer_except
7794
7795            if not self.has_default_clause:
7796                code.put_goto(except_error_label)
7797
7798        for exit_label, old_label in [(except_error_label, old_error_label),
7799                                      (try_break_label, old_break_label),
7800                                      (try_continue_label, old_continue_label),
7801                                      (try_return_label, old_return_label),
7802                                      (except_return_label, old_return_label)]:
7803            if code.label_used(exit_label):
7804                if not normal_case_terminates and not code.label_used(try_end_label):
7805                    code.put_goto(try_end_label)
7806                code.put_label(exit_label)
7807                code.mark_pos(self.pos, trace=False)
7808                if can_raise:
7809                    restore_saved_exception()
7810                code.put_goto(old_label)
7811
7812        if code.label_used(except_end_label):
7813            if not normal_case_terminates and not code.label_used(try_end_label):
7814                code.put_goto(try_end_label)
7815            code.put_label(except_end_label)
7816            if can_raise:
7817                restore_saved_exception()
7818        if code.label_used(try_end_label):
7819            code.put_label(try_end_label)
7820        code.putln("}")
7821
7822        for cname in exc_save_vars:
7823            code.funcstate.release_temp(cname)
7824
7825        code.return_label = old_return_label
7826        code.break_label = old_break_label
7827        code.continue_label = old_continue_label
7828        code.error_label = old_error_label
7829
7830    def generate_function_definitions(self, env, code):
7831        self.body.generate_function_definitions(env, code)
7832        for except_clause in self.except_clauses:
7833            except_clause.generate_function_definitions(env, code)
7834        if self.else_clause is not None:
7835            self.else_clause.generate_function_definitions(env, code)
7836
7837    def annotate(self, code):
7838        self.body.annotate(code)
7839        for except_node in self.except_clauses:
7840            except_node.annotate(code)
7841        if self.else_clause:
7842            self.else_clause.annotate(code)
7843
7844
7845class ExceptClauseNode(Node):
7846    #  Part of try ... except statement.
7847    #
7848    #  pattern        [ExprNode]
7849    #  target         ExprNode or None
7850    #  body           StatNode
7851    #  excinfo_target TupleNode(3*ResultRefNode) or None   optional target for exception info (not owned here!)
7852    #  match_flag     string             result of exception match
7853    #  exc_value      ExcValueNode       used internally
7854    #  function_name  string             qualified name of enclosing function
7855    #  exc_vars       (string * 3)       local exception variables
7856    #  is_except_as   bool               Py3-style "except ... as xyz"
7857
7858    # excinfo_target is never set by the parser, but can be set by a transform
7859    # in order to extract more extensive information about the exception as a
7860    # sys.exc_info()-style tuple into a target variable
7861
7862    child_attrs = ["pattern", "target", "body", "exc_value"]
7863
7864    exc_value = None
7865    excinfo_target = None
7866    is_except_as = False
7867
7868    def analyse_declarations(self, env):
7869        if self.target:
7870            self.target.analyse_target_declaration(env)
7871        self.body.analyse_declarations(env)
7872
7873    def analyse_expressions(self, env):
7874        self.function_name = env.qualified_name
7875        if self.pattern:
7876            # normalise/unpack self.pattern into a list
7877            for i, pattern in enumerate(self.pattern):
7878                pattern = pattern.analyse_expressions(env)
7879                self.pattern[i] = pattern.coerce_to_pyobject(env)
7880
7881        if self.target:
7882            from . import ExprNodes
7883            self.exc_value = ExprNodes.ExcValueNode(self.pos)
7884            self.target = self.target.analyse_target_expression(env, self.exc_value)
7885
7886        self.body = self.body.analyse_expressions(env)
7887        return self
7888
7889    def generate_handling_code(self, code, end_label):
7890        code.mark_pos(self.pos)
7891
7892        if self.pattern:
7893            has_non_literals = not all(
7894                pattern.is_literal or pattern.is_simple() and not pattern.is_temp
7895                for pattern in self.pattern)
7896
7897            if has_non_literals:
7898                # For non-trivial exception check expressions, hide the live exception from C-API calls.
7899                exc_vars = [code.funcstate.allocate_temp(py_object_type, manage_ref=True)
7900                            for _ in range(3)]
7901                code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
7902                code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % tuple(exc_vars))
7903                exc_type = exc_vars[0]
7904            else:
7905                exc_vars = exc_type = None
7906
7907            for pattern in self.pattern:
7908                pattern.generate_evaluation_code(code)
7909            patterns = [pattern.py_result() for pattern in self.pattern]
7910
7911            exc_tests = []
7912            if exc_type:
7913                code.globalstate.use_utility_code(
7914                    UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c"))
7915                if len(patterns) == 2:
7916                    exc_tests.append("__Pyx_PyErr_GivenExceptionMatches2(%s, %s, %s)" % (
7917                        exc_type, patterns[0], patterns[1],
7918                    ))
7919                else:
7920                    exc_tests.extend(
7921                        "__Pyx_PyErr_GivenExceptionMatches(%s, %s)" % (exc_type, pattern)
7922                        for pattern in patterns
7923                    )
7924            elif len(patterns) == 2:
7925                code.globalstate.use_utility_code(
7926                    UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c"))
7927                exc_tests.append("__Pyx_PyErr_ExceptionMatches2(%s, %s)" % (
7928                    patterns[0], patterns[1],
7929                ))
7930            else:
7931                code.globalstate.use_utility_code(
7932                    UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c"))
7933                exc_tests.extend(
7934                    "__Pyx_PyErr_ExceptionMatches(%s)" % pattern
7935                    for pattern in patterns
7936                )
7937
7938            match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
7939            code.putln("%s = %s;" % (match_flag, ' || '.join(exc_tests)))
7940            for pattern in self.pattern:
7941                pattern.generate_disposal_code(code)
7942                pattern.free_temps(code)
7943
7944            if exc_vars:
7945                code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(exc_vars))
7946                code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
7947                for temp in exc_vars:
7948                    code.funcstate.release_temp(temp)
7949
7950            code.putln(
7951                "if (%s) {" %
7952                    match_flag)
7953            code.funcstate.release_temp(match_flag)
7954        else:
7955            code.putln("/*except:*/ {")
7956
7957        if (not getattr(self.body, 'stats', True)
7958                and self.excinfo_target is None
7959                and self.target is None):
7960            # most simple case: no exception variable, empty body (pass)
7961            # => reset the exception state, done
7962            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
7963            code.putln("__Pyx_ErrRestore(0,0,0);")
7964            code.put_goto(end_label)
7965            code.putln("}")
7966            return
7967
7968        exc_vars = [code.funcstate.allocate_temp(py_object_type, manage_ref=True)
7969                    for _ in range(3)]
7970        code.put_add_traceback(self.function_name)
7971        # We always have to fetch the exception value even if
7972        # there is no target, because this also normalises the
7973        # exception and stores it in the thread state.
7974        code.globalstate.use_utility_code(get_exception_utility_code)
7975        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
7976        code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
7977            exc_args, code.error_goto(self.pos)))
7978        for var in exc_vars:
7979            code.put_gotref(var, py_object_type)
7980        if self.target:
7981            self.exc_value.set_var(exc_vars[1])
7982            self.exc_value.generate_evaluation_code(code)
7983            self.target.generate_assignment_code(self.exc_value, code)
7984        if self.excinfo_target is not None:
7985            for tempvar, node in zip(exc_vars, self.excinfo_target.args):
7986                node.set_var(tempvar)
7987
7988        old_break_label, old_continue_label = code.break_label, code.continue_label
7989        code.break_label = code.new_label('except_break')
7990        code.continue_label = code.new_label('except_continue')
7991
7992        old_exc_vars = code.funcstate.exc_vars
7993        code.funcstate.exc_vars = exc_vars
7994        self.body.generate_execution_code(code)
7995        code.funcstate.exc_vars = old_exc_vars
7996
7997        if not self.body.is_terminator:
7998            for var in exc_vars:
7999                # FIXME: XDECREF() is needed to allow re-raising (which clears the exc_vars),
8000                # but I don't think it's the right solution.
8001                code.put_xdecref_clear(var, py_object_type)
8002            code.put_goto(end_label)
8003
8004        for new_label, old_label in [(code.break_label, old_break_label),
8005                                     (code.continue_label, old_continue_label)]:
8006            if code.label_used(new_label):
8007                code.put_label(new_label)
8008                for var in exc_vars:
8009                    code.put_decref_clear(var, py_object_type)
8010                code.put_goto(old_label)
8011        code.break_label = old_break_label
8012        code.continue_label = old_continue_label
8013
8014        for temp in exc_vars:
8015            code.funcstate.release_temp(temp)
8016
8017        code.putln(
8018            "}")
8019
8020    def generate_function_definitions(self, env, code):
8021        if self.target is not None:
8022            self.target.generate_function_definitions(env, code)
8023        self.body.generate_function_definitions(env, code)
8024
8025    def annotate(self, code):
8026        if self.pattern:
8027            for pattern in self.pattern:
8028                pattern.annotate(code)
8029        if self.target:
8030            self.target.annotate(code)
8031        self.body.annotate(code)
8032
8033
8034class TryFinallyStatNode(StatNode):
8035    #  try ... finally statement
8036    #
8037    #  body             StatNode
8038    #  finally_clause   StatNode
8039    #  finally_except_clause  deep-copy of finally_clause for exception case
8040    #  in_generator     inside of generator => must store away current exception also in return case
8041    #
8042    #  Each of the continue, break, return and error gotos runs
8043    #  into its own deep-copy of the finally block code.
8044    #  In addition, if we're doing an error, we save the
8045    #  exception on entry to the finally block and restore
8046    #  it on exit.
8047
8048    child_attrs = ["body", "finally_clause", "finally_except_clause"]
8049
8050    preserve_exception = 1
8051
8052    # handle exception case, in addition to return/break/continue
8053    handle_error_case = True
8054    func_return_type = None
8055    finally_except_clause = None
8056
8057    is_try_finally_in_nogil = False
8058    in_generator = False
8059
8060    @staticmethod
8061    def create_analysed(pos, env, body, finally_clause):
8062        node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
8063        return node
8064
8065    def analyse_declarations(self, env):
8066        self.body.analyse_declarations(env)
8067        self.finally_except_clause = copy.deepcopy(self.finally_clause)
8068        self.finally_except_clause.analyse_declarations(env)
8069        self.finally_clause.analyse_declarations(env)
8070
8071    def analyse_expressions(self, env):
8072        self.body = self.body.analyse_expressions(env)
8073        self.finally_clause = self.finally_clause.analyse_expressions(env)
8074        self.finally_except_clause = self.finally_except_clause.analyse_expressions(env)
8075        if env.return_type and not env.return_type.is_void:
8076            self.func_return_type = env.return_type
8077        return self
8078
8079    nogil_check = Node.gil_error
8080    gil_message = "Try-finally statement"
8081
8082    def generate_execution_code(self, code):
8083        code.mark_pos(self.pos)  # before changing the error label, in case of tracing errors
8084        code.putln("/*try:*/ {")
8085
8086        old_error_label = code.error_label
8087        old_labels = code.all_new_labels()
8088        new_labels = code.get_all_labels()
8089        new_error_label = code.error_label
8090        if not self.handle_error_case:
8091            code.error_label = old_error_label
8092        catch_label = code.new_label()
8093
8094        was_in_try_finally = code.funcstate.in_try_finally
8095        code.funcstate.in_try_finally = 1
8096
8097        self.body.generate_execution_code(code)
8098
8099        code.funcstate.in_try_finally = was_in_try_finally
8100        code.putln("}")
8101
8102        temps_to_clean_up = code.funcstate.all_free_managed_temps()
8103        code.mark_pos(self.finally_clause.pos)
8104        code.putln("/*finally:*/ {")
8105
8106        # Reset labels only after writing out a potential line trace call for correct nogil error handling.
8107        code.set_all_labels(old_labels)
8108
8109        def fresh_finally_clause(_next=[self.finally_clause]):
8110            # generate the original subtree once and always keep a fresh copy
8111            node = _next[0]
8112            node_copy = copy.deepcopy(node)
8113            if node is self.finally_clause:
8114                _next[0] = node_copy
8115            else:
8116                node = node_copy
8117            return node
8118
8119        preserve_error = self.preserve_exception and code.label_used(new_error_label)
8120        needs_success_cleanup = not self.finally_clause.is_terminator
8121
8122        if not self.body.is_terminator:
8123            code.putln('/*normal exit:*/{')
8124            fresh_finally_clause().generate_execution_code(code)
8125            if not self.finally_clause.is_terminator:
8126                code.put_goto(catch_label)
8127            code.putln('}')
8128
8129        if preserve_error:
8130            code.put_label(new_error_label)
8131            code.putln('/*exception exit:*/{')
8132            if not self.in_generator:
8133                code.putln("__Pyx_PyThreadState_declare")
8134            if self.is_try_finally_in_nogil:
8135                code.declare_gilstate()
8136            if needs_success_cleanup:
8137                exc_lineno_cnames = tuple([
8138                    code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
8139                    for _ in range(2)])
8140                exc_filename_cname = code.funcstate.allocate_temp(
8141                    PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
8142                    manage_ref=False)
8143            else:
8144                exc_lineno_cnames = exc_filename_cname = None
8145            exc_vars = tuple([
8146                code.funcstate.allocate_temp(py_object_type, manage_ref=False)
8147                for _ in range(6)])
8148            self.put_error_catcher(
8149                code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
8150            finally_old_labels = code.all_new_labels()
8151
8152            code.putln('{')
8153            old_exc_vars = code.funcstate.exc_vars
8154            code.funcstate.exc_vars = exc_vars[:3]
8155            self.finally_except_clause.generate_execution_code(code)
8156            code.funcstate.exc_vars = old_exc_vars
8157            code.putln('}')
8158
8159            if needs_success_cleanup:
8160                self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
8161                if exc_lineno_cnames:
8162                    for cname in exc_lineno_cnames:
8163                        code.funcstate.release_temp(cname)
8164                if exc_filename_cname:
8165                    code.funcstate.release_temp(exc_filename_cname)
8166                code.put_goto(old_error_label)
8167
8168            for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
8169                if not code.label_used(new_label):
8170                    continue
8171                code.put_label(new_label)
8172                self.put_error_cleaner(code, exc_vars)
8173                code.put_goto(old_label)
8174
8175            for cname in exc_vars:
8176                code.funcstate.release_temp(cname)
8177            code.putln('}')
8178
8179        code.set_all_labels(old_labels)
8180        return_label = code.return_label
8181        exc_vars = ()
8182
8183        for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
8184            if not code.label_used(new_label):
8185                continue
8186            if new_label == new_error_label and preserve_error:
8187                continue  # handled above
8188
8189            code.putln('%s: {' % new_label)
8190            ret_temp = None
8191            if old_label == return_label:
8192                # return actually raises an (uncatchable) exception in generators that we must preserve
8193                if self.in_generator:
8194                    exc_vars = tuple([
8195                        code.funcstate.allocate_temp(py_object_type, manage_ref=False)
8196                        for _ in range(6)])
8197                    self.put_error_catcher(code, [], exc_vars)
8198                if not self.finally_clause.is_terminator:
8199                    # store away return value for later reuse
8200                    if (self.func_return_type and
8201                            not self.is_try_finally_in_nogil and
8202                            not isinstance(self.finally_clause, GILExitNode)):
8203                        ret_temp = code.funcstate.allocate_temp(
8204                            self.func_return_type, manage_ref=False)
8205                        code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
8206                        if self.func_return_type.is_pyobject:
8207                            code.putln("%s = 0;" % Naming.retval_cname)
8208
8209            fresh_finally_clause().generate_execution_code(code)
8210
8211            if old_label == return_label:
8212                if ret_temp:
8213                    code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
8214                    if self.func_return_type.is_pyobject:
8215                        code.putln("%s = 0;" % ret_temp)
8216                    code.funcstate.release_temp(ret_temp)
8217                if self.in_generator:
8218                    self.put_error_uncatcher(code, exc_vars)
8219                    for cname in exc_vars:
8220                        code.funcstate.release_temp(cname)
8221
8222            if not self.finally_clause.is_terminator:
8223                code.put_goto(old_label)
8224            code.putln('}')
8225
8226        # End finally
8227        code.put_label(catch_label)
8228        code.putln(
8229            "}")
8230
8231    def generate_function_definitions(self, env, code):
8232        self.body.generate_function_definitions(env, code)
8233        self.finally_clause.generate_function_definitions(env, code)
8234
8235    def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
8236                          exc_lineno_cnames=None, exc_filename_cname=None):
8237        code.globalstate.use_utility_code(restore_exception_utility_code)
8238        code.globalstate.use_utility_code(get_exception_utility_code)
8239        code.globalstate.use_utility_code(swap_exception_utility_code)
8240
8241        if self.is_try_finally_in_nogil:
8242            code.put_ensure_gil(declare_gilstate=False)
8243        code.putln("__Pyx_PyThreadState_assign")
8244
8245        code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
8246        for temp_name, type in temps_to_clean_up:
8247            code.put_xdecref_clear(temp_name, type)
8248
8249        # not using preprocessor here to avoid warnings about
8250        # unused utility functions and/or temps
8251        code.putln("if (PY_MAJOR_VERSION >= 3)"
8252                   " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
8253        code.putln("if ((PY_MAJOR_VERSION < 3) ||"
8254                   # if __Pyx_GetException() fails in Py3,
8255                   # store the newly raised exception instead
8256                   " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
8257                   "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
8258        for var in exc_vars:
8259            code.put_xgotref(var, py_object_type)
8260        if exc_lineno_cnames:
8261            code.putln("%s = %s; %s = %s; %s = %s;" % (
8262                exc_lineno_cnames[0], Naming.lineno_cname,
8263                exc_lineno_cnames[1], Naming.clineno_cname,
8264                exc_filename_cname, Naming.filename_cname))
8265
8266        if self.is_try_finally_in_nogil:
8267            code.put_release_ensured_gil()
8268
8269    def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames=None, exc_filename_cname=None):
8270        code.globalstate.use_utility_code(restore_exception_utility_code)
8271        code.globalstate.use_utility_code(reset_exception_utility_code)
8272
8273        if self.is_try_finally_in_nogil:
8274            code.put_ensure_gil(declare_gilstate=False)
8275
8276        # not using preprocessor here to avoid warnings about
8277        # unused utility functions and/or temps
8278        code.putln("if (PY_MAJOR_VERSION >= 3) {")
8279        for var in exc_vars[3:]:
8280            code.put_xgiveref(var, py_object_type)
8281        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
8282        code.putln("}")
8283        for var in exc_vars[:3]:
8284            code.put_xgiveref(var, py_object_type)
8285        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
8286
8287        if self.is_try_finally_in_nogil:
8288            code.put_release_ensured_gil()
8289
8290        code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
8291        if exc_lineno_cnames:
8292            code.putln("%s = %s; %s = %s; %s = %s;" % (
8293                Naming.lineno_cname, exc_lineno_cnames[0],
8294                Naming.clineno_cname, exc_lineno_cnames[1],
8295                Naming.filename_cname, exc_filename_cname))
8296
8297    def put_error_cleaner(self, code, exc_vars):
8298        code.globalstate.use_utility_code(reset_exception_utility_code)
8299        if self.is_try_finally_in_nogil:
8300            code.put_ensure_gil(declare_gilstate=False)
8301
8302        # not using preprocessor here to avoid warnings about
8303        # unused utility functions and/or temps
8304        code.putln("if (PY_MAJOR_VERSION >= 3) {")
8305        for var in exc_vars[3:]:
8306            code.put_xgiveref(var, py_object_type)
8307        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
8308        code.putln("}")
8309        for var in exc_vars[:3]:
8310            code.put_xdecref_clear(var, py_object_type)
8311        if self.is_try_finally_in_nogil:
8312            code.put_release_ensured_gil()
8313        code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
8314
8315    def annotate(self, code):
8316        self.body.annotate(code)
8317        self.finally_clause.annotate(code)
8318
8319
8320class NogilTryFinallyStatNode(TryFinallyStatNode):
8321    """
8322    A try/finally statement that may be used in nogil code sections.
8323    """
8324
8325    preserve_exception = False
8326    nogil_check = None
8327
8328
8329class GILStatNode(NogilTryFinallyStatNode):
8330    #  'with gil' or 'with nogil' statement
8331    #
8332    #   state   string   'gil' or 'nogil'
8333
8334    child_attrs = ["condition"] + NogilTryFinallyStatNode.child_attrs
8335    state_temp = None
8336
8337    def __init__(self, pos, state, body, condition=None):
8338        self.state = state
8339        self.condition = condition
8340        self.create_state_temp_if_needed(pos, state, body)
8341        TryFinallyStatNode.__init__(
8342            self, pos,
8343            body=body,
8344            finally_clause=GILExitNode(
8345                pos, state=state, state_temp=self.state_temp))
8346
8347    def create_state_temp_if_needed(self, pos, state, body):
8348        from .ParseTreeTransforms import YieldNodeCollector
8349        collector = YieldNodeCollector()
8350        collector.visitchildren(body)
8351        if not collector.yields:
8352            return
8353
8354        if state == 'gil':
8355            temp_type = PyrexTypes.c_gilstate_type
8356        else:
8357            temp_type = PyrexTypes.c_threadstate_ptr_type
8358        from . import ExprNodes
8359        self.state_temp = ExprNodes.TempNode(pos, temp_type)
8360
8361    def analyse_declarations(self, env):
8362        env._in_with_gil_block = (self.state == 'gil')
8363        if self.state == 'gil':
8364            env.has_with_gil_block = True
8365
8366        if self.condition is not None:
8367            self.condition.analyse_declarations(env)
8368
8369        return super(GILStatNode, self).analyse_declarations(env)
8370
8371    def analyse_expressions(self, env):
8372        env.use_utility_code(
8373            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
8374
8375        if self.condition is not None:
8376            self.condition = self.condition.analyse_expressions(env)
8377
8378        was_nogil = env.nogil
8379        env.nogil = self.state == 'nogil'
8380        node = TryFinallyStatNode.analyse_expressions(self, env)
8381        env.nogil = was_nogil
8382        return node
8383
8384    def generate_execution_code(self, code):
8385        code.mark_pos(self.pos)
8386        code.begin_block()
8387        if self.state_temp:
8388            self.state_temp.allocate(code)
8389            variable = self.state_temp.result()
8390        else:
8391            variable = None
8392
8393        old_gil_config = code.funcstate.gil_owned
8394        if self.state == 'gil':
8395            code.put_ensure_gil(variable=variable)
8396            code.funcstate.gil_owned = True
8397        else:
8398            code.put_release_gil(variable=variable)
8399            code.funcstate.gil_owned = False
8400
8401        TryFinallyStatNode.generate_execution_code(self, code)
8402
8403        if self.state_temp:
8404            self.state_temp.release(code)
8405
8406        code.funcstate.gil_owned = old_gil_config
8407        code.end_block()
8408
8409
8410class GILExitNode(StatNode):
8411    """
8412    Used as the 'finally' block in a GILStatNode
8413
8414    state   string   'gil' or 'nogil'
8415    """
8416
8417    child_attrs = []
8418    state_temp = None
8419
8420    def analyse_expressions(self, env):
8421        return self
8422
8423    def generate_execution_code(self, code):
8424        if self.state_temp:
8425            variable = self.state_temp.result()
8426        else:
8427            variable = None
8428
8429        if self.state == 'gil':
8430            code.put_release_ensured_gil(variable)
8431        else:
8432            code.put_acquire_gil(variable)
8433
8434
8435class EnsureGILNode(GILExitNode):
8436    """
8437    Ensure the GIL in nogil functions for cleanup before returning.
8438    """
8439
8440    def generate_execution_code(self, code):
8441        code.put_ensure_gil(declare_gilstate=False)
8442
8443
8444def cython_view_utility_code():
8445    from . import MemoryView
8446    return MemoryView.view_utility_code
8447
8448
8449utility_code_for_cimports = {
8450    # utility code (or inlining c) in a pxd (or pyx) file.
8451    # TODO: Consider a generic user-level mechanism for importing
8452    'cpython.array'         : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
8453    'cpython.array.array'   : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
8454    'cython.view'           : cython_view_utility_code,
8455}
8456
8457utility_code_for_imports = {
8458    # utility code used when special modules are imported.
8459    # TODO: Consider a generic user-level mechanism for importing
8460    'asyncio': ("__Pyx_patch_asyncio", "PatchAsyncIO", "Coroutine.c"),
8461    'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"),
8462}
8463
8464def cimport_numpy_check(node, code):
8465    # shared code between CImportStatNode and FromCImportStatNode
8466    # check to ensure that import_array is called
8467    for mod in code.globalstate.module_node.scope.cimported_modules:
8468        if mod.name != node.module_name:
8469            continue
8470        # there are sometimes several cimported modules with the same name
8471        # so complete the loop if necessary
8472        import_array = mod.lookup_here("import_array")
8473        _import_array = mod.lookup_here("_import_array")
8474        # at least one entry used
8475        used = (import_array and import_array.used) or (_import_array and _import_array.used)
8476        if ((import_array or _import_array)  # at least one entry found
8477                and not used):
8478            # sanity check that this is actually numpy and not a user pxd called "numpy"
8479            if _import_array and _import_array.type.is_cfunction:
8480                # warning is mainly for the sake of testing
8481                warning(node.pos, "'numpy.import_array()' has been added automatically "
8482                        "since 'numpy' was cimported but 'numpy.import_array' was not called.", 0)
8483                from .Code import TempitaUtilityCode
8484                code.globalstate.use_utility_code(
8485                         TempitaUtilityCode.load_cached("NumpyImportArray", "NumpyImportArray.c",
8486                                            context = {'err_goto': code.error_goto(node.pos)})
8487                    )
8488                return  # no need to continue once the utility code is added
8489
8490
8491
8492class CImportStatNode(StatNode):
8493    #  cimport statement
8494    #
8495    #  module_name   string           Qualified name of module being imported
8496    #  as_name       string or None   Name specified in "as" clause, if any
8497    #  is_absolute   bool             True for absolute imports, False otherwise
8498
8499    child_attrs = []
8500    is_absolute = False
8501
8502    def analyse_declarations(self, env):
8503        if not env.is_module_scope:
8504            error(self.pos, "cimport only allowed at module level")
8505            return
8506        module_scope = env.find_module(
8507            self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
8508        if "." in self.module_name:
8509            names = [EncodedString(name) for name in self.module_name.split(".")]
8510            top_name = names[0]
8511            top_module_scope = env.context.find_submodule(top_name)
8512            module_scope = top_module_scope
8513            for name in names[1:]:
8514                submodule_scope = module_scope.find_submodule(name)
8515                module_scope.declare_module(name, submodule_scope, self.pos)
8516                module_scope = submodule_scope
8517            if self.as_name:
8518                env.declare_module(self.as_name, module_scope, self.pos)
8519            else:
8520                env.add_imported_module(module_scope)
8521                env.declare_module(top_name, top_module_scope, self.pos)
8522        else:
8523            name = self.as_name or self.module_name
8524            env.declare_module(name, module_scope, self.pos)
8525        if self.module_name in utility_code_for_cimports:
8526            env.use_utility_code(utility_code_for_cimports[self.module_name]())
8527
8528    def analyse_expressions(self, env):
8529        return self
8530
8531    def generate_execution_code(self, code):
8532        if self.module_name == "numpy":
8533            cimport_numpy_check(self, code)
8534
8535
8536class FromCImportStatNode(StatNode):
8537    #  from ... cimport statement
8538    #
8539    #  module_name     string                        Qualified name of module
8540    #  relative_level  int or None                   Relative import: number of dots before module_name
8541    #  imported_names  [(pos, name, as_name, kind)]  Names to be imported
8542
8543    child_attrs = []
8544    module_name = None
8545    relative_level = None
8546    imported_names = None
8547
8548    def analyse_declarations(self, env):
8549        if not env.is_module_scope:
8550            error(self.pos, "cimport only allowed at module level")
8551            return
8552        if self.relative_level and self.relative_level > env.qualified_name.count('.'):
8553            error(self.pos, "relative cimport beyond main package is not allowed")
8554            return
8555        module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level)
8556        module_name = module_scope.qualified_name
8557        env.add_imported_module(module_scope)
8558        for pos, name, as_name, kind in self.imported_names:
8559            if name == "*":
8560                for local_name, entry in list(module_scope.entries.items()):
8561                    env.add_imported_entry(local_name, entry, pos)
8562            else:
8563                entry = module_scope.lookup(name)
8564                if entry:
8565                    if kind and not self.declaration_matches(entry, kind):
8566                        entry.redeclared(pos)
8567                    entry.used = 1
8568                else:
8569                    if kind == 'struct' or kind == 'union':
8570                        entry = module_scope.declare_struct_or_union(
8571                            name, kind=kind, scope=None, typedef_flag=0, pos=pos)
8572                    elif kind == 'class':
8573                        entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
8574                    else:
8575                        submodule_scope = env.context.find_module(
8576                            name, relative_to=module_scope, pos=self.pos, absolute_fallback=False)
8577                        if submodule_scope.parent_module is module_scope:
8578                            env.declare_module(as_name or name, submodule_scope, self.pos)
8579                        else:
8580                            error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
8581
8582                if entry:
8583                    local_name = as_name or name
8584                    env.add_imported_entry(local_name, entry, pos)
8585
8586        if module_name.startswith('cpython') or module_name.startswith('cython'):  # enough for now
8587            if module_name in utility_code_for_cimports:
8588                env.use_utility_code(utility_code_for_cimports[module_name]())
8589            for _, name, _, _ in self.imported_names:
8590                fqname = '%s.%s' % (module_name, name)
8591                if fqname in utility_code_for_cimports:
8592                    env.use_utility_code(utility_code_for_cimports[fqname]())
8593
8594    def declaration_matches(self, entry, kind):
8595        if not entry.is_type:
8596            return 0
8597        type = entry.type
8598        if kind == 'class':
8599            if not type.is_extension_type:
8600                return 0
8601        else:
8602            if not type.is_struct_or_union:
8603                return 0
8604            if kind != type.kind:
8605                return 0
8606        return 1
8607
8608    def analyse_expressions(self, env):
8609        return self
8610
8611    def generate_execution_code(self, code):
8612        if self.module_name == "numpy":
8613            cimport_numpy_check(self, code)
8614
8615
8616class FromImportStatNode(StatNode):
8617    #  from ... import statement
8618    #
8619    #  module           ImportNode
8620    #  items            [(string, NameNode)]
8621    #  interned_items   [(string, NameNode, ExprNode)]
8622    #  item             PyTempNode            used internally
8623    #  import_star      boolean               used internally
8624
8625    child_attrs = ["module"]
8626    import_star = 0
8627
8628    def analyse_declarations(self, env):
8629        for name, target in self.items:
8630            if name == "*":
8631                if not env.is_module_scope:
8632                    error(self.pos, "import * only allowed at module level")
8633                    return
8634                env.has_import_star = 1
8635                self.import_star = 1
8636            else:
8637                target.analyse_target_declaration(env)
8638
8639    def analyse_expressions(self, env):
8640        from . import ExprNodes
8641        self.module = self.module.analyse_expressions(env)
8642        self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
8643        self.interned_items = []
8644        for name, target in self.items:
8645            if name == '*':
8646                for _, entry in env.entries.items():
8647                    if not entry.is_type and entry.type.is_extension_type:
8648                        env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
8649                        break
8650            else:
8651                entry = env.lookup(target.name)
8652                # check whether or not entry is already cimported
8653                if (entry.is_type and entry.type.name == name
8654                        and hasattr(entry.type, 'module_name')):
8655                    if entry.type.module_name == self.module.module_name.value:
8656                        # cimported with absolute name
8657                        continue
8658                    try:
8659                        # cimported with relative name
8660                        module = env.find_module(self.module.module_name.value, pos=self.pos,
8661                                                 relative_level=self.module.level)
8662                        if entry.type.module_name == module.qualified_name:
8663                            continue
8664                    except AttributeError:
8665                        pass
8666                target = target.analyse_target_expression(env, None)  # FIXME?
8667                if target.type is py_object_type:
8668                    coerced_item = None
8669                else:
8670                    coerced_item = self.item.coerce_to(target.type, env)
8671                self.interned_items.append((name, target, coerced_item))
8672        return self
8673
8674    def generate_execution_code(self, code):
8675        code.mark_pos(self.pos)
8676        self.module.generate_evaluation_code(code)
8677        if self.import_star:
8678            code.putln(
8679                'if (%s(%s) < 0) %s;' % (
8680                    Naming.import_star,
8681                    self.module.py_result(),
8682                    code.error_goto(self.pos)))
8683        item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
8684        self.item.set_cname(item_temp)
8685        if self.interned_items:
8686            code.globalstate.use_utility_code(
8687                UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
8688        for name, target, coerced_item in self.interned_items:
8689            code.putln(
8690                '%s = __Pyx_ImportFrom(%s, %s); %s' % (
8691                    item_temp,
8692                    self.module.py_result(),
8693                    code.intern_identifier(name),
8694                    code.error_goto_if_null(item_temp, self.pos)))
8695            code.put_gotref(item_temp, py_object_type)
8696            if coerced_item is None:
8697                target.generate_assignment_code(self.item, code)
8698            else:
8699                coerced_item.allocate_temp_result(code)
8700                coerced_item.generate_result_code(code)
8701                target.generate_assignment_code(coerced_item, code)
8702            code.put_decref_clear(item_temp, py_object_type)
8703        code.funcstate.release_temp(item_temp)
8704        self.module.generate_disposal_code(code)
8705        self.module.free_temps(code)
8706
8707
8708class ParallelNode(Node):
8709    """
8710    Base class for cython.parallel constructs.
8711    """
8712
8713    nogil_check = None
8714
8715
8716class ParallelStatNode(StatNode, ParallelNode):
8717    """
8718    Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
8719
8720    assignments     { Entry(var) : (var.pos, inplace_operator_or_None) }
8721                    assignments to variables in this parallel section
8722
8723    parent          parent ParallelStatNode or None
8724    is_parallel     indicates whether this node is OpenMP parallel
8725                    (true for #pragma omp parallel for and
8726                              #pragma omp parallel)
8727
8728    is_parallel is true for:
8729
8730        #pragma omp parallel
8731        #pragma omp parallel for
8732
8733    sections, but NOT for
8734
8735        #pragma omp for
8736
8737    We need this to determine the sharing attributes.
8738
8739    privatization_insertion_point   a code insertion point used to make temps
8740                                    private (esp. the "nsteps" temp)
8741
8742    args         tuple          the arguments passed to the parallel construct
8743    kwargs       DictNode       the keyword arguments passed to the parallel
8744                                construct (replaced by its compile time value)
8745    """
8746
8747    child_attrs = ['body', 'num_threads']
8748
8749    body = None
8750
8751    is_prange = False
8752    is_nested_prange = False
8753
8754    error_label_used = False
8755
8756    num_threads = None
8757    chunksize = None
8758
8759    parallel_exc = (
8760        Naming.parallel_exc_type,
8761        Naming.parallel_exc_value,
8762        Naming.parallel_exc_tb,
8763    )
8764
8765    parallel_pos_info = (
8766        Naming.parallel_filename,
8767        Naming.parallel_lineno,
8768        Naming.parallel_clineno,
8769    )
8770
8771    pos_info = (
8772        Naming.filename_cname,
8773        Naming.lineno_cname,
8774        Naming.clineno_cname,
8775    )
8776
8777    critical_section_counter = 0
8778
8779    def __init__(self, pos, **kwargs):
8780        super(ParallelStatNode, self).__init__(pos, **kwargs)
8781
8782        # All assignments in this scope
8783        self.assignments = kwargs.get('assignments') or {}
8784
8785        # All seen closure cnames and their temporary cnames
8786        self.seen_closure_vars = set()
8787
8788        # Dict of variables that should be declared (first|last|)private or
8789        # reduction { Entry: (op, lastprivate) }.
8790        # If op is not None, it's a reduction.
8791        self.privates = {}
8792
8793        # [NameNode]
8794        self.assigned_nodes = []
8795
8796    def analyse_declarations(self, env):
8797        self.body.analyse_declarations(env)
8798
8799        self.num_threads = None
8800
8801        if self.kwargs:
8802            # Try to find num_threads and chunksize keyword arguments
8803            pairs = []
8804            seen = set()
8805            for dictitem in self.kwargs.key_value_pairs:
8806                if dictitem.key.value in seen:
8807                    error(self.pos, "Duplicate keyword argument found: %s" % dictitem.key.value)
8808                seen.add(dictitem.key.value)
8809                if dictitem.key.value == 'num_threads':
8810                    if not dictitem.value.is_none:
8811                        self.num_threads = dictitem.value
8812                elif self.is_prange and dictitem.key.value == 'chunksize':
8813                    if not dictitem.value.is_none:
8814                        self.chunksize = dictitem.value
8815                else:
8816                    pairs.append(dictitem)
8817
8818            self.kwargs.key_value_pairs = pairs
8819
8820            try:
8821                self.kwargs = self.kwargs.compile_time_value(env)
8822            except Exception as e:
8823                error(self.kwargs.pos, "Only compile-time values may be "
8824                                       "supplied as keyword arguments")
8825        else:
8826            self.kwargs = {}
8827
8828        for kw, val in self.kwargs.items():
8829            if kw not in self.valid_keyword_arguments:
8830                error(self.pos, "Invalid keyword argument: %s" % kw)
8831            else:
8832                setattr(self, kw, val)
8833
8834    def analyse_expressions(self, env):
8835        if self.num_threads:
8836            self.num_threads = self.num_threads.analyse_expressions(env)
8837
8838        if self.chunksize:
8839            self.chunksize = self.chunksize.analyse_expressions(env)
8840
8841        self.body = self.body.analyse_expressions(env)
8842        self.analyse_sharing_attributes(env)
8843
8844        if self.num_threads is not None:
8845            if self.parent and self.parent.num_threads is not None and not self.parent.is_prange:
8846                error(self.pos, "num_threads already declared in outer section")
8847            elif self.parent and not self.parent.is_prange:
8848                error(self.pos, "num_threads must be declared in the parent parallel section")
8849            elif (self.num_threads.type.is_int and
8850                    self.num_threads.is_literal and
8851                    self.num_threads.compile_time_value(env) <= 0):
8852                error(self.pos, "argument to num_threads must be greater than 0")
8853
8854            if not self.num_threads.is_simple() or self.num_threads.type.is_pyobject:
8855                self.num_threads = self.num_threads.coerce_to(
8856                    PyrexTypes.c_int_type, env).coerce_to_temp(env)
8857        return self
8858
8859    def analyse_sharing_attributes(self, env):
8860        """
8861        Analyse the privates for this block and set them in self.privates.
8862        This should be called in a post-order fashion during the
8863        analyse_expressions phase
8864        """
8865        for entry, (pos, op) in self.assignments.items():
8866
8867            if self.is_prange and not self.is_parallel:
8868                # closely nested prange in a with parallel block, disallow
8869                # assigning to privates in the with parallel block (we
8870                # consider it too implicit and magicky for users)
8871                if entry in self.parent.assignments:
8872                    error(pos, "Cannot assign to private of outer parallel block")
8873                    continue
8874
8875            if not self.is_prange and op:
8876                # Again possible, but considered to magicky
8877                error(pos, "Reductions not allowed for parallel blocks")
8878                continue
8879
8880            # By default all variables should have the same values as if
8881            # executed sequentially
8882            lastprivate = True
8883            self.propagate_var_privatization(entry, pos, op, lastprivate)
8884
8885    def propagate_var_privatization(self, entry, pos, op, lastprivate):
8886        """
8887        Propagate the sharing attributes of a variable. If the privatization is
8888        determined by a parent scope, done propagate further.
8889
8890        If we are a prange, we propagate our sharing attributes outwards to
8891        other pranges. If we are a prange in parallel block and the parallel
8892        block does not determine the variable private, we propagate to the
8893        parent of the parent. Recursion stops at parallel blocks, as they have
8894        no concept of lastprivate or reduction.
8895
8896        So the following cases propagate:
8897
8898            sum is a reduction for all loops:
8899
8900                for i in prange(n):
8901                    for j in prange(n):
8902                        for k in prange(n):
8903                            sum += i * j * k
8904
8905            sum is a reduction for both loops, local_var is private to the
8906            parallel with block:
8907
8908                for i in prange(n):
8909                    with parallel:
8910                        local_var = ... # private to the parallel
8911                        for j in prange(n):
8912                            sum += i * j
8913
8914        Nested with parallel blocks are disallowed, because they wouldn't
8915        allow you to propagate lastprivates or reductions:
8916
8917            #pragma omp parallel for lastprivate(i)
8918            for i in prange(n):
8919
8920                sum = 0
8921
8922                #pragma omp parallel private(j, sum)
8923                with parallel:
8924
8925                    #pragma omp parallel
8926                    with parallel:
8927
8928                        #pragma omp for lastprivate(j) reduction(+:sum)
8929                        for j in prange(n):
8930                            sum += i
8931
8932                    # sum and j are well-defined here
8933
8934                # sum and j are undefined here
8935
8936            # sum and j are undefined here
8937        """
8938        self.privates[entry] = (op, lastprivate)
8939
8940        if entry.type.is_memoryviewslice:
8941            error(pos, "Memoryview slices can only be shared in parallel sections")
8942            return
8943
8944        if self.is_prange:
8945            if not self.is_parallel and entry not in self.parent.assignments:
8946                # Parent is a parallel with block
8947                parent = self.parent.parent
8948            else:
8949                parent = self.parent
8950
8951            # We don't need to propagate privates, only reductions and
8952            # lastprivates
8953            if parent and (op or lastprivate):
8954                parent.propagate_var_privatization(entry, pos, op, lastprivate)
8955
8956    def _allocate_closure_temp(self, code, entry):
8957        """
8958        Helper function that allocate a temporary for a closure variable that
8959        is assigned to.
8960        """
8961        if self.parent:
8962            return self.parent._allocate_closure_temp(code, entry)
8963
8964        if entry.cname in self.seen_closure_vars:
8965            return entry.cname
8966
8967        cname = code.funcstate.allocate_temp(entry.type, True)
8968
8969        # Add both the actual cname and the temp cname, as the actual cname
8970        # will be replaced with the temp cname on the entry
8971        self.seen_closure_vars.add(entry.cname)
8972        self.seen_closure_vars.add(cname)
8973
8974        self.modified_entries.append((entry, entry.cname))
8975        code.putln("%s = %s;" % (cname, entry.cname))
8976        entry.cname = cname
8977
8978    def initialize_privates_to_nan(self, code, exclude=None):
8979        first = True
8980
8981        for entry, (op, lastprivate) in sorted(self.privates.items()):
8982            if not op and (not exclude or entry != exclude):
8983                invalid_value = entry.type.invalid_value()
8984
8985                if invalid_value:
8986                    if first:
8987                        code.putln("/* Initialize private variables to "
8988                                   "invalid values */")
8989                        first = False
8990                    code.putln("%s = %s;" % (entry.cname,
8991                                             entry.type.cast_code(invalid_value)))
8992
8993    def evaluate_before_block(self, code, expr):
8994        c = self.begin_of_parallel_control_block_point_after_decls
8995        # we need to set the owner to ourselves temporarily, as
8996        # allocate_temp may generate a comment in the middle of our pragma
8997        # otherwise when DebugFlags.debug_temp_code_comments is in effect
8998        owner = c.funcstate.owner
8999        c.funcstate.owner = c
9000        expr.generate_evaluation_code(c)
9001        c.funcstate.owner = owner
9002
9003        return expr.result()
9004
9005    def put_num_threads(self, code):
9006        """
9007        Write self.num_threads if set as the num_threads OpenMP directive
9008        """
9009        if self.num_threads is not None:
9010            code.put(" num_threads(%s)" % self.evaluate_before_block(code, self.num_threads))
9011
9012
9013    def declare_closure_privates(self, code):
9014        """
9015        If a variable is in a scope object, we need to allocate a temp and
9016        assign the value from the temp to the variable in the scope object
9017        after the parallel section. This kind of copying should be done only
9018        in the outermost parallel section.
9019        """
9020        self.modified_entries = []
9021
9022        for entry in sorted(self.assignments):
9023            if entry.from_closure or entry.in_closure:
9024                self._allocate_closure_temp(code, entry)
9025
9026    def release_closure_privates(self, code):
9027        """
9028        Release any temps used for variables in scope objects. As this is the
9029        outermost parallel block, we don't need to delete the cnames from
9030        self.seen_closure_vars.
9031        """
9032        for entry, original_cname in self.modified_entries:
9033            code.putln("%s = %s;" % (original_cname, entry.cname))
9034            code.funcstate.release_temp(entry.cname)
9035            entry.cname = original_cname
9036
9037    def privatize_temps(self, code, exclude_temps=()):
9038        """
9039        Make any used temporaries private. Before the relevant code block
9040        code.start_collecting_temps() should have been called.
9041        """
9042        c = self.privatization_insertion_point
9043        self.privatization_insertion_point = None
9044
9045        if self.is_parallel:
9046            self.temps = temps = code.funcstate.stop_collecting_temps()
9047            privates, firstprivates = [], []
9048            for temp, type in sorted(temps):
9049                if type.is_pyobject or type.is_memoryviewslice:
9050                    firstprivates.append(temp)
9051                else:
9052                    privates.append(temp)
9053
9054            if privates:
9055                c.put(" private(%s)" % ", ".join(privates))
9056            if firstprivates:
9057                c.put(" firstprivate(%s)" % ", ".join(firstprivates))
9058
9059            if self.breaking_label_used:
9060                shared_vars = [Naming.parallel_why]
9061                if self.error_label_used:
9062                    shared_vars.extend(self.parallel_exc)
9063                    c.put(" private(%s, %s, %s)" % self.pos_info)
9064
9065                c.put(" shared(%s)" % ', '.join(shared_vars))
9066
9067    def cleanup_temps(self, code):
9068        # Now clean up any memoryview slice and object temporaries
9069        if self.is_parallel and not self.is_nested_prange:
9070            code.putln("/* Clean up any temporaries */")
9071            for temp, type in sorted(self.temps):
9072                code.put_xdecref_clear(temp, type, have_gil=False)
9073
9074    def setup_parallel_control_flow_block(self, code):
9075        """
9076        Sets up a block that surrounds the parallel block to determine
9077        how the parallel section was exited. Any kind of return is
9078        trapped (break, continue, return, exceptions). This is the idea:
9079
9080        {
9081            int why = 0;
9082
9083            #pragma omp parallel
9084            {
9085                return # -> goto new_return_label;
9086                goto end_parallel;
9087
9088            new_return_label:
9089                why = 3;
9090                goto end_parallel;
9091
9092            end_parallel:;
9093                #pragma omp flush(why) # we need to flush for every iteration
9094            }
9095
9096            if (why == 3)
9097                goto old_return_label;
9098        }
9099        """
9100        self.old_loop_labels = code.new_loop_labels()
9101        self.old_error_label = code.new_error_label()
9102        self.old_return_label = code.return_label
9103        code.return_label = code.new_label(name="return")
9104
9105        code.begin_block()  # parallel control flow block
9106        self.begin_of_parallel_control_block_point = code.insertion_point()
9107        self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
9108
9109        self.undef_builtin_expect_apple_gcc_bug(code)
9110
9111    def begin_parallel_block(self, code):
9112        """
9113        Each OpenMP thread in a parallel section that contains a with gil block
9114        must have the thread-state initialized. The call to
9115        PyGILState_Release() then deallocates our threadstate. If we wouldn't
9116        do this, each with gil block would allocate and deallocate one, thereby
9117        losing exception information before it can be saved before leaving the
9118        parallel section.
9119        """
9120        self.begin_of_parallel_block = code.insertion_point()
9121
9122    def end_parallel_block(self, code):
9123        """
9124        To ensure all OpenMP threads have thread states, we ensure the GIL
9125        in each thread (which creates a thread state if it doesn't exist),
9126        after which we release the GIL.
9127        On exit, reacquire the GIL and release the thread state.
9128
9129        If compiled without OpenMP support (at the C level), then we still have
9130        to acquire the GIL to decref any object temporaries.
9131        """
9132        begin_code = self.begin_of_parallel_block
9133        self.begin_of_parallel_block = None
9134
9135        if self.error_label_used:
9136            end_code = code
9137
9138            begin_code.putln("#ifdef _OPENMP")
9139            begin_code.put_ensure_gil(declare_gilstate=True)
9140            begin_code.putln("Py_BEGIN_ALLOW_THREADS")
9141            begin_code.putln("#endif /* _OPENMP */")
9142
9143            end_code.putln("#ifdef _OPENMP")
9144            end_code.putln("Py_END_ALLOW_THREADS")
9145            end_code.putln("#else")
9146            end_code.put_safe("{\n")
9147            end_code.put_ensure_gil()
9148            end_code.putln("#endif /* _OPENMP */")
9149            self.cleanup_temps(end_code)
9150            end_code.put_release_ensured_gil()
9151            end_code.putln("#ifndef _OPENMP")
9152            end_code.put_safe("}\n")
9153            end_code.putln("#endif /* _OPENMP */")
9154
9155    def trap_parallel_exit(self, code, should_flush=False):
9156        """
9157        Trap any kind of return inside a parallel construct. 'should_flush'
9158        indicates whether the variable should be flushed, which is needed by
9159        prange to skip the loop. It also indicates whether we need to register
9160        a continue (we need this for parallel blocks, but not for prange
9161        loops, as it is a direct jump there).
9162
9163        It uses the same mechanism as try/finally:
9164            1 continue
9165            2 break
9166            3 return
9167            4 error
9168        """
9169        save_lastprivates_label = code.new_label()
9170        dont_return_label = code.new_label()
9171
9172        self.any_label_used = False
9173        self.breaking_label_used = False
9174        self.error_label_used = False
9175
9176        self.parallel_private_temps = []
9177
9178        all_labels = code.get_all_labels()
9179
9180        # Figure this out before starting to generate any code
9181        for label in all_labels:
9182            if code.label_used(label):
9183                self.breaking_label_used = (self.breaking_label_used or
9184                                            label != code.continue_label)
9185                self.any_label_used = True
9186
9187        if self.any_label_used:
9188            code.put_goto(dont_return_label)
9189
9190        for i, label in enumerate(all_labels):
9191            if not code.label_used(label):
9192                continue
9193
9194            is_continue_label = label == code.continue_label
9195
9196            code.put_label(label)
9197
9198            if not (should_flush and is_continue_label):
9199                if label == code.error_label:
9200                    self.error_label_used = True
9201                    self.fetch_parallel_exception(code)
9202
9203                code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
9204
9205            if (self.breaking_label_used and self.is_prange and not
9206                    is_continue_label):
9207                code.put_goto(save_lastprivates_label)
9208            else:
9209                code.put_goto(dont_return_label)
9210
9211        if self.any_label_used:
9212            if self.is_prange and self.breaking_label_used:
9213                # Don't rely on lastprivate, save our lastprivates
9214                code.put_label(save_lastprivates_label)
9215                self.save_parallel_vars(code)
9216
9217            code.put_label(dont_return_label)
9218
9219            if should_flush and self.breaking_label_used:
9220                code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)
9221
9222    def save_parallel_vars(self, code):
9223        """
9224        The following shenanigans are instated when we break, return or
9225        propagate errors from a prange. In this case we cannot rely on
9226        lastprivate() to do its job, as no iterations may have executed yet
9227        in the last thread, leaving the values undefined. It is most likely
9228        that the breaking thread has well-defined values of the lastprivate
9229        variables, so we keep those values.
9230        """
9231        section_name = "__pyx_parallel_lastprivates%d" % self.critical_section_counter
9232        code.putln_openmp("#pragma omp critical(%s)" % section_name)
9233        ParallelStatNode.critical_section_counter += 1
9234
9235        code.begin_block()  # begin critical section
9236
9237        c = self.begin_of_parallel_control_block_point
9238
9239        temp_count = 0
9240        for entry, (op, lastprivate) in sorted(self.privates.items()):
9241            if not lastprivate or entry.type.is_pyobject:
9242                continue
9243
9244            type_decl = entry.type.empty_declaration_code()
9245            temp_cname = "__pyx_parallel_temp%d" % temp_count
9246            private_cname = entry.cname
9247
9248            temp_count += 1
9249
9250            invalid_value = entry.type.invalid_value()
9251            if invalid_value:
9252                init = ' = ' + entry.type.cast_code(invalid_value)
9253            else:
9254                init = ''
9255            # Declare the parallel private in the outer block
9256            c.putln("%s %s%s;" % (type_decl, temp_cname, init))
9257
9258            # Initialize before escaping
9259            code.putln("%s = %s;" % (temp_cname, private_cname))
9260
9261            self.parallel_private_temps.append((temp_cname, private_cname))
9262
9263        code.end_block()  # end critical section
9264
9265    def fetch_parallel_exception(self, code):
9266        """
9267        As each OpenMP thread may raise an exception, we need to fetch that
9268        exception from the threadstate and save it for after the parallel
9269        section where it can be re-raised in the master thread.
9270
9271        Although it would seem that __pyx_filename, __pyx_lineno and
9272        __pyx_clineno are only assigned to under exception conditions (i.e.,
9273        when we have the GIL), and thus should be allowed to be shared without
9274        any race condition, they are in fact subject to the same race
9275        conditions that they were previously when they were global variables
9276        and functions were allowed to release the GIL:
9277
9278            thread A                thread B
9279                acquire
9280                set lineno
9281                release
9282                                        acquire
9283                                        set lineno
9284                                        release
9285                acquire
9286                fetch exception
9287                release
9288                                        skip the fetch
9289
9290                deallocate threadstate  deallocate threadstate
9291        """
9292        code.begin_block()
9293        code.put_ensure_gil(declare_gilstate=True)
9294
9295        code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
9296        code.putln(
9297            "if (!%s) {" % Naming.parallel_exc_type)
9298
9299        code.putln("__Pyx_ErrFetchWithState(&%s, &%s, &%s);" % self.parallel_exc)
9300        pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
9301        code.funcstate.uses_error_indicator = True
9302        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
9303        code.put_gotref(Naming.parallel_exc_type, py_object_type)
9304
9305        code.putln(
9306            "}")
9307
9308        code.put_release_ensured_gil()
9309        code.end_block()
9310
9311    def restore_parallel_exception(self, code):
9312        "Re-raise a parallel exception"
9313        code.begin_block()
9314        code.put_ensure_gil(declare_gilstate=True)
9315
9316        code.put_giveref(Naming.parallel_exc_type, py_object_type)
9317        code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc)
9318        pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
9319        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
9320
9321        code.put_release_ensured_gil()
9322        code.end_block()
9323
9324    def restore_labels(self, code):
9325        """
9326        Restore all old labels. Call this before the 'else' clause to for
9327        loops and always before ending the parallel control flow block.
9328        """
9329        code.set_all_labels(self.old_loop_labels + (self.old_return_label,
9330                                                    self.old_error_label))
9331
9332    def end_parallel_control_flow_block(
9333            self, code, break_=False, continue_=False, return_=False):
9334        """
9335        This ends the parallel control flow block and based on how the parallel
9336        section was exited, takes the corresponding action. The break_ and
9337        continue_ parameters indicate whether these should be propagated
9338        outwards:
9339
9340            for i in prange(...):
9341                with cython.parallel.parallel():
9342                    continue
9343
9344        Here break should be trapped in the parallel block, and propagated to
9345        the for loop.
9346        """
9347        c = self.begin_of_parallel_control_block_point
9348        self.begin_of_parallel_control_block_point = None
9349        self.begin_of_parallel_control_block_point_after_decls = None
9350
9351        if self.num_threads is not None:
9352            # FIXME: is it the right place? should not normally produce code.
9353            self.num_threads.generate_disposal_code(code)
9354            self.num_threads.free_temps(code)
9355
9356        # Firstly, always prefer errors over returning, continue or break
9357        if self.error_label_used:
9358            c.putln("const char *%s = NULL; int %s = 0, %s = 0;" % self.parallel_pos_info)
9359            c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" % self.parallel_exc)
9360
9361            code.putln(
9362                "if (%s) {" % Naming.parallel_exc_type)
9363            code.putln("/* This may have been overridden by a continue, "
9364                       "break or return in another thread. Prefer the error. */")
9365            code.putln("%s = 4;" % Naming.parallel_why)
9366            code.putln(
9367                "}")
9368
9369        if continue_:
9370            any_label_used = self.any_label_used
9371        else:
9372            any_label_used = self.breaking_label_used
9373
9374        if any_label_used:
9375            # __pyx_parallel_why is used, declare and initialize
9376            c.putln("int %s;" % Naming.parallel_why)
9377            c.putln("%s = 0;" % Naming.parallel_why)
9378
9379            code.putln(
9380                "if (%s) {" % Naming.parallel_why)
9381
9382            for temp_cname, private_cname in self.parallel_private_temps:
9383                code.putln("%s = %s;" % (private_cname, temp_cname))
9384
9385            code.putln("switch (%s) {" % Naming.parallel_why)
9386            if continue_:
9387                code.put("    case 1: ")
9388                code.put_goto(code.continue_label)
9389
9390            if break_:
9391                code.put("    case 2: ")
9392                code.put_goto(code.break_label)
9393
9394            if return_:
9395                code.put("    case 3: ")
9396                code.put_goto(code.return_label)
9397
9398            if self.error_label_used:
9399                code.globalstate.use_utility_code(restore_exception_utility_code)
9400                code.putln("    case 4:")
9401                self.restore_parallel_exception(code)
9402                code.put_goto(code.error_label)
9403
9404            code.putln("}")  # end switch
9405            code.putln(
9406                "}")  # end if
9407
9408        code.end_block()  # end parallel control flow block
9409        self.redef_builtin_expect_apple_gcc_bug(code)
9410
9411    # FIXME: improve with version number for OS X Lion
9412    buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
9413    have_expect_condition = "(defined(__GNUC__) && " \
9414                             "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
9415    redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)
9416
9417    def undef_builtin_expect_apple_gcc_bug(self, code):
9418        """
9419        A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
9420        """
9421        if not self.parent:
9422            code.undef_builtin_expect(self.redef_condition)
9423
9424    def redef_builtin_expect_apple_gcc_bug(self, code):
9425        if not self.parent:
9426            code.redef_builtin_expect(self.redef_condition)
9427
9428
9429class ParallelWithBlockNode(ParallelStatNode):
9430    """
9431    This node represents a 'with cython.parallel.parallel():' block
9432    """
9433
9434    valid_keyword_arguments = ['num_threads']
9435
9436    num_threads = None
9437
9438    def analyse_declarations(self, env):
9439        super(ParallelWithBlockNode, self).analyse_declarations(env)
9440        if self.args:
9441            error(self.pos, "cython.parallel.parallel() does not take "
9442                            "positional arguments")
9443
9444    def generate_execution_code(self, code):
9445        self.declare_closure_privates(code)
9446        self.setup_parallel_control_flow_block(code)
9447
9448        code.putln("#ifdef _OPENMP")
9449        code.put("#pragma omp parallel ")
9450
9451        if self.privates:
9452            privates = [e.cname for e in self.privates
9453                        if not e.type.is_pyobject]
9454            code.put('private(%s)' % ', '.join(sorted(privates)))
9455
9456        self.privatization_insertion_point = code.insertion_point()
9457        self.put_num_threads(code)
9458        code.putln("")
9459
9460        code.putln("#endif /* _OPENMP */")
9461
9462        code.begin_block()  # parallel block
9463        self.begin_parallel_block(code)
9464        self.initialize_privates_to_nan(code)
9465        code.funcstate.start_collecting_temps()
9466        self.body.generate_execution_code(code)
9467        self.trap_parallel_exit(code)
9468        self.privatize_temps(code)
9469        self.end_parallel_block(code)
9470        code.end_block()  # end parallel block
9471
9472        continue_ = code.label_used(code.continue_label)
9473        break_ = code.label_used(code.break_label)
9474        return_ = code.label_used(code.return_label)
9475
9476        self.restore_labels(code)
9477        self.end_parallel_control_flow_block(code, break_=break_,
9478                                             continue_=continue_,
9479                                             return_=return_)
9480        self.release_closure_privates(code)
9481
9482
9483class ParallelRangeNode(ParallelStatNode):
9484    """
9485    This node represents a 'for i in cython.parallel.prange():' construct.
9486
9487    target       NameNode       the target iteration variable
9488    else_clause  Node or None   the else clause of this loop
9489    """
9490
9491    child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
9492                   'chunksize']
9493
9494    body = target = else_clause = args = None
9495
9496    start = stop = step = None
9497
9498    is_prange = True
9499
9500    nogil = None
9501    schedule = None
9502
9503    valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
9504
9505    def __init__(self, pos, **kwds):
9506        super(ParallelRangeNode, self).__init__(pos, **kwds)
9507        # Pretend to be a ForInStatNode for control flow analysis
9508        self.iterator = PassStatNode(pos)
9509
9510    def analyse_declarations(self, env):
9511        super(ParallelRangeNode, self).analyse_declarations(env)
9512        self.target.analyse_target_declaration(env)
9513        if self.else_clause is not None:
9514            self.else_clause.analyse_declarations(env)
9515
9516        if not self.args or len(self.args) > 3:
9517            error(self.pos, "Invalid number of positional arguments to prange")
9518            return
9519
9520        if len(self.args) == 1:
9521            self.stop, = self.args
9522        elif len(self.args) == 2:
9523            self.start, self.stop = self.args
9524        else:
9525            self.start, self.stop, self.step = self.args
9526
9527        if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'):
9528            error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,))
9529
9530    def analyse_expressions(self, env):
9531        was_nogil = env.nogil
9532        if self.nogil:
9533            env.nogil = True
9534
9535        if self.target is None:
9536            error(self.pos, "prange() can only be used as part of a for loop")
9537            return self
9538
9539        self.target = self.target.analyse_target_types(env)
9540
9541        if not self.target.type.is_numeric:
9542            # Not a valid type, assume one for now anyway
9543
9544            if not self.target.type.is_pyobject:
9545                # nogil_check will catch the is_pyobject case
9546                error(self.target.pos,
9547                      "Must be of numeric type, not %s" % self.target.type)
9548
9549            self.index_type = PyrexTypes.c_py_ssize_t_type
9550        else:
9551            self.index_type = self.target.type
9552            if not self.index_type.signed:
9553                warning(self.target.pos,
9554                        "Unsigned index type not allowed before OpenMP 3.0",
9555                        level=2)
9556
9557        # Setup start, stop and step, allocating temps if needed
9558        self.names = 'start', 'stop', 'step'
9559        start_stop_step = self.start, self.stop, self.step
9560
9561        for node, name in zip(start_stop_step, self.names):
9562            if node is not None:
9563                node.analyse_types(env)
9564                if not node.type.is_numeric:
9565                    error(node.pos, "%s argument must be numeric" % name)
9566                    continue
9567
9568                if not node.is_literal:
9569                    node = node.coerce_to_temp(env)
9570                    setattr(self, name, node)
9571
9572                # As we range from 0 to nsteps, computing the index along the
9573                # way, we need a fitting type for 'i' and 'nsteps'
9574                self.index_type = PyrexTypes.widest_numeric_type(
9575                    self.index_type, node.type)
9576
9577        if self.else_clause is not None:
9578            self.else_clause = self.else_clause.analyse_expressions(env)
9579
9580        # Although not actually an assignment in this scope, it should be
9581        # treated as such to ensure it is unpacked if a closure temp, and to
9582        # ensure lastprivate behaviour and propagation. If the target index is
9583        # not a NameNode, it won't have an entry, and an error was issued by
9584        # ParallelRangeTransform
9585        target_entry = getattr(self.target, 'entry', None)
9586        if target_entry:
9587            self.assignments[self.target.entry] = self.target.pos, None
9588
9589        node = super(ParallelRangeNode, self).analyse_expressions(env)
9590
9591        if node.chunksize:
9592            if not node.schedule:
9593                error(node.chunksize.pos,
9594                      "Must provide schedule with chunksize")
9595            elif node.schedule == 'runtime':
9596                error(node.chunksize.pos,
9597                      "Chunksize not valid for the schedule runtime")
9598            elif (node.chunksize.type.is_int and
9599                  node.chunksize.is_literal and
9600                  node.chunksize.compile_time_value(env) <= 0):
9601                error(node.chunksize.pos, "Chunksize must not be negative")
9602
9603            node.chunksize = node.chunksize.coerce_to(
9604                PyrexTypes.c_int_type, env).coerce_to_temp(env)
9605
9606        if node.nogil:
9607            env.nogil = was_nogil
9608
9609        node.is_nested_prange = node.parent and node.parent.is_prange
9610        if node.is_nested_prange:
9611            parent = node
9612            while parent.parent and parent.parent.is_prange:
9613                parent = parent.parent
9614
9615            parent.assignments.update(node.assignments)
9616            parent.privates.update(node.privates)
9617            parent.assigned_nodes.extend(node.assigned_nodes)
9618        return node
9619
9620    def nogil_check(self, env):
9621        names = 'start', 'stop', 'step', 'target'
9622        nodes = self.start, self.stop, self.step, self.target
9623        for name, node in zip(names, nodes):
9624            if node is not None and node.type.is_pyobject:
9625                error(node.pos, "%s may not be a Python object "
9626                                "as we don't have the GIL" % name)
9627
9628    def generate_execution_code(self, code):
9629        """
9630        Generate code in the following steps
9631
9632            1)  copy any closure variables determined thread-private
9633                into temporaries
9634
9635            2)  allocate temps for start, stop and step
9636
9637            3)  generate a loop that calculates the total number of steps,
9638                which then computes the target iteration variable for every step:
9639
9640                    for i in prange(start, stop, step):
9641                        ...
9642
9643                becomes
9644
9645                    nsteps = (stop - start) / step;
9646                    i = start;
9647
9648                    #pragma omp parallel for lastprivate(i)
9649                    for (temp = 0; temp < nsteps; temp++) {
9650                        i = start + step * temp;
9651                        ...
9652                    }
9653
9654                Note that accumulation of 'i' would have a data dependency
9655                between iterations.
9656
9657                Also, you can't do this
9658
9659                    for (i = start; i < stop; i += step)
9660                        ...
9661
9662                as the '<' operator should become '>' for descending loops.
9663                'for i from x < i < y:' does not suffer from this problem
9664                as the relational operator is known at compile time!
9665
9666            4) release our temps and write back any private closure variables
9667        """
9668        self.declare_closure_privates(code)
9669
9670        # This can only be a NameNode
9671        target_index_cname = self.target.entry.cname
9672
9673        # This will be used as the dict to format our code strings, holding
9674        # the start, stop , step, temps and target cnames
9675        fmt_dict = {
9676            'target': target_index_cname,
9677            'target_type': self.target.type.empty_declaration_code()
9678        }
9679
9680        # Setup start, stop and step, allocating temps if needed
9681        start_stop_step = self.start, self.stop, self.step
9682        defaults = '0', '0', '1'
9683        for node, name, default in zip(start_stop_step, self.names, defaults):
9684            if node is None:
9685                result = default
9686            elif node.is_literal:
9687                result = node.get_constant_c_result_code()
9688            else:
9689                node.generate_evaluation_code(code)
9690                result = node.result()
9691
9692            fmt_dict[name] = result
9693
9694        fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
9695        fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)
9696
9697        # TODO: check if the step is 0 and if so, raise an exception in a
9698        # 'with gil' block. For now, just abort
9699        if self.step is not None and self.step.has_constant_result() and self.step.constant_result == 0:
9700            error(node.pos, "Iteration with step 0 is invalid.")
9701        elif not fmt_dict['step'].isdigit() or int(fmt_dict['step']) == 0:
9702            code.putln("if (((%(step)s) == 0)) abort();" % fmt_dict)
9703
9704        self.setup_parallel_control_flow_block(code)  # parallel control flow block
9705
9706        # Note: nsteps is private in an outer scope if present
9707        code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict)
9708
9709        # The target iteration variable might not be initialized, do it only if
9710        # we are executing at least 1 iteration, otherwise we should leave the
9711        # target unaffected. The target iteration variable is firstprivate to
9712        # shut up compiler warnings caused by lastprivate, as the compiler
9713        # erroneously believes that nsteps may be <= 0, leaving the private
9714        # target index uninitialized
9715        code.putln("if (%(nsteps)s > 0)" % fmt_dict)
9716        code.begin_block()  # if block
9717        self.generate_loop(code, fmt_dict)
9718        code.end_block()  # end if block
9719
9720        self.restore_labels(code)
9721
9722        if self.else_clause:
9723            if self.breaking_label_used:
9724                code.put("if (%s < 2)" % Naming.parallel_why)
9725
9726            code.begin_block()  # else block
9727            code.putln("/* else */")
9728            self.else_clause.generate_execution_code(code)
9729            code.end_block()  # end else block
9730
9731        # ------ cleanup ------
9732        self.end_parallel_control_flow_block(code)  # end parallel control flow block
9733
9734        # And finally, release our privates and write back any closure
9735        # variables
9736        for temp in start_stop_step + (self.chunksize,):
9737            if temp is not None:
9738                temp.generate_disposal_code(code)
9739                temp.free_temps(code)
9740
9741        code.funcstate.release_temp(fmt_dict['i'])
9742        code.funcstate.release_temp(fmt_dict['nsteps'])
9743
9744        self.release_closure_privates(code)
9745
9746    def generate_loop(self, code, fmt_dict):
9747        if self.is_nested_prange:
9748            code.putln("#if 0")
9749        else:
9750            code.putln("#ifdef _OPENMP")
9751
9752        if not self.is_parallel:
9753            code.put("#pragma omp for")
9754            self.privatization_insertion_point = code.insertion_point()
9755            reduction_codepoint = self.parent.privatization_insertion_point
9756        else:
9757            code.put("#pragma omp parallel")
9758            self.privatization_insertion_point = code.insertion_point()
9759            reduction_codepoint = self.privatization_insertion_point
9760            code.putln("")
9761            code.putln("#endif /* _OPENMP */")
9762
9763            code.begin_block()  # pragma omp parallel begin block
9764
9765            # Initialize the GIL if needed for this thread
9766            self.begin_parallel_block(code)
9767
9768            if self.is_nested_prange:
9769                code.putln("#if 0")
9770            else:
9771                code.putln("#ifdef _OPENMP")
9772            code.put("#pragma omp for")
9773
9774        for entry, (op, lastprivate) in sorted(self.privates.items()):
9775            # Don't declare the index variable as a reduction
9776            if op and op in "+*-&^|" and entry != self.target.entry:
9777                if entry.type.is_pyobject:
9778                    error(self.pos, "Python objects cannot be reductions")
9779                else:
9780                    #code.put(" reduction(%s:%s)" % (op, entry.cname))
9781                    # This is the only way reductions + nesting works in gcc4.5
9782                    reduction_codepoint.put(
9783                                " reduction(%s:%s)" % (op, entry.cname))
9784            else:
9785                if entry == self.target.entry:
9786                    code.put(" firstprivate(%s)" % entry.cname)
9787                    code.put(" lastprivate(%s)" % entry.cname)
9788                    continue
9789
9790                if not entry.type.is_pyobject:
9791                    if lastprivate:
9792                        private = 'lastprivate'
9793                    else:
9794                        private = 'private'
9795
9796                    code.put(" %s(%s)" % (private, entry.cname))
9797
9798        if self.schedule:
9799            if self.chunksize:
9800                chunksize = ", %s" % self.evaluate_before_block(code, self.chunksize)
9801            else:
9802                chunksize = ""
9803
9804            code.put(" schedule(%s%s)" % (self.schedule, chunksize))
9805
9806        self.put_num_threads(reduction_codepoint)
9807
9808        code.putln("")
9809        code.putln("#endif /* _OPENMP */")
9810
9811        code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
9812        code.begin_block()  # for loop block
9813
9814        guard_around_body_codepoint = code.insertion_point()
9815
9816        # Start if guard block around the body. This may be unnecessary, but
9817        # at least it doesn't spoil indentation
9818        code.begin_block()
9819
9820        code.putln("%(target)s = (%(target_type)s)(%(start)s + %(step)s * %(i)s);" % fmt_dict)
9821        self.initialize_privates_to_nan(code, exclude=self.target.entry)
9822
9823        if self.is_parallel and not self.is_nested_prange:
9824            # nested pranges are not omp'ified, temps go to outer loops
9825            code.funcstate.start_collecting_temps()
9826
9827        self.body.generate_execution_code(code)
9828        self.trap_parallel_exit(code, should_flush=True)
9829        if self.is_parallel and not self.is_nested_prange:
9830            # nested pranges are not omp'ified, temps go to outer loops
9831            self.privatize_temps(code)
9832
9833        if self.breaking_label_used:
9834            # Put a guard around the loop body in case return, break or
9835            # exceptions might be used
9836            guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
9837
9838        code.end_block()  # end guard around loop body
9839        code.end_block()  # end for loop block
9840
9841        if self.is_parallel:
9842            # Release the GIL and deallocate the thread state
9843            self.end_parallel_block(code)
9844            code.end_block()  # pragma omp parallel end block
9845
9846
9847class CnameDecoratorNode(StatNode):
9848    """
9849    This node is for the cname decorator in CythonUtilityCode:
9850
9851        @cname('the_cname')
9852        cdef func(...):
9853            ...
9854
9855    In case of a cdef class the cname specifies the objstruct_cname.
9856
9857    node        the node to which the cname decorator is applied
9858    cname       the cname the node should get
9859    """
9860
9861    child_attrs = ['node']
9862
9863    def analyse_declarations(self, env):
9864        self.node.analyse_declarations(env)
9865
9866        node = self.node
9867        if isinstance(node, CompilerDirectivesNode):
9868            node = node.body.stats[0]
9869
9870        self.is_function = isinstance(node, FuncDefNode)
9871        is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode, CEnumDefNode))
9872        e = node.entry
9873
9874        if self.is_function:
9875            e.cname = self.cname
9876            e.func_cname = self.cname
9877            e.used = True
9878            if e.pyfunc_cname and '.' in e.pyfunc_cname:
9879                e.pyfunc_cname = self.mangle(e.pyfunc_cname)
9880        elif is_struct_or_enum:
9881            e.cname = e.type.cname = self.cname
9882        else:
9883            scope = node.scope
9884
9885            e.cname = self.cname
9886            e.type.objstruct_cname = self.cname + '_obj'
9887            e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
9888            e.type.typeptr_cname = self.cname + '_type'
9889            e.type.scope.namespace_cname = e.type.typeptr_cname
9890
9891            e.as_variable.cname = e.type.typeptr_cname
9892
9893            scope.scope_prefix = self.cname + "_"
9894
9895            for name, entry in scope.entries.items():
9896                if entry.func_cname:
9897                    entry.func_cname = self.mangle(entry.cname)
9898                if entry.pyfunc_cname:
9899                    entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)
9900
9901    def mangle(self, cname):
9902        if '.' in cname:
9903            # remove __pyx_base from func_cname
9904            cname = cname.split('.')[-1]
9905        return '%s_%s' % (self.cname, cname)
9906
9907    def analyse_expressions(self, env):
9908        self.node = self.node.analyse_expressions(env)
9909        return self
9910
9911    def generate_function_definitions(self, env, code):
9912        "Ensure a prototype for every @cname method in the right place"
9913        if self.is_function and env.is_c_class_scope:
9914            # method in cdef class, generate a prototype in the header
9915            h_code = code.globalstate['utility_code_proto']
9916
9917            if isinstance(self.node, DefNode):
9918                self.node.generate_function_header(
9919                    h_code, with_pymethdef=False, proto_only=True)
9920            else:
9921                from . import ModuleNode
9922                entry = self.node.entry
9923                cname = entry.cname
9924                entry.cname = entry.func_cname
9925
9926                ModuleNode.generate_cfunction_declaration(
9927                    entry,
9928                    env.global_scope(),
9929                    h_code,
9930                    definition=True)
9931
9932                entry.cname = cname
9933
9934        self.node.generate_function_definitions(env, code)
9935
9936    def generate_execution_code(self, code):
9937        self.node.generate_execution_code(code)
9938
9939
9940#------------------------------------------------------------------------------------
9941#
9942#  Runtime support code
9943#
9944#------------------------------------------------------------------------------------
9945
9946if Options.gcc_branch_hints:
9947    branch_prediction_macros = """
9948/* Test for GCC > 2.95 */
9949#if defined(__GNUC__) \
9950    && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
9951  #define likely(x)   __builtin_expect(!!(x), 1)
9952  #define unlikely(x) __builtin_expect(!!(x), 0)
9953#else /* !__GNUC__ or GCC < 2.95 */
9954  #define likely(x)   (x)
9955  #define unlikely(x) (x)
9956#endif /* __GNUC__ */
9957"""
9958else:
9959    branch_prediction_macros = """
9960#define likely(x)   (x)
9961#define unlikely(x) (x)
9962"""
9963
9964#------------------------------------------------------------------------------------
9965
9966printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
9967printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
9968
9969#------------------------------------------------------------------------------------
9970
9971# Exception raising code
9972#
9973# Exceptions are raised by __Pyx_Raise() and stored as plain
9974# type/value/tb in PyThreadState->curexc_*.  When being caught by an
9975# 'except' statement, curexc_* is moved over to exc_* by
9976# __Pyx_GetException()
9977
9978restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
9979raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
9980get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
9981swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
9982reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
9983traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
9984
9985#------------------------------------------------------------------------------------
9986
9987get_exception_tuple_utility_code = UtilityCode(
9988    proto="""
9989static PyObject *__Pyx_GetExceptionTuple(PyThreadState *__pyx_tstate); /*proto*/
9990""",
9991    # I doubt that calling __Pyx_GetException() here is correct as it moves
9992    # the exception from tstate->curexc_* to tstate->exc_*, which prevents
9993    # exception handlers later on from receiving it.
9994    # NOTE: "__pyx_tstate" may be used by __Pyx_GetException() macro
9995    impl = """
9996static PyObject *__Pyx_GetExceptionTuple(CYTHON_UNUSED PyThreadState *__pyx_tstate) {
9997    PyObject *type = NULL, *value = NULL, *tb = NULL;
9998    if (__Pyx_GetException(&type, &value, &tb) == 0) {
9999        PyObject* exc_info = PyTuple_New(3);
10000        if (exc_info) {
10001            Py_INCREF(type);
10002            Py_INCREF(value);
10003            Py_INCREF(tb);
10004            PyTuple_SET_ITEM(exc_info, 0, type);
10005            PyTuple_SET_ITEM(exc_info, 1, value);
10006            PyTuple_SET_ITEM(exc_info, 2, tb);
10007            return exc_info;
10008        }
10009    }
10010    return NULL;
10011}
10012""",
10013    requires=[get_exception_utility_code])
10014