1#
2#   Module parse tree node
3#
4
5from __future__ import absolute_import
6
7import cython
8cython.declare(Naming=object, Options=object, PyrexTypes=object, TypeSlots=object,
9               error=object, warning=object, py_object_type=object, UtilityCode=object,
10               EncodedString=object, re=object)
11
12from collections import defaultdict
13import json
14import operator
15import os
16import re
17
18from .PyrexTypes import CPtrType
19from . import Future
20from . import Annotate
21from . import Code
22from . import Naming
23from . import Nodes
24from . import Options
25from . import TypeSlots
26from . import PyrexTypes
27from . import Pythran
28
29from .Errors import error, warning
30from .PyrexTypes import py_object_type
31from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version
32from .Code import UtilityCode, IncludeCode
33from .StringEncoding import EncodedString
34from .Pythran import has_np_pythran
35
36def check_c_declarations_pxd(module_node):
37    module_node.scope.check_c_classes_pxd()
38    return module_node
39
40
41def check_c_declarations(module_node):
42    module_node.scope.check_c_classes()
43    module_node.scope.check_c_functions()
44    return module_node
45
46
47def generate_c_code_config(env, options):
48    if Options.annotate or options.annotate:
49        emit_linenums = False
50    else:
51        emit_linenums = options.emit_linenums
52
53    return Code.CCodeConfig(
54        emit_linenums=emit_linenums,
55        emit_code_comments=env.directives['emit_code_comments'],
56        c_line_in_traceback=options.c_line_in_traceback)
57
58
59class ModuleNode(Nodes.Node, Nodes.BlockNode):
60    #  doc       string or None
61    #  body      StatListNode
62    #
63    #  referenced_modules   [ModuleScope]
64    #  full_module_name     string
65    #
66    #  scope                The module scope.
67    #  compilation_source   A CompilationSource (see Main)
68    #  directives           Top-level compiler directives
69
70    child_attrs = ["body"]
71    directives = None
72
73    def merge_in(self, tree, scope, merge_scope=False):
74        # Merges in the contents of another tree, and possibly scope. With the
75        # current implementation below, this must be done right prior
76        # to code generation.
77        #
78        # Note: This way of doing it seems strange -- I believe the
79        # right concept is to split ModuleNode into a ModuleNode and a
80        # CodeGenerator, and tell that CodeGenerator to generate code
81        # from multiple sources.
82        assert isinstance(self.body, Nodes.StatListNode)
83        if isinstance(tree, Nodes.StatListNode):
84            self.body.stats.extend(tree.stats)
85        else:
86            self.body.stats.append(tree)
87
88        self.scope.utility_code_list.extend(scope.utility_code_list)
89
90        for inc in scope.c_includes.values():
91            self.scope.process_include(inc)
92
93        def extend_if_not_in(L1, L2):
94            for x in L2:
95                if x not in L1:
96                    L1.append(x)
97
98        extend_if_not_in(self.scope.included_files, scope.included_files)
99
100        if merge_scope:
101            # Ensure that we don't generate import code for these entries!
102            for entry in scope.c_class_entries:
103                entry.type.module_name = self.full_module_name
104                entry.type.scope.directives["internal"] = True
105
106            self.scope.merge_in(scope)
107
108    def analyse_declarations(self, env):
109        if has_np_pythran(env):
110            Pythran.include_pythran_generic(env)
111        if self.directives:
112            env.old_style_globals = self.directives['old_style_globals']
113        if not Options.docstrings:
114            env.doc = self.doc = None
115        elif Options.embed_pos_in_docstring:
116            env.doc = EncodedString(u'File: %s (starting at line %s)' % Nodes.relative_position(self.pos))
117            if self.doc is not None:
118                env.doc = EncodedString(env.doc + u'\n' + self.doc)
119                env.doc.encoding = self.doc.encoding
120        else:
121            env.doc = self.doc
122        env.directives = self.directives
123
124        self.body.analyse_declarations(env)
125
126    def prepare_utility_code(self):
127        # prepare any utility code that must be created before code generation
128        # specifically: CythonUtilityCode
129        env = self.scope
130        if env.has_import_star:
131            self.create_import_star_conversion_utility_code(env)
132        for name, entry in sorted(env.entries.items()):
133            if (entry.create_wrapper and entry.scope is env
134                and entry.is_type and entry.type.is_enum):
135                    entry.type.create_type_wrapper(env)
136
137    def process_implementation(self, options, result):
138        env = self.scope
139        env.return_type = PyrexTypes.c_void_type
140        self.referenced_modules = []
141        self.find_referenced_modules(env, self.referenced_modules, {})
142        self.sort_cdef_classes(env)
143        self.generate_c_code(env, options, result)
144        self.generate_h_code(env, options, result)
145        self.generate_api_code(env, options, result)
146
147    def has_imported_c_functions(self):
148        for module in self.referenced_modules:
149            for entry in module.cfunc_entries:
150                if entry.defined_in_pxd:
151                    return 1
152        return 0
153
154    def generate_h_code(self, env, options, result):
155        def h_entries(entries, api=0, pxd=0):
156            return [entry for entry in entries
157                    if ((entry.visibility == 'public') or
158                        (api and entry.api) or
159                        (pxd and entry.defined_in_pxd))]
160        h_types = h_entries(env.type_entries, api=1)
161        h_vars = h_entries(env.var_entries)
162        h_funcs = h_entries(env.cfunc_entries)
163        h_extension_types = h_entries(env.c_class_entries)
164        if h_types or  h_vars or h_funcs or h_extension_types:
165            result.h_file = replace_suffix(result.c_file, ".h")
166            h_code = Code.CCodeWriter()
167            c_code_config = generate_c_code_config(env, options)
168            Code.GlobalState(h_code, self, c_code_config)
169            if options.generate_pxi:
170                result.i_file = replace_suffix(result.c_file, ".pxi")
171                i_code = Code.PyrexCodeWriter(result.i_file)
172            else:
173                i_code = None
174
175            h_code.put_generated_by()
176            h_guard = Naming.h_guard_prefix + self.api_name(env)
177            h_code.put_h_guard(h_guard)
178            h_code.putln("")
179            self.generate_type_header_code(h_types, h_code)
180            if options.capi_reexport_cincludes:
181                self.generate_includes(env, [], h_code)
182            h_code.putln("")
183            api_guard = Naming.api_guard_prefix + self.api_name(env)
184            h_code.putln("#ifndef %s" % api_guard)
185            h_code.putln("")
186            self.generate_extern_c_macro_definition(h_code)
187            h_code.putln("")
188            self.generate_dl_import_macro(h_code)
189            if h_extension_types:
190                h_code.putln("")
191                for entry in h_extension_types:
192                    self.generate_cclass_header_code(entry.type, h_code)
193                    if i_code:
194                        self.generate_cclass_include_code(entry.type, i_code)
195            if h_funcs:
196                h_code.putln("")
197                for entry in h_funcs:
198                    self.generate_public_declaration(entry, h_code, i_code)
199            if h_vars:
200                h_code.putln("")
201                for entry in h_vars:
202                    self.generate_public_declaration(entry, h_code, i_code)
203            h_code.putln("")
204            h_code.putln("#endif /* !%s */" % api_guard)
205            h_code.putln("")
206            h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */")
207            h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */")
208            h_code.putln("")
209            h_code.putln("#if PY_MAJOR_VERSION < 3")
210            h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name)
211            h_code.putln("#else")
212            h_code.putln("PyMODINIT_FUNC %s(void);" % self.mod_init_func_cname('PyInit', env))
213            h_code.putln("#endif")
214            h_code.putln("")
215            h_code.putln("#endif /* !%s */" % h_guard)
216
217            f = open_new_file(result.h_file)
218            try:
219                h_code.copyto(f)
220            finally:
221                f.close()
222
223    def generate_public_declaration(self, entry, h_code, i_code):
224        h_code.putln("%s %s;" % (
225            Naming.extern_c_macro,
226            entry.type.declaration_code(entry.cname)))
227        if i_code:
228            i_code.putln("cdef extern %s" % (
229                entry.type.declaration_code(entry.cname, pyrex=1)))
230
231    def api_name(self, env):
232        return env.qualified_name.replace(".", "__")
233
234    def generate_api_code(self, env, options, result):
235        def api_entries(entries, pxd=0):
236            return [entry for entry in entries
237                    if entry.api or (pxd and entry.defined_in_pxd)]
238        api_vars = api_entries(env.var_entries)
239        api_funcs = api_entries(env.cfunc_entries)
240        api_extension_types = api_entries(env.c_class_entries)
241        if api_vars or api_funcs or api_extension_types:
242            result.api_file = replace_suffix(result.c_file, "_api.h")
243            h_code = Code.CCodeWriter()
244            c_code_config = generate_c_code_config(env, options)
245            Code.GlobalState(h_code, self, c_code_config)
246            h_code.put_generated_by()
247            api_guard = Naming.api_guard_prefix + self.api_name(env)
248            h_code.put_h_guard(api_guard)
249            # Work around https://bugs.python.org/issue4709
250            h_code.putln('#ifdef __MINGW64__')
251            h_code.putln('#define MS_WIN64')
252            h_code.putln('#endif')
253
254            h_code.putln('#include "Python.h"')
255            if result.h_file:
256                h_code.putln('#include "%s"' % os.path.basename(result.h_file))
257            if api_extension_types:
258                h_code.putln("")
259                for entry in api_extension_types:
260                    type = entry.type
261                    h_code.putln("static PyTypeObject *%s = 0;" % type.typeptr_cname)
262                    h_code.putln("#define %s (*%s)" % (
263                        type.typeobj_cname, type.typeptr_cname))
264            if api_funcs:
265                h_code.putln("")
266                for entry in api_funcs:
267                    type = CPtrType(entry.type)
268                    cname = env.mangle(Naming.func_prefix_api, entry.name)
269                    h_code.putln("static %s = 0;" % type.declaration_code(cname))
270                    h_code.putln("#define %s %s" % (entry.name, cname))
271            if api_vars:
272                h_code.putln("")
273                for entry in api_vars:
274                    type = CPtrType(entry.type)
275                    cname = env.mangle(Naming.varptr_prefix_api, entry.name)
276                    h_code.putln("static %s = 0;" %  type.declaration_code(cname))
277                    h_code.putln("#define %s (*%s)" % (entry.name, cname))
278            h_code.put(UtilityCode.load_as_string("PyIdentifierFromString", "ImportExport.c")[0])
279            if api_vars:
280                h_code.put(UtilityCode.load_as_string("VoidPtrImport", "ImportExport.c")[1])
281            if api_funcs:
282                h_code.put(UtilityCode.load_as_string("FunctionImport", "ImportExport.c")[1])
283            if api_extension_types:
284                h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[0])
285                h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[1])
286            h_code.putln("")
287            h_code.putln("static int import_%s(void) {" % self.api_name(env))
288            h_code.putln("PyObject *module = 0;")
289            h_code.putln('module = PyImport_ImportModule("%s");' % env.qualified_name)
290            h_code.putln("if (!module) goto bad;")
291            for entry in api_funcs:
292                cname = env.mangle(Naming.func_prefix_api, entry.name)
293                sig = entry.type.signature_string()
294                h_code.putln(
295                    'if (__Pyx_ImportFunction(module, "%s", (void (**)(void))&%s, "%s") < 0) goto bad;'
296                    % (entry.name, cname, sig))
297            for entry in api_vars:
298                cname = env.mangle(Naming.varptr_prefix_api, entry.name)
299                sig = entry.type.empty_declaration_code()
300                h_code.putln(
301                    'if (__Pyx_ImportVoidPtr(module, "%s", (void **)&%s, "%s") < 0) goto bad;'
302                    % (entry.name, cname, sig))
303            with ModuleImportGenerator(h_code, imported_modules={env.qualified_name: 'module'}) as import_generator:
304                for entry in api_extension_types:
305                    self.generate_type_import_call(entry.type, h_code, import_generator, error_code="goto bad;")
306            h_code.putln("Py_DECREF(module); module = 0;")
307            h_code.putln("return 0;")
308            h_code.putln("bad:")
309            h_code.putln("Py_XDECREF(module);")
310            h_code.putln("return -1;")
311            h_code.putln("}")
312            h_code.putln("")
313            h_code.putln("#endif /* !%s */" % api_guard)
314
315            f = open_new_file(result.api_file)
316            try:
317                h_code.copyto(f)
318            finally:
319                f.close()
320
321    def generate_cclass_header_code(self, type, h_code):
322        h_code.putln("%s %s %s;" % (
323            Naming.extern_c_macro,
324            PyrexTypes.public_decl("PyTypeObject", "DL_IMPORT"),
325            type.typeobj_cname))
326
327    def generate_cclass_include_code(self, type, i_code):
328        i_code.putln("cdef extern class %s.%s:" % (
329            type.module_name, type.name))
330        i_code.indent()
331        var_entries = type.scope.var_entries
332        if var_entries:
333            for entry in var_entries:
334                i_code.putln("cdef %s" % (
335                    entry.type.declaration_code(entry.cname, pyrex=1)))
336        else:
337            i_code.putln("pass")
338        i_code.dedent()
339
340    def generate_c_code(self, env, options, result):
341        modules = self.referenced_modules
342
343        if Options.annotate or options.annotate:
344            rootwriter = Annotate.AnnotationCCodeWriter()
345        else:
346            rootwriter = Code.CCodeWriter()
347
348        c_code_config = generate_c_code_config(env, options)
349
350        globalstate = Code.GlobalState(
351            rootwriter, self,
352            code_config=c_code_config,
353            common_utility_include_dir=options.common_utility_include_dir,
354        )
355        globalstate.initialize_main_c_code()
356        h_code = globalstate['h_code']
357
358        self.generate_module_preamble(env, options, modules, result.embedded_metadata, h_code)
359
360        globalstate.module_pos = self.pos
361        globalstate.directives = self.directives
362
363        globalstate.use_utility_code(refnanny_utility_code)
364
365        code = globalstate['before_global_var']
366        code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name)
367        module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__'))
368        code.putln("extern int %s;" % module_is_main)
369        code.putln("int %s = 0;" % module_is_main)
370        code.putln("")
371        code.putln("/* Implementation of '%s' */" % env.qualified_name)
372
373        code = globalstate['late_includes']
374        code.putln("/* Late includes */")
375        self.generate_includes(env, modules, code, early=False)
376
377        code = globalstate['all_the_rest']
378
379        self.generate_cached_builtins_decls(env, code)
380        self.generate_lambda_definitions(env, code)
381        # generate normal variable and function definitions
382        self.generate_variable_definitions(env, code)
383
384        self.body.generate_function_definitions(env, code)
385
386        code.mark_pos(None)
387        self.generate_typeobj_definitions(env, code)
388        self.generate_method_table(env, code)
389        if env.has_import_star:
390            self.generate_import_star(env, code)
391        self.generate_pymoduledef_struct(env, code)
392
393        # initialise the macro to reduce the code size of one-time functionality
394        code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip())
395
396        # init_globals is inserted before this
397        self.generate_module_init_func(modules[:-1], env, globalstate['init_module'])
398        self.generate_module_cleanup_func(env, globalstate['cleanup_module'])
399        if Options.embed:
400            self.generate_main_method(env, globalstate['main_method'])
401        self.generate_filename_table(globalstate['filename_table'])
402
403        self.generate_declarations_for_modules(env, modules, globalstate)
404        h_code.write('\n')
405
406        for utilcode in env.utility_code_list[:]:
407            globalstate.use_utility_code(utilcode)
408        globalstate.finalize_main_c_code()
409
410        f = open_new_file(result.c_file)
411        try:
412            rootwriter.copyto(f)
413        finally:
414            f.close()
415        result.c_file_generated = 1
416        if options.gdb_debug:
417            self._serialize_lineno_map(env, rootwriter)
418        if Options.annotate or options.annotate:
419            self._generate_annotations(rootwriter, result, options)
420
421    def _generate_annotations(self, rootwriter, result, options):
422        self.annotate(rootwriter)
423
424        coverage_xml_filename = Options.annotate_coverage_xml or options.annotate_coverage_xml
425        if coverage_xml_filename and os.path.exists(coverage_xml_filename):
426            try:
427                import xml.etree.cElementTree as ET
428            except ImportError:
429                import xml.etree.ElementTree as ET
430            coverage_xml = ET.parse(coverage_xml_filename).getroot()
431            for el in coverage_xml.getiterator():
432                el.tail = None  # save some memory
433        else:
434            coverage_xml = None
435
436        rootwriter.save_annotation(result.main_source_file, result.c_file, coverage_xml=coverage_xml)
437
438        # if we included files, additionally generate one annotation file for each
439        if not self.scope.included_files:
440            return
441
442        search_include_file = self.scope.context.search_include_directories
443        target_dir = os.path.abspath(os.path.dirname(result.c_file))
444        for included_file in self.scope.included_files:
445            target_file = os.path.abspath(os.path.join(target_dir, included_file))
446            target_file_dir = os.path.dirname(target_file)
447            if not target_file_dir.startswith(target_dir):
448                # any other directories may not be writable => avoid trying
449                continue
450            source_file = search_include_file(included_file, "", self.pos, include=True)
451            if not source_file:
452                continue
453            if target_file_dir != target_dir and not os.path.exists(target_file_dir):
454                try:
455                    os.makedirs(target_file_dir)
456                except OSError as e:
457                    import errno
458                    if e.errno != errno.EEXIST:
459                        raise
460            rootwriter.save_annotation(source_file, target_file, coverage_xml=coverage_xml)
461
462    def _serialize_lineno_map(self, env, ccodewriter):
463        tb = env.context.gdb_debug_outputwriter
464        markers = ccodewriter.buffer.allmarkers()
465
466        d = defaultdict(list)
467        for c_lineno, cython_lineno in enumerate(markers):
468            if cython_lineno > 0:
469                d[cython_lineno].append(c_lineno + 1)
470
471        tb.start('LineNumberMapping')
472        for cython_lineno, c_linenos in sorted(d.items()):
473            tb.add_entry(
474                'LineNumber',
475                c_linenos=' '.join(map(str, c_linenos)),
476                cython_lineno=str(cython_lineno),
477            )
478        tb.end('LineNumberMapping')
479        tb.serialize()
480
481    def find_referenced_modules(self, env, module_list, modules_seen):
482        if env not in modules_seen:
483            modules_seen[env] = 1
484            for imported_module in env.cimported_modules:
485                self.find_referenced_modules(imported_module, module_list, modules_seen)
486            module_list.append(env)
487
488    def sort_types_by_inheritance(self, type_dict, type_order, getkey):
489        # copy the types into a list moving each parent type before
490        # its first child
491        type_list = []
492        for i, key in enumerate(type_order):
493            new_entry = type_dict[key]
494
495            # collect all base classes to check for children
496            hierarchy = set()
497            base = new_entry
498            while base:
499                base_type = base.type.base_type
500                if not base_type:
501                    break
502                base_key = getkey(base_type)
503                hierarchy.add(base_key)
504                base = type_dict.get(base_key)
505            new_entry.base_keys = hierarchy
506
507            # find the first (sub-)subclass and insert before that
508            for j in range(i):
509                entry = type_list[j]
510                if key in entry.base_keys:
511                    type_list.insert(j, new_entry)
512                    break
513            else:
514                type_list.append(new_entry)
515        return type_list
516
517    def sort_type_hierarchy(self, module_list, env):
518        # poor developer's OrderedDict
519        vtab_dict, vtab_dict_order = {}, []
520        vtabslot_dict, vtabslot_dict_order = {}, []
521
522        for module in module_list:
523            for entry in module.c_class_entries:
524                if entry.used and not entry.in_cinclude:
525                    type = entry.type
526                    key = type.vtabstruct_cname
527                    if not key:
528                        continue
529                    if key in vtab_dict:
530                        # FIXME: this should *never* happen, but apparently it does
531                        # for Cython generated utility code
532                        from .UtilityCode import NonManglingModuleScope
533                        assert isinstance(entry.scope, NonManglingModuleScope), str(entry.scope)
534                        assert isinstance(vtab_dict[key].scope, NonManglingModuleScope), str(vtab_dict[key].scope)
535                    else:
536                        vtab_dict[key] = entry
537                        vtab_dict_order.append(key)
538            all_defined_here = module is env
539            for entry in module.type_entries:
540                if entry.used and (all_defined_here or entry.defined_in_pxd):
541                    type = entry.type
542                    if type.is_extension_type and not entry.in_cinclude:
543                        type = entry.type
544                        key = type.objstruct_cname
545                        assert key not in vtabslot_dict, key
546                        vtabslot_dict[key] = entry
547                        vtabslot_dict_order.append(key)
548
549        def vtabstruct_cname(entry_type):
550            return entry_type.vtabstruct_cname
551        vtab_list = self.sort_types_by_inheritance(
552            vtab_dict, vtab_dict_order, vtabstruct_cname)
553
554        def objstruct_cname(entry_type):
555            return entry_type.objstruct_cname
556        vtabslot_list = self.sort_types_by_inheritance(
557            vtabslot_dict, vtabslot_dict_order, objstruct_cname)
558
559        return (vtab_list, vtabslot_list)
560
561    def sort_cdef_classes(self, env):
562        key_func = operator.attrgetter('objstruct_cname')
563        entry_dict, entry_order = {}, []
564        for entry in env.c_class_entries:
565            key = key_func(entry.type)
566            assert key not in entry_dict, key
567            entry_dict[key] = entry
568            entry_order.append(key)
569        env.c_class_entries[:] = self.sort_types_by_inheritance(
570            entry_dict, entry_order, key_func)
571
572    def generate_type_definitions(self, env, modules, vtab_list, vtabslot_list, code):
573        # TODO: Why are these separated out?
574        for entry in vtabslot_list:
575            self.generate_objstruct_predeclaration(entry.type, code)
576        vtabslot_entries = set(vtabslot_list)
577        for module in modules:
578            definition = module is env
579            if definition:
580                type_entries = module.type_entries
581            else:
582                type_entries = []
583                for entry in module.type_entries:
584                    if entry.defined_in_pxd:
585                        type_entries.append(entry)
586            type_entries = [t for t in type_entries if t not in vtabslot_entries]
587            self.generate_type_header_code(type_entries, code)
588        for entry in vtabslot_list:
589            self.generate_objstruct_definition(entry.type, code)
590            self.generate_typeobj_predeclaration(entry, code)
591        for entry in vtab_list:
592            self.generate_typeobj_predeclaration(entry, code)
593            self.generate_exttype_vtable_struct(entry, code)
594            self.generate_exttype_vtabptr_declaration(entry, code)
595            self.generate_exttype_final_methods_declaration(entry, code)
596
597    def generate_declarations_for_modules(self, env, modules, globalstate):
598        typecode = globalstate['type_declarations']
599        typecode.putln("")
600        typecode.putln("/*--- Type declarations ---*/")
601        # This is to work around the fact that array.h isn't part of the C-API,
602        # but we need to declare it earlier than utility code.
603        if 'cpython.array' in [m.qualified_name for m in modules]:
604            typecode.putln('#ifndef _ARRAYARRAY_H')
605            typecode.putln('struct arrayobject;')
606            typecode.putln('typedef struct arrayobject arrayobject;')
607            typecode.putln('#endif')
608        vtab_list, vtabslot_list = self.sort_type_hierarchy(modules, env)
609        self.generate_type_definitions(
610            env, modules, vtab_list, vtabslot_list, typecode)
611        modulecode = globalstate['module_declarations']
612        for module in modules:
613            defined_here = module is env
614            modulecode.putln("")
615            modulecode.putln("/* Module declarations from '%s' */" % module.qualified_name)
616            self.generate_c_class_declarations(module, modulecode, defined_here)
617            self.generate_cvariable_declarations(module, modulecode, defined_here)
618            self.generate_cfunction_declarations(module, modulecode, defined_here)
619
620    def _put_setup_code(self, code, name):
621        code.put(UtilityCode.load_as_string(name, "ModuleSetupCode.c")[1])
622
623    def generate_module_preamble(self, env, options, cimported_modules, metadata, code):
624        code.put_generated_by()
625        if metadata:
626            code.putln("/* BEGIN: Cython Metadata")
627            code.putln(json.dumps(metadata, indent=4, sort_keys=True))
628            code.putln("END: Cython Metadata */")
629            code.putln("")
630        code.putln("#define PY_SSIZE_T_CLEAN")
631
632        for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey):
633            if inc.location == inc.INITIAL:
634                inc.write(code)
635        code.putln("#ifndef Py_PYTHON_H")
636        code.putln("    #error Python headers needed to compile C extensions, "
637                   "please install development version of Python.")
638        code.putln("#elif PY_VERSION_HEX < 0x02060000 || "
639                   "(0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000)")
640        code.putln("    #error Cython requires Python 2.6+ or Python 3.3+.")
641        code.putln("#else")
642        code.globalstate["end"].putln("#endif /* Py_PYTHON_H */")
643
644        from .. import __version__
645        code.putln('#define CYTHON_ABI "%s"' % __version__.replace('.', '_'))
646        code.putln('#define CYTHON_HEX_VERSION %s' % build_hex_version(__version__))
647        code.putln("#define CYTHON_FUTURE_DIVISION %d" % (
648            Future.division in env.context.future_directives))
649
650        self._put_setup_code(code, "CModulePreamble")
651        if env.context.options.cplus:
652            self._put_setup_code(code, "CppInitCode")
653        else:
654            self._put_setup_code(code, "CInitCode")
655        self._put_setup_code(code, "PythonCompatibility")
656        self._put_setup_code(code, "MathInitCode")
657
658        if options.c_line_in_traceback:
659            cinfo = "%s = %s; " % (Naming.clineno_cname, Naming.line_c_macro)
660        else:
661            cinfo = ""
662        code.put("""
663#define __PYX_ERR(f_index, lineno, Ln_error) \\
664{ \\
665  %s = %s[f_index]; %s = lineno; %sgoto Ln_error; \\
666}
667""" % (Naming.filename_cname, Naming.filetable_cname, Naming.lineno_cname, cinfo))
668
669        code.putln("")
670        self.generate_extern_c_macro_definition(code)
671        code.putln("")
672
673        code.putln("#define %s" % Naming.h_guard_prefix + self.api_name(env))
674        code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env))
675        code.putln("/* Early includes */")
676        self.generate_includes(env, cimported_modules, code, late=False)
677        code.putln("")
678        code.putln("#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS)")
679        code.putln("#define CYTHON_WITHOUT_ASSERTIONS")
680        code.putln("#endif")
681        code.putln("")
682
683        if env.directives['ccomplex']:
684            code.putln("")
685            code.putln("#if !defined(CYTHON_CCOMPLEX)")
686            code.putln("#define CYTHON_CCOMPLEX 1")
687            code.putln("#endif")
688            code.putln("")
689        code.put(UtilityCode.load_as_string("UtilityFunctionPredeclarations", "ModuleSetupCode.c")[0])
690
691        c_string_type = env.directives['c_string_type']
692        c_string_encoding = env.directives['c_string_encoding']
693        if c_string_type not in ('bytes', 'bytearray') and not c_string_encoding:
694            error(self.pos, "a default encoding must be provided if c_string_type is not a byte type")
695        code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII %s' % int(c_string_encoding == 'ascii'))
696        code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 %s' %
697                int(c_string_encoding.replace('-', '').lower() == 'utf8'))
698        if c_string_encoding == 'default':
699            code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 1')
700        else:
701            code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT '
702                    '(PY_MAJOR_VERSION >= 3 && __PYX_DEFAULT_STRING_ENCODING_IS_UTF8)')
703            code.putln('#define __PYX_DEFAULT_STRING_ENCODING "%s"' % c_string_encoding)
704        if c_string_type == 'bytearray':
705            c_string_func_name = 'ByteArray'
706        else:
707            c_string_func_name = c_string_type.title()
708        code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_func_name)
709        code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_func_name)
710        code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0])
711
712        # These utility functions are assumed to exist and used elsewhere.
713        PyrexTypes.c_long_type.create_to_py_utility_code(env)
714        PyrexTypes.c_long_type.create_from_py_utility_code(env)
715        PyrexTypes.c_int_type.create_from_py_utility_code(env)
716
717        code.put(Nodes.branch_prediction_macros)
718        code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }')
719        code.putln('')
720        code.putln('static PyObject *%s = NULL;' % env.module_cname)
721        code.putln('static PyObject *%s;' % env.module_dict_cname)
722        code.putln('static PyObject *%s;' % Naming.builtins_cname)
723        code.putln('static PyObject *%s = NULL;' % Naming.cython_runtime_cname)
724        code.putln('static PyObject *%s;' % Naming.empty_tuple)
725        code.putln('static PyObject *%s;' % Naming.empty_bytes)
726        code.putln('static PyObject *%s;' % Naming.empty_unicode)
727        if Options.pre_import is not None:
728            code.putln('static PyObject *%s;' % Naming.preimport_cname)
729        code.putln('static int %s;' % Naming.lineno_cname)
730        code.putln('static int %s = 0;' % Naming.clineno_cname)
731        code.putln('static const char * %s= %s;' % (Naming.cfilenm_cname, Naming.file_c_macro))
732        code.putln('static const char *%s;' % Naming.filename_cname)
733
734        env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c"))
735        if has_np_pythran(env):
736            env.use_utility_code(UtilityCode.load_cached("PythranConversion", "CppSupport.cpp"))
737
738    def generate_extern_c_macro_definition(self, code):
739        name = Naming.extern_c_macro
740        code.putln("#ifndef %s" % name)
741        code.putln("  #ifdef __cplusplus")
742        code.putln('    #define %s extern "C"' % name)
743        code.putln("  #else")
744        code.putln("    #define %s extern" % name)
745        code.putln("  #endif")
746        code.putln("#endif")
747
748    def generate_dl_import_macro(self, code):
749        code.putln("#ifndef DL_IMPORT")
750        code.putln("  #define DL_IMPORT(_T) _T")
751        code.putln("#endif")
752
753    def generate_includes(self, env, cimported_modules, code, early=True, late=True):
754        includes = []
755        for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey):
756            if inc.location == inc.EARLY:
757                if early:
758                    inc.write(code)
759            elif inc.location == inc.LATE:
760                if late:
761                    inc.write(code)
762        if early:
763            code.putln_openmp("#include <omp.h>")
764
765    def generate_filename_table(self, code):
766        from os.path import isabs, basename
767        code.putln("")
768        code.putln("static const char *%s[] = {" % Naming.filetable_cname)
769        if code.globalstate.filename_list:
770            for source_desc in code.globalstate.filename_list:
771                file_path = source_desc.get_filenametable_entry()
772                if isabs(file_path):
773                    file_path = basename(file_path)  # never include absolute paths
774                escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"')
775                code.putln('"%s",' % escaped_filename)
776        else:
777            # Some C compilers don't like an empty array
778            code.putln("0")
779        code.putln("};")
780
781    def generate_type_predeclarations(self, env, code):
782        pass
783
784    def generate_type_header_code(self, type_entries, code):
785        # Generate definitions of structs/unions/enums/typedefs/objstructs.
786        #self.generate_gcc33_hack(env, code) # Is this still needed?
787        # Forward declarations
788        for entry in type_entries:
789            if not entry.in_cinclude:
790                #print "generate_type_header_code:", entry.name, repr(entry.type) ###
791                type = entry.type
792                if type.is_typedef: # Must test this first!
793                    pass
794                elif type.is_struct_or_union or type.is_cpp_class:
795                    self.generate_struct_union_predeclaration(entry, code)
796                elif type.is_ctuple and entry.used:
797                    self.generate_struct_union_predeclaration(entry.type.struct_entry, code)
798                elif type.is_extension_type:
799                    self.generate_objstruct_predeclaration(type, code)
800        # Actual declarations
801        for entry in type_entries:
802            if not entry.in_cinclude:
803                #print "generate_type_header_code:", entry.name, repr(entry.type) ###
804                type = entry.type
805                if type.is_typedef: # Must test this first!
806                    self.generate_typedef(entry, code)
807                elif type.is_enum:
808                    self.generate_enum_definition(entry, code)
809                elif type.is_struct_or_union:
810                    self.generate_struct_union_definition(entry, code)
811                elif type.is_ctuple and entry.used:
812                    self.generate_struct_union_definition(entry.type.struct_entry, code)
813                elif type.is_cpp_class:
814                    self.generate_cpp_class_definition(entry, code)
815                elif type.is_extension_type:
816                    self.generate_objstruct_definition(type, code)
817
818    def generate_gcc33_hack(self, env, code):
819        # Workaround for spurious warning generation in gcc 3.3
820        code.putln("")
821        for entry in env.c_class_entries:
822            type = entry.type
823            if not type.typedef_flag:
824                name = type.objstruct_cname
825                if name.startswith("__pyx_"):
826                    tail = name[6:]
827                else:
828                    tail = name
829                code.putln("typedef struct %s __pyx_gcc33_%s;" % (
830                    name, tail))
831
832    def generate_typedef(self, entry, code):
833        base_type = entry.type.typedef_base_type
834        if base_type.is_numeric:
835            try:
836                writer = code.globalstate['numeric_typedefs']
837            except KeyError:
838                writer = code
839        else:
840            writer = code
841        writer.mark_pos(entry.pos)
842        writer.putln("typedef %s;" % base_type.declaration_code(entry.cname))
843
844    def sue_predeclaration(self, type, kind, name):
845        if type.typedef_flag:
846            return "%s %s;\ntypedef %s %s %s;" % (
847                kind, name,
848                kind, name, name)
849        else:
850            return "%s %s;" % (kind, name)
851
852    def generate_struct_union_predeclaration(self, entry, code):
853        type = entry.type
854        if type.is_cpp_class and type.templates:
855            code.putln("template <typename %s>" % ", typename ".join(
856                [T.empty_declaration_code() for T in type.templates]))
857        code.putln(self.sue_predeclaration(type, type.kind, type.cname))
858
859    def sue_header_footer(self, type, kind, name):
860        header = "%s %s {" % (kind, name)
861        footer = "};"
862        return header, footer
863
864    def generate_struct_union_definition(self, entry, code):
865        code.mark_pos(entry.pos)
866        type = entry.type
867        scope = type.scope
868        if scope:
869            kind = type.kind
870            packed = type.is_struct and type.packed
871            if packed:
872                kind = "%s %s" % (type.kind, "__Pyx_PACKED")
873                code.globalstate.use_utility_code(packed_struct_utility_code)
874            header, footer = \
875                self.sue_header_footer(type, kind, type.cname)
876            if packed:
877                code.putln("#if defined(__SUNPRO_C)")
878                code.putln("  #pragma pack(1)")
879                code.putln("#elif !defined(__GNUC__)")
880                code.putln("  #pragma pack(push, 1)")
881                code.putln("#endif")
882            code.putln(header)
883            var_entries = scope.var_entries
884            if not var_entries:
885                error(entry.pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block")
886            for attr in var_entries:
887                code.putln(
888                    "%s;" % attr.type.declaration_code(attr.cname))
889            code.putln(footer)
890            if packed:
891                code.putln("#if defined(__SUNPRO_C)")
892                code.putln("  #pragma pack()")
893                code.putln("#elif !defined(__GNUC__)")
894                code.putln("  #pragma pack(pop)")
895                code.putln("#endif")
896
897    def generate_cpp_class_definition(self, entry, code):
898        code.mark_pos(entry.pos)
899        type = entry.type
900        scope = type.scope
901        if scope:
902            if type.templates:
903                code.putln("template <class %s>" % ", class ".join(
904                    [T.empty_declaration_code() for T in type.templates]))
905            # Just let everything be public.
906            code.put("struct %s" % type.cname)
907            if type.base_classes:
908                base_class_decl = ", public ".join(
909                    [base_class.empty_declaration_code() for base_class in type.base_classes])
910                code.put(" : public %s" % base_class_decl)
911            code.putln(" {")
912            py_attrs = [e for e in scope.entries.values()
913                        if e.type.is_pyobject and not e.is_inherited]
914            has_virtual_methods = False
915            constructor = None
916            destructor = None
917            for attr in scope.var_entries:
918                if attr.type.is_cfunction and attr.type.is_static_method:
919                    code.put("static ")
920                elif attr.name == "<init>":
921                    constructor = attr
922                elif attr.name == "<del>":
923                    destructor = attr
924                elif attr.type.is_cfunction:
925                    code.put("virtual ")
926                    has_virtual_methods = True
927                code.putln("%s;" % attr.type.declaration_code(attr.cname))
928            is_implementing = 'init_module' in code.globalstate.parts
929            if constructor or py_attrs:
930                if constructor:
931                    arg_decls = []
932                    arg_names = []
933                    for arg in constructor.type.original_args[
934                            :len(constructor.type.args)-constructor.type.optional_arg_count]:
935                        arg_decls.append(arg.declaration_code())
936                        arg_names.append(arg.cname)
937                    if constructor.type.optional_arg_count:
938                        arg_decls.append(constructor.type.op_arg_struct.declaration_code(Naming.optional_args_cname))
939                        arg_names.append(Naming.optional_args_cname)
940                    if not arg_decls:
941                        arg_decls = ["void"]
942                else:
943                    arg_decls = ["void"]
944                    arg_names = []
945                if is_implementing:
946                  code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls)))
947                  if py_attrs:
948                      code.put_ensure_gil()
949                      for attr in py_attrs:
950                          code.put_init_var_to_py_none(attr, nanny=False);
951                  if constructor:
952                      code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names)))
953                  if py_attrs:
954                      code.put_release_ensured_gil()
955                  code.putln("}")
956                else:
957                  code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls)))
958            if destructor or py_attrs or has_virtual_methods:
959                if has_virtual_methods:
960                    code.put("virtual ")
961                if is_implementing:
962                  code.putln("~%s() {" % type.cname)
963                  if py_attrs:
964                      code.put_ensure_gil()
965                  if destructor:
966                      code.putln("%s();" % destructor.cname)
967                  if py_attrs:
968                      for attr in py_attrs:
969                          code.put_var_xdecref(attr, nanny=False);
970                      code.put_release_ensured_gil()
971                  code.putln("}")
972                else:
973                  code.putln("~%s();" % type.cname)
974            if py_attrs:
975                # Also need copy constructor and assignment operators.
976                if is_implementing:
977                  code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname))
978                  code.put_ensure_gil()
979                  for attr in scope.var_entries:
980                      if not attr.type.is_cfunction:
981                          code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname))
982                          code.put_var_incref(attr, nanny=False)
983                  code.put_release_ensured_gil()
984                  code.putln("}")
985                  code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname))
986                  code.putln("if (this != &__Pyx_other) {")
987                  code.put_ensure_gil()
988                  for attr in scope.var_entries:
989                      if not attr.type.is_cfunction:
990                          code.put_var_xdecref(attr, nanny=False);
991                          code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname))
992                          code.put_var_incref(attr, nanny=False)
993                  code.put_release_ensured_gil()
994                  code.putln("}")
995                  code.putln("return *this;")
996                  code.putln("}")
997                else:
998                  code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname))
999                  code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname))
1000            code.putln("};")
1001
1002    def generate_enum_definition(self, entry, code):
1003        code.mark_pos(entry.pos)
1004        type = entry.type
1005        name = entry.cname or entry.name or ""
1006        header, footer = self.sue_header_footer(type, "enum", name)
1007        code.putln(header)
1008        enum_values = entry.enum_values
1009        if not enum_values:
1010            error(entry.pos, "Empty enum definition not allowed outside a 'cdef extern from' block")
1011        else:
1012            last_entry = enum_values[-1]
1013            # this does not really generate code, just builds the result value
1014            for value_entry in enum_values:
1015                if value_entry.value_node is not None:
1016                    value_entry.value_node.generate_evaluation_code(code)
1017
1018            for value_entry in enum_values:
1019                if value_entry.value_node is None:
1020                    value_code = value_entry.cname
1021                else:
1022                    value_code = ("%s = %s" % (
1023                        value_entry.cname,
1024                        value_entry.value_node.result()))
1025                if value_entry is not last_entry:
1026                    value_code += ","
1027                code.putln(value_code)
1028        code.putln(footer)
1029        if entry.type.typedef_flag:
1030            # Not pre-declared.
1031            code.putln("typedef enum %s %s;" % (name, name))
1032
1033    def generate_typeobj_predeclaration(self, entry, code):
1034        code.putln("")
1035        name = entry.type.typeobj_cname
1036        if name:
1037            if entry.visibility == 'extern' and not entry.in_cinclude:
1038                code.putln("%s %s %s;" % (
1039                    Naming.extern_c_macro,
1040                    PyrexTypes.public_decl("PyTypeObject", "DL_IMPORT"),
1041                    name))
1042            elif entry.visibility == 'public':
1043                code.putln("%s %s %s;" % (
1044                    Naming.extern_c_macro,
1045                    PyrexTypes.public_decl("PyTypeObject", "DL_EXPORT"),
1046                    name))
1047            # ??? Do we really need the rest of this? ???
1048            #else:
1049            #    code.putln("static PyTypeObject %s;" % name)
1050
1051    def generate_exttype_vtable_struct(self, entry, code):
1052        if not entry.used:
1053            return
1054
1055        code.mark_pos(entry.pos)
1056        # Generate struct declaration for an extension type's vtable.
1057        type = entry.type
1058        scope = type.scope
1059
1060        self.specialize_fused_types(scope)
1061
1062        if type.vtabstruct_cname:
1063            code.putln("")
1064            code.putln("struct %s {" % type.vtabstruct_cname)
1065            if type.base_type and type.base_type.vtabstruct_cname:
1066                code.putln("struct %s %s;" % (
1067                    type.base_type.vtabstruct_cname,
1068                    Naming.obj_base_cname))
1069            for method_entry in scope.cfunc_entries:
1070                if not method_entry.is_inherited:
1071                    code.putln("%s;" % method_entry.type.declaration_code("(*%s)" % method_entry.cname))
1072            code.putln("};")
1073
1074    def generate_exttype_vtabptr_declaration(self, entry, code):
1075        if not entry.used:
1076            return
1077
1078        code.mark_pos(entry.pos)
1079        # Generate declaration of pointer to an extension type's vtable.
1080        type = entry.type
1081        if type.vtabptr_cname:
1082            code.putln("static struct %s *%s;" % (
1083                type.vtabstruct_cname,
1084                type.vtabptr_cname))
1085
1086    def generate_exttype_final_methods_declaration(self, entry, code):
1087        if not entry.used:
1088            return
1089
1090        code.mark_pos(entry.pos)
1091        # Generate final methods prototypes
1092        type = entry.type
1093        for method_entry in entry.type.scope.cfunc_entries:
1094            if not method_entry.is_inherited and method_entry.final_func_cname:
1095                declaration = method_entry.type.declaration_code(
1096                    method_entry.final_func_cname)
1097                modifiers = code.build_function_modifiers(method_entry.func_modifiers)
1098                code.putln("static %s%s;" % (modifiers, declaration))
1099
1100    def generate_objstruct_predeclaration(self, type, code):
1101        if not type.scope:
1102            return
1103        code.putln(self.sue_predeclaration(type, "struct", type.objstruct_cname))
1104
1105    def generate_objstruct_definition(self, type, code):
1106        code.mark_pos(type.pos)
1107        # Generate object struct definition for an
1108        # extension type.
1109        if not type.scope:
1110            return # Forward declared but never defined
1111        header, footer = \
1112            self.sue_header_footer(type, "struct", type.objstruct_cname)
1113        code.putln(header)
1114        base_type = type.base_type
1115        if base_type:
1116            basestruct_cname = base_type.objstruct_cname
1117            if basestruct_cname == "PyTypeObject":
1118                # User-defined subclasses of type are heap allocated.
1119                basestruct_cname = "PyHeapTypeObject"
1120            code.putln(
1121                "%s%s %s;" % (
1122                    ("struct ", "")[base_type.typedef_flag],
1123                    basestruct_cname,
1124                    Naming.obj_base_cname))
1125        else:
1126            code.putln(
1127                "PyObject_HEAD")
1128        if type.vtabslot_cname and not (type.base_type and type.base_type.vtabslot_cname):
1129            code.putln(
1130                "struct %s *%s;" % (
1131                    type.vtabstruct_cname,
1132                    type.vtabslot_cname))
1133        for attr in type.scope.var_entries:
1134            if attr.is_declared_generic:
1135                attr_type = py_object_type
1136            else:
1137                attr_type = attr.type
1138            code.putln(
1139                "%s;" % attr_type.declaration_code(attr.cname))
1140        code.putln(footer)
1141        if type.objtypedef_cname is not None:
1142            # Only for exposing public typedef name.
1143            code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname))
1144
1145    def generate_c_class_declarations(self, env, code, definition):
1146        for entry in env.c_class_entries:
1147            if definition or entry.defined_in_pxd:
1148                code.putln("static PyTypeObject *%s = 0;" % (
1149                    entry.type.typeptr_cname))
1150
1151    def generate_cvariable_declarations(self, env, code, definition):
1152        if env.is_cython_builtin:
1153            return
1154        for entry in env.var_entries:
1155            if (entry.in_cinclude or entry.in_closure or
1156                    (entry.visibility == 'private' and not (entry.defined_in_pxd or entry.used))):
1157                continue
1158
1159            storage_class = None
1160            dll_linkage = None
1161            init = None
1162
1163            if entry.visibility == 'extern':
1164                storage_class = Naming.extern_c_macro
1165                dll_linkage = "DL_IMPORT"
1166            elif entry.visibility == 'public':
1167                storage_class = Naming.extern_c_macro
1168                if definition:
1169                    dll_linkage = "DL_EXPORT"
1170                else:
1171                    dll_linkage = "DL_IMPORT"
1172            elif entry.visibility == 'private':
1173                storage_class = "static"
1174                dll_linkage = None
1175                if entry.init is not None:
1176                    init = entry.type.literal_code(entry.init)
1177            type = entry.type
1178            cname = entry.cname
1179
1180            if entry.defined_in_pxd and not definition:
1181                storage_class = "static"
1182                dll_linkage = None
1183                type = CPtrType(type)
1184                cname = env.mangle(Naming.varptr_prefix, entry.name)
1185                init = 0
1186
1187            if storage_class:
1188                code.put("%s " % storage_class)
1189            code.put(type.declaration_code(
1190                cname, dll_linkage=dll_linkage))
1191            if init is not None:
1192                code.put_safe(" = %s" % init)
1193            code.putln(";")
1194            if entry.cname != cname:
1195                code.putln("#define %s (*%s)" % (entry.cname, cname))
1196
1197    def generate_cfunction_declarations(self, env, code, definition):
1198        for entry in env.cfunc_entries:
1199            if entry.used or (entry.visibility == 'public' or entry.api):
1200                generate_cfunction_declaration(entry, env, code, definition)
1201
1202    def generate_variable_definitions(self, env, code):
1203        for entry in env.var_entries:
1204            if not entry.in_cinclude and entry.visibility == "public":
1205                code.put(entry.type.declaration_code(entry.cname))
1206                if entry.init is not None:
1207                    init = entry.type.literal_code(entry.init)
1208                    code.put_safe(" = %s" % init)
1209                code.putln(";")
1210
1211    def generate_typeobj_definitions(self, env, code):
1212        full_module_name = env.qualified_name
1213        for entry in env.c_class_entries:
1214            #print "generate_typeobj_definitions:", entry.name
1215            #print "...visibility =", entry.visibility
1216            if entry.visibility != 'extern':
1217                type = entry.type
1218                scope = type.scope
1219                if scope: # could be None if there was an error
1220                    self.generate_exttype_vtable(scope, code)
1221                    self.generate_new_function(scope, code, entry)
1222                    self.generate_dealloc_function(scope, code)
1223                    if scope.needs_gc():
1224                        self.generate_traverse_function(scope, code, entry)
1225                        if scope.needs_tp_clear():
1226                            self.generate_clear_function(scope, code, entry)
1227                    if scope.defines_any_special(["__getitem__"]):
1228                        self.generate_getitem_int_function(scope, code)
1229                    if scope.defines_any_special(["__setitem__", "__delitem__"]):
1230                        self.generate_ass_subscript_function(scope, code)
1231                    if scope.defines_any_special(["__getslice__", "__setslice__", "__delslice__"]):
1232                        warning(self.pos,
1233                                "__getslice__, __setslice__, and __delslice__ are not supported by Python 3, "
1234                                "use __getitem__, __setitem__, and __delitem__ instead", 1)
1235                        code.putln("#if PY_MAJOR_VERSION >= 3")
1236                        code.putln("#error __getslice__, __setslice__, and __delslice__ not supported in Python 3.")
1237                        code.putln("#endif")
1238                    if scope.defines_any_special(["__setslice__", "__delslice__"]):
1239                        self.generate_ass_slice_function(scope, code)
1240                    if scope.defines_any_special(["__getattr__", "__getattribute__"]):
1241                        self.generate_getattro_function(scope, code)
1242                    if scope.defines_any_special(["__setattr__", "__delattr__"]):
1243                        self.generate_setattro_function(scope, code)
1244                    if scope.defines_any_special(["__get__"]):
1245                        self.generate_descr_get_function(scope, code)
1246                    if scope.defines_any_special(["__set__", "__delete__"]):
1247                        self.generate_descr_set_function(scope, code)
1248                    if not scope.is_closure_class_scope and scope.defines_any(["__dict__"]):
1249                        self.generate_dict_getter_function(scope, code)
1250                    if scope.defines_any_special(TypeSlots.richcmp_special_methods):
1251                        self.generate_richcmp_function(scope, code)
1252                    self.generate_property_accessors(scope, code)
1253                    self.generate_method_table(scope, code)
1254                    self.generate_getset_table(scope, code)
1255                    self.generate_typeobj_definition(full_module_name, entry, code)
1256
1257    def generate_exttype_vtable(self, scope, code):
1258        # Generate the definition of an extension type's vtable.
1259        type = scope.parent_type
1260        if type.vtable_cname:
1261            code.putln("static struct %s %s;" % (
1262                type.vtabstruct_cname,
1263                type.vtable_cname))
1264
1265    def generate_self_cast(self, scope, code):
1266        type = scope.parent_type
1267        code.putln(
1268            "%s = (%s)o;" % (
1269                type.declaration_code("p"),
1270                type.empty_declaration_code()))
1271
1272    def generate_new_function(self, scope, code, cclass_entry):
1273        tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1274        slot_func = scope.mangle_internal("tp_new")
1275        type = scope.parent_type
1276        base_type = type.base_type
1277
1278        have_entries, (py_attrs, py_buffers, memoryview_slices) = \
1279                        scope.get_refcounted_entries()
1280        is_final_type = scope.parent_type.is_final_type
1281        if scope.is_internal:
1282            # internal classes (should) never need None inits, normal zeroing will do
1283            py_attrs = []
1284        cpp_class_attrs = [entry for entry in scope.var_entries
1285                           if entry.type.is_cpp_class]
1286
1287        new_func_entry = scope.lookup_here("__new__")
1288        if base_type or (new_func_entry and new_func_entry.is_special
1289                         and not new_func_entry.trivial_signature):
1290            unused_marker = ''
1291        else:
1292            unused_marker = 'CYTHON_UNUSED '
1293
1294        if base_type:
1295            freelist_size = 0  # not currently supported
1296        else:
1297            freelist_size = scope.directives.get('freelist', 0)
1298        freelist_name = scope.mangle_internal(Naming.freelist_name)
1299        freecount_name = scope.mangle_internal(Naming.freecount_name)
1300
1301        decls = code.globalstate['decls']
1302        decls.putln("static PyObject *%s(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/" %
1303                    slot_func)
1304        code.putln("")
1305        if freelist_size:
1306            code.putln("static %s[%d];" % (
1307                scope.parent_type.declaration_code(freelist_name),
1308                freelist_size))
1309            code.putln("static int %s = 0;" % freecount_name)
1310            code.putln("")
1311        code.putln(
1312            "static PyObject *%s(PyTypeObject *t, %sPyObject *a, %sPyObject *k) {" % (
1313                slot_func, unused_marker, unused_marker))
1314
1315        need_self_cast = (type.vtabslot_cname or
1316                          (py_buffers or memoryview_slices or py_attrs) or
1317                          cpp_class_attrs)
1318        if need_self_cast:
1319            code.putln("%s;" % scope.parent_type.declaration_code("p"))
1320        if base_type:
1321            tp_new = TypeSlots.get_base_slot_function(scope, tp_slot)
1322            if tp_new is None:
1323                tp_new = "%s->tp_new" % base_type.typeptr_cname
1324            code.putln("PyObject *o = %s(t, a, k);" % tp_new)
1325        else:
1326            code.putln("PyObject *o;")
1327            if freelist_size:
1328                code.globalstate.use_utility_code(
1329                    UtilityCode.load_cached("IncludeStringH", "StringTools.c"))
1330                if is_final_type:
1331                    type_safety_check = ''
1332                else:
1333                    type_safety_check = ' & ((t->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)'
1334                obj_struct = type.declaration_code("", deref=True)
1335                code.putln(
1336                    "if (CYTHON_COMPILING_IN_CPYTHON && likely((%s > 0) & (t->tp_basicsize == sizeof(%s))%s)) {" % (
1337                        freecount_name, obj_struct, type_safety_check))
1338                code.putln("o = (PyObject*)%s[--%s];" % (
1339                    freelist_name, freecount_name))
1340                code.putln("memset(o, 0, sizeof(%s));" % obj_struct)
1341                code.putln("(void) PyObject_INIT(o, t);")
1342                if scope.needs_gc():
1343                    code.putln("PyObject_GC_Track(o);")
1344                code.putln("} else {")
1345            if not is_final_type:
1346                code.putln("if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {")
1347            code.putln("o = (*t->tp_alloc)(t, 0);")
1348            if not is_final_type:
1349                code.putln("} else {")
1350                code.putln("o = (PyObject *) PyBaseObject_Type.tp_new(t, %s, 0);" % Naming.empty_tuple)
1351                code.putln("}")
1352        code.putln("if (unlikely(!o)) return 0;")
1353        if freelist_size and not base_type:
1354            code.putln('}')
1355        if need_self_cast:
1356            code.putln("p = %s;" % type.cast_code("o"))
1357        #if need_self_cast:
1358        #    self.generate_self_cast(scope, code)
1359
1360        # from this point on, ensure DECREF(o) on failure
1361        needs_error_cleanup = False
1362
1363        if type.vtabslot_cname:
1364            vtab_base_type = type
1365            while vtab_base_type.base_type and vtab_base_type.base_type.vtabstruct_cname:
1366                vtab_base_type = vtab_base_type.base_type
1367            if vtab_base_type is not type:
1368                struct_type_cast = "(struct %s*)" % vtab_base_type.vtabstruct_cname
1369            else:
1370                struct_type_cast = ""
1371            code.putln("p->%s = %s%s;" % (
1372                type.vtabslot_cname,
1373                struct_type_cast, type.vtabptr_cname))
1374
1375        for entry in cpp_class_attrs:
1376            code.putln("new((void*)&(p->%s)) %s();" % (
1377                entry.cname, entry.type.empty_declaration_code()))
1378
1379        for entry in py_attrs:
1380            if entry.name == "__dict__":
1381                needs_error_cleanup = True
1382                code.put("p->%s = PyDict_New(); if (unlikely(!p->%s)) goto bad;" % (
1383                    entry.cname, entry.cname))
1384            else:
1385                code.put_init_var_to_py_none(entry, "p->%s", nanny=False)
1386
1387        for entry in memoryview_slices:
1388            code.putln("p->%s.data = NULL;" % entry.cname)
1389            code.putln("p->%s.memview = NULL;" % entry.cname)
1390
1391        for entry in py_buffers:
1392            code.putln("p->%s.obj = NULL;" % entry.cname)
1393
1394        if cclass_entry.cname == '__pyx_memoryviewslice':
1395            code.putln("p->from_slice.memview = NULL;")
1396
1397        if new_func_entry and new_func_entry.is_special:
1398            if new_func_entry.trivial_signature:
1399                cinit_args = "o, %s, NULL" % Naming.empty_tuple
1400            else:
1401                cinit_args = "o, a, k"
1402            needs_error_cleanup = True
1403            code.putln("if (unlikely(%s(%s) < 0)) goto bad;" % (
1404                new_func_entry.func_cname, cinit_args))
1405
1406        code.putln(
1407            "return o;")
1408        if needs_error_cleanup:
1409            code.putln("bad:")
1410            code.put_decref_clear("o", py_object_type, nanny=False)
1411            code.putln("return NULL;")
1412        code.putln(
1413            "}")
1414
1415    def generate_dealloc_function(self, scope, code):
1416        tp_slot = TypeSlots.ConstructorSlot("tp_dealloc", '__dealloc__')
1417        slot_func = scope.mangle_internal("tp_dealloc")
1418        base_type = scope.parent_type.base_type
1419        if tp_slot.slot_code(scope) != slot_func:
1420            return  # never used
1421
1422        slot_func_cname = scope.mangle_internal("tp_dealloc")
1423        code.putln("")
1424        code.putln(
1425            "static void %s(PyObject *o) {" % slot_func_cname)
1426
1427        is_final_type = scope.parent_type.is_final_type
1428        needs_gc = scope.needs_gc()
1429
1430        weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
1431        if weakref_slot not in scope.var_entries:
1432            weakref_slot = None
1433
1434        dict_slot = scope.lookup_here("__dict__") if not scope.is_closure_class_scope else None
1435        if dict_slot not in scope.var_entries:
1436            dict_slot = None
1437
1438        _, (py_attrs, _, memoryview_slices) = scope.get_refcounted_entries()
1439        cpp_class_attrs = [entry for entry in scope.var_entries
1440                           if entry.type.is_cpp_class]
1441
1442        if py_attrs or cpp_class_attrs or memoryview_slices or weakref_slot or dict_slot:
1443            self.generate_self_cast(scope, code)
1444
1445        if not is_final_type:
1446            # in Py3.4+, call tp_finalize() as early as possible
1447            code.putln("#if CYTHON_USE_TP_FINALIZE")
1448            if needs_gc:
1449                finalised_check = '!_PyGC_FINALIZED(o)'
1450            else:
1451                finalised_check = (
1452                    '(!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))')
1453            code.putln(
1454                "if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE)"
1455                " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check)
1456            # if instance was resurrected by finaliser, return
1457            code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;")
1458            code.putln("}")
1459            code.putln("#endif")
1460
1461        if needs_gc:
1462            # We must mark this object as (gc) untracked while tearing
1463            # it down, lest the garbage collection is invoked while
1464            # running this destructor.
1465            code.putln("PyObject_GC_UnTrack(o);")
1466
1467        # call the user's __dealloc__
1468        self.generate_usr_dealloc_call(scope, code)
1469
1470        if weakref_slot:
1471            code.putln("if (p->__weakref__) PyObject_ClearWeakRefs(o);")
1472
1473        if dict_slot:
1474            code.putln("if (p->__dict__) PyDict_Clear(p->__dict__);")
1475
1476        for entry in cpp_class_attrs:
1477            code.putln("__Pyx_call_destructor(p->%s);" % entry.cname)
1478
1479        for entry in py_attrs:
1480            code.put_xdecref_clear("p->%s" % entry.cname, entry.type, nanny=False,
1481                                   clear_before_decref=True)
1482
1483        for entry in memoryview_slices:
1484            code.put_xdecref_memoryviewslice("p->%s" % entry.cname,
1485                                             have_gil=True)
1486
1487        if base_type:
1488            if needs_gc:
1489                # The base class deallocator probably expects this to be tracked,
1490                # so undo the untracking above.
1491                if base_type.scope and base_type.scope.needs_gc():
1492                    code.putln("PyObject_GC_Track(o);")
1493                else:
1494                    code.putln("#if CYTHON_USE_TYPE_SLOTS")
1495                    code.putln("if (PyType_IS_GC(Py_TYPE(o)->tp_base))")
1496                    code.putln("#endif")
1497                    code.putln("PyObject_GC_Track(o);")
1498
1499            tp_dealloc = TypeSlots.get_base_slot_function(scope, tp_slot)
1500            if tp_dealloc is not None:
1501                code.putln("%s(o);" % tp_dealloc)
1502            elif base_type.is_builtin_type:
1503                code.putln("%s->tp_dealloc(o);" % base_type.typeptr_cname)
1504            else:
1505                # This is an externally defined type.  Calling through the
1506                # cimported base type pointer directly interacts badly with
1507                # the module cleanup, which may already have cleared it.
1508                # In that case, fall back to traversing the type hierarchy.
1509                base_cname = base_type.typeptr_cname
1510                code.putln("if (likely(%s)) %s->tp_dealloc(o); "
1511                           "else __Pyx_call_next_tp_dealloc(o, %s);" % (
1512                               base_cname, base_cname, slot_func_cname))
1513                code.globalstate.use_utility_code(
1514                    UtilityCode.load_cached("CallNextTpDealloc", "ExtensionTypes.c"))
1515        else:
1516            freelist_size = scope.directives.get('freelist', 0)
1517            if freelist_size:
1518                freelist_name = scope.mangle_internal(Naming.freelist_name)
1519                freecount_name = scope.mangle_internal(Naming.freecount_name)
1520
1521                if is_final_type:
1522                    type_safety_check = ''
1523                else:
1524                    type_safety_check = (
1525                        ' & ((Py_TYPE(o)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)')
1526
1527                type = scope.parent_type
1528                code.putln(
1529                    "if (CYTHON_COMPILING_IN_CPYTHON && ((%s < %d) & (Py_TYPE(o)->tp_basicsize == sizeof(%s))%s)) {" % (
1530                        freecount_name,
1531                        freelist_size,
1532                        type.declaration_code("", deref=True),
1533                        type_safety_check))
1534                code.putln("%s[%s++] = %s;" % (
1535                    freelist_name, freecount_name, type.cast_code("o")))
1536                code.putln("} else {")
1537            code.putln("(*Py_TYPE(o)->tp_free)(o);")
1538            if freelist_size:
1539                code.putln("}")
1540        code.putln(
1541            "}")
1542
1543    def generate_usr_dealloc_call(self, scope, code):
1544        entry = scope.lookup_here("__dealloc__")
1545        if not entry:
1546            return
1547
1548        code.putln("{")
1549        code.putln("PyObject *etype, *eval, *etb;")
1550        code.putln("PyErr_Fetch(&etype, &eval, &etb);")
1551        code.putln("++Py_REFCNT(o);")
1552        code.putln("%s(o);" % entry.func_cname)
1553        code.putln("--Py_REFCNT(o);")
1554        code.putln("PyErr_Restore(etype, eval, etb);")
1555        code.putln("}")
1556
1557    def generate_traverse_function(self, scope, code, cclass_entry):
1558        tp_slot = TypeSlots.GCDependentSlot("tp_traverse")
1559        slot_func = scope.mangle_internal("tp_traverse")
1560        base_type = scope.parent_type.base_type
1561        if tp_slot.slot_code(scope) != slot_func:
1562            return  # never used
1563        code.putln("")
1564        code.putln(
1565            "static int %s(PyObject *o, visitproc v, void *a) {" % slot_func)
1566
1567        have_entries, (py_attrs, py_buffers, memoryview_slices) = (
1568            scope.get_refcounted_entries(include_gc_simple=False))
1569
1570        if base_type or py_attrs:
1571            code.putln("int e;")
1572
1573        if py_attrs or py_buffers:
1574            self.generate_self_cast(scope, code)
1575
1576        if base_type:
1577            # want to call it explicitly if possible so inlining can be performed
1578            static_call = TypeSlots.get_base_slot_function(scope, tp_slot)
1579            if static_call:
1580                code.putln("e = %s(o, v, a); if (e) return e;" % static_call)
1581            elif base_type.is_builtin_type:
1582                base_cname = base_type.typeptr_cname
1583                code.putln("if (!%s->tp_traverse); else { e = %s->tp_traverse(o,v,a); if (e) return e; }" % (
1584                    base_cname, base_cname))
1585            else:
1586                # This is an externally defined type.  Calling through the
1587                # cimported base type pointer directly interacts badly with
1588                # the module cleanup, which may already have cleared it.
1589                # In that case, fall back to traversing the type hierarchy.
1590                base_cname = base_type.typeptr_cname
1591                code.putln(
1592                    "e = ((likely(%s)) ? ((%s->tp_traverse) ? %s->tp_traverse(o, v, a) : 0) : "
1593                    "__Pyx_call_next_tp_traverse(o, v, a, %s)); if (e) return e;" % (
1594                        base_cname, base_cname, base_cname, slot_func))
1595                code.globalstate.use_utility_code(
1596                    UtilityCode.load_cached("CallNextTpTraverse", "ExtensionTypes.c"))
1597
1598        for entry in py_attrs:
1599            var_code = "p->%s" % entry.cname
1600            var_as_pyobject = PyrexTypes.typecast(py_object_type, entry.type, var_code)
1601            code.putln("if (%s) {" % var_code)
1602            code.putln("e = (*v)(%s, a); if (e) return e;" % var_as_pyobject)
1603            code.putln("}")
1604
1605        # Traverse buffer exporting objects.
1606        # Note: not traversing memoryview attributes of memoryview slices!
1607        # When triggered by the GC, it would cause multiple visits (gc_refs
1608        # subtractions which is not matched by its reference count!)
1609        for entry in py_buffers:
1610            cname = entry.cname + ".obj"
1611            code.putln("if (p->%s) {" % cname)
1612            code.putln("e = (*v)(p->%s, a); if (e) return e;" % cname)
1613            code.putln("}")
1614
1615        code.putln("return 0;")
1616        code.putln("}")
1617
1618    def generate_clear_function(self, scope, code, cclass_entry):
1619        tp_slot = TypeSlots.get_slot_by_name("tp_clear")
1620        slot_func = scope.mangle_internal("tp_clear")
1621        base_type = scope.parent_type.base_type
1622        if tp_slot.slot_code(scope) != slot_func:
1623            return # never used
1624
1625        have_entries, (py_attrs, py_buffers, memoryview_slices) = (
1626            scope.get_refcounted_entries(include_gc_simple=False))
1627
1628        if py_attrs or py_buffers or base_type:
1629            unused = ''
1630        else:
1631            unused = 'CYTHON_UNUSED '
1632
1633        code.putln("")
1634        code.putln("static int %s(%sPyObject *o) {" % (slot_func, unused))
1635
1636        if py_attrs and Options.clear_to_none:
1637            code.putln("PyObject* tmp;")
1638
1639        if py_attrs or py_buffers:
1640            self.generate_self_cast(scope, code)
1641
1642        if base_type:
1643            # want to call it explicitly if possible so inlining can be performed
1644            static_call = TypeSlots.get_base_slot_function(scope, tp_slot)
1645            if static_call:
1646                code.putln("%s(o);" % static_call)
1647            elif base_type.is_builtin_type:
1648                base_cname = base_type.typeptr_cname
1649                code.putln("if (!%s->tp_clear); else %s->tp_clear(o);" % (
1650                    base_cname, base_cname))
1651            else:
1652                # This is an externally defined type.  Calling through the
1653                # cimported base type pointer directly interacts badly with
1654                # the module cleanup, which may already have cleared it.
1655                # In that case, fall back to traversing the type hierarchy.
1656                base_cname = base_type.typeptr_cname
1657                code.putln(
1658                    "if (likely(%s)) { if (%s->tp_clear) %s->tp_clear(o); } else __Pyx_call_next_tp_clear(o, %s);" % (
1659                        base_cname, base_cname, base_cname, slot_func))
1660                code.globalstate.use_utility_code(
1661                    UtilityCode.load_cached("CallNextTpClear", "ExtensionTypes.c"))
1662
1663        if Options.clear_to_none:
1664            for entry in py_attrs:
1665                name = "p->%s" % entry.cname
1666                code.putln("tmp = ((PyObject*)%s);" % name)
1667                if entry.is_declared_generic:
1668                    code.put_init_to_py_none(name, py_object_type, nanny=False)
1669                else:
1670                    code.put_init_to_py_none(name, entry.type, nanny=False)
1671                code.putln("Py_XDECREF(tmp);")
1672        else:
1673            for entry in py_attrs:
1674                code.putln("Py_CLEAR(p->%s);" % entry.cname)
1675
1676        for entry in py_buffers:
1677            # Note: shouldn't this call __Pyx_ReleaseBuffer ??
1678            code.putln("Py_CLEAR(p->%s.obj);" % entry.cname)
1679
1680        if cclass_entry.cname == '__pyx_memoryviewslice':
1681            code.putln("__PYX_XDEC_MEMVIEW(&p->from_slice, 1);")
1682
1683        code.putln("return 0;")
1684        code.putln("}")
1685
1686    def generate_getitem_int_function(self, scope, code):
1687        # This function is put into the sq_item slot when
1688        # a __getitem__ method is present. It converts its
1689        # argument to a Python integer and calls mp_subscript.
1690        code.putln(
1691            "static PyObject *%s(PyObject *o, Py_ssize_t i) {" % (
1692                scope.mangle_internal("sq_item")))
1693        code.putln(
1694            "PyObject *r;")
1695        code.putln(
1696            "PyObject *x = PyInt_FromSsize_t(i); if(!x) return 0;")
1697        code.putln(
1698            "r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x);")
1699        code.putln(
1700            "Py_DECREF(x);")
1701        code.putln(
1702            "return r;")
1703        code.putln(
1704            "}")
1705
1706    def generate_ass_subscript_function(self, scope, code):
1707        # Setting and deleting an item are both done through
1708        # the ass_subscript method, so we dispatch to user's __setitem__
1709        # or __delitem__, or raise an exception.
1710        base_type = scope.parent_type.base_type
1711        set_entry = scope.lookup_here("__setitem__")
1712        del_entry = scope.lookup_here("__delitem__")
1713        code.putln("")
1714        code.putln(
1715            "static int %s(PyObject *o, PyObject *i, PyObject *v) {" % (
1716                scope.mangle_internal("mp_ass_subscript")))
1717        code.putln(
1718            "if (v) {")
1719        if set_entry:
1720            code.putln("return %s(o, i, v);" % set_entry.func_cname)
1721        else:
1722            self.generate_guarded_basetype_call(
1723                base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code)
1724            code.putln(
1725                "PyErr_Format(PyExc_NotImplementedError,")
1726            code.putln(
1727                '  "Subscript assignment not supported by %.200s", Py_TYPE(o)->tp_name);')
1728            code.putln(
1729                "return -1;")
1730        code.putln(
1731            "}")
1732        code.putln(
1733            "else {")
1734        if del_entry:
1735            code.putln(
1736                "return %s(o, i);" % (
1737                    del_entry.func_cname))
1738        else:
1739            self.generate_guarded_basetype_call(
1740                base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code)
1741            code.putln(
1742                "PyErr_Format(PyExc_NotImplementedError,")
1743            code.putln(
1744                '  "Subscript deletion not supported by %.200s", Py_TYPE(o)->tp_name);')
1745            code.putln(
1746                "return -1;")
1747        code.putln(
1748            "}")
1749        code.putln(
1750            "}")
1751
1752    def generate_guarded_basetype_call(
1753            self, base_type, substructure, slot, args, code):
1754        if base_type:
1755            base_tpname = base_type.typeptr_cname
1756            if substructure:
1757                code.putln(
1758                    "if (%s->%s && %s->%s->%s)" % (
1759                        base_tpname, substructure, base_tpname, substructure, slot))
1760                code.putln(
1761                    "  return %s->%s->%s(%s);" % (
1762                        base_tpname, substructure, slot, args))
1763            else:
1764                code.putln(
1765                    "if (%s->%s)" % (
1766                        base_tpname, slot))
1767                code.putln(
1768                    "  return %s->%s(%s);" % (
1769                        base_tpname, slot, args))
1770
1771    def generate_ass_slice_function(self, scope, code):
1772        # Setting and deleting a slice are both done through
1773        # the ass_slice method, so we dispatch to user's __setslice__
1774        # or __delslice__, or raise an exception.
1775        base_type = scope.parent_type.base_type
1776        set_entry = scope.lookup_here("__setslice__")
1777        del_entry = scope.lookup_here("__delslice__")
1778        code.putln("")
1779        code.putln(
1780            "static int %s(PyObject *o, Py_ssize_t i, Py_ssize_t j, PyObject *v) {" % (
1781                scope.mangle_internal("sq_ass_slice")))
1782        code.putln(
1783            "if (v) {")
1784        if set_entry:
1785            code.putln(
1786                "return %s(o, i, j, v);" % (
1787                    set_entry.func_cname))
1788        else:
1789            self.generate_guarded_basetype_call(
1790                base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code)
1791            code.putln(
1792                "PyErr_Format(PyExc_NotImplementedError,")
1793            code.putln(
1794                '  "2-element slice assignment not supported by %.200s", Py_TYPE(o)->tp_name);')
1795            code.putln(
1796                "return -1;")
1797        code.putln(
1798            "}")
1799        code.putln(
1800            "else {")
1801        if del_entry:
1802            code.putln(
1803                "return %s(o, i, j);" % (
1804                    del_entry.func_cname))
1805        else:
1806            self.generate_guarded_basetype_call(
1807                base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code)
1808            code.putln(
1809                "PyErr_Format(PyExc_NotImplementedError,")
1810            code.putln(
1811                '  "2-element slice deletion not supported by %.200s", Py_TYPE(o)->tp_name);')
1812            code.putln(
1813                "return -1;")
1814        code.putln(
1815            "}")
1816        code.putln(
1817            "}")
1818
1819    def generate_richcmp_function(self, scope, code):
1820        if scope.lookup_here("__richcmp__"):
1821            # user implemented, nothing to do
1822            return
1823        # otherwise, we have to generate it from the Python special methods
1824        richcmp_cfunc = scope.mangle_internal("tp_richcompare")
1825        code.putln("")
1826        code.putln("static PyObject *%s(PyObject *o1, PyObject *o2, int op) {" % richcmp_cfunc)
1827        code.putln("switch (op) {")
1828
1829        class_scopes = []
1830        cls = scope.parent_type
1831        while cls is not None and not cls.entry.visibility == 'extern':
1832            class_scopes.append(cls.scope)
1833            cls = cls.scope.parent_type.base_type
1834        assert scope in class_scopes
1835
1836        extern_parent = None
1837        if cls and cls.entry.visibility == 'extern':
1838            # need to call up into base classes as we may not know all implemented comparison methods
1839            extern_parent = cls if cls.typeptr_cname else scope.parent_type.base_type
1840
1841        eq_entry = None
1842        has_ne = False
1843        for cmp_method in TypeSlots.richcmp_special_methods:
1844            for class_scope in class_scopes:
1845                entry = class_scope.lookup_here(cmp_method)
1846                if entry is not None:
1847                    break
1848            else:
1849                continue
1850
1851            cmp_type = cmp_method.strip('_').upper()  # e.g. "__eq__" -> EQ
1852            code.putln("case Py_%s: {" % cmp_type)
1853            if cmp_method == '__eq__':
1854                eq_entry = entry
1855                # Python itself does not do this optimisation, it seems...
1856                #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_True);")
1857            elif cmp_method == '__ne__':
1858                has_ne = True
1859                # Python itself does not do this optimisation, it seems...
1860                #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);")
1861            code.putln("return %s(o1, o2);" % entry.func_cname)
1862            code.putln("}")
1863
1864        if eq_entry and not has_ne and not extern_parent:
1865            code.putln("case Py_NE: {")
1866            code.putln("PyObject *ret;")
1867            # Python itself does not do this optimisation, it seems...
1868            #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);")
1869            code.putln("ret = %s(o1, o2);" % eq_entry.func_cname)
1870            code.putln("if (likely(ret && ret != Py_NotImplemented)) {")
1871            code.putln("int b = __Pyx_PyObject_IsTrue(ret); Py_DECREF(ret);")
1872            code.putln("if (unlikely(b < 0)) return NULL;")
1873            code.putln("ret = (b) ? Py_False : Py_True;")
1874            code.putln("Py_INCREF(ret);")
1875            code.putln("}")
1876            code.putln("return ret;")
1877            code.putln("}")
1878
1879        code.putln("default: {")
1880        if extern_parent and extern_parent.typeptr_cname:
1881            code.putln("if (likely(%s->tp_richcompare)) return %s->tp_richcompare(o1, o2, op);" % (
1882                extern_parent.typeptr_cname, extern_parent.typeptr_cname))
1883        code.putln("return __Pyx_NewRef(Py_NotImplemented);")
1884        code.putln("}")
1885
1886        code.putln("}")  # switch
1887        code.putln("}")
1888
1889    def generate_getattro_function(self, scope, code):
1890        # First try to get the attribute using __getattribute__, if defined, or
1891        # PyObject_GenericGetAttr.
1892        #
1893        # If that raises an AttributeError, call the __getattr__ if defined.
1894        #
1895        # In both cases, defined can be in this class, or any base class.
1896        def lookup_here_or_base(n, tp=None, extern_return=None):
1897            # Recursive lookup
1898            if tp is None:
1899                tp = scope.parent_type
1900            r = tp.scope.lookup_here(n)
1901            if r is None:
1902                if tp.is_external and extern_return is not None:
1903                    return extern_return
1904                if tp.base_type is not None:
1905                    return lookup_here_or_base(n, tp.base_type)
1906            return r
1907
1908        has_instance_dict = lookup_here_or_base("__dict__", extern_return="extern")
1909        getattr_entry = lookup_here_or_base("__getattr__")
1910        getattribute_entry = lookup_here_or_base("__getattribute__")
1911        code.putln("")
1912        code.putln(
1913            "static PyObject *%s(PyObject *o, PyObject *n) {" % (
1914                scope.mangle_internal("tp_getattro")))
1915        if getattribute_entry is not None:
1916            code.putln(
1917                "PyObject *v = %s(o, n);" % (
1918                    getattribute_entry.func_cname))
1919        else:
1920            if not has_instance_dict and scope.parent_type.is_final_type:
1921                # Final with no dict => use faster type attribute lookup.
1922                code.globalstate.use_utility_code(
1923                    UtilityCode.load_cached("PyObject_GenericGetAttrNoDict", "ObjectHandling.c"))
1924                generic_getattr_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict"
1925            elif not has_instance_dict or has_instance_dict == "extern":
1926                # No dict in the known ancestors, but don't know about extern ancestors or subtypes.
1927                code.globalstate.use_utility_code(
1928                    UtilityCode.load_cached("PyObject_GenericGetAttr", "ObjectHandling.c"))
1929                generic_getattr_cfunc = "__Pyx_PyObject_GenericGetAttr"
1930            else:
1931                generic_getattr_cfunc = "PyObject_GenericGetAttr"
1932            code.putln(
1933                "PyObject *v = %s(o, n);" % generic_getattr_cfunc)
1934        if getattr_entry is not None:
1935            code.putln(
1936                "if (!v && PyErr_ExceptionMatches(PyExc_AttributeError)) {")
1937            code.putln(
1938                "PyErr_Clear();")
1939            code.putln(
1940                "v = %s(o, n);" % (
1941                    getattr_entry.func_cname))
1942            code.putln(
1943                "}")
1944        code.putln(
1945            "return v;")
1946        code.putln(
1947            "}")
1948
1949    def generate_setattro_function(self, scope, code):
1950        # Setting and deleting an attribute are both done through
1951        # the setattro method, so we dispatch to user's __setattr__
1952        # or __delattr__ or fall back on PyObject_GenericSetAttr.
1953        base_type = scope.parent_type.base_type
1954        set_entry = scope.lookup_here("__setattr__")
1955        del_entry = scope.lookup_here("__delattr__")
1956        code.putln("")
1957        code.putln(
1958            "static int %s(PyObject *o, PyObject *n, PyObject *v) {" % (
1959                scope.mangle_internal("tp_setattro")))
1960        code.putln(
1961            "if (v) {")
1962        if set_entry:
1963            code.putln(
1964                "return %s(o, n, v);" % (
1965                    set_entry.func_cname))
1966        else:
1967            self.generate_guarded_basetype_call(
1968                base_type, None, "tp_setattro", "o, n, v", code)
1969            code.putln(
1970                "return PyObject_GenericSetAttr(o, n, v);")
1971        code.putln(
1972            "}")
1973        code.putln(
1974            "else {")
1975        if del_entry:
1976            code.putln(
1977                "return %s(o, n);" % (
1978                    del_entry.func_cname))
1979        else:
1980            self.generate_guarded_basetype_call(
1981                base_type, None, "tp_setattro", "o, n, v", code)
1982            code.putln(
1983                "return PyObject_GenericSetAttr(o, n, 0);")
1984        code.putln(
1985            "}")
1986        code.putln(
1987            "}")
1988
1989    def generate_descr_get_function(self, scope, code):
1990        # The __get__ function of a descriptor object can be
1991        # called with NULL for the second or third arguments
1992        # under some circumstances, so we replace them with
1993        # None in that case.
1994        user_get_entry = scope.lookup_here("__get__")
1995        code.putln("")
1996        code.putln(
1997            "static PyObject *%s(PyObject *o, PyObject *i, PyObject *c) {" % (
1998                scope.mangle_internal("tp_descr_get")))
1999        code.putln(
2000            "PyObject *r = 0;")
2001        code.putln(
2002            "if (!i) i = Py_None;")
2003        code.putln(
2004            "if (!c) c = Py_None;")
2005        #code.put_incref("i", py_object_type)
2006        #code.put_incref("c", py_object_type)
2007        code.putln(
2008            "r = %s(o, i, c);" % (
2009                user_get_entry.func_cname))
2010        #code.put_decref("i", py_object_type)
2011        #code.put_decref("c", py_object_type)
2012        code.putln(
2013            "return r;")
2014        code.putln(
2015            "}")
2016
2017    def generate_descr_set_function(self, scope, code):
2018        # Setting and deleting are both done through the __set__
2019        # method of a descriptor, so we dispatch to user's __set__
2020        # or __delete__ or raise an exception.
2021        base_type = scope.parent_type.base_type
2022        user_set_entry = scope.lookup_here("__set__")
2023        user_del_entry = scope.lookup_here("__delete__")
2024        code.putln("")
2025        code.putln(
2026            "static int %s(PyObject *o, PyObject *i, PyObject *v) {" % (
2027                scope.mangle_internal("tp_descr_set")))
2028        code.putln(
2029            "if (v) {")
2030        if user_set_entry:
2031            code.putln(
2032                "return %s(o, i, v);" % (
2033                    user_set_entry.func_cname))
2034        else:
2035            self.generate_guarded_basetype_call(
2036                base_type, None, "tp_descr_set", "o, i, v", code)
2037            code.putln(
2038                'PyErr_SetString(PyExc_NotImplementedError, "__set__");')
2039            code.putln(
2040                "return -1;")
2041        code.putln(
2042            "}")
2043        code.putln(
2044            "else {")
2045        if user_del_entry:
2046            code.putln(
2047                "return %s(o, i);" % (
2048                    user_del_entry.func_cname))
2049        else:
2050            self.generate_guarded_basetype_call(
2051                base_type, None, "tp_descr_set", "o, i, v", code)
2052            code.putln(
2053                'PyErr_SetString(PyExc_NotImplementedError, "__delete__");')
2054            code.putln(
2055                "return -1;")
2056        code.putln(
2057            "}")
2058        code.putln(
2059            "}")
2060
2061    def generate_property_accessors(self, cclass_scope, code):
2062        for entry in cclass_scope.property_entries:
2063            property_scope = entry.scope
2064            if property_scope.defines_any(["__get__"]):
2065                self.generate_property_get_function(entry, code)
2066            if property_scope.defines_any(["__set__", "__del__"]):
2067                self.generate_property_set_function(entry, code)
2068
2069    def generate_property_get_function(self, property_entry, code):
2070        property_scope = property_entry.scope
2071        property_entry.getter_cname = property_scope.parent_scope.mangle(
2072            Naming.prop_get_prefix, property_entry.name)
2073        get_entry = property_scope.lookup_here("__get__")
2074        code.putln("")
2075        code.putln(
2076            "static PyObject *%s(PyObject *o, CYTHON_UNUSED void *x) {" % (
2077                property_entry.getter_cname))
2078        code.putln(
2079            "return %s(o);" % (
2080                get_entry.func_cname))
2081        code.putln(
2082            "}")
2083
2084    def generate_property_set_function(self, property_entry, code):
2085        property_scope = property_entry.scope
2086        property_entry.setter_cname = property_scope.parent_scope.mangle(
2087            Naming.prop_set_prefix, property_entry.name)
2088        set_entry = property_scope.lookup_here("__set__")
2089        del_entry = property_scope.lookup_here("__del__")
2090        code.putln("")
2091        code.putln(
2092            "static int %s(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) {" % (
2093                property_entry.setter_cname))
2094        code.putln(
2095            "if (v) {")
2096        if set_entry:
2097            code.putln(
2098                "return %s(o, v);" % (
2099                    set_entry.func_cname))
2100        else:
2101            code.putln(
2102                'PyErr_SetString(PyExc_NotImplementedError, "__set__");')
2103            code.putln(
2104                "return -1;")
2105        code.putln(
2106            "}")
2107        code.putln(
2108            "else {")
2109        if del_entry:
2110            code.putln(
2111                "return %s(o);" % (
2112                    del_entry.func_cname))
2113        else:
2114            code.putln(
2115                'PyErr_SetString(PyExc_NotImplementedError, "__del__");')
2116            code.putln(
2117                "return -1;")
2118        code.putln(
2119            "}")
2120        code.putln(
2121            "}")
2122
2123    def generate_typeobj_definition(self, modname, entry, code):
2124        type = entry.type
2125        scope = type.scope
2126        for suite in TypeSlots.substructures:
2127            suite.generate_substructure(scope, code)
2128        code.putln("")
2129        if entry.visibility == 'public':
2130            header = "DL_EXPORT(PyTypeObject) %s = {"
2131        else:
2132            header = "static PyTypeObject %s = {"
2133        #code.putln(header % scope.parent_type.typeobj_cname)
2134        code.putln(header % type.typeobj_cname)
2135        code.putln(
2136            "PyVarObject_HEAD_INIT(0, 0)")
2137        code.putln(
2138            '"%s.%s", /*tp_name*/' % (
2139                self.full_module_name, scope.class_name))
2140        if type.typedef_flag:
2141            objstruct = type.objstruct_cname
2142        else:
2143            objstruct = "struct %s" % type.objstruct_cname
2144        code.putln(
2145            "sizeof(%s), /*tp_basicsize*/" % objstruct)
2146        code.putln(
2147            "0, /*tp_itemsize*/")
2148        for slot in TypeSlots.slot_table:
2149            slot.generate(scope, code)
2150        code.putln(
2151            "};")
2152
2153    def generate_method_table(self, env, code):
2154        if env.is_c_class_scope and not env.pyfunc_entries:
2155            return
2156        binding = env.directives['binding']
2157
2158        code.putln("")
2159        wrapper_code_writer = code.insertion_point()
2160
2161        code.putln(
2162            "static PyMethodDef %s[] = {" % (
2163                env.method_table_cname))
2164        for entry in env.pyfunc_entries:
2165            if not entry.fused_cfunction and not (binding and entry.is_overridable):
2166                code.put_pymethoddef(entry, ",", wrapper_code_writer=wrapper_code_writer)
2167        code.putln(
2168            "{0, 0, 0, 0}")
2169        code.putln(
2170            "};")
2171
2172        if wrapper_code_writer.getvalue():
2173            wrapper_code_writer.putln("")
2174
2175    def generate_dict_getter_function(self, scope, code):
2176        dict_attr = scope.lookup_here("__dict__")
2177        if not dict_attr or not dict_attr.is_variable:
2178            return
2179        func_name = scope.mangle_internal("__dict__getter")
2180        dict_name = dict_attr.cname
2181        code.putln("")
2182        code.putln("static PyObject *%s(PyObject *o, CYTHON_UNUSED void *x) {" % func_name)
2183        self.generate_self_cast(scope, code)
2184        code.putln("if (unlikely(!p->%s)){" % dict_name)
2185        code.putln("p->%s = PyDict_New();" % dict_name)
2186        code.putln("}")
2187        code.putln("Py_XINCREF(p->%s);" % dict_name)
2188        code.putln("return p->%s;" % dict_name)
2189        code.putln("}")
2190
2191    def generate_getset_table(self, env, code):
2192        if env.property_entries:
2193            code.putln("")
2194            code.putln(
2195                "static struct PyGetSetDef %s[] = {" %
2196                env.getset_table_cname)
2197            for entry in env.property_entries:
2198                doc = entry.doc
2199                if doc:
2200                    if doc.is_unicode:
2201                        doc = doc.as_utf8_string()
2202                    doc_code = doc.as_c_string_literal()
2203                else:
2204                    doc_code = "0"
2205                code.putln(
2206                    '{(char *)"%s", %s, %s, (char *)%s, 0},' % (
2207                        entry.name,
2208                        entry.getter_cname or "0",
2209                        entry.setter_cname or "0",
2210                        doc_code))
2211            code.putln(
2212                "{0, 0, 0, 0, 0}")
2213            code.putln(
2214                "};")
2215
2216    def create_import_star_conversion_utility_code(self, env):
2217        # Create all conversion helpers that are needed for "import *" assignments.
2218        # Must be done before code generation to support CythonUtilityCode.
2219        for name, entry in sorted(env.entries.items()):
2220            if entry.is_cglobal and entry.used:
2221                if not entry.type.is_pyobject:
2222                    entry.type.create_from_py_utility_code(env)
2223
2224    def generate_import_star(self, env, code):
2225        env.use_utility_code(UtilityCode.load_cached("CStringEquals", "StringTools.c"))
2226        code.putln()
2227        code.enter_cfunc_scope()  # as we need labels
2228        code.putln("static int %s(PyObject *o, PyObject* py_name, char *name) {" % Naming.import_star_set)
2229
2230        code.putln("static const char* internal_type_names[] = {")
2231        for name, entry in sorted(env.entries.items()):
2232            if entry.is_type:
2233                code.putln('"%s",' % name)
2234        code.putln("0")
2235        code.putln("};")
2236
2237        code.putln("const char** type_name = internal_type_names;")
2238        code.putln("while (*type_name) {")
2239        code.putln("if (__Pyx_StrEq(name, *type_name)) {")
2240        code.putln('PyErr_Format(PyExc_TypeError, "Cannot overwrite C type %s", name);')
2241        code.putln('goto bad;')
2242        code.putln("}")
2243        code.putln("type_name++;")
2244        code.putln("}")
2245
2246        old_error_label = code.new_error_label()
2247        code.putln("if (0);")  # so the first one can be "else if"
2248        msvc_count = 0
2249        for name, entry in sorted(env.entries.items()):
2250            if entry.is_cglobal and entry.used and not entry.type.is_const:
2251                msvc_count += 1
2252                if msvc_count % 100 == 0:
2253                    code.putln("#ifdef _MSC_VER")
2254                    code.putln("if (0);  /* Workaround for MSVC C1061. */")
2255                    code.putln("#endif")
2256                code.putln('else if (__Pyx_StrEq(name, "%s")) {' % name)
2257                if entry.type.is_pyobject:
2258                    if entry.type.is_extension_type or entry.type.is_builtin_type:
2259                        code.putln("if (!(%s)) %s;" % (
2260                            entry.type.type_test_code("o"),
2261                            code.error_goto(entry.pos)))
2262                    code.putln("Py_INCREF(o);")
2263                    code.put_decref(entry.cname, entry.type, nanny=False)
2264                    code.putln("%s = %s;" % (
2265                        entry.cname,
2266                        PyrexTypes.typecast(entry.type, py_object_type, "o")))
2267                elif entry.type.create_from_py_utility_code(env):
2268                    # if available, utility code was already created in self.prepare_utility_code()
2269                    code.putln(entry.type.from_py_call_code(
2270                        'o', entry.cname, entry.pos, code))
2271                else:
2272                    code.putln('PyErr_Format(PyExc_TypeError, "Cannot convert Python object %s to %s");' % (
2273                        name, entry.type))
2274                    code.putln(code.error_goto(entry.pos))
2275                code.putln("}")
2276        code.putln("else {")
2277        code.putln("if (PyObject_SetAttr(%s, py_name, o) < 0) goto bad;" % Naming.module_cname)
2278        code.putln("}")
2279        code.putln("return 0;")
2280        if code.label_used(code.error_label):
2281            code.put_label(code.error_label)
2282            # This helps locate the offending name.
2283            code.put_add_traceback(self.full_module_name)
2284        code.error_label = old_error_label
2285        code.putln("bad:")
2286        code.putln("return -1;")
2287        code.putln("}")
2288        code.putln("")
2289        code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1])
2290        code.exit_cfunc_scope()  # done with labels
2291
2292    def generate_module_init_func(self, imported_modules, env, code):
2293        subfunction = self.mod_init_subfunction(self.scope, code)
2294
2295        code.enter_cfunc_scope(self.scope)
2296        code.putln("")
2297        code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0])
2298        header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % env.module_name
2299        header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env)
2300        code.putln("#if PY_MAJOR_VERSION < 3")
2301        # Optimise for small code size as the module init function is only executed once.
2302        code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header2)
2303        code.putln(header2)
2304        code.putln("#else")
2305        code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header3)
2306        code.putln(header3)
2307
2308        # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.)
2309        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2310        code.putln("{")
2311        code.putln("return PyModuleDef_Init(&%s);" % Naming.pymoduledef_cname)
2312        code.putln("}")
2313
2314        mod_create_func = UtilityCode.load_as_string("ModuleCreationPEP489", "ModuleSetupCode.c")[1]
2315        code.put(mod_create_func)
2316
2317        code.putln("")
2318        # main module init code lives in Py_mod_exec function, not in PyInit function
2319        code.putln("static CYTHON_SMALL_CODE int %s(PyObject *%s)" % (
2320            self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env),
2321            Naming.pymodinit_module_arg))
2322        code.putln("#endif")  # PEP489
2323
2324        code.putln("#endif")  # Py3
2325
2326        # start of module init/exec function (pre/post PEP 489)
2327        code.putln("{")
2328
2329        tempdecl_code = code.insertion_point()
2330
2331        profile = code.globalstate.directives['profile']
2332        linetrace = code.globalstate.directives['linetrace']
2333        if profile or linetrace:
2334            code.globalstate.use_utility_code(UtilityCode.load_cached("Profile", "Profile.c"))
2335
2336        code.put_declare_refcount_context()
2337        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2338        # Most extension modules simply can't deal with it, and Cython isn't ready either.
2339        # See issues listed here: https://docs.python.org/3/c-api/init.html#sub-interpreter-support
2340        code.putln("if (%s) {" % Naming.module_cname)
2341        # Hack: enforce single initialisation.
2342        code.putln("if (%s == %s) return 0;" % (
2343            Naming.module_cname,
2344            Naming.pymodinit_module_arg,
2345        ))
2346        code.putln('PyErr_SetString(PyExc_RuntimeError,'
2347                   ' "Module \'%s\' has already been imported. Re-initialisation is not supported.");' %
2348                   env.module_name)
2349        code.putln("return -1;")
2350        code.putln("}")
2351        code.putln("#elif PY_MAJOR_VERSION >= 3")
2352        # Hack: enforce single initialisation also on reimports under different names on Python 3 (with PEP 3121/489).
2353        code.putln("if (%s) return __Pyx_NewRef(%s);" % (
2354            Naming.module_cname,
2355            Naming.module_cname,
2356        ))
2357        code.putln("#endif")
2358
2359        if profile or linetrace:
2360            tempdecl_code.put_trace_declarations()
2361            code.put_trace_frame_init()
2362
2363        refnanny_import_code = UtilityCode.load_as_string("ImportRefnannyAPI", "ModuleSetupCode.c")[1]
2364        code.putln(refnanny_import_code.rstrip())
2365        code.put_setup_refcount_context(header3)
2366
2367        env.use_utility_code(UtilityCode.load("CheckBinaryVersion", "ModuleSetupCode.c"))
2368        code.put_error_if_neg(self.pos, "__Pyx_check_binary_version()")
2369
2370        code.putln("#ifdef __Pxy_PyFrame_Initialize_Offsets")
2371        code.putln("__Pxy_PyFrame_Initialize_Offsets();")
2372        code.putln("#endif")
2373        code.putln("%s = PyTuple_New(0); %s" % (
2374            Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)))
2375        code.putln("%s = PyBytes_FromStringAndSize(\"\", 0); %s" % (
2376            Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos)))
2377        code.putln("%s = PyUnicode_FromStringAndSize(\"\", 0); %s" % (
2378            Naming.empty_unicode, code.error_goto_if_null(Naming.empty_unicode, self.pos)))
2379
2380        for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'AsyncGen', 'StopAsyncIteration'):
2381            code.putln("#ifdef __Pyx_%s_USED" % ext_type)
2382            code.put_error_if_neg(self.pos, "__pyx_%s_init()" % ext_type)
2383            code.putln("#endif")
2384
2385        code.putln("/*--- Library function declarations ---*/")
2386        if env.directives['np_pythran']:
2387            code.put_error_if_neg(self.pos, "_import_array()")
2388
2389        code.putln("/*--- Threads initialization code ---*/")
2390        code.putln("#if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS")
2391        code.putln("#ifdef WITH_THREAD /* Python build with threading support? */")
2392        code.putln("PyEval_InitThreads();")
2393        code.putln("#endif")
2394        code.putln("#endif")
2395
2396        code.putln("/*--- Module creation code ---*/")
2397        self.generate_module_creation_code(env, code)
2398
2399        code.putln("/*--- Initialize various global constants etc. ---*/")
2400        code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()")
2401
2402        code.putln("#if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || "
2403                   "__PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT)")
2404        code.put_error_if_neg(self.pos, "__Pyx_init_sys_getdefaultencoding_params()")
2405        code.putln("#endif")
2406
2407        code.putln("if (%s%s) {" % (Naming.module_is_main, self.full_module_name.replace('.', '__')))
2408        code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % (
2409            env.module_cname,
2410            code.intern_identifier(EncodedString("__name__")),
2411            code.intern_identifier(EncodedString("__main__"))))
2412        code.putln("}")
2413
2414        # set up __file__ and __path__, then add the module to sys.modules
2415        self.generate_module_import_setup(env, code)
2416
2417        if Options.cache_builtins:
2418            code.putln("/*--- Builtin init code ---*/")
2419            code.put_error_if_neg(None, "__Pyx_InitCachedBuiltins()")
2420
2421        code.putln("/*--- Constants init code ---*/")
2422        code.put_error_if_neg(None, "__Pyx_InitCachedConstants()")
2423
2424        code.putln("/*--- Global type/function init code ---*/")
2425
2426        with subfunction("Global init code") as inner_code:
2427            self.generate_global_init_code(env, inner_code)
2428
2429        with subfunction("Variable export code") as inner_code:
2430            self.generate_c_variable_export_code(env, inner_code)
2431
2432        with subfunction("Function export code") as inner_code:
2433            self.generate_c_function_export_code(env, inner_code)
2434
2435        with subfunction("Type init code") as inner_code:
2436            self.generate_type_init_code(env, inner_code)
2437
2438        with subfunction("Type import code") as inner_code:
2439            for module in imported_modules:
2440                self.generate_type_import_code_for_module(module, env, inner_code)
2441
2442        with subfunction("Variable import code") as inner_code:
2443            for module in imported_modules:
2444                self.generate_c_variable_import_code_for_module(module, env, inner_code)
2445
2446        with subfunction("Function import code") as inner_code:
2447            for module in imported_modules:
2448                self.specialize_fused_types(module)
2449                self.generate_c_function_import_code_for_module(module, env, inner_code)
2450
2451        code.putln("/*--- Execution code ---*/")
2452        code.mark_pos(None)
2453
2454        code.putln("#if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED)")
2455        code.put_error_if_neg(self.pos, "__Pyx_patch_abc()")
2456        code.putln("#endif")
2457
2458        if profile or linetrace:
2459            code.put_trace_call(header3, self.pos, nogil=not code.funcstate.gil_owned)
2460            code.funcstate.can_trace = True
2461
2462        self.body.generate_execution_code(code)
2463
2464        if profile or linetrace:
2465            code.funcstate.can_trace = False
2466            code.put_trace_return("Py_None", nogil=not code.funcstate.gil_owned)
2467
2468        code.putln()
2469        code.putln("/*--- Wrapped vars code ---*/")
2470        self.generate_wrapped_entries_code(env, code)
2471        code.putln()
2472
2473        if Options.generate_cleanup_code:
2474            code.globalstate.use_utility_code(
2475                UtilityCode.load_cached("RegisterModuleCleanup", "ModuleSetupCode.c"))
2476            code.putln("if (__Pyx_RegisterCleanup()) %s;" % code.error_goto(self.pos))
2477
2478        code.put_goto(code.return_label)
2479        code.put_label(code.error_label)
2480        for cname, type in code.funcstate.all_managed_temps():
2481            code.put_xdecref(cname, type)
2482        code.putln('if (%s) {' % env.module_cname)
2483        code.putln('if (%s) {' % env.module_dict_cname)
2484        code.put_add_traceback("init %s" % env.qualified_name)
2485        code.globalstate.use_utility_code(Nodes.traceback_utility_code)
2486        # Module reference and module dict are in global variables which might still be needed
2487        # for cleanup, atexit code, etc., so leaking is better than crashing.
2488        # At least clearing the module dict here might be a good idea, but could still break
2489        # user code in atexit or other global registries.
2490        ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False)
2491        code.putln('}')
2492        code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True)
2493        code.putln('} else if (!PyErr_Occurred()) {')
2494        code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % env.qualified_name)
2495        code.putln('}')
2496        code.put_label(code.return_label)
2497
2498        code.put_finish_refcount_context()
2499
2500        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2501        code.putln("return (%s != NULL) ? 0 : -1;" % env.module_cname)
2502        code.putln("#elif PY_MAJOR_VERSION >= 3")
2503        code.putln("return %s;" % env.module_cname)
2504        code.putln("#else")
2505        code.putln("return;")
2506        code.putln("#endif")
2507        code.putln('}')
2508
2509        tempdecl_code.put_temp_declarations(code.funcstate)
2510
2511        code.exit_cfunc_scope()
2512
2513    def mod_init_subfunction(self, scope, orig_code):
2514        """
2515        Return a context manager that allows deviating the module init code generation
2516        into a separate function and instead inserts a call to it.
2517
2518        Can be reused sequentially to create multiple functions.
2519        The functions get inserted at the point where the context manager was created.
2520        The call gets inserted where the context manager is used (on entry).
2521        """
2522        prototypes = orig_code.insertion_point()
2523        prototypes.putln("")
2524        function_code = orig_code.insertion_point()
2525        function_code.putln("")
2526
2527        class ModInitSubfunction(object):
2528            def __init__(self, code_type):
2529                cname = '_'.join(code_type.lower().split())
2530                assert re.match("^[a-z0-9_]+$", cname)
2531                self.cfunc_name = "__Pyx_modinit_%s" % cname
2532                self.description = code_type
2533                self.tempdecl_code = None
2534                self.call_code = None
2535
2536            def __enter__(self):
2537                self.call_code = orig_code.insertion_point()
2538                code = function_code
2539                code.enter_cfunc_scope(scope)
2540                prototypes.putln("static CYTHON_SMALL_CODE int %s(void); /*proto*/" % self.cfunc_name)
2541                code.putln("static int %s(void) {" % self.cfunc_name)
2542                code.put_declare_refcount_context()
2543                self.tempdecl_code = code.insertion_point()
2544                code.put_setup_refcount_context(self.cfunc_name)
2545                # Leave a grepable marker that makes it easy to find the generator source.
2546                code.putln("/*--- %s ---*/" % self.description)
2547                return code
2548
2549            def __exit__(self, *args):
2550                code = function_code
2551                code.put_finish_refcount_context()
2552                code.putln("return 0;")
2553
2554                self.tempdecl_code.put_temp_declarations(code.funcstate)
2555                self.tempdecl_code = None
2556
2557                needs_error_handling = code.label_used(code.error_label)
2558                if needs_error_handling:
2559                    code.put_label(code.error_label)
2560                    for cname, type in code.funcstate.all_managed_temps():
2561                        code.put_xdecref(cname, type)
2562                    code.put_finish_refcount_context()
2563                    code.putln("return -1;")
2564                code.putln("}")
2565                code.exit_cfunc_scope()
2566                code.putln("")
2567
2568                if needs_error_handling:
2569                    self.call_code.use_label(orig_code.error_label)
2570                    self.call_code.putln("if (unlikely(%s() != 0)) goto %s;" % (
2571                        self.cfunc_name, orig_code.error_label))
2572                else:
2573                    self.call_code.putln("(void)%s();" % self.cfunc_name)
2574                self.call_code = None
2575
2576        return ModInitSubfunction
2577
2578    def generate_module_import_setup(self, env, code):
2579        module_path = env.directives['set_initial_path']
2580        if module_path == 'SOURCEFILE':
2581            module_path = self.pos[0].filename
2582
2583        if module_path:
2584            code.putln('if (!CYTHON_PEP489_MULTI_PHASE_INIT) {')
2585            code.putln('if (PyObject_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
2586                env.module_cname,
2587                code.globalstate.get_py_string_const(
2588                    EncodedString(decode_filename(module_path))).cname,
2589                code.error_goto(self.pos)))
2590            code.putln("}")
2591
2592            if env.is_package:
2593                # set __path__ to mark the module as package
2594                code.putln('if (!CYTHON_PEP489_MULTI_PHASE_INIT) {')
2595                temp = code.funcstate.allocate_temp(py_object_type, True)
2596                code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
2597                    temp,
2598                    code.globalstate.get_py_string_const(
2599                        EncodedString(decode_filename(
2600                            os.path.dirname(module_path)))).cname,
2601                    code.error_goto_if_null(temp, self.pos)))
2602                code.put_gotref(temp)
2603                code.putln(
2604                    'if (PyObject_SetAttrString(%s, "__path__", %s) < 0) %s;' % (
2605                        env.module_cname, temp, code.error_goto(self.pos)))
2606                code.put_decref_clear(temp, py_object_type)
2607                code.funcstate.release_temp(temp)
2608                code.putln("}")
2609
2610        elif env.is_package:
2611            # packages require __path__, so all we can do is try to figure
2612            # out the module path at runtime by rerunning the import lookup
2613            code.putln("if (!CYTHON_PEP489_MULTI_PHASE_INIT) {")
2614            package_name, _ = self.full_module_name.rsplit('.', 1)
2615            if '.' in package_name:
2616                parent_name = '"%s"' % (package_name.rsplit('.', 1)[0],)
2617            else:
2618                parent_name = 'NULL'
2619            code.globalstate.use_utility_code(UtilityCode.load(
2620                "SetPackagePathFromImportLib", "ImportExport.c"))
2621            code.putln(code.error_goto_if_neg(
2622                '__Pyx_SetPackagePathFromImportLib(%s, %s)' % (
2623                    parent_name,
2624                    code.globalstate.get_py_string_const(
2625                        EncodedString(env.module_name)).cname),
2626                self.pos))
2627            code.putln("}")
2628
2629        # CPython may not have put us into sys.modules yet, but relative imports and reimports require it
2630        fq_module_name = self.full_module_name
2631        if fq_module_name.endswith('.__init__'):
2632            fq_module_name = fq_module_name[:-len('.__init__')]
2633        code.putln("#if PY_MAJOR_VERSION >= 3")
2634        code.putln("{")
2635        code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" %
2636                   code.error_goto_if_null("modules", self.pos))
2637        code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name)
2638        code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % (
2639            fq_module_name, env.module_cname), self.pos))
2640        code.putln("}")
2641        code.putln("}")
2642        code.putln("#endif")
2643
2644    def generate_module_cleanup_func(self, env, code):
2645        if not Options.generate_cleanup_code:
2646            return
2647
2648        code.putln('static void %s(CYTHON_UNUSED PyObject *self) {' %
2649                   Naming.cleanup_cname)
2650        if Options.generate_cleanup_code >= 2:
2651            code.putln("/*--- Global cleanup code ---*/")
2652            rev_entries = list(env.var_entries)
2653            rev_entries.reverse()
2654            for entry in rev_entries:
2655                if entry.visibility != 'extern':
2656                    if entry.type.is_pyobject and entry.used:
2657                        code.put_xdecref_clear(
2658                            entry.cname, entry.type,
2659                            clear_before_decref=True,
2660                            nanny=False)
2661        code.putln("__Pyx_CleanupGlobals();")
2662        if Options.generate_cleanup_code >= 3:
2663            code.putln("/*--- Type import cleanup code ---*/")
2664            for ext_type in sorted(env.types_imported, key=operator.attrgetter('typeptr_cname')):
2665                code.put_xdecref_clear(
2666                    ext_type.typeptr_cname, ext_type,
2667                    clear_before_decref=True,
2668                    nanny=False)
2669        if Options.cache_builtins:
2670            code.putln("/*--- Builtin cleanup code ---*/")
2671            for entry in env.cached_builtins:
2672                code.put_xdecref_clear(
2673                    entry.cname, PyrexTypes.py_object_type,
2674                    clear_before_decref=True,
2675                    nanny=False)
2676        code.putln("/*--- Intern cleanup code ---*/")
2677        code.put_decref_clear(Naming.empty_tuple,
2678                              PyrexTypes.py_object_type,
2679                              clear_before_decref=True,
2680                              nanny=False)
2681        for entry in env.c_class_entries:
2682            cclass_type = entry.type
2683            if cclass_type.is_external or cclass_type.base_type:
2684                continue
2685            if cclass_type.scope.directives.get('freelist', 0):
2686                scope = cclass_type.scope
2687                freelist_name = scope.mangle_internal(Naming.freelist_name)
2688                freecount_name = scope.mangle_internal(Naming.freecount_name)
2689                code.putln("while (%s > 0) {" % freecount_name)
2690                code.putln("PyObject* o = (PyObject*)%s[--%s];" % (
2691                    freelist_name, freecount_name))
2692                code.putln("(*Py_TYPE(o)->tp_free)(o);")
2693                code.putln("}")
2694#        for entry in env.pynum_entries:
2695#            code.put_decref_clear(entry.cname,
2696#                                  PyrexTypes.py_object_type,
2697#                                  nanny=False)
2698#        for entry in env.all_pystring_entries:
2699#            if entry.is_interned:
2700#                code.put_decref_clear(entry.pystring_cname,
2701#                                      PyrexTypes.py_object_type,
2702#                                      nanny=False)
2703#        for entry in env.default_entries:
2704#            if entry.type.is_pyobject and entry.used:
2705#                code.putln("Py_DECREF(%s); %s = 0;" % (
2706#                    code.entry_as_pyobject(entry), entry.cname))
2707        if Options.pre_import is not None:
2708            code.put_decref_clear(Naming.preimport_cname, py_object_type,
2709                                  nanny=False, clear_before_decref=True)
2710        for cname in [env.module_dict_cname, Naming.cython_runtime_cname, Naming.builtins_cname]:
2711            code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True)
2712
2713    def generate_main_method(self, env, code):
2714        module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__'))
2715        if Options.embed == "main":
2716            wmain = "wmain"
2717        else:
2718            wmain = Options.embed
2719        main_method = UtilityCode.load_cached("MainFunction", "Embed.c")
2720        code.globalstate.use_utility_code(
2721            main_method.specialize(
2722                module_name=env.module_name,
2723                module_is_main=module_is_main,
2724                main_method=Options.embed,
2725                wmain_method=wmain))
2726
2727    def mod_init_func_cname(self, prefix, env):
2728        return '%s_%s' % (prefix, env.module_name)
2729
2730    def generate_pymoduledef_struct(self, env, code):
2731        if env.doc:
2732            doc = "%s" % code.get_string_const(env.doc)
2733        else:
2734            doc = "0"
2735        if Options.generate_cleanup_code:
2736            cleanup_func = "(freefunc)%s" % Naming.cleanup_cname
2737        else:
2738            cleanup_func = 'NULL'
2739
2740        code.putln("")
2741        code.putln("#if PY_MAJOR_VERSION >= 3")
2742        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2743        exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env)
2744        code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" %
2745                   Naming.pymodule_create_func_cname)
2746        code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname)
2747
2748        code.putln("static PyModuleDef_Slot %s[] = {" % Naming.pymoduledef_slots_cname)
2749        code.putln("{Py_mod_create, (void*)%s}," % Naming.pymodule_create_func_cname)
2750        code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname)
2751        code.putln("{0, NULL}")
2752        code.putln("};")
2753        code.putln("#endif")
2754
2755        code.putln("")
2756        code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname)
2757        code.putln("  PyModuleDef_HEAD_INIT,")
2758        code.putln('  "%s",' % env.module_name)
2759        code.putln("  %s, /* m_doc */" % doc)
2760        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2761        code.putln("  0, /* m_size */")
2762        code.putln("#else")
2763        code.putln("  -1, /* m_size */")
2764        code.putln("#endif")
2765        code.putln("  %s /* m_methods */," % env.method_table_cname)
2766        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2767        code.putln("  %s, /* m_slots */" % Naming.pymoduledef_slots_cname)
2768        code.putln("#else")
2769        code.putln("  NULL, /* m_reload */")
2770        code.putln("#endif")
2771        code.putln("  NULL, /* m_traverse */")
2772        code.putln("  NULL, /* m_clear */")
2773        code.putln("  %s /* m_free */" % cleanup_func)
2774        code.putln("};")
2775        code.putln("#endif")
2776
2777    def generate_module_creation_code(self, env, code):
2778        # Generate code to create the module object and
2779        # install the builtins.
2780        if env.doc:
2781            doc = "%s" % code.get_string_const(env.doc)
2782        else:
2783            doc = "0"
2784
2785        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
2786        code.putln("%s = %s;" % (
2787            env.module_cname,
2788            Naming.pymodinit_module_arg))
2789        code.put_incref(env.module_cname, py_object_type, nanny=False)
2790        code.putln("#else")
2791        code.putln("#if PY_MAJOR_VERSION < 3")
2792        code.putln(
2793            '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % (
2794                env.module_cname,
2795                env.module_name,
2796                env.method_table_cname,
2797                doc,
2798                env.module_cname))
2799        code.putln("#else")
2800        code.putln(
2801            "%s = PyModule_Create(&%s);" % (
2802                env.module_cname,
2803                Naming.pymoduledef_cname))
2804        code.putln("#endif")
2805        code.putln(code.error_goto_if_null(env.module_cname, self.pos))
2806        code.putln("#endif")  # CYTHON_PEP489_MULTI_PHASE_INIT
2807
2808        code.putln(
2809            "%s = PyModule_GetDict(%s); %s" % (
2810                env.module_dict_cname, env.module_cname,
2811                code.error_goto_if_null(env.module_dict_cname, self.pos)))
2812        code.put_incref(env.module_dict_cname, py_object_type, nanny=False)
2813
2814        code.putln(
2815            '%s = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); %s' % (
2816                Naming.builtins_cname,
2817                code.error_goto_if_null(Naming.builtins_cname, self.pos)))
2818        code.put_incref(Naming.builtins_cname, py_object_type, nanny=False)
2819        code.putln(
2820            '%s = PyImport_AddModule((char *) "cython_runtime"); %s' % (
2821                Naming.cython_runtime_cname,
2822                code.error_goto_if_null(Naming.cython_runtime_cname, self.pos)))
2823        code.put_incref(Naming.cython_runtime_cname, py_object_type, nanny=False)
2824        code.putln(
2825            'if (PyObject_SetAttrString(%s, "__builtins__", %s) < 0) %s;' % (
2826                env.module_cname,
2827                Naming.builtins_cname,
2828                code.error_goto(self.pos)))
2829        if Options.pre_import is not None:
2830            code.putln(
2831                '%s = PyImport_AddModule("%s"); %s' % (
2832                    Naming.preimport_cname,
2833                    Options.pre_import,
2834                    code.error_goto_if_null(Naming.preimport_cname, self.pos)))
2835            code.put_incref(Naming.preimport_cname, py_object_type, nanny=False)
2836
2837    def generate_global_init_code(self, env, code):
2838        # Generate code to initialise global PyObject *
2839        # variables to None.
2840        for entry in env.var_entries:
2841            if entry.visibility != 'extern':
2842                if entry.used:
2843                    entry.type.global_init_code(entry, code)
2844
2845    def generate_wrapped_entries_code(self, env, code):
2846        for name, entry in sorted(env.entries.items()):
2847            if (entry.create_wrapper
2848                    and not entry.is_type
2849                    and entry.scope is env):
2850                if not entry.type.create_to_py_utility_code(env):
2851                    error(entry.pos, "Cannot convert '%s' to Python object" % entry.type)
2852                code.putln("{")
2853                code.putln("PyObject* wrapped = %s(%s);"  % (
2854                    entry.type.to_py_function,
2855                    entry.cname))
2856                code.putln(code.error_goto_if_null("wrapped", entry.pos))
2857                code.putln(
2858                    'if (PyObject_SetAttrString(%s, "%s", wrapped) < 0) %s;' % (
2859                        env.module_cname,
2860                        name,
2861                        code.error_goto(entry.pos)))
2862                code.putln("}")
2863
2864    def generate_c_variable_export_code(self, env, code):
2865        # Generate code to create PyCFunction wrappers for exported C functions.
2866        entries = []
2867        for entry in env.var_entries:
2868            if (entry.api
2869                    or entry.defined_in_pxd
2870                    or (Options.cimport_from_pyx and not entry.visibility == 'extern')):
2871                entries.append(entry)
2872        if entries:
2873            env.use_utility_code(UtilityCode.load_cached("VoidPtrExport", "ImportExport.c"))
2874            for entry in entries:
2875                signature = entry.type.empty_declaration_code()
2876                name = code.intern_identifier(entry.name)
2877                code.putln('if (__Pyx_ExportVoidPtr(%s, (void *)&%s, "%s") < 0) %s' % (
2878                    name, entry.cname, signature,
2879                    code.error_goto(self.pos)))
2880
2881    def generate_c_function_export_code(self, env, code):
2882        # Generate code to create PyCFunction wrappers for exported C functions.
2883        entries = []
2884        for entry in env.cfunc_entries:
2885            if (entry.api
2886                    or entry.defined_in_pxd
2887                    or (Options.cimport_from_pyx and not entry.visibility == 'extern')):
2888                entries.append(entry)
2889        if entries:
2890            env.use_utility_code(
2891                UtilityCode.load_cached("FunctionExport", "ImportExport.c"))
2892            # Note: while this looks like it could be more cheaply stored and read from a struct array,
2893            # investigation shows that the resulting binary is smaller with repeated functions calls.
2894            for entry in entries:
2895                signature = entry.type.signature_string()
2896                code.putln('if (__Pyx_ExportFunction("%s", (void (*)(void))%s, "%s") < 0) %s' % (
2897                    entry.name,
2898                    entry.cname,
2899                    signature,
2900                    code.error_goto(self.pos)))
2901
2902    def generate_type_import_code_for_module(self, module, env, code):
2903        # Generate type import code for all exported extension types in
2904        # an imported module.
2905        #if module.c_class_entries:
2906        with ModuleImportGenerator(code) as import_generator:
2907            for entry in module.c_class_entries:
2908                if entry.defined_in_pxd:
2909                    self.generate_type_import_code(env, entry.type, entry.pos, code, import_generator)
2910
2911    def specialize_fused_types(self, pxd_env):
2912        """
2913        If fused c(p)def functions are defined in an imported pxd, but not
2914        used in this implementation file, we still have fused entries and
2915        not specialized ones. This method replaces any fused entries with their
2916        specialized ones.
2917        """
2918        for entry in pxd_env.cfunc_entries[:]:
2919            if entry.type.is_fused:
2920                # This call modifies the cfunc_entries in-place
2921                entry.type.get_all_specialized_function_types()
2922
2923    def generate_c_variable_import_code_for_module(self, module, env, code):
2924        # Generate import code for all exported C functions in a cimported module.
2925        entries = []
2926        for entry in module.var_entries:
2927            if entry.defined_in_pxd:
2928                entries.append(entry)
2929        if entries:
2930            env.use_utility_code(
2931                UtilityCode.load_cached("VoidPtrImport", "ImportExport.c"))
2932            temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
2933            code.putln(
2934                '%s = PyImport_ImportModule("%s"); if (!%s) %s' % (
2935                    temp,
2936                    module.qualified_name,
2937                    temp,
2938                    code.error_goto(self.pos)))
2939            for entry in entries:
2940                if env is module:
2941                    cname = entry.cname
2942                else:
2943                    cname = module.mangle(Naming.varptr_prefix, entry.name)
2944                signature = entry.type.empty_declaration_code()
2945                code.putln(
2946                    'if (__Pyx_ImportVoidPtr(%s, "%s", (void **)&%s, "%s") < 0) %s' % (
2947                        temp, entry.name, cname, signature,
2948                        code.error_goto(self.pos)))
2949            code.putln("Py_DECREF(%s); %s = 0;" % (temp, temp))
2950
2951    def generate_c_function_import_code_for_module(self, module, env, code):
2952        # Generate import code for all exported C functions in a cimported module.
2953        entries = []
2954        for entry in module.cfunc_entries:
2955            if entry.defined_in_pxd and entry.used:
2956                entries.append(entry)
2957        if entries:
2958            env.use_utility_code(
2959                UtilityCode.load_cached("FunctionImport", "ImportExport.c"))
2960            temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
2961            code.putln(
2962                '%s = PyImport_ImportModule("%s"); if (!%s) %s' % (
2963                    temp,
2964                    module.qualified_name,
2965                    temp,
2966                    code.error_goto(self.pos)))
2967            for entry in entries:
2968                code.putln(
2969                    'if (__Pyx_ImportFunction(%s, "%s", (void (**)(void))&%s, "%s") < 0) %s' % (
2970                        temp,
2971                        entry.name,
2972                        entry.cname,
2973                        entry.type.signature_string(),
2974                        code.error_goto(self.pos)))
2975            code.putln("Py_DECREF(%s); %s = 0;" % (temp, temp))
2976
2977    def generate_type_init_code(self, env, code):
2978        # Generate type import code for extern extension types
2979        # and type ready code for non-extern ones.
2980        with ModuleImportGenerator(code) as import_generator:
2981            for entry in env.c_class_entries:
2982                if entry.visibility == 'extern' and not entry.utility_code_definition:
2983                    self.generate_type_import_code(env, entry.type, entry.pos, code, import_generator)
2984                else:
2985                    self.generate_base_type_import_code(env, entry, code, import_generator)
2986                    self.generate_exttype_vtable_init_code(entry, code)
2987                    if entry.type.early_init:
2988                        self.generate_type_ready_code(entry, code)
2989
2990    def generate_base_type_import_code(self, env, entry, code, import_generator):
2991        base_type = entry.type.base_type
2992        if (base_type and base_type.module_name != env.qualified_name and not
2993                base_type.is_builtin_type and not entry.utility_code_definition):
2994            self.generate_type_import_code(env, base_type, self.pos, code, import_generator)
2995
2996    def generate_type_import_code(self, env, type, pos, code, import_generator):
2997        # If not already done, generate code to import the typeobject of an
2998        # extension type defined in another module, and extract its C method
2999        # table pointer if any.
3000        if type in env.types_imported:
3001            return
3002        if type.name not in Code.ctypedef_builtins_map:
3003            # see corresponding condition in generate_type_import_call() below!
3004            code.globalstate.use_utility_code(
3005                UtilityCode.load_cached("TypeImport", "ImportExport.c"))
3006        self.generate_type_import_call(type, code, import_generator, error_pos=pos)
3007        if type.vtabptr_cname:
3008            code.globalstate.use_utility_code(
3009                UtilityCode.load_cached('GetVTable', 'ImportExport.c'))
3010            code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % (
3011                type.vtabptr_cname,
3012                type.vtabstruct_cname,
3013                type.typeptr_cname,
3014                code.error_goto_if_null(type.vtabptr_cname, pos)))
3015        env.types_imported.add(type)
3016
3017    def generate_type_import_call(self, type, code, import_generator, error_code=None, error_pos=None):
3018        if type.typedef_flag:
3019            objstruct = type.objstruct_cname
3020        else:
3021            objstruct = "struct %s" % type.objstruct_cname
3022        sizeof_objstruct = objstruct
3023        module_name = type.module_name
3024        condition = replacement = None
3025        if module_name not in ('__builtin__', 'builtins'):
3026            module_name = '"%s"' % module_name
3027        elif type.name in Code.ctypedef_builtins_map:
3028            # Fast path for special builtins, don't actually import
3029            ctypename = Code.ctypedef_builtins_map[type.name]
3030            code.putln('%s = %s;' % (type.typeptr_cname, ctypename))
3031            return
3032        else:
3033            module_name = '__Pyx_BUILTIN_MODULE_NAME'
3034            if type.name in Code.non_portable_builtins_map:
3035                condition, replacement = Code.non_portable_builtins_map[type.name]
3036            if objstruct in Code.basicsize_builtins_map:
3037                # Some builtin types have a tp_basicsize which differs from sizeof(...):
3038                sizeof_objstruct = Code.basicsize_builtins_map[objstruct]
3039
3040        if not error_code:
3041            assert error_pos is not None
3042            error_code = code.error_goto(error_pos)
3043
3044        module = import_generator.imported_module(module_name, error_code)
3045        code.put('%s = __Pyx_ImportType(%s, %s,' % (
3046            type.typeptr_cname,
3047            module,
3048            module_name))
3049
3050        if condition and replacement:
3051            code.putln("")  # start in new line
3052            code.putln("#if %s" % condition)
3053            code.putln('"%s",' % replacement)
3054            code.putln("#else")
3055            code.putln('"%s",' % type.name)
3056            code.putln("#endif")
3057        else:
3058            code.put(' "%s", ' % type.name)
3059
3060        if sizeof_objstruct != objstruct:
3061            if not condition:
3062                code.putln("")  # start in new line
3063            code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000")
3064            code.putln('sizeof(%s),' % objstruct)
3065            code.putln("#else")
3066            code.putln('sizeof(%s),' % sizeof_objstruct)
3067            code.putln("#endif")
3068        else:
3069            code.put('sizeof(%s), ' % objstruct)
3070
3071        # check_size
3072        if type.check_size and type.check_size in ('error', 'warn', 'ignore'):
3073            check_size = type.check_size
3074        elif not type.is_external or type.is_subclassed:
3075            check_size = 'error'
3076        else:
3077            raise RuntimeError("invalid value for check_size '%s' when compiling %s.%s" % (
3078                type.check_size, module_name, type.name))
3079        code.putln('__Pyx_ImportType_CheckSize_%s);' % check_size.title())
3080
3081        code.putln(' if (!%s) %s' % (type.typeptr_cname, error_code))
3082
3083    def generate_type_ready_code(self, entry, code):
3084        Nodes.CClassDefNode.generate_type_ready_code(entry, code)
3085
3086    def generate_exttype_vtable_init_code(self, entry, code):
3087        # Generate code to initialise the C method table of an
3088        # extension type.
3089        type = entry.type
3090        if type.vtable_cname:
3091            code.putln(
3092                "%s = &%s;" % (
3093                    type.vtabptr_cname,
3094                    type.vtable_cname))
3095            if type.base_type and type.base_type.vtabptr_cname:
3096                code.putln(
3097                    "%s.%s = *%s;" % (
3098                        type.vtable_cname,
3099                        Naming.obj_base_cname,
3100                        type.base_type.vtabptr_cname))
3101
3102            c_method_entries = [
3103                entry for entry in type.scope.cfunc_entries
3104                if entry.func_cname]
3105            if c_method_entries:
3106                for meth_entry in c_method_entries:
3107                    cast = meth_entry.type.signature_cast_string()
3108                    code.putln(
3109                        "%s.%s = %s%s;" % (
3110                            type.vtable_cname,
3111                            meth_entry.cname,
3112                            cast,
3113                            meth_entry.func_cname))
3114
3115
3116class ModuleImportGenerator(object):
3117    """
3118    Helper to generate module import while importing external types.
3119    This is used to avoid excessive re-imports of external modules when multiple types are looked up.
3120    """
3121    def __init__(self, code, imported_modules=None):
3122        self.code = code
3123        self.imported = {}
3124        if imported_modules:
3125            for name, cname in imported_modules.items():
3126                self.imported['"%s"' % name] = cname
3127        self.temps = []  # remember original import order for freeing
3128
3129    def imported_module(self, module_name_string, error_code):
3130        if module_name_string in self.imported:
3131            return self.imported[module_name_string]
3132
3133        code = self.code
3134        temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
3135        self.temps.append(temp)
3136        code.putln('%s = PyImport_ImportModule(%s); if (unlikely(!%s)) %s' % (
3137            temp, module_name_string, temp, error_code))
3138        code.put_gotref(temp)
3139        self.imported[module_name_string] = temp
3140        return temp
3141
3142    def __enter__(self):
3143        return self
3144
3145    def __exit__(self, *exc):
3146        code = self.code
3147        for temp in self.temps:
3148            code.put_decref_clear(temp, py_object_type)
3149            code.funcstate.release_temp(temp)
3150
3151
3152def generate_cfunction_declaration(entry, env, code, definition):
3153    from_cy_utility = entry.used and entry.utility_code_definition
3154    if entry.used and entry.inline_func_in_pxd or (not entry.in_cinclude and (
3155            definition or entry.defined_in_pxd or entry.visibility == 'extern' or from_cy_utility)):
3156        if entry.visibility == 'extern':
3157            storage_class = Naming.extern_c_macro
3158            dll_linkage = "DL_IMPORT"
3159        elif entry.visibility == 'public':
3160            storage_class = Naming.extern_c_macro
3161            dll_linkage = None
3162        elif entry.visibility == 'private':
3163            storage_class = "static"
3164            dll_linkage = None
3165        else:
3166            storage_class = "static"
3167            dll_linkage = None
3168        type = entry.type
3169
3170        if entry.defined_in_pxd and not definition:
3171            storage_class = "static"
3172            dll_linkage = None
3173            type = CPtrType(type)
3174
3175        header = type.declaration_code(
3176            entry.cname, dll_linkage=dll_linkage)
3177        modifiers = code.build_function_modifiers(entry.func_modifiers)
3178        code.putln("%s %s%s; /*proto*/" % (
3179            storage_class,
3180            modifiers,
3181            header))
3182
3183#------------------------------------------------------------------------------------
3184#
3185#  Runtime support code
3186#
3187#------------------------------------------------------------------------------------
3188
3189refnanny_utility_code = UtilityCode.load("Refnanny", "ModuleSetupCode.c")
3190
3191packed_struct_utility_code = UtilityCode(proto="""
3192#if defined(__GNUC__)
3193#define __Pyx_PACKED __attribute__((__packed__))
3194#else
3195#define __Pyx_PACKED
3196#endif
3197""", impl="", proto_block='utility_code_proto_before_types')
3198
3199capsule_utility_code = UtilityCode.load("Capsule")
3200