Nodes.py 356 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes
William Stein's avatar
William Stein committed
3
#
4

5 6
from __future__ import absolute_import

7
import cython
8
cython.declare(sys=object, os=object, copy=object,
9
               Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
10
               py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
11
               StructOrUnionScope=object, PyClassScope=object,
12
               CppClassScope=object, UtilityCode=object, EncodedString=object,
13
               error_type=object, _py_int_types=object)
William Stein's avatar
William Stein committed
14

15
import sys, os, copy
Stefan Behnel's avatar
Stefan Behnel committed
16
from itertools import chain
Robert Bradshaw's avatar
Robert Bradshaw committed
17

18 19 20 21 22 23 24
from . import Builtin
from .Errors import error, warning, InternalError, CompileError
from . import Naming
from . import PyrexTypes
from . import TypeSlots
from .PyrexTypes import py_object_type, error_type
from .Symtab import (ModuleScope, LocalScope, ClosureScope,
25
                     StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope)
26
from .Code import UtilityCode
27
from .StringEncoding import EncodedString
28
from . import Future
29 30
from . import Options
from . import DebugFlags
31
from .Pythran import has_np_pythran, pythran_type, is_pythran_buffer
32
from ..Utils import add_metaclass
William Stein's avatar
William Stein committed
33

34

35 36 37 38 39
if sys.version_info[0] >= 3:
    _py_int_types = int
else:
    _py_int_types = (int, long)

40

41
def relative_position(pos):
42
    return (pos[0].get_filenametable_entry(), pos[1])
43

44 45 46 47

def embed_position(pos, docstring):
    if not Options.embed_pos_in_docstring:
        return docstring
48
    pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
49 50
    if docstring is None:
        # unicode string
51
        return EncodedString(pos_line)
52 53 54 55 56 57

    # make sure we can encode the filename in the docstring encoding
    # otherwise make the docstring a unicode string
    encoding = docstring.encoding
    if encoding is not None:
        try:
Stefan Behnel's avatar
Stefan Behnel committed
58
            pos_line.encode(encoding)
59 60 61 62 63
        except UnicodeEncodeError:
            encoding = None

    if not docstring:
        # reuse the string encoding of the original docstring
64
        doc = EncodedString(pos_line)
65
    else:
66
        doc = EncodedString(pos_line + u'\n' + docstring)
67 68
    doc.encoding = encoding
    return doc
69

70

71
def analyse_type_annotation(annotation, env):
72 73 74
    base_type = None
    explicit_pytype = explicit_ctype = False
    if annotation.is_dict_literal:
75 76
        warning(annotation.pos,
                "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.")
77 78 79 80 81 82 83 84 85 86 87 88 89
        for name, value in annotation.key_value_pairs:
            if not name.is_string_literal:
                continue
            if name.value in ('type', b'type'):
                explicit_pytype = True
                if not explicit_ctype:
                    annotation = value
            elif name.value in ('ctype', b'ctype'):
                explicit_ctype = True
                annotation = value
        if explicit_pytype and explicit_ctype:
            warning(annotation.pos, "Duplicate type declarations found in signature annotation")
    arg_type = annotation.analyse_as_type(env)
90 91 92 93 94 95 96
    if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'):
        # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations
        if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type):
            arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type
    elif arg_type is not None and annotation.is_string_literal:
        warning(annotation.pos,
                "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.")
97 98 99 100 101 102 103
    if arg_type is not None:
        if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
            warning(annotation.pos,
                    "Python type declaration in signature annotation does not refer to a Python type")
        base_type = CAnalysedBaseTypeNode(
            annotation.pos, type=arg_type, is_arg=True)
    else:
104
        warning(annotation.pos, "Unknown type declaration in annotation, ignoring")
105 106 107
    return base_type, arg_type


108
def write_func_call(func, codewriter_class):
109
    def f(*args, **kwds):
110
        if len(args) > 1 and isinstance(args[1], codewriter_class):
Robert Bradshaw's avatar
Robert Bradshaw committed
111 112
            # here we annotate the code with this function call
            # but only if new code is generated
113
            node, code = args[:2]
Robert Bradshaw's avatar
Robert Bradshaw committed
114
            marker = '                    /* %s -> %s.%s %s */' % (
Stefan Behnel's avatar
Stefan Behnel committed
115 116 117 118
                ' ' * code.call_level,
                node.__class__.__name__,
                func.__name__,
                node.pos[1:])
Robert Bradshaw's avatar
Robert Bradshaw committed
119 120 121 122 123 124 125
            pristine = code.buffer.stream.tell()
            code.putln(marker)
            start = code.buffer.stream.tell()
            code.call_level += 4
            res = func(*args, **kwds)
            code.call_level -= 4
            if start == code.buffer.stream.tell():
126 127
                # no code written => undo writing marker
                code.buffer.stream.truncate(pristine)
Robert Bradshaw's avatar
Robert Bradshaw committed
128
            else:
129
                marker = marker.replace('->', '<-', 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
130 131 132 133
                code.putln(marker)
            return res
        else:
            return func(*args, **kwds)
134 135
    return f

136

137 138
class VerboseCodeWriter(type):
    # Set this as a metaclass to trace function calls in code.
139
    # This slows down code generation and makes much larger files.
140
    def __new__(cls, name, bases, attrs):
141
        from types import FunctionType
142
        from .Code import CCodeWriter
143 144 145
        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType):
146
                attrs[mname] = write_func_call(m, CCodeWriter)
147 148 149
        return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)


150 151 152 153 154 155 156 157 158 159 160 161 162
class CheckAnalysers(type):
    """Metaclass to check that type analysis functions return a node.
    """
    methods = set(['analyse_types',
                   'analyse_expressions',
                   'analyse_target_types'])

    def __new__(cls, name, bases, attrs):
        from types import FunctionType
        def check(name, func):
            def call(*args, **kwargs):
                retval = func(*args, **kwargs)
                if retval is None:
163
                    print('%s %s %s' % (name, args, kwargs))
164 165 166 167 168 169 170 171 172 173
                return retval
            return call

        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType) and mname in cls.methods:
                attrs[mname] = check(mname, m)
        return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)


174 175 176 177 178 179 180 181
def _with_metaclass(cls):
    if DebugFlags.debug_trace_code_generation:
        return add_metaclass(VerboseCodeWriter)(cls)
    #return add_metaclass(CheckAnalysers)(cls)
    return cls


@_with_metaclass
Stefan Behnel's avatar
Stefan Behnel committed
182
class Node(object):
William Stein's avatar
William Stein committed
183 184 185
    #  pos         (string, int, int)   Source file position
    #  is_name     boolean              Is a NameNode
    #  is_literal  boolean              Is a ConstNode
186

William Stein's avatar
William Stein committed
187
    is_name = 0
188
    is_none = 0
189
    is_nonecheck = 0
William Stein's avatar
William Stein committed
190
    is_literal = 0
191
    is_terminator = 0
192
    is_wrapper = False  # is a DefNode wrapper for a C function
193
    temps = None
194

Stefan Behnel's avatar
Stefan Behnel committed
195
    # All descendants should set child_attrs to a list of the attributes
196 197
    # containing nodes considered "children" in the tree. Each such attribute
    # can either contain a single node or a list of nodes. See Visitor.py.
198
    child_attrs = None
199

200 201
    cf_state = None

202 203 204 205 206
    # This may be an additional (or 'actual') type that will be checked when
    # this node is coerced to another type. This could be useful to set when
    # the actual type to which it can coerce is known, but you want to leave
    # the type a py_object_type
    coercion_type = None
207

William Stein's avatar
William Stein committed
208 209 210
    def __init__(self, pos, **kw):
        self.pos = pos
        self.__dict__.update(kw)
211

212 213
    gil_message = "Operation"

214
    nogil_check = None
215

216
    def gil_error(self, env=None):
217
        error(self.pos, "%s not allowed without gil" % self.gil_message)
218

Robert Bradshaw's avatar
Robert Bradshaw committed
219
    cpp_message = "Operation"
220

Robert Bradshaw's avatar
Robert Bradshaw committed
221 222 223 224 225 226
    def cpp_check(self, env):
        if not env.is_cpp():
            self.cpp_error()

    def cpp_error(self):
        error(self.pos, "%s only allowed in c++" % self.cpp_message)
227

228 229 230 231
    def clone_node(self):
        """Clone the node. This is defined as a shallow copy, except for member lists
           amongst the child attributes (from get_child_accessors) which are also
           copied. Lists containing child nodes are thus seen as a way for the node
Stefan Behnel's avatar
Stefan Behnel committed
232
           to hold multiple children directly; the list is not treated as a separate
233
           level in the tree."""
234 235 236
        result = copy.copy(self)
        for attrname in result.child_attrs:
            value = getattr(result, attrname)
237
            if isinstance(value, list):
238
                setattr(result, attrname, [x for x in value])
239
        return result
240 241


William Stein's avatar
William Stein committed
242
    #
243
    #  There are 3 phases of parse tree processing, applied in order to
William Stein's avatar
William Stein committed
244 245
    #  all the statements in a given scope-block:
    #
246
    #  (0) analyse_declarations
William Stein's avatar
William Stein committed
247 248 249 250
    #        Make symbol table entries for all declarations at the current
    #        level, both explicit (def, cdef, etc.) and implicit (assignment
    #        to an otherwise undeclared name).
    #
251
    #  (1) analyse_expressions
William Stein's avatar
William Stein committed
252 253
    #         Determine the result types of expressions and fill in the
    #         'type' attribute of each ExprNode. Insert coercion nodes into the
254
    #         tree where needed to convert to and from Python objects.
William Stein's avatar
William Stein committed
255 256 257 258
    #         Allocate temporary locals for intermediate results. Fill
    #         in the 'result_code' attribute of each ExprNode with a C code
    #         fragment.
    #
259
    #  (2) generate_code
William Stein's avatar
William Stein committed
260 261 262 263
    #         Emit C code for all declarations, statements and expressions.
    #         Recursively applies the 3 processing phases to the bodies of
    #         functions.
    #
264

William Stein's avatar
William Stein committed
265 266
    def analyse_declarations(self, env):
        pass
267

William Stein's avatar
William Stein committed
268 269 270
    def analyse_expressions(self, env):
        raise InternalError("analyse_expressions not implemented for %s" % \
            self.__class__.__name__)
271

William Stein's avatar
William Stein committed
272 273 274
    def generate_code(self, code):
        raise InternalError("generate_code not implemented for %s" % \
            self.__class__.__name__)
275

276 277 278 279
    def annotate(self, code):
        # mro does the wrong thing
        if isinstance(self, BlockNode):
            self.body.annotate(code)
280

281 282 283 284
    def end_pos(self):
        try:
            return self._end_pos
        except AttributeError:
Stefan Behnel's avatar
Stefan Behnel committed
285
            pos = self.pos
286 287 288
            if not self.child_attrs:
                self._end_pos = pos
                return pos
289
            for attr in self.child_attrs:
290
                child = getattr(self, attr)
291
                # Sometimes lists, sometimes nodes
292 293 294
                if child is None:
                    pass
                elif isinstance(child, list):
Stefan Behnel's avatar
Stefan Behnel committed
295 296
                    for c in child:
                        pos = max(pos, c.end_pos())
297
                else:
Stefan Behnel's avatar
Stefan Behnel committed
298 299 300
                    pos = max(pos, child.end_pos())
            self._end_pos = pos
            return pos
William Stein's avatar
William Stein committed
301

302
    def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
303 304
        """Debug helper method that returns a recursive string representation of this node.
        """
305 306
        if cutoff == 0:
            return "<...nesting level cutoff...>"
307 308 309
        if encountered is None:
            encountered = set()
        if id(self) in encountered:
310
            return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
311
        encountered.add(id(self))
312

313 314
        def dump_child(x, level):
            if isinstance(x, Node):
315
                return x.dump(level, filter_out, cutoff-1, encountered)
316
            elif isinstance(x, list):
Robert Bradshaw's avatar
Robert Bradshaw committed
317
                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
318 319
            else:
                return repr(x)
320

321
        attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
322
        if len(attrs) == 0:
323
            return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
324 325
        else:
            indent = "  " * level
326
            res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
327 328 329 330
            for key, value in attrs:
                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
            res += "%s>" % indent
            return res
331

332 333 334 335 336 337
    def dump_pos(self, mark_column=False, marker='(#)'):
        """Debug helper method that returns the source code context of this node as a string.
        """
        if not self.pos:
            return u''
        source_desc, line, col = self.pos
338
        contents = source_desc.get_lines(encoding='ASCII', error_handling='ignore')
339
        # line numbers start at 1
340
        lines = contents[max(0, line-3):line]
341 342 343 344 345 346 347 348
        current = lines[-1]
        if mark_column:
            current = current[:col] + marker + current[col:]
        lines[-1] = current.rstrip() + u'             # <<<<<<<<<<<<<<\n'
        lines += contents[line:line+2]
        return u'"%s":%d:%d\n%s\n' % (
            source_desc.get_escaped_description(), line, col, u''.join(lines))

349 350 351 352 353 354 355 356 357 358 359 360 361 362
class CompilerDirectivesNode(Node):
    """
    Sets compiler directives for the children nodes
    """
    #  directives     {string:value}  A dictionary holding the right value for
    #                                 *all* possible directives.
    #  body           Node
    child_attrs = ["body"]

    def analyse_declarations(self, env):
        old = env.directives
        env.directives = self.directives
        self.body.analyse_declarations(env)
        env.directives = old
363

364 365 366
    def analyse_expressions(self, env):
        old = env.directives
        env.directives = self.directives
367
        self.body = self.body.analyse_expressions(env)
368
        env.directives = old
369
        return self
370 371 372 373 374 375 376 377

    def generate_function_definitions(self, env, code):
        env_old = env.directives
        code_old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_function_definitions(env, code)
        env.directives = env_old
        code.globalstate.directives = code_old
378

379 380 381 382 383
    def generate_execution_code(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_execution_code(code)
        code.globalstate.directives = old
384

385 386 387 388 389
    def annotate(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.annotate(code)
        code.globalstate.directives = old
390

Stefan Behnel's avatar
Stefan Behnel committed
391
class BlockNode(object):
William Stein's avatar
William Stein committed
392 393
    #  Mixin class for nodes representing a declaration block.

394
    def generate_cached_builtins_decls(self, env, code):
395
        entries = env.global_scope().undeclared_cached_builtins
396
        for entry in entries:
397
            code.globalstate.add_cached_builtin_decl(entry)
398
        del entries[:]
399 400 401 402

    def generate_lambda_definitions(self, env, code):
        for node in env.lambda_defs:
            node.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
403 404 405

class StatListNode(Node):
    # stats     a list of StatNode
406

407
    child_attrs = ["stats"]
408

409
    @staticmethod
410 411
    def create_analysed(pos, env, *args, **kw):
        node = StatListNode(pos, *args, **kw)
412
        return node  # No node-specific analysis needed
413

William Stein's avatar
William Stein committed
414 415
    def analyse_declarations(self, env):
        #print "StatListNode.analyse_declarations" ###
416
        for stat in self.stats:
William Stein's avatar
William Stein committed
417
            stat.analyse_declarations(env)
418

William Stein's avatar
William Stein committed
419 420
    def analyse_expressions(self, env):
        #print "StatListNode.analyse_expressions" ###
421 422
        self.stats = [stat.analyse_expressions(env)
                      for stat in self.stats]
423
        return self
424

425
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
426 427
        #print "StatListNode.generate_function_definitions" ###
        for stat in self.stats:
428
            stat.generate_function_definitions(env, code)
429

William Stein's avatar
William Stein committed
430 431 432 433 434
    def generate_execution_code(self, code):
        #print "StatListNode.generate_execution_code" ###
        for stat in self.stats:
            code.mark_pos(stat.pos)
            stat.generate_execution_code(code)
435

436 437 438
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)
439

William Stein's avatar
William Stein committed
440 441 442 443 444 445 446 447 448 449 450 451 452

class StatNode(Node):
    #
    #  Code generation for statements is split into the following subphases:
    #
    #  (1) generate_function_definitions
    #        Emit C code for the definitions of any structs,
    #        unions, enums and functions defined in the current
    #        scope-block.
    #
    #  (2) generate_execution_code
    #        Emit C code for executable statements.
    #
453

454
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
455
        pass
456

William Stein's avatar
William Stein committed
457 458 459 460 461 462 463 464
    def generate_execution_code(self, code):
        raise InternalError("generate_execution_code not implemented for %s" % \
            self.__class__.__name__)


class CDefExternNode(StatNode):
    #  include_file   string or None
    #  body           StatNode
465

466
    child_attrs = ["body"]
467

William Stein's avatar
William Stein committed
468 469 470 471 472 473 474
    def analyse_declarations(self, env):
        if self.include_file:
            env.add_include_file(self.include_file)
        old_cinclude_flag = env.in_cinclude
        env.in_cinclude = 1
        self.body.analyse_declarations(env)
        env.in_cinclude = old_cinclude_flag
475

William Stein's avatar
William Stein committed
476
    def analyse_expressions(self, env):
477
        return self
478

William Stein's avatar
William Stein committed
479 480
    def generate_execution_code(self, code):
        pass
481 482 483

    def annotate(self, code):
        self.body.annotate(code)
484

William Stein's avatar
William Stein committed
485 486 487 488 489 490 491 492

class CDeclaratorNode(Node):
    # Part of a C declaration.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #      Returns (name, type) pair where name is the
493
    #      CNameDeclaratorNode of the name being declared
William Stein's avatar
William Stein committed
494 495
    #      and type is the type it is being declared as.
    #
496
    #  calling_convention  string   Calling convention of CFuncDeclaratorNode
497
    #                               for which this is a base
498

499 500
    child_attrs = []

501 502
    calling_convention = ""

503 504 505
    def analyse_templates(self):
        # Only C++ functions have templates.
        return None
William Stein's avatar
William Stein committed
506

507

William Stein's avatar
William Stein committed
508
class CNameDeclaratorNode(CDeclaratorNode):
509
    #  name    string             The Cython name being declared
Robert Bradshaw's avatar
Robert Bradshaw committed
510 511
    #  cname   string or None     C name, if specified
    #  default ExprNode or None   the value assigned on declaration
512

Robert Bradshaw's avatar
Robert Bradshaw committed
513
    child_attrs = ['default']
514

515
    default = None
516

517
    def analyse(self, base_type, env, nonempty=0):
518
        if nonempty and self.name == '':
519
            # May have mistaken the name for the type.
520
            if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
521
                error(self.pos, "Missing argument name")
522 523
            elif base_type.is_void:
                error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
524 525 526
            else:
                self.name = base_type.declaration_code("", for_display=1, pyrex=1)
                base_type = py_object_type
527 528 529 530

        if base_type.is_fused and env.fused_to_specific:
            base_type = base_type.specialize(env.fused_to_specific)

531
        self.type = base_type
William Stein's avatar
William Stein committed
532
        return self, base_type
533

534

William Stein's avatar
William Stein committed
535 536
class CPtrDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode
537

538 539
    child_attrs = ["base"]

540 541 542
    def analyse_templates(self):
        return self.base.analyse_templates()

543
    def analyse(self, base_type, env, nonempty=0):
William Stein's avatar
William Stein committed
544
        if base_type.is_pyobject:
545
            error(self.pos, "Pointer base type cannot be a Python object")
William Stein's avatar
William Stein committed
546
        ptr_type = PyrexTypes.c_ptr_type(base_type)
547 548
        return self.base.analyse(ptr_type, env, nonempty=nonempty)

Danilo Freitas's avatar
Danilo Freitas committed
549 550 551 552 553 554

class CReferenceDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

555 556 557
    def analyse_templates(self):
        return self.base.analyse_templates()

558
    def analyse(self, base_type, env, nonempty=0):
Danilo Freitas's avatar
Danilo Freitas committed
559
        if base_type.is_pyobject:
560
            error(self.pos, "Reference base type cannot be a Python object")
561
        ref_type = PyrexTypes.c_ref_type(base_type)
562 563
        return self.base.analyse(ref_type, env, nonempty=nonempty)

Danilo Freitas's avatar
Danilo Freitas committed
564

William Stein's avatar
William Stein committed
565 566 567
class CArrayDeclaratorNode(CDeclaratorNode):
    # base        CDeclaratorNode
    # dimension   ExprNode
568 569

    child_attrs = ["base", "dimension"]
570

571
    def analyse(self, base_type, env, nonempty=0):
572
        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
573
            from .ExprNodes import TupleNode
Robert Bradshaw's avatar
Robert Bradshaw committed
574 575 576 577 578 579 580
            if isinstance(self.dimension, TupleNode):
                args = self.dimension.args
            else:
                args = self.dimension,
            values = [v.analyse_as_type(env) for v in args]
            if None in values:
                ix = values.index(None)
581 582 583 584
                error(args[ix].pos, "Template parameter not a type")
                base_type = error_type
            else:
                base_type = base_type.specialize_here(self.pos, values)
585
            return self.base.analyse(base_type, env, nonempty=nonempty)
William Stein's avatar
William Stein committed
586
        if self.dimension:
587
            self.dimension = self.dimension.analyse_const_expression(env)
William Stein's avatar
William Stein committed
588 589
            if not self.dimension.type.is_int:
                error(self.dimension.pos, "Array dimension not integer")
590 591 592 593 594 595 596
            size = self.dimension.get_constant_c_result_code()
            if size is not None:
                try:
                    size = int(size)
                except ValueError:
                    # runtime constant?
                    pass
William Stein's avatar
William Stein committed
597 598 599
        else:
            size = None
        if not base_type.is_complete():
600
            error(self.pos, "Array element type '%s' is incomplete" % base_type)
William Stein's avatar
William Stein committed
601
        if base_type.is_pyobject:
602
            error(self.pos, "Array element cannot be a Python object")
603
        if base_type.is_cfunction:
604
            error(self.pos, "Array element cannot be a function")
William Stein's avatar
William Stein committed
605
        array_type = PyrexTypes.c_array_type(base_type, size)
606
        return self.base.analyse(array_type, env, nonempty=nonempty)
William Stein's avatar
William Stein committed
607 608 609 610 611


class CFuncDeclaratorNode(CDeclaratorNode):
    # base             CDeclaratorNode
    # args             [CArgDeclNode]
612
    # templates        [TemplatePlaceholderType]
William Stein's avatar
William Stein committed
613 614 615
    # has_varargs      boolean
    # exception_value  ConstNode
    # exception_check  boolean    True if PyErr_Occurred check needed
616 617
    # nogil            boolean    Can be called without gil
    # with_gil         boolean    Acquire gil around function body
618
    # is_const_method  boolean    Whether this is a const method
619

620 621
    child_attrs = ["base", "args", "exception_value"]

622
    overridable = 0
623
    optional_arg_count = 0
624
    is_const_method = 0
625 626 627 628
    templates = None

    def analyse_templates(self):
        if isinstance(self.base, CArrayDeclaratorNode):
629
            from .ExprNodes import TupleNode, NameNode
630 631 632 633 634 635 636
            template_node = self.base.dimension
            if isinstance(template_node, TupleNode):
                template_nodes = template_node.args
            elif isinstance(template_node, NameNode):
                template_nodes = [template_node]
            else:
                error(template_node.pos, "Template arguments must be a list of names")
637
                return None
638 639 640 641 642 643 644 645 646 647
            self.templates = []
            for template in template_nodes:
                if isinstance(template, NameNode):
                    self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
                else:
                    error(template.pos, "Template arguments must be a list of names")
            self.base = self.base.base
            return self.templates
        else:
            return None
William Stein's avatar
William Stein committed
648

649 650 651
    def analyse(self, return_type, env, nonempty=0, directive_locals=None):
        if directive_locals is None:
            directive_locals = {}
652 653
        if nonempty:
            nonempty -= 1
William Stein's avatar
William Stein committed
654
        func_type_args = []
655
        for i, arg_node in enumerate(self.args):
656
            name_declarator, type = arg_node.analyse(
657 658
                env, nonempty=nonempty,
                is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
William Stein's avatar
William Stein committed
659
            name = name_declarator.name
660 661 662 663 664
            if name in directive_locals:
                type_node = directive_locals[name]
                other_type = type_node.analyse_as_type(env)
                if other_type is None:
                    error(type_node.pos, "Not a type")
Robert Bradshaw's avatar
Robert Bradshaw committed
665
                elif (type is not PyrexTypes.py_object_type
666 667 668 669 670
                      and not type.same_as(other_type)):
                    error(self.base.pos, "Signature does not agree with previous declaration")
                    error(type_node.pos, "Previous declaration here")
                else:
                    type = other_type
William Stein's avatar
William Stein committed
671
            if name_declarator.cname:
672 673
                error(self.pos, "Function argument cannot have C name specification")
            if i == 0 and env.is_c_class_scope and type.is_unspecified:
674 675
                # fix the type of self
                type = env.parent_type
William Stein's avatar
William Stein committed
676 677 678 679 680
            # Turn *[] argument into **
            if type.is_array:
                type = PyrexTypes.c_ptr_type(type.base_type)
            # Catch attempted C-style func(void) decl
            if type.is_void:
Robert Bradshaw's avatar
Robert Bradshaw committed
681
                error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
William Stein's avatar
William Stein committed
682 683 684
            func_type_args.append(
                PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
            if arg_node.default:
685
                self.optional_arg_count += 1
686 687
            elif self.optional_arg_count:
                error(self.pos, "Non-default argument follows default argument")
688

William Stein's avatar
William Stein committed
689 690
        exc_val = None
        exc_check = 0
691
        if self.exception_check == '+':
692
            env.add_include_file('ios')         # for std::ios_base::failure
693
            env.add_include_file('new')         # for std::bad_alloc
694
            env.add_include_file('stdexcept')
695
            env.add_include_file('typeinfo')    # for std::bad_cast
696 697 698
        if (return_type.is_pyobject
                and (self.exception_value or self.exception_check)
                and self.exception_check != '+'):
699
            error(self.pos, "Exception clause not allowed for function returning Python object")
William Stein's avatar
William Stein committed
700 701
        else:
            if self.exception_value:
702
                self.exception_value = self.exception_value.analyse_const_expression(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
703 704
                if self.exception_check == '+':
                    exc_val_type = self.exception_value.type
705 706 707 708 709
                    if (not exc_val_type.is_error
                            and not exc_val_type.is_pyobject
                            and not (exc_val_type.is_cfunction
                                     and not exc_val_type.return_type.is_pyobject
                                     and not exc_val_type.args)):
Robert Bradshaw's avatar
Robert Bradshaw committed
710
                        error(self.exception_value.pos,
711
                              "Exception value must be a Python exception or cdef function with no arguments.")
712
                    exc_val = self.exception_value
Robert Bradshaw's avatar
Robert Bradshaw committed
713
                else:
714 715 716 717 718 719 720 721 722 723
                    self.exception_value = self.exception_value.coerce_to(
                        return_type, env).analyse_const_expression(env)
                    exc_val = self.exception_value.get_constant_c_result_code()
                    if exc_val is None:
                        raise InternalError(
                            "get_constant_c_result_code not implemented for %s" %
                            self.exception_value.__class__.__name__)
                    if not return_type.assignable_from(self.exception_value.type):
                        error(self.exception_value.pos,
                              "Exception value incompatible with function return type")
William Stein's avatar
William Stein committed
724
            exc_check = self.exception_check
725
        if return_type.is_cfunction:
726
            error(self.pos, "Function cannot return a function")
William Stein's avatar
William Stein committed
727
        func_type = PyrexTypes.CFuncType(
728
            return_type, func_type_args, self.has_varargs,
729 730 731 732 733 734
            optional_arg_count=self.optional_arg_count,
            exception_value=exc_val, exception_check=exc_check,
            calling_convention=self.base.calling_convention,
            nogil=self.nogil, with_gil=self.with_gil, is_overridable=self.overridable,
            is_const_method=self.is_const_method,
            templates=self.templates)
735

736
        if self.optional_arg_count:
737 738 739 740 741 742 743 744 745 746 747
            if func_type.is_fused:
                # This is a bit of a hack... When we need to create specialized CFuncTypes
                # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
                # struct
                def declare_opt_arg_struct(func_type, fused_cname):
                    self.declare_optional_arg_struct(func_type, env, fused_cname)

                func_type.declare_opt_arg_struct = declare_opt_arg_struct
            else:
                self.declare_optional_arg_struct(func_type, env)

748 749 750 751 752 753 754
        callspec = env.directives['callspec']
        if callspec:
            current = func_type.calling_convention
            if current and current != callspec:
                error(self.pos, "cannot have both '%s' and '%s' "
                      "calling conventions" % (current, callspec))
            func_type.calling_convention = callspec
William Stein's avatar
William Stein committed
755 756
        return self.base.analyse(func_type, env)

757 758 759 760 761 762 763 764 765 766 767
    def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
        """
        Declares the optional argument struct (the struct used to hold the
        values for optional arguments). For fused cdef functions, this is
        deferred as analyse_declarations is called only once (on the fused
        cdef function).
        """
        scope = StructOrUnionScope()
        arg_count_member = '%sn' % Naming.pyrex_prefix
        scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)

768 769
        for arg in func_type.args[len(func_type.args) - self.optional_arg_count:]:
            scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject=1)
770 771 772 773 774 775 776

        struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)

        if fused_cname is not None:
            struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)

        op_args_struct = env.global_scope().declare_struct_or_union(
777 778 779 780 781 782
            name=struct_cname,
            kind='struct',
            scope=scope,
            typedef_flag=0,
            pos=self.pos,
            cname=struct_cname)
783 784 785 786 787 788

        op_args_struct.defined_in_pxd = 1
        op_args_struct.used = 1

        func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)

William Stein's avatar
William Stein committed
789

790 791 792 793 794
class CConstDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

795
    def analyse(self, base_type, env, nonempty=0):
796 797 798 799
        if base_type.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        const = PyrexTypes.c_const_type(base_type)
800
        return self.base.analyse(const, env, nonempty=nonempty)
801 802


William Stein's avatar
William Stein committed
803 804 805 806 807 808
class CArgDeclNode(Node):
    # Item in a function declaration argument list.
    #
    # base_type      CBaseTypeNode
    # declarator     CDeclaratorNode
    # not_none       boolean            Tagged with 'not None'
809 810
    # or_none        boolean            Tagged with 'or None'
    # accept_none    boolean            Resolved boolean for not_none/or_none
William Stein's avatar
William Stein committed
811
    # default        ExprNode or None
812
    # default_value  PyObjectConst      constant for default value
813
    # annotation     ExprNode or None   Py3 function arg annotation
William Stein's avatar
William Stein committed
814
    # is_self_arg    boolean            Is the "self" arg of an extension type method
815
    # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
816
    # is_kw_only     boolean            Is a keyword-only argument
817
    # is_dynamic     boolean            Non-literal arg stored inside CyFunction
818

819
    child_attrs = ["base_type", "declarator", "default", "annotation"]
820

William Stein's avatar
William Stein committed
821
    is_self_arg = 0
822
    is_type_arg = 0
823
    is_generic = 1
824 825 826
    kw_only = 0
    not_none = 0
    or_none = 0
827 828
    type = None
    name_declarator = None
829
    default_value = None
830
    annotation = None
831
    is_dynamic = 0
832

833
    def analyse(self, env, nonempty=0, is_self_arg=False):
834 835
        if is_self_arg:
            self.base_type.is_self_arg = self.is_self_arg = True
836
        if self.type is None:
Stefan Behnel's avatar
Stefan Behnel committed
837
            # The parser may misinterpret names as types. We fix that here.
838 839
            if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
                if nonempty:
840 841
                    if self.base_type.is_basic_c_type:
                        # char, short, long called "int"
842
                        type = self.base_type.analyse(env, could_be_name=True)
843
                        arg_name = type.empty_declaration_code()
844 845 846
                    else:
                        arg_name = self.base_type.name
                    self.declarator.name = EncodedString(arg_name)
847 848 849 850 851
                    self.base_type.name = None
                    self.base_type.is_basic_c_type = False
                could_be_name = True
            else:
                could_be_name = False
852
            self.base_type.is_arg = True
853
            base_type = self.base_type.analyse(env, could_be_name=could_be_name)
Robert Bradshaw's avatar
Robert Bradshaw committed
854
            if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name:
855
                self.declarator.name = self.base_type.arg_name
856

857 858
            # The parser is unable to resolve the ambiguity of [] as part of the
            # type (e.g. in buffers) or empty declarator (as with arrays).
859
            # This is only arises for empty multi-dimensional arrays.
860 861
            if (base_type.is_array
                    and isinstance(self.base_type, TemplatedTypeNode)
862 863 864 865 866 867
                    and isinstance(self.declarator, CArrayDeclaratorNode)):
                declarator = self.declarator
                while isinstance(declarator.base, CArrayDeclaratorNode):
                    declarator = declarator.base
                declarator.base = self.base_type.array_declarator
                base_type = base_type.base_type
868 869

            # inject type declaration from annotations
870 871
            # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis
            if self.annotation and env and env.directives['annotation_typing'] and self.base_type.name is None:
872 873 874 875
                arg_type = self.inject_type_from_annotations(env)
                if arg_type is not None:
                    base_type = arg_type
            return self.declarator.analyse(base_type, env, nonempty=nonempty)
876
        else:
877
            return self.name_declarator, self.type
William Stein's avatar
William Stein committed
878

879 880 881
    def inject_type_from_annotations(self, env):
        annotation = self.annotation
        if not annotation:
882
            return None
883
        base_type, arg_type = analyse_type_annotation(annotation, env)
884 885
        if base_type is not None:
            self.base_type = base_type
886 887
        return arg_type

888 889 890 891 892 893 894 895 896
    def calculate_default_value_code(self, code):
        if self.default_value is None:
            if self.default:
                if self.default.is_literal:
                    # will not output any code, just assign the result_code
                    self.default.generate_evaluation_code(code)
                    return self.type.cast_code(self.default.result())
                self.default_value = code.get_argument_default_const(self.type)
        return self.default_value
897

898 899 900 901
    def annotate(self, code):
        if self.default:
            self.default.annotate(code)

902
    def generate_assignment_code(self, code, target=None, overloaded_assignment=False):
903 904 905 906 907 908 909
        default = self.default
        if default is None or default.is_literal:
            return
        if target is None:
            target = self.calculate_default_value_code(code)
        default.generate_evaluation_code(code)
        default.make_owned_reference(code)
910
        result = default.result() if overloaded_assignment else default.result_as(self.type)
911 912 913 914 915 916
        code.putln("%s = %s;" % (target, result))
        if self.type.is_pyobject:
            code.put_giveref(default.result())
        default.generate_post_assignment_code(code)
        default.free_temps(code)

William Stein's avatar
William Stein committed
917 918 919 920 921 922 923 924

class CBaseTypeNode(Node):
    # Abstract base class for C base type nodes.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #     Returns the type.
925

926 927
    def analyse_as_type(self, env):
        return self.analyse(env)
928

Stefan Behnel's avatar
Stefan Behnel committed
929

930 931
class CAnalysedBaseTypeNode(Node):
    # type            type
932

933
    child_attrs = []
934

935
    def analyse(self, env, could_be_name=False):
936
        return self.type
William Stein's avatar
William Stein committed
937

Stefan Behnel's avatar
Stefan Behnel committed
938

William Stein's avatar
William Stein committed
939 940 941 942 943 944
class CSimpleBaseTypeNode(CBaseTypeNode):
    # name             string
    # module_path      [string]     Qualifying name components
    # is_basic_c_type  boolean
    # signed           boolean
    # longness         integer
945
    # complex          boolean
William Stein's avatar
William Stein committed
946
    # is_self_arg      boolean      Is self argument of C method
947
    # ##is_type_arg      boolean      Is type argument of class method
William Stein's avatar
William Stein committed
948

949
    child_attrs = []
950
    arg_name = None   # in case the argument name was interpreted as a type
951 952 953
    module_path = []
    is_basic_c_type = False
    complex = False
954

955
    def analyse(self, env, could_be_name=False):
William Stein's avatar
William Stein committed
956
        # Return type descriptor.
957
        #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
William Stein's avatar
William Stein committed
958 959 960 961 962 963 964 965 966
        type = None
        if self.is_basic_c_type:
            type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
            if not type:
                error(self.pos, "Unrecognised type modifier combination")
        elif self.name == "object" and not self.module_path:
            type = py_object_type
        elif self.name is None:
            if self.is_self_arg and env.is_c_class_scope:
967
                #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
William Stein's avatar
William Stein committed
968
                type = env.parent_type
969 970
            ## elif self.is_type_arg and env.is_c_class_scope:
            ##     type = Builtin.type_type
William Stein's avatar
William Stein committed
971 972 973
            else:
                type = py_object_type
        else:
974
            if self.module_path:
975 976 977 978
                # Maybe it's a nested C++ class.
                scope = env
                for item in self.module_path:
                    entry = scope.lookup(item)
Robert Bradshaw's avatar
Robert Bradshaw committed
979
                    if entry is not None and entry.is_cpp_class:
980 981 982 983
                        scope = entry.type.scope
                    else:
                        scope = None
                        break
984

985 986 987 988 989
                if scope is None:
                    # Maybe it's a cimport.
                    scope = env.find_imported_module(self.module_path, self.pos)
                    if scope:
                        scope.fused_to_specific = env.fused_to_specific
990 991
            else:
                scope = env
992

William Stein's avatar
William Stein committed
993
            if scope:
994 995
                if scope.is_c_class_scope:
                    scope = scope.global_scope()
996 997 998 999

                type = scope.lookup_type(self.name)
                if type is not None:
                    pass
1000 1001 1002
                elif could_be_name:
                    if self.is_self_arg and env.is_c_class_scope:
                        type = env.parent_type
1003 1004
                    ## elif self.is_type_arg and env.is_c_class_scope:
                    ##     type = Builtin.type_type
1005 1006
                    else:
                        type = py_object_type
1007
                    self.arg_name = EncodedString(self.name)
William Stein's avatar
William Stein committed
1008
                else:
Danilo Freitas's avatar
Danilo Freitas committed
1009 1010 1011
                    if self.templates:
                        if not self.name in self.templates:
                            error(self.pos, "'%s' is not a type identifier" % self.name)
1012
                        type = PyrexTypes.TemplatePlaceholderType(self.name)
Danilo Freitas's avatar
Danilo Freitas committed
1013 1014
                    else:
                        error(self.pos, "'%s' is not a type identifier" % self.name)
1015
        if self.complex:
1016 1017 1018
            if not type.is_numeric or type.is_complex:
                error(self.pos, "can only complexify c numeric types")
            type = PyrexTypes.CComplexType(type)
1019
            type.create_declaration_utility_code(env)
1020 1021 1022 1023 1024 1025 1026 1027
        elif type is Builtin.complex_type:
            # Special case: optimise builtin complex type into C's
            # double complex.  The parser cannot do this (as for the
            # normal scalar types) as the user may have redeclared the
            # 'complex' type.  Testing for the exact type here works.
            type = PyrexTypes.c_double_complex_type
            type.create_declaration_utility_code(env)
            self.complex = True
William Stein's avatar
William Stein committed
1028 1029 1030 1031 1032
        if type:
            return type
        else:
            return PyrexTypes.error_type

1033
class MemoryViewSliceTypeNode(CBaseTypeNode):
1034

1035
    name = 'memoryview'
1036 1037
    child_attrs = ['base_type_node', 'axes']

1038
    def analyse(self, env, could_be_name=False):
1039 1040 1041 1042

        base_type = self.base_type_node.analyse(env)
        if base_type.is_error: return base_type

1043
        from . import MemoryView
1044 1045 1046

        try:
            axes_specs = MemoryView.get_axes_specs(env, self.axes)
1047
        except CompileError as e:
1048 1049 1050 1051
            error(e.position, e.message_only)
            self.type = PyrexTypes.ErrorType()
            return self.type

1052 1053 1054 1055
        if not MemoryView.validate_axes(self.pos, axes_specs):
            self.type = error_type
        else:
            self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
1056
            self.type.validate_memslice_dtype(self.pos)
1057
            self.use_memview_utilities(env)
1058

1059
        return self.type
1060

1061
    def use_memview_utilities(self, env):
1062
        from . import MemoryView
1063 1064 1065
        env.use_utility_code(MemoryView.view_utility_code)


Robert Bradshaw's avatar
Robert Bradshaw committed
1066
class CNestedBaseTypeNode(CBaseTypeNode):
1067
    # For C++ classes that live inside other C++ classes.
Robert Bradshaw's avatar
Robert Bradshaw committed
1068 1069 1070

    # name             string
    # base_type        CBaseTypeNode
Stefan Behnel's avatar
Stefan Behnel committed
1071

Robert Bradshaw's avatar
Robert Bradshaw committed
1072
    child_attrs = ['base_type']
Stefan Behnel's avatar
Stefan Behnel committed
1073

1074
    def analyse(self, env, could_be_name=None):
Robert Bradshaw's avatar
Robert Bradshaw committed
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
        base_type = self.base_type.analyse(env)
        if base_type is PyrexTypes.error_type:
            return PyrexTypes.error_type
        if not base_type.is_cpp_class:
            error(self.pos, "'%s' is not a valid type scope" % base_type)
            return PyrexTypes.error_type
        type_entry = base_type.scope.lookup_here(self.name)
        if not type_entry or not type_entry.is_type:
            error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
            return PyrexTypes.error_type
        return type_entry.type

1087

Danilo Freitas's avatar
Danilo Freitas committed
1088
class TemplatedTypeNode(CBaseTypeNode):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1089
    #  After parsing:
1090 1091
    #  positional_args  [ExprNode]        List of positional arguments
    #  keyword_args     DictNode          Keyword arguments
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1092 1093 1094
    #  base_type_node   CBaseTypeNode

    #  After analysis:
1095
    #  type             PyrexTypes.BufferType or PyrexTypes.CppClassType  ...containing the right options
1096 1097 1098

    child_attrs = ["base_type_node", "positional_args",
                   "keyword_args", "dtype_node"]
1099

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1100
    dtype_node = None
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1101 1102

    name = None
1103

1104
    def analyse(self, env, could_be_name=False, base_type=None):
1105 1106
        if base_type is None:
            base_type = self.base_type_node.analyse(env)
1107
        if base_type.is_error: return base_type
1108

1109
        if base_type.is_cpp_class and base_type.is_template_type():
1110
            # Templated class
1111
            if self.keyword_args and self.keyword_args.key_value_pairs:
Stefan Behnel's avatar
Stefan Behnel committed
1112
                error(self.pos, "c++ templates cannot take keyword arguments")
1113 1114 1115 1116
                self.type = PyrexTypes.error_type
            else:
                template_types = []
                for template_node in self.positional_args:
1117 1118 1119 1120 1121
                    type = template_node.analyse_as_type(env)
                    if type is None:
                        error(template_node.pos, "unknown type in template argument")
                        return error_type
                    template_types.append(type)
1122
                self.type = base_type.specialize_here(self.pos, template_types)
1123

1124 1125
        elif base_type.is_pyobject:
            # Buffer
1126
            from . import Buffer
1127

1128 1129 1130 1131 1132 1133
            options = Buffer.analyse_buffer_options(
                self.pos,
                env,
                self.positional_args,
                self.keyword_args,
                base_type.buffer_defaults)
1134

Robert Bradshaw's avatar
Robert Bradshaw committed
1135 1136
            if sys.version_info[0] < 3:
                # Py 2.x enforces byte strings as keyword arguments ...
1137 1138
                options = dict([(name.encode('ASCII'), value)
                                for name, value in options.items()])
Stefan Behnel's avatar
Stefan Behnel committed
1139

1140
            self.type = PyrexTypes.BufferType(base_type, **options)
1141 1142
            if has_np_pythran(env) and is_pythran_buffer(self.type):
                self.type = PyrexTypes.PythranExpr(pythran_type(self.type), self.type)
1143

1144 1145
        else:
            # Array
1146
            empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
1147 1148 1149 1150
            if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
                error(self.pos, "invalid array declaration")
                self.type = PyrexTypes.error_type
            else:
1151
                # It would be nice to merge this class with CArrayDeclaratorNode,
1152 1153 1154 1155 1156
                # but arrays are part of the declaration, not the type...
                if not self.positional_args:
                    dimension = None
                else:
                    dimension = self.positional_args[0]
1157 1158 1159 1160
                self.array_declarator = CArrayDeclaratorNode(
                    self.pos,
                    base=empty_declarator,
                    dimension=dimension)
1161
                self.type = self.array_declarator.analyse(base_type, env)[1]
1162

1163 1164 1165
        if self.type.is_fused and env.fused_to_specific:
            self.type = self.type.specialize(env.fused_to_specific)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1166
        return self.type
William Stein's avatar
William Stein committed
1167

1168

William Stein's avatar
William Stein committed
1169 1170 1171
class CComplexBaseTypeNode(CBaseTypeNode):
    # base_type   CBaseTypeNode
    # declarator  CDeclaratorNode
1172

1173 1174
    child_attrs = ["base_type", "declarator"]

1175
    def analyse(self, env, could_be_name=False):
1176
        base = self.base_type.analyse(env, could_be_name)
William Stein's avatar
William Stein committed
1177 1178 1179 1180
        _, type = self.declarator.analyse(base, env)
        return type


1181 1182 1183 1184 1185
class CTupleBaseTypeNode(CBaseTypeNode):
    # components [CBaseTypeNode]

    child_attrs = ["components"]

Robert Bradshaw's avatar
Robert Bradshaw committed
1186
    def analyse(self, env, could_be_name=False):
1187 1188 1189 1190
        component_types = []
        for c in self.components:
            type = c.analyse(env)
            if type.is_pyobject:
1191 1192
                error(c.pos, "Tuple types can't (yet) contain Python objects.")
                return error_type
1193
            component_types.append(type)
1194
        entry = env.declare_tuple_type(self.pos, component_types)
Robert Bradshaw's avatar
Robert Bradshaw committed
1195
        entry.used = True
1196
        return entry.type
1197 1198


1199 1200 1201 1202 1203 1204
class FusedTypeNode(CBaseTypeNode):
    """
    Represents a fused type in a ctypedef statement:

        ctypedef cython.fused_type(int, long, long long) integral

Mark Florisson's avatar
Mark Florisson committed
1205
    name            str                     name of this fused type
1206 1207 1208 1209 1210
    types           [CSimpleBaseTypeNode]   is the list of types to be fused
    """

    child_attrs = []

Mark Florisson's avatar
Mark Florisson committed
1211 1212 1213 1214 1215 1216 1217
    def analyse_declarations(self, env):
        type = self.analyse(env)
        entry = env.declare_typedef(self.name, type, self.pos)

        # Omit the typedef declaration that self.declarator would produce
        entry.in_cinclude = True

1218
    def analyse(self, env, could_be_name=False):
Mark Florisson's avatar
Mark Florisson committed
1219 1220 1221
        types = []
        for type_node in self.types:
            type = type_node.analyse_as_type(env)
1222

Mark Florisson's avatar
Mark Florisson committed
1223 1224 1225
            if not type:
                error(type_node.pos, "Not a type")
                continue
1226

Mark Florisson's avatar
Mark Florisson committed
1227
            if type in types:
1228 1229
                error(type_node.pos, "Type specified multiple times")
            else:
Mark Florisson's avatar
Mark Florisson committed
1230
                types.append(type)
1231

1232 1233
        # if len(self.types) == 1:
        #     return types[0]
Mark Florisson's avatar
Mark Florisson committed
1234 1235

        return PyrexTypes.FusedType(types, name=self.name)
1236

1237

Robert Bradshaw's avatar
Robert Bradshaw committed
1238 1239 1240 1241 1242
class CConstTypeNode(CBaseTypeNode):
    # base_type     CBaseTypeNode

    child_attrs = ["base_type"]

1243
    def analyse(self, env, could_be_name=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
1244 1245 1246 1247 1248 1249 1250
        base = self.base_type.analyse(env, could_be_name)
        if base.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        return PyrexTypes.c_const_type(base)


William Stein's avatar
William Stein committed
1251 1252 1253 1254 1255 1256
class CVarDefNode(StatNode):
    #  C variable definition or forward/extern function declaration.
    #
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarators   [CDeclaratorNode]
1257
    #  in_pxd        boolean
Stefan Behnel's avatar
Stefan Behnel committed
1258
    #  api           boolean
1259
    #  overridable   boolean        whether it is a cpdef
1260
    #  modifiers     ['inline']
1261

1262
    #  decorators    [cython.locals(...)] or None
1263
    #  directive_locals { string : NameNode } locals defined by cython.locals(...)
1264 1265

    child_attrs = ["base_type", "declarators"]
1266

Robert Bradshaw's avatar
Robert Bradshaw committed
1267
    decorators = None
1268
    directive_locals = None
1269

1270
    def analyse_declarations(self, env, dest_scope=None):
1271 1272
        if self.directive_locals is None:
            self.directive_locals = {}
William Stein's avatar
William Stein committed
1273 1274
        if not dest_scope:
            dest_scope = env
1275
        self.dest_scope = dest_scope
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290

        if self.declarators:
            templates = self.declarators[0].analyse_templates()
        else:
            templates = None
        if templates is not None:
            if self.visibility != 'extern':
                error(self.pos, "Only extern functions allowed")
            if len(self.declarators) > 1:
                error(self.declarators[1].pos, "Can't multiply declare template types")
            env = TemplateScope('func_template', env)
            env.directives = env.outer_scope.directives
            for template_param in templates:
                env.declare_type(template_param.name, template_param, self.pos)

William Stein's avatar
William Stein committed
1291
        base_type = self.base_type.analyse(env)
1292

1293 1294
        if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
                                                       env.is_module_scope):
1295 1296 1297
            error(self.pos, "Fused types not allowed here")
            return error_type

1298
        self.entry = None
1299
        visibility = self.visibility
1300

William Stein's avatar
William Stein committed
1301
        for declarator in self.declarators:
1302

1303
            if (len(self.declarators) > 1
1304 1305 1306 1307 1308
                    and not isinstance(declarator, CNameDeclaratorNode)
                    and env.directives['warn.multiple_declarators']):
                warning(
                    declarator.pos,
                    "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). "
1309
                    "Each pointer declaration should be on its own line.", 1)
1310

1311
            create_extern_wrapper = (self.overridable
1312 1313
                                     and self.visibility == 'extern'
                                     and env.is_module_scope)
1314 1315
            if create_extern_wrapper:
                declarator.overridable = False
1316 1317 1318 1319
            if isinstance(declarator, CFuncDeclaratorNode):
                name_declarator, type = declarator.analyse(base_type, env, directive_locals=self.directive_locals)
            else:
                name_declarator, type = declarator.analyse(base_type, env)
William Stein's avatar
William Stein committed
1320
            if not type.is_complete():
1321
                if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
1322
                    error(declarator.pos, "Variable type '%s' is incomplete" % type)
William Stein's avatar
William Stein committed
1323
            if self.visibility == 'extern' and type.is_pyobject:
1324
                error(declarator.pos, "Python object cannot be declared extern")
William Stein's avatar
William Stein committed
1325 1326
            name = name_declarator.name
            cname = name_declarator.cname
1327 1328 1329
            if name == '':
                error(declarator.pos, "Missing name in declaration.")
                return
1330 1331
            if type.is_reference and self.visibility != 'extern':
                error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
William Stein's avatar
William Stein committed
1332
            if type.is_cfunction:
1333 1334
                if 'staticmethod' in env.directives:
                    type.is_static_method = True
1335 1336
                self.entry = dest_scope.declare_cfunction(
                    name, type, declarator.pos,
1337 1338
                    cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
                    api=self.api, modifiers=self.modifiers, overridable=self.overridable)
1339 1340
                if self.entry is not None:
                    self.entry.directive_locals = copy.copy(self.directive_locals)
1341
                if create_extern_wrapper:
1342
                    self.entry.type.create_to_py_utility_code(env)
1343
                    self.entry.create_wrapper = True
William Stein's avatar
William Stein committed
1344
            else:
1345
                if self.directive_locals:
1346
                    error(self.pos, "Decorators can only be followed by functions")
1347 1348 1349 1350
                self.entry = dest_scope.declare_var(
                    name, type, declarator.pos,
                    cname=cname, visibility=visibility, in_pxd=self.in_pxd,
                    api=self.api, is_cdef=1)
1351 1352
                if Options.docstrings:
                    self.entry.doc = embed_position(self.pos, self.doc)
1353

William Stein's avatar
William Stein committed
1354 1355 1356 1357 1358 1359

class CStructOrUnionDefNode(StatNode):
    #  name          string
    #  cname         string or None
    #  kind          "struct" or "union"
    #  typedef_flag  boolean
1360
    #  visibility    "public" or "private"
1361
    #  api           boolean
Stefan Behnel's avatar
Stefan Behnel committed
1362
    #  in_pxd        boolean
William Stein's avatar
William Stein committed
1363 1364
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1365
    #  packed        boolean
1366

1367
    child_attrs = ["attributes"]
Vitja Makarov's avatar
Vitja Makarov committed
1368

1369
    def declare(self, env, scope=None):
William Stein's avatar
William Stein committed
1370 1371
        self.entry = env.declare_struct_or_union(
            self.name, self.kind, scope, self.typedef_flag, self.pos,
1372 1373
            self.cname, visibility=self.visibility, api=self.api,
            packed=self.packed)
1374 1375 1376 1377 1378 1379

    def analyse_declarations(self, env):
        scope = None
        if self.attributes is not None:
            scope = StructOrUnionScope(self.name)
        self.declare(env, scope)
William Stein's avatar
William Stein committed
1380
        if self.attributes is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1381 1382
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
William Stein's avatar
William Stein committed
1383 1384
            for attr in self.attributes:
                attr.analyse_declarations(env, scope)
1385 1386 1387 1388
            if self.visibility != 'extern':
                for attr in scope.var_entries:
                    type = attr.type
                    while type.is_array:
1389 1390
                        type = type.base_type
                    if type == self.entry.type:
1391
                        error(attr.pos, "Struct cannot contain itself as a member.")
1392

William Stein's avatar
William Stein committed
1393
    def analyse_expressions(self, env):
1394
        return self
1395

William Stein's avatar
William Stein committed
1396 1397 1398 1399
    def generate_execution_code(self, code):
        pass


1400
class CppClassNode(CStructOrUnionDefNode, BlockNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1401 1402 1403

    #  name          string
    #  cname         string or None
1404
    #  visibility    "extern"
Robert Bradshaw's avatar
Robert Bradshaw committed
1405 1406 1407
    #  in_pxd        boolean
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1408
    #  base_classes  [CBaseTypeNode]
1409
    #  templates     [(string, bool)] or None
1410 1411 1412
    #  decorators    [DecoratorNode] or None

    decorators = None
Robert Bradshaw's avatar
Robert Bradshaw committed
1413

1414 1415 1416 1417
    def declare(self, env):
        if self.templates is None:
            template_types = None
        else:
1418 1419 1420 1421 1422
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
            num_optional_templates = sum(not required for _, required in self.templates)
            if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
                error(self.pos, "Required template parameters must precede optional template parameters.")
1423
        self.entry = env.declare_cpp_class(
1424 1425
            self.name, None, self.pos, self.cname,
            base_classes=[], visibility=self.visibility, templates=template_types)
1426

Robert Bradshaw's avatar
Robert Bradshaw committed
1427
    def analyse_declarations(self, env):
1428 1429 1430 1431 1432 1433
        if self.templates is None:
            template_types = template_names = None
        else:
            template_names = [template_name for template_name, _ in self.templates]
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
Robert Bradshaw's avatar
Robert Bradshaw committed
1434
        scope = None
1435
        if self.attributes is not None:
1436
            scope = CppClassScope(self.name, env, templates=template_names)
1437 1438 1439 1440 1441 1442
        def base_ok(base_class):
            if base_class.is_cpp_class or base_class.is_struct:
                return True
            else:
                error(self.pos, "Base class '%s' not a struct or class." % base_class)
        base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
1443
        self.entry = env.declare_cpp_class(
1444
            self.name, scope, self.pos,
1445
            self.cname, base_class_types, visibility=self.visibility, templates=template_types)
1446 1447
        if self.entry is None:
            return
Danilo Freitas's avatar
Danilo Freitas committed
1448
        self.entry.is_cpp_class = 1
Stefan Behnel's avatar
Stefan Behnel committed
1449 1450
        if scope is not None:
            scope.type = self.entry.type
1451
        defined_funcs = []
1452 1453 1454 1455 1456 1457 1458
        def func_attributes(attributes):
            for attr in attributes:
                if isinstance(attr, CFuncDefNode):
                    yield attr
                elif isinstance(attr, CompilerDirectivesNode):
                    for sub_attr in func_attributes(attr.body.stats):
                        yield sub_attr
Robert Bradshaw's avatar
Robert Bradshaw committed
1459 1460 1461 1462
        if self.attributes is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for attr in self.attributes:
1463 1464 1465
                declare = getattr(attr, 'declare', None)
                if declare:
                    attr.declare(scope)
Robert Bradshaw's avatar
Robert Bradshaw committed
1466
                attr.analyse_declarations(scope)
1467 1468 1469
            for func in func_attributes(self.attributes):
                defined_funcs.append(func)
                if self.templates is not None:
1470
                    func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
1471 1472 1473 1474
        self.body = StatListNode(self.pos, stats=defined_funcs)
        self.scope = scope

    def analyse_expressions(self, env):
1475 1476
        self.body = self.body.analyse_expressions(self.entry.type.scope)
        return self
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486

    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(self.entry.type.scope, code)

    def generate_execution_code(self, code):
        self.body.generate_execution_code(code)

    def annotate(self, code):
        self.body.annotate(code)

Robert Bradshaw's avatar
Robert Bradshaw committed
1487

William Stein's avatar
William Stein committed
1488 1489 1490 1491 1492
class CEnumDefNode(StatNode):
    #  name           string or None
    #  cname          string or None
    #  items          [CEnumDefItemNode]
    #  typedef_flag   boolean
1493
    #  visibility     "public" or "private" or "extern"
1494
    #  api            boolean
Stefan Behnel's avatar
Stefan Behnel committed
1495
    #  in_pxd         boolean
1496
    #  create_wrapper boolean
William Stein's avatar
William Stein committed
1497
    #  entry          Entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1498

1499
    child_attrs = ["items"]
1500

1501
    def declare(self, env):
1502 1503 1504 1505
         self.entry = env.declare_enum(
             self.name, self.pos,
             cname=self.cname, typedef_flag=self.typedef_flag,
             visibility=self.visibility, api=self.api,
1506
             create_wrapper=self.create_wrapper)
1507

William Stein's avatar
William Stein committed
1508
    def analyse_declarations(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
1509 1510 1511 1512 1513
        if self.items is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for item in self.items:
                item.analyse_declarations(env, self.entry)
William Stein's avatar
William Stein committed
1514

1515
    def analyse_expressions(self, env):
1516
        return self
1517

William Stein's avatar
William Stein committed
1518
    def generate_execution_code(self, code):
1519
        if self.visibility == 'public' or self.api:
1520
            code.mark_pos(self.pos)
1521
            temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
1522 1523
            for item in self.entry.enum_values:
                code.putln("%s = PyInt_FromLong(%s); %s" % (
1524 1525 1526
                    temp,
                    item.cname,
                    code.error_goto_if_null(temp, item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1527
                code.put_gotref(temp)
1528
                code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
1529 1530 1531 1532
                    Naming.moddict_cname,
                    item.name,
                    temp,
                    code.error_goto(item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1533
                code.put_decref_clear(temp, PyrexTypes.py_object_type)
1534
            code.funcstate.release_temp(temp)
William Stein's avatar
William Stein committed
1535 1536 1537 1538 1539 1540


class CEnumDefItemNode(StatNode):
    #  name     string
    #  cname    string or None
    #  value    ExprNode or None
1541

1542 1543
    child_attrs = ["value"]

William Stein's avatar
William Stein committed
1544 1545
    def analyse_declarations(self, env, enum_entry):
        if self.value:
1546
            self.value = self.value.analyse_const_expression(env)
1547 1548
            if not self.value.type.is_int:
                self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
1549
                self.value = self.value.analyse_const_expression(env)
1550 1551 1552 1553 1554
        entry = env.declare_const(
            self.name, enum_entry.type,
            self.value, self.pos, cname=self.cname,
            visibility=enum_entry.visibility, api=enum_entry.api,
            create_wrapper=enum_entry.create_wrapper and enum_entry.name is None)
William Stein's avatar
William Stein committed
1555
        enum_entry.enum_values.append(entry)
1556
        if enum_entry.name:
1557
            enum_entry.type.values.append(entry.name)
William Stein's avatar
William Stein committed
1558 1559 1560


class CTypeDefNode(StatNode):
Stefan Behnel's avatar
Stefan Behnel committed
1561 1562 1563
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
    #  visibility   "public" or "private"
1564
    #  api          boolean
Stefan Behnel's avatar
Stefan Behnel committed
1565
    #  in_pxd       boolean
1566 1567

    child_attrs = ["base_type", "declarator"]
1568

William Stein's avatar
William Stein committed
1569 1570 1571 1572 1573
    def analyse_declarations(self, env):
        base = self.base_type.analyse(env)
        name_declarator, type = self.declarator.analyse(base, env)
        name = name_declarator.name
        cname = name_declarator.cname
1574

1575 1576 1577
        entry = env.declare_typedef(
            name, type, self.pos,
            cname=cname, visibility=self.visibility, api=self.api)
1578

Mark Florisson's avatar
Mark Florisson committed
1579
        if type.is_fused:
1580 1581
            entry.in_cinclude = True

Mark Florisson's avatar
Mark Florisson committed
1582 1583
        if self.in_pxd and not env.in_cinclude:
            entry.defined_in_pxd = 1
1584

William Stein's avatar
William Stein committed
1585
    def analyse_expressions(self, env):
1586 1587
        return self

William Stein's avatar
William Stein committed
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
    def generate_execution_code(self, code):
        pass


class FuncDefNode(StatNode, BlockNode):
    #  Base class for function definition nodes.
    #
    #  return_type     PyrexType
    #  #filename        string        C name of filename string const
    #  entry           Symtab.Entry
1598
    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
Vitja Makarov's avatar
Vitja Makarov committed
1599
    #  needs_outer_scope boolean      Whether or not this function requires outer scope
1600
    #  pymethdef_required boolean     Force Python method struct generation
1601 1602
    #  directive_locals { string : ExprNode } locals defined by cython.locals(...)
    #  directive_returns [ExprNode] type defined by cython.returns(...)
1603 1604 1605 1606 1607
    #  star_arg      PyArgDeclNode or None  * argument
    #  starstar_arg  PyArgDeclNode or None  ** argument
    #
    #  is_async_def  boolean          is a Coroutine function
    #
1608 1609 1610 1611 1612
    #  has_fused_arguments  boolean
    #       Whether this cdef function has fused parameters. This is needed
    #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
    #       with fused argument types with a FusedCFuncDefNode

1613
    py_func = None
1614
    needs_closure = False
Vitja Makarov's avatar
Vitja Makarov committed
1615
    needs_outer_scope = False
1616
    pymethdef_required = False
1617
    is_generator = False
1618
    is_generator_body = False
1619
    is_async_def = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1620
    modifiers = []
1621
    has_fused_arguments = False
1622 1623
    star_arg = None
    starstar_arg = None
1624
    is_cyfunction = False
1625
    code_object = None
1626

1627
    def analyse_default_values(self, env):
1628
        default_seen = 0
1629 1630
        for arg in self.args:
            if arg.default:
1631
                default_seen = 1
1632
                if arg.is_generic:
1633
                    arg.default = arg.default.analyse_types(env)
1634
                    arg.default = arg.default.coerce_to(arg.type, env)
1635
                else:
1636
                    error(arg.pos, "This argument cannot have a default value")
1637
                    arg.default = None
1638 1639 1640 1641
            elif arg.kw_only:
                default_seen = 1
            elif default_seen:
                error(arg.pos, "Non-default argument following default argument")
1642

1643
    def analyse_annotation(self, env, annotation):
Stefan Behnel's avatar
Stefan Behnel committed
1644
        # Annotations can not only contain valid Python expressions but arbitrary type references.
1645 1646 1647 1648 1649 1650
        if annotation is None:
            return None
        if annotation.analyse_as_type(env) is None:
            annotation = annotation.analyse_types(env)
        return annotation

1651 1652 1653
    def analyse_annotations(self, env):
        for arg in self.args:
            if arg.annotation:
1654
                arg.annotation = self.analyse_annotation(env, arg.annotation)
1655

1656
    def align_argument_type(self, env, arg):
1657
        # @cython.locals()
1658
        directive_locals = self.directive_locals
1659
        orig_type = arg.type
1660 1661 1662
        if arg.name in directive_locals:
            type_node = directive_locals[arg.name]
            other_type = type_node.analyse_as_type(env)
1663
        elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
1664 1665
            type_node = arg.annotation
            other_type = arg.inject_type_from_annotations(env)
1666 1667
            if other_type is None:
                return arg
1668 1669 1670 1671
        else:
            return arg
        if other_type is None:
            error(type_node.pos, "Not a type")
1672
        elif orig_type is not py_object_type and not orig_type.same_as(other_type):
1673 1674 1675 1676
            error(arg.base_type.pos, "Signature does not agree with previous declaration")
            error(type_node.pos, "Previous declaration here")
        else:
            arg.type = other_type
1677 1678
        return arg

1679 1680
    def need_gil_acquisition(self, lenv):
        return 0
1681

1682 1683
    def create_local_scope(self, env):
        genv = env
1684
        while genv.is_py_class_scope or genv.is_c_class_scope:
1685
            genv = genv.outer_scope
1686
        if self.needs_closure:
1687
            lenv = ClosureScope(name=self.entry.name,
1688 1689
                                outer_scope=genv,
                                parent_scope=env,
1690
                                scope_name=self.entry.cname)
1691
        else:
1692 1693 1694
            lenv = LocalScope(name=self.entry.name,
                              outer_scope=genv,
                              parent_scope=env)
William Stein's avatar
William Stein committed
1695
        lenv.return_type = self.return_type
1696 1697 1698
        type = self.entry.type
        if type.is_cfunction:
            lenv.nogil = type.nogil and not type.with_gil
1699
        self.local_scope = lenv
1700
        lenv.directives = env.directives
1701
        return lenv
1702

1703 1704 1705
    def generate_function_body(self, env, code):
        self.body.generate_execution_code(code)

1706
    def generate_function_definitions(self, env, code):
1707
        from . import Buffer
1708
        if self.return_type.is_memoryviewslice:
1709
            from . import MemoryView
1710 1711

        lenv = self.local_scope
Vitja Makarov's avatar
Vitja Makarov committed
1712
        if lenv.is_closure_scope and not lenv.is_passthrough:
1713 1714 1715 1716 1717
            outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
                                            Naming.outer_scope_cname)
        else:
            outer_scope_cname = Naming.outer_scope_cname
        lenv.mangle_closure_cnames(outer_scope_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1718 1719
        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)
Stefan Behnel's avatar
Stefan Behnel committed
1720
        # generate lambda function definitions
1721
        self.generate_lambda_definitions(lenv, code)
1722

1723 1724
        is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
                             self.entry.scope.is_c_class_scope)
1725 1726 1727
        is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
                                 self.entry.scope.is_c_class_scope)
        is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
1728
        if is_buffer_slot:
1729 1730
            if 'cython_unused' not in self.modifiers:
                self.modifiers = self.modifiers + ['cython_unused']
1731

1732
        preprocessor_guard = self.get_preprocessor_guard()
1733

1734
        profile = code.globalstate.directives['profile']
1735 1736
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
1737 1738
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("Profile", "Profile.c"))
1739

1740
        # Generate C code for header and body of function
1741
        code.enter_cfunc_scope(lenv)
1742
        code.return_from_error_cleanup_label = code.new_label()
1743
        code.funcstate.gil_owned = not lenv.nogil
1744

William Stein's avatar
William Stein committed
1745
        # ----- Top-level constants used by this function
1746
        code.mark_pos(self.pos)
1747
        self.generate_cached_builtins_decls(lenv, code)
William Stein's avatar
William Stein committed
1748 1749
        # ----- Function header
        code.putln("")
1750 1751 1752 1753

        if preprocessor_guard:
            code.putln(preprocessor_guard)

1754 1755
        with_pymethdef = (self.needs_assignment_synthesis(env, code) or
                          self.pymethdef_required)
1756
        if self.py_func:
1757 1758 1759
            self.py_func.generate_function_header(
                code, with_pymethdef=with_pymethdef, proto_only=True)
        self.generate_function_header(code, with_pymethdef=with_pymethdef)
William Stein's avatar
William Stein committed
1760
        # ----- Local variable declarations
1761 1762 1763 1764
        # Find function scope
        cenv = env
        while cenv.is_py_class_scope or cenv.is_c_class_scope:
            cenv = cenv.outer_scope
Vitja Makarov's avatar
Vitja Makarov committed
1765
        if self.needs_closure:
1766
            code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
Robert Bradshaw's avatar
Robert Bradshaw committed
1767
            code.putln(";")
Vitja Makarov's avatar
Vitja Makarov committed
1768 1769 1770 1771
        elif self.needs_outer_scope:
            if lenv.is_passthrough:
                code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
                code.putln(";")
1772
            code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
1773
            code.putln(";")
William Stein's avatar
William Stein committed
1774
        self.generate_argument_declarations(lenv, code)
1775

1776
        for entry in lenv.var_entries:
1777
            if not (entry.in_closure or entry.is_arg):
1778
                code.put_var_declaration(entry)
1779

1780
        # Initialize the return variable __pyx_r
William Stein's avatar
William Stein committed
1781 1782
        init = ""
        if not self.return_type.is_void:
1783 1784
            if self.return_type.is_pyobject:
                init = " = NULL"
1785
            elif self.return_type.is_memoryviewslice:
1786
                init = ' = ' + MemoryView.memslice_entry_init
1787

1788 1789 1790
            code.putln("%s%s;" % (
                self.return_type.declaration_code(Naming.retval_cname),
                init))
1791

1792
        tempvardecl_code = code.insertion_point()
William Stein's avatar
William Stein committed
1793
        self.generate_keyword_list(code)
1794

William Stein's avatar
William Stein committed
1795 1796
        # ----- Extern library function declarations
        lenv.generate_library_function_declarations(code)
1797

1798
        # ----- GIL acquisition
1799
        acquire_gil = self.acquire_gil
1800

Mark Florisson's avatar
Mark Florisson committed
1801 1802
        # See if we need to acquire the GIL for variable declarations, or for
        # refnanny only
1803

1804 1805 1806
        # Closures are not currently possible for cdef nogil functions,
        # but check them anyway
        have_object_args = self.needs_closure or self.needs_outer_scope
Mark Florisson's avatar
Mark Florisson committed
1807 1808 1809 1810 1811
        for arg in lenv.arg_entries:
            if arg.type.is_pyobject:
                have_object_args = True
                break

1812 1813
        used_buffer_entries = [entry for entry in lenv.buffer_entries if entry.used]

Mark Florisson's avatar
Mark Florisson committed
1814
        acquire_gil_for_var_decls_only = (
1815 1816
            lenv.nogil and lenv.has_with_gil_block and
            (have_object_args or used_buffer_entries))
Mark Florisson's avatar
Mark Florisson committed
1817 1818

        acquire_gil_for_refnanny_only = (
1819 1820
            lenv.nogil and lenv.has_with_gil_block and not
            acquire_gil_for_var_decls_only)
Mark Florisson's avatar
Mark Florisson committed
1821 1822

        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
1823 1824 1825

        if acquire_gil or acquire_gil_for_var_decls_only:
            code.put_ensure_gil()
1826
            code.funcstate.gil_owned = True
1827 1828
        elif lenv.nogil and lenv.has_with_gil_block:
            code.declare_gilstate()
1829

1830
        if profile or linetrace:
1831 1832 1833 1834 1835
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                tempvardecl_code.put_trace_declarations()
                code_object = self.code_object.calculate_result_code(code) if self.code_object else None
                code.put_trace_frame_init(code_object)
1836

1837
        # ----- set up refnanny
1838
        if use_refnanny:
1839
            tempvardecl_code.put_declare_refcount_context()
1840 1841
            code.put_setup_refcount_context(
                self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
Mark Florisson's avatar
Mark Florisson committed
1842

1843
        # ----- Automatic lead-ins for certain special functions
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1844 1845
        if is_getbuffer_slot:
            self.getbuffer_init(code)
1846
        # ----- Create closure scope object
Robert Bradshaw's avatar
Robert Bradshaw committed
1847
        if self.needs_closure:
1848
            tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1849 1850 1851 1852
            slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
            if not slot_func_cname:
                slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
            code.putln("%s = (%s)%s(%s, %s, NULL);" % (
1853
                Naming.cur_scope_cname,
1854
                lenv.scope_class.type.empty_declaration_code(),
1855
                slot_func_cname,
1856
                lenv.scope_class.type.typeptr_cname,
1857 1858
                Naming.empty_tuple))
            code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
1859 1860 1861
            # Scope unconditionally DECREFed on return.
            code.putln("%s = %s;" % (
                Naming.cur_scope_cname,
1862 1863
                lenv.scope_class.type.cast_code("Py_None")))
            code.put_incref("Py_None", py_object_type)
1864 1865
            code.putln(code.error_goto(self.pos))
            code.putln("} else {")
Robert Bradshaw's avatar
Robert Bradshaw committed
1866
            code.put_gotref(Naming.cur_scope_cname)
1867
            code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
1868
            # Note that it is unsafe to decref the scope at this point.
Vitja Makarov's avatar
Vitja Makarov committed
1869
        if self.needs_outer_scope:
1870 1871 1872
            if self.is_cyfunction:
                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
                    outer_scope_cname,
1873
                    cenv.scope_class.type.empty_declaration_code(),
1874 1875 1876 1877
                    Naming.self_cname))
            else:
                code.putln("%s = (%s) %s;" % (
                    outer_scope_cname,
1878
                    cenv.scope_class.type.empty_declaration_code(),
1879
                    Naming.self_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1880
            if lenv.is_passthrough:
Stefan Behnel's avatar
Stefan Behnel committed
1881
                code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1882
            elif self.needs_closure:
1883
                # inner closures own a reference to their outer parent
1884
                code.put_incref(outer_scope_cname, cenv.scope_class.type)
1885
                code.put_giveref(outer_scope_cname)
1886
        # ----- Trace function call
1887
        if profile or linetrace:
1888 1889
            # this looks a bit late, but if we don't get here due to a
            # fatal error before hand, it's not really worth tracing
1890 1891 1892 1893 1894 1895 1896 1897
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.is_wrapper:
                    trace_name = self.entry.name + " (wrapper)"
                else:
                    trace_name = self.entry.name
                code.put_trace_call(
                    trace_name, self.pos, nogil=not code.funcstate.gil_owned)
1898
            code.funcstate.can_trace = True
William Stein's avatar
William Stein committed
1899
        # ----- Fetch arguments
1900
        self.generate_argument_parsing_code(env, code)
1901
        # If an argument is assigned to in the body, we must
Robert Bradshaw's avatar
Robert Bradshaw committed
1902
        # incref it to properly keep track of refcounts.
1903
        is_cdef = isinstance(self, CFuncDefNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
1904
        for entry in lenv.arg_entries:
1905
            if entry.type.is_pyobject:
1906
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
1907
                    code.put_var_incref(entry)
1908

1909
            # Note: defaults are always incref-ed. For def functions, we
1910 1911 1912
            #       we aquire arguments from object converstion, so we have
            #       new references. If we are a cdef function, we need to
            #       incref our arguments
1913
            elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1:
1914
                code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
1915
        for entry in lenv.var_entries:
1916
            if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure:
1917 1918 1919 1920
                if entry.xdecref_cleanup:
                    code.put_var_xincref(entry)
                else:
                    code.put_var_incref(entry)
1921

1922 1923
        # ----- Initialise local buffer auxiliary variables
        for entry in lenv.var_entries + lenv.arg_entries:
1924 1925
            if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                Buffer.put_init_vars(entry, code)
1926

1927
        # ----- Check and convert arguments
William Stein's avatar
William Stein committed
1928
        self.generate_argument_type_tests(code)
1929 1930 1931
        # ----- Acquire buffer arguments
        for entry in lenv.arg_entries:
            if entry.type.is_buffer:
1932 1933
                Buffer.put_acquire_arg_buffer(entry, code, self.pos)

1934 1935
        if acquire_gil_for_var_decls_only:
            code.put_release_ensured_gil()
1936
            code.funcstate.gil_owned = False
1937

1938 1939 1940
        # -------------------------
        # ----- Function body -----
        # -------------------------
1941
        self.generate_function_body(env, code)
1942

1943
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
1944
        code.putln("")
Robert Bradshaw's avatar
Robert Bradshaw committed
1945
        code.putln("/* function exit code */")
1946 1947

        # ----- Default return value
1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958
        if not self.body.is_terminator:
            if self.return_type.is_pyobject:
                #if self.return_type.is_extension_type:
                #    lhs = "(PyObject *)%s" % Naming.retval_cname
                #else:
                lhs = Naming.retval_cname
                code.put_init_to_py_none(lhs, self.return_type)
            else:
                val = self.return_type.default_value
                if val:
                    code.putln("%s = %s;" % (Naming.retval_cname, val))
1959 1960
                elif not self.return_type.is_void:
                    code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
William Stein's avatar
William Stein committed
1961
        # ----- Error cleanup
1962
        if code.error_label in code.labels_used:
1963 1964
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
1965
            code.put_label(code.error_label)
1966
            for cname, type in code.funcstate.all_managed_temps():
1967
                code.put_xdecref(cname, type, have_gil=not lenv.nogil)
1968 1969 1970

            # Clean up buffers -- this calls a Python function
            # so need to save and restore error state
1971
            buffers_present = len(used_buffer_entries) > 0
1972
            #memslice_entries = [e for e in lenv.entries.values() if e.type.is_memoryviewslice]
1973
            if buffers_present:
1974
                code.globalstate.use_utility_code(restore_exception_utility_code)
1975
                code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
1976 1977
                code.putln("__Pyx_PyThreadState_declare")
                code.putln("__Pyx_PyThreadState_assign")
1978
                code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
1979 1980
                for entry in used_buffer_entries:
                    Buffer.put_release_buffer_code(code, entry)
1981
                    #code.putln("%s = 0;" % entry.cname)
1982
                code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
1983

1984 1985 1986 1987 1988 1989
            if self.return_type.is_memoryviewslice:
                MemoryView.put_init_entry(Naming.retval_cname, code)
                err_val = Naming.retval_cname
            else:
                err_val = self.error_value()

1990 1991
            exc_check = self.caller_will_check_exceptions()
            if err_val is not None or exc_check:
1992
                # TODO: Fix exception tracing (though currently unused by cProfile).
Robert Bradshaw's avatar
Robert Bradshaw committed
1993 1994
                # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                # code.put_trace_exception()
1995

Mark Florisson's avatar
Mark Florisson committed
1996
                if lenv.nogil and not lenv.has_with_gil_block:
1997 1998 1999
                    code.putln("{")
                    code.put_ensure_gil()

2000
                code.put_add_traceback(self.entry.qualified_name)
2001

Mark Florisson's avatar
Mark Florisson committed
2002
                if lenv.nogil and not lenv.has_with_gil_block:
2003 2004
                    code.put_release_ensured_gil()
                    code.putln("}")
2005
            else:
2006 2007 2008
                warning(self.entry.pos,
                        "Unraisable exception in function '%s'." %
                        self.entry.qualified_name, 0)
2009
                code.put_unraisable(self.entry.qualified_name, lenv.nogil)
2010 2011 2012 2013
            default_retval = self.return_type.default_value
            if err_val is None and default_retval:
                err_val = default_retval
            if err_val is not None:
2014
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
2015 2016
            elif not self.return_type.is_void:
                code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
2017 2018 2019 2020 2021 2022 2023

            if is_getbuffer_slot:
                self.getbuffer_error_cleanup(code)

            # If we are using the non-error cleanup section we should
            # jump past it if we have an error. The if-test below determine
            # whether this section is used.
2024
            if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice:
2025 2026 2027
                code.put_goto(code.return_from_error_cleanup_label)

        # ----- Non-error return cleanup
William Stein's avatar
William Stein committed
2028
        code.put_label(code.return_label)
2029 2030
        for entry in used_buffer_entries:
            Buffer.put_release_buffer_code(code, entry)
2031 2032
        if is_getbuffer_slot:
            self.getbuffer_normal_cleanup(code)
2033 2034 2035

        if self.return_type.is_memoryviewslice:
            # See if our return value is uninitialized on non-error return
2036
            # from . import MemoryView
2037
            # MemoryView.err_if_nogil_initialized_check(self.pos, env)
2038
            cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname))
2039 2040 2041 2042 2043
            code.putln(
                'if (%s) {' % cond)
            if env.nogil:
                code.put_ensure_gil()
            code.putln(
2044
                'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
2045 2046 2047 2048 2049
            if env.nogil:
                code.put_release_ensured_gil()
            code.putln(
                '}')

2050 2051
        # ----- Return cleanup for both error and no-error return
        code.put_label(code.return_from_error_cleanup_label)
2052

2053
        for entry in lenv.var_entries:
2054 2055
            if not entry.used or entry.in_closure:
                continue
2056

2057
            if entry.type.is_memoryviewslice:
2058
                code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil)
2059
            elif entry.type.is_pyobject:
2060
                if not entry.is_arg or len(entry.cf_assignments) > 1:
2061 2062 2063 2064
                    if entry.xdecref_cleanup:
                        code.put_var_xdecref(entry)
                    else:
                        code.put_var_decref(entry)
2065

Robert Bradshaw's avatar
Robert Bradshaw committed
2066
        # Decref any increfed args
2067
        for entry in lenv.arg_entries:
2068
            if entry.type.is_pyobject:
2069
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
2070
                    code.put_var_decref(entry)
2071 2072 2073 2074
            elif (entry.type.is_memoryviewslice and
                  (not is_cdef or len(entry.cf_assignments) > 1)):
                # decref slices of def functions and acquired slices from cdef
                # functions, but not borrowed slices from cdef functions.
2075 2076
                code.put_xdecref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
Robert Bradshaw's avatar
Robert Bradshaw committed
2077 2078
        if self.needs_closure:
            code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
2079

2080
        # ----- Return
2081
        # This code is duplicated in ModuleNode.generate_module_init_func
2082 2083 2084 2085
        if not lenv.nogil:
            default_retval = self.return_type.default_value
            err_val = self.error_value()
            if err_val is None and default_retval:
Stefan Behnel's avatar
Stefan Behnel committed
2086
                err_val = default_retval  # FIXME: why is err_val not used?
2087 2088
            if self.return_type.is_pyobject:
                code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
2089

2090 2091
        if self.entry.is_special and self.entry.name == "__hash__":
            # Returning -1 for __hash__ is supposed to signal an error
2092
            # We do as Python instances and coerce -1 into -2.
2093
            code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
2094
                Naming.retval_cname, Naming.retval_cname))
2095

2096 2097
        if profile or linetrace:
            code.funcstate.can_trace = False
2098 2099 2100 2101 2102 2103 2104 2105
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.return_type.is_pyobject:
                    code.put_trace_return(
                        Naming.retval_cname, nogil=not code.funcstate.gil_owned)
                else:
                    code.put_trace_return(
                        "Py_None", nogil=not code.funcstate.gil_owned)
2106

2107
        if not lenv.nogil:
Stefan Behnel's avatar
Stefan Behnel committed
2108
            # GIL holding function
2109
            code.put_finish_refcount_context()
2110

2111 2112
        if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
            # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
2113
            code.put_release_ensured_gil()
2114
            code.funcstate.gil_owned = False
2115

William Stein's avatar
William Stein committed
2116
        if not self.return_type.is_void:
2117
            code.putln("return %s;" % Naming.retval_cname)
2118

William Stein's avatar
William Stein committed
2119
        code.putln("}")
2120 2121 2122 2123

        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

2124
        # ----- Go back and insert temp variable declarations
2125
        tempvardecl_code.put_temp_declarations(code.funcstate)
2126

2127
        # ----- Python version
2128
        code.exit_cfunc_scope()
2129
        if self.py_func:
2130
            self.py_func.generate_function_definitions(env, code)
2131
        self.generate_wrapper_functions(code)
William Stein's avatar
William Stein committed
2132 2133 2134 2135

    def declare_argument(self, env, arg):
        if arg.type.is_void:
            error(arg.pos, "Invalid use of 'void'")
2136
        elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
2137
            error(arg.pos, "Argument type '%s' is incomplete" % arg.type)
William Stein's avatar
William Stein committed
2138
        return env.declare_arg(arg.name, arg.type, arg.pos)
2139

2140 2141 2142
    def generate_arg_type_test(self, arg, code):
        # Generate type test for one argument.
        if arg.type.typeobj_is_available():
2143 2144
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
2145 2146 2147 2148
            typeptr_cname = arg.type.typeptr_cname
            arg_code = "((PyObject *)%s)" % arg.entry.cname
            code.putln(
                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % (
2149
                    arg_code,
2150 2151 2152
                    typeptr_cname,
                    arg.accept_none,
                    arg.name,
2153
                    arg.type.is_builtin_type and arg.type.require_exact,
2154 2155
                    code.error_goto(arg.pos)))
        else:
2156
            error(arg.pos, "Cannot test type of extern C class without type object name specification")
2157 2158 2159

    def generate_arg_none_check(self, arg, code):
        # Generate None check for one argument.
2160 2161 2162 2163 2164 2165
        if arg.type.is_memoryviewslice:
            cname = "%s.memview" % arg.entry.cname
        else:
            cname = arg.entry.cname

        code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
2166 2167
        code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % (
            max(200, len(arg.name)), arg.name,
2168 2169
            code.error_goto(arg.pos)))
        code.putln('}')
2170

2171
    def generate_wrapper_functions(self, code):
William Stein's avatar
William Stein committed
2172 2173 2174
        pass

    def generate_execution_code(self, code):
2175
        code.mark_pos(self.pos)
2176 2177
        # Evaluate and store argument default values
        for arg in self.args:
2178 2179
            if not arg.is_dynamic:
                arg.generate_assignment_code(code)
William Stein's avatar
William Stein committed
2180

2181
    #
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2182
    # Special code for the __getbuffer__ function
2183 2184 2185 2186 2187
    #
    def getbuffer_init(self, code):
        info = self.local_scope.arg_entries[1].cname
        # Python 3.0 betas have a bug in memoryview which makes it call
        # getbuffer with a NULL parameter. For now we work around this;
2188 2189
        # the following block should be removed when this bug is fixed.
        code.putln("if (%s != NULL) {" % info)
2190
        code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info)
2191
        code.put_giveref("%s->obj" % info) # Do not refnanny object within structs
2192
        code.putln("}")
2193 2194 2195

    def getbuffer_error_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2196 2197
        code.putln("if (%s != NULL && %s->obj != NULL) {"
                   % (info, info))
2198
        code.put_gotref("%s->obj" % info)
2199 2200 2201
        code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;"
                   % (info, info))
        code.putln("}")
2202 2203 2204

    def getbuffer_normal_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2205
        code.putln("if (%s != NULL && %s->obj == Py_None) {" % (info, info))
2206 2207 2208
        code.put_gotref("Py_None")
        code.putln("__Pyx_DECREF(Py_None); %s->obj = NULL;" % info)
        code.putln("}")
William Stein's avatar
William Stein committed
2209

2210
    def get_preprocessor_guard(self):
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221
        if not self.entry.is_special:
            return None
        name = self.entry.name
        slot = TypeSlots.method_name_to_slot.get(name)
        if not slot:
            return None
        if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
            return None
        if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
            return None
        return slot.preprocessor_guard_code()
2222 2223


William Stein's avatar
William Stein committed
2224 2225 2226
class CFuncDefNode(FuncDefNode):
    #  C function definition.
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
2227
    #  modifiers     ['inline']
William Stein's avatar
William Stein committed
2228 2229 2230
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarator    CDeclaratorNode
2231 2232 2233
    #  cfunc_declarator  the CFuncDeclarator of this function
    #                    (this is also available through declarator or a
    #                     base thereof)
William Stein's avatar
William Stein committed
2234
    #  body          StatListNode
2235
    #  api           boolean
2236
    #  decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2237
    #
2238
    #  with_gil      boolean    Acquire GIL around body
William Stein's avatar
William Stein committed
2239
    #  type          CFuncType
2240
    #  py_func       wrapper for calling from Python
2241
    #  overridable   whether or not this is a cpdef function
2242
    #  inline_in_pxd whether this is an inline function in a pxd file
2243
    #  template_declaration  String or None   Used for c++ class methods
Robert Bradshaw's avatar
Robert Bradshaw committed
2244
    #  is_const_method whether this is a const method
Robert Bradshaw's avatar
Robert Bradshaw committed
2245
    #  is_static_method whether this is a static method
2246
    #  is_c_class_method whether this is a cclass method
2247

2248
    child_attrs = ["base_type", "declarator", "body", "py_func_stat"]
2249 2250

    inline_in_pxd = False
2251
    decorators = None
2252
    directive_locals = None
2253
    directive_returns = None
2254
    override = None
2255
    template_declaration = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2256
    is_const_method = False
2257
    py_func_stat = None
2258

William Stein's avatar
William Stein committed
2259 2260
    def unqualified_name(self):
        return self.entry.name
2261

2262 2263 2264 2265 2266
    @property
    def code_object(self):
        # share the CodeObject with the cpdef wrapper (if available)
        return self.py_func.code_object if self.py_func else None

William Stein's avatar
William Stein committed
2267
    def analyse_declarations(self, env):
2268
        self.is_c_class_method = env.is_c_class_scope
2269 2270
        if self.directive_locals is None:
            self.directive_locals = {}
2271
        self.directive_locals.update(env.directives['locals'])
2272 2273 2274 2275 2276 2277 2278
        if self.directive_returns is not None:
            base_type = self.directive_returns.analyse_as_type(env)
            if base_type is None:
                error(self.directive_returns.pos, "Not a type")
                base_type = PyrexTypes.error_type
        else:
            base_type = self.base_type.analyse(env)
2279
        self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
2280
        # The 2 here is because we need both function and argument names.
2281
        if isinstance(self.declarator, CFuncDeclaratorNode):
2282 2283 2284
            name_declarator, type = self.declarator.analyse(
                base_type, env, nonempty=2 * (self.body is not None),
                directive_locals=self.directive_locals)
2285
        else:
2286 2287
            name_declarator, type = self.declarator.analyse(
                base_type, env, nonempty=2 * (self.body is not None))
2288
        if not type.is_cfunction:
2289
            error(self.pos, "Suite attached to non-function declaration")
William Stein's avatar
William Stein committed
2290 2291 2292 2293 2294
        # Remember the actual type according to the function header
        # written here, because the type in the symbol table entry
        # may be different if we're overriding a C method inherited
        # from the base type of an extension type.
        self.type = type
2295
        type.is_overridable = self.overridable
2296 2297 2298
        declarator = self.declarator
        while not hasattr(declarator, 'args'):
            declarator = declarator.base
2299 2300

        self.cfunc_declarator = declarator
2301
        self.args = declarator.args
2302

2303 2304 2305
        opt_arg_count = self.cfunc_declarator.optional_arg_count
        if (self.visibility == 'public' or self.api) and opt_arg_count:
            error(self.cfunc_declarator.pos,
2306
                  "Function with optional arguments may not be declared public or api")
2307

2308
        if type.exception_check == '+' and self.visibility != 'extern':
2309 2310 2311
            warning(self.cfunc_declarator.pos,
                    "Only extern functions can throw C++ exceptions.")

2312
        for formal_arg, type_arg in zip(self.args, type.args):
2313
            self.align_argument_type(env, type_arg)
2314
            formal_arg.type = type_arg.type
2315
            formal_arg.name = type_arg.name
2316
            formal_arg.cname = type_arg.cname
2317

2318 2319
            self._validate_type_visibility(type_arg.type, type_arg.pos, env)

2320 2321 2322
            if type_arg.type.is_fused:
                self.has_fused_arguments = True

2323 2324
            if type_arg.type.is_buffer and 'inline' in self.modifiers:
                warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2325

2326
            if type_arg.type.is_buffer or type_arg.type.is_pythran_expr:
2327 2328
                if self.type.nogil:
                    error(formal_arg.pos,
2329
                          "Buffer may not be acquired without the GIL. Consider using memoryview slices instead.")
2330 2331 2332
                elif 'inline' in self.modifiers:
                    warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)

2333 2334
        self._validate_type_visibility(type.return_type, self.pos, env)

William Stein's avatar
William Stein committed
2335 2336
        name = name_declarator.name
        cname = name_declarator.cname
2337

Robert Bradshaw's avatar
Robert Bradshaw committed
2338
        type.is_const_method = self.is_const_method
Robert Bradshaw's avatar
Robert Bradshaw committed
2339
        type.is_static_method = self.is_static_method
William Stein's avatar
William Stein committed
2340
        self.entry = env.declare_cfunction(
2341
            name, type, self.pos,
2342 2343 2344
            cname=cname, visibility=self.visibility, api=self.api,
            defining=self.body is not None, modifiers=self.modifiers,
            overridable=self.overridable)
2345
        self.entry.inline_func_in_pxd = self.inline_in_pxd
William Stein's avatar
William Stein committed
2346
        self.return_type = type.return_type
2347
        if self.return_type.is_array and self.visibility != 'extern':
2348
            error(self.pos, "Function cannot return an array")
2349 2350
        if self.return_type.is_cpp_class:
            self.return_type.check_nullary_constructor(self.pos, "used as a return value")
2351

Robert Bradshaw's avatar
Robert Bradshaw committed
2352
        if self.overridable and not env.is_module_scope and not self.is_static_method:
2353 2354 2355
            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
                # An error will be produced in the cdef function
                self.overridable = False
2356

2357 2358 2359 2360
        self.declare_cpdef_wrapper(env)
        self.create_local_scope(env)

    def declare_cpdef_wrapper(self, env):
2361
        if self.overridable:
2362 2363 2364
            if self.is_static_method:
                # TODO(robertwb): Finish this up, perhaps via more function refactoring.
                error(self.pos, "static cpdef methods not yet supported")
2365
            name = self.entry.name
2366
            py_func_body = self.call_self_node(is_module_scope=env.is_module_scope)
2367 2368 2369 2370 2371 2372
            if self.is_static_method:
                from .ExprNodes import NameNode
                decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))]
                decorators[0].decorator.analyse_types(env)
            else:
                decorators = []
2373 2374 2375 2376 2377 2378 2379 2380 2381
            self.py_func = DefNode(pos=self.pos,
                                   name=self.entry.name,
                                   args=self.args,
                                   star_arg=None,
                                   starstar_arg=None,
                                   doc=self.doc,
                                   body=py_func_body,
                                   decorators=decorators,
                                   is_wrapper=1)
2382
            self.py_func.is_module_scope = env.is_module_scope
2383
            self.py_func.analyse_declarations(env)
2384
            self.py_func.entry.is_overridable = True
2385
            self.py_func_stat = StatListNode(self.pos, stats=[self.py_func])
2386
            self.py_func.type = PyrexTypes.py_object_type
2387
            self.entry.as_variable = self.py_func.entry
2388
            self.entry.used = self.entry.as_variable.used = True
2389 2390
            # Reset scope entry the above cfunction
            env.entries[name] = self.entry
2391
            if (not self.entry.is_final_cmethod and
2392 2393
                    (not env.is_module_scope or Options.lookup_module_cpdef)):
                self.override = OverrideCheckNode(self.pos, py_func=self.py_func)
2394
                self.body = StatListNode(self.pos, stats=[self.override, self.body])
2395

2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
    def _validate_type_visibility(self, type, pos, env):
        """
        Ensure that types used in cdef functions are public or api, or
        defined in a C header.
        """
        public_or_api = (self.visibility == 'public' or self.api)
        entry = getattr(type, 'entry', None)
        if public_or_api and entry and env.is_module_scope:
            if not (entry.visibility in ('public', 'extern') or
                    entry.api or entry.in_cinclude):
2406
                error(pos, "Function declared public or api may not have private types")
2407

2408
    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
2409
        from . import ExprNodes
2410 2411 2412 2413
        args = self.type.args
        if omit_optional_args:
            args = args[:len(args) - self.type.optional_arg_count]
        arg_names = [arg.name for arg in args]
2414
        if is_module_scope:
2415
            cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
2416 2417 2418 2419
            call_arg_names = arg_names
            skip_dispatch = Options.lookup_module_cpdef
        elif self.type.is_static_method:
            class_entry = self.entry.scope.parent_type.entry
2420 2421 2422
            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
            class_node.entry = class_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
2423 2424 2425
            # Calling static c(p)def methods on an instance disallowed.
            # TODO(robertwb): Support by passing self to check for override?
            skip_dispatch = True
2426
        else:
2427 2428 2429 2430
            type_entry = self.type.args[0].type.entry
            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
            type_arg.entry = type_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
2431
        skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
2432 2433 2434 2435 2436
        c_call = ExprNodes.SimpleCallNode(
            self.pos,
            function=cfunc,
            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
            wrapper_call=skip_dispatch)
2437 2438
        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)

William Stein's avatar
William Stein committed
2439 2440 2441 2442 2443
    def declare_arguments(self, env):
        for arg in self.type.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            self.declare_argument(env, arg)
2444

2445
    def need_gil_acquisition(self, lenv):
2446 2447
        return self.type.with_gil

2448
    def nogil_check(self, env):
2449
        type = self.type
2450
        with_gil = type.with_gil
2451 2452 2453 2454
        if type.nogil and not with_gil:
            if type.return_type.is_pyobject:
                error(self.pos,
                      "Function with Python return type cannot be declared nogil")
2455
            for entry in self.local_scope.var_entries:
2456
                if entry.type.is_pyobject and not entry.in_with_gil_block:
2457 2458
                    error(self.pos, "Function declared nogil has Python locals or temporaries")

2459
    def analyse_expressions(self, env):
2460
        self.local_scope.directives = env.directives
2461 2462 2463 2464
        if self.py_func_stat is not None:
            # this will also analyse the default values and the function name assignment
            self.py_func_stat = self.py_func_stat.analyse_expressions(env)
        elif self.py_func is not None:
2465
            # this will also analyse the default values
2466
            self.py_func = self.py_func.analyse_expressions(env)
2467 2468
        else:
            self.analyse_default_values(env)
2469
            self.analyse_annotations(env)
2470
        self.acquire_gil = self.need_gil_acquisition(self.local_scope)
2471
        return self
2472

Robert Bradshaw's avatar
Robert Bradshaw committed
2473 2474 2475
    def needs_assignment_synthesis(self, env, code=None):
        return False

2476
    def generate_function_header(self, code, with_pymethdef, with_opt_args=1, with_dispatch=1, cname=None):
2477
        scope = self.local_scope
William Stein's avatar
William Stein committed
2478 2479
        arg_decls = []
        type = self.type
2480
        for arg in type.args[:len(type.args)-type.optional_arg_count]:
2481 2482 2483 2484 2485
            arg_decl = arg.declaration_code()
            entry = scope.lookup(arg.name)
            if not entry.cf_used:
                arg_decl = 'CYTHON_UNUSED %s' % arg_decl
            arg_decls.append(arg_decl)
2486
        if with_dispatch and self.overridable:
2487 2488 2489 2490 2491 2492
            dispatch_arg = PyrexTypes.c_int_type.declaration_code(
                Naming.skip_dispatch_cname)
            if self.override:
                arg_decls.append(dispatch_arg)
            else:
                arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
2493 2494
        if type.optional_arg_count and with_opt_args:
            arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
William Stein's avatar
William Stein committed
2495 2496 2497 2498
        if type.has_varargs:
            arg_decls.append("...")
        if not arg_decls:
            arg_decls = ["void"]
2499 2500
        if cname is None:
            cname = self.entry.func_cname
2501
        entity = type.function_header_code(cname, ', '.join(arg_decls))
2502
        if self.entry.visibility == 'private' and '::' not in cname:
2503
            storage_class = "static "
William Stein's avatar
William Stein committed
2504
        else:
2505
            storage_class = ""
2506
        dll_linkage = None
2507
        modifiers = code.build_function_modifiers(self.entry.func_modifiers)
Robert Bradshaw's avatar
Robert Bradshaw committed
2508

2509 2510
        header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
        #print (storage_class, modifiers, header)
2511
        needs_proto = self.is_c_class_method
2512
        if self.template_declaration:
2513 2514
            if needs_proto:
                code.globalstate.parts['module_declarations'].putln(self.template_declaration)
2515
            code.putln(self.template_declaration)
2516
        if needs_proto:
2517 2518
            code.globalstate.parts['module_declarations'].putln(
                "%s%s%s; /* proto*/" % (storage_class, modifiers, header))
2519
        code.putln("%s%s%s {" % (storage_class, modifiers, header))
William Stein's avatar
William Stein committed
2520 2521

    def generate_argument_declarations(self, env, code):
2522
        scope = self.local_scope
2523
        for arg in self.args:
2524
            if arg.default:
2525
                entry = scope.lookup(arg.name)
2526
                if self.override or entry.cf_used:
2527 2528 2529
                    result = arg.calculate_default_value_code(code)
                    code.putln('%s = %s;' % (
                        arg.type.declaration_code(arg.cname), result))
2530

William Stein's avatar
William Stein committed
2531 2532
    def generate_keyword_list(self, code):
        pass
2533

2534
    def generate_argument_parsing_code(self, env, code):
2535
        i = 0
2536
        used = 0
2537
        scope = self.local_scope
2538 2539
        if self.type.optional_arg_count:
            code.putln('if (%s) {' % Naming.optional_args_cname)
2540
            for arg in self.args:
2541
                if arg.default:
2542
                    entry = scope.lookup(arg.name)
2543
                    if self.override or entry.cf_used:
2544 2545 2546 2547 2548 2549 2550 2551 2552 2553
                        code.putln('if (%s->%sn > %s) {' %
                                   (Naming.optional_args_cname,
                                    Naming.pyrex_prefix, i))
                        declarator = arg.declarator
                        while not hasattr(declarator, 'name'):
                            declarator = declarator.base
                        code.putln('%s = %s->%s;' %
                                   (arg.cname, Naming.optional_args_cname,
                                    self.type.opt_arg_cname(declarator.name)))
                        used += 1
2554
                    i += 1
2555
            for _ in range(used):
2556
                code.putln('}')
2557
            code.putln('}')
2558

2559 2560 2561 2562 2563 2564 2565 2566 2567 2568
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure and not arg.default:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(scope.lookup_here(arg.name))


William Stein's avatar
William Stein committed
2569 2570
    def generate_argument_conversion_code(self, code):
        pass
2571

William Stein's avatar
William Stein committed
2572
    def generate_argument_type_tests(self, code):
2573 2574 2575 2576 2577
        # Generate type tests for args whose type in a parent
        # class is a supertype of the declared type.
        for arg in self.type.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
2578 2579
            elif arg.type.is_pyobject and not arg.accept_none:
                self.generate_arg_none_check(arg, code)
2580

2581 2582 2583 2584 2585
    def generate_execution_code(self, code):
        super(CFuncDefNode, self).generate_execution_code(code)
        if self.py_func_stat:
            self.py_func_stat.generate_execution_code(code)

William Stein's avatar
William Stein committed
2586 2587 2588 2589
    def error_value(self):
        if self.return_type.is_pyobject:
            return "0"
        else:
2590 2591
            #return None
            return self.entry.type.exception_value
2592

William Stein's avatar
William Stein committed
2593
    def caller_will_check_exceptions(self):
2594
        return self.entry.type.exception_check
2595

2596 2597
    def generate_wrapper_functions(self, code):
        # If the C signature of a function has changed, we need to generate
2598
        # wrappers to put in the slots here.
2599 2600 2601 2602 2603 2604 2605
        k = 0
        entry = self.entry
        func_type = entry.type
        while entry.prev_entry is not None:
            k += 1
            entry = entry.prev_entry
            entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
2606
            code.putln()
2607 2608 2609 2610 2611
            self.generate_function_header(
                code, 0,
                with_dispatch=entry.type.is_overridable,
                with_opt_args=entry.type.optional_arg_count,
                cname=entry.func_cname)
2612 2613 2614 2615
            if not self.return_type.is_void:
                code.put('return ')
            args = self.type.args
            arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
2616 2617 2618 2619 2620 2621 2622 2623
            if entry.type.is_overridable:
                arglist.append(Naming.skip_dispatch_cname)
            elif func_type.is_overridable:
                arglist.append('0')
            if entry.type.optional_arg_count:
                arglist.append(Naming.optional_args_cname)
            elif func_type.optional_arg_count:
                arglist.append('NULL')
2624 2625
            code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
            code.putln('}')
2626

William Stein's avatar
William Stein committed
2627 2628 2629 2630 2631

class PyArgDeclNode(Node):
    # Argument which must be a Python object (used
    # for * and ** arguments).
    #
2632 2633 2634
    # name        string
    # entry       Symtab.Entry
    # annotation  ExprNode or None   Py3 argument annotation
2635
    child_attrs = []
2636 2637
    is_self_arg = False
    is_type_arg = False
2638 2639 2640

    def generate_function_definitions(self, env, code):
        self.entry.generate_function_definitions(env, code)
2641

2642

2643 2644 2645
class DecoratorNode(Node):
    # A decorator
    #
2646
    # decorator    NameNode or CallNode or AttributeNode
2647 2648
    child_attrs = ['decorator']

William Stein's avatar
William Stein committed
2649 2650 2651 2652 2653

class DefNode(FuncDefNode):
    # A Python function definition.
    #
    # name          string                 the Python name of the function
Stefan Behnel's avatar
Stefan Behnel committed
2654
    # lambda_name   string                 the internal name of a lambda 'function'
2655
    # decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2656
    # args          [CArgDeclNode]         formal arguments
2657
    # doc           EncodedString or None
William Stein's avatar
William Stein committed
2658
    # body          StatListNode
2659 2660
    # return_type_annotation
    #               ExprNode or None       the Py3 return type annotation
William Stein's avatar
William Stein committed
2661 2662 2663 2664
    #
    #  The following subnode is constructed internally
    #  when the def statement is inside a Python class definition.
    #
2665 2666 2667
    #  fused_py_func        DefNode     The original fused cpdef DefNode
    #                                   (in case this is a specialization)
    #  specialized_cpdefs   [DefNode]   list of specialized cpdef DefNodes
2668
    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
2669 2670
    #
    # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
2671

2672
    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
2673

2674 2675 2676
    is_staticmethod = False
    is_classmethod = False

Stefan Behnel's avatar
Stefan Behnel committed
2677
    lambda_name = None
2678
    reqd_kw_flags_cname = "0"
2679
    is_wrapper = 0
2680
    no_assignment_synthesis = 0
2681
    decorators = None
2682
    return_type_annotation = None
2683
    entry = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2684
    acquire_gil = 0
2685
    self_in_stararg = 0
2686
    py_cfunc_node = None
2687
    requires_classobj = False
2688
    defaults_struct = None # Dynamic kwrds structure name
2689
    doc = None
2690

2691 2692
    fused_py_func = False
    specialized_cpdefs = None
2693 2694 2695
    py_wrapper = None
    py_wrapper_required = True
    func_cname = None
2696

2697 2698
    defaults_getter = None

2699 2700
    def __init__(self, pos, **kwds):
        FuncDefNode.__init__(self, pos, **kwds)
2701
        k = rk = r = 0
2702 2703
        for arg in self.args:
            if arg.kw_only:
2704
                k += 1
2705
                if not arg.default:
2706 2707 2708 2709 2710 2711
                    rk += 1
            if not arg.default:
                r += 1
        self.num_kwonly_args = k
        self.num_required_kw_args = rk
        self.num_required_args = r
2712

2713
    def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, except_val=None, modifiers=None):
2714 2715 2716 2717
        if self.star_arg:
            error(self.star_arg.pos, "cdef function cannot have star argument")
        if self.starstar_arg:
            error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
2718 2719
        exception_value, exception_check = except_val or (None, False)

2720 2721 2722 2723
        if cfunc is None:
            cfunc_args = []
            for formal_arg in self.args:
                name_declarator, type = formal_arg.analyse(scope, nonempty=1)
2724 2725 2726 2727 2728 2729 2730 2731
                cfunc_args.append(PyrexTypes.CFuncTypeArg(name=name_declarator.name,
                                                          cname=None,
                                                          type=py_object_type,
                                                          pos=formal_arg.pos))
            cfunc_type = PyrexTypes.CFuncType(return_type=py_object_type,
                                              args=cfunc_args,
                                              has_varargs=False,
                                              exception_value=None,
2732
                                              exception_check=exception_check,
2733 2734 2735
                                              nogil=False,
                                              with_gil=False,
                                              is_overridable=overridable)
2736
            cfunc = CVarDefNode(self.pos, type=cfunc_type)
2737
        else:
2738 2739
            if scope is None:
                scope = cfunc.scope
2740 2741 2742
            cfunc_type = cfunc.type
            if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
                error(self.pos, "wrong number of arguments")
Stefan Behnel's avatar
Stefan Behnel committed
2743
                error(cfunc.pos, "previous declaration here")
2744 2745
            for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
                name_declarator, type = formal_arg.analyse(scope, nonempty=1,
2746
                                                           is_self_arg=(i == 0 and scope.is_c_class_scope))
Stefan Behnel's avatar
Stefan Behnel committed
2747
                if type is None or type is PyrexTypes.py_object_type:
2748 2749
                    formal_arg.type = type_arg.type
                    formal_arg.name_declarator = name_declarator
2750 2751 2752 2753

        if exception_value is None and cfunc_type.exception_value is not None:
            from .ExprNodes import ConstNode
            exception_value = ConstNode(
2754
                self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
2755
        declarator = CFuncDeclaratorNode(self.pos,
2756 2757 2758 2759 2760 2761 2762
                                         base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
                                         args=self.args,
                                         has_varargs=False,
                                         exception_check=cfunc_type.exception_check,
                                         exception_value=exception_value,
                                         with_gil=cfunc_type.with_gil,
                                         nogil=cfunc_type.nogil)
2763
        return CFuncDefNode(self.pos,
2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776
                            modifiers=modifiers or [],
                            base_type=CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
                            declarator=declarator,
                            body=self.body,
                            doc=self.doc,
                            overridable=cfunc_type.is_overridable,
                            type=cfunc_type,
                            with_gil=cfunc_type.with_gil,
                            nogil=cfunc_type.nogil,
                            visibility='private',
                            api=False,
                            directive_locals=getattr(cfunc, 'directive_locals', {}),
                            directive_returns=returns)
2777

2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788
    def is_cdef_func_compatible(self):
        """Determines if the function's signature is compatible with a
        cdef function.  This can be used before calling
        .as_cfunction() to see if that will be successful.
        """
        if self.needs_closure:
            return False
        if self.star_arg or self.starstar_arg:
            return False
        return True

William Stein's avatar
William Stein committed
2789
    def analyse_declarations(self, env):
2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803
        if self.decorators:
            for decorator in self.decorators:
                func = decorator.decorator
                if func.is_name:
                    self.is_classmethod |= func.name == 'classmethod'
                    self.is_staticmethod |= func.name == 'staticmethod'

        if self.is_classmethod and env.lookup_here('classmethod'):
            # classmethod() was overridden - not much we can do here ...
            self.is_classmethod = False
        if self.is_staticmethod and env.lookup_here('staticmethod'):
            # staticmethod() was overridden - not much we can do here ...
            self.is_staticmethod = False

2804
        if self.name == '__new__' and env.is_py_class_scope:
Vitja Makarov's avatar
Vitja Makarov committed
2805 2806
            self.is_staticmethod = 1

2807
        self.analyse_argument_types(env)
2808 2809 2810 2811
        if self.name == '<lambda>':
            self.declare_lambda_function(env)
        else:
            self.declare_pyfunction(env)
2812

2813 2814
        self.analyse_signature(env)
        self.return_type = self.entry.signature.return_type()
2815 2816 2817
        # if a signature annotation provides a more specific return object type, use it
        if self.return_type is py_object_type and self.return_type_annotation:
            if env.directives['annotation_typing'] and not self.entry.is_special:
2818
                _, return_type = analyse_type_annotation(self.return_type_annotation, env)
2819 2820 2821
                if return_type and return_type.is_pyobject:
                    self.return_type = return_type

2822 2823
        self.create_local_scope(env)

2824 2825 2826 2827 2828 2829 2830 2831 2832 2833
        self.py_wrapper = DefNodeWrapper(
            self.pos,
            target=self,
            name=self.entry.name,
            args=self.args,
            star_arg=self.star_arg,
            starstar_arg=self.starstar_arg,
            return_type=self.return_type)
        self.py_wrapper.analyse_declarations(env)

2834
    def analyse_argument_types(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
2835
        self.directive_locals = env.directives['locals']
2836
        allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
2837 2838 2839 2840

        f2s = env.fused_to_specific
        env.fused_to_specific = None

William Stein's avatar
William Stein committed
2841
        for arg in self.args:
2842 2843 2844 2845
            if hasattr(arg, 'name'):
                name_declarator = None
            else:
                base_type = arg.base_type.analyse(env)
2846 2847 2848 2849 2850 2851 2852
                # If we hare in pythran mode and we got a buffer supported by
                # Pythran, we change this node to a fused type
                if has_np_pythran(env) and base_type.is_pythran_expr:
                    base_type = PyrexTypes.FusedType([
                        base_type,
                        #PyrexTypes.PythranExpr(pythran_type(self.type, "numpy_texpr")),
                        base_type.org_buffer])
2853 2854 2855
                name_declarator, type = \
                    arg.declarator.analyse(base_type, env)
                arg.name = name_declarator.name
2856
                arg.type = type
2857 2858 2859 2860

                if type.is_fused:
                    self.has_fused_arguments = True

2861
            self.align_argument_type(env, arg)
2862
            if name_declarator and name_declarator.cname:
2863
                error(self.pos, "Python function argument cannot have C name specification")
2864
            arg.type = arg.type.as_argument_type()
William Stein's avatar
William Stein committed
2865 2866 2867 2868
            arg.hdr_type = None
            arg.needs_conversion = 0
            arg.needs_type_test = 0
            arg.is_generic = 1
2869
            if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
2870 2871 2872 2873
                if arg.or_none:
                    arg.accept_none = True
                elif arg.not_none:
                    arg.accept_none = False
2874
                elif (arg.type.is_extension_type or arg.type.is_builtin_type
2875
                        or arg.type.is_buffer or arg.type.is_memoryviewslice):
2876 2877 2878 2879 2880 2881
                    if arg.default and arg.default.constant_result is None:
                        # special case: def func(MyType obj = None)
                        arg.accept_none = True
                    else:
                        # default depends on compiler directive
                        arg.accept_none = allow_none_for_extension_args
2882 2883 2884
                else:
                    # probably just a plain 'object'
                    arg.accept_none = True
2885
            else:
2886
                arg.accept_none = True # won't be used, but must be there
2887
                if arg.not_none:
2888
                    error(arg.pos, "Only Python type arguments can have 'not None'")
2889
                if arg.or_none:
2890
                    error(arg.pos, "Only Python type arguments can have 'or None'")
2891 2892
        env.fused_to_specific = f2s

2893 2894 2895 2896 2897
        if has_np_pythran(env):
            self.np_args_idx = [i for i,a in enumerate(self.args) if a.type.is_numpy_buffer]
        else:
            self.np_args_idx = []

William Stein's avatar
William Stein committed
2898
    def analyse_signature(self, env):
2899
        if self.entry.is_special:
2900
            if self.decorators:
2901
                error(self.pos, "special functions of cdef classes cannot have decorators")
2902 2903 2904 2905
            self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
        elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg):
            # Use the simpler calling signature for zero- and one-argument functions.
            if self.entry.signature is TypeSlots.pyfunction_signature:
2906 2907
                if len(self.args) == 0:
                    self.entry.signature = TypeSlots.pyfunction_noargs
2908 2909 2910
                elif len(self.args) == 1:
                    if self.args[0].default is None and not self.args[0].kw_only:
                        self.entry.signature = TypeSlots.pyfunction_onearg
2911 2912 2913
            elif self.entry.signature is TypeSlots.pymethod_signature:
                if len(self.args) == 1:
                    self.entry.signature = TypeSlots.unaryfunc
2914 2915 2916
                elif len(self.args) == 2:
                    if self.args[1].default is None and not self.args[1].kw_only:
                        self.entry.signature = TypeSlots.ibinaryfunc
2917

William Stein's avatar
William Stein committed
2918 2919
        sig = self.entry.signature
        nfixed = sig.num_fixed_args()
2920 2921
        if (sig is TypeSlots.pymethod_signature and nfixed == 1
               and len(self.args) == 0 and self.star_arg):
2922 2923 2924 2925 2926 2927 2928
            # this is the only case where a diverging number of
            # arguments is not an error - when we have no explicit
            # 'self' parameter as in method(*args)
            sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used
            self.self_in_stararg = 1
            nfixed = 0

2929 2930
        if self.is_staticmethod and env.is_c_class_scope:
            nfixed = 0
2931
            self.self_in_stararg = True  # FIXME: why for staticmethods?
2932 2933 2934 2935 2936 2937

            self.entry.signature = sig = copy.copy(sig)
            sig.fixed_arg_format = "*"
            sig.is_staticmethod = True
            sig.has_generic_args = True

2938
        if ((self.is_classmethod or self.is_staticmethod) and
2939
                self.has_fused_arguments and env.is_c_class_scope):
2940 2941
            del self.decorator_indirection.stats[:]

2942 2943 2944 2945 2946 2947 2948
        for i in range(min(nfixed, len(self.args))):
            arg = self.args[i]
            arg.is_generic = 0
            if sig.is_self_arg(i) and not self.is_staticmethod:
                if self.is_classmethod:
                    arg.is_type_arg = 1
                    arg.hdr_type = arg.type = Builtin.type_type
William Stein's avatar
William Stein committed
2949
                else:
2950 2951 2952
                    arg.is_self_arg = 1
                    arg.hdr_type = arg.type = env.parent_type
                arg.needs_conversion = 0
William Stein's avatar
William Stein committed
2953
            else:
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968
                arg.hdr_type = sig.fixed_arg_type(i)
                if not arg.type.same_as(arg.hdr_type):
                    if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
                        arg.needs_type_test = 1
                    else:
                        arg.needs_conversion = 1
            if arg.needs_conversion:
                arg.hdr_cname = Naming.arg_prefix + arg.name
            else:
                arg.hdr_cname = Naming.var_prefix + arg.name

        if nfixed > len(self.args):
            self.bad_signature()
            return
        elif nfixed < len(self.args):
William Stein's avatar
William Stein committed
2969 2970 2971
            if not sig.has_generic_args:
                self.bad_signature()
            for arg in self.args:
2972
                if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type):
William Stein's avatar
William Stein committed
2973
                    arg.needs_type_test = 1
2974

William Stein's avatar
William Stein committed
2975 2976 2977 2978
    def bad_signature(self):
        sig = self.entry.signature
        expected_str = "%d" % sig.num_fixed_args()
        if sig.has_generic_args:
Stefan Behnel's avatar
Stefan Behnel committed
2979
            expected_str += " or more"
William Stein's avatar
William Stein committed
2980 2981 2982 2983 2984
        name = self.name
        if name.startswith("__") and name.endswith("__"):
            desc = "Special method"
        else:
            desc = "Method"
2985 2986
        error(self.pos, "%s %s has wrong number of arguments (%d declared, %s expected)" % (
            desc, self.name, len(self.args), expected_str))
2987

William Stein's avatar
William Stein committed
2988
    def declare_pyfunction(self, env):
2989 2990
        #print "DefNode.declare_pyfunction:", self.name, "in", env ###
        name = self.name
Stefan Behnel's avatar
Stefan Behnel committed
2991
        entry = env.lookup_here(name)
2992 2993
        if entry:
            if entry.is_final_cmethod and not env.parent_type.is_final_type:
Stefan Behnel's avatar
Stefan Behnel committed
2994
                error(self.pos, "Only final types can have final Python (def/cpdef) methods")
2995
            if entry.type.is_cfunction and not entry.is_builtin_cmethod and not self.is_wrapper:
2996
                warning(self.pos, "Overriding cdef method with def method.", 5)
2997
        entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
2998
        self.entry = entry
2999
        prefix = env.next_id(env.scope_prefix)
3000
        self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name
3001 3002
        if Options.docstrings:
            entry.doc = embed_position(self.pos, self.doc)
3003
            entry.doc_cname = Naming.funcdoc_prefix + prefix + name
3004
            if entry.is_special:
3005 3006
                if entry.name in TypeSlots.invisible or not entry.doc or (
                        entry.name in '__getattr__' and env.directives['fast_getattr']):
3007 3008 3009
                    entry.wrapperbase_cname = None
                else:
                    entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name
3010 3011
        else:
            entry.doc = None
3012

Stefan Behnel's avatar
Stefan Behnel committed
3013
    def declare_lambda_function(self, env):
3014
        entry = env.declare_lambda_function(self.lambda_name, self.pos)
Stefan Behnel's avatar
Stefan Behnel committed
3015 3016
        entry.doc = None
        self.entry = entry
3017
        self.entry.pyfunc_cname = entry.cname
Stefan Behnel's avatar
Stefan Behnel committed
3018

William Stein's avatar
William Stein committed
3019 3020 3021 3022 3023 3024 3025 3026 3027 3028
    def declare_arguments(self, env):
        for arg in self.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            if arg.needs_conversion:
                arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
                if arg.type.is_pyobject:
                    arg.entry.init = "0"
            else:
                arg.entry = self.declare_argument(env, arg)
3029
            arg.entry.is_arg = 1
3030
            arg.entry.used = 1
William Stein's avatar
William Stein committed
3031 3032 3033 3034 3035 3036
            arg.entry.is_self_arg = arg.is_self_arg
        self.declare_python_arg(env, self.star_arg)
        self.declare_python_arg(env, self.starstar_arg)

    def declare_python_arg(self, env, arg):
        if arg:
3037
            if env.directives['infer_types'] != False:
3038 3039 3040 3041
                type = PyrexTypes.unspecified_type
            else:
                type = py_object_type
            entry = env.declare_var(arg.name, type, arg.pos)
3042
            entry.is_arg = 1
3043 3044 3045 3046
            entry.used = 1
            entry.init = "0"
            entry.xdecref_cleanup = 1
            arg.entry = entry
3047

William Stein's avatar
William Stein committed
3048
    def analyse_expressions(self, env):
3049
        self.local_scope.directives = env.directives
William Stein's avatar
William Stein committed
3050
        self.analyse_default_values(env)
3051 3052
        self.analyse_annotations(env)
        if self.return_type_annotation:
3053
            self.return_type_annotation = self.analyse_annotation(env, self.return_type_annotation)
3054

3055
        if not self.needs_assignment_synthesis(env) and self.decorators:
3056
            for decorator in self.decorators[::-1]:
3057
                decorator.decorator = decorator.decorator.analyse_expressions(env)
3058

3059
        self.py_wrapper.prepare_argument_coercion(env)
3060
        return self
3061

3062
    def needs_assignment_synthesis(self, env, code=None):
3063
        if self.is_staticmethod:
3064
            return True
3065
        if self.specialized_cpdefs or self.entry.is_fused_specialized:
3066
            return False
3067 3068
        if self.no_assignment_synthesis:
            return False
3069 3070
        if self.entry.is_special:
            return False
3071
        if self.entry.is_anonymous:
3072
            return True
3073
        if env.is_module_scope or env.is_c_class_scope:
3074
            if code is None:
3075
                return self.local_scope.directives['binding']
3076 3077 3078 3079
            else:
                return code.globalstate.directives['binding']
        return env.is_py_class_scope or env.is_closure_scope

3080 3081 3082 3083
    def error_value(self):
        return self.entry.signature.error_value

    def caller_will_check_exceptions(self):
3084
        return self.entry.signature.exception_check
3085 3086

    def generate_function_definitions(self, env, code):
3087
        if self.defaults_getter:
3088 3089
            # defaults getter must never live in class scopes, it's always a module function
            self.defaults_getter.generate_function_definitions(env.global_scope(), code)
3090

3091 3092 3093 3094 3095 3096 3097
        # Before closure cnames are mangled
        if self.py_wrapper_required:
            # func_cname might be modified by @cname
            self.py_wrapper.func_cname = self.entry.func_cname
            self.py_wrapper.generate_function_definitions(env, code)
        FuncDefNode.generate_function_definitions(self, env, code)

3098
    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3099 3100 3101 3102 3103
        if proto_only:
            if self.py_wrapper_required:
                self.py_wrapper.generate_function_header(
                    code, with_pymethdef, True)
            return
William Stein's avatar
William Stein committed
3104
        arg_code_list = []
3105
        if self.entry.signature.has_dummy_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3106 3107 3108
            self_arg = 'PyObject *%s' % Naming.self_cname
            if not self.needs_outer_scope:
                self_arg = 'CYTHON_UNUSED ' + self_arg
3109 3110 3111 3112 3113 3114 3115 3116
            arg_code_list.append(self_arg)

        def arg_decl_code(arg):
            entry = arg.entry
            if entry.in_closure:
                cname = entry.original_cname
            else:
                cname = entry.cname
3117
            decl = entry.type.declaration_code(cname)
Stefan Behnel's avatar
Stefan Behnel committed
3118 3119 3120
            if not entry.cf_used:
                decl = 'CYTHON_UNUSED ' + decl
            return decl
3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186

        for arg in self.args:
            arg_code_list.append(arg_decl_code(arg))
        if self.star_arg:
            arg_code_list.append(arg_decl_code(self.star_arg))
        if self.starstar_arg:
            arg_code_list.append(arg_decl_code(self.starstar_arg))
        arg_code = ', '.join(arg_code_list)
        dc = self.return_type.declaration_code(self.entry.pyfunc_cname)

        decls_code = code.globalstate['decls']
        preprocessor_guard = self.get_preprocessor_guard()
        if preprocessor_guard:
            decls_code.putln(preprocessor_guard)
        decls_code.putln(
            "static %s(%s); /* proto */" % (dc, arg_code))
        if preprocessor_guard:
            decls_code.putln("#endif")
        code.putln("static %s(%s) {" % (dc, arg_code))

    def generate_argument_declarations(self, env, code):
        pass

    def generate_keyword_list(self, code):
        pass

    def generate_argument_parsing_code(self, env, code):
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(arg.entry)
        for arg in self.star_arg, self.starstar_arg:
            if arg:
                put_into_closure(arg.entry)

    def generate_argument_type_tests(self, code):
        pass


class DefNodeWrapper(FuncDefNode):
    # DefNode python wrapper code generator

    defnode = None
    target = None # Target DefNode

    def __init__(self, *args, **kwargs):
        FuncDefNode.__init__(self, *args, **kwargs)
        self.num_kwonly_args = self.target.num_kwonly_args
        self.num_required_kw_args = self.target.num_required_kw_args
        self.num_required_args = self.target.num_required_args
        self.self_in_stararg = self.target.self_in_stararg
        self.signature = None

    def analyse_declarations(self, env):
        target_entry = self.target.entry
        name = self.name
        prefix = env.next_id(env.scope_prefix)
        target_entry.func_cname = Naming.pywrap_prefix + prefix + name
        target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name

        self.signature = target_entry.signature

3187 3188
        self.np_args_idx = self.target.np_args_idx

3189
    def prepare_argument_coercion(self, env):
3190 3191 3192 3193 3194 3195 3196 3197
        # This is only really required for Cython utility code at this time,
        # everything else can be done during code generation.  But we expand
        # all utility code here, simply because we cannot easily distinguish
        # different code types.
        for arg in self.args:
            if not arg.type.is_pyobject:
                if not arg.type.create_from_py_utility_code(env):
                    pass # will fail later
Robert Bradshaw's avatar
Robert Bradshaw committed
3198
            elif arg.hdr_type and not arg.hdr_type.is_pyobject:
3199 3200
                if not arg.hdr_type.create_to_py_utility_code(env):
                    pass # will fail later
3201

3202 3203 3204 3205 3206 3207 3208 3209 3210
        if self.starstar_arg and not self.starstar_arg.entry.cf_used:
            # we will set the kwargs argument to NULL instead of a new dict
            # and must therefore correct the control flow state
            entry = self.starstar_arg.entry
            entry.xdecref_cleanup = 1
            for ass in entry.cf_assignments:
                if not ass.is_arg and ass.lhs.is_name:
                    ass.lhs.cf_maybe_null = True

3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226
    def signature_has_nongeneric_args(self):
        argcount = len(self.args)
        if argcount == 0 or (
                argcount == 1 and (self.args[0].is_self_arg or
                                   self.args[0].is_type_arg)):
            return 0
        return 1

    def signature_has_generic_args(self):
        return self.signature.has_generic_args

    def generate_function_body(self, code):
        args = []
        if self.signature.has_dummy_arg:
            args.append(Naming.self_cname)
        for arg in self.args:
3227 3228 3229
            if arg.hdr_type and not (arg.type.is_memoryviewslice or
                                     arg.type.is_struct or
                                     arg.type.is_complex):
3230 3231 3232
                args.append(arg.type.cast_code(arg.entry.cname))
            else:
                args.append(arg.entry.cname)
3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245
        if self.star_arg:
            args.append(self.star_arg.entry.cname)
        if self.starstar_arg:
            args.append(self.starstar_arg.entry.cname)
        args = ', '.join(args)
        if not self.return_type.is_void:
            code.put('%s = ' % Naming.retval_cname)
        code.putln('%s(%s);' % (
            self.target.entry.pyfunc_cname, args))

    def generate_function_definitions(self, env, code):
        lenv = self.target.local_scope
        # Generate C code for header and body of function
3246
        code.mark_pos(self.pos)
3247 3248 3249 3250 3251 3252
        code.putln("")
        code.putln("/* Python wrapper */")
        preprocessor_guard = self.target.get_preprocessor_guard()
        if preprocessor_guard:
            code.putln(preprocessor_guard)

3253
        code.enter_cfunc_scope(lenv)
3254 3255
        code.return_from_error_cleanup_label = code.new_label()

Vitja Makarov's avatar
Vitja Makarov committed
3256
        with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279
                          self.target.pymethdef_required)
        self.generate_function_header(code, with_pymethdef)
        self.generate_argument_declarations(lenv, code)
        tempvardecl_code = code.insertion_point()

        if self.return_type.is_pyobject:
            retval_init = ' = 0'
        else:
            retval_init = ''
        if not self.return_type.is_void:
            code.putln('%s%s;' % (
                self.return_type.declaration_code(Naming.retval_cname),
                retval_init))
        code.put_declare_refcount_context()
        code.put_setup_refcount_context('%s (wrapper)' % self.name)

        self.generate_argument_parsing_code(lenv, code)
        self.generate_argument_type_tests(code)
        self.generate_function_body(code)

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)

3280 3281 3282 3283
        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

3284 3285 3286 3287 3288 3289
        # ----- Error cleanup
        if code.error_label in code.labels_used:
            code.put_goto(code.return_label)
            code.put_label(code.error_label)
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
3290 3291 3292
            err_val = self.error_value()
            if err_val is not None:
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
        for entry in lenv.var_entries:
            if entry.is_arg and entry.type.is_pyobject:
                code.put_var_decref(entry)

        code.put_finish_refcount_context()
        if not self.return_type.is_void:
            code.putln("return %s;" % Naming.retval_cname)
        code.putln('}')
        code.exit_cfunc_scope()
        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

    def generate_function_header(self, code, with_pymethdef, proto_only=0):
        arg_code_list = []
        sig = self.signature

3312
        if sig.has_dummy_arg or self.self_in_stararg:
3313 3314 3315 3316
            arg_code = "PyObject *%s" % Naming.self_cname
            if not sig.has_dummy_arg:
                arg_code = 'CYTHON_UNUSED ' + arg_code
            arg_code_list.append(arg_code)
3317

William Stein's avatar
William Stein committed
3318 3319
        for arg in self.args:
            if not arg.is_generic:
3320
                if arg.is_self_arg or arg.is_type_arg:
William Stein's avatar
William Stein committed
3321 3322
                    arg_code_list.append("PyObject *%s" % arg.hdr_cname)
                else:
3323 3324 3325 3326
                    arg_code_list.append(
                        arg.hdr_type.declaration_code(arg.hdr_cname))
        entry = self.target.entry
        if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
3327
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3328
        if entry.scope.is_c_class_scope and entry.name == "__ipow__":
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3329
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
William Stein's avatar
William Stein committed
3330 3331
        if sig.has_generic_args:
            arg_code_list.append(
3332 3333
                "PyObject *%s, PyObject *%s" % (
                    Naming.args_cname, Naming.kwds_cname))
William Stein's avatar
William Stein committed
3334
        arg_code = ", ".join(arg_code_list)
3335 3336 3337 3338

        # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
        mf = ""
        if (entry.name in ("__getbuffer__", "__releasebuffer__")
3339
                and entry.scope.is_c_class_scope):
3340
            mf = "CYTHON_UNUSED "
3341
            with_pymethdef = False
3342

3343
        dc = self.return_type.declaration_code(entry.func_cname)
3344
        header = "static %s%s(%s)" % (mf, dc, arg_code)
William Stein's avatar
William Stein committed
3345
        code.putln("%s; /*proto*/" % header)
3346

3347
        if proto_only:
3348
            if self.target.fused_py_func:
3349 3350 3351
                # If we are the specialized version of the cpdef, we still
                # want the prototype for the "fused cpdef", in case we're
                # checking to see if our method was overridden in Python
3352
                self.target.fused_py_func.generate_function_header(
3353
                    code, with_pymethdef, proto_only=True)
3354
            return
3355

3356 3357 3358 3359
        if (Options.docstrings and entry.doc and
                not self.target.fused_py_func and
                not entry.scope.is_property_scope and
                (not entry.is_special or entry.wrapperbase_cname)):
3360
            # h_code = code.globalstate['h_code']
3361
            docstr = entry.doc
3362

Stefan Behnel's avatar
Stefan Behnel committed
3363
            if docstr.is_unicode:
3364
                docstr = docstr.as_utf8_string()
3365

William Stein's avatar
William Stein committed
3366
            code.putln(
3367
                'static char %s[] = %s;' % (
3368
                    entry.doc_cname,
3369
                    docstr.as_c_string_literal()))
3370

3371
            if entry.is_special:
3372
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
3373
                code.putln(
3374
                    "struct wrapperbase %s;" % entry.wrapperbase_cname)
3375
                code.putln('#endif')
3376

3377
        if with_pymethdef or self.target.fused_py_func:
William Stein's avatar
William Stein committed
3378
            code.put(
3379
                "static PyMethodDef %s = " % entry.pymethdef_cname)
3380
            code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
William Stein's avatar
William Stein committed
3381 3382 3383 3384
        code.putln("%s {" % header)

    def generate_argument_declarations(self, env, code):
        for arg in self.args:
3385
            if arg.is_generic:
3386 3387
                if arg.needs_conversion:
                    code.putln("PyObject *%s = 0;" % arg.hdr_cname)
3388
                else:
3389
                    code.put_var_declaration(arg.entry)
3390 3391 3392
        for entry in env.var_entries:
            if entry.is_arg:
                code.put_var_declaration(entry)
3393

3394
    def generate_argument_parsing_code(self, env, code):
Stefan Behnel's avatar
Stefan Behnel committed
3395 3396
        # Generate fast equivalent of PyArg_ParseTuple call for
        # generic arguments, if any, including args/kwargs
3397 3398
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
Stefan Behnel's avatar
Stefan Behnel committed
3399
        end_label = code.new_label("argument_unpacking_done")
3400

3401 3402 3403
        has_kwonly_args = self.num_kwonly_args > 0
        has_star_or_kw_args = self.star_arg is not None \
            or self.starstar_arg is not None or has_kwonly_args
3404

3405
        for arg in self.args:
3406
            if not arg.type.is_pyobject:
3407
                if not arg.type.create_from_py_utility_code(env):
3408
                    pass  # will fail later
3409

3410
        if not self.signature_has_generic_args():
3411 3412
            if has_star_or_kw_args:
                error(self.pos, "This method cannot have * or keyword arguments")
3413
            self.generate_argument_conversion_code(code)
3414

3415 3416
        elif not self.signature_has_nongeneric_args():
            # func(*args) or func(**kw) or func(*args, **kw)
3417
            self.generate_stararg_copy_code(code)
3418

3419
        else:
3420
            self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
3421

3422 3423
        code.error_label = old_error_label
        if code.label_used(our_error_label):
3424 3425
            if not code.label_used(end_label):
                code.put_goto(end_label)
3426 3427 3428 3429 3430
            code.put_label(our_error_label)
            if has_star_or_kw_args:
                self.generate_arg_decref(self.star_arg, code)
                if self.starstar_arg:
                    if self.starstar_arg.entry.xdecref_cleanup:
3431
                        code.put_var_xdecref_clear(self.starstar_arg.entry)
3432
                    else:
3433
                        code.put_var_decref_clear(self.starstar_arg.entry)
3434
            code.put_add_traceback(self.target.entry.qualified_name)
3435
            code.put_finish_refcount_context()
3436
            code.putln("return %s;" % self.error_value())
3437
        if code.label_used(end_label):
3438 3439
            code.put_label(end_label)

William Stein's avatar
William Stein committed
3440 3441
    def generate_arg_xdecref(self, arg, code):
        if arg:
3442
            code.put_var_xdecref_clear(arg.entry)
3443

3444 3445
    def generate_arg_decref(self, arg, code):
        if arg:
3446
            code.put_var_decref_clear(arg.entry)
William Stein's avatar
William Stein committed
3447

3448 3449
    def generate_stararg_copy_code(self, code):
        if not self.star_arg:
3450 3451
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3452 3453 3454
            code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" %
                       Naming.args_cname)
            code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % (
3455
                self.name, Naming.args_cname, self.error_value()))
3456
            code.putln("}")
3457

3458
        if self.starstar_arg:
3459
            if self.star_arg or not self.starstar_arg.entry.cf_used:
3460 3461 3462 3463 3464 3465
                kwarg_check = "unlikely(%s)" % Naming.kwds_cname
            else:
                kwarg_check = "%s" % Naming.kwds_cname
        else:
            kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                Naming.kwds_cname, Naming.kwds_cname)
3466 3467
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
3468
        code.putln(
3469 3470
            "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                kwarg_check, Naming.kwds_cname, self.name,
3471
                bool(self.starstar_arg), self.error_value()))
3472

3473 3474
        if self.starstar_arg and self.starstar_arg.entry.cf_used:
            if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references):
3475 3476
                code.putln("if (%s) {" % kwarg_check)
                code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % (
3477 3478 3479 3480
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    self.starstar_arg.entry.cname,
                    self.error_value()))
3481 3482
                code.put_gotref(self.starstar_arg.entry.cname)
                code.putln("} else {")
3483
                code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,))
3484 3485 3486
                code.putln("}")
                self.starstar_arg.entry.xdecref_cleanup = 1
            else:
3487
                code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % (
3488 3489 3490
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    Naming.kwds_cname))
3491
                code.putln("if (unlikely(!%s)) return %s;" % (
3492
                    self.starstar_arg.entry.cname, self.error_value()))
3493 3494
                self.starstar_arg.entry.xdecref_cleanup = 0
                code.put_gotref(self.starstar_arg.entry.cname)
3495

3496
        if self.self_in_stararg and not self.target.is_staticmethod:
3497 3498
            # need to create a new tuple with 'self' inserted as first item
            code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % (
3499 3500 3501
                self.star_arg.entry.cname,
                Naming.args_cname,
                self.star_arg.entry.cname))
3502
            if self.starstar_arg and self.starstar_arg.entry.cf_used:
3503
                code.putln("{")
3504
                code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type)
3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526
                code.putln("return %s;" % self.error_value())
                code.putln("}")
            else:
                code.putln("return %s;" % self.error_value())
            code.put_gotref(self.star_arg.entry.cname)
            code.put_incref(Naming.self_cname, py_object_type)
            code.put_giveref(Naming.self_cname)
            code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
                self.star_arg.entry.cname, Naming.self_cname))
            temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % (
                temp, temp, Naming.args_cname, temp))
            code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
                Naming.args_cname, temp))
            code.put_incref("item", py_object_type)
            code.put_giveref("item")
            code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
                self.star_arg.entry.cname, temp))
            code.putln("}")
            code.funcstate.release_temp(temp)
            self.star_arg.entry.xdecref_cleanup = 0
        elif self.star_arg:
3527 3528
            code.put_incref(Naming.args_cname, py_object_type)
            code.putln("%s = %s;" % (
3529 3530
                self.star_arg.entry.cname,
                Naming.args_cname))
3531 3532
            self.star_arg.entry.xdecref_cleanup = 0

3533
    def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
3534
        argtuple_error_label = code.new_label("argtuple_error")
3535

3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555
        positional_args = []
        required_kw_only_args = []
        optional_kw_only_args = []
        for arg in args:
            if arg.is_generic:
                if arg.default:
                    if not arg.is_self_arg and not arg.is_type_arg:
                        if arg.kw_only:
                            optional_kw_only_args.append(arg)
                        else:
                            positional_args.append(arg)
                elif arg.kw_only:
                    required_kw_only_args.append(arg)
                elif not arg.is_self_arg and not arg.is_type_arg:
                    positional_args.append(arg)

        # sort required kw-only args before optional ones to avoid special
        # cases in the unpacking code
        kw_only_args = required_kw_only_args + optional_kw_only_args

3556
        min_positional_args = self.num_required_args - self.num_required_kw_args
3557
        if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
3558 3559
            min_positional_args -= 1
        max_positional_args = len(positional_args)
3560 3561
        has_fixed_positional_count = not self.star_arg and \
            min_positional_args == max_positional_args
3562
        has_kw_only_args = bool(kw_only_args)
3563

Stefan Behnel's avatar
Stefan Behnel committed
3564
        if self.num_required_kw_args:
3565 3566
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
Stefan Behnel's avatar
Stefan Behnel committed
3567

3568 3569 3570
        if self.starstar_arg or self.star_arg:
            self.generate_stararg_init_code(max_positional_args, code)

3571 3572
        code.putln('{')
        all_args = tuple(positional_args) + tuple(kw_only_args)
3573 3574
        code.putln("static PyObject **%s[] = {%s,0};" % (
            Naming.pykwdlist_cname,
3575 3576
            ','.join(['&%s' % code.intern_identifier(arg.name)
                      for arg in all_args])))
3577 3578 3579 3580 3581 3582 3583 3584 3585 3586

        # Before being converted and assigned to the target variables,
        # borrowed references to all unpacked argument values are
        # collected into a local PyObject* array called "values",
        # regardless if they were taken from default arguments,
        # positional arguments or keyword arguments.  Note that
        # C-typed default arguments are handled at conversion time,
        # so their array value is NULL in the end if no argument
        # was passed for them.
        self.generate_argument_values_setup_code(all_args, code)
3587

3588
        # --- optimised code when we receive keyword arguments
3589 3590 3591
        code.putln("if (%s(%s)) {" % (
            (self.num_required_kw_args > 0) and "likely" or "unlikely",
            Naming.kwds_cname))
3592 3593
        self.generate_keyword_unpacking_code(
            min_positional_args, max_positional_args,
3594 3595
            has_fixed_positional_count, has_kw_only_args,
            all_args, argtuple_error_label, code)
3596 3597

        # --- optimised code when we do not receive any keyword arguments
3598
        if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
3599 3600 3601 3602 3603 3604 3605
            # Python raises arg tuple related errors first, so we must
            # check the length here
            if min_positional_args == max_positional_args and not self.star_arg:
                compare = '!='
            else:
                compare = '<'
            code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % (
3606
                Naming.args_cname, compare, min_positional_args))
3607
            code.put_goto(argtuple_error_label)
3608

3609 3610 3611 3612
        if self.num_required_kw_args:
            # pure error case: keywords required but not passed
            if max_positional_args > min_positional_args and not self.star_arg:
                code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % (
3613
                    Naming.args_cname, max_positional_args))
3614 3615 3616 3617
                code.put_goto(argtuple_error_label)
            code.putln('} else {')
            for i, arg in enumerate(kw_only_args):
                if not arg.default:
3618
                    pystring_cname = code.intern_identifier(arg.name)
3619
                    # required keyword-only argument missing
3620
                    code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
3621 3622
                        self.name,
                        pystring_cname))
3623 3624
                    code.putln(code.error_goto(self.pos))
                    break
3625

3626
        else:
3627
            # optimised tuple unpacking code
3628
            code.putln('} else {')
3629 3630 3631 3632 3633
            if min_positional_args == max_positional_args:
                # parse the exact number of positional arguments from
                # the args tuple
                for i, arg in enumerate(positional_args):
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
3634
            else:
3635 3636 3637 3638 3639 3640 3641 3642
                # parse the positional arguments from the variable length
                # args tuple and reject illegal argument tuple sizes
                code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname)
                if self.star_arg:
                    code.putln('default:')
                reversed_args = list(enumerate(positional_args))[::-1]
                for i, arg in reversed_args:
                    if i >= min_positional_args-1:
Robert Bradshaw's avatar
Robert Bradshaw committed
3643 3644
                        if i != reversed_args[0][0]:
                            code.putln('CYTHON_FALLTHROUGH;')
3645
                        code.put('case %2d: ' % (i+1))
3646 3647
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
                if min_positional_args == 0:
Robert Bradshaw's avatar
Robert Bradshaw committed
3648
                    code.putln('CYTHON_FALLTHROUGH;')
3649 3650 3651 3652 3653 3654 3655 3656 3657
                    code.put('case  0: ')
                code.putln('break;')
                if self.star_arg:
                    if min_positional_args:
                        for i in range(min_positional_args-1, -1, -1):
                            code.putln('case %2d:' % i)
                        code.put_goto(argtuple_error_label)
                else:
                    code.put('default: ')
3658
                    code.put_goto(argtuple_error_label)
3659 3660
                code.putln('}')

3661
        code.putln('}') # end of the conditional unpacking blocks
3662

3663 3664 3665
        # Convert arg values to their final type and assign them.
        # Also inject non-Python default arguments, which do cannot
        # live in the values[] array.
3666 3667
        for i, arg in enumerate(all_args):
            self.generate_arg_assignment(arg, "values[%d]" % i, code)
3668

3669
        code.putln('}') # end of the whole argument unpacking block
3670 3671 3672 3673

        if code.label_used(argtuple_error_label):
            code.put_goto(success_label)
            code.put_label(argtuple_error_label)
3674 3675
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3676
            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
3677 3678 3679
                self.name, has_fixed_positional_count,
                min_positional_args, max_positional_args,
                Naming.args_cname))
3680 3681
            code.putln(code.error_goto(self.pos))

3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703
    def generate_arg_assignment(self, arg, item, code):
        if arg.type.is_pyobject:
            # Python default arguments were already stored in 'item' at the very beginning
            if arg.is_generic:
                item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
            entry = arg.entry
            code.putln("%s = %s;" % (entry.cname, item))
        else:
            func = arg.type.from_py_function
            if func:
                if arg.default:
                    # C-typed default arguments must be handled here
                    code.putln('if (%s) {' % item)
                rhs = "%s(%s)" % (func, item)
                if arg.type.is_enum:
                    rhs = arg.type.cast_code(rhs)
                code.putln("%s = %s; %s" % (
                    arg.entry.cname,
                    rhs,
                    code.error_goto_if(arg.type.error_condition(arg.entry.cname), arg.pos)))
                if arg.default:
                    code.putln('} else {')
3704 3705 3706
                    code.putln("%s = %s;" % (
                        arg.entry.cname,
                        arg.calculate_default_value_code(code)))
3707 3708 3709 3710 3711 3712
                    if arg.type.is_memoryviewslice:
                        code.put_incref_memoryviewslice(arg.entry.cname,
                                                        have_gil=True)
                    code.putln('}')
            else:
                error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
3713

3714
    def generate_stararg_init_code(self, max_positional_args, code):
3715
        if self.starstar_arg:
3716
            self.starstar_arg.entry.xdecref_cleanup = 0
Stefan Behnel's avatar
Stefan Behnel committed
3717
            code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
3718 3719 3720
                self.starstar_arg.entry.cname,
                self.starstar_arg.entry.cname,
                self.error_value()))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3721
            code.put_gotref(self.starstar_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3722 3723 3724
        if self.star_arg:
            self.star_arg.entry.xdecref_cleanup = 0
            code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
3725 3726
                Naming.args_cname,
                max_positional_args))
3727
            code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
3728 3729
                self.star_arg.entry.cname, Naming.args_cname,
                max_positional_args, Naming.args_cname))
3730
            code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3731
            if self.starstar_arg:
3732
                code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
3733 3734 3735 3736
            code.put_finish_refcount_context()
            code.putln('return %s;' % self.error_value())
            code.putln('}')
            code.put_gotref(self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3737 3738 3739 3740
            code.putln('} else {')
            code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
            code.put_incref(Naming.empty_tuple, py_object_type)
            code.putln('}')
3741

3742
    def generate_argument_values_setup_code(self, args, code):
3743
        max_args = len(args)
Stefan Behnel's avatar
Stefan Behnel committed
3744 3745
        # the 'values' array collects borrowed references to arguments
        # before doing any type coercion etc.
3746
        code.putln("PyObject* values[%d] = {%s};" % (
Stefan Behnel's avatar
Stefan Behnel committed
3747
            max_args, ','.join('0'*max_args)))
3748

3749
        if self.target.defaults_struct:
3750
            code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
3751 3752
                self.target.defaults_struct, Naming.dynamic_args_cname,
                self.target.defaults_struct, Naming.self_cname))
3753

3754 3755
        # assign borrowed Python default values to the values array,
        # so that they can be overwritten by received arguments below
3756
        for i, arg in enumerate(args):
3757 3758 3759 3760
            if arg.default and arg.type.is_pyobject:
                default_value = arg.calculate_default_value_code(code)
                code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))

3761 3762 3763 3764
    def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
                                        has_fixed_positional_count, has_kw_only_args,
                                        all_args, argtuple_error_label, code):
        code.putln('Py_ssize_t kw_args;')
3765
        code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
3766
        # copy the values from the args tuple and check that it's not too long
3767
        code.putln('switch (pos_args) {')
Stefan Behnel's avatar
Stefan Behnel committed
3768 3769
        if self.star_arg:
            code.putln('default:')
3770
        for i in range(max_positional_args-1, -1, -1):
3771
            code.put('case %2d: ' % (i+1))
3772
            code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (
3773
                i, Naming.args_cname, i))
3774
            code.putln('CYTHON_FALLTHROUGH;')
3775
        code.putln('case  0: break;')
3776
        if not self.star_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3777
            code.put('default: ') # more arguments than allowed
3778
            code.put_goto(argtuple_error_label)
3779 3780
        code.putln('}')

3781 3782 3783 3784 3785 3786 3787 3788 3789
        # The code above is very often (but not always) the same as
        # the optimised non-kwargs tuple unpacking code, so we keep
        # the code block above at the very top, before the following
        # 'external' PyDict_Size() call, to make it easy for the C
        # compiler to merge the two separate tuple unpacking
        # implementations into one when they turn out to be identical.

        # If we received kwargs, fill up the positional/required
        # arguments with values from the kw dict
3790
        code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
3791
        if self.num_required_args or max_positional_args > 0:
Stefan Behnel's avatar
Stefan Behnel committed
3792 3793 3794 3795
            last_required_arg = -1
            for i, arg in enumerate(all_args):
                if not arg.default:
                    last_required_arg = i
3796 3797
            if last_required_arg < max_positional_args:
                last_required_arg = max_positional_args-1
Stefan Behnel's avatar
Stefan Behnel committed
3798
            if max_positional_args > 0:
3799
                code.putln('switch (pos_args) {')
3800
            for i, arg in enumerate(all_args[:last_required_arg+1]):
Stefan Behnel's avatar
Stefan Behnel committed
3801
                if max_positional_args > 0 and i <= max_positional_args:
3802 3803 3804
                    if self.star_arg and i == max_positional_args:
                        code.putln('default:')
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
3805 3806
                        if i != 0:
                            code.putln('CYTHON_FALLTHROUGH;')
3807
                        code.putln('case %2d:' % i)
3808
                pystring_cname = code.intern_identifier(arg.name)
3809
                if arg.default:
3810
                    if arg.kw_only:
3811
                        # optional kw-only args are handled separately below
3812
                        continue
3813
                    code.putln('if (kw_args > 0) {')
3814
                    # don't overwrite default argument
3815
                    code.putln('PyObject* value = PyDict_GetItem(%s, %s);' % (
3816
                        Naming.kwds_cname, pystring_cname))
3817
                    code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
3818 3819
                    code.putln('}')
                else:
3820
                    code.putln('if (likely((values[%d] = PyDict_GetItem(%s, %s)) != 0)) kw_args--;' % (
3821
                        i, Naming.kwds_cname, pystring_cname))
3822 3823 3824 3825 3826 3827 3828 3829 3830 3831
                    if i < min_positional_args:
                        if i == 0:
                            # special case: we know arg 0 is missing
                            code.put('else ')
                            code.put_goto(argtuple_error_label)
                        else:
                            # print the correct number of values (args or
                            # kwargs) that were passed into positional
                            # arguments up to this point
                            code.putln('else {')
3832 3833
                            code.globalstate.use_utility_code(
                                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3834
                            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
3835 3836
                                self.name, has_fixed_positional_count,
                                min_positional_args, max_positional_args, i))
3837 3838 3839
                            code.putln(code.error_goto(self.pos))
                            code.putln('}')
                    elif arg.kw_only:
3840
                        code.putln('else {')
3841 3842
                        code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
                            self.name, pystring_cname))
3843 3844
                        code.putln(code.error_goto(self.pos))
                        code.putln('}')
Stefan Behnel's avatar
Stefan Behnel committed
3845 3846
            if max_positional_args > 0:
                code.putln('}')
3847

3848
        if has_kw_only_args:
3849
            # unpack optional keyword-only arguments separately because
3850
            # checking for interned strings in a dict is faster than iterating
3851
            self.generate_optional_kwonly_args_unpacking_code(all_args, code)
3852

3853
        code.putln('if (unlikely(kw_args > 0)) {')
3854 3855
        # non-positional/-required kw args left in dict: default args,
        # kw-only args, **kwargs or error
Stefan Behnel's avatar
Stefan Behnel committed
3856 3857 3858 3859 3860
        #
        # This is sort of a catch-all: except for checking required
        # arguments, this will always do the right thing for unpacking
        # keyword arguments, so that we can concentrate on optimising
        # common cases above.
3861 3862 3863
        if max_positional_args == 0:
            pos_arg_count = "0"
        elif self.star_arg:
3864
            code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % (
3865
                max_positional_args, max_positional_args))
Stefan Behnel's avatar
Stefan Behnel committed
3866 3867
            pos_arg_count = "used_pos_args"
        else:
3868
            pos_arg_count = "pos_args"
3869 3870
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
3871 3872 3873 3874 3875 3876 3877
        code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % (
            Naming.kwds_cname,
            Naming.pykwdlist_cname,
            self.starstar_arg and self.starstar_arg.entry.cname or '0',
            pos_arg_count,
            self.name,
            code.error_goto(self.pos)))
3878
        code.putln('}')
3879

3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910
    def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
        optional_args = []
        first_optional_arg = -1
        for i, arg in enumerate(all_args):
            if not arg.kw_only or not arg.default:
                continue
            if not optional_args:
                first_optional_arg = i
            optional_args.append(arg.name)
        if optional_args:
            if len(optional_args) > 1:
                # if we receive more than the named kwargs, we either have **kwargs
                # (in which case we must iterate anyway) or it's an error (which we
                # also handle during iteration) => skip this part if there are more
                code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
                    not self.starstar_arg and 'likely' or '',
                    len(optional_args)))
                code.putln('Py_ssize_t index;')
                # not unrolling the loop here reduces the C code overhead
                code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
                    first_optional_arg, first_optional_arg + len(optional_args)))
            else:
                code.putln('if (kw_args == 1) {')
                code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
            code.putln('PyObject* value = PyDict_GetItem(%s, *%s[index]);' % (
                Naming.kwds_cname, Naming.pykwdlist_cname))
            code.putln('if (value) { values[index] = value; kw_args--; }')
            if len(optional_args) > 1:
                code.putln('}')
            code.putln('}')

William Stein's avatar
William Stein committed
3911
    def generate_argument_conversion_code(self, code):
3912 3913 3914
        # Generate code to convert arguments from signature type to
        # declared type, if needed.  Also copies signature arguments
        # into closure fields.
William Stein's avatar
William Stein committed
3915 3916 3917 3918 3919 3920 3921 3922 3923
        for arg in self.args:
            if arg.needs_conversion:
                self.generate_arg_conversion(arg, code)

    def generate_arg_conversion(self, arg, code):
        # Generate conversion code for one argument.
        old_type = arg.hdr_type
        new_type = arg.type
        if old_type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
3924 3925 3926 3927
            if arg.default:
                code.putln("if (%s) {" % arg.hdr_cname)
            else:
                code.putln("assert(%s); {" % arg.hdr_cname)
William Stein's avatar
William Stein committed
3928
            self.generate_arg_conversion_from_pyobject(arg, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
3929
            code.putln("}")
William Stein's avatar
William Stein committed
3930 3931 3932 3933
        elif new_type.is_pyobject:
            self.generate_arg_conversion_to_pyobject(arg, code)
        else:
            if new_type.assignable_from(old_type):
3934
                code.putln("%s = %s;" % (arg.entry.cname, arg.hdr_cname))
William Stein's avatar
William Stein committed
3935
            else:
3936
                error(arg.pos, "Cannot convert 1 argument from '%s' to '%s'" % (old_type, new_type))
3937

William Stein's avatar
William Stein committed
3938 3939 3940
    def generate_arg_conversion_from_pyobject(self, arg, code):
        new_type = arg.type
        func = new_type.from_py_function
3941
        # copied from CoerceFromPyTypeNode
William Stein's avatar
William Stein committed
3942
        if func:
3943 3944 3945 3946 3947
            lhs = arg.entry.cname
            rhs = "%s(%s)" % (func, arg.hdr_cname)
            if new_type.is_enum:
                rhs = PyrexTypes.typecast(new_type, PyrexTypes.c_long_type, rhs)
            code.putln("%s = %s; %s" % (
3948
                lhs,
3949
                rhs,
3950
                code.error_goto_if(new_type.error_condition(arg.entry.cname), arg.pos)))
William Stein's avatar
William Stein committed
3951
        else:
3952
            error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
3953

William Stein's avatar
William Stein committed
3954 3955 3956 3957
    def generate_arg_conversion_to_pyobject(self, arg, code):
        old_type = arg.hdr_type
        func = old_type.to_py_function
        if func:
Robert Bradshaw's avatar
Robert Bradshaw committed
3958
            code.putln("%s = %s(%s); %s" % (
William Stein's avatar
William Stein committed
3959 3960 3961
                arg.entry.cname,
                func,
                arg.hdr_cname,
Robert Bradshaw's avatar
Robert Bradshaw committed
3962
                code.error_goto_if_null(arg.entry.cname, arg.pos)))
3963
            code.put_var_gotref(arg.entry)
William Stein's avatar
William Stein committed
3964
        else:
3965
            error(arg.pos, "Cannot convert argument of type '%s' to Python object" % old_type)
William Stein's avatar
William Stein committed
3966 3967 3968 3969 3970 3971 3972 3973

    def generate_argument_type_tests(self, code):
        # Generate type tests for args whose signature
        # type is PyObject * and whose declared type is
        # a subtype thereof.
        for arg in self.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
3974 3975 3976
            elif not arg.accept_none and (arg.type.is_pyobject or
                                          arg.type.is_buffer or
                                          arg.type.is_memoryviewslice):
3977 3978
                self.generate_arg_none_check(arg, code)

William Stein's avatar
William Stein committed
3979
    def error_value(self):
3980
        return self.signature.error_value
3981

3982 3983

class GeneratorDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
3984
    # Generator function node that creates a new generator instance when called.
3985
    #
Stefan Behnel's avatar
Stefan Behnel committed
3986
    # gbody          GeneratorBodyDefNode   the function implementing the generator
3987 3988 3989
    #

    is_generator = True
3990
    is_coroutine = False
3991 3992
    is_asyncgen = False
    gen_type_name = 'Generator'
3993 3994
    needs_closure = True

Stefan Behnel's avatar
Stefan Behnel committed
3995
    child_attrs = DefNode.child_attrs + ["gbody"]
3996

3997
    def __init__(self, pos, **kwargs):
3998
        # XXX: don't actually needs a body
3999 4000
        kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
        super(GeneratorDefNode, self).__init__(pos, **kwargs)
4001 4002 4003 4004 4005 4006 4007 4008

    def analyse_declarations(self, env):
        super(GeneratorDefNode, self).analyse_declarations(env)
        self.gbody.local_scope = self.local_scope
        self.gbody.analyse_declarations(env)

    def generate_function_body(self, env, code):
        body_cname = self.gbody.entry.func_cname
4009 4010
        name = code.intern_identifier(self.name)
        qualname = code.intern_identifier(self.qualname)
4011
        module_name = code.intern_identifier(self.module_name)
4012

4013
        code.putln('{')
4014
        code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
4015
                   '(__pyx_coroutine_body_t) %s, (PyObject *) %s, %s, %s, %s); %s' % (
4016
                       self.gen_type_name,
4017
                       body_cname, Naming.cur_scope_cname, name, qualname, module_name,
4018 4019
                       code.error_goto_if_null('gen', self.pos)))
        code.put_decref(Naming.cur_scope_cname, py_object_type)
4020
        if self.requires_classobj:
4021
            classobj_cname = 'gen->classobj'
4022 4023 4024 4025
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                classobj_cname, Naming.self_cname))
            code.put_incref(classobj_cname, py_object_type)
            code.put_giveref(classobj_cname)
4026
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4027
        code.putln('return (PyObject *) gen;')
4028
        code.putln('}')
4029 4030

    def generate_function_definitions(self, env, code):
4031
        env.use_utility_code(UtilityCode.load_cached(self.gen_type_name, "Coroutine.c"))
4032 4033 4034 4035 4036
        self.gbody.generate_function_header(code, proto=True)
        super(GeneratorDefNode, self).generate_function_definitions(env, code)
        self.gbody.generate_function_definitions(env, code)


4037
class AsyncDefNode(GeneratorDefNode):
4038
    gen_type_name = 'Coroutine'
4039 4040 4041
    is_coroutine = True


4042 4043 4044 4045 4046
class AsyncGenNode(AsyncDefNode):
    gen_type_name = 'AsyncGen'
    is_asyncgen = True


4047
class GeneratorBodyDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
4048
    # Main code body of a generator implemented as a DefNode.
4049 4050 4051
    #

    is_generator_body = True
4052
    is_inlined = False
4053
    is_async_gen_body = False
4054
    inlined_comprehension_type = None  # container type for inlined comprehensions
4055

4056
    def __init__(self, pos=None, name=None, body=None, is_async_gen_body=False):
4057
        super(GeneratorBodyDefNode, self).__init__(
4058
            pos=pos, body=body, name=name, is_async_gen_body=is_async_gen_body,
4059
            doc=None, args=[], star_arg=None, starstar_arg=None)
4060

4061 4062
    def declare_generator_body(self, env):
        prefix = env.next_id(env.scope_prefix)
Vitja Makarov's avatar
Vitja Makarov committed
4063
        name = env.next_id('generator')
4064 4065 4066 4067
        cname = Naming.genbody_prefix + prefix + name
        entry = env.declare_var(None, py_object_type, self.pos,
                                cname=cname, visibility='private')
        entry.func_cname = cname
4068 4069 4070 4071 4072 4073
        entry.qualified_name = EncodedString(self.name)
        self.entry = entry

    def analyse_declarations(self, env):
        self.analyse_argument_types(env)
        self.declare_generator_body(env)
4074 4075

    def generate_function_header(self, code, proto=False):
4076
        header = "static PyObject *%s(__pyx_CoroutineObject *%s, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % (
4077
            self.entry.func_cname,
4078
            Naming.generator_cname,
4079
            Naming.local_tstate_cname,
4080 4081 4082 4083
            Naming.sent_value_cname)
        if proto:
            code.putln('%s; /* proto */' % header)
        else:
Stefan Behnel's avatar
Stefan Behnel committed
4084
            code.putln('%s /* generator body */\n{' % header)
4085 4086 4087 4088 4089 4090 4091 4092

    def generate_function_definitions(self, env, code):
        lenv = self.local_scope

        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)

        # Generate C code for header and body of function
4093
        code.enter_cfunc_scope(lenv)
4094 4095 4096 4097 4098 4099 4100 4101
        code.return_from_error_cleanup_label = code.new_label()

        # ----- Top-level constants used by this function
        code.mark_pos(self.pos)
        self.generate_cached_builtins_decls(lenv, code)
        # ----- Function header
        code.putln("")
        self.generate_function_header(code)
4102
        closure_init_code = code.insertion_point()
4103 4104 4105 4106
        # ----- Local variables
        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
        tempvardecl_code = code.insertion_point()
        code.put_declare_refcount_context()
4107
        code.put_setup_refcount_context(self.entry.name or self.entry.qualified_name)
4108 4109 4110
        profile = code.globalstate.directives['profile']
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
4111
            tempvardecl_code.put_trace_declarations()
4112 4113 4114 4115 4116 4117 4118 4119 4120 4121

        # ----- Resume switch point.
        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
        resume_code = code.insertion_point()
        first_run_label = code.new_label('first_run')
        code.use_label(first_run_label)
        code.put_label(first_run_label)
        code.putln('%s' %
                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))

4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138
        # ----- prepare target container for inlined comprehension
        if self.is_inlined and self.inlined_comprehension_type is not None:
            target_type = self.inlined_comprehension_type
            if target_type is Builtin.list_type:
                comp_init = 'PyList_New(0)'
            elif target_type is Builtin.set_type:
                comp_init = 'PySet_New(NULL)'
            elif target_type is Builtin.dict_type:
                comp_init = 'PyDict_New()'
            else:
                raise InternalError(
                    "invalid type of inlined comprehension: %s" % target_type)
            code.putln("%s = %s; %s" % (
                Naming.retval_cname, comp_init,
                code.error_goto_if_null(Naming.retval_cname, self.pos)))
            code.put_gotref(Naming.retval_cname)

4139 4140
        # ----- Function body
        self.generate_function_body(env, code)
4141
        # ----- Closure initialization
4142
        if lenv.scope_class.type.scope.var_entries:
4143 4144 4145 4146
            closure_init_code.putln('%s = %s;' % (
                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
                lenv.scope_class.type.cast_code('%s->closure' %
                                                Naming.generator_cname)))
4147
            # FIXME: this silences a potential "unused" warning => try to avoid unused closures in more cases
4148
            code.putln("CYTHON_MAYBE_UNUSED_VAR(%s);" % Naming.cur_scope_cname)
4149 4150 4151 4152 4153

        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

Stefan Behnel's avatar
Stefan Behnel committed
4154 4155
        # on normal generator termination, we do not take the exception propagation
        # path: no traceback info is required and not creating it is much faster
4156
        if not self.is_inlined and not self.body.is_terminator:
4157
            code.putln('PyErr_SetNone(%s);' % (
4158
                '__Pyx_PyExc_StopAsyncIteration' if self.is_async_gen_body else 'PyExc_StopIteration'))
4159
        # ----- Error cleanup
4160
        if code.label_used(code.error_label):
4161 4162
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
4163
            code.put_label(code.error_label)
4164 4165
            if self.is_inlined and self.inlined_comprehension_type is not None:
                code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4166 4167
            if Future.generator_stop in env.global_scope().context.future_directives:
                # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
4168
                code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
4169
                code.putln("__Pyx_Generator_Replace_StopIteration(%d);" % bool(self.is_async_gen_body))
4170 4171
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
4172
            code.put_add_traceback(self.entry.qualified_name)
4173 4174 4175

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
4176 4177 4178 4179
        if self.is_inlined:
            code.put_xgiveref(Naming.retval_cname)
        else:
            code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4180
        code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
4181
        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
4182
        # clean up as early as possible to help breaking any reference cycles
4183
        code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
4184 4185 4186
        if profile or linetrace:
            code.put_trace_return(Naming.retval_cname,
                                  nogil=not code.funcstate.gil_owned)
4187
        code.put_finish_refcount_context()
4188
        code.putln("return %s;" % Naming.retval_cname)
4189 4190 4191 4192 4193
        code.putln("}")

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)
        # ----- Generator resume code
4194 4195 4196
        if profile or linetrace:
            resume_code.put_trace_call(self.entry.qualified_name, self.pos,
                                       nogil=not code.funcstate.gil_owned)
4197
        resume_code.putln("switch (%s->resume_label) {" % (
Stefan Behnel's avatar
Stefan Behnel committed
4198
                       Naming.generator_cname))
4199

4200
        resume_code.putln("case 0: goto %s;" % first_run_label)
4201

4202 4203
        for i, label in code.yield_labels:
            resume_code.putln("case %d: goto %s;" % (i, label))
Stefan Behnel's avatar
Stefan Behnel committed
4204
        resume_code.putln("default: /* CPython raises the right error here */")
4205 4206 4207
        if profile or linetrace:
            resume_code.put_trace_return("Py_None",
                                         nogil=not code.funcstate.gil_owned)
4208
        resume_code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4209 4210
        resume_code.putln("return NULL;")
        resume_code.putln("}")
4211 4212 4213 4214

        code.exit_cfunc_scope()


4215 4216
class OverrideCheckNode(StatNode):
    # A Node for dispatching to the def method if it
4217
    # is overriden.
4218 4219 4220 4221 4222 4223
    #
    #  py_func
    #
    #  args
    #  func_temp
    #  body
4224

Robert Bradshaw's avatar
Robert Bradshaw committed
4225
    child_attrs = ['body']
4226

4227
    body = None
Robert Bradshaw's avatar
Robert Bradshaw committed
4228

4229 4230
    def analyse_expressions(self, env):
        self.args = env.arg_entries
4231 4232 4233 4234
        if self.py_func.is_module_scope:
            first_arg = 0
        else:
            first_arg = 1
4235
        from . import ExprNodes
4236
        self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
Stefan Behnel's avatar
Stefan Behnel committed
4237 4238
        call_node = ExprNodes.SimpleCallNode(
            self.pos, function=self.func_node,
4239 4240
            args=[ExprNodes.NameNode(self.pos, name=arg.name)
                  for arg in self.args[first_arg:]])
4241 4242
        if env.return_type.is_void or env.return_type.is_returncode:
            self.body = StatListNode(self.pos, stats=[
4243 4244
                ExprStatNode(self.pos, expr=call_node),
                ReturnStatNode(self.pos, value=None)])
4245 4246
        else:
            self.body = ReturnStatNode(self.pos, value=call_node)
4247 4248
        self.body = self.body.analyse_expressions(env)
        return self
4249

4250
    def generate_execution_code(self, code):
4251
        interned_attr_cname = code.intern_identifier(self.py_func.entry.name)
4252
        # Check to see if we are an extension type
4253 4254 4255 4256
        if self.py_func.is_module_scope:
            self_arg = "((PyObject *)%s)" % Naming.module_cname
        else:
            self_arg = "((PyObject *)%s)" % self.args[0].cname
4257
        code.putln("/* Check if called by wrapper */")
4258
        code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
Stefan Behnel's avatar
Stefan Behnel committed
4259
        code.putln("/* Check if overridden in Python */")
4260 4261 4262
        if self.py_func.is_module_scope:
            code.putln("else {")
        else:
4263
            code.putln("else if (unlikely(Py_TYPE(%s)->tp_dictoffset != 0)) {" % self_arg)
4264 4265
        func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.func_node.set_cname(func_node_temp)
4266
        # need to get attribute manually--scope would return cdef method
4267 4268
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
4269
        err = code.error_goto_if_null(func_node_temp, self.pos)
4270
        code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
4271 4272 4273
            func_node_temp, self_arg, interned_attr_cname, err))
        code.put_gotref(func_node_temp)
        is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp
4274
        is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)%s)" % (
4275
            func_node_temp, self.py_func.entry.func_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
4276
        code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden))
4277 4278
        self.body.generate_execution_code(code)
        code.putln("}")
4279 4280
        code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
        code.funcstate.release_temp(func_node_temp)
Robert Bradshaw's avatar
Robert Bradshaw committed
4281
        code.putln("}")
4282

4283

Robert Bradshaw's avatar
Robert Bradshaw committed
4284 4285
class ClassDefNode(StatNode, BlockNode):
    pass
4286

4287

Robert Bradshaw's avatar
Robert Bradshaw committed
4288
class PyClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4289 4290
    #  A Python class definition.
    #
Stefan Behnel's avatar
Stefan Behnel committed
4291
    #  name     EncodedString   Name of the class
William Stein's avatar
William Stein committed
4292 4293 4294 4295
    #  doc      string or None
    #  body     StatNode        Attribute definition code
    #  entry    Symtab.Entry
    #  scope    PyClassScope
4296
    #  decorators    [DecoratorNode]        list of decorators or None
William Stein's avatar
William Stein committed
4297 4298 4299
    #
    #  The following subnodes are constructed internally:
    #
4300
    #  dict     DictNode   Class dictionary or Py3 namespace
William Stein's avatar
William Stein committed
4301 4302
    #  classobj ClassNode  Class object
    #  target   NameNode   Variable to assign class object to
4303

4304
    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result",
4305
                   "target", "class_cell", "decorators"]
4306
    decorators = None
4307
    class_result = None
4308 4309 4310
    is_py3_style_class = False  # Python3 style class (kwargs)
    metaclass = None
    mkw = None
4311

4312
    def __init__(self, pos, name, bases, doc, body, decorators=None,
4313
                 keyword_args=None, force_py3_semantics=False):
William Stein's avatar
William Stein committed
4314 4315 4316 4317
        StatNode.__init__(self, pos)
        self.name = name
        self.doc = doc
        self.body = body
4318
        self.decorators = decorators
4319
        self.bases = bases
4320
        from . import ExprNodes
4321
        if self.doc and Options.docstrings:
4322
            doc = embed_position(self.pos, self.doc)
4323
            doc_node = ExprNodes.StringNode(pos, value=doc)
William Stein's avatar
William Stein committed
4324 4325
        else:
            doc_node = None
4326 4327

        allow_py2_metaclass = not force_py3_semantics
4328
        if keyword_args:
4329 4330
            allow_py2_metaclass = False
            self.is_py3_style_class = True
4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344
            if keyword_args.is_dict_literal:
                if keyword_args.key_value_pairs:
                    for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
                        if item.key.value == 'metaclass':
                            if self.metaclass is not None:
                                error(item.pos, "keyword argument 'metaclass' passed multiple times")
                            # special case: we already know the metaclass,
                            # so we don't need to do the "build kwargs,
                            # find metaclass" dance at runtime
                            self.metaclass = item.value
                            del keyword_args.key_value_pairs[i]
                    self.mkw = keyword_args
                else:
                    assert self.metaclass is not None
4345
            else:
4346
                # MergedDictNode
4347
                self.mkw = ExprNodes.ProxyNode(keyword_args)
4348 4349

        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
4350
            if self.metaclass is None:
4351
                if keyword_args and not keyword_args.is_dict_literal:
4352 4353 4354 4355
                    # **kwargs may contain 'metaclass' arg
                    mkdict = self.mkw
                else:
                    mkdict = None
4356 4357 4358 4359 4360 4361 4362
                if (not mkdict and
                        self.bases.is_sequence_constructor and
                        not self.bases.args):
                    pass  # no base classes => no inherited metaclass
                else:
                    self.metaclass = ExprNodes.PyClassMetaclassNode(
                        pos, mkw=mkdict, bases=self.bases)
4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375
                needs_metaclass_calculation = False
            else:
                needs_metaclass_calculation = True

            self.dict = ExprNodes.PyClassNamespaceNode(
                pos, name=name, doc=doc_node,
                metaclass=self.metaclass, bases=self.bases, mkw=self.mkw)
            self.classobj = ExprNodes.Py3ClassNode(
                pos, name=name,
                bases=self.bases, dict=self.dict, doc=doc_node,
                metaclass=self.metaclass, mkw=self.mkw,
                calculate_metaclass=needs_metaclass_calculation,
                allow_py2_metaclass=allow_py2_metaclass)
4376
        else:
4377 4378 4379 4380 4381 4382 4383
            # no bases, no metaclass => old style class creation
            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
            self.classobj = ExprNodes.ClassNode(
                pos, name=name,
                bases=bases, dict=self.dict, doc=doc_node)

        self.target = ExprNodes.NameNode(pos, name=name)
4384
        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
4385

4386 4387
    def as_cclass(self):
        """
4388
        Return this node as if it were declared as an extension class
4389
        """
4390
        if self.is_py3_style_class:
4391 4392
            error(self.classobj.pos, "Python3 style class could not be represented as C class")
            return
4393 4394 4395 4396 4397 4398 4399
        bases = self.classobj.bases.args
        if len(bases) == 0:
            base_class_name = None
            base_class_module = None
        elif len(bases) == 1:
            base = bases[0]
            path = []
4400
            from .ExprNodes import AttributeNode, NameNode
4401
            while isinstance(base, AttributeNode):
4402 4403
                path.insert(0, base.attribute)
                base = base.obj
4404
            if isinstance(base, NameNode):
4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415
                path.insert(0, base.name)
                base_class_name = path[-1]
                if len(path) > 1:
                    base_class_module = u'.'.join(path[:-1])
                else:
                    base_class_module = None
            else:
                error(self.classobj.bases.args.pos, "Invalid base class")
        else:
            error(self.classobj.bases.args.pos, "C class may only have one base class")
            return None
4416 4417

        return CClassDefNode(self.pos,
4418 4419 4420 4421 4422 4423 4424 4425 4426
                             visibility='private',
                             module_name=None,
                             class_name=self.name,
                             base_class_module=base_class_module,
                             base_class_name=base_class_name,
                             decorators=self.decorators,
                             body=self.body,
                             in_pxd=False,
                             doc=self.doc)
4427

4428 4429
    def create_scope(self, env):
        genv = env
4430 4431
        while genv.is_py_class_scope or genv.is_c_class_scope:
            genv = genv.outer_scope
4432
        cenv = self.scope = PyClassScope(name=self.name, outer_scope=genv)
4433
        return cenv
4434

William Stein's avatar
William Stein committed
4435
    def analyse_declarations(self, env):
4436 4437
        class_result = self.classobj
        if self.decorators:
4438
            from .ExprNodes import SimpleCallNode
4439 4440 4441
            for decorator in self.decorators[::-1]:
                class_result = SimpleCallNode(
                    decorator.pos,
4442 4443
                    function=decorator.decorator,
                    args=[class_result])
4444
            self.decorators = None
4445 4446
        self.class_result = class_result
        self.class_result.analyse_declarations(env)
William Stein's avatar
William Stein committed
4447
        self.target.analyse_target_declaration(env)
4448
        cenv = self.create_scope(env)
4449
        cenv.directives = env.directives
4450 4451
        cenv.class_obj_cname = self.target.entry.cname
        self.body.analyse_declarations(cenv)
4452

William Stein's avatar
William Stein committed
4453
    def analyse_expressions(self, env):
4454
        if self.bases:
4455
            self.bases = self.bases.analyse_expressions(env)
4456
        if self.metaclass:
4457
            self.metaclass = self.metaclass.analyse_expressions(env)
4458
        if self.mkw:
4459 4460 4461
            self.mkw = self.mkw.analyse_expressions(env)
        self.dict = self.dict.analyse_expressions(env)
        self.class_result = self.class_result.analyse_expressions(env)
4462
        cenv = self.scope
4463
        self.body = self.body.analyse_expressions(cenv)
4464
        self.target.analyse_target_expression(env, self.classobj)
4465 4466
        self.class_cell = self.class_cell.analyse_expressions(cenv)
        return self
4467

4468
    def generate_function_definitions(self, env, code):
4469
        self.generate_lambda_definitions(self.scope, code)
4470
        self.body.generate_function_definitions(self.scope, code)
4471

William Stein's avatar
William Stein committed
4472
    def generate_execution_code(self, code):
4473
        code.mark_pos(self.pos)
4474 4475
        code.pyclass_stack.append(self)
        cenv = self.scope
4476
        if self.bases:
4477
            self.bases.generate_evaluation_code(code)
4478
        if self.mkw:
4479
            self.mkw.generate_evaluation_code(code)
4480
        if self.metaclass:
4481
            self.metaclass.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
4482
        self.dict.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4483
        cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
4484
        self.class_cell.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4485
        self.body.generate_execution_code(code)
4486
        self.class_result.generate_evaluation_code(code)
4487 4488 4489
        self.class_cell.generate_injection_code(
            code, self.class_result.result())
        self.class_cell.generate_disposal_code(code)
4490
        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
4491
        self.target.generate_assignment_code(self.class_result, code)
William Stein's avatar
William Stein committed
4492
        self.dict.generate_disposal_code(code)
4493
        self.dict.free_temps(code)
4494
        if self.metaclass:
4495 4496
            self.metaclass.generate_disposal_code(code)
            self.metaclass.free_temps(code)
4497 4498 4499 4500
        if self.mkw:
            self.mkw.generate_disposal_code(code)
            self.mkw.free_temps(code)
        if self.bases:
4501 4502
            self.bases.generate_disposal_code(code)
            self.bases.free_temps(code)
4503
        code.pyclass_stack.pop()
William Stein's avatar
William Stein committed
4504

Robert Bradshaw's avatar
Robert Bradshaw committed
4505
class CClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4506 4507 4508 4509
    #  An extension type definition.
    #
    #  visibility         'private' or 'public' or 'extern'
    #  typedef_flag       boolean
Stefan Behnel's avatar
Stefan Behnel committed
4510
    #  api                boolean
William Stein's avatar
William Stein committed
4511 4512 4513 4514 4515 4516 4517 4518
    #  module_name        string or None    For import of extern type objects
    #  class_name         string            Unqualified name of class
    #  as_name            string or None    Name to declare as in this scope
    #  base_class_module  string or None    Module containing the base class
    #  base_class_name    string or None    Name of the base class
    #  objstruct_name     string or None    Specified C name of object struct
    #  typeobj_name       string or None    Specified C name of type object
    #  in_pxd             boolean           Is in a .pxd file
4519
    #  decorators         [DecoratorNode]   list of decorators or None
William Stein's avatar
William Stein committed
4520 4521 4522 4523
    #  doc                string or None
    #  body               StatNode or None
    #  entry              Symtab.Entry
    #  base_type          PyExtensionType or None
4524 4525
    #  buffer_defaults_node DictNode or None Declares defaults for a buffer
    #  buffer_defaults_pos
4526

4527
    child_attrs = ["body"]
4528 4529
    buffer_defaults_node = None
    buffer_defaults_pos = None
4530 4531 4532 4533
    typedef_flag = False
    api = False
    objstruct_name = None
    typeobj_name = None
4534
    decorators = None
4535
    shadow = False
4536

Robert Bradshaw's avatar
Robert Bradshaw committed
4537 4538
    def buffer_defaults(self, env):
        if not hasattr(self, '_buffer_defaults'):
4539
            from . import Buffer
Robert Bradshaw's avatar
Robert Bradshaw committed
4540 4541 4542 4543 4544 4545 4546 4547 4548
            if self.buffer_defaults_node:
                self._buffer_defaults = Buffer.analyse_buffer_options(
                    self.buffer_defaults_pos,
                    env, [], self.buffer_defaults_node,
                    need_complete=False)
            else:
                self._buffer_defaults = None
        return self._buffer_defaults

4549 4550 4551 4552 4553 4554 4555 4556
    def declare(self, env):
        if self.module_name and self.visibility != 'extern':
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return None
        else:
            home_scope = env
4557

4558
        self.entry = home_scope.declare_c_class(
4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571
            name=self.class_name,
            pos=self.pos,
            defining=0,
            implementing=0,
            module_name=self.module_name,
            base_type=None,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4572

William Stein's avatar
William Stein committed
4573 4574 4575 4576
    def analyse_declarations(self, env):
        #print "CClassDefNode.analyse_declarations:", self.class_name
        #print "...visibility =", self.visibility
        #print "...module_name =", self.module_name
4577

William Stein's avatar
William Stein committed
4578
        if env.in_cinclude and not self.objstruct_name:
4579
            error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
4580
        if self.decorators:
4581
            error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
William Stein's avatar
William Stein committed
4582
        self.base_type = None
4583 4584
        # Now that module imports are cached, we need to
        # import the modules for extern classes.
4585 4586 4587 4588 4589 4590 4591
        if self.module_name:
            self.module = None
            for module in env.cimported_modules:
                if module.name == self.module_name:
                    self.module = module
            if self.module is None:
                self.module = ModuleScope(self.module_name, None, env.context)
4592
                self.module.has_extern_class = 1
Robert Bradshaw's avatar
Robert Bradshaw committed
4593
                env.add_imported_module(self.module)
4594

William Stein's avatar
William Stein committed
4595 4596
        if self.base_class_name:
            if self.base_class_module:
4597 4598 4599 4600
                base_class_scope = env.find_imported_module(self.base_class_module.split('.'), self.pos)
                if not base_class_scope:
                    error(self.pos, "'%s' is not a cimported module" % self.base_class_module)
                    return
William Stein's avatar
William Stein committed
4601 4602
            else:
                base_class_scope = env
4603 4604 4605 4606 4607 4608
            if self.base_class_name == 'object':
                # extension classes are special and don't need to inherit from object
                if base_class_scope is None or base_class_scope.lookup('object') is None:
                    self.base_class_name = None
                    self.base_class_module = None
                    base_class_scope = None
William Stein's avatar
William Stein committed
4609 4610 4611 4612 4613
            if base_class_scope:
                base_class_entry = base_class_scope.find(self.base_class_name, self.pos)
                if base_class_entry:
                    if not base_class_entry.is_type:
                        error(self.pos, "'%s' is not a type name" % self.base_class_name)
4614
                    elif not base_class_entry.type.is_extension_type and \
Stefan Behnel's avatar
Stefan Behnel committed
4615
                             not (base_class_entry.type.is_builtin_type and
4616
                                  base_class_entry.type.objstruct_cname):
William Stein's avatar
William Stein committed
4617 4618
                        error(self.pos, "'%s' is not an extension type" % self.base_class_name)
                    elif not base_class_entry.type.is_complete():
4619 4620 4621
                        error(self.pos, "Base class '%s' of type '%s' is incomplete" % (
                            self.base_class_name, self.class_name))
                    elif base_class_entry.type.scope and base_class_entry.type.scope.directives and \
4622
                             base_class_entry.type.is_final_type:
4623 4624
                        error(self.pos, "Base class '%s' of type '%s' is final" % (
                            self.base_class_name, self.class_name))
4625 4626 4627 4628
                    elif base_class_entry.type.is_builtin_type and \
                             base_class_entry.type.name in ('tuple', 'str', 'bytes'):
                        error(self.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
                              % base_class_entry.type.name)
William Stein's avatar
William Stein committed
4629 4630
                    else:
                        self.base_type = base_class_entry.type
4631
                if env.directives.get('freelist', 0) > 0:
Stefan Behnel's avatar
Stefan Behnel committed
4632
                    warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
4633

William Stein's avatar
William Stein committed
4634
        has_body = self.body is not None
4635 4636 4637 4638 4639 4640
        if has_body and self.base_type and not self.base_type.scope:
            # To properly initialize inherited attributes, the base type must
            # be analysed before this type.
            self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
            return

4641
        if self.module_name and self.visibility != 'extern':
4642 4643 4644 4645 4646 4647
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return
        else:
            home_scope = env
4648 4649

        if self.visibility == 'extern':
4650
            if (self.module_name == '__builtin__' and
4651 4652
                    self.class_name in Builtin.builtin_types and
                    env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython
4653
                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
4654

4655
        self.entry = home_scope.declare_c_class(
4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668
            name=self.class_name,
            pos=self.pos,
            defining=has_body and self.in_pxd,
            implementing=has_body and not self.in_pxd,
            module_name=self.module_name,
            base_type=self.base_type,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4669

4670 4671
        if self.shadow:
            home_scope.lookup(self.class_name).as_variable = self.entry
4672
        if home_scope is not env and self.visibility == 'extern':
4673
            env.add_imported_entry(self.class_name, self.entry, self.pos)
4674
        self.scope = scope = self.entry.type.scope
4675 4676
        if scope is not None:
            scope.directives = env.directives
4677

4678
        if self.doc and Options.docstrings:
4679
            scope.doc = embed_position(self.pos, self.doc)
4680

William Stein's avatar
William Stein committed
4681 4682
        if has_body:
            self.body.analyse_declarations(scope)
4683
            dict_entry = self.scope.lookup_here("__dict__")
4684
            if dict_entry and dict_entry.is_variable and (not scope.defined and not scope.implemented):
4685 4686
                dict_entry.getter_cname = self.scope.mangle_internal("__dict__getter")
                self.scope.declare_property("__dict__", dict_entry.doc, dict_entry.pos)
William Stein's avatar
William Stein committed
4687 4688 4689 4690
            if self.in_pxd:
                scope.defined = 1
            else:
                scope.implemented = 1
4691

William Stein's avatar
William Stein committed
4692
        env.allocate_vtable_names(self.entry)
4693

4694 4695 4696
        for thunk in self.entry.type.defered_declarations:
            thunk()

William Stein's avatar
William Stein committed
4697 4698
    def analyse_expressions(self, env):
        if self.body:
Robert Bradshaw's avatar
Robert Bradshaw committed
4699
            scope = self.entry.type.scope
4700 4701
            self.body = self.body.analyse_expressions(scope)
        return self
4702

4703
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
4704
        if self.body:
4705 4706
            self.generate_lambda_definitions(self.scope, code)
            self.body.generate_function_definitions(self.scope, code)
4707

William Stein's avatar
William Stein committed
4708 4709 4710
    def generate_execution_code(self, code):
        # This is needed to generate evaluation code for
        # default values of method arguments.
4711
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4712 4713
        if self.body:
            self.body.generate_execution_code(code)
4714

4715 4716 4717
    def annotate(self, code):
        if self.body:
            self.body.annotate(code)
William Stein's avatar
William Stein committed
4718 4719 4720 4721 4722 4723


class PropertyNode(StatNode):
    #  Definition of a property in an extension type.
    #
    #  name   string
4724
    #  doc    EncodedString or None    Doc string
4725
    #  entry  Symtab.Entry
William Stein's avatar
William Stein committed
4726
    #  body   StatListNode
4727

4728 4729
    child_attrs = ["body"]

William Stein's avatar
William Stein committed
4730
    def analyse_declarations(self, env):
4731 4732 4733
        self.entry = env.declare_property(self.name, self.doc, self.pos)
        self.entry.scope.directives = env.directives
        self.body.analyse_declarations(self.entry.scope)
4734

William Stein's avatar
William Stein committed
4735
    def analyse_expressions(self, env):
4736 4737
        self.body = self.body.analyse_expressions(env)
        return self
4738

4739 4740
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
4741 4742 4743 4744

    def generate_execution_code(self, code):
        pass

4745 4746 4747
    def annotate(self, code):
        self.body.annotate(code)

William Stein's avatar
William Stein committed
4748 4749 4750 4751 4752

class GlobalNode(StatNode):
    # Global variable declaration.
    #
    # names    [string]
4753

4754 4755
    child_attrs = []

William Stein's avatar
William Stein committed
4756 4757 4758 4759 4760
    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_global(name, self.pos)

    def analyse_expressions(self, env):
4761
        return self
4762

William Stein's avatar
William Stein committed
4763 4764 4765 4766
    def generate_execution_code(self, code):
        pass


4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778
class NonlocalNode(StatNode):
    # Nonlocal variable declaration via the 'nonlocal' keyword.
    #
    # names    [string]

    child_attrs = []

    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_nonlocal(name, self.pos)

    def analyse_expressions(self, env):
4779
        return self
4780 4781 4782 4783 4784

    def generate_execution_code(self, code):
        pass


William Stein's avatar
William Stein committed
4785 4786 4787 4788
class ExprStatNode(StatNode):
    #  Expression used as a statement.
    #
    #  expr   ExprNode
4789 4790

    child_attrs = ["expr"]
4791

Robert Bradshaw's avatar
Robert Bradshaw committed
4792
    def analyse_declarations(self, env):
4793
        from . import ExprNodes
4794 4795 4796
        expr = self.expr
        if isinstance(expr, ExprNodes.GeneralCallNode):
            func = expr.function.as_cython_attribute()
Robert Bradshaw's avatar
Robert Bradshaw committed
4797
            if func == u'declare':
4798
                args, kwds = expr.explicit_args_kwds()
Robert Bradshaw's avatar
Robert Bradshaw committed
4799
                if len(args):
4800
                    error(expr.pos, "Variable names must be specified.")
Robert Bradshaw's avatar
Robert Bradshaw committed
4801 4802 4803 4804 4805
                for var, type_node in kwds.key_value_pairs:
                    type = type_node.analyse_as_type(env)
                    if type is None:
                        error(type_node.pos, "Unknown type")
                    else:
4806
                        env.declare_var(var.value, type, var.pos, is_cdef=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
4807
                self.__class__ = PassStatNode
4808
        elif getattr(expr, 'annotation', None) is not None:
4809 4810 4811 4812 4813 4814 4815
            if expr.is_name:
                # non-code variable annotation, e.g. "name: type"
                expr.declare_from_annotation(env)
                self.__class__ = PassStatNode
            elif expr.is_attribute or expr.is_subscript:
                # unused expression with annotation, e.g. "a[0]: type" or "a.xyz : type"
                self.__class__ = PassStatNode
4816

William Stein's avatar
William Stein committed
4817
    def analyse_expressions(self, env):
4818
        self.expr.result_is_used = False  # hint that .result() may safely be left empty
4819 4820
        self.expr = self.expr.analyse_expressions(env)
        return self
4821

4822
    def nogil_check(self, env):
4823
        if self.expr.type.is_pyobject and self.expr.is_temp:
4824 4825 4826 4827
            self.gil_error()

    gil_message = "Discarding owned Python object"

William Stein's avatar
William Stein committed
4828
    def generate_execution_code(self, code):
4829
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4830
        self.expr.generate_evaluation_code(code)
4831 4832
        if not self.expr.is_temp and self.expr.result():
            code.putln("%s;" % self.expr.result())
William Stein's avatar
William Stein committed
4833
        self.expr.generate_disposal_code(code)
4834
        self.expr.free_temps(code)
William Stein's avatar
William Stein committed
4835

4836 4837 4838
    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

4839 4840 4841
    def annotate(self, code):
        self.expr.annotate(code)

William Stein's avatar
William Stein committed
4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852

class AssignmentNode(StatNode):
    #  Abstract base class for assignment nodes.
    #
    #  The analyse_expressions and generate_execution_code
    #  phases of assignments are split into two sub-phases
    #  each, to enable all the right hand sides of a
    #  parallel assignment to be evaluated before assigning
    #  to any of the left hand sides.

    def analyse_expressions(self, env):
4853
        node = self.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
4854
        if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
4855
            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
4856 4857
                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
        return node
4858

4859 4860 4861
#       def analyse_expressions(self, env):
#           self.analyse_expressions_1(env)
#           self.analyse_expressions_2(env)
William Stein's avatar
William Stein committed
4862 4863

    def generate_execution_code(self, code):
4864
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4865 4866
        self.generate_rhs_evaluation_code(code)
        self.generate_assignment_code(code)
4867

William Stein's avatar
William Stein committed
4868 4869 4870 4871 4872 4873

class SingleAssignmentNode(AssignmentNode):
    #  The simplest case:
    #
    #    a = b
    #
4874 4875 4876 4877
    #  lhs                      ExprNode      Left hand side
    #  rhs                      ExprNode      Right hand side
    #  first                    bool          Is this guaranteed the first assignment to lhs?
    #  is_overloaded_assignment bool          Is this assignment done via an overloaded operator=
4878 4879
    #  exception_check
    #  exception_value
4880

4881
    child_attrs = ["lhs", "rhs"]
4882
    first = False
4883
    is_overloaded_assignment = False
4884
    declaration_only = False
William Stein's avatar
William Stein committed
4885 4886

    def analyse_declarations(self, env):
4887
        from . import ExprNodes
4888

4889 4890
        # handle declarations of the form x = cython.foo()
        if isinstance(self.rhs, ExprNodes.CallNode):
4891
            func_name = self.rhs.function.as_cython_attribute()
4892 4893 4894
            if func_name:
                args, kwds = self.rhs.explicit_args_kwds()
                if func_name in ['declare', 'typedef']:
4895 4896
                    if len(args) > 2:
                        error(args[2].pos, "Invalid positional argument.")
4897
                        return
4898 4899 4900 4901 4902 4903 4904 4905
                    if kwds is not None:
                        kwdict = kwds.compile_time_value(None)
                        if func_name == 'typedef' or 'visibility' not in kwdict:
                            error(kwds.pos, "Invalid keyword argument.")
                            return
                        visibility = kwdict['visibility']
                    else:
                        visibility = 'private'
4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919
                    type = args[0].analyse_as_type(env)
                    if type is None:
                        error(args[0].pos, "Unknown type")
                        return
                    lhs = self.lhs
                    if func_name == 'declare':
                        if isinstance(lhs, ExprNodes.NameNode):
                            vars = [(lhs.name, lhs.pos)]
                        elif isinstance(lhs, ExprNodes.TupleNode):
                            vars = [(var.name, var.pos) for var in lhs.args]
                        else:
                            error(lhs.pos, "Invalid declaration")
                            return
                        for var, pos in vars:
4920
                            env.declare_var(var, type, pos, is_cdef=True, visibility=visibility)
Robert Bradshaw's avatar
Robert Bradshaw committed
4921 4922 4923 4924 4925
                        if len(args) == 2:
                            # we have a value
                            self.rhs = args[1]
                        else:
                            self.declaration_only = True
4926
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
4927
                        self.declaration_only = True
4928 4929
                        if not isinstance(lhs, ExprNodes.NameNode):
                            error(lhs.pos, "Invalid declaration.")
4930
                        env.declare_typedef(lhs.name, type, self.pos, visibility='private')
4931

4932 4933 4934
                elif func_name in ['struct', 'union']:
                    self.declaration_only = True
                    if len(args) > 0 or kwds is None:
4935
                        error(self.rhs.pos, "Struct or union members must be given by name.")
4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952
                        return
                    members = []
                    for member, type_node in kwds.key_value_pairs:
                        type = type_node.analyse_as_type(env)
                        if type is None:
                            error(type_node.pos, "Unknown type")
                        else:
                            members.append((member.value, type, member.pos))
                    if len(members) < len(kwds.key_value_pairs):
                        return
                    if not isinstance(self.lhs, ExprNodes.NameNode):
                        error(self.lhs.pos, "Invalid declaration.")
                    name = self.lhs.name
                    scope = StructOrUnionScope(name)
                    env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
                    for member, type, pos in members:
                        scope.declare_var(member, type, pos)
4953

Mark Florisson's avatar
Mark Florisson committed
4954 4955 4956 4957 4958 4959 4960 4961
                elif func_name == 'fused_type':
                    # dtype = cython.fused_type(...)
                    self.declaration_only = True
                    if kwds:
                        error(self.rhs.function.pos,
                              "fused_type does not take keyword arguments")

                    fusednode = FusedTypeNode(self.rhs.pos,
4962
                                              name=self.lhs.name, types=args)
Mark Florisson's avatar
Mark Florisson committed
4963 4964
                    fusednode.analyse_declarations(env)

4965 4966 4967 4968
        if self.declaration_only:
            return
        else:
            self.lhs.analyse_target_declaration(env)
4969

4970 4971
    def analyse_types(self, env, use_temp=0):
        from . import ExprNodes
4972

4973
        self.rhs = self.rhs.analyse_types(env)
4974

4975 4976 4977
        unrolled_assignment = self.unroll_rhs(env)
        if unrolled_assignment:
            return unrolled_assignment
Robert Bradshaw's avatar
Robert Bradshaw committed
4978

4979
        self.lhs = self.lhs.analyse_target_types(env)
4980
        self.lhs.gil_assignment_check(env)
4981 4982 4983
        unrolled_assignment = self.unroll_lhs(env)
        if unrolled_assignment:
            return unrolled_assignment
4984

4985 4986 4987
        if isinstance(self.lhs, ExprNodes.MemoryViewIndexNode):
            self.lhs.analyse_broadcast_operation(self.rhs)
            self.lhs = self.lhs.analyse_as_memview_scalar_assignment(self.rhs)
4988
        elif self.lhs.type.is_array:
4989
            if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
4990
                # cannot assign to C array, only to its full slice
4991
                self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None)
4992
                self.lhs = self.lhs.analyse_target_types(env)
4993

4994 4995 4996 4997
        if self.lhs.type.is_cpp_class:
            op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type])
            if op:
                rhs = self.rhs
4998
                self.is_overloaded_assignment = True
4999 5000 5001 5002
                self.exception_check = op.type.exception_check
                self.exception_value = op.type.exception_value
                if self.exception_check == '+' and self.exception_value is None:
                    env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
5003
            else:
5004
                rhs = self.rhs.coerce_to(self.lhs.type, env)
5005
        else:
5006 5007
            rhs = self.rhs.coerce_to(self.lhs.type, env)

5008 5009 5010
        if use_temp or rhs.is_attribute or (
                not rhs.is_name and not rhs.is_literal and
                rhs.type.is_pyobject):
5011
            # things like (cdef) attribute access are not safe (traverses pointers)
5012 5013 5014 5015
            rhs = rhs.coerce_to_temp(env)
        elif rhs.type.is_pyobject:
            rhs = rhs.coerce_to_simple(env)
        self.rhs = rhs
5016
        return self
5017

5018 5019
    def unroll(self, node, target_size, env):
        from . import ExprNodes, UtilNodes
5020 5021 5022 5023

        base = node
        start_node = stop_node = step_node = check_node = None

5024
        if node.type.is_ctuple:
5025
            slice_size = node.type.size
5026

5027 5028 5029
        elif node.type.is_ptr or node.type.is_array:
            while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
                base = node = node.base
5030 5031 5032
            if isinstance(node, ExprNodes.SliceIndexNode):
                base = node.base
                start_node = node.start
5033 5034
                if start_node:
                    start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5035
                stop_node = node.stop
5036 5037 5038
                if stop_node:
                    stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
                else:
5039
                    if node.type.is_array and node.type.size:
5040 5041
                        stop_node = ExprNodes.IntNode(
                            self.pos, value=str(node.type.size),
5042
                            constant_result=(node.type.size if isinstance(node.type.size, _py_int_types)
5043
                                             else ExprNodes.constant_value_not_set))
5044 5045 5046
                    else:
                        error(self.pos, "C array iteration requires known end index")
                        return
5047
                step_node = None  #node.step
5048 5049
                if step_node:
                    step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5050

5051 5052 5053 5054
                # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
                def get_const(node, none_value):
                    if node is None:
                        return none_value
5055
                    elif node.has_constant_result():
5056 5057
                        return node.constant_result
                    else:
5058 5059
                        raise ValueError("Not a constant.")

5060 5061 5062 5063 5064
                try:
                    slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
                except ValueError:
                    error(self.pos, "C array assignment currently requires known endpoints")
                    return
5065 5066 5067

            elif node.type.is_array:
                slice_size = node.type.size
5068
                if not isinstance(slice_size, _py_int_types):
5069
                    return  # might still work when coercing to Python
5070 5071 5072 5073 5074 5075
            else:
                return

        else:
            return

5076 5077 5078 5079 5080
        if slice_size != target_size:
            error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
                slice_size, target_size))
            return

5081
        items = []
5082 5083 5084
        base = UtilNodes.LetRefNode(base)
        refs = [base]
        if start_node and not start_node.is_literal:
5085 5086
            start_node = UtilNodes.LetRefNode(start_node)
            refs.append(start_node)
5087
        if stop_node and not stop_node.is_literal:
5088 5089
            stop_node = UtilNodes.LetRefNode(stop_node)
            refs.append(stop_node)
5090
        if step_node and not step_node.is_literal:
5091 5092
            step_node = UtilNodes.LetRefNode(step_node)
            refs.append(step_node)
5093

5094
        for ix in range(target_size):
5095
            ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
5096
            if step_node is not None:
5097 5098 5099 5100 5101
                if step_node.has_constant_result():
                    step_value = ix_node.constant_result * step_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
                else:
                    ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
5102
            if start_node is not None:
5103 5104 5105 5106 5107 5108 5109
                if start_node.has_constant_result() and ix_node.has_constant_result():
                    index_value = ix_node.constant_result + start_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
                else:
                    ix_node = ExprNodes.AddNode(
                        self.pos, operator='+', operand1=start_node, operand2=ix_node)
            items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
5110 5111
        return check_node, refs, items

5112
    def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
5113
        from . import UtilNodes
5114
        assignments = []
5115
        for lhs, rhs in zip(lhs_list, rhs_list):
5116
            assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
5117
        node = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
5118
        if check_node:
5119
            node = StatListNode(pos=self.pos, stats=[check_node, node])
5120
        for ref in refs[::-1]:
5121 5122
            node = UtilNodes.LetNode(ref, node)
        return node
5123

5124
    def unroll_rhs(self, env):
5125
        from . import ExprNodes
5126 5127
        if not isinstance(self.lhs, ExprNodes.TupleNode):
            return
5128 5129
        if any(arg.is_starred for arg in self.lhs.args):
            return
5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140

        unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
        if not unrolled:
            return
        check_node, refs, rhs = unrolled
        return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)

    def unroll_lhs(self, env):
        if self.lhs.type.is_ctuple:
            # Handled directly.
            return
5141
        from . import ExprNodes
5142 5143 5144 5145 5146 5147 5148 5149 5150
        if not isinstance(self.rhs, ExprNodes.TupleNode):
            return

        unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
        if not unrolled:
            return
        check_node, refs, lhs = unrolled
        return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)

William Stein's avatar
William Stein committed
5151 5152
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5153

5154
    def generate_assignment_code(self, code, overloaded_assignment=False):
5155 5156 5157 5158 5159 5160 5161 5162 5163
        if self.is_overloaded_assignment:
            self.lhs.generate_assignment_code(
                self.rhs,
                code,
                overloaded_assignment=self.is_overloaded_assignment,
                exception_check=self.exception_check,
                exception_value=self.exception_value)
        else:
            self.lhs.generate_assignment_code(self.rhs, code)
William Stein's avatar
William Stein committed
5164

5165 5166 5167
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5168 5169 5170 5171
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

William Stein's avatar
William Stein committed
5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182

class CascadedAssignmentNode(AssignmentNode):
    #  An assignment with multiple left hand sides:
    #
    #    a = b = c
    #
    #  lhs_list   [ExprNode]   Left hand sides
    #  rhs        ExprNode     Right hand sides
    #
    #  Used internally:
    #
5183 5184 5185
    #  coerced_values       [ExprNode]   RHS coerced to all distinct LHS types
    #  cloned_values        [ExprNode]   cloned RHS value for each LHS
    #  assignment_overloads [Bool]       If each assignment uses a C++ operator=
5186

5187 5188
    child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
    cloned_values = None
5189
    coerced_values = None
5190
    assignment_overloads = None
5191

William Stein's avatar
William Stein committed
5192 5193 5194
    def analyse_declarations(self, env):
        for lhs in self.lhs_list:
            lhs.analyse_target_declaration(env)
5195

5196
    def analyse_types(self, env, use_temp=0):
5197
        from .ExprNodes import CloneNode, ProxyNode
5198

5199
        # collect distinct types used on the LHS
5200
        lhs_types = set()
5201 5202
        for i, lhs in enumerate(self.lhs_list):
            lhs = self.lhs_list[i] = lhs.analyse_target_types(env)
5203 5204 5205
            lhs.gil_assignment_check(env)
            lhs_types.add(lhs.type)

5206
        rhs = self.rhs.analyse_types(env)
5207
        # common special case: only one type needed on the LHS => coerce only once
5208
        if len(lhs_types) == 1:
5209 5210 5211 5212 5213 5214 5215
            # Avoid coercion for overloaded assignment operators.
            if next(iter(lhs_types)).is_cpp_class:
                op = env.lookup_operator('=', [lhs, self.rhs])
                if not op:
                    rhs = rhs.coerce_to(lhs_types.pop(), env)
            else:
                rhs = rhs.coerce_to(lhs_types.pop(), env)
5216 5217 5218

        if not rhs.is_name and not rhs.is_literal and (
                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
5219
            rhs = rhs.coerce_to_temp(env)
5220
        else:
5221
            rhs = rhs.coerce_to_simple(env)
5222 5223
        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs

5224
        # clone RHS and coerce it to all distinct LHS types
5225 5226
        self.coerced_values = []
        coerced_values = {}
5227
        self.assignment_overloads = []
5228
        for lhs in self.lhs_list:
5229 5230
            overloaded = lhs.type.is_cpp_class and env.lookup_operator('=', [lhs, self.rhs])
            self.assignment_overloads.append(overloaded)
5231
            if lhs.type not in coerced_values and lhs.type != rhs.type:
5232
                rhs = CloneNode(self.rhs)
5233
                if not overloaded:
5234
                    rhs = rhs.coerce_to(lhs.type, env)
5235 5236
                self.coerced_values.append(rhs)
                coerced_values[lhs.type] = rhs
5237

5238 5239
        # clone coerced values for all LHS assignments
        self.cloned_values = []
William Stein's avatar
William Stein committed
5240
        for lhs in self.lhs_list:
5241
            rhs = coerced_values.get(lhs.type, self.rhs)
5242
            self.cloned_values.append(CloneNode(rhs))
5243
        return self
5244

William Stein's avatar
William Stein committed
5245 5246
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5247

5248
    def generate_assignment_code(self, code, overloaded_assignment=False):
5249
        # prepare all coercions
5250 5251
        for rhs in self.coerced_values:
            rhs.generate_evaluation_code(code)
5252
        # assign clones to LHS
5253
        for lhs, rhs, overload in zip(self.lhs_list, self.cloned_values, self.assignment_overloads):
William Stein's avatar
William Stein committed
5254
            rhs.generate_evaluation_code(code)
5255
            lhs.generate_assignment_code(rhs, code, overloaded_assignment=overload)
5256 5257 5258 5259
        # dispose of coerced values and original RHS
        for rhs_value in self.coerced_values:
            rhs_value.generate_disposal_code(code)
            rhs_value.free_temps(code)
William Stein's avatar
William Stein committed
5260
        self.rhs.generate_disposal_code(code)
5261
        self.rhs.free_temps(code)
William Stein's avatar
William Stein committed
5262

5263 5264 5265
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5266
    def annotate(self, code):
5267 5268
        for rhs in self.coerced_values:
            rhs.annotate(code)
5269
        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
5270 5271
            lhs.annotate(code)
            rhs.annotate(code)
5272
        self.rhs.annotate(code)
5273

5274

William Stein's avatar
William Stein committed
5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287
class ParallelAssignmentNode(AssignmentNode):
    #  A combined packing/unpacking assignment:
    #
    #    a, b, c =  d, e, f
    #
    #  This has been rearranged by the parser into
    #
    #    a = d ; b = e ; c = f
    #
    #  but we must evaluate all the right hand sides
    #  before assigning to any of the left hand sides.
    #
    #  stats     [AssignmentNode]   The constituent assignments
5288

5289 5290
    child_attrs = ["stats"]

William Stein's avatar
William Stein committed
5291 5292 5293
    def analyse_declarations(self, env):
        for stat in self.stats:
            stat.analyse_declarations(env)
5294

William Stein's avatar
William Stein committed
5295
    def analyse_expressions(self, env):
5296 5297
        self.stats = [stat.analyse_types(env, use_temp=1)
                      for stat in self.stats]
5298
        return self
5299

Robert Bradshaw's avatar
Robert Bradshaw committed
5300 5301
#    def analyse_expressions(self, env):
#        for stat in self.stats:
5302
#            stat.analyse_expressions_1(env, use_temp=1)
Robert Bradshaw's avatar
Robert Bradshaw committed
5303 5304
#        for stat in self.stats:
#            stat.analyse_expressions_2(env)
5305

William Stein's avatar
William Stein committed
5306
    def generate_execution_code(self, code):
5307
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5308 5309 5310 5311 5312
        for stat in self.stats:
            stat.generate_rhs_evaluation_code(code)
        for stat in self.stats:
            stat.generate_assignment_code(code)

5313 5314 5315 5316
    def generate_function_definitions(self, env, code):
        for stat in self.stats:
            stat.generate_function_definitions(env, code)

5317 5318 5319 5320 5321
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)


5322
class InPlaceAssignmentNode(AssignmentNode):
Craig Citro's avatar
Craig Citro committed
5323
    #  An in place arithmetic operand:
5324 5325 5326 5327 5328 5329 5330
    #
    #    a += b
    #    a -= b
    #    ...
    #
    #  lhs      ExprNode      Left hand side
    #  rhs      ExprNode      Right hand side
Stefan Behnel's avatar
Stefan Behnel committed
5331
    #  operator char          one of "+-*/%^&|"
5332
    #
5333 5334 5335 5336 5337 5338 5339
    #  This code is a bit tricky because in order to obey Python
    #  semantics the sub-expressions (e.g. indices) of the lhs must
    #  not be evaluated twice. So we must re-use the values calculated
    #  in evaluation phase for the assignment phase as well.
    #  Fortunately, the type of the lhs node is fairly constrained
    #  (it must be a NameNode, AttributeNode, or IndexNode).

5340
    child_attrs = ["lhs", "rhs"]
5341

5342 5343
    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)
5344

5345
    def analyse_types(self, env):
5346 5347
        self.rhs = self.rhs.analyse_types(env)
        self.lhs = self.lhs.analyse_target_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5348

5349
        # When assigning to a fully indexed buffer or memoryview, coerce the rhs
5350
        if self.lhs.is_memview_index or self.lhs.is_buffer_access:
5351
            self.rhs = self.rhs.coerce_to(self.lhs.type, env)
5352 5353 5354
        elif self.lhs.type.is_string and self.operator in '+-':
            # use pointer arithmetic for char* LHS instead of string concat
            self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5355
        return self
5356

5357
    def generate_execution_code(self, code):
5358
        code.mark_pos(self.pos)
5359 5360 5361
        lhs, rhs = self.lhs, self.rhs
        rhs.generate_evaluation_code(code)
        lhs.generate_subexpr_evaluation_code(code)
5362 5363 5364 5365 5366
        c_op = self.operator
        if c_op == "//":
            c_op = "/"
        elif c_op == "**":
            error(self.pos, "No C inplace power operator")
5367 5368
        if lhs.is_buffer_access or lhs.is_memview_index:
            if lhs.type.is_pyobject:
5369
                error(self.pos, "In-place operators not allowed on object buffers in this release.")
5370
            if c_op in ('/', '%') and lhs.type.is_int and not code.globalstate.directives['cdivision']:
Robert Bradshaw's avatar
Robert Bradshaw committed
5371
                error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
5372 5373 5374
            lhs.generate_buffer_setitem_code(rhs, code, c_op)
        elif lhs.is_memview_slice:
            error(self.pos, "Inplace operators not supported on memoryview slices")
5375 5376 5377
        else:
            # C++
            # TODO: make sure overload is declared
5378 5379 5380 5381 5382
            code.putln("%s %s= %s;" % (lhs.result(), c_op, rhs.result()))
        lhs.generate_subexpr_disposal_code(code)
        lhs.free_subexpr_temps(code)
        rhs.generate_disposal_code(code)
        rhs.free_temps(code)
5383

5384 5385 5386
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)
5387

5388
    def create_binop_node(self):
5389
        from . import ExprNodes
5390
        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
5391

William Stein's avatar
William Stein committed
5392 5393 5394 5395

class PrintStatNode(StatNode):
    #  print statement
    #
5396
    #  arg_tuple         TupleNode
5397
    #  stream            ExprNode or None (stdout)
5398
    #  append_newline    boolean
5399

5400
    child_attrs = ["arg_tuple", "stream"]
5401

William Stein's avatar
William Stein committed
5402
    def analyse_expressions(self, env):
5403
        if self.stream:
5404 5405 5406 5407
            stream = self.stream.analyse_expressions(env)
            self.stream = stream.coerce_to_pyobject(env)
        arg_tuple = self.arg_tuple.analyse_expressions(env)
        self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
5408
        env.use_utility_code(printing_utility_code)
5409 5410
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            env.use_utility_code(printing_one_utility_code)
5411
        return self
5412

5413
    nogil_check = Node.gil_error
5414
    gil_message = "Python print statement"
5415

William Stein's avatar
William Stein committed
5416
    def generate_execution_code(self, code):
5417
        code.mark_pos(self.pos)
5418 5419 5420 5421 5422
        if self.stream:
            self.stream.generate_evaluation_code(code)
            stream_result = self.stream.py_result()
        else:
            stream_result = '0'
5423 5424 5425
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            arg = self.arg_tuple.args[0]
            arg.generate_evaluation_code(code)
5426

5427
            code.putln(
5428 5429
                "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
                    stream_result,
5430 5431 5432 5433 5434 5435 5436
                    arg.py_result(),
                    code.error_goto(self.pos)))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
        else:
            self.arg_tuple.generate_evaluation_code(code)
            code.putln(
5437 5438
                "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
                    stream_result,
5439 5440 5441 5442 5443
                    self.arg_tuple.py_result(),
                    self.append_newline,
                    code.error_goto(self.pos)))
            self.arg_tuple.generate_disposal_code(code)
            self.arg_tuple.free_temps(code)
5444

5445 5446 5447 5448
        if self.stream:
            self.stream.generate_disposal_code(code)
            self.stream.free_temps(code)

5449
    def generate_function_definitions(self, env, code):
5450 5451
        if self.stream:
            self.stream.generate_function_definitions(env, code)
5452
        self.arg_tuple.generate_function_definitions(env, code)
5453

5454
    def annotate(self, code):
5455 5456
        if self.stream:
            self.stream.annotate(code)
5457
        self.arg_tuple.annotate(code)
William Stein's avatar
William Stein committed
5458 5459


5460 5461 5462 5463 5464 5465 5466 5467 5468
class ExecStatNode(StatNode):
    #  exec statement
    #
    #  args     [ExprNode]

    child_attrs = ["args"]

    def analyse_expressions(self, env):
        for i, arg in enumerate(self.args):
5469
            arg = arg.analyse_expressions(env)
5470 5471 5472
            arg = arg.coerce_to_pyobject(env)
            self.args[i] = arg
        env.use_utility_code(Builtin.pyexec_utility_code)
5473
        return self
5474

5475
    nogil_check = Node.gil_error
5476 5477 5478
    gil_message = "Python exec statement"

    def generate_execution_code(self, code):
5479
        code.mark_pos(self.pos)
5480 5481 5482
        args = []
        for arg in self.args:
            arg.generate_evaluation_code(code)
5483
            args.append(arg.py_result())
5484
        args = tuple(args + ['0', '0'][:3-len(args)])
5485
        temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
5486
        code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % ((temp_result,) + args))
5487 5488
        for arg in self.args:
            arg.generate_disposal_code(code)
5489
            arg.free_temps(code)
5490
        code.putln(
5491 5492 5493 5494
            code.error_goto_if_null(temp_result, self.pos))
        code.put_gotref(temp_result)
        code.put_decref_clear(temp_result, py_object_type)
        code.funcstate.release_temp(temp_result)
5495 5496 5497 5498 5499 5500

    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)


William Stein's avatar
William Stein committed
5501 5502 5503 5504
class DelStatNode(StatNode):
    #  del statement
    #
    #  args     [ExprNode]
5505

5506
    child_attrs = ["args"]
5507
    ignore_nonexisting = False
5508

William Stein's avatar
William Stein committed
5509 5510 5511
    def analyse_declarations(self, env):
        for arg in self.args:
            arg.analyse_target_declaration(env)
5512

William Stein's avatar
William Stein committed
5513
    def analyse_expressions(self, env):
5514 5515
        for i, arg in enumerate(self.args):
            arg = self.args[i] = arg.analyse_target_expression(env, None)
5516
            if arg.type.is_pyobject or (arg.is_name and arg.type.is_memoryviewslice):
5517 5518
                if arg.is_name and arg.entry.is_cglobal:
                    error(arg.pos, "Deletion of global C variable")
Robert Bradshaw's avatar
Robert Bradshaw committed
5519
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
Robert Bradshaw's avatar
Robert Bradshaw committed
5520
                self.cpp_check(env)
5521
            elif arg.type.is_cpp_class:
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5522
                error(arg.pos, "Deletion of non-heap C++ object")
5523 5524
            elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
                pass  # del ba[i]
5525
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5526
                error(arg.pos, "Deletion of non-Python, non-C++ object")
5527
            #arg.release_target_temp(env)
5528
        return self
5529

5530
    def nogil_check(self, env):
5531 5532
        for arg in self.args:
            if arg.type.is_pyobject:
5533
                self.gil_error()
5534

5535 5536
    gil_message = "Deleting Python object"

William Stein's avatar
William Stein committed
5537
    def generate_execution_code(self, code):
5538
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5539
        for arg in self.args:
5540 5541 5542
            if (arg.type.is_pyobject or
                    arg.type.is_memoryviewslice or
                    arg.is_subscript and arg.base.type is Builtin.bytearray_type):
5543 5544
                arg.generate_deletion_code(
                    code, ignore_nonexisting=self.ignore_nonexisting)
5545
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
5546
                arg.generate_evaluation_code(code)
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5547
                code.putln("delete %s;" % arg.result())
5548
                arg.generate_disposal_code(code)
William Stein's avatar
William Stein committed
5549 5550
            # else error reported earlier

5551 5552 5553 5554
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)

William Stein's avatar
William Stein committed
5555 5556 5557

class PassStatNode(StatNode):
    #  pass statement
5558 5559

    child_attrs = []
5560

William Stein's avatar
William Stein committed
5561
    def analyse_expressions(self, env):
5562
        return self
5563

William Stein's avatar
William Stein committed
5564 5565 5566 5567
    def generate_execution_code(self, code):
        pass


5568 5569 5570 5571 5572 5573 5574 5575 5576
class IndirectionNode(StatListNode):
    """
    This adds an indirection so that the node can be shared and a subtree can
    be removed at any time by clearing self.stats.
    """

    def __init__(self, stats):
        super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)

5577

William Stein's avatar
William Stein committed
5578 5579
class BreakStatNode(StatNode):

5580
    child_attrs = []
5581
    is_terminator = True
5582

William Stein's avatar
William Stein committed
5583
    def analyse_expressions(self, env):
5584
        return self
5585

William Stein's avatar
William Stein committed
5586
    def generate_execution_code(self, code):
5587
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5588 5589 5590
        if not code.break_label:
            error(self.pos, "break statement not inside loop")
        else:
5591
            code.put_goto(code.break_label)
William Stein's avatar
William Stein committed
5592 5593 5594 5595


class ContinueStatNode(StatNode):

5596
    child_attrs = []
5597
    is_terminator = True
5598

William Stein's avatar
William Stein committed
5599
    def analyse_expressions(self, env):
5600
        return self
5601

William Stein's avatar
William Stein committed
5602
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
5603
        if not code.continue_label:
William Stein's avatar
William Stein committed
5604
            error(self.pos, "continue statement not inside loop")
Stefan Behnel's avatar
Stefan Behnel committed
5605 5606 5607
            return
        code.mark_pos(self.pos)
        code.put_goto(code.continue_label)
William Stein's avatar
William Stein committed
5608 5609 5610 5611 5612 5613 5614


class ReturnStatNode(StatNode):
    #  return statement
    #
    #  value         ExprNode or None
    #  return_type   PyrexType
5615
    #  in_generator  return inside of generator => raise StopIteration
5616
    #  in_async_gen  return inside of async generator
5617

5618
    child_attrs = ["value"]
5619
    is_terminator = True
5620
    in_generator = False
5621
    in_async_gen = False
5622

5623 5624 5625
    # Whether we are in a parallel section
    in_parallel = False

William Stein's avatar
William Stein committed
5626 5627 5628 5629 5630
    def analyse_expressions(self, env):
        return_type = env.return_type
        self.return_type = return_type
        if not return_type:
            error(self.pos, "Return not inside a function body")
5631
            return self
William Stein's avatar
William Stein committed
5632
        if self.value:
5633 5634
            if self.in_async_gen:
                error(self.pos, "Return with value in async generator")
5635
            self.value = self.value.analyse_types(env)
William Stein's avatar
William Stein committed
5636
            if return_type.is_void or return_type.is_returncode:
5637
                error(self.value.pos, "Return with value in void function")
William Stein's avatar
William Stein committed
5638 5639 5640 5641
            else:
                self.value = self.value.coerce_to(env.return_type, env)
        else:
            if (not return_type.is_void
5642 5643 5644
                    and not return_type.is_pyobject
                    and not return_type.is_returncode):
                error(self.pos, "Return value required")
5645
        return self
5646

5647
    def nogil_check(self, env):
5648
        if self.return_type.is_pyobject:
5649
            self.gil_error()
5650 5651 5652

    gil_message = "Returning Python object"

William Stein's avatar
William Stein committed
5653
    def generate_execution_code(self, code):
5654
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5655 5656 5657
        if not self.return_type:
            # error reported earlier
            return
5658 5659 5660
        if self.return_type.is_pyobject:
            code.put_xdecref(Naming.retval_cname,
                             self.return_type)
5661

William Stein's avatar
William Stein committed
5662 5663
        if self.value:
            self.value.generate_evaluation_code(code)
5664
            if self.return_type.is_memoryviewslice:
5665
                from . import MemoryView
5666
                MemoryView.put_acquire_memoryviewslice(
5667 5668 5669 5670 5671 5672
                    lhs_cname=Naming.retval_cname,
                    lhs_type=self.return_type,
                    lhs_pos=self.value.pos,
                    rhs=self.value,
                    code=code,
                    have_gil=self.in_nogil_context)
5673 5674
            elif self.in_generator:
                # return value == raise StopIteration(value), but uncatchable
5675
                code.globalstate.use_utility_code(
5676
                    UtilityCode.load_cached("ReturnWithStopIteration", "Coroutine.c"))
5677 5678 5679
                code.putln("%s = NULL; __Pyx_ReturnWithStopIteration(%s);" % (
                    Naming.retval_cname,
                    self.value.py_result()))
5680
                self.value.generate_disposal_code(code)
5681 5682
            else:
                self.value.make_owned_reference(code)
5683 5684 5685
                code.putln("%s = %s;" % (
                    Naming.retval_cname,
                    self.value.result_as(self.return_type)))
5686
            self.value.generate_post_assignment_code(code)
5687
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5688 5689
        else:
            if self.return_type.is_pyobject:
5690
                if self.in_generator:
5691 5692
                    if self.in_async_gen:
                        code.put("PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); ")
5693 5694 5695
                    code.putln("%s = NULL;" % Naming.retval_cname)
                else:
                    code.put_init_to_py_none(Naming.retval_cname, self.return_type)
William Stein's avatar
William Stein committed
5696
            elif self.return_type.is_returncode:
5697 5698
                self.put_return(code, self.return_type.default_value)

5699
        for cname, type in code.funcstate.temps_holding_reference():
5700
            code.put_decref_clear(cname, type)
5701

5702
        code.put_goto(code.return_label)
5703

5704 5705 5706 5707 5708
    def put_return(self, code, value):
        if self.in_parallel:
            code.putln_openmp("#pragma omp critical(__pyx_returning)")
        code.putln("%s = %s;" % (Naming.retval_cname, value))

5709 5710 5711
    def generate_function_definitions(self, env, code):
        if self.value is not None:
            self.value.generate_function_definitions(env, code)
5712

5713 5714 5715
    def annotate(self, code):
        if self.value:
            self.value.annotate(code)
William Stein's avatar
William Stein committed
5716 5717 5718 5719 5720 5721 5722 5723


class RaiseStatNode(StatNode):
    #  raise statement
    #
    #  exc_type    ExprNode or None
    #  exc_value   ExprNode or None
    #  exc_tb      ExprNode or None
Haoyu Bai's avatar
Haoyu Bai committed
5724
    #  cause       ExprNode or None
5725

Haoyu Bai's avatar
Haoyu Bai committed
5726
    child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
5727
    is_terminator = True
5728

William Stein's avatar
William Stein committed
5729 5730
    def analyse_expressions(self, env):
        if self.exc_type:
5731 5732
            exc_type = self.exc_type.analyse_types(env)
            self.exc_type = exc_type.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5733
        if self.exc_value:
5734 5735
            exc_value = self.exc_value.analyse_types(env)
            self.exc_value = exc_value.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5736
        if self.exc_tb:
5737 5738
            exc_tb = self.exc_tb.analyse_types(env)
            self.exc_tb = exc_tb.coerce_to_pyobject(env)
Haoyu Bai's avatar
Haoyu Bai committed
5739
        if self.cause:
5740 5741
            cause = self.cause.analyse_types(env)
            self.cause = cause.coerce_to_pyobject(env)
5742 5743 5744 5745
        # special cases for builtin exceptions
        self.builtin_exc_name = None
        if self.exc_type and not self.exc_value and not self.exc_tb:
            exc = self.exc_type
5746
            from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
5747
            if (isinstance(exc, ExprNodes.SimpleCallNode) and
5748 5749
                    not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))):
                exc = exc.function  # extract the exception type
5750 5751 5752 5753
            if exc.is_name and exc.entry.is_builtin:
                self.builtin_exc_name = exc.name
                if self.builtin_exc_name == 'MemoryError':
                    self.exc_type = None # has a separate implementation
5754
        return self
5755

5756
    nogil_check = Node.gil_error
5757 5758
    gil_message = "Raising exception"

William Stein's avatar
William Stein committed
5759
    def generate_execution_code(self, code):
5760
        code.mark_pos(self.pos)
5761 5762 5763 5764
        if self.builtin_exc_name == 'MemoryError':
            code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
            return

William Stein's avatar
William Stein committed
5765 5766 5767
        if self.exc_type:
            self.exc_type.generate_evaluation_code(code)
            type_code = self.exc_type.py_result()
5768 5769
            if self.exc_type.is_name:
                code.globalstate.use_entry_utility_code(self.exc_type.entry)
William Stein's avatar
William Stein committed
5770
        else:
Stefan Behnel's avatar
Stefan Behnel committed
5771
            type_code = "0"
William Stein's avatar
William Stein committed
5772 5773 5774 5775 5776 5777 5778 5779 5780 5781
        if self.exc_value:
            self.exc_value.generate_evaluation_code(code)
            value_code = self.exc_value.py_result()
        else:
            value_code = "0"
        if self.exc_tb:
            self.exc_tb.generate_evaluation_code(code)
            tb_code = self.exc_tb.py_result()
        else:
            tb_code = "0"
Haoyu Bai's avatar
Haoyu Bai committed
5782 5783 5784 5785 5786
        if self.cause:
            self.cause.generate_evaluation_code(code)
            cause_code = self.cause.py_result()
        else:
            cause_code = "0"
5787
        code.globalstate.use_utility_code(raise_utility_code)
5788
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
5789
            "__Pyx_Raise(%s, %s, %s, %s);" % (
5790 5791
                type_code,
                value_code,
Haoyu Bai's avatar
Haoyu Bai committed
5792 5793 5794
                tb_code,
                cause_code))
        for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
5795 5796 5797
            if obj:
                obj.generate_disposal_code(code)
                obj.free_temps(code)
William Stein's avatar
William Stein committed
5798 5799 5800
        code.putln(
            code.error_goto(self.pos))

5801 5802 5803 5804 5805 5806 5807
    def generate_function_definitions(self, env, code):
        if self.exc_type is not None:
            self.exc_type.generate_function_definitions(env, code)
        if self.exc_value is not None:
            self.exc_value.generate_function_definitions(env, code)
        if self.exc_tb is not None:
            self.exc_tb.generate_function_definitions(env, code)
Haoyu Bai's avatar
Haoyu Bai committed
5808 5809
        if self.cause is not None:
            self.cause.generate_function_definitions(env, code)
5810

5811 5812 5813 5814 5815 5816 5817
    def annotate(self, code):
        if self.exc_type:
            self.exc_type.annotate(code)
        if self.exc_value:
            self.exc_value.annotate(code)
        if self.exc_tb:
            self.exc_tb.annotate(code)
Haoyu Bai's avatar
Haoyu Bai committed
5818 5819
        if self.cause:
            self.cause.annotate(code)
5820

William Stein's avatar
William Stein committed
5821

5822 5823
class ReraiseStatNode(StatNode):

5824
    child_attrs = []
5825
    is_terminator = True
5826

5827
    def analyse_expressions(self, env):
5828
        return self
5829

5830
    nogil_check = Node.gil_error
5831 5832
    gil_message = "Raising exception"

5833
    def generate_execution_code(self, code):
5834
        code.mark_pos(self.pos)
5835
        vars = code.funcstate.exc_vars
5836
        if vars:
5837
            code.globalstate.use_utility_code(restore_exception_utility_code)
5838 5839 5840 5841
            code.put_giveref(vars[0])
            code.put_giveref(vars[1])
            # fresh exceptions may not have a traceback yet (-> finally!)
            code.put_xgiveref(vars[2])
5842
            code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars))
5843 5844 5845
            for varname in vars:
                code.put("%s = 0; " % varname)
            code.putln()
5846 5847
            code.putln(code.error_goto(self.pos))
        else:
5848 5849 5850
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
            code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
5851

William Stein's avatar
William Stein committed
5852 5853 5854 5855 5856
class AssertStatNode(StatNode):
    #  assert statement
    #
    #  cond    ExprNode
    #  value   ExprNode or None
5857

5858 5859
    child_attrs = ["cond", "value"]

William Stein's avatar
William Stein committed
5860 5861 5862
    def analyse_expressions(self, env):
        self.cond = self.cond.analyse_boolean_expression(env)
        if self.value:
5863
            value = self.value.analyse_types(env)
5864 5865
            if value.type is Builtin.tuple_type or not value.type.is_builtin_type:
                # prevent tuple values from being interpreted as argument value tuples
5866
                from .ExprNodes import TupleNode
5867
                value = TupleNode(value.pos, args=[value], slow=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
5868
                self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env)
5869 5870
            else:
                self.value = value.coerce_to_pyobject(env)
5871
        return self
5872

5873
    nogil_check = Node.gil_error
5874
    gil_message = "Raising exception"
5875

William Stein's avatar
William Stein committed
5876
    def generate_execution_code(self, code):
5877
        code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
5878
        code.putln("if (unlikely(!Py_OptimizeFlag)) {")
5879
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5880 5881
        self.cond.generate_evaluation_code(code)
        code.putln(
5882
            "if (unlikely(!%s)) {" % self.cond.result())
William Stein's avatar
William Stein committed
5883
        if self.value:
5884
            self.value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
5885
            code.putln(
5886
                "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result())
5887
            self.value.generate_disposal_code(code)
5888
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5889 5890 5891 5892
        else:
            code.putln(
                "PyErr_SetNone(PyExc_AssertionError);")
        code.putln(
5893
            code.error_goto(self.pos))
William Stein's avatar
William Stein committed
5894 5895 5896
        code.putln(
            "}")
        self.cond.generate_disposal_code(code)
5897
        self.cond.free_temps(code)
5898 5899
        code.putln(
            "}")
5900
        code.putln("#endif")
William Stein's avatar
William Stein committed
5901

5902 5903 5904 5905 5906
    def generate_function_definitions(self, env, code):
        self.cond.generate_function_definitions(env, code)
        if self.value is not None:
            self.value.generate_function_definitions(env, code)

5907 5908 5909 5910 5911 5912
    def annotate(self, code):
        self.cond.annotate(code)
        if self.value:
            self.value.annotate(code)


William Stein's avatar
William Stein committed
5913 5914 5915 5916 5917
class IfStatNode(StatNode):
    #  if statement
    #
    #  if_clauses   [IfClauseNode]
    #  else_clause  StatNode or None
5918 5919

    child_attrs = ["if_clauses", "else_clause"]
5920

William Stein's avatar
William Stein committed
5921 5922 5923 5924 5925
    def analyse_declarations(self, env):
        for if_clause in self.if_clauses:
            if_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
5926

William Stein's avatar
William Stein committed
5927
    def analyse_expressions(self, env):
5928
        self.if_clauses = [if_clause.analyse_expressions(env) for if_clause in self.if_clauses]
William Stein's avatar
William Stein committed
5929
        if self.else_clause:
5930 5931
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
5932

William Stein's avatar
William Stein committed
5933
    def generate_execution_code(self, code):
5934
        code.mark_pos(self.pos)
5935
        end_label = code.new_label()
5936 5937 5938
        last = len(self.if_clauses)
        if not self.else_clause:
            last -= 1  # avoid redundant goto at end of last if-clause
5939 5940
        for i, if_clause in enumerate(self.if_clauses):
            if_clause.generate_execution_code(code, end_label, is_last=i == last)
5941
        if self.else_clause:
5942
            code.mark_pos(self.else_clause.pos)
5943
            code.putln("/*else*/ {")
William Stein's avatar
William Stein committed
5944
            self.else_clause.generate_execution_code(code)
5945 5946
            code.putln("}")
        code.put_label(end_label)
5947

5948 5949 5950 5951 5952
    def generate_function_definitions(self, env, code):
        for clause in self.if_clauses:
            clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
5953

5954 5955 5956 5957 5958
    def annotate(self, code):
        for if_clause in self.if_clauses:
            if_clause.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
5959 5960 5961 5962 5963 5964 5965


class IfClauseNode(Node):
    #  if or elif clause in an if statement
    #
    #  condition   ExprNode
    #  body        StatNode
5966

5967 5968
    child_attrs = ["condition", "body"]

William Stein's avatar
William Stein committed
5969 5970
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
5971

William Stein's avatar
William Stein committed
5972
    def analyse_expressions(self, env):
5973
        self.condition = self.condition.analyse_temp_boolean_expression(env)
5974 5975
        self.body = self.body.analyse_expressions(env)
        return self
5976

5977
    def generate_execution_code(self, code, end_label, is_last):
William Stein's avatar
William Stein committed
5978
        self.condition.generate_evaluation_code(code)
5979 5980
        code.mark_pos(self.pos)
        code.putln("if (%s) {" % self.condition.result())
5981 5982
        self.condition.generate_disposal_code(code)
        self.condition.free_temps(code)
William Stein's avatar
William Stein committed
5983
        self.body.generate_execution_code(code)
5984 5985
        code.mark_pos(self.pos, trace=False)
        if not (is_last or self.body.is_terminator):
5986
            code.put_goto(end_label)
William Stein's avatar
William Stein committed
5987
        code.putln("}")
5988

5989 5990 5991 5992
    def generate_function_definitions(self, env, code):
        self.condition.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)

5993 5994 5995
    def annotate(self, code):
        self.condition.annotate(code)
        self.body.annotate(code)
5996

5997 5998 5999 6000 6001 6002

class SwitchCaseNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # conditions    [ExprNode]
    # body          StatNode
6003

6004
    child_attrs = ['conditions', 'body']
6005

6006 6007
    def generate_execution_code(self, code):
        for cond in self.conditions:
6008
            code.mark_pos(cond.pos)
6009 6010
            cond.generate_evaluation_code(code)
            code.putln("case %s:" % cond.result())
6011
        self.body.generate_execution_code(code)
6012
        code.mark_pos(self.pos, trace=False)
6013
        code.putln("break;")
6014 6015 6016 6017 6018

    def generate_function_definitions(self, env, code):
        for cond in self.conditions:
            cond.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
6019

6020 6021 6022
    def annotate(self, code):
        for cond in self.conditions:
            cond.annotate(code)
6023
        self.body.annotate(code)
6024

6025

6026 6027 6028 6029 6030 6031
class SwitchStatNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # test          ExprNode
    # cases         [SwitchCaseNode]
    # else_clause   StatNode or None
6032

6033
    child_attrs = ['test', 'cases', 'else_clause']
6034

6035
    def generate_execution_code(self, code):
6036
        self.test.generate_evaluation_code(code)
6037
        code.mark_pos(self.pos)
6038
        code.putln("switch (%s) {" % self.test.result())
6039 6040 6041 6042 6043
        for case in self.cases:
            case.generate_execution_code(code)
        if self.else_clause is not None:
            code.putln("default:")
            self.else_clause.generate_execution_code(code)
6044
            code.putln("break;")
6045 6046 6047 6048 6049
        else:
            # Always generate a default clause to prevent C compiler warnings
            # about unmatched enum values (it was not the user who decided to
            # generate the switch statement, so shouldn't be bothered).
            code.putln("default: break;")
6050 6051
        code.putln("}")

6052 6053 6054 6055 6056 6057 6058
    def generate_function_definitions(self, env, code):
        self.test.generate_function_definitions(env, code)
        for case in self.cases:
            case.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6059 6060 6061 6062
    def annotate(self, code):
        self.test.annotate(code)
        for case in self.cases:
            case.annotate(code)
6063 6064
        if self.else_clause is not None:
            self.else_clause.annotate(code)
6065

6066

6067
class LoopNode(object):
6068
    pass
6069

6070

6071
class WhileStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6072 6073 6074 6075 6076
    #  while statement
    #
    #  condition    ExprNode
    #  body         StatNode
    #  else_clause  StatNode
6077 6078

    child_attrs = ["condition", "body", "else_clause"]
6079

William Stein's avatar
William Stein committed
6080 6081 6082 6083
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6084

William Stein's avatar
William Stein committed
6085
    def analyse_expressions(self, env):
6086 6087
        if self.condition:
            self.condition = self.condition.analyse_temp_boolean_expression(env)
6088
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6089
        if self.else_clause:
6090 6091
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6092

William Stein's avatar
William Stein committed
6093
    def generate_execution_code(self, code):
6094
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6095 6096 6097
        old_loop_labels = code.new_loop_labels()
        code.putln(
            "while (1) {")
6098 6099 6100 6101
        if self.condition:
            self.condition.generate_evaluation_code(code)
            self.condition.generate_disposal_code(code)
            code.putln(
6102
                "if (!%s) break;" % self.condition.result())
6103
            self.condition.free_temps(code)
William Stein's avatar
William Stein committed
6104
        self.body.generate_execution_code(code)
6105
        code.put_label(code.continue_label)
William Stein's avatar
William Stein committed
6106 6107 6108 6109
        code.putln("}")
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
6110
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6111 6112 6113 6114 6115
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)

6116
    def generate_function_definitions(self, env, code):
6117 6118
        if self.condition:
            self.condition.generate_function_definitions(env, code)
6119 6120 6121 6122
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6123
    def annotate(self, code):
6124 6125
        if self.condition:
            self.condition.annotate(code)
6126 6127 6128 6129
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6130

6131 6132 6133 6134
class DictIterationNextNode(Node):
    # Helper node for calling PyDict_Next() inside of a WhileStatNode
    # and checking the dictionary size for changes.  Created in
    # Optimize.py.
6135 6136 6137
    child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
                   'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
                   'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
6138

6139 6140 6141 6142 6143 6144
    coerced_key_var = key_ref = None
    coerced_value_var = value_ref = None
    coerced_tuple_var = tuple_ref = None

    def __init__(self, dict_obj, expected_size, pos_index_var,
                 key_target, value_target, tuple_target, is_dict_flag):
6145 6146
        Node.__init__(
            self, dict_obj.pos,
6147 6148 6149 6150 6151 6152 6153 6154 6155
            dict_obj=dict_obj,
            expected_size=expected_size,
            pos_index_var=pos_index_var,
            key_target=key_target,
            value_target=value_target,
            tuple_target=tuple_target,
            is_dict_flag=is_dict_flag,
            is_temp=True,
            type=PyrexTypes.c_bint_type)
6156 6157

    def analyse_expressions(self, env):
6158
        from . import ExprNodes
6159 6160 6161 6162
        self.dict_obj = self.dict_obj.analyse_types(env)
        self.expected_size = self.expected_size.analyse_types(env)
        if self.pos_index_var:
            self.pos_index_var = self.pos_index_var.analyse_types(env)
6163
        if self.key_target:
6164
            self.key_target = self.key_target.analyse_target_types(env)
6165
            self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
6166 6167
            self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
        if self.value_target:
6168
            self.value_target = self.value_target.analyse_target_types(env)
6169
            self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
6170 6171
            self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
        if self.tuple_target:
6172
            self.tuple_target = self.tuple_target.analyse_target_types(env)
6173
            self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
6174
            self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
6175 6176
        self.is_dict_flag = self.is_dict_flag.analyse_types(env)
        return self
6177 6178 6179 6180 6181

    def generate_function_definitions(self, env, code):
        self.dict_obj.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
6182
        code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
6183 6184
        self.dict_obj.generate_evaluation_code(code)

6185 6186 6187 6188 6189 6190 6191 6192
        assignments = []
        temp_addresses = []
        for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
                                    (self.value_ref, self.coerced_value_var, self.value_target),
                                    (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
            if target is None:
                addr = 'NULL'
            else:
6193 6194 6195
                assignments.append((var, result, target))
                var.allocate(code)
                addr = '&%s' % var.result()
6196 6197 6198 6199 6200
            temp_addresses.append(addr)

        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
        code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
            result_temp,
6201
            self.dict_obj.py_result(),
6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213
            self.expected_size.result(),
            self.pos_index_var.result(),
            temp_addresses[0],
            temp_addresses[1],
            temp_addresses[2],
            self.is_dict_flag.result()
        ))
        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
        code.funcstate.release_temp(result_temp)

        # evaluate all coercions before the assignments
6214 6215
        for var, result, target in assignments:
            code.put_gotref(var.result())
6216
        for var, result, target in assignments:
6217
            result.generate_evaluation_code(code)
6218
        for var, result, target in assignments:
6219
            target.generate_assignment_code(result, code)
6220
            var.release(code)
6221

6222

Robert Bradshaw's avatar
Robert Bradshaw committed
6223
def ForStatNode(pos, **kw):
6224
    if 'iterator' in kw:
6225 6226 6227 6228
        if kw['iterator'].is_async:
            return AsyncForStatNode(pos, **kw)
        else:
            return ForInStatNode(pos, **kw)
Robert Bradshaw's avatar
Robert Bradshaw committed
6229 6230 6231
    else:
        return ForFromStatNode(pos, **kw)

6232 6233 6234

class _ForInStatNode(LoopNode, StatNode):
    #  Base class of 'for-in' statements.
William Stein's avatar
William Stein committed
6235 6236
    #
    #  target        ExprNode
6237
    #  iterator      IteratorNode | AIterAwaitExprNode(AsyncIteratorNode)
William Stein's avatar
William Stein committed
6238 6239
    #  body          StatNode
    #  else_clause   StatNode
6240 6241
    #  item          NextNode | AwaitExprNode(AsyncNextNode)
    #  is_async      boolean        true for 'async for' statements
6242

6243
    child_attrs = ["target", "item", "iterator", "body", "else_clause"]
6244
    item = None
6245 6246 6247 6248
    is_async = False

    def _create_item_node(self):
        raise NotImplementedError("must be implemented by subclasses")
6249

William Stein's avatar
William Stein committed
6250 6251 6252 6253 6254
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6255
        self._create_item_node()
6256

William Stein's avatar
William Stein committed
6257
    def analyse_expressions(self, env):
6258
        self.target = self.target.analyse_target_types(env)
6259
        self.iterator = self.iterator.analyse_expressions(env)
6260
        self._create_item_node()  # must rewrap self.item after analysis
6261
        self.item = self.item.analyse_expressions(env)
6262 6263 6264
        if (not self.is_async and
                (self.iterator.type.is_ptr or self.iterator.type.is_array) and
                self.target.type.assignable_from(self.iterator.type)):
6265 6266 6267 6268
            # C array slice optimization.
            pass
        else:
            self.item = self.item.coerce_to(self.target.type, env)
6269
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6270
        if self.else_clause:
6271 6272
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
William Stein's avatar
William Stein committed
6273 6274

    def generate_execution_code(self, code):
6275
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6276 6277
        old_loop_labels = code.new_loop_labels()
        self.iterator.generate_evaluation_code(code)
Mark Florisson's avatar
Mark Florisson committed
6278
        code.putln("for (;;) {")
William Stein's avatar
William Stein committed
6279 6280 6281
        self.item.generate_evaluation_code(code)
        self.target.generate_assignment_code(self.item, code)
        self.body.generate_execution_code(code)
6282
        code.mark_pos(self.pos)
6283
        code.put_label(code.continue_label)
Mark Florisson's avatar
Mark Florisson committed
6284
        code.putln("}")
William Stein's avatar
William Stein committed
6285 6286
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
6287

William Stein's avatar
William Stein committed
6288
        if self.else_clause:
6289 6290 6291 6292 6293 6294 6295
            # in nested loops, the 'else' block can contain a
            # 'continue' statement for the outer loop, but we may need
            # to generate cleanup code before taking that path, so we
            # intercept it here
            orig_continue_label = code.continue_label
            code.continue_label = code.new_label('outer_continue')

William Stein's avatar
William Stein committed
6296 6297 6298
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
6299 6300 6301

            if code.label_used(code.continue_label):
                code.put_goto(break_label)
6302
                code.mark_pos(self.pos)
6303 6304 6305 6306 6307
                code.put_label(code.continue_label)
                self.iterator.generate_disposal_code(code)
                code.put_goto(orig_continue_label)
            code.set_loop_labels(old_loop_labels)

6308
        code.mark_pos(self.pos)
6309 6310
        if code.label_used(break_label):
            code.put_label(break_label)
William Stein's avatar
William Stein committed
6311
        self.iterator.generate_disposal_code(code)
6312
        self.iterator.free_temps(code)
William Stein's avatar
William Stein committed
6313

6314 6315 6316 6317 6318 6319 6320
    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.iterator.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6321 6322 6323 6324 6325 6326 6327 6328
    def annotate(self, code):
        self.target.annotate(code)
        self.iterator.annotate(code)
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
        self.item.annotate(code)

William Stein's avatar
William Stein committed
6329

6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342
class ForInStatNode(_ForInStatNode):
    #  'for' statement

    is_async = False

    def _create_item_node(self):
        from .ExprNodes import NextNode
        self.item = NextNode(self.iterator)


class AsyncForStatNode(_ForInStatNode):
    #  'async for' statement
    #
6343
    #  iterator      AIterAwaitExprNode(AsyncIteratorNode)
6344 6345 6346 6347
    #  item          AwaitIterNextExprNode(AsyncIteratorNode)

    is_async = True

6348
    def __init__(self, pos, **kw):
6349 6350 6351
        assert 'item' not in kw
        from . import ExprNodes
        # AwaitExprNodes must appear before running MarkClosureVisitor
6352
        kw['item'] = ExprNodes.AwaitIterNextExprNode(kw['iterator'].pos, arg=None)
6353 6354 6355 6356 6357 6358 6359
        _ForInStatNode.__init__(self, pos, **kw)

    def _create_item_node(self):
        from . import ExprNodes
        self.item.arg = ExprNodes.AsyncNextNode(self.iterator)


6360
class ForFromStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6361 6362 6363 6364 6365 6366 6367
    #  for name from expr rel name rel expr
    #
    #  target        NameNode
    #  bound1        ExprNode
    #  relation1     string
    #  relation2     string
    #  bound2        ExprNode
6368
    #  step          ExprNode or None
William Stein's avatar
William Stein committed
6369 6370 6371 6372 6373
    #  body          StatNode
    #  else_clause   StatNode or None
    #
    #  Used internally:
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
6374
    #  from_range         bool
6375
    #  is_py_target       bool
6376
    #  loopvar_node       ExprNode (usually a NameNode or temp node)
William Stein's avatar
William Stein committed
6377
    #  py_loopvar_node    PyTempNode or None
6378
    child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
6379 6380

    is_py_target = False
6381
    loopvar_node = None
6382
    py_loopvar_node = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6383
    from_range = False
6384

6385 6386 6387 6388 6389 6390 6391
    gil_message = "For-loop using object bounds or target"

    def nogil_check(self, env):
        for x in (self.target, self.bound1, self.bound2):
            if x.type.is_pyobject:
                self.gil_error()

Robert Bradshaw's avatar
Robert Bradshaw committed
6392 6393 6394 6395 6396
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6397

William Stein's avatar
William Stein committed
6398
    def analyse_expressions(self, env):
6399
        from . import ExprNodes
6400 6401 6402
        self.target = self.target.analyse_target_types(env)
        self.bound1 = self.bound1.analyse_types(env)
        self.bound2 = self.bound2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6403 6404
        if self.step is not None:
            if isinstance(self.step, ExprNodes.UnaryMinusNode):
6405 6406
                warning(self.step.pos, "Probable infinite loop in for-from-by statement. "
                        "Consider switching the directions of the relations.", 2)
6407
            self.step = self.step.analyse_types(env)
6408

6409
        if self.target.type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6410
            loop_type = self.target.type
6411
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6412 6413 6414 6415 6416 6417 6418 6419 6420
            loop_type = PyrexTypes.c_int_type
            if not self.bound1.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
            if not self.bound2.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
            if self.step is not None and not self.step.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
        self.bound1 = self.bound1.coerce_to(loop_type, env)
        self.bound2 = self.bound2.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6421 6422
        if not self.bound2.is_literal:
            self.bound2 = self.bound2.coerce_to_temp(env)
6423
        if self.step is not None:
6424
            self.step = self.step.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6425 6426
            if not self.step.is_literal:
                self.step = self.step.coerce_to_temp(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6427

William Stein's avatar
William Stein committed
6428
        target_type = self.target.type
6429
        if not (target_type.is_pyobject or target_type.is_numeric):
6430
            error(self.target.pos, "for-from loop variable must be c numeric type or Python object")
6431
        if target_type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6432
            self.is_py_target = False
6433 6434
            if isinstance(self.target, ExprNodes.BufferIndexNode):
                raise error(self.pos, "Buffer or memoryview slicing/indexing not allowed as for-loop target.")
6435
            self.loopvar_node = self.target
William Stein's avatar
William Stein committed
6436 6437
            self.py_loopvar_node = None
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6438 6439
            self.is_py_target = True
            c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
6440
            self.loopvar_node = c_loopvar_node
William Stein's avatar
William Stein committed
6441 6442
            self.py_loopvar_node = \
                ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
6443
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6444
        if self.else_clause:
6445 6446
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6447

William Stein's avatar
William Stein committed
6448
    def generate_execution_code(self, code):
6449
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6450
        old_loop_labels = code.new_loop_labels()
Robert Bradshaw's avatar
Robert Bradshaw committed
6451
        from_range = self.from_range
William Stein's avatar
William Stein committed
6452 6453 6454
        self.bound1.generate_evaluation_code(code)
        self.bound2.generate_evaluation_code(code)
        offset, incop = self.relation_table[self.relation1]
6455 6456
        if self.step is not None:
            self.step.generate_evaluation_code(code)
Magnus Lie Hetland's avatar
Magnus Lie Hetland committed
6457 6458
            step = self.step.result()
            incop = "%s=%s" % (incop[0], step)
6459
        from . import ExprNodes
6460 6461 6462 6463
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.allocate(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.allocate(code)
6464 6465 6466

        loopvar_type = PyrexTypes.c_long_type if self.target.type.is_enum else self.target.type

6467
        if from_range:
6468
            loopvar_name = code.funcstate.allocate_temp(loopvar_type, False)
6469
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6470
            loopvar_name = self.loopvar_node.result()
6471
        if loopvar_type.is_int and not loopvar_type.signed and self.relation2[0] == '>':
6472 6473 6474 6475
            # Handle the case where the endpoint of an unsigned int iteration
            # is within step of 0.
            if not self.step:
                step = 1
6476 6477 6478 6479 6480
            code.putln("for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
                loopvar_name,
                self.bound1.result(), offset, step,
                loopvar_name, self.relation2, self.bound2.result(), step,
                loopvar_name, incop))
6481
        else:
6482 6483 6484 6485 6486
            code.putln("for (%s = %s%s; %s %s %s; %s%s) {" % (
                loopvar_name,
                self.bound1.result(), offset,
                loopvar_name, self.relation2, self.bound2.result(),
                loopvar_name, incop))
6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497

        coerced_loopvar_node = self.py_loopvar_node
        if coerced_loopvar_node is None and from_range:
            loopvar_cvalue = loopvar_name
            if self.target.type.is_enum:
                loopvar_cvalue = '(%s)%s' % (self.target.type.declaration_code(''), loopvar_cvalue)
            coerced_loopvar_node = ExprNodes.RawCNameExprNode(self.target.pos, loopvar_type, loopvar_cvalue)
        if coerced_loopvar_node is not None:
            coerced_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(coerced_loopvar_node, code)

William Stein's avatar
William Stein committed
6498 6499
        self.body.generate_execution_code(code)
        code.put_label(code.continue_label)
6500

Robert Bradshaw's avatar
Robert Bradshaw committed
6501
        if self.py_loopvar_node:
6502 6503 6504
            # This mess is to make for..from loops with python targets behave
            # exactly like those with C targets with regards to re-assignment
            # of the loop variable.
6505
            if self.target.entry.is_pyglobal:
6506
                # We know target is a NameNode, this is the only ugly case.
6507
                target_node = ExprNodes.PyTempNode(self.target.pos, None)
6508 6509
                target_node.allocate(code)
                interned_cname = code.intern_identifier(self.target.entry.name)
6510 6511 6512 6513 6514 6515 6516 6517 6518 6519
                if self.target.entry.scope.is_module_scope:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetModuleGlobalName(%s)'
                else:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetNameInClass(%s, %%s)' % (
                        self.target.entry.scope.namespace_cname)
                code.putln("%s = %s; %s" % (
6520
                    target_node.result(),
6521
                    lookup_func % interned_cname,
6522
                    code.error_goto_if_null(target_node.result(), self.target.pos)))
6523
                code.put_gotref(target_node.result())
6524 6525
            else:
                target_node = self.target
6526 6527
            from_py_node = ExprNodes.CoerceFromPyTypeNode(
                self.loopvar_node.type, target_node, self.target.entry.scope)
6528 6529
            from_py_node.temp_code = loopvar_name
            from_py_node.generate_result_code(code)
6530
            if self.target.entry.is_pyglobal:
6531 6532
                code.put_decref(target_node.result(), target_node.type)
                target_node.release(code)
6533

Robert Bradshaw's avatar
Robert Bradshaw committed
6534
        code.putln("}")
6535

6536
        if self.py_loopvar_node:
6537 6538
            # This is potentially wasteful, but we don't want the semantics to
            # depend on whether or not the loop is a python type.
6539 6540
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
6541 6542
        if from_range:
            code.funcstate.release_temp(loopvar_name)
6543

William Stein's avatar
William Stein committed
6544 6545 6546 6547 6548 6549 6550 6551
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)
        self.bound1.generate_disposal_code(code)
6552
        self.bound1.free_temps(code)
William Stein's avatar
William Stein committed
6553
        self.bound2.generate_disposal_code(code)
6554
        self.bound2.free_temps(code)
6555 6556 6557 6558
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.release(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.release(code)
6559 6560
        if self.step is not None:
            self.step.generate_disposal_code(code)
6561
            self.step.free_temps(code)
6562

William Stein's avatar
William Stein committed
6563 6564 6565 6566 6567
    relation_table = {
        # {relop : (initial offset, increment op)}
        '<=': ("",   "++"),
        '<' : ("+1", "++"),
        '>=': ("",   "--"),
6568
        '>' : ("-1", "--"),
William Stein's avatar
William Stein committed
6569
    }
6570 6571 6572 6573 6574 6575 6576 6577 6578 6579

    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.bound1.generate_function_definitions(env, code)
        self.bound2.generate_function_definitions(env, code)
        if self.step is not None:
            self.step.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
6580

6581 6582 6583 6584 6585
    def annotate(self, code):
        self.target.annotate(code)
        self.bound1.annotate(code)
        self.bound2.annotate(code)
        if self.step:
6586
            self.step.annotate(code)
6587 6588 6589
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
6590 6591


6592 6593 6594
class WithStatNode(StatNode):
    """
    Represents a Python with statement.
6595

6596
    Implemented by the WithTransform as follows:
6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613

        MGR = EXPR
        EXIT = MGR.__exit__
        VALUE = MGR.__enter__()
        EXC = True
        try:
            try:
                TARGET = VALUE  # optional
                BODY
            except:
                EXC = False
                if not EXIT(*EXCINFO):
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
            MGR = EXIT = VALUE = None
6614 6615
    """
    #  manager          The with statement manager object
6616
    #  target           ExprNode  the target lhs of the __enter__() call
6617
    #  body             StatNode
6618
    #  enter_call       ExprNode  the call to the __enter__() method
6619
    #  exit_var         String    the cname of the __exit__() method reference
6620

6621
    child_attrs = ["manager", "enter_call", "target", "body"]
6622

6623
    enter_call = None
6624
    target_temp = None
6625 6626 6627

    def analyse_declarations(self, env):
        self.manager.analyse_declarations(env)
6628
        self.enter_call.analyse_declarations(env)
6629 6630 6631
        self.body.analyse_declarations(env)

    def analyse_expressions(self, env):
6632 6633
        self.manager = self.manager.analyse_types(env)
        self.enter_call = self.enter_call.analyse_types(env)
6634 6635 6636 6637
        if self.target:
            # set up target_temp before descending into body (which uses it)
            from .ExprNodes import TempNode
            self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
6638 6639
        self.body = self.body.analyse_expressions(env)
        return self
6640

6641 6642
    def generate_function_definitions(self, env, code):
        self.manager.generate_function_definitions(env, code)
6643
        self.enter_call.generate_function_definitions(env, code)
6644 6645
        self.body.generate_function_definitions(env, code)

6646
    def generate_execution_code(self, code):
6647
        code.mark_pos(self.pos)
6648 6649 6650
        code.putln("/*with:*/ {")
        self.manager.generate_evaluation_code(code)
        self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
6651
        code.globalstate.use_utility_code(
6652 6653
            UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
        code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
6654 6655
            self.exit_var,
            self.manager.py_result(),
6656
            code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')),
6657 6658 6659 6660 6661 6662 6663 6664
            code.error_goto_if_null(self.exit_var, self.pos),
            ))
        code.put_gotref(self.exit_var)

        # need to free exit_var in the face of exceptions during setup
        old_error_label = code.new_error_label()
        intermediate_error_label = code.error_label

6665
        self.enter_call.generate_evaluation_code(code)
6666 6667 6668 6669 6670 6671 6672
        if self.target:
            # The temp result will be cleaned up by the WithTargetAssignmentStatNode
            # after assigning its result to the target of the 'with' statement.
            self.target_temp.allocate(code)
            self.enter_call.make_owned_reference(code)
            code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
            self.enter_call.generate_post_assignment_code(code)
6673
        else:
6674 6675 6676
            self.enter_call.generate_disposal_code(code)
        self.enter_call.free_temps(code)

6677 6678 6679 6680 6681 6682
        self.manager.generate_disposal_code(code)
        self.manager.free_temps(code)

        code.error_label = old_error_label
        self.body.generate_execution_code(code)

6683 6684 6685 6686 6687 6688 6689
        if code.label_used(intermediate_error_label):
            step_over_label = code.new_label()
            code.put_goto(step_over_label)
            code.put_label(intermediate_error_label)
            code.put_decref_clear(self.exit_var, py_object_type)
            code.put_goto(old_error_label)
            code.put_label(step_over_label)
6690 6691 6692 6693

        code.funcstate.release_temp(self.exit_var)
        code.putln('}')

6694

6695 6696 6697 6698
class WithTargetAssignmentStatNode(AssignmentNode):
    # The target assignment of the 'with' statement value (return
    # value of the __enter__() call).
    #
6699
    # This is a special cased assignment that properly cleans up the RHS.
6700
    #
6701 6702 6703
    # lhs       ExprNode      the assignment target
    # rhs       ExprNode      a (coerced) TempNode for the rhs (from WithStatNode)
    # with_node WithStatNode  the surrounding with-statement
6704

6705 6706 6707
    child_attrs = ["rhs", "lhs"]
    with_node = None
    rhs = None
6708 6709 6710 6711

    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)

6712
    def analyse_expressions(self, env):
6713
        self.lhs = self.lhs.analyse_target_types(env)
6714
        self.lhs.gil_assignment_check(env)
6715
        self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
6716
        return self
6717 6718 6719 6720

    def generate_execution_code(self, code):
        self.rhs.generate_evaluation_code(code)
        self.lhs.generate_assignment_code(self.rhs, code)
6721
        self.with_node.target_temp.release(code)
6722 6723 6724 6725 6726

    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

6727

William Stein's avatar
William Stein committed
6728 6729 6730 6731 6732 6733
class TryExceptStatNode(StatNode):
    #  try .. except statement
    #
    #  body             StatNode
    #  except_clauses   [ExceptClauseNode]
    #  else_clause      StatNode or None
6734

6735
    child_attrs = ["body", "except_clauses", "else_clause"]
6736
    in_generator = False
6737

William Stein's avatar
William Stein committed
6738 6739 6740 6741 6742 6743
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        for except_clause in self.except_clauses:
            except_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6744

William Stein's avatar
William Stein committed
6745
    def analyse_expressions(self, env):
6746
        self.body = self.body.analyse_expressions(env)
6747
        default_clause_seen = 0
6748 6749
        for i, except_clause in enumerate(self.except_clauses):
            except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
6750 6751 6752 6753 6754
            if default_clause_seen:
                error(except_clause.pos, "default 'except:' must be last")
            if not except_clause.pattern:
                default_clause_seen = 1
        self.has_default_clause = default_clause_seen
William Stein's avatar
William Stein committed
6755
        if self.else_clause:
6756 6757
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6758

6759
    nogil_check = Node.gil_error
6760 6761
    gil_message = "Try-except statement"

William Stein's avatar
William Stein committed
6762
    def generate_execution_code(self, code):
6763
        old_return_label = code.return_label
6764
        old_break_label = code.break_label
6765
        old_continue_label = code.continue_label
William Stein's avatar
William Stein committed
6766 6767
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
6768 6769 6770
        except_end_label = code.new_label('exception_handled')
        except_error_label = code.new_label('except_error')
        except_return_label = code.new_label('except_return')
6771
        try_return_label = code.new_label('try_return')
6772 6773
        try_break_label = code.new_label('try_break') if old_break_label else None
        try_continue_label = code.new_label('try_continue') if old_continue_label else None
6774
        try_end_label = code.new_label('try_end')
6775

6776
        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
6777
                         for _ in range(3)]
6778
        code.mark_pos(self.pos)
6779
        code.putln("{")
6780
        save_exc = code.insertion_point()
William Stein's avatar
William Stein committed
6781 6782
        code.putln(
            "/*try:*/ {")
6783
        code.return_label = try_return_label
6784
        code.break_label = try_break_label
6785
        code.continue_label = try_continue_label
William Stein's avatar
William Stein committed
6786
        self.body.generate_execution_code(code)
6787
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
6788 6789
        code.putln(
            "}")
6790
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
6791 6792 6793 6794
        can_raise = code.label_used(our_error_label)

        if can_raise:
            # inject code before the try block to save away the exception state
6795
            code.globalstate.use_utility_code(reset_exception_utility_code)
6796 6797 6798
            if not self.in_generator:
                save_exc.putln("__Pyx_PyThreadState_declare")
                save_exc.putln("__Pyx_PyThreadState_assign")
6799 6800
            save_exc.putln("__Pyx_ExceptionSave(%s);" % (
                ', '.join(['&%s' % var for var in exc_save_vars])))
6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811
            for var in exc_save_vars:
                save_exc.put_xgotref(var)

            def restore_saved_exception():
                for name in exc_save_vars:
                    code.put_xgiveref(name)
                code.putln("__Pyx_ExceptionReset(%s);" %
                           ', '.join(exc_save_vars))
        else:
            # try block cannot raise exceptions, but we had to allocate the temps above,
            # so just keep the C compiler from complaining about them being unused
6812
            save_exc.putln("if (%s); else {/*mark used*/}" % '||'.join(exc_save_vars))
6813 6814 6815 6816

            def restore_saved_exception():
                pass

6817 6818
        code.error_label = except_error_label
        code.return_label = except_return_label
6819
        normal_case_terminates = self.body.is_terminator
William Stein's avatar
William Stein committed
6820
        if self.else_clause:
6821
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6822 6823 6824 6825 6826
            code.putln(
                "/*else:*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln(
                "}")
6827 6828
            if not normal_case_terminates:
                normal_case_terminates = self.else_clause.is_terminator
6829

6830
        if can_raise:
6831 6832 6833 6834
            if not normal_case_terminates:
                for var in exc_save_vars:
                    code.put_xdecref_clear(var, py_object_type)
                code.put_goto(try_end_label)
6835 6836 6837
            code.put_label(our_error_label)
            for temp_name, temp_type in temps_to_clean_up:
                code.put_xdecref_clear(temp_name, temp_type)
6838 6839 6840 6841

            outer_except = code.funcstate.current_except
            # Currently points to self, but the ExceptClauseNode would also be ok. Change if needed.
            code.funcstate.current_except = self
6842 6843
            for except_clause in self.except_clauses:
                except_clause.generate_handling_code(code, except_end_label)
6844 6845
            code.funcstate.current_except = outer_except

6846 6847 6848 6849 6850 6851 6852 6853
            if not self.has_default_clause:
                code.put_goto(except_error_label)

        for exit_label, old_label in [(except_error_label, old_error_label),
                                      (try_break_label, old_break_label),
                                      (try_continue_label, old_continue_label),
                                      (try_return_label, old_return_label),
                                      (except_return_label, old_return_label)]:
6854
            if code.label_used(exit_label):
6855
                if not normal_case_terminates and not code.label_used(try_end_label):
6856
                    code.put_goto(try_end_label)
6857
                code.put_label(exit_label)
6858
                code.mark_pos(self.pos, trace=False)
6859 6860
                if can_raise:
                    restore_saved_exception()
6861
                code.put_goto(old_label)
6862 6863

        if code.label_used(except_end_label):
6864
            if not normal_case_terminates and not code.label_used(try_end_label):
6865
                code.put_goto(try_end_label)
6866
            code.put_label(except_end_label)
6867 6868
            if can_raise:
                restore_saved_exception()
6869 6870
        if code.label_used(try_end_label):
            code.put_label(try_end_label)
6871 6872
        code.putln("}")

6873 6874 6875
        for cname in exc_save_vars:
            code.funcstate.release_temp(cname)

6876
        code.return_label = old_return_label
6877
        code.break_label = old_break_label
6878
        code.continue_label = old_continue_label
6879
        code.error_label = old_error_label
William Stein's avatar
William Stein committed
6880

6881 6882 6883 6884 6885 6886 6887
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        for except_clause in self.except_clauses:
            except_clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6888 6889 6890 6891 6892 6893 6894
    def annotate(self, code):
        self.body.annotate(code)
        for except_node in self.except_clauses:
            except_node.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6895 6896 6897 6898

class ExceptClauseNode(Node):
    #  Part of try ... except statement.
    #
6899
    #  pattern        [ExprNode]
William Stein's avatar
William Stein committed
6900 6901
    #  target         ExprNode or None
    #  body           StatNode
6902
    #  excinfo_target TupleNode(3*ResultRefNode) or None   optional target for exception info (not owned here!)
William Stein's avatar
William Stein committed
6903 6904 6905
    #  match_flag     string             result of exception match
    #  exc_value      ExcValueNode       used internally
    #  function_name  string             qualified name of enclosing function
6906
    #  exc_vars       (string * 3)       local exception variables
6907
    #  is_except_as   bool               Py3-style "except ... as xyz"
6908 6909 6910 6911

    # excinfo_target is never set by the parser, but can be set by a transform
    # in order to extract more extensive information about the exception as a
    # sys.exc_info()-style tuple into a target variable
6912

6913
    child_attrs = ["pattern", "target", "body", "exc_value"]
6914

6915
    exc_value = None
6916
    excinfo_target = None
6917
    is_except_as = False
6918

William Stein's avatar
William Stein committed
6919 6920 6921 6922
    def analyse_declarations(self, env):
        if self.target:
            self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
6923

William Stein's avatar
William Stein committed
6924 6925 6926
    def analyse_expressions(self, env):
        self.function_name = env.qualified_name
        if self.pattern:
6927 6928
            # normalise/unpack self.pattern into a list
            for i, pattern in enumerate(self.pattern):
6929
                pattern = pattern.analyse_expressions(env)
6930
                self.pattern[i] = pattern.coerce_to_pyobject(env)
6931

William Stein's avatar
William Stein committed
6932
        if self.target:
6933
            from . import ExprNodes
6934
            self.exc_value = ExprNodes.ExcValueNode(self.pos)
6935
            self.target = self.target.analyse_target_expression(env, self.exc_value)
6936

6937 6938
        self.body = self.body.analyse_expressions(env)
        return self
6939

William Stein's avatar
William Stein committed
6940 6941 6942
    def generate_handling_code(self, code, end_label):
        code.mark_pos(self.pos)
        if self.pattern:
6943
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c"))
6944 6945 6946
            exc_tests = []
            for pattern in self.pattern:
                pattern.generate_evaluation_code(code)
6947
                exc_tests.append("__Pyx_PyErr_ExceptionMatches(%s)" % pattern.py_result())
6948

6949
            match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
William Stein's avatar
William Stein committed
6950
            code.putln(
6951 6952 6953 6954
                "%s = %s;" % (match_flag, ' || '.join(exc_tests)))
            for pattern in self.pattern:
                pattern.generate_disposal_code(code)
                pattern.free_temps(code)
William Stein's avatar
William Stein committed
6955 6956
            code.putln(
                "if (%s) {" %
6957 6958
                    match_flag)
            code.funcstate.release_temp(match_flag)
William Stein's avatar
William Stein committed
6959
        else:
6960
            code.putln("/*except:*/ {")
6961

6962 6963
        if (not getattr(self.body, 'stats', True)
                and self.excinfo_target is None
6964
                and self.target is None):
Stefan Behnel's avatar
Stefan Behnel committed
6965 6966
            # most simple case: no exception variable, empty body (pass)
            # => reset the exception state, done
6967 6968
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
            code.putln("__Pyx_ErrRestore(0,0,0);")
6969 6970 6971
            code.put_goto(end_label)
            code.putln("}")
            return
6972

6973 6974
        exc_vars = [code.funcstate.allocate_temp(py_object_type,
                                                 manage_ref=True)
6975
                    for _ in range(3)]
6976
        code.put_add_traceback(self.function_name)
William Stein's avatar
William Stein committed
6977
        # We always have to fetch the exception value even if
6978
        # there is no target, because this also normalises the
William Stein's avatar
William Stein committed
6979
        # exception and stores it in the thread state.
6980 6981
        code.globalstate.use_utility_code(get_exception_utility_code)
        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
6982 6983
        code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
            exc_args, code.error_goto(self.pos)))
Stefan Behnel's avatar
Stefan Behnel committed
6984 6985
        for var in exc_vars:
            code.put_gotref(var)
William Stein's avatar
William Stein committed
6986
        if self.target:
6987
            self.exc_value.set_var(exc_vars[1])
6988
            self.exc_value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
6989
            self.target.generate_assignment_code(self.exc_value, code)
6990
        if self.excinfo_target is not None:
6991
            for tempvar, node in zip(exc_vars, self.excinfo_target.args):
6992
                node.set_var(tempvar)
6993

6994 6995 6996 6997
        old_break_label, old_continue_label = code.break_label, code.continue_label
        code.break_label = code.new_label('except_break')
        code.continue_label = code.new_label('except_continue')

6998
        old_exc_vars = code.funcstate.exc_vars
6999
        code.funcstate.exc_vars = exc_vars
William Stein's avatar
William Stein committed
7000
        self.body.generate_execution_code(code)
7001
        code.funcstate.exc_vars = old_exc_vars
Stefan Behnel's avatar
Stefan Behnel committed
7002

7003 7004 7005 7006
        if not self.body.is_terminator:
            for var in exc_vars:
                code.put_decref_clear(var, py_object_type)
            code.put_goto(end_label)
7007

Stefan Behnel's avatar
Stefan Behnel committed
7008 7009 7010 7011 7012 7013 7014
        for new_label, old_label in [(code.break_label, old_break_label),
                                     (code.continue_label, old_continue_label)]:
            if code.label_used(new_label):
                code.put_label(new_label)
                for var in exc_vars:
                    code.put_decref_clear(var, py_object_type)
                code.put_goto(old_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
7015
        code.break_label = old_break_label
7016
        code.continue_label = old_continue_label
7017

7018 7019
        for temp in exc_vars:
            code.funcstate.release_temp(temp)
7020

William Stein's avatar
William Stein committed
7021 7022 7023
        code.putln(
            "}")

7024 7025 7026 7027
    def generate_function_definitions(self, env, code):
        if self.target is not None:
            self.target.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
7028

7029
    def annotate(self, code):
7030
        if self.pattern:
7031 7032
            for pattern in self.pattern:
                pattern.annotate(code)
7033 7034 7035 7036
        if self.target:
            self.target.annotate(code)
        self.body.annotate(code)

William Stein's avatar
William Stein committed
7037 7038 7039 7040 7041 7042

class TryFinallyStatNode(StatNode):
    #  try ... finally statement
    #
    #  body             StatNode
    #  finally_clause   StatNode
7043
    #  finally_except_clause  deep-copy of finally_clause for exception case
7044
    #  in_generator     inside of generator => must store away current exception also in return case
7045
    #
Stefan Behnel's avatar
Stefan Behnel committed
7046 7047
    #  Each of the continue, break, return and error gotos runs
    #  into its own deep-copy of the finally block code.
William Stein's avatar
William Stein committed
7048 7049 7050
    #  In addition, if we're doing an error, we save the
    #  exception on entry to the finally block and restore
    #  it on exit.
7051

7052
    child_attrs = ["body", "finally_clause", "finally_except_clause"]
7053

7054
    preserve_exception = 1
7055

7056 7057
    # handle exception case, in addition to return/break/continue
    handle_error_case = True
7058
    func_return_type = None
7059
    finally_except_clause = None
7060

7061
    is_try_finally_in_nogil = False
7062
    in_generator = False
7063

7064
    @staticmethod
7065 7066 7067
    def create_analysed(pos, env, body, finally_clause):
        node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
        return node
7068

William Stein's avatar
William Stein committed
7069 7070
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
7071 7072
        self.finally_except_clause = copy.deepcopy(self.finally_clause)
        self.finally_except_clause.analyse_declarations(env)
William Stein's avatar
William Stein committed
7073
        self.finally_clause.analyse_declarations(env)
7074

William Stein's avatar
William Stein committed
7075
    def analyse_expressions(self, env):
7076 7077
        self.body = self.body.analyse_expressions(env)
        self.finally_clause = self.finally_clause.analyse_expressions(env)
7078
        self.finally_except_clause = self.finally_except_clause.analyse_expressions(env)
Stefan Behnel's avatar
Stefan Behnel committed
7079 7080
        if env.return_type and not env.return_type.is_void:
            self.func_return_type = env.return_type
7081
        return self
7082

7083
    nogil_check = Node.gil_error
7084 7085
    gil_message = "Try-finally statement"

William Stein's avatar
William Stein committed
7086
    def generate_execution_code(self, code):
7087
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
7088 7089 7090 7091
        old_error_label = code.error_label
        old_labels = code.all_new_labels()
        new_labels = code.get_all_labels()
        new_error_label = code.error_label
7092 7093
        if not self.handle_error_case:
            code.error_label = old_error_label
William Stein's avatar
William Stein committed
7094
        catch_label = code.new_label()
7095 7096

        code.putln("/*try:*/ {")
Stefan Behnel's avatar
Stefan Behnel committed
7097 7098
        was_in_try_finally = code.funcstate.in_try_finally
        code.funcstate.in_try_finally = 1
7099

William Stein's avatar
William Stein committed
7100
        self.body.generate_execution_code(code)
7101

Stefan Behnel's avatar
Stefan Behnel committed
7102
        code.funcstate.in_try_finally = was_in_try_finally
7103
        code.putln("}")
7104
        code.set_all_labels(old_labels)
7105

7106
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
7107
        code.mark_pos(self.finally_clause.pos)
7108 7109
        code.putln("/*finally:*/ {")

7110 7111 7112 7113 7114 7115 7116 7117 7118
        def fresh_finally_clause(_next=[self.finally_clause]):
            # generate the original subtree once and always keep a fresh copy
            node = _next[0]
            node_copy = copy.deepcopy(node)
            if node is self.finally_clause:
                _next[0] = node_copy
            else:
                node = node_copy
            return node
7119 7120

        preserve_error = self.preserve_exception and code.label_used(new_error_label)
7121
        needs_success_cleanup = not self.finally_clause.is_terminator
7122 7123 7124

        if not self.body.is_terminator:
            code.putln('/*normal exit:*/{')
7125 7126
            fresh_finally_clause().generate_execution_code(code)
            if not self.finally_clause.is_terminator:
7127
                code.put_goto(catch_label)
7128
            code.putln('}')
7129

7130 7131
        if preserve_error:
            code.putln('/*exception exit:*/{')
7132 7133
            if not self.in_generator:
                code.putln("__Pyx_PyThreadState_declare")
7134 7135
            if self.is_try_finally_in_nogil:
                code.declare_gilstate()
7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146
            if needs_success_cleanup:
                exc_lineno_cnames = tuple([
                    code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
                    for _ in range(2)])
                exc_filename_cname = code.funcstate.allocate_temp(
                    PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
                    manage_ref=False)
            else:
                exc_lineno_cnames = exc_filename_cname = None
            exc_vars = tuple([
                code.funcstate.allocate_temp(py_object_type, manage_ref=False)
7147
                for _ in range(6)])
7148
            code.put_label(new_error_label)
7149
            self.put_error_catcher(
7150
                code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
7151
            finally_old_labels = code.all_new_labels()
7152

7153
            code.putln('{')
7154 7155
            old_exc_vars = code.funcstate.exc_vars
            code.funcstate.exc_vars = exc_vars[:3]
7156
            self.finally_except_clause.generate_execution_code(code)
7157
            code.funcstate.exc_vars = old_exc_vars
7158 7159
            code.putln('}')

7160
            if needs_success_cleanup:
7161 7162 7163 7164 7165 7166
                self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
                if exc_lineno_cnames:
                    for cname in exc_lineno_cnames:
                        code.funcstate.release_temp(cname)
                if exc_filename_cname:
                    code.funcstate.release_temp(exc_filename_cname)
7167
                code.put_goto(old_error_label)
7168

7169 7170 7171 7172
            for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
                if not code.label_used(new_label):
                    continue
                code.put_label(new_label)
7173
                self.put_error_cleaner(code, exc_vars)
7174
                code.put_goto(old_label)
7175 7176 7177

            for cname in exc_vars:
                code.funcstate.release_temp(cname)
7178
            code.putln('}')
7179

7180
        code.set_all_labels(old_labels)
7181
        return_label = code.return_label
7182 7183
        exc_vars = ()

7184 7185 7186 7187 7188
        for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
            if not code.label_used(new_label):
                continue
            if new_label == new_error_label and preserve_error:
                continue  # handled above
7189

7190
            code.putln('%s: {' % new_label)
7191
            ret_temp = None
7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209
            if old_label == return_label:
                # return actually raises an (uncatchable) exception in generators that we must preserve
                if self.in_generator:
                    exc_vars = tuple([
                        code.funcstate.allocate_temp(py_object_type, manage_ref=False)
                        for _ in range(6)])
                    self.put_error_catcher(code, [], exc_vars)
                if not self.finally_clause.is_terminator:
                    # store away return value for later reuse
                    if (self.func_return_type and
                            not self.is_try_finally_in_nogil and
                            not isinstance(self.finally_clause, GILExitNode)):
                        ret_temp = code.funcstate.allocate_temp(
                            self.func_return_type, manage_ref=False)
                        code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
                        if self.func_return_type.is_pyobject:
                            code.putln("%s = 0;" % Naming.retval_cname)

7210
            fresh_finally_clause().generate_execution_code(code)
7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221

            if old_label == return_label:
                if ret_temp:
                    code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
                    if self.func_return_type.is_pyobject:
                        code.putln("%s = 0;" % ret_temp)
                    code.funcstate.release_temp(ret_temp)
                    ret_temp = None
                if self.in_generator:
                    self.put_error_uncatcher(code, exc_vars)

7222
            if not self.finally_clause.is_terminator:
7223 7224
                code.put_goto(old_label)
            code.putln('}')
7225 7226

        # End finally
7227
        code.put_label(catch_label)
William Stein's avatar
William Stein committed
7228 7229 7230
        code.putln(
            "}")

7231 7232 7233 7234
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        self.finally_clause.generate_function_definitions(env, code)

7235
    def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
7236
                          exc_lineno_cnames=None, exc_filename_cname=None):
7237
        code.globalstate.use_utility_code(restore_exception_utility_code)
7238 7239
        code.globalstate.use_utility_code(get_exception_utility_code)
        code.globalstate.use_utility_code(swap_exception_utility_code)
7240

7241
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
7242 7243
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7244
        code.putln("__Pyx_PyThreadState_assign")
7245

7246 7247
        for temp_name, type in temps_to_clean_up:
            code.put_xdecref_clear(temp_name, type)
7248

7249 7250 7251 7252 7253 7254 7255 7256 7257
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3)"
                   " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
        code.putln("if ((PY_MAJOR_VERSION < 3) ||"
                   # if __Pyx_GetException() fails in Py3,
                   # store the newly raised exception instead
                   " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
                   "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
7258 7259 7260 7261 7262 7263 7264
        for var in exc_vars:
            code.put_xgotref(var)
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                exc_lineno_cnames[0], Naming.lineno_cname,
                exc_lineno_cnames[1], Naming.clineno_cname,
                exc_filename_cname, Naming.filename_cname))
7265 7266 7267 7268

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7269
    def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames=None, exc_filename_cname=None):
7270
        code.globalstate.use_utility_code(restore_exception_utility_code)
7271
        code.globalstate.use_utility_code(reset_exception_utility_code)
7272 7273 7274 7275

        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)

7276 7277 7278
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7279 7280 7281
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7282
        code.putln("}")
7283
        for var in exc_vars[:3]:
7284
            code.put_xgiveref(var)
7285
        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
7286 7287 7288 7289

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7290
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
7291 7292 7293 7294 7295
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                Naming.lineno_cname, exc_lineno_cnames[0],
                Naming.clineno_cname, exc_lineno_cnames[1],
                Naming.filename_cname, exc_filename_cname))
7296

7297
    def put_error_cleaner(self, code, exc_vars):
7298
        code.globalstate.use_utility_code(reset_exception_utility_code)
7299 7300
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7301

7302 7303 7304
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7305 7306 7307
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7308
        code.putln("}")
7309
        for var in exc_vars[:3]:
7310
            code.put_xdecref_clear(var, py_object_type)
7311 7312
        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()
7313
        code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
William Stein's avatar
William Stein committed
7314

7315 7316 7317 7318
    def annotate(self, code):
        self.body.annotate(code)
        self.finally_clause.annotate(code)

William Stein's avatar
William Stein committed
7319

7320 7321 7322 7323 7324 7325 7326 7327 7328 7329
class NogilTryFinallyStatNode(TryFinallyStatNode):
    """
    A try/finally statement that may be used in nogil code sections.
    """

    preserve_exception = False
    nogil_check = None


class GILStatNode(NogilTryFinallyStatNode):
7330 7331 7332
    #  'with gil' or 'with nogil' statement
    #
    #   state   string   'gil' or 'nogil'
7333

7334 7335
    state_temp = None

7336 7337
    def __init__(self, pos, state, body):
        self.state = state
7338
        self.create_state_temp_if_needed(pos, state, body)
7339 7340
        TryFinallyStatNode.__init__(
            self, pos,
7341 7342 7343 7344 7345
            body=body,
            finally_clause=GILExitNode(
                pos, state=state, state_temp=self.state_temp))

    def create_state_temp_if_needed(self, pos, state, body):
7346
        from .ParseTreeTransforms import YieldNodeCollector
7347 7348
        collector = YieldNodeCollector()
        collector.visitchildren(body)
7349
        if not collector.yields:
7350 7351
            return

7352 7353 7354 7355
        if state == 'gil':
            temp_type = PyrexTypes.c_gilstate_type
        else:
            temp_type = PyrexTypes.c_threadstate_ptr_type
7356
        from . import ExprNodes
7357
        self.state_temp = ExprNodes.TempNode(pos, temp_type)
7358

7359 7360 7361 7362
    def analyse_declarations(self, env):
        env._in_with_gil_block = (self.state == 'gil')
        if self.state == 'gil':
            env.has_with_gil_block = True
7363

7364 7365
        return super(GILStatNode, self).analyse_declarations(env)

7366
    def analyse_expressions(self, env):
7367 7368
        env.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
7369
        was_nogil = env.nogil
7370
        env.nogil = self.state == 'nogil'
7371
        node = TryFinallyStatNode.analyse_expressions(self, env)
7372
        env.nogil = was_nogil
7373
        return node
7374

7375
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
7376
        code.mark_pos(self.pos)
7377
        code.begin_block()
7378 7379 7380 7381 7382
        if self.state_temp:
            self.state_temp.allocate(code)
            variable = self.state_temp.result()
        else:
            variable = None
7383

7384
        old_gil_config = code.funcstate.gil_owned
7385
        if self.state == 'gil':
7386
            code.put_ensure_gil(variable=variable)
7387
            code.funcstate.gil_owned = True
7388
        else:
7389
            code.put_release_gil(variable=variable)
7390
            code.funcstate.gil_owned = False
7391

7392
        TryFinallyStatNode.generate_execution_code(self, code)
7393

7394 7395
        if self.state_temp:
            self.state_temp.release(code)
7396

7397
        code.funcstate.gil_owned = old_gil_config
7398
        code.end_block()
7399 7400 7401


class GILExitNode(StatNode):
7402 7403 7404 7405 7406
    """
    Used as the 'finally' block in a GILStatNode

    state   string   'gil' or 'nogil'
    """
7407

7408
    child_attrs = []
7409
    state_temp = None
7410

7411
    def analyse_expressions(self, env):
7412
        return self
7413 7414

    def generate_execution_code(self, code):
7415 7416 7417 7418 7419
        if self.state_temp:
            variable = self.state_temp.result()
        else:
            variable = None

7420
        if self.state == 'gil':
7421
            code.put_release_ensured_gil(variable)
7422
        else:
7423
            code.put_acquire_gil(variable)
7424 7425


7426 7427 7428 7429 7430 7431 7432
class EnsureGILNode(GILExitNode):
    """
    Ensure the GIL in nogil functions for cleanup before returning.
    """

    def generate_execution_code(self, code):
        code.put_ensure_gil(declare_gilstate=False)
7433

7434

7435 7436 7437 7438 7439
def cython_view_utility_code():
    from . import MemoryView
    return MemoryView.view_utility_code


7440 7441 7442
utility_code_for_cimports = {
    # utility code (or inlining c) in a pxd (or pyx) file.
    # TODO: Consider a generic user-level mechanism for importing
7443 7444 7445
    'cpython.array'         : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cpython.array.array'   : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cython.view'           : cython_view_utility_code,
7446
}
7447

7448 7449 7450
utility_code_for_imports = {
    # utility code used when special modules are imported.
    # TODO: Consider a generic user-level mechanism for importing
7451 7452
    'asyncio': ("__Pyx_patch_asyncio", "PatchAsyncIO", "Coroutine.c"),
    'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"),
7453 7454 7455
}


William Stein's avatar
William Stein committed
7456 7457 7458 7459 7460
class CImportStatNode(StatNode):
    #  cimport statement
    #
    #  module_name   string           Qualified name of module being imported
    #  as_name       string or None   Name specified in "as" clause, if any
7461
    #  is_absolute   bool             True for absolute imports, False otherwise
7462 7463

    child_attrs = []
7464
    is_absolute = False
7465

William Stein's avatar
William Stein committed
7466
    def analyse_declarations(self, env):
7467 7468 7469
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7470 7471
        module_scope = env.find_module(
            self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
William Stein's avatar
William Stein committed
7472
        if "." in self.module_name:
7473
            names = [EncodedString(name) for name in self.module_name.split(".")]
William Stein's avatar
William Stein committed
7474 7475 7476 7477 7478 7479 7480 7481 7482 7483
            top_name = names[0]
            top_module_scope = env.context.find_submodule(top_name)
            module_scope = top_module_scope
            for name in names[1:]:
                submodule_scope = module_scope.find_submodule(name)
                module_scope.declare_module(name, submodule_scope, self.pos)
                module_scope = submodule_scope
            if self.as_name:
                env.declare_module(self.as_name, module_scope, self.pos)
            else:
7484
                env.add_imported_module(module_scope)
William Stein's avatar
William Stein committed
7485 7486 7487 7488
                env.declare_module(top_name, top_module_scope, self.pos)
        else:
            name = self.as_name or self.module_name
            env.declare_module(name, module_scope, self.pos)
7489
        if self.module_name in utility_code_for_cimports:
7490
            env.use_utility_code(utility_code_for_cimports[self.module_name]())
William Stein's avatar
William Stein committed
7491 7492

    def analyse_expressions(self, env):
7493
        return self
7494

William Stein's avatar
William Stein committed
7495 7496
    def generate_execution_code(self, code):
        pass
7497

William Stein's avatar
William Stein committed
7498 7499 7500 7501

class FromCImportStatNode(StatNode):
    #  from ... cimport statement
    #
7502
    #  module_name     string                        Qualified name of module
7503
    #  relative_level  int or None                   Relative import: number of dots before module_name
7504
    #  imported_names  [(pos, name, as_name, kind)]  Names to be imported
7505

7506
    child_attrs = []
7507 7508 7509
    module_name = None
    relative_level = None
    imported_names = None
7510

William Stein's avatar
William Stein committed
7511
    def analyse_declarations(self, env):
7512 7513 7514
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7515 7516
        if self.relative_level and self.relative_level > env.qualified_name.count('.'):
            error(self.pos, "relative cimport beyond main package is not allowed")
7517
            return
7518 7519
        module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level)
        module_name = module_scope.qualified_name
William Stein's avatar
William Stein committed
7520
        env.add_imported_module(module_scope)
7521
        for pos, name, as_name, kind in self.imported_names:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7522
            if name == "*":
7523
                for local_name, entry in list(module_scope.entries.items()):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7524 7525
                    env.add_imported_entry(local_name, entry, pos)
            else:
7526 7527 7528 7529
                entry = module_scope.lookup(name)
                if entry:
                    if kind and not self.declaration_matches(entry, kind):
                        entry.redeclared(pos)
7530
                    entry.used = 1
7531 7532
                else:
                    if kind == 'struct' or kind == 'union':
7533 7534
                        entry = module_scope.declare_struct_or_union(
                            name, kind=kind, scope=None, typedef_flag=0, pos=pos)
7535
                    elif kind == 'class':
7536
                        entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
7537
                    else:
7538 7539
                        submodule_scope = env.context.find_module(
                            name, relative_to=module_scope, pos=self.pos, absolute_fallback=False)
7540 7541 7542
                        if submodule_scope.parent_module is module_scope:
                            env.declare_module(as_name or name, submodule_scope, self.pos)
                        else:
7543
                            error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
7544

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7545 7546 7547
                if entry:
                    local_name = as_name or name
                    env.add_imported_entry(local_name, entry, pos)
7548

7549
        if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now
7550
            if module_name in utility_code_for_cimports:
7551
                env.use_utility_code(utility_code_for_cimports[module_name]())
7552
            for _, name, _, _ in self.imported_names:
7553
                fqname = '%s.%s' % (module_name, name)
7554
                if fqname in utility_code_for_cimports:
7555
                    env.use_utility_code(utility_code_for_cimports[fqname]())
7556

7557
    def declaration_matches(self, entry, kind):
7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569
        if not entry.is_type:
            return 0
        type = entry.type
        if kind == 'class':
            if not type.is_extension_type:
                return 0
        else:
            if not type.is_struct_or_union:
                return 0
            if kind != type.kind:
                return 0
        return 1
William Stein's avatar
William Stein committed
7570 7571

    def analyse_expressions(self, env):
7572
        return self
7573

William Stein's avatar
William Stein committed
7574 7575 7576 7577 7578 7579 7580 7581 7582
    def generate_execution_code(self, code):
        pass


class FromImportStatNode(StatNode):
    #  from ... import statement
    #
    #  module           ImportNode
    #  items            [(string, NameNode)]
7583
    #  interned_items   [(string, NameNode, ExprNode)]
William Stein's avatar
William Stein committed
7584
    #  item             PyTempNode            used internally
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7585
    #  import_star      boolean               used internally
7586 7587

    child_attrs = ["module"]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7588
    import_star = 0
7589

William Stein's avatar
William Stein committed
7590
    def analyse_declarations(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7591 7592 7593 7594 7595 7596 7597 7598 7599
        for name, target in self.items:
            if name == "*":
                if not env.is_module_scope:
                    error(self.pos, "import * only allowed at module level")
                    return
                env.has_import_star = 1
                self.import_star = 1
            else:
                target.analyse_target_declaration(env)
7600

William Stein's avatar
William Stein committed
7601
    def analyse_expressions(self, env):
7602
        from . import ExprNodes
7603
        self.module = self.module.analyse_expressions(env)
7604
        self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
William Stein's avatar
William Stein committed
7605 7606
        self.interned_items = []
        for name, target in self.items:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7607 7608 7609
            if name == '*':
                for _, entry in env.entries.items():
                    if not entry.is_type and entry.type.is_extension_type:
7610
                        env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7611 7612
                        break
            else:
7613
                entry = env.lookup(target.name)
7614 7615
                # check whether or not entry is already cimported
                if (entry.is_type and entry.type.name == name
Stefan Behnel's avatar
Stefan Behnel committed
7616
                        and hasattr(entry.type, 'module_name')):
7617 7618 7619 7620 7621
                    if entry.type.module_name == self.module.module_name.value:
                        # cimported with absolute name
                        continue
                    try:
                        # cimported with relative name
7622 7623
                        module = env.find_module(self.module.module_name.value, pos=self.pos,
                                                 relative_level=self.module.level)
7624 7625 7626
                        if entry.type.module_name == module.qualified_name:
                            continue
                    except AttributeError:
7627
                        pass
7628
                target = target.analyse_target_expression(env, None)  # FIXME?
7629 7630 7631 7632
                if target.type is py_object_type:
                    coerced_item = None
                else:
                    coerced_item = self.item.coerce_to(target.type, env)
7633
                self.interned_items.append((name, target, coerced_item))
7634
        return self
7635

William Stein's avatar
William Stein committed
7636
    def generate_execution_code(self, code):
7637
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
7638
        self.module.generate_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7639 7640 7641 7642 7643 7644
        if self.import_star:
            code.putln(
                'if (%s(%s) < 0) %s;' % (
                    Naming.import_star,
                    self.module.py_result(),
                    code.error_goto(self.pos)))
7645 7646
        item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.item.set_cname(item_temp)
7647 7648 7649
        if self.interned_items:
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
7650
        for name, target, coerced_item in self.interned_items:
7651
            code.putln(
7652
                '%s = __Pyx_ImportFrom(%s, %s); %s' % (
7653
                    item_temp,
7654
                    self.module.py_result(),
7655 7656
                    code.intern_identifier(name),
                    code.error_goto_if_null(item_temp, self.pos)))
7657
            code.put_gotref(item_temp)
7658 7659 7660 7661 7662 7663
            if coerced_item is None:
                target.generate_assignment_code(self.item, code)
            else:
                coerced_item.allocate_temp_result(code)
                coerced_item.generate_result_code(code)
                target.generate_assignment_code(coerced_item, code)
7664 7665
            code.put_decref_clear(item_temp, py_object_type)
        code.funcstate.release_temp(item_temp)
William Stein's avatar
William Stein committed
7666
        self.module.generate_disposal_code(code)
7667
        self.module.free_temps(code)
William Stein's avatar
William Stein committed
7668

7669

Mark Florisson's avatar
Mark Florisson committed
7670 7671 7672 7673 7674 7675 7676 7677 7678 7679
class ParallelNode(Node):
    """
    Base class for cython.parallel constructs.
    """

    nogil_check = None


class ParallelStatNode(StatNode, ParallelNode):
    """
7680
    Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
Mark Florisson's avatar
Mark Florisson committed
7681 7682 7683 7684 7685

    assignments     { Entry(var) : (var.pos, inplace_operator_or_None) }
                    assignments to variables in this parallel section

    parent          parent ParallelStatNode or None
7686 7687 7688
    is_parallel     indicates whether this node is OpenMP parallel
                    (true for #pragma omp parallel for and
                              #pragma omp parallel)
Mark Florisson's avatar
Mark Florisson committed
7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699

    is_parallel is true for:

        #pragma omp parallel
        #pragma omp parallel for

    sections, but NOT for

        #pragma omp for

    We need this to determine the sharing attributes.
7700 7701 7702

    privatization_insertion_point   a code insertion point used to make temps
                                    private (esp. the "nsteps" temp)
7703 7704 7705 7706

    args         tuple          the arguments passed to the parallel construct
    kwargs       DictNode       the keyword arguments passed to the parallel
                                construct (replaced by its compile time value)
Mark Florisson's avatar
Mark Florisson committed
7707 7708
    """

7709
    child_attrs = ['body', 'num_threads']
Mark Florisson's avatar
Mark Florisson committed
7710 7711 7712 7713

    body = None

    is_prange = False
7714
    is_nested_prange = False
Mark Florisson's avatar
Mark Florisson committed
7715

7716
    error_label_used = False
7717

7718
    num_threads = None
7719
    chunksize = None
7720

7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737
    parallel_exc = (
        Naming.parallel_exc_type,
        Naming.parallel_exc_value,
        Naming.parallel_exc_tb,
    )

    parallel_pos_info = (
        Naming.parallel_filename,
        Naming.parallel_lineno,
        Naming.parallel_clineno,
    )

    pos_info = (
        Naming.filename_cname,
        Naming.lineno_cname,
        Naming.clineno_cname,
    )
7738

7739 7740
    critical_section_counter = 0

Mark Florisson's avatar
Mark Florisson committed
7741 7742
    def __init__(self, pos, **kwargs):
        super(ParallelStatNode, self).__init__(pos, **kwargs)
7743 7744

        # All assignments in this scope
Mark Florisson's avatar
Mark Florisson committed
7745 7746
        self.assignments = kwargs.get('assignments') or {}

7747 7748 7749 7750
        # All seen closure cnames and their temporary cnames
        self.seen_closure_vars = set()

        # Dict of variables that should be declared (first|last|)private or
7751 7752
        # reduction { Entry: (op, lastprivate) }.
        # If op is not None, it's a reduction.
7753
        self.privates = {}
Mark Florisson's avatar
Mark Florisson committed
7754

Mark Florisson's avatar
Mark Florisson committed
7755 7756 7757
        # [NameNode]
        self.assigned_nodes = []

Mark Florisson's avatar
Mark Florisson committed
7758 7759 7760
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)

7761 7762
        self.num_threads = None

7763
        if self.kwargs:
7764 7765 7766
            # Try to find num_threads and chunksize keyword arguments
            pairs = []
            for dictitem in self.kwargs.key_value_pairs:
7767 7768
                if dictitem.key.value == 'num_threads':
                    self.num_threads = dictitem.value
7769 7770 7771 7772 7773 7774
                elif self.is_prange and dictitem.key.value == 'chunksize':
                    self.chunksize = dictitem.value
                else:
                    pairs.append(dictitem)

            self.kwargs.key_value_pairs = pairs
7775

7776 7777
            try:
                self.kwargs = self.kwargs.compile_time_value(env)
7778
            except Exception as e:
7779 7780
                error(self.kwargs.pos, "Only compile-time values may be "
                                       "supplied as keyword arguments")
7781 7782 7783
        else:
            self.kwargs = {}

7784
        for kw, val in self.kwargs.items():
7785 7786 7787 7788 7789
            if kw not in self.valid_keyword_arguments:
                error(self.pos, "Invalid keyword argument: %s" % kw)
            else:
                setattr(self, kw, val)

7790
    def analyse_expressions(self, env):
7791
        if self.num_threads:
7792
            self.num_threads = self.num_threads.analyse_expressions(env)
7793 7794

        if self.chunksize:
7795
            self.chunksize = self.chunksize.analyse_expressions(env)
7796

7797
        self.body = self.body.analyse_expressions(env)
7798
        self.analyse_sharing_attributes(env)
7799

7800
        if self.num_threads is not None:
7801 7802
            if self.parent and self.parent.num_threads is not None and not self.parent.is_prange:
                error(self.pos, "num_threads already declared in outer section")
7803
            elif self.parent and not self.parent.is_prange:
7804
                error(self.pos, "num_threads must be declared in the parent parallel section")
7805
            elif (self.num_threads.type.is_int and
7806 7807 7808
                    self.num_threads.is_literal and
                    self.num_threads.compile_time_value(env) <= 0):
                error(self.pos, "argument to num_threads must be greater than 0")
7809

Mark Florisson's avatar
Mark Florisson committed
7810 7811
            if not self.num_threads.is_simple():
                self.num_threads = self.num_threads.coerce_to(
7812 7813
                    PyrexTypes.c_int_type, env).coerce_to_temp(env)
        return self
7814 7815

    def analyse_sharing_attributes(self, env):
Mark Florisson's avatar
Mark Florisson committed
7816
        """
7817 7818 7819
        Analyse the privates for this block and set them in self.privates.
        This should be called in a post-order fashion during the
        analyse_expressions phase
Mark Florisson's avatar
Mark Florisson committed
7820
        """
7821
        for entry, (pos, op) in self.assignments.items():
Mark Florisson's avatar
Mark Florisson committed
7822

7823 7824 7825 7826 7827
            if self.is_prange and not self.is_parallel:
                # closely nested prange in a with parallel block, disallow
                # assigning to privates in the with parallel block (we
                # consider it too implicit and magicky for users)
                if entry in self.parent.assignments:
7828
                    error(pos, "Cannot assign to private of outer parallel block")
7829 7830 7831 7832 7833 7834 7835
                    continue

            if not self.is_prange and op:
                # Again possible, but considered to magicky
                error(pos, "Reductions not allowed for parallel blocks")
                continue

Mark Florisson's avatar
Mark Florisson committed
7836 7837 7838
            # By default all variables should have the same values as if
            # executed sequentially
            lastprivate = True
7839
            self.propagate_var_privatization(entry, pos, op, lastprivate)
7840

7841
    def propagate_var_privatization(self, entry, pos, op, lastprivate):
7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869
        """
        Propagate the sharing attributes of a variable. If the privatization is
        determined by a parent scope, done propagate further.

        If we are a prange, we propagate our sharing attributes outwards to
        other pranges. If we are a prange in parallel block and the parallel
        block does not determine the variable private, we propagate to the
        parent of the parent. Recursion stops at parallel blocks, as they have
        no concept of lastprivate or reduction.

        So the following cases propagate:

            sum is a reduction for all loops:

                for i in prange(n):
                    for j in prange(n):
                        for k in prange(n):
                            sum += i * j * k

            sum is a reduction for both loops, local_var is private to the
            parallel with block:

                for i in prange(n):
                    with parallel:
                        local_var = ... # private to the parallel
                        for j in prange(n):
                            sum += i * j

Mark Florisson's avatar
Mark Florisson committed
7870 7871
        Nested with parallel blocks are disallowed, because they wouldn't
        allow you to propagate lastprivates or reductions:
7872 7873 7874 7875

            #pragma omp parallel for lastprivate(i)
            for i in prange(n):

Mark Florisson's avatar
Mark Florisson committed
7876 7877
                sum = 0

7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893
                #pragma omp parallel private(j, sum)
                with parallel:

                    #pragma omp parallel
                    with parallel:

                        #pragma omp for lastprivate(j) reduction(+:sum)
                        for j in prange(n):
                            sum += i

                    # sum and j are well-defined here

                # sum and j are undefined here

            # sum and j are undefined here
        """
7894
        self.privates[entry] = (op, lastprivate)
7895 7896 7897 7898 7899

        if entry.type.is_memoryviewslice:
            error(pos, "Memoryview slices can only be shared in parallel sections")
            return

7900 7901 7902 7903 7904 7905
        if self.is_prange:
            if not self.is_parallel and entry not in self.parent.assignments:
                # Parent is a parallel with block
                parent = self.parent.parent
            else:
                parent = self.parent
Mark Florisson's avatar
Mark Florisson committed
7906

7907 7908 7909
            # We don't need to propagate privates, only reductions and
            # lastprivates
            if parent and (op or lastprivate):
7910
                parent.propagate_var_privatization(entry, pos, op, lastprivate)
Mark Florisson's avatar
Mark Florisson committed
7911 7912 7913 7914 7915 7916 7917 7918 7919

    def _allocate_closure_temp(self, code, entry):
        """
        Helper function that allocate a temporary for a closure variable that
        is assigned to.
        """
        if self.parent:
            return self.parent._allocate_closure_temp(code, entry)

7920 7921 7922
        if entry.cname in self.seen_closure_vars:
            return entry.cname

7923
        cname = code.funcstate.allocate_temp(entry.type, True)
7924 7925 7926 7927 7928 7929

        # Add both the actual cname and the temp cname, as the actual cname
        # will be replaced with the temp cname on the entry
        self.seen_closure_vars.add(entry.cname)
        self.seen_closure_vars.add(cname)

Mark Florisson's avatar
Mark Florisson committed
7930 7931 7932 7933
        self.modified_entries.append((entry, entry.cname))
        code.putln("%s = %s;" % (cname, entry.cname))
        entry.cname = cname

7934
    def initialize_privates_to_nan(self, code, exclude=None):
7935
        first = True
7936

7937
        for entry, (op, lastprivate) in sorted(self.privates.items()):
7938
            if not op and (not exclude or entry != exclude):
7939
                invalid_value = entry.type.invalid_value()
7940

7941
                if invalid_value:
7942 7943 7944 7945
                    if first:
                        code.putln("/* Initialize private variables to "
                                   "invalid values */")
                        first = False
7946
                    code.putln("%s = %s;" % (entry.cname,
7947
                                             entry.type.cast_code(invalid_value)))
7948

7949
    def evaluate_before_block(self, code, expr):
Mark Florisson's avatar
Mark Florisson committed
7950
        c = self.begin_of_parallel_control_block_point_after_decls
7951 7952 7953 7954 7955 7956 7957 7958 7959 7960
        # we need to set the owner to ourselves temporarily, as
        # allocate_temp may generate a comment in the middle of our pragma
        # otherwise when DebugFlags.debug_temp_code_comments is in effect
        owner = c.funcstate.owner
        c.funcstate.owner = c
        expr.generate_evaluation_code(c)
        c.funcstate.owner = owner

        return expr.result()

7961 7962 7963 7964 7965
    def put_num_threads(self, code):
        """
        Write self.num_threads if set as the num_threads OpenMP directive
        """
        if self.num_threads is not None:
7966
            code.put(" num_threads(%s)" % self.evaluate_before_block(code, self.num_threads))
7967

7968

Mark Florisson's avatar
Mark Florisson committed
7969 7970 7971 7972 7973 7974 7975 7976 7977
    def declare_closure_privates(self, code):
        """
        If a variable is in a scope object, we need to allocate a temp and
        assign the value from the temp to the variable in the scope object
        after the parallel section. This kind of copying should be done only
        in the outermost parallel section.
        """
        self.modified_entries = []

7978
        for entry in sorted(self.assignments):
Mark Florisson's avatar
Mark Florisson committed
7979
            if entry.from_closure or entry.in_closure:
7980
                self._allocate_closure_temp(code, entry)
Mark Florisson's avatar
Mark Florisson committed
7981 7982

    def release_closure_privates(self, code):
7983 7984 7985
        """
        Release any temps used for variables in scope objects. As this is the
        outermost parallel block, we don't need to delete the cnames from
7986
        self.seen_closure_vars.
7987
        """
Mark Florisson's avatar
Mark Florisson committed
7988 7989 7990 7991 7992
        for entry, original_cname in self.modified_entries:
            code.putln("%s = %s;" % (original_cname, entry.cname))
            code.funcstate.release_temp(entry.cname)
            entry.cname = original_cname

7993 7994 7995 7996 7997 7998
    def privatize_temps(self, code, exclude_temps=()):
        """
        Make any used temporaries private. Before the relevant code block
        code.start_collecting_temps() should have been called.
        """
        if self.is_parallel:
7999
            c = self.privatization_insertion_point
8000

8001
            self.temps = temps = code.funcstate.stop_collecting_temps()
8002
            privates, firstprivates = [], []
8003
            for temp, type in sorted(temps):
8004
                if type.is_pyobject or type.is_memoryviewslice:
8005 8006 8007
                    firstprivates.append(temp)
                else:
                    privates.append(temp)
8008

8009 8010 8011 8012
            if privates:
                c.put(" private(%s)" % ", ".join(privates))
            if firstprivates:
                c.put(" firstprivate(%s)" % ", ".join(firstprivates))
8013 8014 8015 8016 8017 8018 8019 8020

            if self.breaking_label_used:
                shared_vars = [Naming.parallel_why]
                if self.error_label_used:
                    shared_vars.extend(self.parallel_exc)
                    c.put(" private(%s, %s, %s)" % self.pos_info)

                c.put(" shared(%s)" % ', '.join(shared_vars))
8021

8022 8023 8024 8025
    def cleanup_temps(self, code):
        # Now clean up any memoryview slice and object temporaries
        if self.is_parallel and not self.is_nested_prange:
            code.putln("/* Clean up any temporaries */")
8026
            for temp, type in sorted(self.temps):
8027 8028 8029 8030 8031
                if type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(temp, have_gil=False)
                elif type.is_pyobject:
                    code.put_xdecref(temp, type)
                    code.putln("%s = NULL;" % temp)
8032

8033
    def setup_parallel_control_flow_block(self, code):
8034
        """
8035 8036
        Sets up a block that surrounds the parallel block to determine
        how the parallel section was exited. Any kind of return is
8037 8038 8039
        trapped (break, continue, return, exceptions). This is the idea:

        {
8040
            int why = 0;
8041 8042 8043 8044 8045 8046 8047

            #pragma omp parallel
            {
                return # -> goto new_return_label;
                goto end_parallel;

            new_return_label:
8048
                why = 3;
8049
                goto end_parallel;
8050

8051
            end_parallel:;
8052
                #pragma omp flush(why) # we need to flush for every iteration
8053 8054
            }

8055
            if (why == 3)
8056 8057 8058 8059 8060 8061 8062 8063
                goto old_return_label;
        }
        """
        self.old_loop_labels = code.new_loop_labels()
        self.old_error_label = code.new_error_label()
        self.old_return_label = code.return_label
        code.return_label = code.new_label(name="return")

8064 8065
        code.begin_block() # parallel control flow block
        self.begin_of_parallel_control_block_point = code.insertion_point()
Mark Florisson's avatar
Mark Florisson committed
8066
        self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
8067

8068 8069
        self.undef_builtin_expect_apple_gcc_bug(code)

8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081
    def begin_parallel_block(self, code):
        """
        Each OpenMP thread in a parallel section that contains a with gil block
        must have the thread-state initialized. The call to
        PyGILState_Release() then deallocates our threadstate. If we wouldn't
        do this, each with gil block would allocate and deallocate one, thereby
        losing exception information before it can be saved before leaving the
        parallel section.
        """
        self.begin_of_parallel_block = code.insertion_point()

    def end_parallel_block(self, code):
8082 8083 8084 8085 8086 8087 8088 8089 8090
        """
        To ensure all OpenMP threads have thread states, we ensure the GIL
        in each thread (which creates a thread state if it doesn't exist),
        after which we release the GIL.
        On exit, reacquire the GIL and release the thread state.

        If compiled without OpenMP support (at the C level), then we still have
        to acquire the GIL to decref any object temporaries.
        """
8091 8092 8093 8094
        if self.error_label_used:
            begin_code = self.begin_of_parallel_block
            end_code = code

8095
            begin_code.putln("#ifdef _OPENMP")
8096 8097
            begin_code.put_ensure_gil(declare_gilstate=True)
            begin_code.putln("Py_BEGIN_ALLOW_THREADS")
8098
            begin_code.putln("#endif /* _OPENMP */")
8099

8100
            end_code.putln("#ifdef _OPENMP")
8101
            end_code.putln("Py_END_ALLOW_THREADS")
8102 8103 8104 8105 8106
            end_code.putln("#else")
            end_code.put_safe("{\n")
            end_code.put_ensure_gil()
            end_code.putln("#endif /* _OPENMP */")
            self.cleanup_temps(end_code)
8107
            end_code.put_release_ensured_gil()
8108 8109
            end_code.putln("#ifndef _OPENMP")
            end_code.put_safe("}\n")
8110
            end_code.putln("#endif /* _OPENMP */")
8111

8112 8113 8114 8115
    def trap_parallel_exit(self, code, should_flush=False):
        """
        Trap any kind of return inside a parallel construct. 'should_flush'
        indicates whether the variable should be flushed, which is needed by
8116 8117 8118 8119 8120 8121 8122 8123 8124
        prange to skip the loop. It also indicates whether we need to register
        a continue (we need this for parallel blocks, but not for prange
        loops, as it is a direct jump there).

        It uses the same mechanism as try/finally:
            1 continue
            2 break
            3 return
            4 error
8125
        """
8126
        save_lastprivates_label = code.new_label()
8127
        dont_return_label = code.new_label()
8128 8129 8130

        self.any_label_used = False
        self.breaking_label_used = False
8131
        self.error_label_used = False
8132

8133 8134 8135 8136 8137 8138
        self.parallel_private_temps = []

        all_labels = code.get_all_labels()

        # Figure this out before starting to generate any code
        for label in all_labels:
8139 8140
            if code.label_used(label):
                self.breaking_label_used = (self.breaking_label_used or
8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151
                                            label != code.continue_label)
                self.any_label_used = True

        if self.any_label_used:
            code.put_goto(dont_return_label)

        for i, label in enumerate(all_labels):
            if not code.label_used(label):
                continue

            is_continue_label = label == code.continue_label
8152

8153
            code.put_label(label)
8154

8155 8156 8157 8158
            if not (should_flush and is_continue_label):
                if label == code.error_label:
                    self.error_label_used = True
                    self.fetch_parallel_exception(code)
8159

8160
                code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
8161

8162 8163 8164 8165
            if (self.breaking_label_used and self.is_prange and not
                    is_continue_label):
                code.put_goto(save_lastprivates_label)
            else:
8166 8167
                code.put_goto(dont_return_label)

8168
        if self.any_label_used:
8169 8170 8171 8172 8173
            if self.is_prange and self.breaking_label_used:
                # Don't rely on lastprivate, save our lastprivates
                code.put_label(save_lastprivates_label)
                self.save_parallel_vars(code)

8174 8175 8176
            code.put_label(dont_return_label)

            if should_flush and self.breaking_label_used:
8177 8178
                code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)

8179 8180 8181 8182 8183 8184 8185 8186 8187
    def save_parallel_vars(self, code):
        """
        The following shenanigans are instated when we break, return or
        propagate errors from a prange. In this case we cannot rely on
        lastprivate() to do its job, as no iterations may have executed yet
        in the last thread, leaving the values undefined. It is most likely
        that the breaking thread has well-defined values of the lastprivate
        variables, so we keep those values.
        """
8188
        section_name = "__pyx_parallel_lastprivates%d" % self.critical_section_counter
8189 8190 8191 8192 8193 8194 8195 8196
        code.putln_openmp("#pragma omp critical(%s)" % section_name)
        ParallelStatNode.critical_section_counter += 1

        code.begin_block() # begin critical section

        c = self.begin_of_parallel_control_block_point

        temp_count = 0
8197
        for entry, (op, lastprivate) in sorted(self.privates.items()):
8198 8199 8200
            if not lastprivate or entry.type.is_pyobject:
                continue

8201
            type_decl = entry.type.empty_declaration_code()
8202 8203 8204 8205 8206
            temp_cname = "__pyx_parallel_temp%d" % temp_count
            private_cname = entry.cname

            temp_count += 1

8207 8208
            invalid_value = entry.type.invalid_value()
            if invalid_value:
8209
                init = ' = ' + entry.type.cast_code(invalid_value)
8210 8211
            else:
                init = ''
8212
            # Declare the parallel private in the outer block
8213
            c.putln("%s %s%s;" % (type_decl, temp_cname, init))
8214 8215 8216 8217 8218 8219 8220 8221

            # Initialize before escaping
            code.putln("%s = %s;" % (temp_cname, private_cname))

            self.parallel_private_temps.append((temp_cname, private_cname))

        code.end_block() # end critical section

8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255
    def fetch_parallel_exception(self, code):
        """
        As each OpenMP thread may raise an exception, we need to fetch that
        exception from the threadstate and save it for after the parallel
        section where it can be re-raised in the master thread.

        Although it would seem that __pyx_filename, __pyx_lineno and
        __pyx_clineno are only assigned to under exception conditions (i.e.,
        when we have the GIL), and thus should be allowed to be shared without
        any race condition, they are in fact subject to the same race
        conditions that they were previously when they were global variables
        and functions were allowed to release the GIL:

            thread A                thread B
                acquire
                set lineno
                release
                                        acquire
                                        set lineno
                                        release
                acquire
                fetch exception
                release
                                        skip the fetch

                deallocate threadstate  deallocate threadstate
        """
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

        code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
        code.putln(
            "if (!%s) {" % Naming.parallel_exc_type)

8256
        code.putln("__Pyx_ErrFetchWithState(&%s, &%s, &%s);" % self.parallel_exc)
8257
        pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
8258
        code.funcstate.uses_error_indicator = True
8259
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
8260
        code.put_gotref(Naming.parallel_exc_type)
8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272

        code.putln(
            "}")

        code.put_release_ensured_gil()
        code.end_block()

    def restore_parallel_exception(self, code):
        "Re-raise a parallel exception"
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

8273
        code.put_giveref(Naming.parallel_exc_type)
8274
        code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc)
8275 8276 8277 8278 8279
        pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))

        code.put_release_ensured_gil()
        code.end_block()
8280 8281

    def restore_labels(self, code):
8282 8283 8284 8285 8286 8287
        """
        Restore all old labels. Call this before the 'else' clause to for
        loops and always before ending the parallel control flow block.
        """
        code.set_all_labels(self.old_loop_labels + (self.old_return_label,
                                                    self.old_error_label))
8288

8289 8290
    def end_parallel_control_flow_block(
            self, code, break_=False, continue_=False, return_=False):
8291 8292 8293 8294 8295
        """
        This ends the parallel control flow block and based on how the parallel
        section was exited, takes the corresponding action. The break_ and
        continue_ parameters indicate whether these should be propagated
        outwards:
8296

8297 8298 8299 8300 8301 8302 8303
            for i in prange(...):
                with cython.parallel.parallel():
                    continue

        Here break should be trapped in the parallel block, and propagated to
        the for loop.
        """
8304 8305 8306 8307
        c = self.begin_of_parallel_control_block_point

        # Firstly, always prefer errors over returning, continue or break
        if self.error_label_used:
8308 8309
            c.putln("const char *%s = NULL; int %s = 0, %s = 0;" % self.parallel_pos_info)
            c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" % self.parallel_exc)
8310 8311 8312 8313 8314 8315 8316 8317 8318

            code.putln(
                "if (%s) {" % Naming.parallel_exc_type)
            code.putln("/* This may have been overridden by a continue, "
                       "break or return in another thread. Prefer the error. */")
            code.putln("%s = 4;" % Naming.parallel_why)
            code.putln(
                "}")

8319 8320 8321 8322 8323 8324 8325
        if continue_:
            any_label_used = self.any_label_used
        else:
            any_label_used = self.breaking_label_used

        if any_label_used:
            # __pyx_parallel_why is used, declare and initialize
8326 8327
            c.putln("int %s;" % Naming.parallel_why)
            c.putln("%s = 0;" % Naming.parallel_why)
8328

8329 8330 8331 8332 8333 8334
            code.putln(
                "if (%s) {" % Naming.parallel_why)

            for temp_cname, private_cname in self.parallel_private_temps:
                code.putln("%s = %s;" % (private_cname, temp_cname))

8335
            code.putln("switch (%s) {" % Naming.parallel_why)
8336 8337 8338 8339 8340 8341 8342 8343
            if continue_:
                code.put("    case 1: ")
                code.put_goto(code.continue_label)

            if break_:
                code.put("    case 2: ")
                code.put_goto(code.break_label)

8344 8345 8346
            if return_:
                code.put("    case 3: ")
                code.put_goto(code.return_label)
8347 8348

            if self.error_label_used:
8349
                code.globalstate.use_utility_code(restore_exception_utility_code)
8350 8351 8352
                code.putln("    case 4:")
                self.restore_parallel_exception(code)
                code.put_goto(code.error_label)
8353

8354 8355 8356
            code.putln("}") # end switch
            code.putln(
                "}") # end if
8357 8358

        code.end_block() # end parallel control flow block
8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370
        self.redef_builtin_expect_apple_gcc_bug(code)

    # FIXME: improve with version number for OS X Lion
    buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
    have_expect_condition = "(defined(__GNUC__) && " \
                             "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
    redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)

    def undef_builtin_expect_apple_gcc_bug(self, code):
        """
        A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
        """
8371
        if not self.parent:
8372
            code.undef_builtin_expect(self.redef_condition)
8373 8374

    def redef_builtin_expect_apple_gcc_bug(self, code):
8375
        if not self.parent:
8376
            code.redef_builtin_expect(self.redef_condition)
8377

Mark Florisson's avatar
Mark Florisson committed
8378 8379 8380

class ParallelWithBlockNode(ParallelStatNode):
    """
8381
    This node represents a 'with cython.parallel.parallel():' block
Mark Florisson's avatar
Mark Florisson committed
8382 8383
    """

8384 8385 8386 8387 8388 8389 8390 8391 8392
    valid_keyword_arguments = ['num_threads']

    num_threads = None

    def analyse_declarations(self, env):
        super(ParallelWithBlockNode, self).analyse_declarations(env)
        if self.args:
            error(self.pos, "cython.parallel.parallel() does not take "
                            "positional arguments")
8393

Mark Florisson's avatar
Mark Florisson committed
8394 8395
    def generate_execution_code(self, code):
        self.declare_closure_privates(code)
8396
        self.setup_parallel_control_flow_block(code)
Mark Florisson's avatar
Mark Florisson committed
8397 8398 8399

        code.putln("#ifdef _OPENMP")
        code.put("#pragma omp parallel ")
8400 8401

        if self.privates:
8402
            privates = [e.cname for e in self.privates
8403
                        if not e.type.is_pyobject]
8404
            code.put('private(%s)' % ', '.join(sorted(privates)))
8405

8406
        self.privatization_insertion_point = code.insertion_point()
8407
        self.put_num_threads(code)
8408
        code.putln("")
8409

8410 8411
        code.putln("#endif /* _OPENMP */")

8412
        code.begin_block()  # parallel block
8413
        self.begin_parallel_block(code)
8414
        self.initialize_privates_to_nan(code)
8415
        code.funcstate.start_collecting_temps()
Mark Florisson's avatar
Mark Florisson committed
8416
        self.body.generate_execution_code(code)
8417
        self.trap_parallel_exit(code)
8418
        self.privatize_temps(code)
8419
        self.end_parallel_block(code)
8420
        code.end_block()  # end parallel block
8421

8422 8423
        continue_ = code.label_used(code.continue_label)
        break_ = code.label_used(code.break_label)
8424
        return_ = code.label_used(code.return_label)
Mark Florisson's avatar
Mark Florisson committed
8425

8426 8427
        self.restore_labels(code)
        self.end_parallel_control_flow_block(code, break_=break_,
8428 8429
                                             continue_=continue_,
                                             return_=return_)
Mark Florisson's avatar
Mark Florisson committed
8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440
        self.release_closure_privates(code)


class ParallelRangeNode(ParallelStatNode):
    """
    This node represents a 'for i in cython.parallel.prange():' construct.

    target       NameNode       the target iteration variable
    else_clause  Node or None   the else clause of this loop
    """

8441 8442
    child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
                   'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8443 8444 8445 8446 8447 8448

    body = target = else_clause = args = None

    start = stop = step = None

    is_prange = True
8449

8450
    nogil = None
8451 8452
    schedule = None

8453
    valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8454

8455 8456 8457 8458 8459
    def __init__(self, pos, **kwds):
        super(ParallelRangeNode, self).__init__(pos, **kwds)
        # Pretend to be a ForInStatNode for control flow analysis
        self.iterator = PassStatNode(pos)

Mark Florisson's avatar
Mark Florisson committed
8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476
    def analyse_declarations(self, env):
        super(ParallelRangeNode, self).analyse_declarations(env)
        self.target.analyse_target_declaration(env)
        if self.else_clause is not None:
            self.else_clause.analyse_declarations(env)

        if not self.args or len(self.args) > 3:
            error(self.pos, "Invalid number of positional arguments to prange")
            return

        if len(self.args) == 1:
            self.stop, = self.args
        elif len(self.args) == 2:
            self.start, self.stop = self.args
        else:
            self.start, self.stop, self.step = self.args

Mark Florisson's avatar
Mark Florisson committed
8477 8478 8479
        if hasattr(self.schedule, 'decode'):
            self.schedule = self.schedule.decode('ascii')

8480 8481
        if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'):
            error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,))
Mark Florisson's avatar
Mark Florisson committed
8482 8483

    def analyse_expressions(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
8484
        was_nogil = env.nogil
8485 8486 8487
        if self.nogil:
            env.nogil = True

8488 8489
        if self.target is None:
            error(self.pos, "prange() can only be used as part of a for loop")
8490
            return self
Mark Florisson's avatar
Mark Florisson committed
8491

8492
        self.target = self.target.analyse_target_types(env)
Mark Florisson's avatar
Mark Florisson committed
8493

8494 8495 8496 8497 8498 8499 8500 8501
        if not self.target.type.is_numeric:
            # Not a valid type, assume one for now anyway

            if not self.target.type.is_pyobject:
                # nogil_check will catch the is_pyobject case
                error(self.target.pos,
                      "Must be of numeric type, not %s" % self.target.type)

8502
            self.index_type = PyrexTypes.c_py_ssize_t_type
8503 8504
        else:
            self.index_type = self.target.type
8505 8506 8507 8508
            if not self.index_type.signed:
                warning(self.target.pos,
                        "Unsigned index type not allowed before OpenMP 3.0",
                        level=2)
Mark Florisson's avatar
Mark Florisson committed
8509 8510 8511 8512 8513 8514 8515 8516 8517

        # Setup start, stop and step, allocating temps if needed
        self.names = 'start', 'stop', 'step'
        start_stop_step = self.start, self.stop, self.step

        for node, name in zip(start_stop_step, self.names):
            if node is not None:
                node.analyse_types(env)
                if not node.type.is_numeric:
8518 8519
                    error(node.pos, "%s argument must be numeric" % name)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8520 8521 8522 8523 8524 8525 8526 8527

                if not node.is_literal:
                    node = node.coerce_to_temp(env)
                    setattr(self, name, node)

                # As we range from 0 to nsteps, computing the index along the
                # way, we need a fitting type for 'i' and 'nsteps'
                self.index_type = PyrexTypes.widest_numeric_type(
8528
                    self.index_type, node.type)
Mark Florisson's avatar
Mark Florisson committed
8529 8530

        if self.else_clause is not None:
8531
            self.else_clause = self.else_clause.analyse_expressions(env)
Mark Florisson's avatar
Mark Florisson committed
8532

8533 8534 8535 8536 8537 8538 8539 8540
        # Although not actually an assignment in this scope, it should be
        # treated as such to ensure it is unpacked if a closure temp, and to
        # ensure lastprivate behaviour and propagation. If the target index is
        # not a NameNode, it won't have an entry, and an error was issued by
        # ParallelRangeTransform
        if hasattr(self.target, 'entry'):
            self.assignments[self.target.entry] = self.target.pos, None

8541
        node = super(ParallelRangeNode, self).analyse_expressions(env)
8542

8543 8544 8545
        if node.chunksize:
            if not node.schedule:
                error(node.chunksize.pos,
8546
                      "Must provide schedule with chunksize")
8547 8548
            elif node.schedule == 'runtime':
                error(node.chunksize.pos,
8549
                      "Chunksize not valid for the schedule runtime")
8550 8551 8552 8553
            elif (node.chunksize.type.is_int and
                  node.chunksize.is_literal and
                  node.chunksize.compile_time_value(env) <= 0):
                error(node.chunksize.pos, "Chunksize must not be negative")
8554

8555 8556
            node.chunksize = node.chunksize.coerce_to(
                PyrexTypes.c_int_type, env).coerce_to_temp(env)
8557

8558
        if node.nogil:
8559 8560
            env.nogil = was_nogil

8561 8562 8563
        node.is_nested_prange = node.parent and node.parent.is_prange
        if node.is_nested_prange:
            parent = node
8564 8565 8566
            while parent.parent and parent.parent.is_prange:
                parent = parent.parent

8567 8568 8569 8570
            parent.assignments.update(node.assignments)
            parent.privates.update(node.privates)
            parent.assigned_nodes.extend(node.assigned_nodes)
        return node
8571

Mark Florisson's avatar
Mark Florisson committed
8572 8573
    def nogil_check(self, env):
        names = 'start', 'stop', 'step', 'target'
8574
        nodes = self.start, self.stop, self.step, self.target
Mark Florisson's avatar
Mark Florisson committed
8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619
        for name, node in zip(names, nodes):
            if node is not None and node.type.is_pyobject:
                error(node.pos, "%s may not be a Python object "
                                "as we don't have the GIL" % name)

    def generate_execution_code(self, code):
        """
        Generate code in the following steps

            1)  copy any closure variables determined thread-private
                into temporaries

            2)  allocate temps for start, stop and step

            3)  generate a loop that calculates the total number of steps,
                which then computes the target iteration variable for every step:

                    for i in prange(start, stop, step):
                        ...

                becomes

                    nsteps = (stop - start) / step;
                    i = start;

                    #pragma omp parallel for lastprivate(i)
                    for (temp = 0; temp < nsteps; temp++) {
                        i = start + step * temp;
                        ...
                    }

                Note that accumulation of 'i' would have a data dependency
                between iterations.

                Also, you can't do this

                    for (i = start; i < stop; i += step)
                        ...

                as the '<' operator should become '>' for descending loops.
                'for i from x < i < y:' does not suffer from this problem
                as the relational operator is known at compile time!

            4) release our temps and write back any private closure variables
        """
8620
        self.declare_closure_privates(code)
Mark Florisson's avatar
Mark Florisson committed
8621 8622 8623 8624 8625 8626 8627 8628

        # This can only be a NameNode
        target_index_cname = self.target.entry.cname

        # This will be used as the dict to format our code strings, holding
        # the start, stop , step, temps and target cnames
        fmt_dict = {
            'target': target_index_cname,
8629
            'target_type': self.target.type.empty_declaration_code()
Mark Florisson's avatar
Mark Florisson committed
8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652
        }

        # Setup start, stop and step, allocating temps if needed
        start_stop_step = self.start, self.stop, self.step
        defaults = '0', '0', '1'
        for node, name, default in zip(start_stop_step, self.names, defaults):
            if node is None:
                result = default
            elif node.is_literal:
                result = node.get_constant_c_result_code()
            else:
                node.generate_evaluation_code(code)
                result = node.result()

            fmt_dict[name] = result

        fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
        fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)

        # TODO: check if the step is 0 and if so, raise an exception in a
        # 'with gil' block. For now, just abort
        code.putln("if (%(step)s == 0) abort();" % fmt_dict)

8653
        self.setup_parallel_control_flow_block(code) # parallel control flow block
8654

8655
        self.control_flow_var_code_point = code.insertion_point()
8656

8657
        # Note: nsteps is private in an outer scope if present
8658
        code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict)
Mark Florisson's avatar
Mark Florisson committed
8659

8660 8661 8662 8663 8664 8665 8666
        # The target iteration variable might not be initialized, do it only if
        # we are executing at least 1 iteration, otherwise we should leave the
        # target unaffected. The target iteration variable is firstprivate to
        # shut up compiler warnings caused by lastprivate, as the compiler
        # erroneously believes that nsteps may be <= 0, leaving the private
        # target index uninitialized
        code.putln("if (%(nsteps)s > 0)" % fmt_dict)
8667
        code.begin_block() # if block
Mark Florisson's avatar
Mark Florisson committed
8668
        self.generate_loop(code, fmt_dict)
8669
        code.end_block() # end if block
Mark Florisson's avatar
Mark Florisson committed
8670

8671 8672 8673
        self.restore_labels(code)

        if self.else_clause:
8674
            if self.breaking_label_used:
8675
                code.put("if (%s < 2)" % Naming.parallel_why)
8676 8677 8678 8679 8680 8681 8682

            code.begin_block() # else block
            code.putln("/* else */")
            self.else_clause.generate_execution_code(code)
            code.end_block() # end else block

        # ------ cleanup ------
8683
        self.end_parallel_control_flow_block(code) # end parallel control flow block
8684

Mark Florisson's avatar
Mark Florisson committed
8685 8686
        # And finally, release our privates and write back any closure
        # variables
8687
        for temp in start_stop_step + (self.chunksize, self.num_threads):
Mark Florisson's avatar
Mark Florisson committed
8688 8689 8690 8691 8692 8693 8694 8695 8696 8697
            if temp is not None:
                temp.generate_disposal_code(code)
                temp.free_temps(code)

        code.funcstate.release_temp(fmt_dict['i'])
        code.funcstate.release_temp(fmt_dict['nsteps'])

        self.release_closure_privates(code)

    def generate_loop(self, code, fmt_dict):
8698 8699 8700 8701
        if self.is_nested_prange:
            code.putln("#if 0")
        else:
            code.putln("#ifdef _OPENMP")
Mark Florisson's avatar
Mark Florisson committed
8702 8703 8704

        if not self.is_parallel:
            code.put("#pragma omp for")
8705
            self.privatization_insertion_point = code.insertion_point()
8706
            reduction_codepoint = self.parent.privatization_insertion_point
Mark Florisson's avatar
Mark Florisson committed
8707
        else:
8708 8709
            code.put("#pragma omp parallel")
            self.privatization_insertion_point = code.insertion_point()
8710
            reduction_codepoint = self.privatization_insertion_point
8711 8712 8713 8714 8715 8716 8717 8718
            code.putln("")
            code.putln("#endif /* _OPENMP */")

            code.begin_block() # pragma omp parallel begin block

            # Initialize the GIL if needed for this thread
            self.begin_parallel_block(code)

8719 8720 8721 8722
            if self.is_nested_prange:
                code.putln("#if 0")
            else:
                code.putln("#ifdef _OPENMP")
8723
            code.put("#pragma omp for")
Mark Florisson's avatar
Mark Florisson committed
8724

8725
        for entry, (op, lastprivate) in sorted(self.privates.items()):
Mark Florisson's avatar
Mark Florisson committed
8726
            # Don't declare the index variable as a reduction
8727
            if op and op in "+*-&^|" and entry != self.target.entry:
8728 8729 8730
                if entry.type.is_pyobject:
                    error(self.pos, "Python objects cannot be reductions")
                else:
8731 8732 8733 8734
                    #code.put(" reduction(%s:%s)" % (op, entry.cname))
                    # This is the only way reductions + nesting works in gcc4.5
                    reduction_codepoint.put(
                                " reduction(%s:%s)" % (op, entry.cname))
8735
            else:
8736 8737
                if entry == self.target.entry:
                    code.put(" firstprivate(%s)" % entry.cname)
8738 8739
                    code.put(" lastprivate(%s)" % entry.cname)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8740

8741
                if not entry.type.is_pyobject:
8742 8743 8744 8745
                    if lastprivate:
                        private = 'lastprivate'
                    else:
                        private = 'private'
Mark Florisson's avatar
Mark Florisson committed
8746

8747
                    code.put(" %s(%s)" % (private, entry.cname))
8748

Mark Florisson's avatar
Mark Florisson committed
8749
        if self.schedule:
8750
            if self.chunksize:
8751
                chunksize = ", %s" % self.evaluate_before_block(code, self.chunksize)
8752 8753 8754 8755
            else:
                chunksize = ""

            code.put(" schedule(%s%s)" % (self.schedule, chunksize))
8756

8757
        self.put_num_threads(reduction_codepoint)
8758

8759 8760
        code.putln("")
        code.putln("#endif /* _OPENMP */")
Mark Florisson's avatar
Mark Florisson committed
8761 8762

        code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
8763
        code.begin_block()  # for loop block
8764

8765
        guard_around_body_codepoint = code.insertion_point()
8766

8767 8768
        # Start if guard block around the body. This may be unnecessary, but
        # at least it doesn't spoil indentation
Mark Florisson's avatar
Mark Florisson committed
8769
        code.begin_block()
8770

8771
        code.putln("%(target)s = (%(target_type)s)(%(start)s + %(step)s * %(i)s);" % fmt_dict)
8772 8773
        self.initialize_privates_to_nan(code, exclude=self.target.entry)

8774 8775 8776
        if self.is_parallel:
            code.funcstate.start_collecting_temps()

Mark Florisson's avatar
Mark Florisson committed
8777
        self.body.generate_execution_code(code)
8778
        self.trap_parallel_exit(code, should_flush=True)
8779 8780
        self.privatize_temps(code)

8781 8782 8783
        if self.breaking_label_used:
            # Put a guard around the loop body in case return, break or
            # exceptions might be used
8784
            guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
8785

8786 8787
        code.end_block()  # end guard around loop body
        code.end_block()  # end for loop block
Mark Florisson's avatar
Mark Florisson committed
8788

8789 8790 8791
        if self.is_parallel:
            # Release the GIL and deallocate the thread state
            self.end_parallel_block(code)
8792
            code.end_block()  # pragma omp parallel end block
8793

8794

8795 8796 8797 8798 8799 8800 8801 8802
class CnameDecoratorNode(StatNode):
    """
    This node is for the cname decorator in CythonUtilityCode:

        @cname('the_cname')
        cdef func(...):
            ...

8803 8804
    In case of a cdef class the cname specifies the objstruct_cname.

8805 8806 8807 8808 8809 8810 8811 8812
    node        the node to which the cname decorator is applied
    cname       the cname the node should get
    """

    child_attrs = ['node']

    def analyse_declarations(self, env):
        self.node.analyse_declarations(env)
8813

8814 8815 8816 8817 8818
        node = self.node
        if isinstance(node, CompilerDirectivesNode):
            node = node.body.stats[0]

        self.is_function = isinstance(node, FuncDefNode)
8819
        is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode, CEnumDefNode))
8820
        e = node.entry
8821 8822 8823 8824

        if self.is_function:
            e.cname = self.cname
            e.func_cname = self.cname
8825
            e.used = True
8826 8827
            if e.pyfunc_cname and '.' in e.pyfunc_cname:
                e.pyfunc_cname = self.mangle(e.pyfunc_cname)
8828 8829
        elif is_struct_or_enum:
            e.cname = e.type.cname = self.cname
8830
        else:
8831
            scope = node.scope
8832 8833

            e.cname = self.cname
8834
            e.type.objstruct_cname = self.cname + '_obj'
8835
            e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
8836
            e.type.typeptr_cname = self.cname + '_type'
8837
            e.type.scope.namespace_cname = e.type.typeptr_cname
8838

8839
            e.as_variable.cname = e.type.typeptr_cname
8840 8841 8842

            scope.scope_prefix = self.cname + "_"

8843
            for name, entry in scope.entries.items():
8844
                if entry.func_cname:
8845 8846 8847 8848 8849 8850 8851 8852 8853
                    entry.func_cname = self.mangle(entry.cname)
                if entry.pyfunc_cname:
                    entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)

    def mangle(self, cname):
        if '.' in cname:
            # remove __pyx_base from func_cname
            cname = cname.split('.')[-1]
        return '%s_%s' % (self.cname, cname)
8854 8855

    def analyse_expressions(self, env):
8856 8857
        self.node = self.node.analyse_expressions(env)
        return self
8858 8859

    def generate_function_definitions(self, env, code):
8860
        "Ensure a prototype for every @cname method in the right place"
8861 8862 8863 8864 8865 8866
        if self.is_function and env.is_c_class_scope:
            # method in cdef class, generate a prototype in the header
            h_code = code.globalstate['utility_code_proto']

            if isinstance(self.node, DefNode):
                self.node.generate_function_header(
8867
                    h_code, with_pymethdef=False, proto_only=True)
8868
            else:
8869
                from . import ModuleNode
8870 8871 8872 8873 8874
                entry = self.node.entry
                cname = entry.cname
                entry.cname = entry.func_cname

                ModuleNode.generate_cfunction_declaration(
8875 8876 8877 8878
                    entry,
                    env.global_scope(),
                    h_code,
                    definition=True)
8879 8880 8881 8882 8883 8884 8885

                entry.cname = cname

        self.node.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
        self.node.generate_execution_code(code)
Mark Florisson's avatar
Mark Florisson committed
8886

8887

William Stein's avatar
William Stein committed
8888 8889 8890 8891 8892 8893
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

Robert Bradshaw's avatar
Robert Bradshaw committed
8894
if Options.gcc_branch_hints:
Stefan Behnel's avatar
Stefan Behnel committed
8895
    branch_prediction_macros = """
8896 8897 8898 8899 8900 8901
/* Test for GCC > 2.95 */
#if defined(__GNUC__) \
    && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  #define likely(x)   __builtin_expect(!!(x), 1)
  #define unlikely(x) __builtin_expect(!!(x), 0)
#else /* !__GNUC__ or GCC < 2.95 */
8902 8903
  #define likely(x)   (x)
  #define unlikely(x) (x)
8904
#endif /* __GNUC__ */
Stefan Behnel's avatar
Stefan Behnel committed
8905
"""
Robert Bradshaw's avatar
Robert Bradshaw committed
8906
else:
Stefan Behnel's avatar
Stefan Behnel committed
8907
    branch_prediction_macros = """
Robert Bradshaw's avatar
Robert Bradshaw committed
8908 8909
#define likely(x)   (x)
#define unlikely(x) (x)
Stefan Behnel's avatar
Stefan Behnel committed
8910
"""
William Stein's avatar
William Stein committed
8911 8912 8913

#------------------------------------------------------------------------------------

8914 8915
printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
8916

William Stein's avatar
William Stein committed
8917 8918
#------------------------------------------------------------------------------------

8919 8920 8921 8922 8923 8924 8925
# Exception raising code
#
# Exceptions are raised by __Pyx_Raise() and stored as plain
# type/value/tb in PyThreadState->curexc_*.  When being caught by an
# 'except' statement, curexc_* is moved over to exc_* by
# __Pyx_GetException()

8926 8927 8928 8929 8930
restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
8931
traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
8932 8933 8934

#------------------------------------------------------------------------------------

8935 8936
get_exception_tuple_utility_code = UtilityCode(
    proto="""
8937
static PyObject *__Pyx_GetExceptionTuple(PyThreadState *__pyx_tstate); /*proto*/
8938
""",
8939 8940 8941 8942 8943
    # I doubt that calling __Pyx_GetException() here is correct as it moves
    # the exception from tstate->curexc_* to tstate->exc_*, which prevents
    # exception handlers later on from receiving it.
    # NOTE: "__pyx_tstate" may be used by __Pyx_GetException() macro
    impl = """
8944
static PyObject *__Pyx_GetExceptionTuple(CYTHON_UNUSED PyThreadState *__pyx_tstate) {
8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960
    PyObject *type = NULL, *value = NULL, *tb = NULL;
    if (__Pyx_GetException(&type, &value, &tb) == 0) {
        PyObject* exc_info = PyTuple_New(3);
        if (exc_info) {
            Py_INCREF(type);
            Py_INCREF(value);
            Py_INCREF(tb);
            PyTuple_SET_ITEM(exc_info, 0, type);
            PyTuple_SET_ITEM(exc_info, 1, value);
            PyTuple_SET_ITEM(exc_info, 2, tb);
            return exc_info;
        }
    }
    return NULL;
}
""",
8961
    requires=[get_exception_utility_code])