ExprNodes.py 379 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes for expressions
William Stein's avatar
William Stein committed
3 4
#

5 6
import cython
cython.declare(error=object, warning=object, warn_once=object, InternalError=object,
7 8
               CompileError=object, UtilityCode=object, TempitaUtilityCode=object,
               StringEncoding=object, operator=object,
9
               Naming=object, Nodes=object, PyrexTypes=object, py_object_type=object,
Stefan Behnel's avatar
Stefan Behnel committed
10
               list_type=object, tuple_type=object, set_type=object, dict_type=object,
11 12 13 14
               unicode_type=object, str_type=object, bytes_type=object, type_type=object,
               Builtin=object, Symtab=object, Utils=object, find_coercion_error=object,
               debug_disposal_code=object, debug_temp_alloc=object, debug_coercion=object)

15
import sys
16
import copy
17
import operator
William Stein's avatar
William Stein committed
18

19
from Errors import error, warning, warn_once, InternalError, CompileError
20
from Errors import hold_errors, release_errors, held_errors, report_error
21
from Code import UtilityCode, TempitaUtilityCode
22
import StringEncoding
William Stein's avatar
William Stein committed
23
import Naming
Robert Bradshaw's avatar
Robert Bradshaw committed
24
import Nodes
William Stein's avatar
William Stein committed
25 26
from Nodes import Node
import PyrexTypes
27
from PyrexTypes import py_object_type, c_long_type, typecast, error_type, \
28
     unspecified_type, cython_memoryview_ptr_type
29
import TypeSlots
30 31
from Builtin import list_type, tuple_type, set_type, dict_type, \
     unicode_type, str_type, bytes_type, type_type
32
import Builtin
William Stein's avatar
William Stein committed
33 34
import Symtab
import Options
35
from Cython import Utils
36
from Annotate import AnnotationItem
William Stein's avatar
William Stein committed
37

William Stein's avatar
William Stein committed
38
from Cython.Debugging import print_call_chain
William Stein's avatar
William Stein committed
39 40 41
from DebugFlags import debug_disposal_code, debug_temp_alloc, \
    debug_coercion

42 43 44 45 46
try:
    from __builtin__ import basestring
except ImportError:
    basestring = str # Python 3

Stefan Behnel's avatar
Stefan Behnel committed
47
class NotConstant(object):
48 49 50 51 52 53 54 55
    _obj = None

    def __new__(cls):
        if NotConstant._obj is None:
            NotConstant._obj = super(NotConstant, cls).__new__(cls)

        return NotConstant._obj

Stefan Behnel's avatar
Stefan Behnel committed
56 57 58
    def __repr__(self):
        return "<NOT CONSTANT>"

59
not_a_constant = NotConstant()
60
constant_value_not_set = object()
61

62 63 64 65 66 67 68 69 70 71
# error messages when coercing from key[0] to key[1]
find_coercion_error = {
    # string related errors
    (Builtin.unicode_type, Builtin.bytes_type) : "Cannot convert Unicode string to 'bytes' implicitly, encoding required.",
    (Builtin.unicode_type, Builtin.str_type)   : "Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.",
    (Builtin.unicode_type, PyrexTypes.c_char_ptr_type) : "Unicode objects do not support coercion to C types.",
    (Builtin.bytes_type, Builtin.unicode_type) : "Cannot convert 'bytes' object to unicode implicitly, decoding required",
    (Builtin.bytes_type, Builtin.str_type) : "Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.",
    (Builtin.str_type, Builtin.unicode_type) : "str objects do not support coercion to unicode, use a unicode string literal instead (u'')",
    (Builtin.str_type, Builtin.bytes_type) : "Cannot convert 'str' to 'bytes' implicitly. This is not portable.",
72
    (Builtin.str_type, PyrexTypes.c_char_ptr_type) : "'str' objects do not support coercion to C types (use 'bytes'?).",
73 74 75 76 77
    (PyrexTypes.c_char_ptr_type, Builtin.unicode_type) : "Cannot convert 'char*' to unicode implicitly, decoding required",
    (PyrexTypes.c_uchar_ptr_type, Builtin.unicode_type) : "Cannot convert 'char*' to unicode implicitly, decoding required",
    }.get


William Stein's avatar
William Stein committed
78 79 80 81 82 83
class ExprNode(Node):
    #  subexprs     [string]     Class var holding names of subexpr node attrs
    #  type         PyrexType    Type of the result
    #  result_code  string       Code fragment
    #  result_ctype string       C type of result_code if different from type
    #  is_temp      boolean      Result is in a temporary variable
84
    #  is_sequence_constructor
William Stein's avatar
William Stein committed
85
    #               boolean      Is a list or tuple constructor expression
86
    #  is_starred   boolean      Is a starred expression (e.g. '*a')
William Stein's avatar
William Stein committed
87 88 89
    #  saved_subexpr_nodes
    #               [ExprNode or [ExprNode or None] or None]
    #                            Cached result of subexpr_nodes()
90
    #  use_managed_ref boolean   use ref-counted temps/assignments/etc.
91 92 93
    #  result_is_used  boolean   indicates that the result will be dropped and the
    #                            result_code/temp_result can safely be set to None

William Stein's avatar
William Stein committed
94
    result_ctype = None
95
    type = None
96 97
    temp_code = None
    old_temp = None # error checker for multiple frees etc.
98
    use_managed_ref = True # can be set by optimisation transforms
99
    result_is_used = True
William Stein's avatar
William Stein committed
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126

    #  The Analyse Expressions phase for expressions is split
    #  into two sub-phases:
    #
    #    Analyse Types
    #      Determines the result type of the expression based
    #      on the types of its sub-expressions, and inserts
    #      coercion nodes into the expression tree where needed.
    #      Marks nodes which will need to have temporary variables
    #      allocated.
    #
    #    Allocate Temps
    #      Allocates temporary variables where needed, and fills
    #      in the result_code field of each node.
    #
    #  ExprNode provides some convenience routines which
    #  perform both of the above phases. These should only
    #  be called from statement nodes, and only when no
    #  coercion nodes need to be added around the expression
    #  being analysed. In that case, the above two phases
    #  should be invoked separately.
    #
    #  Framework code in ExprNode provides much of the common
    #  processing for the various phases. It makes use of the
    #  'subexprs' class attribute of ExprNodes, which should
    #  contain a list of the names of attributes which can
    #  hold sub-nodes or sequences of sub-nodes.
127 128
    #
    #  The framework makes use of a number of abstract methods.
William Stein's avatar
William Stein committed
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
    #  Their responsibilities are as follows.
    #
    #    Declaration Analysis phase
    #
    #      analyse_target_declaration
    #        Called during the Analyse Declarations phase to analyse
    #        the LHS of an assignment or argument of a del statement.
    #        Nodes which cannot be the LHS of an assignment need not
    #        implement it.
    #
    #    Expression Analysis phase
    #
    #      analyse_types
    #        - Call analyse_types on all sub-expressions.
    #        - Check operand types, and wrap coercion nodes around
    #          sub-expressions where needed.
    #        - Set the type of this node.
    #        - If a temporary variable will be required for the
    #          result, set the is_temp flag of this node.
    #
    #      analyse_target_types
    #        Called during the Analyse Types phase to analyse
151
    #        the LHS of an assignment or argument of a del
William Stein's avatar
William Stein committed
152 153
    #        statement. Similar responsibilities to analyse_types.
    #
154 155 156 157
    #      target_code
    #        Called by the default implementation of allocate_target_temps.
    #        Should return a C lvalue for assigning to the node. The default
    #        implementation calls calculate_result_code.
William Stein's avatar
William Stein committed
158 159 160 161
    #
    #      check_const
    #        - Check that this node and its subnodes form a
    #          legal constant expression. If so, do nothing,
162
    #          otherwise call not_const.
William Stein's avatar
William Stein committed
163
    #
164
    #        The default implementation of check_const
William Stein's avatar
William Stein committed
165 166 167 168 169 170 171 172
    #        assumes that the expression is not constant.
    #
    #      check_const_addr
    #        - Same as check_const, except check that the
    #          expression is a C lvalue whose address is
    #          constant. Otherwise, call addr_not_const.
    #
    #        The default implementation of calc_const_addr
173
    #        assumes that the expression is not a constant
William Stein's avatar
William Stein committed
174 175 176 177 178 179 180 181 182 183 184 185
    #        lvalue.
    #
    #   Code Generation phase
    #
    #      generate_evaluation_code
    #        - Call generate_evaluation_code for sub-expressions.
    #        - Perform the functions of generate_result_code
    #          (see below).
    #        - If result is temporary, call generate_disposal_code
    #          on all sub-expressions.
    #
    #        A default implementation of generate_evaluation_code
186
    #        is provided which uses the following abstract methods:
William Stein's avatar
William Stein committed
187 188 189 190 191 192
    #
    #          generate_result_code
    #            - Generate any C statements necessary to calculate
    #              the result of this node from the results of its
    #              sub-expressions.
    #
193
    #          calculate_result_code
194 195
    #            - Should return a C code fragment evaluating to the
    #              result. This is only called when the result is not
196 197
    #              a temporary.
    #
William Stein's avatar
William Stein committed
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
    #      generate_assignment_code
    #        Called on the LHS of an assignment.
    #        - Call generate_evaluation_code for sub-expressions.
    #        - Generate code to perform the assignment.
    #        - If the assignment absorbed a reference, call
    #          generate_post_assignment_code on the RHS,
    #          otherwise call generate_disposal_code on it.
    #
    #      generate_deletion_code
    #        Called on an argument of a del statement.
    #        - Call generate_evaluation_code for sub-expressions.
    #        - Generate code to perform the deletion.
    #        - Call generate_disposal_code on all sub-expressions.
    #
    #
213

William Stein's avatar
William Stein committed
214
    is_sequence_constructor = 0
215
    is_string_literal = 0
William Stein's avatar
William Stein committed
216
    is_attribute = 0
217

William Stein's avatar
William Stein committed
218 219
    saved_subexpr_nodes = None
    is_temp = 0
220
    is_target = 0
221
    is_starred = 0
William Stein's avatar
William Stein committed
222

223 224
    constant_result = constant_value_not_set

225 226 227
    # whether this node with a memoryview type should be broadcast
    memslice_broadcast = False

228 229 230 231
    try:
        _get_child_attrs = operator.attrgetter('subexprs')
    except AttributeError:
        # Python 2.3
232
        def __get_child_attrs(self):
233
            return self.subexprs
234
        _get_child_attrs = __get_child_attrs
235
    child_attrs = property(fget=_get_child_attrs)
236

William Stein's avatar
William Stein committed
237 238 239 240
    def not_implemented(self, method_name):
        print_call_chain(method_name, "not implemented") ###
        raise InternalError(
            "%s.%s not implemented" %
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
241
                (self.__class__.__name__, method_name))
242

William Stein's avatar
William Stein committed
243 244
    def is_lvalue(self):
        return 0
245

246 247 248
    def is_addressable(self):
        return self.is_lvalue()

William Stein's avatar
William Stein committed
249 250 251 252 253 254 255 256 257 258 259
    def is_ephemeral(self):
        #  An ephemeral node is one whose result is in
        #  a Python temporary and we suspect there are no
        #  other references to it. Certain operations are
        #  disallowed on such values, since they are
        #  likely to result in a dangling pointer.
        return self.type.is_pyobject and self.is_temp

    def subexpr_nodes(self):
        #  Extract a list of subexpression nodes based
        #  on the contents of the subexprs class attribute.
260 261 262
        nodes = []
        for name in self.subexprs:
            item = getattr(self, name)
Stefan Behnel's avatar
Stefan Behnel committed
263 264
            if item is not None:
                if type(item) is list:
265
                    nodes.extend(item)
Stefan Behnel's avatar
Stefan Behnel committed
266 267
                else:
                    nodes.append(item)
268
        return nodes
269

270
    def result(self):
271 272 273
        if self.is_temp:
            return self.temp_code
        else:
274
            return self.calculate_result_code()
275

William Stein's avatar
William Stein committed
276 277
    def result_as(self, type = None):
        #  Return the result code cast to the specified C type.
278
        return typecast(type, self.ctype(), self.result())
279

William Stein's avatar
William Stein committed
280 281 282
    def py_result(self):
        #  Return the result code cast to PyObject *.
        return self.result_as(py_object_type)
283

William Stein's avatar
William Stein committed
284 285 286 287
    def ctype(self):
        #  Return the native C type of the result (i.e. the
        #  C type of the result_code expression).
        return self.result_ctype or self.type
288

289
    def get_constant_c_result_code(self):
290
        # Return the constant value of this node as a result code
291 292 293 294 295 296 297
        # string, or None if the node is not constant.  This method
        # can be called when the constant result code is required
        # before the code generation phase.
        #
        # The return value is a string that can represent a simple C
        # value, a constant C name or a constant C expression.  If the
        # node type depends on Python code, this must return None.
298 299
        return None

300
    def calculate_constant_result(self):
301 302 303 304 305
        # Calculate the constant compile time result value of this
        # expression and store it in ``self.constant_result``.  Does
        # nothing by default, thus leaving ``self.constant_result``
        # unknown.  If valid, the result can be an arbitrary Python
        # value.
306 307 308 309 310 311
        #
        # This must only be called when it is assured that all
        # sub-expressions have a valid constant_result value.  The
        # ConstantFolding transform will do this.
        pass

312 313 314 315
    def has_constant_result(self):
        return self.constant_result is not constant_value_not_set and \
               self.constant_result is not not_a_constant

316 317 318
    def compile_time_value(self, denv):
        #  Return value of compile-time expression, or report error.
        error(self.pos, "Invalid compile-time expression")
319

320 321 322
    def compile_time_value_error(self, e):
        error(self.pos, "Error in compile-time expression: %s: %s" % (
            e.__class__.__name__, e))
323

William Stein's avatar
William Stein committed
324
    # ------------- Declaration Analysis ----------------
325

William Stein's avatar
William Stein committed
326 327
    def analyse_target_declaration(self, env):
        error(self.pos, "Cannot assign to or delete this")
328

William Stein's avatar
William Stein committed
329
    # ------------- Expression Analysis ----------------
330

William Stein's avatar
William Stein committed
331 332 333 334 335 336
    def analyse_const_expression(self, env):
        #  Called during the analyse_declarations phase of a
        #  constant expression. Analyses the expression's type,
        #  checks whether it is a legal const expression,
        #  and determines its value.
        self.analyse_types(env)
337
        return self.check_const()
338

William Stein's avatar
William Stein committed
339 340
    def analyse_expressions(self, env):
        #  Convenience routine performing both the Type
341
        #  Analysis and Temp Allocation phases for a whole
William Stein's avatar
William Stein committed
342 343
        #  expression.
        self.analyse_types(env)
344

345
    def analyse_target_expression(self, env, rhs):
William Stein's avatar
William Stein committed
346 347 348 349
        #  Convenience routine performing both the Type
        #  Analysis and Temp Allocation phases for the LHS of
        #  an assignment.
        self.analyse_target_types(env)
350

William Stein's avatar
William Stein committed
351 352 353 354 355
    def analyse_boolean_expression(self, env):
        #  Analyse expression and coerce to a boolean.
        self.analyse_types(env)
        bool = self.coerce_to_boolean(env)
        return bool
356

William Stein's avatar
William Stein committed
357 358 359 360 361 362 363 364 365
    def analyse_temp_boolean_expression(self, env):
        #  Analyse boolean expression and coerce result into
        #  a temporary. This is used when a branch is to be
        #  performed on the result and we won't have an
        #  opportunity to ensure disposal code is executed
        #  afterwards. By forcing the result into a temporary,
        #  we ensure that all disposal has been done by the
        #  time we get the result.
        self.analyse_types(env)
Stefan Behnel's avatar
Stefan Behnel committed
366 367
        return self.coerce_to_boolean(env).coerce_to_simple(env)

368
    # --------------- Type Inference -----------------
369

Robert Bradshaw's avatar
Robert Bradshaw committed
370
    def type_dependencies(self, env):
371 372 373 374
        # Returns the list of entries whose types must be determined
        # before the type of self can be infered.
        if hasattr(self, 'type') and self.type is not None:
            return ()
Robert Bradshaw's avatar
Robert Bradshaw committed
375
        return sum([node.type_dependencies(env) for node in self.subexpr_nodes()], ())
376

377
    def infer_type(self, env):
378 379
        # Attempt to deduce the type of self.
        # Differs from analyse_types as it avoids unnecessary
380 381 382 383 384 385 386 387
        # analysis of subexpressions, but can assume everything
        # in self.type_dependencies() has been resolved.
        if hasattr(self, 'type') and self.type is not None:
            return self.type
        elif hasattr(self, 'entry') and self.entry is not None:
            return self.entry.type
        else:
            self.not_implemented("infer_type")
388

389 390 391
    def nonlocally_immutable(self):
        # Returns whether this variable is a safe reference, i.e.
        # can't be modified as part of globals or closures.
392
        return self.is_temp or self.type.is_array or self.type.is_cfunction
393

William Stein's avatar
William Stein committed
394
    # --------------- Type Analysis ------------------
395

William Stein's avatar
William Stein committed
396 397 398 399
    def analyse_as_module(self, env):
        # If this node can be interpreted as a reference to a
        # cimported module, return its scope, else None.
        return None
400

401 402 403 404
    def analyse_as_type(self, env):
        # If this node can be interpreted as a reference to a
        # type, return that type, else None.
        return None
405

William Stein's avatar
William Stein committed
406 407 408 409
    def analyse_as_extension_type(self, env):
        # If this node can be interpreted as a reference to an
        # extension type, return its type, else None.
        return None
410

William Stein's avatar
William Stein committed
411 412
    def analyse_types(self, env):
        self.not_implemented("analyse_types")
413

William Stein's avatar
William Stein committed
414 415
    def analyse_target_types(self, env):
        self.analyse_types(env)
416

417
    def nogil_check(self, env):
418 419 420
        # By default, any expression based on Python objects is
        # prevented in nogil environments.  Subtypes must override
        # this if they can work without the GIL.
421
        if self.type and self.type.is_pyobject:
422
            self.gil_error()
423

424 425 426 427
    def gil_assignment_check(self, env):
        if env.nogil and self.type.is_pyobject:
            error(self.pos, "Assignment of Python object not allowed without gil")

William Stein's avatar
William Stein committed
428 429
    def check_const(self):
        self.not_const()
430
        return False
431

William Stein's avatar
William Stein committed
432 433
    def not_const(self):
        error(self.pos, "Not allowed in a constant expression")
434

William Stein's avatar
William Stein committed
435 436
    def check_const_addr(self):
        self.addr_not_const()
437
        return False
438

William Stein's avatar
William Stein committed
439 440
    def addr_not_const(self):
        error(self.pos, "Address is not constant")
441

William Stein's avatar
William Stein committed
442
    # ----------------- Result Allocation -----------------
443

William Stein's avatar
William Stein committed
444 445 446 447 448 449
    def result_in_temp(self):
        #  Return true if result is in a temporary owned by
        #  this node or one of its subexpressions. Overridden
        #  by certain nodes which can share the result of
        #  a subnode.
        return self.is_temp
450

William Stein's avatar
William Stein committed
451 452 453
    def target_code(self):
        #  Return code fragment for use as LHS of a C assignment.
        return self.calculate_result_code()
454

William Stein's avatar
William Stein committed
455 456
    def calculate_result_code(self):
        self.not_implemented("calculate_result_code")
457

Robert Bradshaw's avatar
Robert Bradshaw committed
458 459 460
#    def release_target_temp(self, env):
#        #  Release temporaries used by LHS of an assignment.
#        self.release_subexpr_temps(env)
William Stein's avatar
William Stein committed
461

462 463
    def allocate_temp_result(self, code):
        if self.temp_code:
464
            raise RuntimeError("Temp allocated multiple times in %r: %r" % (self.__class__.__name__, self.pos))
465 466 467 468 469
        type = self.type
        if not type.is_void:
            if type.is_pyobject:
                type = PyrexTypes.py_object_type
            self.temp_code = code.funcstate.allocate_temp(
470
                type, manage_ref=self.use_managed_ref)
471 472 473 474 475
        else:
            self.temp_code = None

    def release_temp_result(self, code):
        if not self.temp_code:
476 477 478
            if not self.result_is_used:
                # not used anyway, so ignore if not set up
                return
479 480 481 482 483 484 485 486 487 488
            if self.old_temp:
                raise RuntimeError("temp %s released multiple times in %s" % (
                        self.old_temp, self.__class__.__name__))
            else:
                raise RuntimeError("no temp, but release requested in %s" % (
                        self.__class__.__name__))
        code.funcstate.release_temp(self.temp_code)
        self.old_temp = self.temp_code
        self.temp_code = None

William Stein's avatar
William Stein committed
489
    # ---------------- Code Generation -----------------
490

William Stein's avatar
William Stein committed
491
    def make_owned_reference(self, code):
492 493 494 495
        """
        If result is a pyobject, make sure we own a reference to it.
        If the result is in a temp, it is already a new reference.
        """
William Stein's avatar
William Stein committed
496
        if self.type.is_pyobject and not self.result_in_temp():
497
            code.put_incref(self.result(), self.ctype())
498

499 500 501 502 503 504 505 506
    def make_owned_memoryviewslice(self, code):
        """
        Make sure we own the reference to this memoryview slice.
        """
        if not self.result_in_temp():
            code.put_incref_memoryviewslice(self.result(),
                                            have_gil=self.in_nogil_context)

William Stein's avatar
William Stein committed
507
    def generate_evaluation_code(self, code):
508
        code.mark_pos(self.pos)
509

William Stein's avatar
William Stein committed
510 511 512 513
        #  Generate code to evaluate this node and
        #  its sub-expressions, and dispose of any
        #  temporary results of its sub-expressions.
        self.generate_subexpr_evaluation_code(code)
514 515 516 517

        if self.is_temp:
            self.allocate_temp_result(code)

William Stein's avatar
William Stein committed
518 519
        self.generate_result_code(code)
        if self.is_temp:
520 521
            # If we are temp we do not need to wait until this node is disposed
            # before disposing children.
William Stein's avatar
William Stein committed
522
            self.generate_subexpr_disposal_code(code)
523
            self.free_subexpr_temps(code)
524

William Stein's avatar
William Stein committed
525 526 527
    def generate_subexpr_evaluation_code(self, code):
        for node in self.subexpr_nodes():
            node.generate_evaluation_code(code)
528

William Stein's avatar
William Stein committed
529 530
    def generate_result_code(self, code):
        self.not_implemented("generate_result_code")
531

532 533
    def generate_disposal_code(self, code):
        if self.is_temp:
534 535 536 537 538 539
            if self.result():
                if self.type.is_pyobject:
                    code.put_decref_clear(self.result(), self.ctype())
                elif self.type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(
                            self.result(), have_gil=not self.in_nogil_context)
William Stein's avatar
William Stein committed
540
        else:
541
            # Already done if self.is_temp
542
            self.generate_subexpr_disposal_code(code)
543

William Stein's avatar
William Stein committed
544 545 546 547 548
    def generate_subexpr_disposal_code(self, code):
        #  Generate code to dispose of temporary results
        #  of all sub-expressions.
        for node in self.subexpr_nodes():
            node.generate_disposal_code(code)
549

William Stein's avatar
William Stein committed
550 551 552
    def generate_post_assignment_code(self, code):
        if self.is_temp:
            if self.type.is_pyobject:
553
                code.putln("%s = 0;" % self.result())
554 555 556
            elif self.type.is_memoryviewslice:
                code.putln("%s.memview = NULL;" % self.result())
                code.putln("%s.data = NULL;" % self.result())
William Stein's avatar
William Stein committed
557 558
        else:
            self.generate_subexpr_disposal_code(code)
559

William Stein's avatar
William Stein committed
560 561
    def generate_assignment_code(self, rhs, code):
        #  Stub method for nodes which are not legal as
562
        #  the LHS of an assignment. An error will have
William Stein's avatar
William Stein committed
563 564
        #  been reported earlier.
        pass
565

William Stein's avatar
William Stein committed
566 567 568 569 570
    def generate_deletion_code(self, code):
        #  Stub method for nodes that are not legal as
        #  the argument of a del statement. An error
        #  will have been reported earlier.
        pass
571 572

    def free_temps(self, code):
573 574 575 576
        if self.is_temp:
            if not self.type.is_void:
                self.release_temp_result(code)
        else:
577
            self.free_subexpr_temps(code)
578

579 580 581 582
    def free_subexpr_temps(self, code):
        for sub in self.subexpr_nodes():
            sub.free_temps(code)

583 584 585
    def generate_function_definitions(self, env, code):
        pass

586
    # ---------------- Annotation ---------------------
587

588 589 590
    def annotate(self, code):
        for node in self.subexpr_nodes():
            node.annotate(code)
591

William Stein's avatar
William Stein committed
592
    # ----------------- Coercion ----------------------
593

William Stein's avatar
William Stein committed
594 595 596 597 598 599 600 601
    def coerce_to(self, dst_type, env):
        #   Coerce the result so that it can be assigned to
        #   something of type dst_type. If processing is necessary,
        #   wraps this node in a coercion node and returns that.
        #   Otherwise, returns this node unchanged.
        #
        #   This method is called during the analyse_expressions
        #   phase of the src_node's processing.
602 603 604 605 606 607 608 609
        #
        #   Note that subclasses that override this (especially
        #   ConstNodes) must not (re-)set their own .type attribute
        #   here.  Since expression nodes may turn up in different
        #   places in the tree (e.g. inside of CloneNodes in cascaded
        #   assignments), this method must return a new node instance
        #   if it changes the type.
        #
William Stein's avatar
William Stein committed
610 611 612 613
        src = self
        src_type = self.type
        src_is_py_type = src_type.is_pyobject
        dst_is_py_type = dst_type.is_pyobject
614

615 616 617
        if self.check_for_coercion_error(dst_type):
            return self

618
        if dst_type.is_reference and not src_type.is_reference:
619
            dst_type = dst_type.ref_base_type
620

621
        if src_type.is_fused or dst_type.is_fused:
622 623 624 625 626 627 628
            # See if we are coercing a fused function to a pointer to a
            # specialized function
            if (src_type.is_cfunction and not dst_type.is_fused and
                    dst_type.is_ptr and dst_type.base_type.is_cfunction):

                dst_type = dst_type.base_type

629
                for signature in src_type.get_all_specialized_function_types():
630
                    if signature.same_as(dst_type):
Mark Florisson's avatar
Mark Florisson committed
631 632 633 634
                        src.type = signature
                        src.entry = src.type.entry
                        src.entry.used = True
                        return self
635

636
            if src_type.is_fused:
Mark Florisson's avatar
Mark Florisson committed
637
                error(self.pos, "Type is not specialized")
638 639 640
            else:
                error(self.pos, "Cannot coerce to a type that is not specialized")

641 642 643
            self.type = error_type
            return self

644 645 646 647 648
        if self.coercion_type is not None:
            # This is purely for error checking purposes!
            node = NameNode(self.pos, name='', type=self.coercion_type)
            node.coerce_to(dst_type, env)

649
        if dst_type.is_memoryviewslice:
650
            import MemoryView
651
            if not src.type.is_memoryviewslice:
652 653
                if src.type.is_pyobject:
                    src = CoerceToMemViewSliceNode(src, dst_type, env)
654 655 656
                elif src.type.is_array:
                    src = CythonArrayNode.from_carray(src, env).coerce_to(
                                                            dst_type, env)
657
                elif not src_type.is_error:
658 659 660
                    error(self.pos,
                          "Cannot convert '%s' to memoryviewslice" %
                                                                (src_type,))
661 662
            elif not MemoryView.src_conforms_to_dst(
                        src.type, dst_type, broadcast=self.memslice_broadcast):
663 664 665 666 667 668 669 670
                if src.type.dtype.same_as(dst_type.dtype):
                    msg = "Memoryview '%s' not conformable to memoryview '%s'."
                    tup = src.type, dst_type
                else:
                    msg = "Different base types for memoryviews (%s, %s)"
                    tup = src.type.dtype, dst_type.dtype

                error(self.pos, msg % tup)
671

672
        elif dst_type.is_pyobject:
William Stein's avatar
William Stein committed
673
            if not src.type.is_pyobject:
674 675 676 677
                if dst_type is bytes_type and src.type.is_int:
                    src = CoerceIntToBytesNode(src, env)
                else:
                    src = CoerceToPyTypeNode(src, env)
William Stein's avatar
William Stein committed
678
            if not src.type.subtype_of(dst_type):
679 680
                if not isinstance(src, NoneNode):
                    src = PyTypeTestNode(src, dst_type, env)
William Stein's avatar
William Stein committed
681 682
        elif src.type.is_pyobject:
            src = CoerceFromPyTypeNode(dst_type, src, env)
683
        elif (dst_type.is_complex
684 685
              and src_type != dst_type
              and dst_type.assignable_from(src_type)):
686
            src = CoerceToComplexNode(src, dst_type, env)
William Stein's avatar
William Stein committed
687
        else: # neither src nor dst are py types
688
            # Added the string comparison, since for c types that
689
            # is enough, but Cython gets confused when the types are
690
            # in different pxi files.
691
            if not (str(src.type) == str(dst_type) or dst_type.assignable_from(src_type)):
692
                self.fail_assignment(dst_type)
William Stein's avatar
William Stein committed
693 694
        return src

695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
    def fail_assignment(self, dst_type):
        error(self.pos, "Cannot assign type '%s' to '%s'" % (self.type, dst_type))

    def check_for_coercion_error(self, dst_type, fail=False, default=None):
        if fail and not default:
            default = "Cannot assign type '%(FROM)s' to '%(TO)s'"
        message = find_coercion_error((self.type, dst_type), default)
        if message is not None:
            error(self.pos, message % {'FROM': self.type, 'TO': dst_type})
            return True
        if fail:
            self.fail_assignment(dst_type)
            return True
        return False

William Stein's avatar
William Stein committed
710 711 712 713 714 715
    def coerce_to_pyobject(self, env):
        return self.coerce_to(PyrexTypes.py_object_type, env)

    def coerce_to_boolean(self, env):
        #  Coerce result to something acceptable as
        #  a boolean value.
716 717 718 719 720 721 722

        # if it's constant, calculate the result now
        if self.has_constant_result():
            bool_value = bool(self.constant_result)
            return BoolNode(self.pos, value=bool_value,
                            constant_result=bool_value)

William Stein's avatar
William Stein committed
723 724 725 726
        type = self.type
        if type.is_pyobject or type.is_ptr or type.is_float:
            return CoerceToBooleanNode(self, env)
        else:
727
            if not (type.is_int or type.is_enum or type.is_error):
728
                error(self.pos,
William Stein's avatar
William Stein committed
729 730
                    "Type '%s' not acceptable as a boolean" % type)
            return self
731

William Stein's avatar
William Stein committed
732 733 734 735 736 737
    def coerce_to_integer(self, env):
        # If not already some C integer type, coerce to longint.
        if self.type.is_int:
            return self
        else:
            return self.coerce_to(PyrexTypes.c_long_type, env)
738

William Stein's avatar
William Stein committed
739 740 741 742 743 744
    def coerce_to_temp(self, env):
        #  Ensure that the result is in a temporary.
        if self.result_in_temp():
            return self
        else:
            return CoerceToTempNode(self, env)
745

William Stein's avatar
William Stein committed
746 747 748 749 750 751
    def coerce_to_simple(self, env):
        #  Ensure that the result is simple (see is_simple).
        if self.is_simple():
            return self
        else:
            return self.coerce_to_temp(env)
752

William Stein's avatar
William Stein committed
753 754 755 756 757 758
    def is_simple(self):
        #  A node is simple if its result is something that can
        #  be referred to without performing any operations, e.g.
        #  a constant, local var, C global var, struct member
        #  reference, or temporary.
        return self.result_in_temp()
759 760

    def may_be_none(self):
761
        if self.type and not self.type.is_pyobject:
762 763 764 765
            return False
        if self.constant_result not in (not_a_constant, constant_value_not_set):
            return self.constant_result is not None
        return True
766

767
    def as_cython_attribute(self):
768
        return None
William Stein's avatar
William Stein committed
769

770
    def as_none_safe_node(self, message, error="PyExc_TypeError", format_args=()):
771 772 773
        # Wraps the node in a NoneCheckNode if it is not known to be
        # not-None (e.g. because it is a Python literal).
        if self.may_be_none():
774
            return NoneCheckNode(self, error, message, format_args)
775 776 777 778
        else:
            return self


William Stein's avatar
William Stein committed
779
class AtomicExprNode(ExprNode):
780 781
    #  Abstract base class for expression nodes which have
    #  no sub-expressions.
782

783 784 785
    subexprs = []

    # Override to optimize -- we know we have no children
786 787 788 789
    def generate_subexpr_evaluation_code(self, code):
        pass
    def generate_subexpr_disposal_code(self, code):
        pass
790

791
class PyConstNode(AtomicExprNode):
William Stein's avatar
William Stein committed
792
    #  Abstract base class for constant Python values.
793

794
    is_literal = 1
795
    type = py_object_type
796

William Stein's avatar
William Stein committed
797 798
    def is_simple(self):
        return 1
799 800 801 802

    def may_be_none(self):
        return False

William Stein's avatar
William Stein committed
803
    def analyse_types(self, env):
804
        pass
805

William Stein's avatar
William Stein committed
806 807 808 809 810 811 812 813 814
    def calculate_result_code(self):
        return self.value

    def generate_result_code(self, code):
        pass


class NoneNode(PyConstNode):
    #  The constant value None
815

816
    is_none = 1
William Stein's avatar
William Stein committed
817
    value = "Py_None"
818 819

    constant_result = None
820

821
    nogil_check = None
822

823 824
    def compile_time_value(self, denv):
        return None
825 826 827 828 829

    def may_be_none(self):
        return True


William Stein's avatar
William Stein committed
830 831
class EllipsisNode(PyConstNode):
    #  '...' in a subscript list.
832

William Stein's avatar
William Stein committed
833 834
    value = "Py_Ellipsis"

835 836
    constant_result = Ellipsis

837 838 839
    def compile_time_value(self, denv):
        return Ellipsis

William Stein's avatar
William Stein committed
840

841
class ConstNode(AtomicExprNode):
William Stein's avatar
William Stein committed
842 843 844
    # Abstract base type for literal constant nodes.
    #
    # value     string      C code fragment
845

William Stein's avatar
William Stein committed
846
    is_literal = 1
847
    nogil_check = None
848

William Stein's avatar
William Stein committed
849 850
    def is_simple(self):
        return 1
851

852 853 854
    def nonlocally_immutable(self):
        return 1

855 856 857
    def may_be_none(self):
        return False

William Stein's avatar
William Stein committed
858 859
    def analyse_types(self, env):
        pass # Types are held in class variables
860

William Stein's avatar
William Stein committed
861
    def check_const(self):
862
        return True
863

864
    def get_constant_c_result_code(self):
865 866
        return self.calculate_result_code()

William Stein's avatar
William Stein committed
867 868 869 870 871 872 873
    def calculate_result_code(self):
        return str(self.value)

    def generate_result_code(self, code):
        pass


874 875 876
class BoolNode(ConstNode):
    type = PyrexTypes.c_bint_type
    #  The constant value True or False
877 878 879 880

    def calculate_constant_result(self):
        self.constant_result = self.value

881 882
    def compile_time_value(self, denv):
        return self.value
883

884
    def calculate_result_code(self):
885
        return str(int(self.value))
886

887

William Stein's avatar
William Stein committed
888 889
class NullNode(ConstNode):
    type = PyrexTypes.c_null_ptr_type
890
    value = "NULL"
891
    constant_result = 0
William Stein's avatar
William Stein committed
892

893
    def get_constant_c_result_code(self):
894 895
        return self.value

William Stein's avatar
William Stein committed
896 897 898

class CharNode(ConstNode):
    type = PyrexTypes.c_char_type
899 900 901

    def calculate_constant_result(self):
        self.constant_result = ord(self.value)
902

903
    def compile_time_value(self, denv):
904
        return ord(self.value)
905

William Stein's avatar
William Stein committed
906
    def calculate_result_code(self):
907
        return "'%s'" % StringEncoding.escape_char(self.value)
William Stein's avatar
William Stein committed
908 909 910


class IntNode(ConstNode):
911 912 913

    # unsigned     "" or "U"
    # longness     "" or "L" or "LL"
914
    # is_c_literal   True/False/None   creator considers this a C integer literal
915 916 917

    unsigned = ""
    longness = ""
918
    is_c_literal = None # unknown
919 920 921

    def __init__(self, pos, **kwds):
        ExprNode.__init__(self, pos, **kwds)
Robert Bradshaw's avatar
Robert Bradshaw committed
922
        if 'type' not in kwds:
923 924 925 926 927 928 929 930
            self.type = self.find_suitable_type_for_value()

    def find_suitable_type_for_value(self):
        if self.constant_result is constant_value_not_set:
            try:
                self.calculate_constant_result()
            except ValueError:
                pass
931 932 933 934
        # we ignore 'is_c_literal = True' and instead map signed 32bit
        # integers as C long values
        if self.is_c_literal or \
               self.constant_result in (constant_value_not_set, not_a_constant) or \
935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
               self.unsigned or self.longness == 'LL':
            # clearly a C literal
            rank = (self.longness == 'LL') and 2 or 1
            suitable_type = PyrexTypes.modifiers_and_name_to_type[not self.unsigned, rank, "int"]
            if self.type:
                suitable_type = PyrexTypes.widest_numeric_type(suitable_type, self.type)
        else:
            # C literal or Python literal - split at 32bit boundary
            if self.constant_result >= -2**31 and self.constant_result < 2**31:
                if self.type and self.type.is_int:
                    suitable_type = self.type
                else:
                    suitable_type = PyrexTypes.c_long_type
            else:
                suitable_type = PyrexTypes.py_object_type
        return suitable_type
William Stein's avatar
William Stein committed
951

952
    def coerce_to(self, dst_type, env):
953
        if self.type is dst_type:
954
            return self
955
        elif dst_type.is_float:
956
            if self.constant_result is not not_a_constant:
957 958
                return FloatNode(self.pos, value='%d.0' % int(self.constant_result), type=dst_type,
                                 constant_result=float(self.constant_result))
959 960 961
            else:
                return FloatNode(self.pos, value=self.value, type=dst_type,
                                 constant_result=not_a_constant)
962
        if dst_type.is_numeric and not dst_type.is_complex:
963
            node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
964 965
                           type = dst_type, is_c_literal = True,
                           unsigned=self.unsigned, longness=self.longness)
966
            return node
967 968
        elif dst_type.is_pyobject:
            node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
969 970
                           type = PyrexTypes.py_object_type, is_c_literal = False,
                           unsigned=self.unsigned, longness=self.longness)
971
        else:
972 973
            # FIXME: not setting the type here to keep it working with
            # complex numbers. Should they be special cased?
974 975
            node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
                           unsigned=self.unsigned, longness=self.longness)
976 977 978
        # We still need to perform normal coerce_to processing on the
        # result, because we might be coercing to an extension type,
        # in which case a type test node will be needed.
979 980
        return ConstNode.coerce_to(node, dst_type, env)

981
    def coerce_to_boolean(self, env):
982 983 984 985
        return IntNode(
            self.pos, value=self.value,
            type = PyrexTypes.c_bint_type,
            unsigned=self.unsigned, longness=self.longness)
986

987
    def generate_evaluation_code(self, code):
988
        if self.type.is_pyobject:
989
            # pre-allocate a Python version of the number
990 991
            plain_integer_string = self.value_as_c_integer_string(plain_digits=True)
            self.result_code = code.get_py_num(plain_integer_string, self.longness)
992
        else:
993
            self.result_code = self.get_constant_c_result_code()
994

995
    def get_constant_c_result_code(self):
996 997 998
        return self.value_as_c_integer_string() + self.unsigned + self.longness

    def value_as_c_integer_string(self, plain_digits=False):
999 1000 1001 1002
        value = self.value
        if isinstance(value, basestring) and len(value) > 2:
            # must convert C-incompatible Py3 oct/bin notations
            if value[1] in 'oO':
1003 1004 1005 1006
                if plain_digits:
                    value = int(value[2:], 8)
                else:
                    value = value[0] + value[2:] # '0o123' => '0123'
1007 1008
            elif value[1] in 'bB':
                value = int(value[2:], 2)
1009 1010 1011
            elif plain_digits and value[1] in 'xX':
                value = int(value[2:], 16)
        return str(value)
1012 1013 1014

    def calculate_result_code(self):
        return self.result_code
William Stein's avatar
William Stein committed
1015

1016
    def calculate_constant_result(self):
1017
        self.constant_result = Utils.str_to_number(self.value)
1018

1019
    def compile_time_value(self, denv):
1020
        return Utils.str_to_number(self.value)
1021 1022


William Stein's avatar
William Stein committed
1023 1024 1025
class FloatNode(ConstNode):
    type = PyrexTypes.c_double_type

1026
    def calculate_constant_result(self):
1027
        self.constant_result = float(self.value)
1028

1029 1030
    def compile_time_value(self, denv):
        return float(self.value)
1031

Stefan Behnel's avatar
Stefan Behnel committed
1032
    def calculate_result_code(self):
1033 1034 1035 1036
        strval = self.value
        assert isinstance(strval, (str, unicode))
        cmpval = repr(float(strval))
        if cmpval == 'nan':
1037
            return "(Py_HUGE_VAL * 0)"
1038
        elif cmpval == 'inf':
1039
            return "Py_HUGE_VAL"
1040
        elif cmpval == '-inf':
1041
            return "(-Py_HUGE_VAL)"
Stefan Behnel's avatar
Stefan Behnel committed
1042 1043
        else:
            return strval
1044

William Stein's avatar
William Stein committed
1045

1046
class BytesNode(ConstNode):
1047 1048 1049 1050
    # A char* or bytes literal
    #
    # value      BytesLiteral

1051
    is_string_literal = True
1052 1053
    # start off as Python 'bytes' to support len() in O(1)
    type = bytes_type
1054 1055

    def compile_time_value(self, denv):
1056
        return self.value
1057

1058
    def analyse_as_type(self, env):
1059
        type = PyrexTypes.parse_basic_type(self.value)
1060
        if type is not None:
1061
            return type
1062 1063 1064 1065 1066 1067 1068
        from TreeFragment import TreeFragment
        pos = (self.pos[0], self.pos[1], self.pos[2]-7)
        declaration = TreeFragment(u"sizeof(%s)" % self.value, name=pos[0].filename, initial_pos=pos)
        sizeof_node = declaration.root.stats[0].expr
        sizeof_node.analyse_types(env)
        if isinstance(sizeof_node, SizeofTypeNode):
            return sizeof_node.arg_type
1069

1070 1071 1072
    def can_coerce_to_char_literal(self):
        return len(self.value) == 1

1073
    def coerce_to_boolean(self, env):
1074 1075
        # This is special because testing a C char* for truth directly
        # would yield the wrong result.
1076 1077
        bool_value = bool(self.value)
        return BoolNode(self.pos, value=bool_value, constant_result=bool_value)
1078

William Stein's avatar
William Stein committed
1079
    def coerce_to(self, dst_type, env):
1080 1081
        if self.type == dst_type:
            return self
1082
        if dst_type.is_int:
1083
            if not self.can_coerce_to_char_literal():
1084 1085
                error(self.pos, "Only single-character string literals can be coerced into ints.")
                return self
Stefan Behnel's avatar
Stefan Behnel committed
1086 1087
            if dst_type.is_unicode_char:
                error(self.pos, "Bytes literals cannot coerce to Py_UNICODE/Py_UCS4, use a unicode literal instead.")
1088
                return self
1089 1090
            return CharNode(self.pos, value=self.value)

1091
        node = BytesNode(self.pos, value=self.value)
1092 1093 1094 1095 1096 1097 1098 1099
        if dst_type.is_pyobject:
            if dst_type in (py_object_type, Builtin.bytes_type):
                node.type = Builtin.bytes_type
            else:
                self.check_for_coercion_error(dst_type, fail=True)
                return node
        elif dst_type == PyrexTypes.c_char_ptr_type:
            node.type = dst_type
1100 1101 1102 1103
            return node
        elif dst_type == PyrexTypes.c_uchar_ptr_type:
            node.type = PyrexTypes.c_char_ptr_type
            return CastNode(node, PyrexTypes.c_uchar_ptr_type)
1104 1105
        elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
            node.type = dst_type
1106
            return node
1107

William Stein's avatar
William Stein committed
1108 1109 1110 1111 1112
        # We still need to perform normal coerce_to processing on the
        # result, because we might be coercing to an extension type,
        # in which case a type test node will be needed.
        return ConstNode.coerce_to(node, dst_type, env)

1113
    def generate_evaluation_code(self, code):
William Stein's avatar
William Stein committed
1114
        if self.type.is_pyobject:
1115
            self.result_code = code.get_py_string_const(self.value)
William Stein's avatar
William Stein committed
1116
        else:
1117
            self.result_code = code.get_string_const(self.value)
1118

1119
    def get_constant_c_result_code(self):
1120
        return None # FIXME
1121

1122 1123
    def calculate_result_code(self):
        return self.result_code
William Stein's avatar
William Stein committed
1124 1125


1126
class UnicodeNode(PyConstNode):
1127 1128
    # A Python unicode object
    #
1129 1130
    # value        EncodedString
    # bytes_value  BytesLiteral    the literal parsed as bytes string ('-3' unicode literals only)
Robert Bradshaw's avatar
Robert Bradshaw committed
1131

1132
    is_string_literal = True
1133
    bytes_value = None
1134
    type = unicode_type
1135

1136
    def coerce_to(self, dst_type, env):
1137 1138
        if dst_type is self.type:
            pass
Stefan Behnel's avatar
Stefan Behnel committed
1139
        elif dst_type.is_unicode_char:
1140
            if not self.can_coerce_to_char_literal():
Stefan Behnel's avatar
Stefan Behnel committed
1141
                error(self.pos, "Only single-character Unicode string literals or surrogate pairs can be coerced into Py_UCS4/Py_UNICODE.")
1142 1143
                return self
            int_value = ord(self.value)
Stefan Behnel's avatar
Stefan Behnel committed
1144
            return IntNode(self.pos, type=dst_type, value=str(int_value), constant_result=int_value)
1145
        elif not dst_type.is_pyobject:
1146 1147 1148
            if dst_type.is_string and self.bytes_value is not None:
                # special case: '-3' enforced unicode literal used in a C char* context
                return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env)
Stefan Behnel's avatar
Stefan Behnel committed
1149
            error(self.pos, "Unicode literals do not support coercion to C types other than Py_UNICODE or Py_UCS4.")
1150 1151 1152 1153
        elif dst_type is not py_object_type:
            if not self.check_for_coercion_error(dst_type):
                self.fail_assignment(dst_type)
        return self
1154

1155 1156
    def can_coerce_to_char_literal(self):
        return len(self.value) == 1
Stefan Behnel's avatar
Stefan Behnel committed
1157 1158 1159
            ## or (len(self.value) == 2
            ##     and (0xD800 <= self.value[0] <= 0xDBFF)
            ##     and (0xDC00 <= self.value[1] <= 0xDFFF))
1160

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
    def contains_surrogates(self):
        # Check if the unicode string contains surrogate code points
        # on a CPython platform with wide (UCS-4) or narrow (UTF-16)
        # Unicode, i.e. characters that would be spelled as two
        # separate code units on a narrow platform.
        for c in map(ord, self.value):
            if c > 65535: # can only happen on wide platforms
                return True
            # We only look for the first code unit (D800-DBFF) of a
            # surrogate pair - if we find one, the other one
            # (DC00-DFFF) is likely there, too.  If we don't find it,
            # any second code unit cannot make for a surrogate pair by
            # itself.
            if c >= 0xD800 and c <= 0xDBFF:
                return True
        return False

1178
    def generate_evaluation_code(self, code):
1179
        self.result_code = code.get_py_string_const(self.value)
1180 1181 1182

    def calculate_result_code(self):
        return self.result_code
1183

1184 1185
    def compile_time_value(self, env):
        return self.value
1186 1187


1188 1189 1190 1191
class StringNode(PyConstNode):
    # A Python str object, i.e. a byte string in Python 2.x and a
    # unicode string in Python 3.x
    #
1192 1193
    # value          BytesLiteral (or EncodedString with ASCII content)
    # unicode_value  EncodedString or None
1194
    # is_identifier  boolean
1195

1196
    type = str_type
1197
    is_string_literal = True
1198
    is_identifier = None
1199
    unicode_value = None
1200

1201
    def coerce_to(self, dst_type, env):
1202
        if dst_type is not py_object_type and not str_type.subtype_of(dst_type):
1203 1204 1205 1206 1207
#            if dst_type is Builtin.bytes_type:
#                # special case: bytes = 'str literal'
#                return BytesNode(self.pos, value=self.value)
            if not dst_type.is_pyobject:
                return BytesNode(self.pos, value=self.value).coerce_to(dst_type, env)
1208
            self.check_for_coercion_error(dst_type, fail=True)
1209
        return self
1210

1211 1212
    def can_coerce_to_char_literal(self):
        return not self.is_identifier and len(self.value) == 1
1213

1214
    def generate_evaluation_code(self, code):
1215
        self.result_code = code.get_py_string_const(
1216 1217
            self.value, identifier=self.is_identifier, is_str=True,
            unicode_value=self.unicode_value)
1218

1219
    def get_constant_c_result_code(self):
1220 1221
        return None

1222
    def calculate_result_code(self):
1223
        return self.result_code
1224

1225 1226
    def compile_time_value(self, env):
        return self.value
1227 1228


1229 1230 1231 1232
class IdentifierStringNode(StringNode):
    # A special str value that represents an identifier (bytes in Py2,
    # unicode in Py3).
    is_identifier = True
1233 1234


1235
class LongNode(AtomicExprNode):
William Stein's avatar
William Stein committed
1236 1237 1238
    #  Python long integer literal
    #
    #  value   string
1239

1240 1241
    type = py_object_type

1242
    def calculate_constant_result(self):
1243
        self.constant_result = Utils.str_to_number(self.value)
1244

1245
    def compile_time_value(self, denv):
1246
        return Utils.str_to_number(self.value)
1247

William Stein's avatar
William Stein committed
1248 1249
    def analyse_types(self, env):
        self.is_temp = 1
1250

1251 1252 1253
    def may_be_none(self):
        return False

1254 1255
    gil_message = "Constructing Python long int"

1256
    def generate_result_code(self, code):
William Stein's avatar
William Stein committed
1257
        code.putln(
1258
            '%s = PyLong_FromString((char *)"%s", 0, 0); %s' % (
1259
                self.result(),
William Stein's avatar
William Stein committed
1260
                self.value,
1261
                code.error_goto_if_null(self.result(), self.pos)))
1262
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
1263 1264


1265
class ImagNode(AtomicExprNode):
William Stein's avatar
William Stein committed
1266 1267 1268
    #  Imaginary number literal
    #
    #  value   float    imaginary part
1269

1270
    type = PyrexTypes.c_double_complex_type
1271 1272 1273

    def calculate_constant_result(self):
        self.constant_result = complex(0.0, self.value)
1274

1275 1276
    def compile_time_value(self, denv):
        return complex(0.0, self.value)
1277

William Stein's avatar
William Stein committed
1278
    def analyse_types(self, env):
1279 1280
        self.type.create_declaration_utility_code(env)

1281 1282 1283
    def may_be_none(self):
        return False

1284
    def coerce_to(self, dst_type, env):
1285 1286 1287
        if self.type is dst_type:
            return self
        node = ImagNode(self.pos, value=self.value)
1288
        if dst_type.is_pyobject:
1289 1290
            node.is_temp = 1
            node.type = PyrexTypes.py_object_type
1291 1292 1293
        # We still need to perform normal coerce_to processing on the
        # result, because we might be coercing to an extension type,
        # in which case a type test node will be needed.
1294
        return AtomicExprNode.coerce_to(node, dst_type, env)
1295 1296 1297

    gil_message = "Constructing complex number"

1298 1299 1300 1301 1302 1303
    def calculate_result_code(self):
        if self.type.is_pyobject:
            return self.result()
        else:
            return "%s(0, %r)" % (self.type.from_parts, float(self.value))

1304
    def generate_result_code(self, code):
1305 1306 1307 1308 1309 1310 1311
        if self.type.is_pyobject:
            code.putln(
                "%s = PyComplex_FromDoubles(0.0, %r); %s" % (
                    self.result(),
                    float(self.value),
                    code.error_goto_if_null(self.result(), self.pos)))
            code.put_gotref(self.py_result())
1312

William Stein's avatar
William Stein committed
1313

Danilo Freitas's avatar
Danilo Freitas committed
1314
class NewExprNode(AtomicExprNode):
1315 1316 1317

    # C++ new statement
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
1318
    # cppclass              node                 c++ class to create
1319

Robert Bradshaw's avatar
Robert Bradshaw committed
1320
    type = None
1321

1322
    def infer_type(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
1323 1324
        type = self.cppclass.analyse_as_type(env)
        if type is None or not type.is_cpp_class:
Danilo Freitas's avatar
Danilo Freitas committed
1325
            error(self.pos, "new operator can only be applied to a C++ class")
Robert Bradshaw's avatar
Robert Bradshaw committed
1326
            self.type = error_type
Danilo Freitas's avatar
Danilo Freitas committed
1327
            return
Robert Bradshaw's avatar
Robert Bradshaw committed
1328
        self.cpp_check(env)
1329
        constructor = type.scope.lookup(u'<init>')
Danilo Freitas's avatar
Danilo Freitas committed
1330
        if constructor is None:
1331 1332
            return_type = PyrexTypes.CFuncType(type, [])
            return_type = PyrexTypes.CPtrType(return_type)
1333 1334
            type.scope.declare_cfunction(u'<init>', return_type, self.pos)
            constructor = type.scope.lookup(u'<init>')
1335
        self.class_type = type
DaniloFreitas's avatar
DaniloFreitas committed
1336
        self.entry = constructor
Robert Bradshaw's avatar
Robert Bradshaw committed
1337
        self.type = constructor.type
1338
        return self.type
1339

1340
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
1341 1342
        if self.type is None:
            self.infer_type(env)
1343 1344 1345 1346

    def may_be_none(self):
        return False

Danilo Freitas's avatar
Danilo Freitas committed
1347 1348
    def generate_result_code(self, code):
        pass
1349

Danilo Freitas's avatar
Danilo Freitas committed
1350
    def calculate_result_code(self):
1351
        return "new " + self.class_type.declaration_code("")
Danilo Freitas's avatar
Danilo Freitas committed
1352

William Stein's avatar
William Stein committed
1353

1354
class NameNode(AtomicExprNode):
William Stein's avatar
William Stein committed
1355 1356 1357 1358
    #  Reference to a local or global variable name.
    #
    #  name            string    Python name of the variable
    #  entry           Entry     Symbol table entry
1359
    #  type_entry      Entry     For extension type names, the original type entry
1360 1361
    #  cf_is_null      boolean   Is uninitialized before this node
    #  cf_maybe_null   boolean   Maybe uninitialized before this node
Vitja Makarov's avatar
Vitja Makarov committed
1362
    #  allow_null      boolean   Don't raise UnboundLocalError
1363
    #  nogil           boolean   Whether it is used in a nogil context
1364

1365 1366
    is_name = True
    is_cython_module = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1367
    cython_attribute = None
1368
    lhs_of_first_assignment = False # TODO: remove me
1369
    is_used_as_rvalue = 0
1370
    entry = None
1371
    type_entry = None
1372 1373
    cf_maybe_null = True
    cf_is_null = False
Vitja Makarov's avatar
Vitja Makarov committed
1374
    allow_null = False
1375
    nogil = False
1376

1377
    def as_cython_attribute(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
1378
        return self.cython_attribute
1379

Robert Bradshaw's avatar
Robert Bradshaw committed
1380 1381 1382 1383 1384 1385 1386
    def type_dependencies(self, env):
        if self.entry is None:
            self.entry = env.lookup(self.name)
        if self.entry is not None and self.entry.type.is_unspecified:
            return (self.entry,)
        else:
            return ()
1387

Robert Bradshaw's avatar
Robert Bradshaw committed
1388 1389 1390 1391 1392
    def infer_type(self, env):
        if self.entry is None:
            self.entry = env.lookup(self.name)
        if self.entry is None:
            return py_object_type
Robert Bradshaw's avatar
Robert Bradshaw committed
1393 1394 1395
        elif (self.entry.type.is_extension_type or self.entry.type.is_builtin_type) and \
                self.name == self.entry.type.name:
            # Unfortunately the type attribute of type objects
1396
            # is used for the pointer to the type they represent.
Robert Bradshaw's avatar
Robert Bradshaw committed
1397
            return type_type
1398
        elif self.entry.type.is_cfunction:
1399 1400 1401 1402 1403 1404
            if self.entry.scope.is_builtin_scope:
                # special case: optimised builtin functions must be treated as Python objects
                return py_object_type
            else:
                # special case: referring to a C function must return its pointer
                return PyrexTypes.CPtrType(self.entry.type)
Robert Bradshaw's avatar
Robert Bradshaw committed
1405 1406
        else:
            return self.entry.type
1407

1408 1409 1410 1411
    def compile_time_value(self, denv):
        try:
            return denv.lookup(self.name)
        except KeyError:
Stefan Behnel's avatar
Stefan Behnel committed
1412
            error(self.pos, "Compile-time name '%s' not defined" % self.name)
1413 1414 1415 1416 1417

    def get_constant_c_result_code(self):
        if not self.entry or self.entry.type.is_pyobject:
            return None
        return self.entry.cname
1418

1419 1420 1421 1422 1423 1424 1425
    def coerce_to(self, dst_type, env):
        #  If coercing to a generic pyobject and this is a builtin
        #  C function with a Python equivalent, manufacture a NameNode
        #  referring to the Python builtin.
        #print "NameNode.coerce_to:", self.name, dst_type ###
        if dst_type is py_object_type:
            entry = self.entry
1426
            if entry and entry.is_cfunction:
1427 1428
                var_entry = entry.as_variable
                if var_entry:
1429
                    if var_entry.is_builtin and var_entry.is_const:
1430
                        var_entry = env.declare_builtin(var_entry.name, self.pos)
1431 1432 1433 1434
                    node = NameNode(self.pos, name = self.name)
                    node.entry = var_entry
                    node.analyse_rvalue_entry(env)
                    return node
1435

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1436
        return super(NameNode, self).coerce_to(dst_type, env)
1437

William Stein's avatar
William Stein committed
1438 1439 1440
    def analyse_as_module(self, env):
        # Try to interpret this as a reference to a cimported module.
        # Returns the module scope, or None.
1441 1442 1443
        entry = self.entry
        if not entry:
            entry = env.lookup(self.name)
William Stein's avatar
William Stein committed
1444 1445 1446
        if entry and entry.as_module:
            return entry.as_module
        return None
1447

1448
    def analyse_as_type(self, env):
1449 1450 1451 1452
        if self.cython_attribute:
            type = PyrexTypes.parse_basic_type(self.cython_attribute)
        else:
            type = PyrexTypes.parse_basic_type(self.name)
1453 1454
        if type:
            return type
1455 1456 1457 1458 1459 1460 1461
        entry = self.entry
        if not entry:
            entry = env.lookup(self.name)
        if entry and entry.is_type:
            return entry.type
        else:
            return None
1462

William Stein's avatar
William Stein committed
1463 1464 1465
    def analyse_as_extension_type(self, env):
        # Try to interpret this as a reference to an extension type.
        # Returns the extension type, or None.
1466 1467 1468
        entry = self.entry
        if not entry:
            entry = env.lookup(self.name)
William Stein's avatar
William Stein committed
1469
        if entry and entry.is_type and entry.type.is_extension_type:
1470 1471 1472
            return entry.type
        else:
            return None
1473

William Stein's avatar
William Stein committed
1474
    def analyse_target_declaration(self, env):
1475 1476
        if not self.entry:
            self.entry = env.lookup_here(self.name)
William Stein's avatar
William Stein committed
1477
        if not self.entry:
1478 1479
            if env.directives['warn.undeclared']:
                warning(self.pos, "implicit declaration of '%s'" % self.name, 1)
1480
            if env.directives['infer_types'] != False:
1481 1482 1483 1484
                type = unspecified_type
            else:
                type = py_object_type
            self.entry = env.declare_var(self.name, type, self.pos)
1485 1486
        if self.entry.is_declared_generic:
            self.result_ctype = py_object_type
1487

1488
    def analyse_types(self, env):
1489
        self.initialized_check = env.directives['initializedcheck']
1490 1491
        if self.entry is None:
            self.entry = env.lookup(self.name)
William Stein's avatar
William Stein committed
1492 1493
        if not self.entry:
            self.entry = env.declare_builtin(self.name, self.pos)
1494 1495 1496
        if not self.entry:
            self.type = PyrexTypes.error_type
            return
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1497 1498 1499 1500 1501 1502 1503 1504
        entry = self.entry
        if entry:
            entry.used = 1
            if entry.type.is_buffer:
                import Buffer
                Buffer.used_buffer_aux_vars(entry)
            if entry.utility_code:
                env.use_utility_code(entry.utility_code)
1505
        self.analyse_rvalue_entry(env)
1506

1507
    def analyse_target_types(self, env):
William Stein's avatar
William Stein committed
1508
        self.analyse_entry(env)
1509 1510 1511

        if (not self.is_lvalue() and self.entry.is_cfunction and
                self.entry.fused_cfunction and self.entry.as_variable):
1512
            # We need this for the fused 'def' TreeFragment
1513 1514 1515
            self.entry = self.entry.as_variable
            self.type = self.entry.type

1516 1517 1518 1519
        if not self.is_lvalue():
            error(self.pos, "Assignment to non-lvalue '%s'"
                % self.name)
            self.type = PyrexTypes.error_type
Stefan Behnel's avatar
Stefan Behnel committed
1520
        self.entry.used = 1
1521
        if self.entry.type.is_buffer:
1522 1523
            import Buffer
            Buffer.used_buffer_aux_vars(self.entry)
1524

1525 1526 1527 1528
    def analyse_rvalue_entry(self, env):
        #print "NameNode.analyse_rvalue_entry:", self.name ###
        #print "Entry:", self.entry.__dict__ ###
        self.analyse_entry(env)
1529
        entry = self.entry
1530

1531
        if entry.is_declared_generic:
William Stein's avatar
William Stein committed
1532
            self.result_ctype = py_object_type
1533

1534
        if entry.is_pyglobal or entry.is_builtin:
1535
            if entry.is_builtin and entry.is_const:
1536 1537 1538
                self.is_temp = 0
            else:
                self.is_temp = 1
1539
                env.use_utility_code(get_name_interned_utility_code)
1540

1541
            self.is_used_as_rvalue = 1
1542 1543 1544
        elif entry.type.is_memoryviewslice:
            self.is_temp = False
            self.is_used_as_rvalue = True
1545
            self.use_managed_ref = True
1546

1547
    def nogil_check(self, env):
1548
        self.nogil = True
1549 1550 1551
        if self.is_used_as_rvalue:
            entry = self.entry
            if entry.is_builtin:
1552
                if not entry.is_const: # cached builtins are ok
1553
                    self.gil_error()
1554
            elif entry.is_pyglobal:
1555
                self.gil_error()
1556 1557 1558 1559
            elif self.entry.type.is_memoryviewslice:
                if self.cf_is_null or self.cf_maybe_null:
                    import MemoryView
                    MemoryView.err_if_nogil_initialized_check(self.pos, env)
1560 1561 1562

    gil_message = "Accessing Python global or builtin"

1563 1564
    def analyse_entry(self, env):
        #print "NameNode.analyse_entry:", self.name ###
William Stein's avatar
William Stein committed
1565
        self.check_identifier_kind()
1566 1567 1568 1569
        entry = self.entry
        type = entry.type
        self.type = type

William Stein's avatar
William Stein committed
1570
    def check_identifier_kind(self):
1571 1572 1573
        # Check that this is an appropriate kind of name for use in an
        # expression.  Also finds the variable entry associated with
        # an extension type.
William Stein's avatar
William Stein committed
1574
        entry = self.entry
1575 1576
        if entry.is_type and entry.type.is_extension_type:
            self.type_entry = entry
1577
        if not (entry.is_const or entry.is_variable
Danilo Freitas's avatar
Danilo Freitas committed
1578 1579
            or entry.is_builtin or entry.is_cfunction
            or entry.is_cpp_class):
William Stein's avatar
William Stein committed
1580 1581 1582
                if self.entry.as_variable:
                    self.entry = self.entry.as_variable
                else:
1583
                    error(self.pos,
1584 1585
                          "'%s' is not a constant, variable or function identifier" % self.name)

William Stein's avatar
William Stein committed
1586 1587 1588
    def is_simple(self):
        #  If it's not a C variable, it'll be in a temp.
        return 1
1589

1590
    def may_be_none(self):
1591
        if self.cf_state and self.type and self.type.is_pyobject:
1592 1593 1594 1595 1596 1597 1598
            # gard against infinite recursion on self-dependencies
            if getattr(self, '_none_checking', False):
                # self-dependency - either this node receives a None
                # value from *another* node, or it can not reference
                # None at this point => safe to assume "not None"
                return False
            self._none_checking = True
1599 1600
            # evaluate control flow state to see if there were any
            # potential None values assigned to the node so far
1601
            may_be_none = False
1602 1603
            for assignment in self.cf_state:
                if assignment.rhs.may_be_none():
1604 1605 1606 1607
                    may_be_none = True
                    break
            del self._none_checking
            return may_be_none
1608 1609
        return super(NameNode, self).may_be_none()

1610
    def nonlocally_immutable(self):
1611 1612
        if ExprNode.nonlocally_immutable(self):
            return True
1613 1614 1615
        entry = self.entry
        return entry and (entry.is_local or entry.is_arg) and not entry.in_closure

William Stein's avatar
William Stein committed
1616 1617
    def calculate_target_results(self, env):
        pass
1618

William Stein's avatar
William Stein committed
1619 1620
    def check_const(self):
        entry = self.entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1621
        if entry is not None and not (entry.is_const or entry.is_cfunction or entry.is_builtin):
William Stein's avatar
William Stein committed
1622
            self.not_const()
1623 1624
            return False
        return True
1625

William Stein's avatar
William Stein committed
1626 1627
    def check_const_addr(self):
        entry = self.entry
1628
        if not (entry.is_cglobal or entry.is_cfunction or entry.is_builtin):
William Stein's avatar
William Stein committed
1629
            self.addr_not_const()
1630 1631
            return False
        return True
William Stein's avatar
William Stein committed
1632 1633 1634 1635 1636

    def is_lvalue(self):
        return self.entry.is_variable and \
            not self.entry.type.is_array and \
            not self.entry.is_readonly
1637

1638 1639 1640
    def is_addressable(self):
        return self.entry.is_variable

William Stein's avatar
William Stein committed
1641 1642 1643 1644
    def is_ephemeral(self):
        #  Name nodes are never ephemeral, even if the
        #  result is in a temporary.
        return 0
1645

William Stein's avatar
William Stein committed
1646
    def calculate_result_code(self):
Stefan Behnel's avatar
Stefan Behnel committed
1647 1648
        entry = self.entry
        if not entry:
William Stein's avatar
William Stein committed
1649
            return "<error>" # There was an error earlier
Stefan Behnel's avatar
Stefan Behnel committed
1650
        return entry.cname
1651

William Stein's avatar
William Stein committed
1652
    def generate_result_code(self, code):
1653
        assert hasattr(self, 'entry')
William Stein's avatar
William Stein committed
1654 1655 1656
        entry = self.entry
        if entry is None:
            return # There was an error earlier
1657
        if entry.is_builtin and entry.is_const:
1658
            return # Lookup already cached
Stefan Behnel's avatar
Stefan Behnel committed
1659
        elif entry.is_pyclass_attr:
Vitja Makarov's avatar
Vitja Makarov committed
1660 1661 1662 1663 1664 1665
            assert entry.type.is_pyobject, "Python global or builtin not a Python object"
            interned_cname = code.intern_identifier(self.entry.name)
            if entry.is_builtin:
                namespace = Naming.builtins_cname
            else: # entry.is_pyglobal
                namespace = entry.scope.namespace_cname
1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
            if not self.cf_is_null:
                code.putln(
                    '%s = PyObject_GetItem(%s, %s);' % (
                        self.result(),
                        namespace,
                        interned_cname))
            if self.cf_maybe_null:
                if not self.cf_is_null:
                    code.putln('if (unlikely(!%s)) {' % self.result())
                    code.putln('PyErr_Clear();')
                code.putln(
                    '%s = __Pyx_GetName(%s, %s);' % (
                    self.result(),
                    Naming.module_cname,
                    interned_cname))
                if not self.cf_is_null:
                    code.putln("}");
            code.putln(code.error_goto_if_null(self.result(), self.pos))
Vitja Makarov's avatar
Vitja Makarov committed
1684
            code.put_gotref(self.py_result())
1685

1686
        elif entry.is_pyglobal or entry.is_builtin:
1687 1688
            assert entry.type.is_pyobject, "Python global or builtin not a Python object"
            interned_cname = code.intern_identifier(self.entry.name)
William Stein's avatar
William Stein committed
1689 1690 1691
            if entry.is_builtin:
                namespace = Naming.builtins_cname
            else: # entry.is_pyglobal
1692
                namespace = entry.scope.namespace_cname
1693
            code.globalstate.use_utility_code(get_name_interned_utility_code)
1694 1695
            code.putln(
                '%s = __Pyx_GetName(%s, %s); %s' % (
1696
                self.result(),
1697
                namespace,
1698
                interned_cname,
1699
                code.error_goto_if_null(self.result(), self.pos)))
1700
            code.put_gotref(self.py_result())
1701

1702
        elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice:
1703 1704 1705 1706
            # Raise UnboundLocalError for objects and memoryviewslices
            raise_unbound = (
                (self.cf_maybe_null or self.cf_is_null) and not self.allow_null)
            null_code = entry.type.check_for_null_code(entry.cname)
1707

1708 1709 1710
            memslice_check = entry.type.is_memoryviewslice and self.initialized_check

            if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check):
1711
                code.put_error_if_unbound(self.pos, entry, self.in_nogil_context)
William Stein's avatar
William Stein committed
1712 1713

    def generate_assignment_code(self, rhs, code):
1714
        #print "NameNode.generate_assignment_code:", self.name ###
William Stein's avatar
William Stein committed
1715 1716 1717
        entry = self.entry
        if entry is None:
            return # There was an error earlier
1718 1719 1720 1721

        if (self.entry.type.is_ptr and isinstance(rhs, ListNode)
            and not self.lhs_of_first_assignment):
            error(self.pos, "Literal list must be assigned to pointer at time of declaration")
1722

1723 1724
        # is_pyglobal seems to be True for module level-globals only.
        # We use this to access class->tp_dict if necessary.
William Stein's avatar
William Stein committed
1725
        if entry.is_pyglobal:
1726 1727
            assert entry.type.is_pyobject, "Python global or builtin not a Python object"
            interned_cname = code.intern_identifier(self.entry.name)
1728
            namespace = self.entry.scope.namespace_cname
1729
            if entry.is_member:
Stefan Behnel's avatar
Stefan Behnel committed
1730
                # if the entry is a member we have to cheat: SetAttr does not work
1731
                # on types, so we create a descriptor which is then added to tp_dict
1732 1733 1734
                code.put_error_if_neg(self.pos,
                    'PyDict_SetItem(%s->tp_dict, %s, %s)' % (
                        namespace,
1735
                        interned_cname,
1736
                        rhs.py_result()))
1737 1738
                rhs.generate_disposal_code(code)
                rhs.free_temps(code)
1739
                # in Py2.6+, we need to invalidate the method cache
1740
                code.putln("PyType_Modified(%s);" %
Vitja Makarov's avatar
Vitja Makarov committed
1741
                            entry.scope.parent_type.typeptr_cname)
Stefan Behnel's avatar
Stefan Behnel committed
1742
            elif entry.is_pyclass_attr:
Vitja Makarov's avatar
Vitja Makarov committed
1743
                code.put_error_if_neg(self.pos,
Stefan Behnel's avatar
Stefan Behnel committed
1744
                    'PyObject_SetItem(%s, %s, %s)' % (
Vitja Makarov's avatar
Vitja Makarov committed
1745 1746 1747 1748 1749 1750
                        namespace,
                        interned_cname,
                        rhs.py_result()))
                rhs.generate_disposal_code(code)
                rhs.free_temps(code)
            else:
1751 1752 1753
                code.put_error_if_neg(self.pos,
                    'PyObject_SetAttr(%s, %s, %s)' % (
                        namespace,
1754
                        interned_cname,
1755
                        rhs.py_result()))
1756
                if debug_disposal_code:
Stefan Behnel's avatar
Stefan Behnel committed
1757 1758
                    print("NameNode.generate_assignment_code:")
                    print("...generating disposal code for %s" % rhs)
1759
                rhs.generate_disposal_code(code)
1760
                rhs.free_temps(code)
William Stein's avatar
William Stein committed
1761
        else:
1762
            if self.type.is_memoryviewslice:
1763
                self.generate_acquire_memoryviewslice(rhs, code)
1764

1765
            elif self.type.is_buffer:
1766 1767 1768 1769 1770 1771 1772 1773 1774
                # Generate code for doing the buffer release/acquisition.
                # This might raise an exception in which case the assignment (done
                # below) will not happen.
                #
                # The reason this is not in a typetest-like node is because the
                # variables that the acquired buffer info is stored to is allocated
                # per entry and coupled with it.
                self.generate_acquire_buffer(rhs, code)

1775
            if self.type.is_pyobject:
William Stein's avatar
William Stein committed
1776 1777 1778 1779
                #print "NameNode.generate_assignment_code: to", self.name ###
                #print "...from", rhs ###
                #print "...LHS type", self.type, "ctype", self.ctype() ###
                #print "...RHS type", rhs.type, "ctype", rhs.ctype() ###
1780 1781
                if self.use_managed_ref:
                    rhs.make_owned_reference(code)
1782
                    is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure
1783 1784 1785 1786 1787 1788
                    if is_external_ref:
                        if not self.cf_is_null:
                            if self.cf_maybe_null:
                                code.put_xgotref(self.py_result())
                            else:
                                code.put_gotref(self.py_result())
1789 1790 1791
                    if entry.is_cglobal:
                        code.put_decref(self.result(), self.ctype())
                    else:
1792 1793
                        if not self.cf_is_null:
                            if self.cf_maybe_null:
1794
                                code.put_xdecref(self.result(), self.ctype())
1795 1796
                            else:
                                code.put_decref(self.result(), self.ctype())
1797
                    if is_external_ref:
1798
                        code.put_giveref(rhs.py_result())
1799
            if not self.type.is_memoryviewslice:
1800
                code.putln('%s = %s;' % (self.result(), rhs.result_as(self.ctype())))
1801 1802 1803 1804
                if debug_disposal_code:
                    print("NameNode.generate_assignment_code:")
                    print("...generating post-assignment code for %s" % rhs)
                rhs.generate_post_assignment_code(code)
1805 1806
            elif rhs.result_in_temp():
                rhs.generate_post_assignment_code(code)
1807

1808
            rhs.free_temps(code)
1809

1810 1811
    def generate_acquire_memoryviewslice(self, rhs, code):
        """
1812 1813
        Slices, coercions from objects, return values etc are new references.
        We have a borrowed reference in case of dst = src
1814 1815 1816 1817 1818 1819 1820 1821 1822
        """
        import MemoryView

        MemoryView.put_acquire_memoryviewslice(
            lhs_cname=self.result(),
            lhs_type=self.type,
            lhs_pos=self.pos,
            rhs=rhs,
            code=code,
1823 1824
            have_gil=not self.in_nogil_context,
            first_assignment=self.cf_is_null)
1825

1826
    def generate_acquire_buffer(self, rhs, code):
1827 1828 1829
        # rhstmp is only used in case the rhs is a complicated expression leading to
        # the object, to avoid repeating the same C expression for every reference
        # to the rhs. It does NOT hold a reference.
1830 1831 1832 1833 1834 1835 1836
        pretty_rhs = isinstance(rhs, NameNode) or rhs.is_temp
        if pretty_rhs:
            rhstmp = rhs.result_as(self.ctype())
        else:
            rhstmp = code.funcstate.allocate_temp(self.entry.type, manage_ref=False)
            code.putln('%s = %s;' % (rhstmp, rhs.result_as(self.ctype())))

1837
        import Buffer
1838
        Buffer.put_assign_to_buffer(self.result(), rhstmp, self.entry,
1839
                                    is_initialized=not self.lhs_of_first_assignment,
1840
                                    pos=self.pos, code=code)
1841

1842 1843 1844
        if not pretty_rhs:
            code.putln("%s = 0;" % rhstmp)
            code.funcstate.release_temp(rhstmp)
1845

William Stein's avatar
William Stein committed
1846 1847 1848
    def generate_deletion_code(self, code):
        if self.entry is None:
            return # There was an error earlier
1849
        elif self.entry.is_pyclass_attr:
Vitja Makarov's avatar
Vitja Makarov committed
1850
            namespace = self.entry.scope.namespace_cname
1851
            interned_cname = code.intern_identifier(self.entry.name)
Vitja Makarov's avatar
Vitja Makarov committed
1852
            code.put_error_if_neg(self.pos,
1853
                'PyObject_DelItem(%s, %s)' % (
Vitja Makarov's avatar
Vitja Makarov committed
1854
                    namespace,
1855
                    interned_cname))
1856 1857 1858 1859 1860
        elif self.entry.is_pyglobal:
            code.put_error_if_neg(self.pos,
                '__Pyx_DelAttrString(%s, "%s")' % (
                    Naming.module_cname,
                    self.entry.name))
1861
        elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice:
1862 1863
            if not self.cf_is_null:
                if self.cf_maybe_null:
1864
                    code.put_error_if_unbound(self.pos, self.entry)
1865 1866 1867 1868 1869 1870 1871

                if self.entry.type.is_pyobject:
                    code.put_decref(self.result(), self.ctype())
                    code.putln('%s = NULL;' % self.result())
                else:
                    code.put_xdecref_memoryviewslice(self.entry.cname,
                                                     have_gil=not self.nogil)
Vitja Makarov's avatar
Vitja Makarov committed
1872
        else:
1873
            error(self.pos, "Deletion of C names not supported")
1874

1875 1876 1877 1878 1879 1880 1881
    def annotate(self, code):
        if hasattr(self, 'is_called') and self.is_called:
            pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1)
            if self.type.is_pyobject:
                code.annotate(pos, AnnotationItem('py_call', 'python function', size=len(self.name)))
            else:
                code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
1882

1883
class BackquoteNode(ExprNode):
William Stein's avatar
William Stein committed
1884 1885 1886
    #  `expr`
    #
    #  arg    ExprNode
1887

1888
    type = py_object_type
1889

William Stein's avatar
William Stein committed
1890
    subexprs = ['arg']
1891

William Stein's avatar
William Stein committed
1892 1893 1894 1895
    def analyse_types(self, env):
        self.arg.analyse_types(env)
        self.arg = self.arg.coerce_to_pyobject(env)
        self.is_temp = 1
1896 1897 1898

    gil_message = "Backquote expression"

1899 1900 1901
    def calculate_constant_result(self):
        self.constant_result = repr(self.arg.constant_result)

William Stein's avatar
William Stein committed
1902 1903
    def generate_result_code(self, code):
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
1904
            "%s = PyObject_Repr(%s); %s" % (
1905
                self.result(),
William Stein's avatar
William Stein committed
1906
                self.arg.py_result(),
1907
                code.error_goto_if_null(self.result(), self.pos)))
1908
        code.put_gotref(self.py_result())
1909

William Stein's avatar
William Stein committed
1910

1911
class ImportNode(ExprNode):
William Stein's avatar
William Stein committed
1912
    #  Used as part of import statement implementation.
1913
    #  Implements result =
Haoyu Bai's avatar
Haoyu Bai committed
1914
    #    __import__(module_name, globals(), None, name_list, level)
William Stein's avatar
William Stein committed
1915
    #
Haoyu Bai's avatar
Haoyu Bai committed
1916 1917 1918
    #  module_name   StringNode            dotted name of module. Empty module
    #                       name means importing the parent package accourding
    #                       to level
1919
    #  name_list     ListNode or None      list of names to be imported
Haoyu Bai's avatar
Haoyu Bai committed
1920 1921 1922 1923 1924
    #  level         int                   relative import level:
    #                       -1: attempt both relative import and absolute import;
    #                        0: absolute import;
    #                       >0: the number of parent directories to search
    #                           relative to the current module.
1925 1926
    #                     None: decide the level according to language level and
    #                           directives
1927

1928
    type = py_object_type
1929

William Stein's avatar
William Stein committed
1930
    subexprs = ['module_name', 'name_list']
1931

William Stein's avatar
William Stein committed
1932
    def analyse_types(self, env):
1933 1934 1935 1936 1937
        if self.level is None:
            if env.directives['language_level'] < 3 or env.directives['py2_import']:
                self.level = -1
            else:
                self.level = 0
William Stein's avatar
William Stein committed
1938 1939 1940 1941
        self.module_name.analyse_types(env)
        self.module_name = self.module_name.coerce_to_pyobject(env)
        if self.name_list:
            self.name_list.analyse_types(env)
1942
            self.name_list.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
1943 1944
        self.is_temp = 1
        env.use_utility_code(import_utility_code)
1945 1946 1947

    gil_message = "Python import"

William Stein's avatar
William Stein committed
1948 1949 1950 1951 1952 1953
    def generate_result_code(self, code):
        if self.name_list:
            name_list_code = self.name_list.py_result()
        else:
            name_list_code = "0"
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
1954
            "%s = __Pyx_Import(%s, %s, %d); %s" % (
1955
                self.result(),
William Stein's avatar
William Stein committed
1956 1957
                self.module_name.py_result(),
                name_list_code,
Haoyu Bai's avatar
Haoyu Bai committed
1958
                self.level,
1959
                code.error_goto_if_null(self.result(), self.pos)))
1960
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
1961 1962


1963
class IteratorNode(ExprNode):
William Stein's avatar
William Stein committed
1964
    #  Used as part of for statement implementation.
1965
    #
William Stein's avatar
William Stein committed
1966 1967 1968
    #  Implements result = iter(sequence)
    #
    #  sequence   ExprNode
1969

1970
    type = py_object_type
1971
    iter_func_ptr = None
1972
    counter_cname = None
1973
    reversed = False      # currently only used for list/tuple types (see Optimize.py)
1974

William Stein's avatar
William Stein committed
1975
    subexprs = ['sequence']
1976

William Stein's avatar
William Stein committed
1977 1978
    def analyse_types(self, env):
        self.sequence.analyse_types(env)
1979 1980
        if (self.sequence.type.is_array or self.sequence.type.is_ptr) and \
                not self.sequence.type.is_string:
1981
            # C array iteration will be transformed later on
1982
            self.type = self.sequence.type
1983 1984
        else:
            self.sequence = self.sequence.coerce_to_pyobject(env)
1985 1986 1987
            if self.sequence.type is list_type or \
                   self.sequence.type is tuple_type:
                self.sequence = self.sequence.as_none_safe_node("'NoneType' object is not iterable")
William Stein's avatar
William Stein committed
1988
        self.is_temp = 1
1989 1990 1991

    gil_message = "Iterating over Python object"

1992 1993 1994 1995
    _func_iternext_type = PyrexTypes.CPtrType(PyrexTypes.CFuncType(
        PyrexTypes.py_object_type, [
            PyrexTypes.CFuncTypeArg("it", PyrexTypes.py_object_type, None),
            ]))
1996

William Stein's avatar
William Stein committed
1997
    def generate_result_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
1998 1999
        sequence_type = self.sequence.type
        if sequence_type.is_array or sequence_type.is_ptr:
2000
            raise InternalError("for in carray slice not transformed")
Stefan Behnel's avatar
Stefan Behnel committed
2001 2002
        is_builtin_sequence = sequence_type is list_type or \
                              sequence_type is tuple_type
2003 2004 2005
        if not is_builtin_sequence:
            # reversed() not currently optimised (see Optimize.py)
            assert not self.reversed, "internal error: reversed() only implemented for list/tuple objects"
Stefan Behnel's avatar
Stefan Behnel committed
2006 2007
        self.may_be_a_sequence = not sequence_type.is_builtin_type
        if self.may_be_a_sequence:
2008 2009 2010 2011
            code.putln(
                "if (PyList_CheckExact(%s) || PyTuple_CheckExact(%s)) {" % (
                    self.sequence.py_result(),
                    self.sequence.py_result()))
Stefan Behnel's avatar
Stefan Behnel committed
2012
        if is_builtin_sequence or self.may_be_a_sequence:
2013 2014
            self.counter_cname = code.funcstate.allocate_temp(
                PyrexTypes.c_py_ssize_t_type, manage_ref=False)
2015 2016 2017 2018 2019 2020 2021
            if self.reversed:
                if sequence_type is list_type:
                    init_value = 'PyList_GET_SIZE(%s) - 1' % self.result()
                else:
                    init_value = 'PyTuple_GET_SIZE(%s) - 1' % self.result()
            else:
                init_value = '0'
2022
            code.putln(
2023
                "%s = %s; __Pyx_INCREF(%s); %s = %s;" % (
2024 2025
                    self.result(),
                    self.sequence.py_result(),
2026 2027 2028 2029
                    self.result(),
                    self.counter_cname,
                    init_value
                    ))
2030
        if not is_builtin_sequence:
Stefan Behnel's avatar
Stefan Behnel committed
2031
            self.iter_func_ptr = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False)
Stefan Behnel's avatar
Stefan Behnel committed
2032
            if self.may_be_a_sequence:
Stefan Behnel's avatar
Stefan Behnel committed
2033
                code.putln("%s = NULL;" % self.iter_func_ptr)
2034
                code.putln("} else {")
2035 2036
                code.put("%s = -1; " % self.counter_cname)
            code.putln("%s = PyObject_GetIter(%s); %s" % (
2037 2038 2039
                    self.result(),
                    self.sequence.py_result(),
                    code.error_goto_if_null(self.result(), self.pos)))
2040
            code.put_gotref(self.py_result())
2041
            code.putln("%s = Py_TYPE(%s)->tp_iternext;" % (self.iter_func_ptr, self.py_result()))
Stefan Behnel's avatar
Stefan Behnel committed
2042 2043 2044 2045
        if self.may_be_a_sequence:
            code.putln("}")

    def generate_next_sequence_item(self, test_name, result_name, code):
2046
        assert self.counter_cname, "internal error: counter_cname temp not prepared"
Stefan Behnel's avatar
Stefan Behnel committed
2047 2048 2049 2050 2051
        code.putln(
            "if (%s >= Py%s_GET_SIZE(%s)) break;" % (
                self.counter_cname,
                test_name,
                self.py_result()))
2052 2053 2054 2055
        if self.reversed:
            inc_dec = '--'
        else:
            inc_dec = '++'
Stefan Behnel's avatar
Stefan Behnel committed
2056
        code.putln(
2057
            "%s = Py%s_GET_ITEM(%s, %s); __Pyx_INCREF(%s); %s%s;" % (
Stefan Behnel's avatar
Stefan Behnel committed
2058 2059 2060 2061 2062
                result_name,
                test_name,
                self.py_result(),
                self.counter_cname,
                result_name,
2063 2064
                self.counter_cname,
                inc_dec))
Stefan Behnel's avatar
Stefan Behnel committed
2065 2066 2067

    def generate_iter_next_result_code(self, result_name, code):
        sequence_type = self.sequence.type
2068 2069
        if self.reversed:
            code.putln("if (%s < 0) break;" % self.counter_cname)
Stefan Behnel's avatar
Stefan Behnel committed
2070 2071 2072 2073 2074 2075 2076 2077 2078
        if sequence_type is list_type:
            self.generate_next_sequence_item('List', result_name, code)
            return
        elif sequence_type is tuple_type:
            self.generate_next_sequence_item('Tuple', result_name, code)
            return

        if self.may_be_a_sequence:
            for test_name in ('List', 'Tuple'):
2079 2080
                code.putln("if (!%s && Py%s_CheckExact(%s)) {" % (
                    self.iter_func_ptr, test_name, self.py_result()))
Stefan Behnel's avatar
Stefan Behnel committed
2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
                self.generate_next_sequence_item(test_name, result_name, code)
                code.put("} else ")

        code.putln("{")
        code.putln(
            "%s = %s(%s);" % (
                result_name,
                self.iter_func_ptr,
                self.py_result()))
        code.putln("if (unlikely(!%s)) {" % result_name)
        code.putln("if (PyErr_Occurred()) {")
        code.putln("if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) PyErr_Clear();")
        code.putln("else %s" % code.error_goto(self.pos))
        code.putln("}")
        code.putln("break;")
        code.putln("}")
        code.put_gotref(result_name)
        code.putln("}")
William Stein's avatar
William Stein committed
2099

2100
    def free_temps(self, code):
2101 2102
        if self.counter_cname:
            code.funcstate.release_temp(self.counter_cname)
2103 2104 2105 2106
        if self.iter_func_ptr:
            code.funcstate.release_temp(self.iter_func_ptr)
            self.iter_func_ptr = None
        ExprNode.free_temps(self, code)
William Stein's avatar
William Stein committed
2107 2108


2109
class NextNode(AtomicExprNode):
William Stein's avatar
William Stein committed
2110 2111 2112 2113 2114
    #  Used as part of for statement implementation.
    #  Implements result = iterator.next()
    #  Created during analyse_types phase.
    #  The iterator is not owned by this node.
    #
2115
    #  iterator   IteratorNode
2116

2117
    type = py_object_type
2118

2119
    def __init__(self, iterator):
William Stein's avatar
William Stein committed
2120 2121
        self.pos = iterator.pos
        self.iterator = iterator
2122 2123
        if iterator.type.is_ptr or iterator.type.is_array:
            self.type = iterator.type.base_type
William Stein's avatar
William Stein committed
2124
        self.is_temp = 1
2125

William Stein's avatar
William Stein committed
2126
    def generate_result_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
2127
        self.iterator.generate_iter_next_result_code(self.result(), code)
2128

William Stein's avatar
William Stein committed
2129

2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
class WithExitCallNode(ExprNode):
    # The __exit__() call of a 'with' statement.  Used in both the
    # except and finally clauses.

    # with_stat  WithStatNode                the surrounding 'with' statement
    # args       TupleNode or ResultStatNode the exception info tuple

    subexprs = ['args']

    def analyse_types(self, env):
        self.args.analyse_types(env)
        self.type = PyrexTypes.c_bint_type
        self.is_temp = True

    def generate_result_code(self, code):
        if isinstance(self.args, TupleNode):
            # call only if it was not already called (and decref-cleared)
            code.putln("if (%s) {" % self.with_stat.exit_var)
        result_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
        code.putln("%s = PyObject_Call(%s, %s, NULL);" % (
            result_var,
            self.with_stat.exit_var,
            self.args.result()))
        code.put_decref_clear(self.with_stat.exit_var, type=py_object_type)
        code.putln(code.error_goto_if_null(result_var, self.pos))
        code.put_gotref(result_var)
        code.putln("%s = __Pyx_PyObject_IsTrue(%s);" % (self.result(), result_var))
        code.put_decref_clear(result_var, type=py_object_type)
        code.putln(code.error_goto_if_neg(self.result(), self.pos))
        code.funcstate.release_temp(result_var)
        if isinstance(self.args, TupleNode):
            code.putln("}")


2164
class ExcValueNode(AtomicExprNode):
William Stein's avatar
William Stein committed
2165 2166 2167
    #  Node created during analyse_types phase
    #  of an ExceptClauseNode to fetch the current
    #  exception value.
2168

2169
    type = py_object_type
2170

2171
    def __init__(self, pos, env):
William Stein's avatar
William Stein committed
2172
        ExprNode.__init__(self, pos)
2173 2174

    def set_var(self, var):
2175
        self.var = var
2176

2177 2178 2179
    def calculate_result_code(self):
        return self.var

William Stein's avatar
William Stein committed
2180
    def generate_result_code(self, code):
2181
        pass
William Stein's avatar
William Stein committed
2182

2183 2184 2185
    def analyse_types(self, env):
        pass

William Stein's avatar
William Stein committed
2186

2187
class TempNode(ExprNode):
2188 2189 2190 2191 2192 2193 2194
    # Node created during analyse_types phase
    # of some nodes to hold a temporary value.
    #
    # Note: One must call "allocate" and "release" on
    # the node during code generation to get/release the temp.
    # This is because the temp result is often used outside of
    # the regular cycle.
2195 2196

    subexprs = []
2197

2198
    def __init__(self, pos, type, env=None):
William Stein's avatar
William Stein committed
2199 2200 2201 2202 2203
        ExprNode.__init__(self, pos)
        self.type = type
        if type.is_pyobject:
            self.result_ctype = py_object_type
        self.is_temp = 1
2204

2205 2206
    def analyse_types(self, env):
        return self.type
2207

2208 2209 2210
    def analyse_target_declaration(self, env):
        pass

William Stein's avatar
William Stein committed
2211 2212 2213
    def generate_result_code(self, code):
        pass

2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230
    def allocate(self, code):
        self.temp_cname = code.funcstate.allocate_temp(self.type, manage_ref=True)

    def release(self, code):
        code.funcstate.release_temp(self.temp_cname)
        self.temp_cname = None

    def result(self):
        try:
            return self.temp_cname
        except:
            assert False, "Remember to call allocate/release on TempNode"
            raise

    # Do not participate in normal temp alloc/dealloc:
    def allocate_temp_result(self, code):
        pass
2231

2232 2233
    def release_temp_result(self, code):
        pass
William Stein's avatar
William Stein committed
2234 2235 2236

class PyTempNode(TempNode):
    #  TempNode holding a Python value.
2237

William Stein's avatar
William Stein committed
2238 2239 2240
    def __init__(self, pos, env):
        TempNode.__init__(self, pos, PyrexTypes.py_object_type, env)

2241 2242
class RawCNameExprNode(ExprNode):
    subexprs = []
2243

2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
    def __init__(self, pos, type=None):
        self.pos = pos
        self.type = type

    def analyse_types(self, env):
        return self.type

    def set_cname(self, cname):
        self.cname = cname

    def result(self):
        return self.cname

    def generate_result_code(self, code):
        pass

William Stein's avatar
William Stein committed
2260

Mark Florisson's avatar
Mark Florisson committed
2261 2262 2263 2264 2265 2266 2267 2268
#-------------------------------------------------------------------
#
#  Parallel nodes (cython.parallel.thread(savailable|id))
#
#-------------------------------------------------------------------

class ParallelThreadsAvailableNode(AtomicExprNode):
    """
Mark Florisson's avatar
Mark Florisson committed
2269 2270
    Note: this is disabled and not a valid directive at this moment

Mark Florisson's avatar
Mark Florisson committed
2271 2272 2273 2274 2275 2276 2277 2278 2279
    Implements cython.parallel.threadsavailable(). If we are called from the
    sequential part of the application, we need to call omp_get_max_threads(),
    and in the parallel part we can just call omp_get_num_threads()
    """

    type = PyrexTypes.c_int_type

    def analyse_types(self, env):
        self.is_temp = True
2280
        # env.add_include_file("omp.h")
Mark Florisson's avatar
Mark Florisson committed
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304
        return self.type

    def generate_result_code(self, code):
        code.putln("#ifdef _OPENMP")
        code.putln("if (omp_in_parallel()) %s = omp_get_max_threads();" %
                                                            self.temp_code)
        code.putln("else %s = omp_get_num_threads();" % self.temp_code)
        code.putln("#else")
        code.putln("%s = 1;" % self.temp_code)
        code.putln("#endif")

    def result(self):
        return self.temp_code


class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode):
    """
    Implements cython.parallel.threadid()
    """

    type = PyrexTypes.c_int_type

    def analyse_types(self, env):
        self.is_temp = True
2305
        # env.add_include_file("omp.h")
Mark Florisson's avatar
Mark Florisson committed
2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
        return self.type

    def generate_result_code(self, code):
        code.putln("#ifdef _OPENMP")
        code.putln("%s = omp_get_thread_num();" % self.temp_code)
        code.putln("#else")
        code.putln("%s = 0;" % self.temp_code)
        code.putln("#endif")

    def result(self):
        return self.temp_code


William Stein's avatar
William Stein committed
2319 2320 2321 2322 2323 2324
#-------------------------------------------------------------------
#
#  Trailer nodes
#
#-------------------------------------------------------------------

2325
class IndexNode(ExprNode):
William Stein's avatar
William Stein committed
2326 2327 2328 2329
    #  Sequence indexing.
    #
    #  base     ExprNode
    #  index    ExprNode
2330 2331 2332 2333 2334 2335
    #  indices  [ExprNode]
    #  is_buffer_access boolean Whether this is a buffer access.
    #
    #  indices is used on buffer access, index on non-buffer access.
    #  The former contains a clean list of index parameters, the
    #  latter whatever Python object is needed for index access.
2336 2337 2338
    #
    #  is_fused_index boolean   Whether the index is used to specialize a
    #                           c(p)def function
2339

2340 2341 2342
    subexprs = ['base', 'index', 'indices']
    indices = None

2343 2344
    is_fused_index = False

2345 2346 2347 2348
    # Whether we're assigning to a buffer (in that case it needs to be
    # writable)
    writable_needed = False

2349 2350 2351
    # Whether we are indexing or slicing a memoryviewslice
    memslice_index = False
    memslice_slice = False
2352 2353
    is_memslice_copy = False
    memslice_ellipsis_noop = False
2354
    warned_untyped_idx = False
2355 2356
    # set by SingleAssignmentNode after analyse_types()
    is_memslice_scalar_assignment = False
2357

2358 2359 2360
    def __init__(self, pos, index, *args, **kw):
        ExprNode.__init__(self, pos, index=index, *args, **kw)
        self._index = index
2361 2362 2363 2364 2365

    def calculate_constant_result(self):
        self.constant_result = \
            self.base.constant_result[self.index.constant_result]

2366 2367 2368 2369 2370 2371 2372
    def compile_time_value(self, denv):
        base = self.base.compile_time_value(denv)
        index = self.index.compile_time_value(denv)
        try:
            return base[index]
        except Exception, e:
            self.compile_time_value_error(e)
2373

William Stein's avatar
William Stein committed
2374 2375
    def is_ephemeral(self):
        return self.base.is_ephemeral()
2376

2377
    def is_simple(self):
2378
        if self.is_buffer_access or self.memslice_index:
2379
            return False
2380 2381 2382
        elif self.memslice_slice:
            return True

2383 2384 2385 2386
        base = self.base
        return (base.is_simple() and self.index.is_simple()
                and base.type and (base.type.is_ptr or base.type.is_array))

William Stein's avatar
William Stein committed
2387 2388
    def analyse_target_declaration(self, env):
        pass
2389

2390 2391 2392
    def analyse_as_type(self, env):
        base_type = self.base.analyse_as_type(env)
        if base_type and not base_type.is_pyobject:
2393
            if base_type.is_cpp_class:
2394
                if isinstance(self.index, TupleNode):
2395 2396 2397 2398 2399
                    template_values = self.index.args
                else:
                    template_values = [self.index]
                import Nodes
                type_node = Nodes.TemplatedTypeNode(
2400 2401
                    pos = self.pos,
                    positional_args = template_values,
2402 2403 2404 2405
                    keyword_args = None)
                return type_node.analyse(env, base_type = base_type)
            else:
                return PyrexTypes.CArrayType(base_type, int(self.index.compile_time_value(env)))
2406
        return None
2407

Robert Bradshaw's avatar
Robert Bradshaw committed
2408
    def type_dependencies(self, env):
2409
        return self.base.type_dependencies(env) + self.index.type_dependencies(env)
2410

2411
    def infer_type(self, env):
2412 2413 2414 2415
        base_type = self.base.infer_type(env)
        if isinstance(self.index, SliceNode):
            # slicing!
            if base_type.is_string:
2416
                # sliced C strings must coerce to Python
2417
                return bytes_type
2418 2419 2420
            elif base_type in (unicode_type, bytes_type, str_type, list_type, tuple_type):
                # slicing these returns the same type
                return base_type
2421
            else:
2422 2423 2424
                # TODO: Handle buffers (hopefully without too much redundancy).
                return py_object_type

2425 2426
        index_type = self.index.infer_type(env)
        if index_type and index_type.is_int or isinstance(self.index, (IntNode, LongNode)):
2427 2428
            # indexing!
            if base_type is unicode_type:
2429 2430 2431
                # Py_UCS4 will automatically coerce to a unicode string
                # if required, so this is safe.  We only infer Py_UCS4
                # when the index is a C integer type.  Otherwise, we may
2432 2433 2434 2435
                # need to use normal Python item access, in which case
                # it's faster to return the one-char unicode string than
                # to receive it, throw it away, and potentially rebuild it
                # on a subsequent PyObject coercion.
2436
                return PyrexTypes.c_py_ucs4_type
2437 2438 2439
            elif base_type is str_type:
                # always returns str - Py2: bytes, Py3: unicode
                return base_type
2440 2441 2442 2443 2444 2445
            elif isinstance(self.base, BytesNode):
                #if env.global_scope().context.language_level >= 3:
                #    # infering 'char' can be made to work in Python 3 mode
                #    return PyrexTypes.c_char_type
                # Py2/3 return different types on indexing bytes objects
                return py_object_type
2446 2447
            elif base_type.is_ptr or base_type.is_array:
                return base_type.base_type
2448

2449
        # may be slicing or indexing, we don't know
2450 2451
        if base_type in (unicode_type, str_type):
            # these types always returns their own type on Python indexing/slicing
2452
            return base_type
2453 2454 2455
        else:
            # TODO: Handle buffers (hopefully without too much redundancy).
            return py_object_type
2456

William Stein's avatar
William Stein committed
2457
    def analyse_types(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2458
        self.analyse_base_and_index_types(env, getting = 1)
2459

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2460 2461
    def analyse_target_types(self, env):
        self.analyse_base_and_index_types(env, setting = 1)
2462 2463
        if not self.is_lvalue():
            error(self.pos, "Assignment to non-lvalue of type '%s'" % self.type)
2464

2465
    def analyse_base_and_index_types(self, env, getting = 0, setting = 0, analyse_base = True):
2466 2467 2468
        # Note: This might be cleaned up by having IndexNode
        # parsed in a saner way and only construct the tuple if
        # needed.
2469 2470 2471 2472

        # Note that this function must leave IndexNode in a cloneable state.
        # For buffers, self.index is packed out on the initial analysis, and
        # when cloning self.indices is copied.
2473 2474
        self.is_buffer_access = False

2475
        # a[...] = b
2476
        self.is_memslice_copy = False
2477 2478 2479 2480
        # incomplete indexing, Ellipsis indexing or slicing
        self.memslice_slice = False
        # integer indexing
        self.memslice_index = False
2481

2482 2483 2484
        if analyse_base:
            self.base.analyse_types(env)

2485 2486 2487 2488 2489
        if self.base.type.is_error:
            # Do not visit child tree if base is undeclared to avoid confusing
            # error messages
            self.type = PyrexTypes.error_type
            return
2490

2491
        is_slice = isinstance(self.index, SliceNode)
2492

2493
        # Potentially overflowing index value.
2494
        if not is_slice and isinstance(self.index, IntNode) and Utils.long_literal(self.index.value):
2495
            self.index = self.index.coerce_to_pyobject(env)
2496

2497 2498
        is_memslice = self.base.type.is_memoryviewslice

2499
        # Handle the case where base is a literal char* (and we expect a string, not an int)
2500
        if not is_memslice and (isinstance(self.base, BytesNode) or is_slice):
Robert Bradshaw's avatar
Robert Bradshaw committed
2501
            if self.base.type.is_string or not (self.base.type.is_ptr or self.base.type.is_array):
2502
                self.base = self.base.coerce_to_pyobject(env)
2503 2504 2505

        skip_child_analysis = False
        buffer_access = False
Mark Florisson's avatar
Mark Florisson committed
2506

2507 2508 2509 2510 2511 2512 2513
        if self.indices:
            indices = self.indices
        elif isinstance(self.index, TupleNode):
            indices = self.index.args
        else:
            indices = [self.index]

2514
        if (is_memslice and not self.indices and
Mark Florisson's avatar
Mark Florisson committed
2515
                isinstance(self.index, EllipsisNode)):
2516
            # Memoryviewslice copying
2517
            self.is_memslice_copy = True
2518 2519 2520 2521 2522 2523

        elif is_memslice:
            # memoryviewslice indexing or slicing
            import MemoryView

            skip_child_analysis = True
2524 2525
            have_slices, indices = MemoryView.unellipsify(indices,
                                                          self.base.type.ndim)
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544
            self.memslice_index = len(indices) == self.base.type.ndim
            axes = []

            index_type = PyrexTypes.c_py_ssize_t_type
            new_indices = []

            if len(indices) > self.base.type.ndim:
                self.type = error_type
                return error(indices[self.base.type.ndim].pos,
                             "Too many indices specified for type %s" %
                                                        self.base.type)

            suboffsets_dim = -1
            for i, index in enumerate(indices[:]):
                index.analyse_types(env)
                access, packing = self.base.type.axes[i]
                if isinstance(index, SliceNode):
                    suboffsets_dim = i
                    self.memslice_slice = True
2545 2546
                    if index.step.is_none:
                        axes.append((access, packing))
2547 2548
                    else:
                        axes.append((access, 'strided'))
2549 2550 2551 2552 2553 2554

                    # Coerce start, stop and step to temps of the right type
                    for attr in ('start', 'stop', 'step'):
                        value = getattr(index, attr)
                        if not value.is_none:
                            value = value.coerce_to(index_type, env)
2555
                            #value = value.coerce_to_temp(env)
2556 2557 2558
                            setattr(index, attr, value)
                            new_indices.append(value)

2559 2560 2561 2562 2563
                elif index.type.is_int or index.type.is_pyobject:
                    if index.type.is_pyobject and not self.warned_untyped_idx:
                        warning(index.pos, "Index should be typed for more "
                                           "efficient access", level=2)
                        IndexNode.warned_untyped_idx = True
2564

2565
                    self.memslice_index = True
2566
                    index = index.coerce_to(index_type, env)
2567 2568 2569
                    indices[i] = index
                    new_indices.append(index)

2570
                else:
2571 2572
                    self.type = error_type
                    return error(index.pos, "Invalid index for memoryview specified")
2573

2574 2575
            self.memslice_index = self.memslice_index and not self.memslice_slice
            self.original_indices = indices
2576 2577
            # All indices with all start/stop/step for slices.
            # We need to keep this around
2578
            self.indices = new_indices
2579 2580
            self.env = env

2581 2582
        elif self.base.type.is_buffer:
            # Buffer indexing
2583
            if len(indices) == self.base.type.ndim:
2584 2585 2586 2587 2588 2589
                buffer_access = True
                skip_child_analysis = True
                for x in indices:
                    x.analyse_types(env)
                    if not x.type.is_int:
                        buffer_access = False
2590

2591
            if buffer_access and not self.base.type.is_memoryviewslice:
Robert Bradshaw's avatar
Robert Bradshaw committed
2592
                assert hasattr(self.base, "entry") # Must be a NameNode-like node
2593

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2594 2595 2596
        # On cloning, indices is cloned. Otherwise, unpack index into indices
        assert not (buffer_access and isinstance(self.index, CloneNode))

2597 2598
        self.nogil = env.nogil

2599
        if buffer_access or self.memslice_index:
2600 2601 2602
            #if self.base.type.is_memoryviewslice and not self.base.is_name:
            #    self.base = self.base.coerce_to_temp(env)
            self.base = self.base.coerce_to_simple(env)
2603

2604
            self.indices = indices
2605
            self.index = None
2606 2607
            self.type = self.base.type.dtype
            self.is_buffer_access = True
2608
            self.buffer_type = self.base.type #self.base.entry.type
2609 2610

            if getting and self.type.is_pyobject:
2611
                self.is_temp = True
2612 2613

            if setting and self.base.type.is_memoryviewslice:
2614
                self.base.type.writable_needed = True
2615
            elif setting:
2616 2617 2618
                if not self.base.entry.type.writable:
                    error(self.pos, "Writing to readonly buffer")
                else:
2619
                    self.writable_needed = True
2620
                    if self.base.type.is_buffer:
2621
                        self.base.entry.buffer_aux.writable_needed = True
2622

2623
        elif self.is_memslice_copy:
Mark Florisson's avatar
Mark Florisson committed
2624 2625
            self.type = self.base.type
            if getting:
2626 2627 2628
                self.memslice_ellipsis_noop = True
            else:
                self.memslice_broadcast = True
2629

2630
        elif self.memslice_slice:
2631
            self.index = None
2632
            self.is_temp = True
2633
            self.use_managed_ref = True
2634 2635
            self.type = PyrexTypes.MemoryViewSliceType(
                            self.base.type.dtype, axes)
2636 2637 2638 2639 2640 2641

            if (self.base.type.is_memoryviewslice and not
                    self.base.is_name and not
                    self.base.result_in_temp()):
                self.base = self.base.coerce_to_temp(env)

2642 2643
            if setting:
                self.memslice_broadcast = True
2644

2645
        else:
2646
            base_type = self.base.type
2647 2648 2649 2650 2651 2652 2653 2654 2655

            fused_index_operation = base_type.is_cfunction and base_type.is_fused
            if not fused_index_operation:
                if isinstance(self.index, TupleNode):
                    self.index.analyse_types(env, skip_children=skip_child_analysis)
                elif not skip_child_analysis:
                    self.index.analyse_types(env)
                self.original_index_type = self.index.type

Stefan Behnel's avatar
Stefan Behnel committed
2656 2657
            if base_type.is_unicode_char:
                # we infer Py_UNICODE/Py_UCS4 for unicode strings in some
2658 2659 2660 2661 2662 2663 2664
                # cases, but indexing must still work for them
                if self.index.constant_result in (0, -1):
                    # FIXME: we know that this node is redundant -
                    # currently, this needs to get handled in Optimize.py
                    pass
                self.base = self.base.coerce_to_pyobject(env)
                base_type = self.base.type
2665
            if base_type.is_pyobject:
2666
                if self.index.type.is_int:
2667
                    if (not setting
2668
                        and (base_type in (list_type, tuple_type))
2669 2670
                        and (not self.index.type.signed
                             or not env.directives['wraparound']
2671 2672
                             or (isinstance(self.index, IntNode) and
                                 self.index.has_constant_result() and self.index.constant_result >= 0))
2673 2674 2675 2676
                        and not env.directives['boundscheck']):
                        self.is_temp = 0
                    else:
                        self.is_temp = 1
2677 2678 2679
                    self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
                else:
                    self.index = self.index.coerce_to_pyobject(env)
2680
                    self.is_temp = 1
2681
                if self.index.type.is_int and base_type is unicode_type:
Stefan Behnel's avatar
Stefan Behnel committed
2682
                    # Py_UNICODE/Py_UCS4 will automatically coerce to a unicode string
2683
                    # if required, so this is fast and safe
2684
                    self.type = PyrexTypes.c_py_ucs4_type
2685 2686
                elif is_slice and base_type in (bytes_type, str_type, unicode_type, list_type, tuple_type):
                    self.type = base_type
2687 2688
                else:
                    self.type = py_object_type
William Stein's avatar
William Stein committed
2689
            else:
2690 2691
                if base_type.is_ptr or base_type.is_array:
                    self.type = base_type.base_type
2692 2693 2694
                    if is_slice:
                        self.type = base_type
                    elif self.index.type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
2695 2696
                        self.index = self.index.coerce_to(
                            PyrexTypes.c_py_ssize_t_type, env)
2697
                    elif not self.index.type.is_int:
Robert Bradshaw's avatar
Robert Bradshaw committed
2698 2699 2700
                        error(self.pos,
                            "Invalid index type '%s'" %
                                self.index.type)
2701
                elif base_type.is_cpp_class:
2702
                    function = env.lookup_operator("[]", [self.base, self.index])
Robert Bradshaw's avatar
Robert Bradshaw committed
2703
                    if function is None:
2704
                        error(self.pos, "Indexing '%s' not supported for index type '%s'" % (base_type, self.index.type))
Robert Bradshaw's avatar
Robert Bradshaw committed
2705 2706 2707 2708 2709 2710 2711 2712 2713
                        self.type = PyrexTypes.error_type
                        self.result_code = "<error>"
                        return
                    func_type = function.type
                    if func_type.is_ptr:
                        func_type = func_type.base_type
                    self.index = self.index.coerce_to(func_type.args[0].type, env)
                    self.type = func_type.return_type
                    if setting and not func_type.return_type.is_reference:
Robert Bradshaw's avatar
Robert Bradshaw committed
2714
                        error(self.pos, "Can't set non-reference result '%s'" % self.type)
2715 2716
                elif fused_index_operation:
                    self.parse_indexed_fused_cdef(env)
2717 2718 2719
                else:
                    error(self.pos,
                        "Attempting to index non-array type '%s'" %
2720
                            base_type)
2721
                    self.type = PyrexTypes.error_type
Stefan Behnel's avatar
Stefan Behnel committed
2722

2723 2724 2725 2726 2727 2728 2729 2730 2731
    def parse_indexed_fused_cdef(self, env):
        """
        Interpret fused_cdef_func[specific_type1, ...]

        Note that if this method is called, we are an indexed cdef function
        with fused argument types, and this IndexNode will be replaced by the
        NameNode with specific entry just after analysis of expressions by
        AnalyseExpressionsTransform.
        """
2732
        self.type = PyrexTypes.error_type
2733

2734 2735
        self.is_fused_index = True

Mark Florisson's avatar
Mark Florisson committed
2736
        base_type = self.base.type
2737 2738 2739
        specific_types = []
        positions = []

2740
        if self.index.is_name or self.index.is_attribute:
2741 2742 2743 2744 2745
            positions.append(self.index.pos)
            specific_types.append(self.index.analyse_as_type(env))
        elif isinstance(self.index, TupleNode):
            for arg in self.index.args:
                positions.append(arg.pos)
2746 2747
                specific_type = arg.analyse_as_type(env)
                specific_types.append(specific_type)
2748
        else:
2749
            specific_types = [False]
2750

2751 2752 2753 2754
        if not Utils.all(specific_types):
            self.index.analyse_types(env)

            if not self.base.entry.as_variable:
2755
                error(self.pos, "Can only index fused functions with types")
2756 2757
            else:
                # A cpdef function indexed with Python objects
2758 2759
                self.base.entry = self.entry = self.base.entry.as_variable
                self.base.type = self.type = self.entry.type
2760

2761 2762 2763 2764 2765 2766
                self.base.is_temp = True
                self.is_temp = True

                self.entry.used = True

            self.is_fused_index = False
2767 2768
            return

2769 2770
        fused_types = base_type.get_fused_types()
        if len(specific_types) > len(fused_types):
2771 2772 2773 2774 2775
            return error(self.pos, "Too many types specified")
        elif len(specific_types) < len(fused_types):
            t = fused_types[len(specific_types)]
            return error(self.pos, "Not enough types specified to specialize "
                                   "the function, %s is still fused" % t)
2776 2777 2778 2779 2780 2781 2782

        # See if our index types form valid specializations
        for pos, specific_type, fused_type in zip(positions,
                                                  specific_types,
                                                  fused_types):
            if not Utils.any([specific_type.same_as(t)
                                  for t in fused_type.types]):
2783
                return error(pos, "Type not in fused type")
2784 2785 2786 2787 2788 2789 2790

            if specific_type is None or specific_type.is_error:
                return

        fused_to_specific = dict(zip(fused_types, specific_types))
        type = base_type.specialize(fused_to_specific)

2791 2792 2793 2794 2795
        if type.is_fused:
            # Only partially specific, this is invalid
            error(self.pos,
                  "Index operation makes function only partially specific")
        else:
2796
            # Fully specific, find the signature with the specialized entry
2797
            for signature in self.base.type.get_all_specialized_function_types():
2798 2799
                if type.same_as(signature):
                    self.type = signature
Mark Florisson's avatar
Mark Florisson committed
2800 2801 2802 2803 2804

                    if self.base.is_attribute:
                        # Pretend to be a normal attribute, for cdef extension
                        # methods
                        self.entry = signature.entry
2805
                        self.is_attribute = True
Mark Florisson's avatar
Mark Florisson committed
2806
                        self.obj = self.base.obj
2807 2808 2809 2810

                    self.type.entry.used = True
                    self.base.type = signature
                    self.base.entry = signature.entry
Mark Florisson's avatar
Mark Florisson committed
2811

2812 2813
                    break
            else:
Mark Florisson's avatar
Mark Florisson committed
2814 2815
                # This is a bug
                raise InternalError("Couldn't find the right signature")
2816

2817 2818
    gil_message = "Indexing Python object"

2819
    def nogil_check(self, env):
2820 2821
        if self.is_buffer_access or self.memslice_index or self.memslice_slice:
            if not self.memslice_slice and env.directives['boundscheck']:
2822 2823 2824 2825 2826
                # error(self.pos, "Cannot check buffer index bounds without gil; "
                #                 "use boundscheck(False) directive")
                warning(self.pos, "Use boundscheck(False) for faster access",
                        level=1)
            if self.type.is_pyobject:
2827 2828
                error(self.pos, "Cannot access buffer with object dtype without gil")
                return
2829
        super(IndexNode, self).nogil_check(env)
2830 2831


William Stein's avatar
William Stein committed
2832
    def check_const_addr(self):
2833
        return self.base.check_const_addr() and self.index.check_const()
2834

William Stein's avatar
William Stein committed
2835
    def is_lvalue(self):
2836 2837 2838 2839 2840
        base_type = self.base.type
        if self.type.is_ptr or self.type.is_array:
            return not base_type.base_type.is_array
        else:
            return True
Dag Sverre Seljebotn's avatar
merge  
Dag Sverre Seljebotn committed
2841

William Stein's avatar
William Stein committed
2842
    def calculate_result_code(self):
2843
        if self.is_buffer_access:
2844
            return "(*%s)" % self.buffer_ptr_code
2845 2846
        elif self.is_memslice_copy:
            return self.base.result()
2847 2848 2849 2850
        elif self.base.type is list_type:
            return "PyList_GET_ITEM(%s, %s)" % (self.base.result(), self.index.result())
        elif self.base.type is tuple_type:
            return "PyTuple_GET_ITEM(%s, %s)" % (self.base.result(), self.index.result())
2851 2852
        elif (self.type.is_ptr or self.type.is_array) and self.type == self.base.type:
            error(self.pos, "Invalid use of pointer slice")
2853 2854
        else:
            return "(%s[%s])" % (
2855
                self.base.result(), self.index.result())
2856

2857
    def extra_index_params(self):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2858 2859
        if self.index.type.is_int:
            if self.original_index_type.signed:
2860
                size_adjustment = ""
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2861
            else:
2862 2863
                size_adjustment = "+1"
            return ", sizeof(%s)%s, %s" % (self.original_index_type.declaration_code(""), size_adjustment, self.original_index_type.to_py_function)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2864 2865
        else:
            return ""
2866 2867 2868

    def generate_subexpr_evaluation_code(self, code):
        self.base.generate_evaluation_code(code)
2869
        if self.indices is None:
2870 2871
            self.index.generate_evaluation_code(code)
        else:
2872 2873
            for i in self.indices:
                i.generate_evaluation_code(code)
2874

2875 2876
    def generate_subexpr_disposal_code(self, code):
        self.base.generate_disposal_code(code)
2877
        if self.indices is None:
2878 2879
            self.index.generate_disposal_code(code)
        else:
2880 2881
            for i in self.indices:
                i.generate_disposal_code(code)
2882

2883 2884
    def free_subexpr_temps(self, code):
        self.base.free_temps(code)
2885
        if self.indices is None:
2886 2887 2888 2889 2890
            self.index.free_temps(code)
        else:
            for i in self.indices:
                i.free_temps(code)

William Stein's avatar
William Stein committed
2891
    def generate_result_code(self, code):
2892
        if self.is_buffer_access or self.memslice_index:
2893
            self.nonecheck(code)
2894
            buffer_entry, self.buffer_ptr_code = self.buffer_lookup_code(code)
2895 2896 2897
            if self.type.is_pyobject:
                # is_temp is True, so must pull out value and incref it.
                code.putln("%s = *%s;" % (self.result(), self.buffer_ptr_code))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2898
                code.putln("__Pyx_INCREF((PyObject*)%s);" % self.result())
2899 2900

        elif self.memslice_slice:
2901
            self.nonecheck(code)
2902 2903
            self.put_memoryviewslice_slice_code(code)

2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914
        elif self.is_temp:
            if self.type.is_pyobject:
                if self.index.type.is_int:
                    index_code = self.index.result()
                    if self.base.type is list_type:
                        function = "__Pyx_GetItemInt_List"
                    elif self.base.type is tuple_type:
                        function = "__Pyx_GetItemInt_Tuple"
                    else:
                        function = "__Pyx_GetItemInt"
                    code.globalstate.use_utility_code(getitem_int_utility_code)
2915
                else:
2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931
                    index_code = self.index.py_result()
                    if self.base.type is dict_type:
                        function = "__Pyx_PyDict_GetItem"
                        code.globalstate.use_utility_code(getitem_dict_utility_code)
                    else:
                        function = "PyObject_GetItem"
                code.putln(
                    "%s = %s(%s, %s%s); if (!%s) %s" % (
                        self.result(),
                        function,
                        self.base.py_result(),
                        index_code,
                        self.extra_index_params(),
                        self.result(),
                        code.error_goto(self.pos)))
                code.put_gotref(self.py_result())
Stefan Behnel's avatar
Stefan Behnel committed
2932
            elif self.type.is_unicode_char and self.base.type is unicode_type:
2933 2934 2935
                assert self.index.type.is_int
                index_code = self.index.result()
                function = "__Pyx_GetItemInt_Unicode"
2936 2937
                code.globalstate.use_utility_code(getitem_int_pyunicode_utility_code)
                code.putln(
2938
                    "%s = %s(%s, %s%s); if (unlikely(%s == (Py_UCS4)-1)) %s;" % (
2939 2940 2941 2942 2943 2944 2945
                        self.result(),
                        function,
                        self.base.py_result(),
                        index_code,
                        self.extra_index_params(),
                        self.result(),
                        code.error_goto(self.pos)))
2946

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2947 2948 2949
    def generate_setitem_code(self, value_code, code):
        if self.index.type.is_int:
            function = "__Pyx_SetItemInt"
2950
            index_code = self.index.result()
2951
            code.globalstate.use_utility_code(setitem_int_utility_code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2952 2953
        else:
            index_code = self.index.py_result()
2954 2955
            if self.base.type is dict_type:
                function = "PyDict_SetItem"
Craig Citro's avatar
Craig Citro committed
2956
            # It would seem that we could specialized lists/tuples, but that
2957 2958 2959 2960 2961 2962
            # shouldn't happen here.
            # Both PyList_SetItem PyTuple_SetItem and a Py_ssize_t as input,
            # not a PyObject*, and bad conversion here would give the wrong
            # exception. Also, tuples are supposed to be immutable, and raise
            # TypeErrors when trying to set their entries (PyTuple_SetItem
            # is for creating new tuples from).
2963 2964
            else:
                function = "PyObject_SetItem"
2965
        code.putln(
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2966 2967
            "if (%s(%s, %s, %s%s) < 0) %s" % (
                function,
2968
                self.base.py_result(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2969 2970
                index_code,
                value_code,
2971
                self.extra_index_params(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2972
                code.error_goto(self.pos)))
2973 2974 2975

    def generate_buffer_setitem_code(self, rhs, code, op=""):
        # Used from generate_assignment_code and InPlaceAssignmentNode
2976
        self.nonecheck(code)
2977 2978
        buffer_entry, ptrexpr = self.buffer_lookup_code(code)

2979 2980 2981
        if self.buffer_type.dtype.is_pyobject:
            # Must manage refcounts. Decref what is already there
            # and incref what we put in.
2982 2983
            ptr = code.funcstate.allocate_temp(buffer_entry.buf_ptr_type,
                                               manage_ref=False)
2984
            rhs_code = rhs.result()
2985
            code.putln("%s = %s;" % (ptr, ptrexpr))
2986
            code.put_gotref("*%s" % ptr)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2987
            code.putln("__Pyx_DECREF(*%s); __Pyx_INCREF(%s);" % (
2988 2989 2990
                ptr, rhs_code
                ))
            code.putln("*%s %s= %s;" % (ptr, op, rhs_code))
2991
            code.put_giveref("*%s" % ptr)
2992
            code.funcstate.release_temp(ptr)
2993
        else:
2994
            # Simple case
2995
            code.putln("*%s %s= %s;" % (ptrexpr, op, rhs.result()))
2996

William Stein's avatar
William Stein committed
2997
    def generate_assignment_code(self, rhs, code):
2998 2999 3000 3001 3002 3003 3004
        generate_evaluation_code = (self.is_memslice_scalar_assignment or
                                    self.memslice_slice)
        if generate_evaluation_code:
            self.generate_evaluation_code(code)
        else:
            self.generate_subexpr_evaluation_code(code)

3005
        if self.is_buffer_access or self.memslice_index:
3006
            self.generate_buffer_setitem_code(rhs, code)
3007 3008
        elif self.is_memslice_scalar_assignment:
            self.generate_memoryviewslice_assign_scalar_code(rhs, code)
3009
        elif self.memslice_slice or self.is_memslice_copy:
3010
            self.generate_memoryviewslice_setslice_code(rhs, code)
3011
        elif self.type.is_pyobject:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3012
            self.generate_setitem_code(rhs.py_result(), code)
William Stein's avatar
William Stein committed
3013 3014 3015
        else:
            code.putln(
                "%s = %s;" % (
3016
                    self.result(), rhs.result()))
3017 3018 3019 3020 3021 3022 3023

        if generate_evaluation_code:
            self.generate_disposal_code(code)
        else:
            self.generate_subexpr_disposal_code(code)
            self.free_subexpr_temps(code)

William Stein's avatar
William Stein committed
3024
        rhs.generate_disposal_code(code)
3025
        rhs.free_temps(code)
3026

William Stein's avatar
William Stein committed
3027 3028
    def generate_deletion_code(self, code):
        self.generate_subexpr_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3029 3030
        #if self.type.is_pyobject:
        if self.index.type.is_int:
3031
            function = "__Pyx_DelItemInt"
3032
            index_code = self.index.result()
3033
            code.globalstate.use_utility_code(delitem_int_utility_code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3034 3035
        else:
            index_code = self.index.py_result()
3036 3037 3038 3039
            if self.base.type is dict_type:
                function = "PyDict_DelItem"
            else:
                function = "PyObject_DelItem"
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3040
        code.putln(
3041
            "if (%s(%s, %s%s) < 0) %s" % (
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3042
                function,
William Stein's avatar
William Stein committed
3043
                self.base.py_result(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3044
                index_code,
3045
                self.extra_index_params(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3046
                code.error_goto(self.pos)))
William Stein's avatar
William Stein committed
3047
        self.generate_subexpr_disposal_code(code)
3048
        self.free_subexpr_temps(code)
3049

3050 3051 3052 3053 3054 3055
    def buffer_entry(self):
        import Buffer, MemoryView

        if self.base.is_name:
            entry = self.base.entry
        else:
3056 3057
            # SimpleCallNode is_simple is not consistent with coerce_to_simple
            assert self.base.is_simple() or self.base.is_temp
3058 3059 3060 3061 3062 3063 3064 3065 3066 3067
            cname = self.base.result()
            entry = Symtab.Entry(cname, cname, self.base.type, self.base.pos)

        if entry.type.is_buffer:
            buffer_entry = Buffer.BufferEntry(entry)
        else:
            buffer_entry = MemoryView.MemoryViewSliceBufferEntry(entry)

        return buffer_entry

3068
    def buffer_lookup_code(self, code):
3069
        "ndarray[1, 2, 3] and memslice[1, 2, 3]"
3070
        # Assign indices to temps
3071 3072 3073
        index_temps = [code.funcstate.allocate_temp(i.type, manage_ref=False)
                           for i in self.indices]

3074
        for temp, index in zip(index_temps, self.indices):
3075
            code.putln("%s = %s;" % (temp, index.result()))
3076

3077
        # Generate buffer access code using these temps
3078
        import Buffer, MemoryView
3079

3080
        buffer_entry = self.buffer_entry()
3081

3082
        if buffer_entry.type.is_buffer:
3083
            negative_indices = buffer_entry.type.negative_indices
3084 3085 3086
        else:
            negative_indices = Buffer.buffer_defaults['negative_indices']

3087 3088 3089 3090 3091 3092
        return buffer_entry, Buffer.put_buffer_lookup_code(
               entry=buffer_entry,
               index_signeds=[i.type.signed for i in self.indices],
               index_cnames=index_temps,
               directives=code.globalstate.directives,
               pos=self.pos, code=code,
3093 3094
               negative_indices=negative_indices,
               in_nogil_context=self.in_nogil_context)
William Stein's avatar
William Stein committed
3095

3096
    def put_memoryviewslice_slice_code(self, code):
3097
        "memslice[:]"
3098
        buffer_entry = self.buffer_entry()
3099
        have_gil = not self.in_nogil_context
3100

3101 3102 3103 3104 3105 3106
        if sys.version_info < (3,):
            def next_(it):
                return it.next()
        else:
            next_ = next

3107 3108 3109 3110 3111 3112 3113
        have_slices = False
        it = iter(self.indices)
        for index in self.original_indices:
            is_slice = isinstance(index, SliceNode)
            have_slices = have_slices or is_slice
            if is_slice:
                if not index.start.is_none:
3114
                    index.start = next_(it)
3115
                if not index.stop.is_none:
3116
                    index.stop = next_(it)
3117
                if not index.step.is_none:
3118
                    index.step = next_(it)
3119
            else:
3120
                next_(it)
3121 3122 3123 3124

        assert not list(it)

        buffer_entry.generate_buffer_slice_code(code, self.original_indices,
3125
                                                self.result(),
3126 3127
                                                have_gil=have_gil,
                                                have_slices=have_slices)
William Stein's avatar
William Stein committed
3128

3129
    def generate_memoryviewslice_setslice_code(self, rhs, code):
3130
        "memslice1[...] = memslice2 or memslice1[:] = memslice2"
3131 3132 3133
        import MemoryView
        MemoryView.copy_broadcast_memview_src_to_dst(rhs, self, code)

3134 3135 3136 3137 3138
    def generate_memoryviewslice_assign_scalar_code(self, rhs, code):
        "memslice1[...] = 0.0 or memslice1[:] = 0.0"
        import MemoryView
        MemoryView.assign_scalar(self, rhs, code)

3139 3140 3141 3142
    def nonecheck(self, code):
        if code.globalstate.directives['nonecheck']:
            self.put_nonecheck(code)

3143
    def put_nonecheck(self, code):
3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155
        if self.base.type.is_memoryviewslice:
            code.globalstate.use_utility_code(
                raise_noneindex_memview_error_utility_code)
            code.putln("if (unlikely((PyObject *) %s.memview == Py_None)) {" %
                                                         self.base.result())
            code.putln("__Pyx_RaiseNoneMemviewIndexingError();")
        else:
            code.globalstate.use_utility_code(raise_noneindex_error_utility_code)
            code.putln("if (%s) {" % code.unlikely("%s == Py_None") %
                            self.base.result_as(PyrexTypes.py_object_type))
            code.putln("__Pyx_RaiseNoneIndexingError();")

3156 3157 3158
        code.putln(code.error_goto(self.pos))
        code.putln("}")

3159

3160
class SliceIndexNode(ExprNode):
William Stein's avatar
William Stein committed
3161 3162 3163 3164 3165
    #  2-element slice indexing
    #
    #  base      ExprNode
    #  start     ExprNode or None
    #  stop      ExprNode or None
3166

William Stein's avatar
William Stein committed
3167
    subexprs = ['base', 'start', 'stop']
3168

3169 3170 3171 3172 3173 3174 3175
    def infer_type(self, env):
        base_type = self.base.infer_type(env)
        if base_type.is_string:
            return bytes_type
        elif base_type in (bytes_type, str_type, unicode_type,
                           list_type, tuple_type):
            return base_type
3176 3177
        elif base_type.is_ptr or base_type.is_array:
            return PyrexTypes.c_array_type(base_type.base_type, None)
3178 3179
        return py_object_type

3180 3181 3182 3183
    def calculate_constant_result(self):
        self.constant_result = self.base.constant_result[
            self.start.constant_result : self.stop.constant_result]

3184 3185
    def compile_time_value(self, denv):
        base = self.base.compile_time_value(denv)
3186 3187 3188 3189 3190 3191 3192 3193
        if self.start is None:
            start = 0
        else:
            start = self.start.compile_time_value(denv)
        if self.stop is None:
            stop = None
        else:
            stop = self.stop.compile_time_value(denv)
3194 3195 3196 3197
        try:
            return base[start:stop]
        except Exception, e:
            self.compile_time_value_error(e)
3198

William Stein's avatar
William Stein committed
3199 3200
    def analyse_target_declaration(self, env):
        pass
3201

3202
    def analyse_target_types(self, env):
3203
        self.analyse_types(env, getting=False)
3204
        # when assigning, we must accept any Python type
3205 3206
        if self.type.is_pyobject:
            self.type = py_object_type
William Stein's avatar
William Stein committed
3207

3208
    def analyse_types(self, env, getting=True):
William Stein's avatar
William Stein committed
3209
        self.base.analyse_types(env)
3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229

        if self.base.type.is_memoryviewslice:
            # Gross hack here! But we do not know the type until this point,
            # and we cannot create and return a new node. So we change the
            # type...
            none_node = NoneNode(self.pos)
            index = SliceNode(self.pos,
                              start=self.start or none_node,
                              stop=self.stop or none_node,
                              step=none_node)
            del self.start
            del self.stop
            self.index = index
            self.__class__ = IndexNode
            self.analyse_base_and_index_types(env,
                                              getting=getting,
                                              setting=not getting,
                                              analyse_base=False)
            return

William Stein's avatar
William Stein committed
3230 3231 3232 3233
        if self.start:
            self.start.analyse_types(env)
        if self.stop:
            self.stop.analyse_types(env)
3234 3235 3236
        base_type = self.base.type
        if base_type.is_string:
            self.type = bytes_type
3237 3238 3239
        elif base_type.is_ptr:
            self.type = base_type
        elif base_type.is_array:
3240 3241 3242
            # we need a ptr type here instead of an array type, as
            # array types can result in invalid type casts in the C
            # code
3243
            self.type = PyrexTypes.CPtrType(base_type.base_type)
3244 3245 3246
        else:
            self.base = self.base.coerce_to_pyobject(env)
            self.type = py_object_type
3247 3248 3249
        if base_type.is_builtin_type:
            # slicing builtin types returns something of the same type
            self.type = base_type
3250
        c_int = PyrexTypes.c_py_ssize_t_type
William Stein's avatar
William Stein committed
3251 3252 3253 3254 3255
        if self.start:
            self.start = self.start.coerce_to(c_int, env)
        if self.stop:
            self.stop = self.stop.coerce_to(c_int, env)
        self.is_temp = 1
3256

3257
    nogil_check = Node.gil_error
3258 3259
    gil_message = "Slicing Python object"

William Stein's avatar
William Stein committed
3260
    def generate_result_code(self, code):
3261 3262 3263 3264
        if not self.type.is_pyobject:
            error(self.pos,
                  "Slicing is not currently supported for '%s'." % self.type)
            return
Robert Bradshaw's avatar
Robert Bradshaw committed
3265 3266 3267
        if self.base.type.is_string:
            if self.stop is None:
                code.putln(
3268
                    "%s = PyBytes_FromString(%s + %s); %s" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
3269 3270 3271 3272 3273 3274
                        self.result(),
                        self.base.result(),
                        self.start_code(),
                        code.error_goto_if_null(self.result(), self.pos)))
            else:
                code.putln(
3275
                    "%s = PyBytes_FromStringAndSize(%s + %s, %s - %s); %s" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
3276 3277 3278 3279 3280 3281 3282 3283
                        self.result(),
                        self.base.result(),
                        self.start_code(),
                        self.stop_code(),
                        self.start_code(),
                        code.error_goto_if_null(self.result(), self.pos)))
        else:
            code.putln(
3284
                "%s = __Pyx_PySequence_GetSlice(%s, %s, %s); %s" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
3285 3286 3287 3288 3289
                    self.result(),
                    self.base.py_result(),
                    self.start_code(),
                    self.stop_code(),
                    code.error_goto_if_null(self.result(), self.pos)))
3290
        code.put_gotref(self.py_result())
3291

William Stein's avatar
William Stein committed
3292 3293
    def generate_assignment_code(self, rhs, code):
        self.generate_subexpr_evaluation_code(code)
3294
        if self.type.is_pyobject:
3295
            code.put_error_if_neg(self.pos,
3296
                "__Pyx_PySequence_SetSlice(%s, %s, %s, %s)" % (
3297 3298 3299
                    self.base.py_result(),
                    self.start_code(),
                    self.stop_code(),
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3300
                    rhs.py_result()))
3301 3302 3303 3304 3305 3306 3307 3308
        else:
            start_offset = ''
            if self.start:
                start_offset = self.start_code()
                if start_offset == '0':
                    start_offset = ''
                else:
                    start_offset += '+'
Stefan Behnel's avatar
Stefan Behnel committed
3309 3310
            if rhs.type.is_array:
                array_length = rhs.type.size
3311
                self.generate_slice_guard_code(code, array_length)
Stefan Behnel's avatar
Stefan Behnel committed
3312
            else:
Stefan Behnel's avatar
Stefan Behnel committed
3313 3314
                error(self.pos,
                      "Slice assignments from pointers are not yet supported.")
Stefan Behnel's avatar
Stefan Behnel committed
3315 3316
                # FIXME: fix the array size according to start/stop
                array_length = self.base.type.size
3317 3318 3319 3320
            for i in range(array_length):
                code.putln("%s[%s%s] = %s[%d];" % (
                        self.base.result(), start_offset, i,
                        rhs.result(), i))
William Stein's avatar
William Stein committed
3321
        self.generate_subexpr_disposal_code(code)
3322
        self.free_subexpr_temps(code)
William Stein's avatar
William Stein committed
3323
        rhs.generate_disposal_code(code)
3324
        rhs.free_temps(code)
William Stein's avatar
William Stein committed
3325 3326

    def generate_deletion_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
3327
        if not self.base.type.is_pyobject:
3328 3329 3330
            error(self.pos,
                  "Deleting slices is only supported for Python types, not '%s'." % self.type)
            return
William Stein's avatar
William Stein committed
3331
        self.generate_subexpr_evaluation_code(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
3332
        code.put_error_if_neg(self.pos,
3333
            "__Pyx_PySequence_DelSlice(%s, %s, %s)" % (
William Stein's avatar
William Stein committed
3334 3335
                self.base.py_result(),
                self.start_code(),
Robert Bradshaw's avatar
Robert Bradshaw committed
3336
                self.stop_code()))
William Stein's avatar
William Stein committed
3337
        self.generate_subexpr_disposal_code(code)
3338
        self.free_subexpr_temps(code)
3339 3340 3341 3342 3343 3344 3345 3346 3347 3348

    def generate_slice_guard_code(self, code, target_size):
        if not self.base.type.is_array:
            return
        slice_size = self.base.type.size
        start = stop = None
        if self.stop:
            stop = self.stop.result()
            try:
                stop = int(stop)
Stefan Behnel's avatar
Stefan Behnel committed
3349
                if stop < 0:
3350
                    slice_size = self.base.type.size + stop
Stefan Behnel's avatar
Stefan Behnel committed
3351 3352
                else:
                    slice_size = stop
3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382
                stop = None
            except ValueError:
                pass
        if self.start:
            start = self.start.result()
            try:
                start = int(start)
                if start < 0:
                    start = self.base.type.size + start
                slice_size -= start
                start = None
            except ValueError:
                pass
        check = None
        if slice_size < 0:
            if target_size > 0:
                error(self.pos, "Assignment to empty slice.")
        elif start is None and stop is None:
            # we know the exact slice length
            if target_size != slice_size:
                error(self.pos, "Assignment to slice of wrong length, expected %d, got %d" % (
                        slice_size, target_size))
        elif start is not None:
            if stop is None:
                stop = slice_size
            check = "(%s)-(%s)" % (stop, start)
        else: # stop is not None:
            check = stop
        if check:
            code.putln("if (unlikely((%s) != %d)) {" % (check, target_size))
3383
            code.putln('PyErr_Format(PyExc_ValueError, "Assignment to slice of wrong length, expected %%" PY_FORMAT_SIZE_T "d, got %%" PY_FORMAT_SIZE_T "d", (Py_ssize_t)%d, (Py_ssize_t)(%s));' % (
3384 3385 3386
                        target_size, check))
            code.putln(code.error_goto(self.pos))
            code.putln("}")
3387

William Stein's avatar
William Stein committed
3388 3389
    def start_code(self):
        if self.start:
3390
            return self.start.result()
William Stein's avatar
William Stein committed
3391 3392
        else:
            return "0"
3393

William Stein's avatar
William Stein committed
3394 3395
    def stop_code(self):
        if self.stop:
3396
            return self.stop.result()
3397 3398
        elif self.base.type.is_array:
            return self.base.type.size
William Stein's avatar
William Stein committed
3399
        else:
3400
            return "PY_SSIZE_T_MAX"
3401

William Stein's avatar
William Stein committed
3402
    def calculate_result_code(self):
3403
        # self.result() is not used, but this method must exist
William Stein's avatar
William Stein committed
3404
        return "<unused>"
3405

William Stein's avatar
William Stein committed
3406

3407
class SliceNode(ExprNode):
William Stein's avatar
William Stein committed
3408 3409 3410 3411 3412
    #  start:stop:step in subscript list
    #
    #  start     ExprNode
    #  stop      ExprNode
    #  step      ExprNode
3413

3414 3415
    subexprs = ['start', 'stop', 'step']

3416 3417
    type = py_object_type
    is_temp = 1
3418 3419

    def calculate_constant_result(self):
3420 3421 3422 3423
        self.constant_result = slice(
            self.start.constant_result,
            self.stop.constant_result,
            self.step.constant_result)
3424

3425 3426
    def compile_time_value(self, denv):
        start = self.start.compile_time_value(denv)
Stefan Behnel's avatar
Stefan Behnel committed
3427 3428
        stop = self.stop.compile_time_value(denv)
        step = self.step.compile_time_value(denv)
3429 3430 3431 3432 3433
        try:
            return slice(start, stop, step)
        except Exception, e:
            self.compile_time_value_error(e)

William Stein's avatar
William Stein committed
3434 3435 3436 3437 3438 3439 3440
    def analyse_types(self, env):
        self.start.analyse_types(env)
        self.stop.analyse_types(env)
        self.step.analyse_types(env)
        self.start = self.start.coerce_to_pyobject(env)
        self.stop = self.stop.coerce_to_pyobject(env)
        self.step = self.step.coerce_to_pyobject(env)
3441 3442 3443
        if self.start.is_literal and self.stop.is_literal and self.step.is_literal:
            self.is_literal = True
            self.is_temp = False
3444 3445 3446

    gil_message = "Constructing Python slice object"

3447 3448 3449
    def calculate_result_code(self):
        return self.result_code

William Stein's avatar
William Stein committed
3450
    def generate_result_code(self, code):
3451 3452 3453 3454 3455
        if self.is_literal:
            self.result_code = code.get_py_const(py_object_type, 'slice_', cleanup_level=2)
            code = code.get_cached_constants_writer()
            code.mark_pos(self.pos)

William Stein's avatar
William Stein committed
3456
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
3457
            "%s = PySlice_New(%s, %s, %s); %s" % (
3458
                self.result(),
3459 3460
                self.start.py_result(),
                self.stop.py_result(),
William Stein's avatar
William Stein committed
3461
                self.step.py_result(),
3462
                code.error_goto_if_null(self.result(), self.pos)))
3463
        code.put_gotref(self.py_result())
3464 3465
        if self.is_literal:
            code.put_giveref(self.py_result())
William Stein's avatar
William Stein committed
3466

3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479
    def __deepcopy__(self, memo):
        """
        There is a copy bug in python 2.4 for slice objects.
        """
        return SliceNode(
            self.pos,
            start=copy.deepcopy(self.start, memo),
            stop=copy.deepcopy(self.stop, memo),
            step=copy.deepcopy(self.step, memo),
            is_temp=self.is_temp,
            is_literal=self.is_literal,
            constant_result=self.constant_result)

3480

3481
class CallNode(ExprNode):
3482

Stefan Behnel's avatar
Stefan Behnel committed
3483 3484 3485
    # allow overriding the default 'may_be_none' behaviour
    may_return_none = None

3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506
    def infer_type(self, env):
        function = self.function
        func_type = function.infer_type(env)
        if isinstance(self.function, NewExprNode):
            return PyrexTypes.CPtrType(self.function.class_type)
        if func_type.is_ptr:
            func_type = func_type.base_type
        if func_type.is_cfunction:
            return func_type.return_type
        elif func_type is type_type:
            if function.is_name and function.entry and function.entry.type:
                result_type = function.entry.type
                if result_type.is_extension_type:
                    return result_type
                elif result_type.is_builtin_type:
                    if function.entry.name == 'float':
                        return PyrexTypes.c_double_type
                    elif function.entry.name in Builtin.types_that_construct_their_instance:
                        return result_type
        return py_object_type

Robert Bradshaw's avatar
Robert Bradshaw committed
3507 3508 3509 3510 3511
    def type_dependencies(self, env):
        # TODO: Update when Danilo's C++ code merged in to handle the
        # the case of function overloading.
        return self.function.type_dependencies(env)

Stefan Behnel's avatar
Stefan Behnel committed
3512 3513 3514 3515 3516
    def may_be_none(self):
        if self.may_return_none is not None:
            return self.may_return_none
        return ExprNode.may_be_none(self)

Robert Bradshaw's avatar
Robert Bradshaw committed
3517 3518 3519 3520 3521 3522
    def analyse_as_type_constructor(self, env):
        type = self.function.analyse_as_type(env)
        if type and type.is_struct_or_union:
            args, kwds = self.explicit_args_kwds()
            items = []
            for arg, member in zip(args, type.scope.var_entries):
3523
                items.append(DictItemNode(pos=arg.pos, key=StringNode(pos=arg.pos, value=member.name), value=arg))
Robert Bradshaw's avatar
Robert Bradshaw committed
3524 3525 3526 3527 3528 3529 3530
            if kwds:
                items += kwds.key_value_pairs
            self.key_value_pairs = items
            self.__class__ = DictNode
            self.analyse_types(env)
            self.coerce_to(type, env)
            return True
3531 3532 3533 3534 3535 3536 3537 3538 3539
        elif type and type.is_cpp_class:
            for arg in self.args:
                arg.analyse_types(env)
            constructor = type.scope.lookup("<init>")
            self.function = RawCNameExprNode(self.function.pos, constructor.type)
            self.function.entry = constructor
            self.function.set_cname(type.declaration_code(""))
            self.analyse_c_function_call(env)
            return True
3540

3541 3542
    def is_lvalue(self):
        return self.type.is_reference
3543

3544
    def nogil_check(self, env):
3545 3546
        func_type = self.function_type()
        if func_type.is_pyobject:
3547
            self.gil_error()
3548
        elif not getattr(func_type, 'nogil', False):
3549
            self.gil_error()
3550 3551 3552

    gil_message = "Calling gil-requiring function"

3553 3554

class SimpleCallNode(CallNode):
William Stein's avatar
William Stein committed
3555 3556 3557 3558 3559 3560 3561
    #  Function call without keyword, * or ** args.
    #
    #  function       ExprNode
    #  args           [ExprNode]
    #  arg_tuple      ExprNode or None     used internally
    #  self           ExprNode or None     used internally
    #  coerced_self   ExprNode or None     used internally
3562
    #  wrapper_call   bool                 used internally
3563
    #  has_optional_args   bool            used internally
3564
    #  nogil          bool                 used internally
3565

William Stein's avatar
William Stein committed
3566
    subexprs = ['self', 'coerced_self', 'function', 'args', 'arg_tuple']
3567

William Stein's avatar
William Stein committed
3568 3569 3570
    self = None
    coerced_self = None
    arg_tuple = None
3571
    wrapper_call = False
3572
    has_optional_args = False
3573
    nogil = False
3574
    analysed = False
3575

3576 3577 3578 3579 3580 3581 3582
    def compile_time_value(self, denv):
        function = self.function.compile_time_value(denv)
        args = [arg.compile_time_value(denv) for arg in self.args]
        try:
            return function(*args)
        except Exception, e:
            self.compile_time_value_error(e)
3583

3584
    def analyse_as_type(self, env):
3585
        attr = self.function.as_cython_attribute()
3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597
        if attr == 'pointer':
            if len(self.args) != 1:
                error(self.args.pos, "only one type allowed.")
            else:
                type = self.args[0].analyse_as_type(env)
                if not type:
                    error(self.args[0].pos, "Unknown type")
                else:
                    return PyrexTypes.CPtrType(type)

    def explicit_args_kwds(self):
        return self.args, None
3598

William Stein's avatar
William Stein committed
3599
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
3600 3601
        if self.analyse_as_type_constructor(env):
            return
3602 3603 3604
        if self.analysed:
            return
        self.analysed = True
William Stein's avatar
William Stein committed
3605 3606 3607
        function = self.function
        function.is_called = 1
        self.function.analyse_types(env)
Mark Florisson's avatar
Mark Florisson committed
3608

William Stein's avatar
William Stein committed
3609 3610 3611 3612 3613
        if function.is_attribute and function.entry and function.entry.is_cmethod:
            # Take ownership of the object from which the attribute
            # was obtained, because we need to pass it as 'self'.
            self.self = function.obj
            function.obj = CloneNode(self.self)
Mark Florisson's avatar
Mark Florisson committed
3614

William Stein's avatar
William Stein committed
3615 3616
        func_type = self.function_type()
        if func_type.is_pyobject:
3617 3618
            self.arg_tuple = TupleNode(self.pos, args = self.args)
            self.arg_tuple.analyse_types(env)
William Stein's avatar
William Stein committed
3619
            self.args = None
3620 3621 3622
            if func_type is Builtin.type_type and function.is_name and \
                   function.entry and \
                   function.entry.is_builtin and \
3623 3624 3625 3626 3627 3628 3629 3630 3631
                   function.entry.name in Builtin.types_that_construct_their_instance:
                # calling a builtin type that returns a specific object type
                if function.entry.name == 'float':
                    # the following will come true later on in a transform
                    self.type = PyrexTypes.c_double_type
                    self.result_ctype = PyrexTypes.c_double_type
                else:
                    self.type = Builtin.builtin_types[function.entry.name]
                    self.result_ctype = py_object_type
Stefan Behnel's avatar
Stefan Behnel committed
3632
                self.may_return_none = False
3633
            elif function.is_name and function.type_entry:
3634 3635 3636 3637 3638
                # We are calling an extension type constructor.  As
                # long as we do not support __new__(), the result type
                # is clear
                self.type = function.type_entry.type
                self.result_ctype = py_object_type
Stefan Behnel's avatar
Stefan Behnel committed
3639
                self.may_return_none = False
3640 3641
            else:
                self.type = py_object_type
William Stein's avatar
William Stein committed
3642 3643 3644 3645
            self.is_temp = 1
        else:
            for arg in self.args:
                arg.analyse_types(env)
3646

William Stein's avatar
William Stein committed
3647 3648
            if self.self and func_type.args:
                # Coerce 'self' to the type expected by the method.
3649 3650 3651
                self_arg = func_type.args[0]
                if self_arg.not_none: # C methods must do the None test for self at *call* time
                    self.self = self.self.as_none_safe_node(
3652 3653 3654
                        "'NoneType' object has no attribute '%s'",
                        error = 'PyExc_AttributeError',
                        format_args = [self.function.entry.name])
3655
                expected_type = self_arg.type
Stefan Behnel's avatar
Stefan Behnel committed
3656 3657 3658 3659 3660
                if self_arg.accept_builtin_subtypes:
                    self.coerced_self = CMethodSelfCloneNode(self.self)
                else:
                    self.coerced_self = CloneNode(self.self)
                self.coerced_self = self.coerced_self.coerce_to(expected_type, env)
William Stein's avatar
William Stein committed
3661 3662 3663
                # Insert coerced 'self' argument into argument list.
                self.args.insert(0, self.coerced_self)
            self.analyse_c_function_call(env)
3664

William Stein's avatar
William Stein committed
3665 3666
    def function_type(self):
        # Return the type of the function being called, coercing a function
3667 3668
        # pointer to a function if necessary. If the function has fused
        # arguments, return the specific type.
William Stein's avatar
William Stein committed
3669
        func_type = self.function.type
3670

William Stein's avatar
William Stein committed
3671 3672
        if func_type.is_ptr:
            func_type = func_type.base_type
3673

William Stein's avatar
William Stein committed
3674
        return func_type
3675

3676 3677 3678 3679 3680 3681 3682
    def is_simple(self):
        # C function calls could be considered simple, but they may
        # have side-effects that may hit when multiple operations must
        # be effected in order, e.g. when constructing the argument
        # sequence for a function call or comparing values.
        return False

William Stein's avatar
William Stein committed
3683
    def analyse_c_function_call(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
3684
        if self.function.type is error_type:
3685
            self.type = error_type
Robert Bradshaw's avatar
Robert Bradshaw committed
3686
            return
3687

Robert Bradshaw's avatar
Robert Bradshaw committed
3688
        if self.function.type.is_cpp_class:
3689 3690
            overloaded_entry = self.function.type.scope.lookup("operator()")
            if overloaded_entry is None:
Robert Bradshaw's avatar
Robert Bradshaw committed
3691 3692 3693
                self.type = PyrexTypes.error_type
                self.result_code = "<error>"
                return
3694 3695
        elif hasattr(self.function, 'entry'):
            overloaded_entry = self.function.entry
3696
        elif (isinstance(self.function, IndexNode) and
3697
              self.function.is_fused_index):
3698
            overloaded_entry = self.function.type.entry
Robert Bradshaw's avatar
Robert Bradshaw committed
3699
        else:
3700
            overloaded_entry = None
3701

3702
        if overloaded_entry:
3703
            if self.function.type.is_fused:
3704
                functypes = self.function.type.get_all_specialized_function_types()
Mark Florisson's avatar
Mark Florisson committed
3705
                alternatives = [f.entry for f in functypes]
3706 3707 3708 3709 3710
            else:
                alternatives = overloaded_entry.all_alternatives()

            entry = PyrexTypes.best_match(self.args, alternatives, self.pos, env)

3711 3712 3713 3714
            if not entry:
                self.type = PyrexTypes.error_type
                self.result_code = "<error>"
                return
Mark Florisson's avatar
Mark Florisson committed
3715 3716

            entry.used = True
3717 3718
            self.function.entry = entry
            self.function.type = entry.type
3719 3720 3721 3722 3723 3724 3725 3726
            func_type = self.function_type()
        else:
            func_type = self.function_type()
            if not func_type.is_cfunction:
                error(self.pos, "Calling non-function type '%s'" % func_type)
                self.type = PyrexTypes.error_type
                self.result_code = "<error>"
                return
William Stein's avatar
William Stein committed
3727
        # Check no. of args
3728 3729
        max_nargs = len(func_type.args)
        expected_nargs = max_nargs - func_type.optional_arg_count
William Stein's avatar
William Stein committed
3730
        actual_nargs = len(self.args)
3731 3732 3733
        if func_type.optional_arg_count and expected_nargs != actual_nargs:
            self.has_optional_args = 1
            self.is_temp = 1
William Stein's avatar
William Stein committed
3734
        # Coerce arguments
3735
        some_args_in_temps = False
3736
        for i in xrange(min(max_nargs, actual_nargs)):
William Stein's avatar
William Stein committed
3737
            formal_type = func_type.args[i].type
3738
            arg = self.args[i].coerce_to(formal_type, env)
3739
            if arg.is_temp:
3740 3741
                if i > 0:
                    # first argument in temp doesn't impact subsequent arguments
3742
                    some_args_in_temps = True
3743
            elif arg.type.is_pyobject and not env.nogil:
3744 3745
                if i == 0 and self.self is not None:
                    # a method's cloned "self" argument is ok
3746
                    pass
3747
                elif arg.nonlocally_immutable():
3748 3749 3750
                    # plain local variables are ok
                    pass
                else:
3751 3752 3753 3754
                    # we do not safely own the argument's reference,
                    # but we must make sure it cannot be collected
                    # before we return from the function, so we create
                    # an owned temp reference to it
3755 3756
                    if i > 0: # first argument doesn't matter
                        some_args_in_temps = True
3757
                    arg = arg.coerce_to_temp(env)
3758
            self.args[i] = arg
3759
        # handle additional varargs parameters
3760
        for i in xrange(max_nargs, actual_nargs):
3761 3762 3763 3764 3765 3766 3767
            arg = self.args[i]
            if arg.type.is_pyobject:
                arg_ctype = arg.type.default_coerced_ctype()
                if arg_ctype is None:
                    error(self.args[i].pos,
                          "Python object cannot be passed as a varargs parameter")
                else:
3768
                    self.args[i] = arg = arg.coerce_to(arg_ctype, env)
3769 3770
            if arg.is_temp and i > 0:
                some_args_in_temps = True
3771 3772 3773
        if some_args_in_temps:
            # if some args are temps and others are not, they may get
            # constructed in the wrong order (temps first) => make
3774 3775 3776 3777
            # sure they are either all temps or all not temps (except
            # for the last argument, which is evaluated last in any
            # case)
            for i in xrange(actual_nargs-1):
3778 3779
                if i == 0 and self.self is not None:
                    continue # self is ok
3780
                arg = self.args[i]
3781 3782
                if arg.nonlocally_immutable():
                    # locals, C functions, unassignable types are safe.
3783
                    pass
3784 3785
                elif arg.type.is_cpp_class:
                    # Assignment has side effects, avoid.
3786 3787
                    pass
                elif env.nogil and arg.type.is_pyobject:
3788 3789 3790
                    # can't copy a Python reference into a temp in nogil
                    # env (this is safe: a construction would fail in
                    # nogil anyway)
3791 3792
                    pass
                else:
3793 3794 3795 3796 3797
                    #self.args[i] = arg.coerce_to_temp(env)
                    # instead: issue a warning
                    if i > 0 or i == 1 and self.self is not None: # skip first arg
                        warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0)
                        break
3798

William Stein's avatar
William Stein committed
3799
        # Calc result type and code fragment
Robert Bradshaw's avatar
Robert Bradshaw committed
3800
        if isinstance(self.function, NewExprNode):
3801
            self.type = PyrexTypes.CPtrType(self.function.class_type)
Robert Bradshaw's avatar
Robert Bradshaw committed
3802 3803
        else:
            self.type = func_type.return_type
3804

3805 3806 3807
        if self.function.is_name or self.function.is_attribute:
            if self.function.entry and self.function.entry.utility_code:
                self.is_temp = 1 # currently doesn't work for self.calculate_result_code()
3808

Stefan Behnel's avatar
Stefan Behnel committed
3809 3810 3811 3812 3813 3814
        if self.type.is_pyobject:
            self.result_ctype = py_object_type
            self.is_temp = 1
        elif func_type.exception_value is not None \
                 or func_type.exception_check:
            self.is_temp = 1
3815 3816 3817 3818
        elif self.type.is_memoryviewslice:
            self.is_temp = 1
            # func_type.exception_check = True

3819
        # Called in 'nogil' context?
3820
        self.nogil = env.nogil
3821 3822 3823 3824 3825
        if (self.nogil and
            func_type.exception_check and
            func_type.exception_check != '+'):
            env.use_utility_code(pyerr_occurred_withgil_utility_code)
        # C++ exception handler
Robert Bradshaw's avatar
Robert Bradshaw committed
3826 3827 3828 3829
        if func_type.exception_check == '+':
            if func_type.exception_value is None:
                env.use_utility_code(cpp_exception_utility_code)

William Stein's avatar
William Stein committed
3830 3831
    def calculate_result_code(self):
        return self.c_call_code()
3832

William Stein's avatar
William Stein committed
3833 3834
    def c_call_code(self):
        func_type = self.function_type()
3835
        if self.type is PyrexTypes.error_type or not func_type.is_cfunction:
William Stein's avatar
William Stein committed
3836 3837 3838
            return "<error>"
        formal_args = func_type.args
        arg_list_code = []
3839
        args = list(zip(formal_args, self.args))
3840 3841 3842 3843
        max_nargs = len(func_type.args)
        expected_nargs = max_nargs - func_type.optional_arg_count
        actual_nargs = len(self.args)
        for formal_arg, actual_arg in args[:expected_nargs]:
William Stein's avatar
William Stein committed
3844 3845
                arg_code = actual_arg.result_as(formal_arg.type)
                arg_list_code.append(arg_code)
3846

3847 3848
        if func_type.is_overridable:
            arg_list_code.append(str(int(self.wrapper_call or self.function.entry.is_unbound_cmethod)))
3849

3850
        if func_type.optional_arg_count:
3851
            if expected_nargs == actual_nargs:
3852
                optional_args = 'NULL'
3853
            else:
3854
                optional_args = "&%s" % self.opt_arg_struct
3855
            arg_list_code.append(optional_args)
3856

William Stein's avatar
William Stein committed
3857
        for actual_arg in self.args[len(formal_args):]:
3858
            arg_list_code.append(actual_arg.result())
3859 3860

        result = "%s(%s)" % (self.function.result(), ', '.join(arg_list_code))
William Stein's avatar
William Stein committed
3861
        return result
3862

William Stein's avatar
William Stein committed
3863 3864
    def generate_result_code(self, code):
        func_type = self.function_type()
3865 3866 3867
        if self.function.is_name or self.function.is_attribute:
            if self.function.entry and self.function.entry.utility_code:
                code.globalstate.use_utility_code(self.function.entry.utility_code)
William Stein's avatar
William Stein committed
3868
        if func_type.is_pyobject:
3869
            arg_code = self.arg_tuple.py_result()
William Stein's avatar
William Stein committed
3870
            code.putln(
3871
                "%s = PyObject_Call(%s, %s, NULL); %s" % (
3872
                    self.result(),
William Stein's avatar
William Stein committed
3873
                    self.function.py_result(),
3874
                    arg_code,
3875
                    code.error_goto_if_null(self.result(), self.pos)))
3876
            code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
3877
        elif func_type.is_cfunction:
3878 3879 3880
            if self.has_optional_args:
                actual_nargs = len(self.args)
                expected_nargs = len(func_type.args) - func_type.optional_arg_count
3881 3882
                self.opt_arg_struct = code.funcstate.allocate_temp(
                    func_type.op_arg_struct.base_type, manage_ref=True)
3883 3884 3885 3886
                code.putln("%s.%s = %s;" % (
                        self.opt_arg_struct,
                        Naming.pyrex_prefix + "n",
                        len(self.args) - expected_nargs))
3887
                args = list(zip(func_type.args, self.args))
3888 3889 3890
                for formal_arg, actual_arg in args[expected_nargs:actual_nargs]:
                    code.putln("%s.%s = %s;" % (
                            self.opt_arg_struct,
3891
                            func_type.opt_arg_cname(formal_arg.name),
3892
                            actual_arg.result_as(formal_arg.type)))
William Stein's avatar
William Stein committed
3893
            exc_checks = []
3894
            if self.type.is_pyobject and self.is_temp:
3895
                exc_checks.append("!%s" % self.result())
3896 3897 3898
            elif self.type.is_memoryviewslice:
                assert self.is_temp
                exc_checks.append(self.type.error_condition(self.result()))
William Stein's avatar
William Stein committed
3899
            else:
3900 3901
                exc_val = func_type.exception_value
                exc_check = func_type.exception_check
William Stein's avatar
William Stein committed
3902
                if exc_val is not None:
3903
                    exc_checks.append("%s == %s" % (self.result(), exc_val))
William Stein's avatar
William Stein committed
3904
                if exc_check:
3905 3906
                    if self.nogil:
                        exc_checks.append("__Pyx_ErrOccurredWithGIL()")
3907
                    else:
3908
                        exc_checks.append("PyErr_Occurred()")
William Stein's avatar
William Stein committed
3909 3910
            if self.is_temp or exc_checks:
                rhs = self.c_call_code()
3911 3912
                if self.result():
                    lhs = "%s = " % self.result()
William Stein's avatar
William Stein committed
3913 3914 3915
                    if self.is_temp and self.type.is_pyobject:
                        #return_type = self.type # func_type.return_type
                        #print "SimpleCallNode.generate_result_code: casting", rhs, \
Robert Bradshaw's avatar
Robert Bradshaw committed
3916
                        #    "from", return_type, "to pyobject" ###
William Stein's avatar
William Stein committed
3917 3918 3919
                        rhs = typecast(py_object_type, self.type, rhs)
                else:
                    lhs = ""
Felix Wu's avatar
Felix Wu committed
3920
                if func_type.exception_check == '+':
Robert Bradshaw's avatar
Robert Bradshaw committed
3921 3922 3923
                    if func_type.exception_value is None:
                        raise_py_exception = "__Pyx_CppExn2PyErr()"
                    elif func_type.exception_value.type.is_pyobject:
3924 3925 3926
                        raise_py_exception = ' try { throw; } catch(const std::exception& exn) { PyErr_SetString(%s, exn.what()); } catch(...) { PyErr_SetNone(%s); }' % (
                            func_type.exception_value.entry.cname,
                            func_type.exception_value.entry.cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
3927 3928
                    else:
                        raise_py_exception = '%s(); if (!PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError , "Error converting c++ exception.")' % func_type.exception_value.entry.cname
3929 3930
                    if self.nogil:
                        raise_py_exception = 'Py_BLOCK_THREADS; %s; Py_UNBLOCK_THREADS' % raise_py_exception
Felix Wu's avatar
Felix Wu committed
3931
                    code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
3932
                    "try {%s%s;} catch(...) {%s; %s}" % (
Felix Wu's avatar
Felix Wu committed
3933 3934
                        lhs,
                        rhs,
Robert Bradshaw's avatar
Robert Bradshaw committed
3935
                        raise_py_exception,
Felix Wu's avatar
Felix Wu committed
3936
                        code.error_goto(self.pos)))
3937 3938 3939 3940 3941 3942
                else:
                    if exc_checks:
                        goto_error = code.error_goto_if(" && ".join(exc_checks), self.pos)
                    else:
                        goto_error = ""
                    code.putln("%s%s; %s" % (lhs, rhs, goto_error))
3943
                if self.type.is_pyobject and self.result():
3944
                    code.put_gotref(self.py_result())
3945 3946
            if self.has_optional_args:
                code.funcstate.release_temp(self.opt_arg_struct)
3947 3948


3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032
class InlinedDefNodeCallNode(CallNode):
    #  Inline call to defnode
    #
    #  function       PyCFunctionNode
    #  function_name  NameNode
    #  args           [ExprNode]

    subexprs = ['args', 'function_name']
    is_temp = 1
    type = py_object_type
    function = None
    function_name = None

    def can_be_inlined(self):
        func_type= self.function.def_node
        if func_type.star_arg or func_type.starstar_arg:
            return False
        if len(func_type.args) != len(self.args):
            return False
        return True

    def analyse_types(self, env):
        self.function_name.analyse_types(env)

        for arg in self.args:
            arg.analyse_types(env)

        func_type = self.function.def_node
        actual_nargs = len(self.args)

        # Coerce arguments
        some_args_in_temps = False
        for i in xrange(actual_nargs):
            formal_type = func_type.args[i].type
            arg = self.args[i].coerce_to(formal_type, env)
            if arg.is_temp:
                if i > 0:
                    # first argument in temp doesn't impact subsequent arguments
                    some_args_in_temps = True
            elif arg.type.is_pyobject and not env.nogil:
                if arg.nonlocally_immutable():
                    # plain local variables are ok
                    pass
                else:
                    # we do not safely own the argument's reference,
                    # but we must make sure it cannot be collected
                    # before we return from the function, so we create
                    # an owned temp reference to it
                    if i > 0: # first argument doesn't matter
                        some_args_in_temps = True
                    arg = arg.coerce_to_temp(env)
            self.args[i] = arg

        if some_args_in_temps:
            # if some args are temps and others are not, they may get
            # constructed in the wrong order (temps first) => make
            # sure they are either all temps or all not temps (except
            # for the last argument, which is evaluated last in any
            # case)
            for i in xrange(actual_nargs-1):
                arg = self.args[i]
                if arg.nonlocally_immutable():
                    # locals, C functions, unassignable types are safe.
                    pass
                elif arg.type.is_cpp_class:
                    # Assignment has side effects, avoid.
                    pass
                elif env.nogil and arg.type.is_pyobject:
                    # can't copy a Python reference into a temp in nogil
                    # env (this is safe: a construction would fail in
                    # nogil anyway)
                    pass
                else:
                    #self.args[i] = arg.coerce_to_temp(env)
                    # instead: issue a warning
                    if i > 0:
                        warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0)
                        break

    def generate_result_code(self, code):
        arg_code = [self.function_name.py_result()]
        func_type = self.function.def_node
        for arg, proto_arg in zip(self.args, func_type.args):
            if arg.type.is_pyobject:
4033
                arg_code.append(arg.result_as(proto_arg.type))
4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045
            else:
                arg_code.append(arg.result())
        arg_code = ', '.join(arg_code)
        code.putln(
            "%s = %s(%s); %s" % (
                self.result(),
                self.function.def_node.entry.pyfunc_cname,
                arg_code,
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())


4046 4047
class PythonCapiFunctionNode(ExprNode):
    subexprs = []
4048
    def __init__(self, pos, py_name, cname, func_type, utility_code = None):
4049
        self.pos = pos
4050 4051
        self.name = py_name
        self.cname = cname
4052 4053 4054
        self.type = func_type
        self.utility_code = utility_code

4055 4056 4057
    def analyse_types(self, env):
        pass

4058 4059 4060 4061 4062
    def generate_result_code(self, code):
        if self.utility_code:
            code.globalstate.use_utility_code(self.utility_code)

    def calculate_result_code(self):
4063
        return self.cname
4064 4065 4066 4067

class PythonCapiCallNode(SimpleCallNode):
    # Python C-API Function call (only created in transforms)

Stefan Behnel's avatar
Stefan Behnel committed
4068 4069 4070 4071 4072 4073
    # By default, we assume that the call never returns None, as this
    # is true for most C-API functions in CPython.  If this does not
    # apply to a call, set the following to True (or None to inherit
    # the default behaviour).
    may_return_none = False

4074
    def __init__(self, pos, function_name, func_type,
4075
                 utility_code = None, py_name=None, **kwargs):
4076 4077 4078
        self.type = func_type.return_type
        self.result_ctype = self.type
        self.function = PythonCapiFunctionNode(
4079
            pos, py_name, function_name, func_type,
4080 4081 4082 4083 4084
            utility_code = utility_code)
        # call this last so that we can override the constructed
        # attributes above with explicit keyword arguments if required
        SimpleCallNode.__init__(self, pos, **kwargs)

William Stein's avatar
William Stein committed
4085

4086
class GeneralCallNode(CallNode):
William Stein's avatar
William Stein committed
4087 4088 4089 4090 4091 4092
    #  General Python function call, including keyword,
    #  * and ** arguments.
    #
    #  function         ExprNode
    #  positional_args  ExprNode          Tuple of positional arguments
    #  keyword_args     ExprNode or None  Dict of keyword arguments
4093

4094
    type = py_object_type
4095

4096
    subexprs = ['function', 'positional_args', 'keyword_args']
William Stein's avatar
William Stein committed
4097

4098
    nogil_check = Node.gil_error
4099

4100 4101 4102 4103 4104 4105 4106 4107
    def compile_time_value(self, denv):
        function = self.function.compile_time_value(denv)
        positional_args = self.positional_args.compile_time_value(denv)
        keyword_args = self.keyword_args.compile_time_value(denv)
        try:
            return function(*positional_args, **keyword_args)
        except Exception, e:
            self.compile_time_value_error(e)
4108

4109
    def explicit_args_kwds(self):
4110 4111
        if (self.keyword_args and not isinstance(self.keyword_args, DictNode) or
            not isinstance(self.positional_args, TupleNode)):
4112
            raise CompileError(self.pos,
4113 4114
                'Compile-time keyword arguments must be explicit.')
        return self.positional_args.args, self.keyword_args
4115

William Stein's avatar
William Stein committed
4116
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
4117 4118
        if self.analyse_as_type_constructor(env):
            return
William Stein's avatar
William Stein committed
4119 4120 4121 4122
        self.function.analyse_types(env)
        self.positional_args.analyse_types(env)
        if self.keyword_args:
            self.keyword_args.analyse_types(env)
4123
        if not self.function.type.is_pyobject:
4124 4125
            if self.function.type.is_error:
                self.type = error_type
Stefan Behnel's avatar
Stefan Behnel committed
4126
                return
4127
            if hasattr(self.function, 'entry') and not self.function.entry.as_variable:
4128
                error(self.pos, "Keyword and starred arguments not allowed in cdef functions.")
4129 4130
            else:
                self.function = self.function.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
4131 4132
        self.positional_args = \
            self.positional_args.coerce_to_pyobject(env)
Stefan Behnel's avatar
Stefan Behnel committed
4133
        function = self.function
4134 4135 4136 4137 4138
        if function.is_name and function.type_entry:
            # We are calling an extension type constructor.  As long
            # as we do not support __new__(), the result type is clear
            self.type = function.type_entry.type
            self.result_ctype = py_object_type
Stefan Behnel's avatar
Stefan Behnel committed
4139
            self.may_return_none = False
4140 4141
        else:
            self.type = py_object_type
William Stein's avatar
William Stein committed
4142
        self.is_temp = 1
4143

William Stein's avatar
William Stein committed
4144
    def generate_result_code(self, code):
4145
        if self.type.is_error: return
4146 4147
        if self.keyword_args:
            kwargs = self.keyword_args.py_result()
William Stein's avatar
William Stein committed
4148
        else:
4149
            kwargs = 'NULL'
William Stein's avatar
William Stein committed
4150
        code.putln(
4151
            "%s = PyObject_Call(%s, %s, %s); %s" % (
4152
                self.result(),
4153 4154 4155
                self.function.py_result(),
                self.positional_args.py_result(),
                kwargs,
4156
                code.error_goto_if_null(self.result(), self.pos)))
4157
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
4158 4159


4160
class AsTupleNode(ExprNode):
William Stein's avatar
William Stein committed
4161 4162 4163 4164
    #  Convert argument to tuple. Used for normalising
    #  the * argument of a function call.
    #
    #  arg    ExprNode
4165

William Stein's avatar
William Stein committed
4166
    subexprs = ['arg']
4167 4168 4169

    def calculate_constant_result(self):
        self.constant_result = tuple(self.base.constant_result)
4170

4171 4172 4173 4174 4175 4176 4177
    def compile_time_value(self, denv):
        arg = self.arg.compile_time_value(denv)
        try:
            return tuple(arg)
        except Exception, e:
            self.compile_time_value_error(e)

William Stein's avatar
William Stein committed
4178 4179 4180
    def analyse_types(self, env):
        self.arg.analyse_types(env)
        self.arg = self.arg.coerce_to_pyobject(env)
4181
        self.type = tuple_type
William Stein's avatar
William Stein committed
4182
        self.is_temp = 1
4183

4184 4185 4186
    def may_be_none(self):
        return False

4187
    nogil_check = Node.gil_error
4188 4189
    gil_message = "Constructing Python tuple"

William Stein's avatar
William Stein committed
4190 4191
    def generate_result_code(self, code):
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
4192
            "%s = PySequence_Tuple(%s); %s" % (
4193
                self.result(),
William Stein's avatar
William Stein committed
4194
                self.arg.py_result(),
4195
                code.error_goto_if_null(self.result(), self.pos)))
4196
        code.put_gotref(self.py_result())
4197

William Stein's avatar
William Stein committed
4198

4199
class AttributeNode(ExprNode):
William Stein's avatar
William Stein committed
4200 4201 4202 4203
    #  obj.attribute
    #
    #  obj          ExprNode
    #  attribute    string
4204
    #  needs_none_check boolean        Used if obj is an extension type.
4205
    #                                  If set to True, it is known that the type is not None.
William Stein's avatar
William Stein committed
4206 4207 4208 4209 4210 4211 4212
    #
    #  Used internally:
    #
    #  is_py_attr           boolean   Is a Python getattr operation
    #  member               string    C name of struct member
    #  is_called            boolean   Function call is being done on result
    #  entry                Entry     Symbol table entry of attribute
4213

William Stein's avatar
William Stein committed
4214 4215
    is_attribute = 1
    subexprs = ['obj']
4216

William Stein's avatar
William Stein committed
4217 4218 4219
    type = PyrexTypes.error_type
    entry = None
    is_called = 0
4220
    needs_none_check = True
4221
    is_memslice_transpose = False
William Stein's avatar
William Stein committed
4222

4223
    def as_cython_attribute(self):
Mark Florisson's avatar
Mark Florisson committed
4224 4225 4226
        if (isinstance(self.obj, NameNode) and
                self.obj.is_cython_module and not
                self.attribute == u"parallel"):
4227
            return self.attribute
Mark Florisson's avatar
Mark Florisson committed
4228

4229 4230 4231
        cy = self.obj.as_cython_attribute()
        if cy:
            return "%s.%s" % (cy, self.attribute)
4232
        return None
4233

4234 4235 4236 4237 4238 4239 4240 4241 4242
    def coerce_to(self, dst_type, env):
        #  If coercing to a generic pyobject and this is a cpdef function
        #  we can create the corresponding attribute
        if dst_type is py_object_type:
            entry = self.entry
            if entry and entry.is_cfunction and entry.as_variable:
                # must be a cpdef function
                self.is_temp = 1
                self.entry = entry.as_variable
4243
                self.analyse_as_python_attribute(env)
4244
                return self
4245
        return ExprNode.coerce_to(self, dst_type, env)
4246 4247 4248

    def calculate_constant_result(self):
        attr = self.attribute
4249
        if attr.startswith("__") and attr.endswith("__"):
4250 4251 4252
            return
        self.constant_result = getattr(self.obj.constant_result, attr)

4253 4254
    def compile_time_value(self, denv):
        attr = self.attribute
4255
        if attr.startswith("__") and attr.endswith("__"):
Stefan Behnel's avatar
Stefan Behnel committed
4256 4257
            error(self.pos,
                  "Invalid attribute name '%s' in compile-time expression" % attr)
4258
            return None
4259
        obj = self.obj.compile_time_value(denv)
4260 4261 4262 4263
        try:
            return getattr(obj, attr)
        except Exception, e:
            self.compile_time_value_error(e)
4264

Robert Bradshaw's avatar
Robert Bradshaw committed
4265 4266
    def type_dependencies(self, env):
        return self.obj.type_dependencies(env)
4267

4268 4269 4270 4271 4272 4273
    def infer_type(self, env):
        if self.analyse_as_cimported_attribute(env, 0):
            return self.entry.type
        elif self.analyse_as_unbound_cmethod(env):
            return self.entry.type
        else:
4274 4275 4276 4277 4278 4279 4280 4281
            obj_type = self.obj.infer_type(env)
            self.analyse_attribute(env, obj_type = obj_type)
            if obj_type.is_builtin_type and self.type.is_cfunction:
                # special case: C-API replacements for C methods of
                # builtin types cannot be inferred as C functions as
                # that would prevent their use as bound methods
                self.type = py_object_type
                return py_object_type
4282
            return self.type
4283

William Stein's avatar
William Stein committed
4284 4285
    def analyse_target_declaration(self, env):
        pass
4286

William Stein's avatar
William Stein committed
4287 4288
    def analyse_target_types(self, env):
        self.analyse_types(env, target = 1)
4289 4290
        if not self.is_lvalue():
            error(self.pos, "Assignment to non-lvalue of type '%s'" % self.type)
4291

William Stein's avatar
William Stein committed
4292
    def analyse_types(self, env, target = 0):
4293
        self.initialized_check = env.directives['initializedcheck']
William Stein's avatar
William Stein committed
4294
        if self.analyse_as_cimported_attribute(env, target):
4295 4296 4297 4298 4299 4300 4301
            self.entry.used = True
        elif not target and self.analyse_as_unbound_cmethod(env):
            self.entry.used = True
        else:
            self.analyse_as_ordinary_attribute(env, target)
            if self.entry:
                self.entry.used = True
4302

William Stein's avatar
William Stein committed
4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
    def analyse_as_cimported_attribute(self, env, target):
        # Try to interpret this as a reference to an imported
        # C const, type, var or function. If successful, mutates
        # this node into a NameNode and returns 1, otherwise
        # returns 0.
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
            entry = module_scope.lookup_here(self.attribute)
            if entry and (
                entry.is_cglobal or entry.is_cfunction
                or entry.is_type or entry.is_const):
                    self.mutate_into_name_node(env, entry, target)
4315
                    entry.used = 1
William Stein's avatar
William Stein committed
4316 4317
                    return 1
        return 0
4318

William Stein's avatar
William Stein committed
4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334
    def analyse_as_unbound_cmethod(self, env):
        # Try to interpret this as a reference to an unbound
        # C method of an extension type. If successful, mutates
        # this node into a NameNode and returns 1, otherwise
        # returns 0.
        type = self.obj.analyse_as_extension_type(env)
        if type:
            entry = type.scope.lookup_here(self.attribute)
            if entry and entry.is_cmethod:
                # Create a temporary entry describing the C method
                # as an ordinary function.
                ubcm_entry = Symtab.Entry(entry.name,
                    "%s->%s" % (type.vtabptr_cname, entry.cname),
                    entry.type)
                ubcm_entry.is_cfunction = 1
                ubcm_entry.func_cname = entry.func_cname
4335
                ubcm_entry.is_unbound_cmethod = 1
William Stein's avatar
William Stein committed
4336 4337 4338
                self.mutate_into_name_node(env, ubcm_entry, None)
                return 1
        return 0
4339

4340 4341 4342
    def analyse_as_type(self, env):
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
4343
            return module_scope.lookup_type(self.attribute)
4344
        if not self.obj.is_string_literal:
Robert Bradshaw's avatar
Robert Bradshaw committed
4345
            base_type = self.obj.analyse_as_type(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
4346
            if base_type and hasattr(base_type, 'scope') and base_type.scope is not None:
Robert Bradshaw's avatar
Robert Bradshaw committed
4347
                return base_type.scope.lookup_type(self.attribute)
4348
        return None
4349

William Stein's avatar
William Stein committed
4350 4351 4352 4353 4354 4355 4356 4357 4358
    def analyse_as_extension_type(self, env):
        # Try to interpret this as a reference to an extension type
        # in a cimported module. Returns the extension type, or None.
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
            entry = module_scope.lookup_here(self.attribute)
            if entry and entry.is_type and entry.type.is_extension_type:
                return entry.type
        return None
4359

William Stein's avatar
William Stein committed
4360 4361 4362 4363 4364 4365 4366 4367 4368
    def analyse_as_module(self, env):
        # Try to interpret this as a reference to a cimported module
        # in another cimported module. Returns the module scope, or None.
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
            entry = module_scope.lookup_here(self.attribute)
            if entry and entry.as_module:
                return entry.as_module
        return None
4369

William Stein's avatar
William Stein committed
4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380
    def mutate_into_name_node(self, env, entry, target):
        # Mutate this node into a NameNode and complete the
        # analyse_types phase.
        self.__class__ = NameNode
        self.name = self.attribute
        self.entry = entry
        del self.obj
        del self.attribute
        if target:
            NameNode.analyse_target_types(self, env)
        else:
4381
            NameNode.analyse_rvalue_entry(self, env)
4382

William Stein's avatar
William Stein committed
4383 4384 4385 4386
    def analyse_as_ordinary_attribute(self, env, target):
        self.obj.analyse_types(env)
        self.analyse_attribute(env)
        if self.entry and self.entry.is_cmethod and not self.is_called:
4387 4388
#            error(self.pos, "C method can only be called")
            pass
4389 4390
        ## Reference to C array turns into pointer to first element.
        #while self.type.is_array:
Robert Bradshaw's avatar
Robert Bradshaw committed
4391
        #    self.type = self.type.element_ptr_type()
William Stein's avatar
William Stein committed
4392 4393 4394 4395
        if self.is_py_attr:
            if not target:
                self.is_temp = 1
                self.result_ctype = py_object_type
4396 4397
        elif target and self.obj.type.is_builtin_type:
            error(self.pos, "Assignment to an immutable object field")
4398 4399
        #elif self.type.is_memoryviewslice and not target:
        #    self.is_temp = True
4400

Robert Bradshaw's avatar
Robert Bradshaw committed
4401
    def analyse_attribute(self, env, obj_type = None):
William Stein's avatar
William Stein committed
4402 4403 4404
        # Look up attribute and set self.type and self.member.
        self.is_py_attr = 0
        self.member = self.attribute
Robert Bradshaw's avatar
Robert Bradshaw committed
4405 4406 4407 4408 4409 4410 4411
        if obj_type is None:
            if self.obj.type.is_string:
                self.obj = self.obj.coerce_to_pyobject(env)
            obj_type = self.obj.type
        else:
            if obj_type.is_string:
                obj_type = py_object_type
4412
        if obj_type.is_ptr or obj_type.is_array:
William Stein's avatar
William Stein committed
4413 4414
            obj_type = obj_type.base_type
            self.op = "->"
4415
        elif obj_type.is_extension_type or obj_type.is_builtin_type:
William Stein's avatar
William Stein committed
4416 4417 4418 4419 4420 4421
            self.op = "->"
        else:
            self.op = "."
        if obj_type.has_attributes:
            entry = None
            if obj_type.attributes_known():
4422 4423
                if (obj_type.is_memoryviewslice and not
                        obj_type.scope.lookup_here(self.attribute)):
4424 4425 4426 4427 4428 4429 4430
                    if self.attribute == 'T':
                        self.is_memslice_transpose = True
                        self.is_temp = True
                        self.use_managed_ref = True
                        self.type = self.obj.type
                        return
                    else:
4431
                        obj_type.declare_attribute(self.attribute, env, self.pos)
William Stein's avatar
William Stein committed
4432
                entry = obj_type.scope.lookup_here(self.attribute)
Robert Bradshaw's avatar
Robert Bradshaw committed
4433 4434
                if entry and entry.is_member:
                    entry = None
William Stein's avatar
William Stein committed
4435
            else:
4436 4437
                error(self.pos,
                    "Cannot select attribute of incomplete type '%s'"
William Stein's avatar
William Stein committed
4438
                    % obj_type)
Robert Bradshaw's avatar
Robert Bradshaw committed
4439 4440
                self.type = PyrexTypes.error_type
                return
William Stein's avatar
William Stein committed
4441 4442
            self.entry = entry
            if entry:
4443 4444
                if obj_type.is_extension_type and entry.name == "__weakref__":
                    error(self.pos, "Illegal use of special attribute __weakref__")
4445 4446

                # def methods need the normal attribute lookup
4447
                # because they do not have struct entries
4448 4449 4450 4451
                # fused function go through assignment synthesis
                # (foo = pycfunction(foo_func_obj)) and need to go through
                # regular Python lookup as well
                if (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod:
4452 4453 4454
                    self.type = entry.type
                    self.member = entry.cname
                    return
William Stein's avatar
William Stein committed
4455 4456 4457 4458 4459
                else:
                    # If it's not a variable or C method, it must be a Python
                    # method of an extension type, so we treat it like a Python
                    # attribute.
                    pass
4460
        # If we get here, the base object is not a struct/union/extension
William Stein's avatar
William Stein committed
4461 4462 4463
        # type, or it is an extension type and the attribute is either not
        # declared or is declared as a Python method. Treat it as a Python
        # attribute reference.
Robert Bradshaw's avatar
Robert Bradshaw committed
4464
        self.analyse_as_python_attribute(env, obj_type)
Stefan Behnel's avatar
Stefan Behnel committed
4465

Robert Bradshaw's avatar
Robert Bradshaw committed
4466 4467 4468
    def analyse_as_python_attribute(self, env, obj_type = None):
        if obj_type is None:
            obj_type = self.obj.type
4469 4470
        # mangle private '__*' Python attributes used inside of a class
        self.attribute = env.mangle_class_private_name(self.attribute)
4471
        self.member = self.attribute
4472 4473
        self.type = py_object_type
        self.is_py_attr = 1
4474
        if not obj_type.is_pyobject and not obj_type.is_error:
4475
            if obj_type.can_coerce_to_pyobject(env):
4476 4477 4478 4479 4480
                self.obj = self.obj.coerce_to_pyobject(env)
            else:
                error(self.pos,
                      "Object of type '%s' has no attribute '%s'" %
                      (obj_type, self.attribute))
4481

4482
    def nogil_check(self, env):
4483
        if self.is_py_attr:
4484
            self.gil_error()
4485 4486 4487
        elif self.type.is_memoryviewslice:
            import MemoryView
            MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
4488

4489 4490
    gil_message = "Accessing Python attribute"

William Stein's avatar
William Stein committed
4491 4492 4493 4494 4495 4496 4497 4498
    def is_simple(self):
        if self.obj:
            return self.result_in_temp() or self.obj.is_simple()
        else:
            return NameNode.is_simple(self)

    def is_lvalue(self):
        if self.obj:
4499
            return not self.type.is_array
William Stein's avatar
William Stein committed
4500 4501
        else:
            return NameNode.is_lvalue(self)
4502

William Stein's avatar
William Stein committed
4503 4504 4505 4506 4507
    def is_ephemeral(self):
        if self.obj:
            return self.obj.is_ephemeral()
        else:
            return NameNode.is_ephemeral(self)
4508

William Stein's avatar
William Stein committed
4509 4510
    def calculate_result_code(self):
        #print "AttributeNode.calculate_result_code:", self.member ###
4511
        #print "...obj node =", self.obj, "code", self.obj.result() ###
William Stein's avatar
William Stein committed
4512 4513 4514 4515 4516
        #print "...obj type", self.obj.type, "ctype", self.obj.ctype() ###
        obj = self.obj
        obj_code = obj.result_as(obj.type)
        #print "...obj_code =", obj_code ###
        if self.entry and self.entry.is_cmethod:
Stefan Behnel's avatar
Stefan Behnel committed
4517
            if obj.type.is_extension_type and not self.entry.is_builtin_cmethod:
4518 4519
                if self.entry.final_func_cname:
                    return self.entry.final_func_cname
4520

Mark Florisson's avatar
Mark Florisson committed
4521
                if self.type.from_fused:
4522 4523 4524 4525
                    # If the attribute was specialized through indexing, make
                    # sure to get the right fused name, as our entry was
                    # replaced by our parent index node
                    # (AnalyseExpressionsTransform)
Mark Florisson's avatar
Mark Florisson committed
4526 4527
                    self.member = self.entry.cname

Robert Bradshaw's avatar
Robert Bradshaw committed
4528
                return "((struct %s *)%s%s%s)->%s" % (
4529
                    obj.type.vtabstruct_cname, obj_code, self.op,
Robert Bradshaw's avatar
Robert Bradshaw committed
4530 4531 4532
                    obj.type.vtabslot_cname, self.member)
            else:
                return self.member
4533
        elif obj.type.is_complex:
4534
            return "__Pyx_C%s(%s)" % (self.member.upper(), obj_code)
William Stein's avatar
William Stein committed
4535
        else:
4536 4537 4538
            if obj.type.is_builtin_type and self.entry and self.entry.is_variable:
                # accessing a field of a builtin type, need to cast better than result_as() does
                obj_code = obj.type.cast_code(obj.result(), to_object_struct = True)
William Stein's avatar
William Stein committed
4539
            return "%s%s%s" % (obj_code, self.op, self.member)
4540

William Stein's avatar
William Stein committed
4541 4542
    def generate_result_code(self, code):
        if self.is_py_attr:
4543 4544
            code.putln(
                '%s = PyObject_GetAttr(%s, %s); %s' % (
4545
                    self.result(),
4546
                    self.obj.py_result(),
4547
                    code.intern_identifier(self.attribute),
4548
                    code.error_goto_if_null(self.result(), self.pos)))
4549
            code.put_gotref(self.py_result())
4550
        elif self.type.is_memoryviewslice:
4551 4552 4553
            if code.globalstate.directives['nonecheck']:
                self.put_nonecheck(code)

4554 4555 4556 4557 4558 4559 4560 4561 4562
            if self.is_memslice_transpose:
                # transpose the slice
                for access, packing in self.type.axes:
                    if access == 'ptr':
                        error(self.pos, "Transposing not supported for slices "
                                        "with indirect dimensions")
                        return

                code.putln("%s = %s;" % (self.result(), self.obj.result()))
4563 4564
                if self.obj.is_name or (self.obj.is_attribute and
                                        self.obj.is_memslice_transpose):
4565 4566 4567 4568 4569
                    code.put_incref_memoryviewslice(self.result(), have_gil=True)

                T = "__pyx_memslice_transpose(&%s) == 0"
                code.putln(code.error_goto_if(T % self.result(), self.pos))
            elif self.initialized_check:
4570 4571 4572 4573 4574 4575
                code.putln(
                    'if (unlikely(!%s.memview)) {'
                        'PyErr_SetString(PyExc_AttributeError,'
                                        '"Memoryview is not initialized");'
                        '%s'
                    '}' % (self.result(), code.error_goto(self.pos)))
4576 4577 4578
        elif (self.obj.type.is_memoryviewslice and
                code.globalstate.directives['nonecheck']):
            self.put_nonecheck(code)
4579 4580 4581
        else:
            # result_code contains what is needed, but we may need to insert
            # a check and raise an exception
4582 4583 4584 4585 4586 4587
            if self.obj.type.is_extension_type:
                if self.needs_none_check and code.globalstate.directives['nonecheck']:
                    self.put_nonecheck(code)
            elif self.entry and self.entry.is_cmethod and self.entry.utility_code:
                # C method implemented as function call with utility code
                code.globalstate.use_utility_code(self.entry.utility_code)
4588

William Stein's avatar
William Stein committed
4589 4590 4591
    def generate_assignment_code(self, rhs, code):
        self.obj.generate_evaluation_code(code)
        if self.is_py_attr:
4592
            code.put_error_if_neg(self.pos,
4593 4594
                'PyObject_SetAttr(%s, %s, %s)' % (
                    self.obj.py_result(),
4595
                    code.intern_identifier(self.attribute),
4596
                    rhs.py_result()))
William Stein's avatar
William Stein committed
4597
            rhs.generate_disposal_code(code)
4598
            rhs.free_temps(code)
4599 4600 4601 4602 4603
        elif self.obj.type.is_complex:
            code.putln("__Pyx_SET_C%s(%s, %s);" % (
                self.member.upper(),
                self.obj.result_as(self.obj.type),
                rhs.result_as(self.ctype())))
William Stein's avatar
William Stein committed
4604
        else:
4605
            if (self.obj.type.needs_nonecheck()
4606 4607 4608 4609
                  and self.needs_none_check
                  and code.globalstate.directives['nonecheck']):
                self.put_nonecheck(code)

4610
            select_code = self.result()
4611
            if self.type.is_pyobject and self.use_managed_ref:
William Stein's avatar
William Stein committed
4612
                rhs.make_owned_reference(code)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
4613
                code.put_giveref(rhs.py_result())
4614
                code.put_gotref(select_code)
William Stein's avatar
William Stein committed
4615
                code.put_decref(select_code, self.ctype())
4616
            elif self.type.is_memoryviewslice:
4617
                import MemoryView
4618
                MemoryView.put_assign_to_memviewslice(
4619
                        select_code, rhs, rhs.result(), self.type, code)
4620

4621
            if not self.type.is_memoryviewslice:
4622 4623 4624 4625 4626
                code.putln(
                    "%s = %s;" % (
                        select_code,
                        rhs.result_as(self.ctype())))
                        #rhs.result()))
William Stein's avatar
William Stein committed
4627
            rhs.generate_post_assignment_code(code)
4628
            rhs.free_temps(code)
William Stein's avatar
William Stein committed
4629
        self.obj.generate_disposal_code(code)
4630
        self.obj.free_temps(code)
4631

William Stein's avatar
William Stein committed
4632 4633
    def generate_deletion_code(self, code):
        self.obj.generate_evaluation_code(code)
4634
        if self.is_py_attr or (isinstance(self.entry.scope, Symtab.PropertyScope)
4635
                               and u'__del__' in self.entry.scope.entries):
4636 4637 4638
            code.put_error_if_neg(self.pos,
                'PyObject_DelAttr(%s, %s)' % (
                    self.obj.py_result(),
4639
                    code.intern_identifier(self.attribute)))
William Stein's avatar
William Stein committed
4640 4641 4642
        else:
            error(self.pos, "Cannot delete C attribute of extension type")
        self.obj.generate_disposal_code(code)
4643
        self.obj.free_temps(code)
4644

4645 4646 4647 4648 4649
    def annotate(self, code):
        if self.is_py_attr:
            code.annotate(self.pos, AnnotationItem('py_attr', 'python attribute', size=len(self.attribute)))
        else:
            code.annotate(self.pos, AnnotationItem('c_attr', 'c attribute', size=len(self.attribute)))
William Stein's avatar
William Stein committed
4650

4651 4652
    def put_nonecheck(self, code):
        code.globalstate.use_utility_code(raise_noneattr_error_utility_code)
4653 4654 4655
        if self.obj.type.is_extension_type:
            test = "%s == Py_None" % self.obj.result_as(PyrexTypes.py_object_type)
        elif self.obj.type.is_memoryviewslice:
4656
            test = "(PyObject *) %s.memview == Py_None" % self.obj.result()
4657 4658 4659
        else:
            assert False
        code.putln("if (%s) {" % code.unlikely(test))
Stefan Behnel's avatar
Stefan Behnel committed
4660
        code.putln("__Pyx_RaiseNoneAttributeError(\"%s\");" % self.attribute)
4661 4662 4663 4664
        code.putln(code.error_goto(self.pos))
        code.putln("}")


William Stein's avatar
William Stein committed
4665 4666 4667 4668 4669 4670
#-------------------------------------------------------------------
#
#  Constructor nodes
#
#-------------------------------------------------------------------

4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685
class StarredTargetNode(ExprNode):
    #  A starred expression like "*a"
    #
    #  This is only allowed in sequence assignment targets such as
    #
    #      a, *b = (1,2,3,4)    =>     a = 1 ; b = [2,3,4]
    #
    #  and will be removed during type analysis (or generate an error
    #  if it's found at unexpected places).
    #
    #  target          ExprNode

    subexprs = ['target']
    is_starred = 1
    type = py_object_type
Robert Bradshaw's avatar
Robert Bradshaw committed
4686
    is_temp = 1
4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714

    def __init__(self, pos, target):
        self.pos = pos
        self.target = target

    def analyse_declarations(self, env):
        error(self.pos, "can use starred expression only as assignment target")
        self.target.analyse_declarations(env)

    def analyse_types(self, env):
        error(self.pos, "can use starred expression only as assignment target")
        self.target.analyse_types(env)
        self.type = self.target.type

    def analyse_target_declaration(self, env):
        self.target.analyse_target_declaration(env)

    def analyse_target_types(self, env):
        self.target.analyse_target_types(env)
        self.type = self.target.type

    def calculate_result_code(self):
        return ""

    def generate_result_code(self, code):
        pass


4715
class SequenceNode(ExprNode):
William Stein's avatar
William Stein committed
4716 4717 4718 4719 4720 4721
    #  Base class for list and tuple constructor nodes.
    #  Contains common code for performing sequence unpacking.
    #
    #  args                    [ExprNode]
    #  unpacked_items          [ExprNode] or None
    #  coerced_unpacked_items  [ExprNode] or None
4722
    # mult_factor              ExprNode     the integer number of content repetitions ([1,2]*3)
4723

4724
    subexprs = ['args', 'mult_factor']
4725

William Stein's avatar
William Stein committed
4726 4727
    is_sequence_constructor = 1
    unpacked_items = None
4728
    mult_factor = None
4729

4730 4731 4732
    def compile_time_value_list(self, denv):
        return [arg.compile_time_value(denv) for arg in self.args]

4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746
    def replace_starred_target_node(self):
        # replace a starred node in the targets by the contained expression
        self.starred_assignment = False
        args = []
        for arg in self.args:
            if arg.is_starred:
                if self.starred_assignment:
                    error(arg.pos, "more than 1 starred expression in assignment")
                self.starred_assignment = True
                arg = arg.target
                arg.is_starred = True
            args.append(arg)
        self.args = args

William Stein's avatar
William Stein committed
4747
    def analyse_target_declaration(self, env):
4748
        self.replace_starred_target_node()
William Stein's avatar
William Stein committed
4749 4750 4751
        for arg in self.args:
            arg.analyse_target_declaration(env)

4752
    def analyse_types(self, env, skip_children=False):
William Stein's avatar
William Stein committed
4753 4754
        for i in range(len(self.args)):
            arg = self.args[i]
4755
            if not skip_children: arg.analyse_types(env)
William Stein's avatar
William Stein committed
4756
            self.args[i] = arg.coerce_to_pyobject(env)
4757 4758 4759
        if self.mult_factor:
            self.mult_factor.analyse_types(env)
            if not self.mult_factor.type.is_int:
4760
                self.mult_factor = self.mult_factor.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
4761
        self.is_temp = 1
Stefan Behnel's avatar
Stefan Behnel committed
4762
        # not setting self.type here, subtypes do this
4763

4764 4765 4766
    def may_be_none(self):
        return False

William Stein's avatar
William Stein committed
4767
    def analyse_target_types(self, env):
4768
        if self.mult_factor:
Stefan Behnel's avatar
Stefan Behnel committed
4769
            error(self.pos, "can't assign to multiplied sequence")
4770
        self.unpacked_items = []
William Stein's avatar
William Stein committed
4771
        self.coerced_unpacked_items = []
4772
        self.any_coerced_items = False
William Stein's avatar
William Stein committed
4773 4774
        for arg in self.args:
            arg.analyse_target_types(env)
4775 4776 4777 4778 4779 4780
            if arg.is_starred:
                if not arg.type.assignable_from(Builtin.list_type):
                    error(arg.pos,
                          "starred target must have Python object (list) type")
                if arg.type is py_object_type:
                    arg.type = Builtin.list_type
William Stein's avatar
William Stein committed
4781 4782
            unpacked_item = PyTempNode(self.pos, env)
            coerced_unpacked_item = unpacked_item.coerce_to(arg.type, env)
4783 4784
            if unpacked_item is not coerced_unpacked_item:
                self.any_coerced_items = True
William Stein's avatar
William Stein committed
4785 4786 4787
            self.unpacked_items.append(unpacked_item)
            self.coerced_unpacked_items.append(coerced_unpacked_item)
        self.type = py_object_type
4788

William Stein's avatar
William Stein committed
4789 4790
    def generate_result_code(self, code):
        self.generate_operation_code(code)
4791

4792 4793 4794 4795 4796 4797
    def generate_sequence_packing_code(self, code, target=None, plain=False):
        if target is None:
            target = self.result()
        py_multiply = self.mult_factor and not self.mult_factor.type.is_int
        if plain or py_multiply:
            mult_factor = None
4798
        else:
4799 4800 4801 4802 4803 4804
            mult_factor = self.mult_factor
        if mult_factor:
            mult = mult_factor.result()
            if isinstance(mult_factor.constant_result, (int,long)) \
                   and mult_factor.constant_result > 0:
                size_factor = ' * %s' % mult_factor.constant_result
4805 4806 4807 4808 4809
            else:
                size_factor = ' * ((%s<0) ? 0:%s)' % (mult, mult)
        else:
            size_factor = ''
            mult = ''
4810 4811 4812 4813 4814 4815

        if self.type is Builtin.list_type:
            create_func, set_item_func = 'PyList_New', 'PyList_SET_ITEM'
        elif self.type is Builtin.tuple_type:
            create_func, set_item_func = 'PyTuple_New', 'PyTuple_SET_ITEM'
        else:
Stefan Behnel's avatar
Stefan Behnel committed
4816
            raise InternalError("sequence packing for unexpected type %s" % self.type)
4817 4818
        arg_count = len(self.args)
        code.putln("%s = %s(%s%s); %s" % (
4819 4820 4821 4822
            target, create_func, arg_count, size_factor,
            code.error_goto_if_null(target, self.pos)))
        code.put_gotref(target)

4823
        if mult:
4824 4825 4826 4827
            # FIXME: can't use a temp variable here as the code may
            # end up in the constant building function.  Temps
            # currently don't work there.

4828 4829
            #counter = code.funcstate.allocate_temp(mult_factor.type, manage_ref=False)
            counter = Naming.quick_temp_cname
4830 4831
            code.putln('{ Py_ssize_t %s;' % counter)
            if arg_count == 1:
Stefan Behnel's avatar
Stefan Behnel committed
4832
                offset = counter
4833
            else:
Stefan Behnel's avatar
Stefan Behnel committed
4834
                offset = '%s * %s' % (counter, arg_count)
4835 4836 4837 4838 4839 4840 4841 4842 4843
            code.putln('for (%s=0; %s < %s; %s++) {' % (
                counter, counter, mult, counter
                ))
        else:
            offset = ''
        for i in xrange(arg_count):
            arg = self.args[i]
            if mult or not arg.result_in_temp():
                code.put_incref(arg.result(), arg.ctype())
Stefan Behnel's avatar
Stefan Behnel committed
4844
            code.putln("%s(%s, %s, %s);" % (
4845
                set_item_func,
4846
                target,
Stefan Behnel's avatar
Stefan Behnel committed
4847
                (offset and i) and ('%s + %s' % (offset, i)) or (offset or i),
4848 4849 4850 4851
                arg.py_result()))
            code.put_giveref(arg.py_result())
        if mult:
            code.putln('}')
4852 4853
            #code.funcstate.release_temp(counter)
            code.putln('}')
4854
        elif py_multiply and not plain:
4855
            code.putln('{ PyObject* %s = PyNumber_InPlaceMultiply(%s, %s); %s' % (
4856 4857 4858 4859 4860 4861 4862
                Naming.quick_temp_cname, target, self.mult_factor.py_result(),
                code.error_goto_if_null(Naming.quick_temp_cname, self.pos)
                ))
            code.put_gotref(Naming.quick_temp_cname)
            code.put_decref(target, py_object_type)
            code.putln('%s = %s;' % (target, Naming.quick_temp_cname))
            code.putln('}')
4863 4864

    def generate_subexpr_disposal_code(self, code):
4865
        if self.mult_factor and self.mult_factor.type.is_int:
4866 4867 4868 4869 4870 4871 4872 4873 4874
            super(SequenceNode, self).generate_subexpr_disposal_code(code)
        else:
            # We call generate_post_assignment_code here instead
            # of generate_disposal_code, because values were stored
            # in the tuple using a reference-stealing operation.
            for arg in self.args:
                arg.generate_post_assignment_code(code)
                # Should NOT call free_temps -- this is invoked by the default
                # generate_evaluation_code which will do that.
4875 4876
            if self.mult_factor:
                self.mult_factor.generate_disposal_code(code)
4877

William Stein's avatar
William Stein committed
4878
    def generate_assignment_code(self, rhs, code):
4879 4880 4881
        if self.starred_assignment:
            self.generate_starred_assignment_code(rhs, code)
        else:
4882
            self.generate_parallel_assignment_code(rhs, code)
4883 4884 4885 4886 4887

        for item in self.unpacked_items:
            item.release(code)
        rhs.free_temps(code)

4888 4889 4890 4891 4892
    _func_iternext_type = PyrexTypes.CPtrType(PyrexTypes.CFuncType(
        PyrexTypes.py_object_type, [
            PyrexTypes.CFuncTypeArg("it", PyrexTypes.py_object_type, None),
            ]))

4893
    def generate_parallel_assignment_code(self, rhs, code):
4894 4895 4896
        # Need to work around the fact that generate_evaluation_code
        # allocates the temps in a rather hacky way -- the assignment
        # is evaluated twice, within each if-block.
4897 4898
        for item in self.unpacked_items:
            item.allocate(code)
4899 4900 4901
        special_unpack = (rhs.type is py_object_type
                          or rhs.type in (tuple_type, list_type)
                          or not rhs.type.is_builtin_type)
4902
        long_enough_for_a_loop = len(self.unpacked_items) > 3
4903 4904 4905
        if special_unpack:
            tuple_check = 'likely(PyTuple_CheckExact(%s))' % rhs.py_result()
            list_check  = 'PyList_CheckExact(%s)' % rhs.py_result()
4906
            sequence_type_test = '1'
4907 4908
            if rhs.type is list_type:
                sequence_types = ['List']
4909 4910
                if rhs.may_be_none():
                    sequence_type_test = list_check
4911 4912
            elif rhs.type is tuple_type:
                sequence_types = ['Tuple']
4913 4914
                if rhs.may_be_none():
                    sequence_type_test = tuple_check
4915 4916 4917
            else:
                sequence_types = ['Tuple', 'List']
                sequence_type_test = "(%s) || (%s)" % (tuple_check, list_check)
4918
            code.putln("#if CYTHON_COMPILING_IN_CPYTHON")
4919 4920 4921 4922
            code.putln("if (%s) {" % sequence_type_test)
            code.putln("PyObject* sequence = %s;" % rhs.py_result())
            if len(sequence_types) == 2:
                code.putln("if (likely(Py%s_CheckExact(sequence))) {" % sequence_types[0])
4923 4924 4925
            self.generate_special_parallel_unpacking_code(
                code, sequence_types[0],
                use_loop=long_enough_for_a_loop and sequence_types[0] != 'Tuple')
4926 4927
            if len(sequence_types) == 2:
                code.putln("} else {")
4928 4929
                self.generate_special_parallel_unpacking_code(
                    code, sequence_types[1], use_loop=long_enough_for_a_loop)
4930 4931
                code.putln("}")
            rhs.generate_disposal_code(code)
4932
            code.putln("} else")
Stefan Behnel's avatar
Stefan Behnel committed
4933 4934 4935 4936 4937 4938
            if rhs.type is tuple_type:
                code.putln("if (1) {")
                code.globalstate.use_utility_code(tuple_unpacking_error_code)
                code.putln("__Pyx_UnpackTupleError(%s, %s); %s" % (
                    rhs.py_result(), len(self.args), code.error_goto(self.pos)))
                code.putln("} else")
4939
            code.putln("#endif")
4940

4941
        code.putln("{")
Stefan Behnel's avatar
Stefan Behnel committed
4942 4943
        self.generate_generic_parallel_unpacking_code(
            code, rhs, self.unpacked_items, use_loop=long_enough_for_a_loop)
4944
        code.putln("}")
4945

4946 4947
        for value_node in self.coerced_unpacked_items:
            value_node.generate_evaluation_code(code)
4948 4949 4950
        for i in range(len(self.args)):
            self.args[i].generate_assignment_code(
                self.coerced_unpacked_items[i], code)
4951

4952
    def generate_special_parallel_unpacking_code(self, code, sequence_type, use_loop):
4953 4954
        code.globalstate.use_utility_code(raise_need_more_values_to_unpack)
        code.globalstate.use_utility_code(raise_too_many_values_to_unpack)
4955 4956 4957 4958 4959 4960 4961

        if use_loop:
            # must be at the start of a C block!
            code.putln("PyObject** temps[%s] = {%s};" % (
                len(self.unpacked_items),
                ','.join(['&%s' % item.result() for item in self.unpacked_items])))

4962 4963 4964 4965 4966 4967 4968
        code.putln("if (unlikely(Py%s_GET_SIZE(sequence) != %d)) {" % (
            sequence_type, len(self.args)))
        code.putln("if (Py%s_GET_SIZE(sequence) > %d) __Pyx_RaiseTooManyValuesError(%d);" % (
            sequence_type, len(self.args), len(self.args)))
        code.putln("else __Pyx_RaiseNeedMoreValuesError(Py%s_GET_SIZE(sequence));" % sequence_type)
        code.putln(code.error_goto(self.pos))
        code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
4969

4970 4971 4972 4973 4974
        if use_loop:
            # shorter code in a loop works better for lists in CPython
            counter = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln("for (%s=0; %s < %s; %s++) {" % (
                counter, counter, len(self.unpacked_items), counter))
Stefan Behnel's avatar
Stefan Behnel committed
4975 4976 4977
            code.putln("PyObject* item = Py%s_GET_ITEM(sequence, %s);" % (
                sequence_type, counter))
            code.putln("*(temps[%s]) = item;" % counter)
4978 4979 4980 4981 4982 4983 4984 4985 4986
            code.put_incref("item", PyrexTypes.py_object_type)
            code.putln("}")
            code.funcstate.release_temp(counter)
        else:
            # unrolling the loop is very fast for tuples in CPython
            for i, item in enumerate(self.unpacked_items):
                code.putln("%s = Py%s_GET_ITEM(sequence, %d); " % (item.result(), sequence_type, i))
                code.put_incref(item.result(), item.ctype())

4987
    def generate_generic_parallel_unpacking_code(self, code, rhs, unpacked_items, use_loop, terminate=True):
4988
        code.globalstate.use_utility_code(raise_need_more_values_to_unpack)
4989
        code.globalstate.use_utility_code(UtilityCode.load_cached("IterFinish", "ObjectHandling.c"))
4990
        code.putln("Py_ssize_t index = -1;") # must be at the start of a C block!
4991

4992 4993 4994
        if use_loop:
            code.putln("PyObject** temps[%s] = {%s};" % (
                len(self.unpacked_items),
4995
                ','.join(['&%s' % item.result() for item in unpacked_items])))
4996

4997 4998 4999 5000 5001 5002 5003 5004
        iterator_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        code.putln(
            "%s = PyObject_GetIter(%s); %s" % (
                iterator_temp,
                rhs.py_result(),
                code.error_goto_if_null(iterator_temp, self.pos)))
        code.put_gotref(iterator_temp)
        rhs.generate_disposal_code(code)
5005

5006 5007 5008
        iternext_func = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False)
        code.putln("%s = Py_TYPE(%s)->tp_iternext;" % (
            iternext_func, iterator_temp))
William Stein's avatar
William Stein committed
5009

5010 5011
        unpacking_error_label = code.new_label('unpacking_failed')
        unpack_code = "%s(%s)" % (iternext_func, iterator_temp)
5012
        if use_loop:
5013
            code.putln("for (index=0; index < %s; index++) {" % len(unpacked_items))
5014 5015 5016 5017 5018 5019
            code.put("PyObject* item = %s; if (unlikely(!item)) " % unpack_code)
            code.put_goto(unpacking_error_label)
            code.put_gotref("item")
            code.putln("*(temps[index]) = item;")
            code.putln("}")
        else:
5020
            for i, item in enumerate(unpacked_items):
5021 5022 5023 5024 5025 5026 5027 5028
                code.put(
                    "index = %d; %s = %s; if (unlikely(!%s)) " % (
                        i,
                        item.result(),
                        unpack_code,
                        item.result()))
                code.put_goto(unpacking_error_label)
                code.put_gotref(item.py_result())
5029 5030

        if terminate:
5031 5032
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("UnpackItemEndCheck", "ObjectHandling.c"))
5033 5034 5035 5036 5037 5038
            code.put_error_if_neg(self.pos, "__Pyx_IternextUnpackEndCheck(%s, %d)" % (
                unpack_code,
                len(unpacked_items)))
            code.putln("%s = NULL;" % iternext_func)
            code.put_decref_clear(iterator_temp, py_object_type)

5039 5040 5041 5042 5043
        unpacking_done_label = code.new_label('unpacking_done')
        code.put_goto(unpacking_done_label)

        code.put_label(unpacking_error_label)
        code.put_decref_clear(iterator_temp, py_object_type)
5044
        code.putln("%s = NULL;" % iternext_func)
5045
        code.putln("if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);")
5046 5047
        code.putln(code.error_goto(self.pos))
        code.put_label(unpacking_done_label)
5048

5049 5050 5051 5052 5053 5054 5055
        code.funcstate.release_temp(iternext_func)
        if terminate:
            code.funcstate.release_temp(iterator_temp)
            iterator_temp = None

        return iterator_temp

5056 5057 5058 5059
    def generate_starred_assignment_code(self, rhs, code):
        for i, arg in enumerate(self.args):
            if arg.is_starred:
                starred_target = self.unpacked_items[i]
5060 5061
                unpacked_fixed_items_left  = self.unpacked_items[:i]
                unpacked_fixed_items_right = self.unpacked_items[i+1:]
5062
                break
5063 5064
        else:
            assert False
5065

5066 5067
        iterator_temp = None
        if unpacked_fixed_items_left:
5068
            for item in unpacked_fixed_items_left:
5069
                item.allocate(code)
5070 5071 5072 5073 5074
            code.putln('{')
            iterator_temp = self.generate_generic_parallel_unpacking_code(
                code, rhs, unpacked_fixed_items_left,
                use_loop=True, terminate=False)
            for i, item in enumerate(unpacked_fixed_items_left):
5075 5076
                value_node = self.coerced_unpacked_items[i]
                value_node.generate_evaluation_code(code)
5077
            code.putln('}')
5078

5079
        starred_target.allocate(code)
5080 5081
        target_list = starred_target.result()
        code.putln("%s = PySequence_List(%s); %s" % (
5082 5083
            target_list,
            iterator_temp or rhs.py_result(),
5084 5085
            code.error_goto_if_null(target_list, self.pos)))
        code.put_gotref(target_list)
5086 5087 5088 5089 5090 5091 5092 5093

        if iterator_temp:
            code.put_decref_clear(iterator_temp, py_object_type)
            code.funcstate.release_temp(iterator_temp)
        else:
            rhs.generate_disposal_code(code)

        if unpacked_fixed_items_right:
5094
            code.globalstate.use_utility_code(raise_need_more_values_to_unpack)
5095 5096 5097 5098 5099 5100
            length_temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln('%s = PyList_GET_SIZE(%s);' % (length_temp, target_list))
            code.putln("if (unlikely(%s < %d)) {" % (length_temp, len(unpacked_fixed_items_right)))
            code.putln("__Pyx_RaiseNeedMoreValuesError(%d+%s); %s" % (
                 len(unpacked_fixed_items_left), length_temp,
                 code.error_goto(self.pos)))
5101
            code.putln('}')
5102 5103 5104 5105 5106

            for item in unpacked_fixed_items_right[::-1]:
                item.allocate(code)
            for i, (item, coerced_arg) in enumerate(zip(unpacked_fixed_items_right[::-1],
                                                        self.coerced_unpacked_items[::-1])):
5107 5108 5109
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
                code.putln("%s = PyList_GET_ITEM(%s, %s-%d); " % (
                    item.py_result(), target_list, length_temp, i+1))
5110
                # resize the list the hard way
5111
                code.putln("((PyVarObject*)%s)->ob_size--;" % target_list)
5112 5113 5114 5115
                code.putln('#else')
                code.putln("%s = PySequence_GetItem(%s, %s-%d); " % (
                    item.py_result(), target_list, length_temp, i+1))
                code.putln('#endif')
5116
                code.put_gotref(item.py_result())
5117 5118
                coerced_arg.generate_evaluation_code(code)

5119
            code.putln('#if !CYTHON_COMPILING_IN_CPYTHON')
5120
            sublist_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
Stefan Behnel's avatar
Stefan Behnel committed
5121
            code.putln('%s = PySequence_GetSlice(%s, 0, %s-%d); %s' % (
5122 5123 5124 5125 5126 5127
                sublist_temp, target_list, length_temp, len(unpacked_fixed_items_right),
                code.error_goto_if_null(sublist_temp, self.pos)))
            code.put_gotref(sublist_temp)
            code.funcstate.release_temp(length_temp)
            code.put_decref(target_list, py_object_type)
            code.putln('%s = %s; %s = NULL;' % (target_list, sublist_temp, sublist_temp))
5128 5129
            code.putln('#else')
            code.putln('%s = %s;' % (sublist_temp, sublist_temp)) # avoid warning about unused variable
5130 5131 5132 5133 5134
            code.funcstate.release_temp(sublist_temp)
            code.putln('#endif')

        for i, arg in enumerate(self.args):
            arg.generate_assignment_code(self.coerced_unpacked_items[i], code)
5135

5136 5137 5138 5139 5140 5141 5142 5143
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)
        if self.unpacked_items:
            for arg in self.unpacked_items:
                arg.annotate(code)
            for arg in self.coerced_unpacked_items:
                arg.annotate(code)
William Stein's avatar
William Stein committed
5144 5145 5146 5147


class TupleNode(SequenceNode):
    #  Tuple constructor.
5148

5149
    type = tuple_type
5150
    is_partly_literal = False
5151 5152 5153

    gil_message = "Constructing Python tuple"

5154
    def analyse_types(self, env, skip_children=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
5155
        if len(self.args) == 0:
5156 5157
            self.is_temp = False
            self.is_literal = True
Robert Bradshaw's avatar
Robert Bradshaw committed
5158
        else:
5159
            SequenceNode.analyse_types(self, env, skip_children)
5160 5161 5162 5163
            for child in self.args:
                if not child.is_literal:
                    break
            else:
5164 5165 5166 5167 5168
                if not self.mult_factor or self.mult_factor.is_literal and \
                       isinstance(self.mult_factor.constant_result, (int, long)):
                    self.is_temp = False
                    self.is_literal = True
                else:
5169 5170
                    if not self.mult_factor.type.is_pyobject:
                        self.mult_factor = self.mult_factor.coerce_to_pyobject(env)
5171 5172
                    self.is_temp = True
                    self.is_partly_literal = True
5173

Stefan Behnel's avatar
Stefan Behnel committed
5174 5175 5176 5177
    def is_simple(self):
        # either temp or constant => always simple
        return True

5178 5179 5180 5181
    def nonlocally_immutable(self):
        # either temp or constant => always safe
        return True

Robert Bradshaw's avatar
Robert Bradshaw committed
5182 5183
    def calculate_result_code(self):
        if len(self.args) > 0:
5184
            return self.result_code
Robert Bradshaw's avatar
Robert Bradshaw committed
5185 5186
        else:
            return Naming.empty_tuple
William Stein's avatar
William Stein committed
5187

5188 5189 5190 5191
    def calculate_constant_result(self):
        self.constant_result = tuple([
                arg.constant_result for arg in self.args])

5192 5193 5194 5195 5196 5197
    def compile_time_value(self, denv):
        values = self.compile_time_value_list(denv)
        try:
            return tuple(values)
        except Exception, e:
            self.compile_time_value_error(e)
5198

William Stein's avatar
William Stein committed
5199
    def generate_operation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5200 5201 5202
        if len(self.args) == 0:
            # result_code is Naming.empty_tuple
            return
5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215
        if self.is_partly_literal:
            # underlying tuple is const, but factor is not
            tuple_target = code.get_py_const(py_object_type, 'tuple_', cleanup_level=2)
            const_code = code.get_cached_constants_writer()
            const_code.mark_pos(self.pos)
            self.generate_sequence_packing_code(const_code, tuple_target, plain=True)
            const_code.put_giveref(tuple_target)
            code.putln('%s = PyNumber_Multiply(%s, %s); %s' % (
                self.result(), tuple_target, self.mult_factor.py_result(),
                code.error_goto_if_null(self.result(), self.pos)
                ))
            code.put_gotref(self.py_result())
        elif self.is_literal:
5216 5217
            # non-empty cached tuple => result is global constant,
            # creation code goes into separate code writer
5218
            self.result_code = code.get_py_const(py_object_type, 'tuple_', cleanup_level=2)
5219 5220
            code = code.get_cached_constants_writer()
            code.mark_pos(self.pos)
5221
            self.generate_sequence_packing_code(code)
5222
            code.put_giveref(self.py_result())
5223 5224
        else:
            self.generate_sequence_packing_code(code)
William Stein's avatar
William Stein committed
5225 5226 5227 5228


class ListNode(SequenceNode):
    #  List constructor.
5229

5230 5231
    # obj_conversion_errors    [PyrexError]   used internally
    # orignial_args            [ExprNode]     used internally
5232

5233
    obj_conversion_errors = []
Stefan Behnel's avatar
Stefan Behnel committed
5234
    type = list_type
5235

5236
    gil_message = "Constructing Python list"
5237

Robert Bradshaw's avatar
Robert Bradshaw committed
5238
    def type_dependencies(self, env):
5239
        return ()
5240

5241 5242 5243
    def infer_type(self, env):
        # TOOD: Infer non-object list arrays.
        return list_type
5244

5245
    def analyse_expressions(self, env):
5246
        SequenceNode.analyse_expressions(self, env)
5247 5248
        self.coerce_to_pyobject(env)

Robert Bradshaw's avatar
Robert Bradshaw committed
5249
    def analyse_types(self, env):
5250 5251 5252 5253 5254
        hold_errors()
        self.original_args = list(self.args)
        SequenceNode.analyse_types(self, env)
        self.obj_conversion_errors = held_errors()
        release_errors(ignore=True)
5255

Robert Bradshaw's avatar
Robert Bradshaw committed
5256 5257
    def coerce_to(self, dst_type, env):
        if dst_type.is_pyobject:
5258 5259 5260
            for err in self.obj_conversion_errors:
                report_error(err)
            self.obj_conversion_errors = []
Robert Bradshaw's avatar
Robert Bradshaw committed
5261 5262
            if not self.type.subtype_of(dst_type):
                error(self.pos, "Cannot coerce list to type '%s'" % dst_type)
5263 5264
        elif self.mult_factor:
            error(self.pos, "Cannot coerce multiplied list to '%s'" % dst_type)
5265
        elif dst_type.is_ptr and dst_type.base_type is not PyrexTypes.c_void_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
5266
            base_type = dst_type.base_type
Robert Bradshaw's avatar
Robert Bradshaw committed
5267
            self.type = PyrexTypes.CArrayType(base_type, len(self.args))
5268
            for i in range(len(self.original_args)):
Robert Bradshaw's avatar
Robert Bradshaw committed
5269
                arg = self.args[i]
5270 5271
                if isinstance(arg, CoerceToPyTypeNode):
                    arg = arg.arg
Robert Bradshaw's avatar
Robert Bradshaw committed
5272
                self.args[i] = arg.coerce_to(base_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5273 5274 5275 5276 5277 5278
        elif dst_type.is_struct:
            if len(self.args) > len(dst_type.scope.var_entries):
                error(self.pos, "Too may members for '%s'" % dst_type)
            else:
                if len(self.args) < len(dst_type.scope.var_entries):
                    warning(self.pos, "Too few members for '%s'" % dst_type, 1)
5279 5280 5281
                for i, (arg, member) in enumerate(zip(self.original_args, dst_type.scope.var_entries)):
                    if isinstance(arg, CoerceToPyTypeNode):
                        arg = arg.arg
Robert Bradshaw's avatar
Robert Bradshaw committed
5282 5283
                    self.args[i] = arg.coerce_to(member.type, env)
            self.type = dst_type
Robert Bradshaw's avatar
Robert Bradshaw committed
5284 5285 5286 5287
        else:
            self.type = error_type
            error(self.pos, "Cannot coerce list to type '%s'" % dst_type)
        return self
5288

Robert Bradshaw's avatar
Robert Bradshaw committed
5289 5290
    def release_temp(self, env):
        if self.type.is_array:
5291 5292
            # To be valid C++, we must allocate the memory on the stack
            # manually and be sure not to reuse it for something else.
Robert Bradshaw's avatar
Robert Bradshaw committed
5293 5294 5295
            pass
        else:
            SequenceNode.release_temp(self, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5296

5297
    def calculate_constant_result(self):
5298 5299
        if self.mult_factor:
            raise ValueError() # may exceed the compile time memory
5300 5301 5302
        self.constant_result = [
            arg.constant_result for arg in self.args]

5303
    def compile_time_value(self, denv):
5304 5305 5306 5307
        l = self.compile_time_value_list(denv)
        if self.mult_factor:
            l *= self.mult_factor.compile_time_value(denv)
        return l
5308

William Stein's avatar
William Stein committed
5309
    def generate_operation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5310
        if self.type.is_pyobject:
5311 5312
            for err in self.obj_conversion_errors:
                report_error(err)
5313
            self.generate_sequence_packing_code(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
5314 5315 5316 5317 5318 5319
        elif self.type.is_array:
            for i, arg in enumerate(self.args):
                code.putln("%s[%s] = %s;" % (
                                self.result(),
                                i,
                                arg.result()))
Robert Bradshaw's avatar
Robert Bradshaw committed
5320
        elif self.type.is_struct:
Robert Bradshaw's avatar
Robert Bradshaw committed
5321 5322 5323 5324 5325
            for arg, member in zip(self.args, self.type.scope.var_entries):
                code.putln("%s.%s = %s;" % (
                        self.result(),
                        member.cname,
                        arg.result()))
5326 5327
        else:
            raise InternalError("List type never specified")
5328

Robert Bradshaw's avatar
Robert Bradshaw committed
5329

5330 5331 5332 5333 5334 5335 5336 5337 5338
class ScopedExprNode(ExprNode):
    # Abstract base class for ExprNodes that have their own local
    # scope, such as generator expressions.
    #
    # expr_scope    Scope  the inner scope of the expression

    subexprs = []
    expr_scope = None

5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357
    # does this node really have a local scope, e.g. does it leak loop
    # variables or not?  non-leaking Py3 behaviour is default, except
    # for list comprehensions where the behaviour differs in Py2 and
    # Py3 (set in Parsing.py based on parser context)
    has_local_scope = True

    def init_scope(self, outer_scope, expr_scope=None):
        if expr_scope is not None:
            self.expr_scope = expr_scope
        elif self.has_local_scope:
            self.expr_scope = Symtab.GeneratorExpressionScope(outer_scope)
        else:
            self.expr_scope = None

    def analyse_declarations(self, env):
        self.init_scope(env)

    def analyse_scoped_declarations(self, env):
        # this is called with the expr_scope as env
5358 5359
        pass

5360 5361
    def analyse_types(self, env):
        # no recursion here, the children will be analysed separately below
5362 5363 5364 5365 5366 5367
        pass

    def analyse_scoped_expressions(self, env):
        # this is called with the expr_scope as env
        pass

5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414
    def generate_evaluation_code(self, code):
        # set up local variables and free their references on exit
        generate_inner_evaluation_code = super(ScopedExprNode, self).generate_evaluation_code
        if not self.has_local_scope or not self.expr_scope.var_entries:
            # no local variables => delegate, done
            generate_inner_evaluation_code(code)
            return

        code.putln('{ /* enter inner scope */')
        py_entries = []
        for entry in self.expr_scope.var_entries:
            if not entry.in_closure:
                code.put_var_declaration(entry)
                if entry.type.is_pyobject and entry.used:
                    py_entries.append(entry)
        if not py_entries:
            # no local Python references => no cleanup required
            generate_inner_evaluation_code(code)
            code.putln('} /* exit inner scope */')
            return

        # must free all local Python references at each exit point
        old_loop_labels = tuple(code.new_loop_labels())
        old_error_label = code.new_error_label()

        generate_inner_evaluation_code(code)

        # normal (non-error) exit
        for entry in py_entries:
            code.put_var_decref(entry)

        # error/loop body exit points
        exit_scope = code.new_label('exit_scope')
        code.put_goto(exit_scope)
        for label, old_label in ([(code.error_label, old_error_label)] +
                                 list(zip(code.get_loop_labels(), old_loop_labels))):
            if code.label_used(label):
                code.put_label(label)
                for entry in py_entries:
                    code.put_var_decref(entry)
                code.put_goto(old_label)
        code.put_label(exit_scope)
        code.putln('} /* exit inner scope */')

        code.set_loop_labels(old_loop_labels)
        code.error_label = old_error_label

5415 5416

class ComprehensionNode(ScopedExprNode):
5417
    subexprs = ["target"]
5418
    child_attrs = ["loop"]
5419

5420 5421
    def infer_type(self, env):
        return self.target.infer_type(env)
5422 5423 5424

    def analyse_declarations(self, env):
        self.append.target = self # this is used in the PyList_Append of the inner loop
5425 5426
        self.init_scope(env)

5427 5428
    def analyse_scoped_declarations(self, env):
        self.loop.analyse_declarations(env)
5429

5430 5431 5432
    def analyse_types(self, env):
        self.target.analyse_expressions(env)
        self.type = self.target.type
5433 5434
        if not self.has_local_scope:
            self.loop.analyse_expressions(env)
5435

5436 5437 5438
    def analyse_scoped_expressions(self, env):
        if self.has_local_scope:
            self.loop.analyse_expressions(env)
5439

5440 5441 5442
    def may_be_none(self):
        return False

5443 5444
    def calculate_result_code(self):
        return self.target.result()
5445

5446 5447
    def generate_result_code(self, code):
        self.generate_operation_code(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
5448

5449 5450 5451
    def generate_operation_code(self, code):
        self.loop.generate_execution_code(code)

5452 5453
    def annotate(self, code):
        self.loop.annotate(code)
5454 5455


5456
class ComprehensionAppendNode(Node):
5457 5458
    # Need to be careful to avoid infinite recursion:
    # target must not be in child_attrs/subexprs
5459 5460

    child_attrs = ['expr']
5461 5462

    type = PyrexTypes.c_int_type
5463

5464 5465
    def analyse_expressions(self, env):
        self.expr.analyse_expressions(env)
5466
        if not self.expr.type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
5467
            self.expr = self.expr.coerce_to_pyobject(env)
5468

5469
    def generate_execution_code(self, code):
5470
        if self.target.type is list_type:
5471 5472
            code.globalstate.use_utility_code(UtilityCode.load_cached("InternalListAppend", "Optimize.c"))
            function = "__Pyx_PyList_Append"
5473 5474 5475 5476 5477
        elif self.target.type is set_type:
            function = "PySet_Add"
        else:
            raise InternalError(
                "Invalid type for comprehension node: %s" % self.target.type)
5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492

        self.expr.generate_evaluation_code(code)
        code.putln(code.error_goto_if("%s(%s, (PyObject*)%s)" % (
            function,
            self.target.result(),
            self.expr.result()
            ), self.pos))
        self.expr.generate_disposal_code(code)
        self.expr.free_temps(code)

    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

    def annotate(self, code):
        self.expr.annotate(code)
5493 5494

class DictComprehensionAppendNode(ComprehensionAppendNode):
5495
    child_attrs = ['key_expr', 'value_expr']
5496

5497 5498
    def analyse_expressions(self, env):
        self.key_expr.analyse_expressions(env)
5499 5500
        if not self.key_expr.type.is_pyobject:
            self.key_expr = self.key_expr.coerce_to_pyobject(env)
5501
        self.value_expr.analyse_expressions(env)
5502 5503 5504
        if not self.value_expr.type.is_pyobject:
            self.value_expr = self.value_expr.coerce_to_pyobject(env)

5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524
    def generate_execution_code(self, code):
        self.key_expr.generate_evaluation_code(code)
        self.value_expr.generate_evaluation_code(code)
        code.putln(code.error_goto_if("PyDict_SetItem(%s, (PyObject*)%s, (PyObject*)%s)" % (
            self.target.result(),
            self.key_expr.result(),
            self.value_expr.result()
            ), self.pos))
        self.key_expr.generate_disposal_code(code)
        self.key_expr.free_temps(code)
        self.value_expr.generate_disposal_code(code)
        self.value_expr.free_temps(code)

    def generate_function_definitions(self, env, code):
        self.key_expr.generate_function_definitions(env, code)
        self.value_expr.generate_function_definitions(env, code)

    def annotate(self, code):
        self.key_expr.annotate(code)
        self.value_expr.annotate(code)
5525 5526


5527 5528 5529 5530 5531
class InlinedGeneratorExpressionNode(ScopedExprNode):
    # An inlined generator expression for which the result is
    # calculated inside of the loop.  This will only be created by
    # transforms when replacing builtin calls on generator
    # expressions.
5532
    #
5533 5534 5535
    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
    # result_node    ResultRefNode    the reference to the result value temp
    # orig_func      String           the name of the builtin function this node replaces
5536

5537
    child_attrs = ["loop"]
5538
    loop_analysed = False
5539 5540
    type = py_object_type

5541 5542
    def analyse_scoped_declarations(self, env):
        self.loop.analyse_declarations(env)
5543

5544 5545 5546 5547 5548 5549
    def may_be_none(self):
        return False

    def annotate(self, code):
        self.loop.annotate(code)

5550 5551
    def infer_type(self, env):
        return self.result_node.infer_type(env)
5552 5553

    def analyse_types(self, env):
5554 5555 5556
        if not self.has_local_scope:
            self.loop_analysed = True
            self.loop.analyse_expressions(env)
5557 5558 5559
        self.type = self.result_node.type
        self.is_temp = True

5560 5561
    def analyse_scoped_expressions(self, env):
        self.loop_analysed = True
5562 5563
        if self.has_local_scope:
            self.loop.analyse_expressions(env)
5564

5565
    def coerce_to(self, dst_type, env):
5566 5567 5568 5569 5570 5571
        if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
            # We can optimise by dropping the aggregation variable and
            # the add operations into C.  This can only be done safely
            # before analysing the loop body, after that, the result
            # reference type will have infected expressions and
            # assignments.
5572 5573
            self.result_node.type = self.type = dst_type
            return self
5574
        return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
5575

5576 5577 5578 5579 5580
    def generate_result_code(self, code):
        self.result_node.result_code = self.result()
        self.loop.generate_execution_code(code)


5581
class SetNode(ExprNode):
5582 5583
    #  Set constructor.

5584 5585
    type = set_type

5586 5587 5588
    subexprs = ['args']

    gil_message = "Constructing Python set"
5589

5590 5591 5592 5593 5594 5595 5596 5597
    def analyse_types(self, env):
        for i in range(len(self.args)):
            arg = self.args[i]
            arg.analyse_types(env)
            self.args[i] = arg.coerce_to_pyobject(env)
        self.type = set_type
        self.is_temp = 1

5598 5599 5600
    def may_be_none(self):
        return False

5601 5602 5603 5604
    def calculate_constant_result(self):
        self.constant_result = set([
                arg.constant_result for arg in self.args])

5605 5606 5607 5608 5609 5610 5611 5612
    def compile_time_value(self, denv):
        values = [arg.compile_time_value(denv) for arg in self.args]
        try:
            return set(values)
        except Exception, e:
            self.compile_time_value_error(e)

    def generate_evaluation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5613
        code.globalstate.use_utility_code(Builtin.py_set_utility_code)
5614 5615 5616 5617 5618
        self.allocate_temp_result(code)
        code.putln(
            "%s = PySet_New(0); %s" % (
                self.result(),
                code.error_goto_if_null(self.result(), self.pos)))
5619
        code.put_gotref(self.py_result())
5620 5621 5622 5623 5624 5625 5626 5627
        for arg in self.args:
            arg.generate_evaluation_code(code)
            code.putln(
                code.error_goto_if_neg(
                    "PySet_Add(%s, %s)" % (self.result(), arg.py_result()),
                    self.pos))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
5628

William Stein's avatar
William Stein committed
5629

5630
class DictNode(ExprNode):
William Stein's avatar
William Stein committed
5631 5632
    #  Dictionary constructor.
    #
Vitja Makarov's avatar
Vitja Makarov committed
5633 5634
    #  key_value_pairs     [DictItemNode]
    #  exclude_null_values [boolean]          Do not add NULL values to dict
5635 5636
    #
    # obj_conversion_errors    [PyrexError]   used internally
5637

5638
    subexprs = ['key_value_pairs']
5639
    is_temp = 1
Vitja Makarov's avatar
Vitja Makarov committed
5640
    exclude_null_values = False
5641
    type = dict_type
5642

5643
    obj_conversion_errors = []
5644

5645 5646 5647 5648 5649
    @classmethod
    def from_pairs(cls, pos, pairs):
        return cls(pos, key_value_pairs=[
                DictItemNode(pos, key=k, value=v) for k, v in pairs])

5650 5651 5652
    def calculate_constant_result(self):
        self.constant_result = dict([
                item.constant_result for item in self.key_value_pairs])
5653

5654
    def compile_time_value(self, denv):
Robert Bradshaw's avatar
Robert Bradshaw committed
5655 5656
        pairs = [(item.key.compile_time_value(denv), item.value.compile_time_value(denv))
            for item in self.key_value_pairs]
5657 5658 5659 5660
        try:
            return dict(pairs)
        except Exception, e:
            self.compile_time_value_error(e)
5661

Robert Bradshaw's avatar
Robert Bradshaw committed
5662
    def type_dependencies(self, env):
5663
        return ()
5664

5665 5666 5667 5668
    def infer_type(self, env):
        # TOOD: Infer struct constructors.
        return dict_type

William Stein's avatar
William Stein committed
5669
    def analyse_types(self, env):
5670
        hold_errors()
Robert Bradshaw's avatar
Robert Bradshaw committed
5671 5672
        for item in self.key_value_pairs:
            item.analyse_types(env)
5673 5674
        self.obj_conversion_errors = held_errors()
        release_errors(ignore=True)
5675 5676 5677

    def may_be_none(self):
        return False
5678

5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692
    def coerce_to(self, dst_type, env):
        if dst_type.is_pyobject:
            self.release_errors()
            if not self.type.subtype_of(dst_type):
                error(self.pos, "Cannot interpret dict as type '%s'" % dst_type)
        elif dst_type.is_struct_or_union:
            self.type = dst_type
            if not dst_type.is_struct and len(self.key_value_pairs) != 1:
                error(self.pos, "Exactly one field must be specified to convert to union '%s'" % dst_type)
            elif dst_type.is_struct and len(self.key_value_pairs) < len(dst_type.scope.var_entries):
                warning(self.pos, "Not all members given for struct '%s'" % dst_type, 1)
            for item in self.key_value_pairs:
                if isinstance(item.key, CoerceToPyTypeNode):
                    item.key = item.key.arg
5693
                if not item.key.is_string_literal:
5694
                    error(item.key.pos, "Invalid struct field identifier")
5695
                    item.key = StringNode(item.key.pos, value="<error>")
5696
                else:
Stefan Behnel's avatar
Stefan Behnel committed
5697 5698
                    key = str(item.key.value) # converts string literals to unicode in Py3
                    member = dst_type.scope.lookup_here(key)
5699
                    if not member:
Stefan Behnel's avatar
Stefan Behnel committed
5700
                        error(item.key.pos, "struct '%s' has no field '%s'" % (dst_type, key))
5701 5702 5703 5704 5705 5706 5707 5708 5709
                    else:
                        value = item.value
                        if isinstance(value, CoerceToPyTypeNode):
                            value = value.arg
                        item.value = value.coerce_to(member.type, env)
        else:
            self.type = error_type
            error(self.pos, "Cannot interpret dict as type '%s'" % dst_type)
        return self
5710

5711 5712 5713 5714
    def release_errors(self):
        for err in self.obj_conversion_errors:
            report_error(err)
        self.obj_conversion_errors = []
5715 5716 5717

    gil_message = "Constructing Python dict"

William Stein's avatar
William Stein committed
5718 5719 5720
    def generate_evaluation_code(self, code):
        #  Custom method used here because key-value
        #  pairs are evaluated and used one at a time.
5721 5722
        code.mark_pos(self.pos)
        self.allocate_temp_result(code)
5723 5724 5725 5726 5727 5728
        if self.type.is_pyobject:
            self.release_errors()
            code.putln(
                "%s = PyDict_New(); %s" % (
                    self.result(),
                    code.error_goto_if_null(self.result(), self.pos)))
5729
            code.put_gotref(self.py_result())
Robert Bradshaw's avatar
Robert Bradshaw committed
5730 5731
        for item in self.key_value_pairs:
            item.generate_evaluation_code(code)
5732
            if self.type.is_pyobject:
Vitja Makarov's avatar
Vitja Makarov committed
5733 5734
                if self.exclude_null_values:
                    code.putln('if (%s) {' % item.value.py_result())
5735
                code.put_error_if_neg(self.pos,
5736 5737 5738 5739
                    "PyDict_SetItem(%s, %s, %s)" % (
                        self.result(),
                        item.key.py_result(),
                        item.value.py_result()))
Vitja Makarov's avatar
Vitja Makarov committed
5740 5741
                if self.exclude_null_values:
                    code.putln('}')
5742 5743 5744
            else:
                code.putln("%s.%s = %s;" % (
                        self.result(),
5745
                        item.key.value,
5746
                        item.value.result()))
Robert Bradshaw's avatar
Robert Bradshaw committed
5747
            item.generate_disposal_code(code)
5748
            item.free_temps(code)
5749

5750
    def annotate(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5751 5752
        for item in self.key_value_pairs:
            item.annotate(code)
5753

5754
class DictItemNode(ExprNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
5755 5756 5757 5758 5759
    # Represents a single item in a DictNode
    #
    # key          ExprNode
    # value        ExprNode
    subexprs = ['key', 'value']
5760

5761
    nogil_check = None # Parent DictNode takes care of it
5762

5763 5764 5765
    def calculate_constant_result(self):
        self.constant_result = (
            self.key.constant_result, self.value.constant_result)
5766

Robert Bradshaw's avatar
Robert Bradshaw committed
5767 5768 5769 5770 5771
    def analyse_types(self, env):
        self.key.analyse_types(env)
        self.value.analyse_types(env)
        self.key = self.key.coerce_to_pyobject(env)
        self.value = self.value.coerce_to_pyobject(env)
5772

Robert Bradshaw's avatar
Robert Bradshaw committed
5773 5774 5775
    def generate_evaluation_code(self, code):
        self.key.generate_evaluation_code(code)
        self.value.generate_evaluation_code(code)
Stefan Behnel's avatar
Stefan Behnel committed
5776

5777 5778 5779
    def generate_disposal_code(self, code):
        self.key.generate_disposal_code(code)
        self.value.generate_disposal_code(code)
5780 5781 5782 5783

    def free_temps(self, code):
        self.key.free_temps(code)
        self.value.free_temps(code)
5784

5785 5786
    def __iter__(self):
        return iter([self.key, self.value])
William Stein's avatar
William Stein committed
5787

5788

5789 5790 5791 5792 5793 5794 5795
class ModuleNameMixin(object):
    def set_mod_name(self, env):
        self.module_name = env.global_scope().qualified_name

    def get_py_mod_name(self, code):
        return code.get_py_string_const(
                 self.module_name, identifier=True)
Stefan Behnel's avatar
Stefan Behnel committed
5796

5797
class ClassNode(ExprNode, ModuleNameMixin):
William Stein's avatar
William Stein committed
5798 5799 5800 5801
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs a class object given
    #  a name, tuple of bases and class dictionary.
    #
Stefan Behnel's avatar
Stefan Behnel committed
5802
    #  name         EncodedString      Name of the class
William Stein's avatar
William Stein committed
5803 5804 5805
    #  bases        ExprNode           Base class tuple
    #  dict         ExprNode           Class dict (not owned by this node)
    #  doc          ExprNode or None   Doc string
5806
    #  module_name  EncodedString      Name of defining module
5807

5808
    subexprs = ['bases', 'doc']
5809

William Stein's avatar
William Stein committed
5810 5811 5812 5813 5814 5815 5816 5817
    def analyse_types(self, env):
        self.bases.analyse_types(env)
        if self.doc:
            self.doc.analyse_types(env)
            self.doc = self.doc.coerce_to_pyobject(env)
        self.type = py_object_type
        self.is_temp = 1
        env.use_utility_code(create_class_utility_code);
5818 5819
        #TODO(craig,haoyu) This should be moved to a better place
        self.set_mod_name(env)
5820

5821
    def may_be_none(self):
Stefan Behnel's avatar
Stefan Behnel committed
5822
        return True
5823

5824 5825
    gil_message = "Constructing Python class"

William Stein's avatar
William Stein committed
5826
    def generate_result_code(self, code):
5827
        cname = code.intern_identifier(self.name)
5828

William Stein's avatar
William Stein committed
5829
        if self.doc:
5830
            code.put_error_if_neg(self.pos,
Robert Bradshaw's avatar
Robert Bradshaw committed
5831
                'PyDict_SetItemString(%s, "__doc__", %s)' % (
William Stein's avatar
William Stein committed
5832
                    self.dict.py_result(),
Robert Bradshaw's avatar
Robert Bradshaw committed
5833
                    self.doc.py_result()))
5834
        py_mod_name = self.get_py_mod_name(code)
William Stein's avatar
William Stein committed
5835
        code.putln(
5836
            '%s = __Pyx_CreateClass(%s, %s, %s, %s); %s' % (
5837
                self.result(),
William Stein's avatar
William Stein committed
5838 5839
                self.bases.py_result(),
                self.dict.py_result(),
5840
                cname,
5841
                py_mod_name,
5842
                code.error_goto_if_null(self.result(), self.pos)))
5843
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
5844

Stefan Behnel's avatar
Stefan Behnel committed
5845

5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863
class Py3ClassNode(ExprNode):
    #  Helper class used in the implementation of Python3+
    #  class definitions. Constructs a class object given
    #  a name, tuple of bases and class dictionary.
    #
    #  name         EncodedString      Name of the class
    #  dict         ExprNode           Class dict (not owned by this node)
    #  module_name  EncodedString      Name of defining module

    subexprs = []

    def analyse_types(self, env):
        self.type = py_object_type
        self.is_temp = 1

    def may_be_none(self):
        return True

5864
    gil_message = "Constructing Python class"
5865 5866

    def generate_result_code(self, code):
5867
        code.globalstate.use_utility_code(create_py3class_utility_code)
5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880
        cname = code.intern_identifier(self.name)
        code.putln(
            '%s = __Pyx_Py3ClassCreate(%s, %s, %s, %s, %s); %s' % (
                self.result(),
                self.metaclass.result(),
                cname,
                self.bases.py_result(),
                self.dict.py_result(),
                self.mkw.py_result(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())

class KeywordArgsNode(ExprNode):
5881
    #  Helper class for keyword arguments.
5882
    #
5883 5884
    #  starstar_arg      DictNode
    #  keyword_args      [DictItemNode]
5885

5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911
    subexprs = ['starstar_arg', 'keyword_args']
    is_temp = 1
    type = dict_type

    def calculate_constant_result(self):
        result = dict(self.starstar_arg.constant_result)
        for item in self.keyword_args:
            key, value = item.constant_result
            if key in result:
                raise ValueError("duplicate keyword argument found: %s" % key)
            result[key] = value
        self.constant_result = result

    def compile_time_value(self, denv):
        result = self.starstar_arg.compile_time_value(denv)
        pairs = [ (item.key.compile_time_value(denv), item.value.compile_time_value(denv))
                  for item in self.keyword_args ]
        try:
            result = dict(result)
            for key, value in pairs:
                if key in result:
                    raise ValueError("duplicate keyword argument found: %s" % key)
                result[key] = value
        except Exception, e:
            self.compile_time_value_error(e)
        return result
5912

5913 5914 5915 5916 5917
    def type_dependencies(self, env):
        return ()

    def infer_type(self, env):
        return dict_type
5918 5919

    def analyse_types(self, env):
5920 5921 5922 5923 5924 5925
        self.starstar_arg.analyse_types(env)
        self.starstar_arg = self.starstar_arg.coerce_to_pyobject(env).as_none_safe_node(
            # FIXME: CPython's error message starts with the runtime function name
            'argument after ** must be a mapping, not NoneType')
        for item in self.keyword_args:
            item.analyse_types(env)
5926

5927 5928
    def may_be_none(self):
        return False
5929

5930 5931 5932 5933 5934 5935 5936 5937 5938 5939
    gil_message = "Constructing Python dict"

    def generate_evaluation_code(self, code):
        code.mark_pos(self.pos)
        self.allocate_temp_result(code)
        self.starstar_arg.generate_evaluation_code(code)
        if self.starstar_arg.type is not Builtin.dict_type:
            # CPython supports calling functions with non-dicts, so do we
            code.putln('if (likely(PyDict_Check(%s))) {' %
                       self.starstar_arg.py_result())
5940 5941 5942 5943 5944 5945
        if self.keyword_args:
            code.putln(
                "%s = PyDict_Copy(%s); %s" % (
                    self.result(),
                    self.starstar_arg.py_result(),
                    code.error_goto_if_null(self.result(), self.pos)))
5946
            code.put_gotref(self.py_result())
5947
        else:
5948 5949 5950 5951 5952 5953
            code.putln("%s = %s;" % (
                self.result(),
                self.starstar_arg.py_result()))
            code.put_incref(self.result(), py_object_type)
        if self.starstar_arg.type is not Builtin.dict_type:
            code.putln('} else {')
5954
            code.putln(
5955 5956
                "%s = PyObject_CallFunctionObjArgs("
                "(PyObject*)&PyDict_Type, %s, NULL); %s" % (
5957
                    self.result(),
5958
                    self.starstar_arg.py_result(),
5959
                    code.error_goto_if_null(self.result(), self.pos)))
5960
            code.put_gotref(self.py_result())
5961 5962 5963 5964 5965 5966 5967
            code.putln('}')
        self.starstar_arg.generate_disposal_code(code)
        self.starstar_arg.free_temps(code)

        if not self.keyword_args:
            return

5968 5969
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("RaiseDoubleKeywords", "FunctionArguments.c"))
5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991
        for item in self.keyword_args:
            item.generate_evaluation_code(code)
            code.putln("if (unlikely(PyDict_GetItem(%s, %s))) {" % (
                    self.result(),
                    item.key.py_result()))
            # FIXME: find out function name at runtime!
            code.putln('__Pyx_RaiseDoubleKeywordsError("function", %s); %s' % (
                item.key.py_result(),
                code.error_goto(self.pos)))
            code.putln("}")
            code.put_error_if_neg(self.pos,
                "PyDict_SetItem(%s, %s, %s)" % (
                    self.result(),
                    item.key.py_result(),
                    item.value.py_result()))
            item.generate_disposal_code(code)
            item.free_temps(code)

    def annotate(self, code):
        self.starstar_arg.annotate(code)
        for item in self.keyword_args:
            item.annotate(code)
5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059

class PyClassMetaclassNode(ExprNode):
    # Helper class holds Python3 metaclass object
    #
    #  bases        ExprNode           Base class tuple (not owned by this node)
    #  mkw          ExprNode           Class keyword arguments (not owned by this node)

    subexprs = []

    def analyse_types(self, env):
        self.type = py_object_type
        self.is_temp = True

    def may_be_none(self):
        return True

    def generate_result_code(self, code):
        code.putln(
            "%s = __Pyx_Py3MetaclassGet(%s, %s); %s" % (
                self.result(),
                self.bases.result(),
                self.mkw.result(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())

class PyClassNamespaceNode(ExprNode, ModuleNameMixin):
    # Helper class holds Python3 namespace object
    #
    # All this are not owned by this node
    #  metaclass    ExprNode           Metaclass object
    #  bases        ExprNode           Base class tuple
    #  mkw          ExprNode           Class keyword arguments
    #  doc          ExprNode or None   Doc string (owned)

    subexprs = ['doc']

    def analyse_types(self, env):
        self.bases.analyse_types(env)
        if self.doc:
            self.doc.analyse_types(env)
            self.doc = self.doc.coerce_to_pyobject(env)
        self.type = py_object_type
        self.is_temp = 1
        #TODO(craig,haoyu) This should be moved to a better place
        self.set_mod_name(env)

    def may_be_none(self):
        return True

    def generate_result_code(self, code):
        cname = code.intern_identifier(self.name)
        py_mod_name = self.get_py_mod_name(code)
        if self.doc:
            doc_code = self.doc.result()
        else:
            doc_code = '(PyObject *) NULL'
        code.putln(
            "%s = __Pyx_Py3MetaclassPrepare(%s, %s, %s, %s, %s, %s); %s" % (
                self.result(),
                self.metaclass.result(),
                self.bases.result(),
                cname,
                self.mkw.result(),
                py_mod_name,
                doc_code,
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())

6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103

class ClassCellInjectorNode(ExprNode):
    # Initialize CyFunction.func_classobj
    is_temp = True
    type = py_object_type
    subexprs = []
    is_active = False

    def analyse_expressions(self, env):
        if self.is_active:
            env.use_utility_code(cyfunction_class_cell_utility_code)

    def generate_evaluation_code(self, code):
        if self.is_active:
            self.allocate_temp_result(code)
            code.putln(
                '%s = PyList_New(0); %s' % (
                    self.result(),
                    code.error_goto_if_null(self.result(), self.pos)))
            code.put_gotref(self.result())

    def generate_injection_code(self, code, classobj_cname):
        if self.is_active:
            code.putln('__Pyx_CyFunction_InitClassCell(%s, %s);' % (
                self.result(), classobj_cname))


class ClassCellNode(ExprNode):
    # Class Cell for noargs super()
    subexprs = []
    is_temp = True
    is_generator = False
    type = py_object_type

    def analyse_types(self, env):
        pass

    def generate_result_code(self, code):
        if not self.is_generator:
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                self.result(),
                Naming.self_cname))
        else:
            code.putln('%s =  %s->classobj;' % (
6104
                self.result(), Naming.generator_cname))
6105 6106 6107 6108 6109 6110 6111 6112
        code.putln(
            'if (!%s) { PyErr_SetString(PyExc_SystemError, '
            '"super(): empty __class__ cell"); %s }' % (
                self.result(),
                code.error_goto(self.pos)));
        code.put_incref(self.result(), py_object_type)


Robert Bradshaw's avatar
Robert Bradshaw committed
6113 6114 6115 6116 6117 6118 6119
class BoundMethodNode(ExprNode):
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs an bound method
    #  object from a class and a function.
    #
    #  function      ExprNode   Function object
    #  self_object   ExprNode   self object
6120

Robert Bradshaw's avatar
Robert Bradshaw committed
6121
    subexprs = ['function']
6122

Robert Bradshaw's avatar
Robert Bradshaw committed
6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138
    def analyse_types(self, env):
        self.function.analyse_types(env)
        self.type = py_object_type
        self.is_temp = 1

    gil_message = "Constructing an bound method"

    def generate_result_code(self, code):
        code.putln(
            "%s = PyMethod_New(%s, %s, (PyObject*)%s->ob_type); %s" % (
                self.result(),
                self.function.py_result(),
                self.self_object.py_result(),
                self.self_object.py_result(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6139

6140
class UnboundMethodNode(ExprNode):
William Stein's avatar
William Stein committed
6141 6142 6143 6144 6145
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs an unbound method
    #  object from a class and a function.
    #
    #  function      ExprNode   Function object
6146

6147 6148
    type = py_object_type
    is_temp = 1
6149

William Stein's avatar
William Stein committed
6150
    subexprs = ['function']
6151

William Stein's avatar
William Stein committed
6152 6153
    def analyse_types(self, env):
        self.function.analyse_types(env)
6154

6155 6156 6157
    def may_be_none(self):
        return False

6158 6159
    gil_message = "Constructing an unbound method"

William Stein's avatar
William Stein committed
6160
    def generate_result_code(self, code):
6161
        class_cname = code.pyclass_stack[-1].classobj.result()
William Stein's avatar
William Stein committed
6162
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
6163
            "%s = PyMethod_New(%s, 0, %s); %s" % (
6164
                self.result(),
William Stein's avatar
William Stein committed
6165
                self.function.py_result(),
6166
                class_cname,
6167
                code.error_goto_if_null(self.result(), self.pos)))
6168
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6169

Robert Bradshaw's avatar
Robert Bradshaw committed
6170

6171
class PyCFunctionNode(ExprNode, ModuleNameMixin):
William Stein's avatar
William Stein committed
6172 6173 6174 6175
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs a PyCFunction object
    #  from a PyMethodDef struct.
    #
6176
    #  pymethdef_cname   string             PyMethodDef structure
Robert Bradshaw's avatar
Robert Bradshaw committed
6177
    #  self_object       ExprNode or None
Robert Bradshaw's avatar
Robert Bradshaw committed
6178
    #  binding           bool
6179
    #  def_node          DefNode            the Python function node
6180
    #  module_name       EncodedString      Name of defining module
6181 6182
    #  code_object       CodeObjectNode     the PyCodeObject creator node

6183
    subexprs = ['code_object', 'defaults_tuple']
Stefan Behnel's avatar
Stefan Behnel committed
6184

Robert Bradshaw's avatar
Robert Bradshaw committed
6185
    self_object = None
6186
    code_object = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6187
    binding = False
6188
    def_node = None
6189 6190 6191
    defaults = None
    defaults_struct = None
    defaults_pyobjects = 0
6192
    defaults_tuple = None
6193

6194 6195
    type = py_object_type
    is_temp = 1
6196

6197
    specialized_cpdefs = None
6198
    is_specialization = False
6199

6200 6201 6202 6203 6204 6205 6206 6207
    @classmethod
    def from_defnode(cls, node, binding):
        return cls(node.pos,
                   def_node=node,
                   pymethdef_cname=node.entry.pymethdef_cname,
                   binding=binding or node.specialized_cpdefs,
                   specialized_cpdefs=node.specialized_cpdefs,
                   code_object=CodeObjectNode(node))
6208

6209
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
6210
        if self.binding:
6211
            if self.specialized_cpdefs or self.is_specialization:
6212 6213 6214
                env.use_utility_code(fused_function_utility_code)
            else:
                env.use_utility_code(binding_cfunc_utility_code)
6215
                self.analyse_default_args(env)
6216

6217 6218 6219
        #TODO(craig,haoyu) This should be moved to a better place
        self.set_mod_name(env)

6220 6221 6222 6223 6224 6225
    def analyse_default_args(self, env):
        """
        Handle non-literal function's default arguments.
        """
        nonliteral_objects = []
        nonliteral_other = []
6226
        default_args = []
6227
        for arg in self.def_node.args:
6228 6229 6230 6231 6232 6233 6234
            if arg.default:
                if not arg.default.is_literal:
                    arg.is_dynamic = True
                    if arg.type.is_pyobject:
                        nonliteral_objects.append(arg)
                    else:
                        nonliteral_other.append(arg)
6235 6236
                else:
                    arg.default = DefaultLiteralArgNode(arg.pos, arg.default)
6237
                default_args.append(arg)
6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261
        if nonliteral_objects or nonliteral_objects:
            module_scope = env.global_scope()
            cname = module_scope.next_id(Naming.defaults_struct_prefix)
            scope = Symtab.StructOrUnionScope(cname)
            self.defaults = []
            for arg in nonliteral_objects:
                entry = scope.declare_var(arg.name, arg.type, None,
                                          Naming.arg_prefix + arg.name,
                                          allow_pyobject=True)
                self.defaults.append((arg, entry))
            for arg in nonliteral_other:
                entry = scope.declare_var(arg.name, arg.type, None,
                                          Naming.arg_prefix + arg.name,
                                          allow_pyobject=False)
                self.defaults.append((arg, entry))
            entry = module_scope.declare_struct_or_union(
                None, 'struct', scope, 1, None, cname=cname)
            self.defaults_struct = scope
            self.defaults_pyobjects = len(nonliteral_objects)
            for arg, entry in self.defaults:
                arg.default_value = '%s->%s' % (
                    Naming.dynamic_args_cname, entry.cname)
            self.def_node.defaults_struct = self.defaults_struct.name

6262 6263 6264 6265 6266
        if default_args:
            if self.defaults_struct is None:
                self.defaults_tuple = TupleNode(self.pos, args=[
                    arg.default for arg in default_args])
                self.defaults_tuple.analyse_types(env)
6267 6268 6269 6270 6271 6272 6273 6274
            else:
                defaults_getter = Nodes.DefNode(
                    self.pos, args=[], star_arg=None, starstar_arg=None,
                    body=Nodes.ReturnStatNode(
                        self.pos, return_type=py_object_type,
                        value=DefaultsTupleNode(
                            self.pos, default_args,
                            self.defaults_struct)),
6275
                    decorators=None, name=StringEncoding.EncodedString("__defaults__"))
6276 6277 6278 6279 6280 6281 6282
                defaults_getter.analyse_declarations(env)
                defaults_getter.analyse_expressions(env)
                defaults_getter.body.analyse_expressions(
                    defaults_getter.local_scope)
                defaults_getter.py_wrapper_required = False
                defaults_getter.pymethdef_required = False
                self.def_node.defaults_getter = defaults_getter
6283

6284 6285
    def may_be_none(self):
        return False
6286

6287 6288
    gil_message = "Constructing Python function"

Stefan Behnel's avatar
Stefan Behnel committed
6289
    def self_result_code(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
6290 6291 6292 6293
        if self.self_object is None:
            self_result = "NULL"
        else:
            self_result = self.self_object.py_result()
Stefan Behnel's avatar
Stefan Behnel committed
6294 6295 6296
        return self_result

    def generate_result_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
6297
        if self.binding:
6298 6299 6300
            self.generate_cyfunction_code(code)
        else:
            self.generate_pycfunction_code(code)
6301

6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314
    def generate_pycfunction_code(self, code):
        py_mod_name = self.get_py_mod_name(code)
        code.putln(
            '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % (
                self.result(),
                self.pymethdef_cname,
                self.self_result_code(),
                py_mod_name,
                code.error_goto_if_null(self.result(), self.pos)))

        code.put_gotref(self.py_result())

    def generate_cyfunction_code(self, code):
6315 6316
        def_node = self.def_node

6317 6318
        if self.specialized_cpdefs:
            constructor = "__pyx_FusedFunction_NewEx"
6319
            def_node = self.specialized_cpdefs[0]
6320 6321
        elif self.is_specialization:
            constructor = "__pyx_FusedFunction_NewEx"
Robert Bradshaw's avatar
Robert Bradshaw committed
6322
        else:
6323 6324 6325 6326 6327 6328 6329 6330
            constructor = "__Pyx_CyFunction_NewEx"

        if self.code_object:
            code_object_result = self.code_object.py_result()
        else:
            code_object_result = 'NULL'

        flags = []
6331
        if def_node.is_staticmethod:
6332
            flags.append('__Pyx_CYFUNCTION_STATICMETHOD')
6333
        elif def_node.is_classmethod:
6334
            flags.append('__Pyx_CYFUNCTION_CLASSMETHOD')
6335 6336 6337 6338

        if def_node.local_scope.parent_scope.is_c_class_scope:
            flags.append('__Pyx_CYFUNCTION_CCLASS')

6339 6340 6341 6342
        if flags:
            flags = ' | '.join(flags)
        else:
            flags = '0'
6343

6344
        py_mod_name = self.get_py_mod_name(code)
William Stein's avatar
William Stein committed
6345
        code.putln(
6346
            '%s = %s(&%s, %s, %s, %s, %s); %s' % (
6347
                self.result(),
Robert Bradshaw's avatar
Robert Bradshaw committed
6348
                constructor,
William Stein's avatar
William Stein committed
6349
                self.pymethdef_cname,
6350
                flags,
Stefan Behnel's avatar
Stefan Behnel committed
6351
                self.self_result_code(),
6352
                py_mod_name,
6353
                code_object_result,
6354
                code.error_goto_if_null(self.result(), self.pos)))
6355

6356
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6357

6358
        if def_node.requires_classobj:
6359 6360 6361 6362 6363 6364 6365 6366 6367
            assert code.pyclass_stack, "pyclass_stack is empty"
            class_node = code.pyclass_stack[-1]
            code.put_incref(self.py_result(), py_object_type)
            code.putln(
                'PyList_Append(%s, %s);' % (
                    class_node.class_cell.result(),
                    self.result()))
            code.put_giveref(self.py_result())

6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378
        if self.defaults:
            code.putln(
                'if (!__Pyx_CyFunction_InitDefaults(%s, sizeof(%s), %d)) %s' % (
                    self.result(), self.defaults_struct.name,
                    self.defaults_pyobjects, code.error_goto(self.pos)))
            defaults = '__Pyx_CyFunction_Defaults(%s, %s)' % (
                self.defaults_struct.name, self.result())
            for arg, entry in self.defaults:
                arg.generate_assignment_code(code, target='%s->%s' % (
                    defaults, entry.cname))

6379 6380 6381
        if self.defaults_tuple:
            code.putln('__Pyx_CyFunction_SetDefaultsTuple(%s, %s);' % (
                self.result(), self.defaults_tuple.py_result()))
6382 6383 6384
        if def_node.defaults_getter:
            code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % (
                self.result(), def_node.defaults_getter.entry.pyfunc_cname))
6385

6386

Stefan Behnel's avatar
Stefan Behnel committed
6387 6388 6389
class InnerFunctionNode(PyCFunctionNode):
    # Special PyCFunctionNode that depends on a closure class
    #
Vitja Makarov's avatar
Vitja Makarov committed
6390

Robert Bradshaw's avatar
Robert Bradshaw committed
6391
    binding = True
Vitja Makarov's avatar
Vitja Makarov committed
6392 6393
    needs_self_code = True

Stefan Behnel's avatar
Stefan Behnel committed
6394
    def self_result_code(self):
Vitja Makarov's avatar
Vitja Makarov committed
6395 6396 6397
        if self.needs_self_code:
            return "((PyObject*)%s)" % (Naming.cur_scope_cname)
        return "NULL"
Stefan Behnel's avatar
Stefan Behnel committed
6398

6399 6400 6401 6402
class CodeObjectNode(ExprNode):
    # Create a PyCodeObject for a CyFunction instance.
    #
    # def_node   DefNode    the Python function node
6403
    # varnames   TupleNode  a tuple with all local variable names
6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414

    subexprs = ['varnames']
    is_temp = False

    def __init__(self, def_node):
        ExprNode.__init__(self, def_node.pos, def_node=def_node)
        args = list(def_node.args)
        if def_node.star_arg:
            args.append(def_node.star_arg)
        if def_node.starstar_arg:
            args.append(def_node.starstar_arg)
6415
        local_vars = [ arg for arg in def_node.local_scope.var_entries
6416
                       if arg.name ]
6417 6418
        self.varnames = TupleNode(
            def_node.pos,
6419
            args = [ IdentifierStringNode(arg.pos, value=arg.name)
6420
                     for arg in args + local_vars ],
6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445
            is_temp = 0,
            is_literal = 1)

    def calculate_result_code(self):
        return self.result_code

    def generate_result_code(self, code):
        self.result_code = code.get_py_const(py_object_type, 'codeobj_', cleanup_level=2)

        code = code.get_cached_constants_writer()
        code.mark_pos(self.pos)
        func = self.def_node
        func_name = code.get_py_string_const(
            func.name, identifier=True, is_str=False, unicode_value=func.name)
        # FIXME: better way to get the module file path at module init time? Encoding to use?
        file_path = StringEncoding.BytesLiteral(func.pos[0].get_filenametable_entry().encode('utf8'))
        file_path_const = code.get_py_string_const(file_path, identifier=False, is_str=True)

        code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, 0, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % (
            self.result_code,
            len(func.args),            # argcount
            func.num_kwonly_args,      # kwonlyargcount (Py3 only)
            len(self.varnames.args),   # nlocals
            Naming.empty_bytes,        # code
            Naming.empty_tuple,        # consts
6446 6447
            Naming.empty_tuple,        # names (FIXME)
            self.varnames.result(),    # varnames
6448 6449 6450 6451 6452 6453 6454 6455 6456 6457
            Naming.empty_tuple,        # freevars (FIXME)
            Naming.empty_tuple,        # cellvars (FIXME)
            file_path_const,           # filename
            func_name,                 # name
            self.pos[1],               # firstlineno
            Naming.empty_bytes,        # lnotab
            code.error_goto_if_null(self.result_code, self.pos),
            ))


6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488
class DefaultLiteralArgNode(ExprNode):
    # CyFunction's literal argument default value
    #
    # Evaluate literal only once.

    subexprs = []
    is_literal = True
    is_temp = False

    def __init__(self, pos, arg):
        super(DefaultLiteralArgNode, self).__init__(pos)
        self.arg = arg
        self.type = self.arg.type
        self.evaluated = False

    def analyse_types(self, env):
        pass

    def generate_result_code(self, code):
        pass

    def generate_evaluation_code(self, code):
        if not self.evaluated:
            self.arg.generate_evaluation_code(code)
            self.evaluated = True

    def result(self):
        return self.type.cast_code(self.arg.result())


class DefaultNonLiteralArgNode(ExprNode):
6489 6490 6491 6492 6493
    # CyFunction's non-literal argument default value

    subexprs = []

    def __init__(self, pos, arg, defaults_struct):
6494
        super(DefaultNonLiteralArgNode, self).__init__(pos)
6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517
        self.arg = arg
        self.defaults_struct = defaults_struct

    def analyse_types(self, env):
        self.type = self.arg.type
        self.is_temp = False

    def generate_result_code(self, code):
        pass

    def result(self):
        return '__Pyx_CyFunction_Defaults(%s, %s)->%s' % (
            self.defaults_struct.name, Naming.self_cname,
            self.defaults_struct.lookup(self.arg.name).cname)


class DefaultsTupleNode(TupleNode):
    # CyFunction's __defaults__ tuple

    def __init__(self, pos, defaults, defaults_struct):
        args = []
        for arg in defaults:
            if not arg.default.is_literal:
6518
                arg = DefaultNonLiteralArgNode(pos, arg, defaults_struct)
6519 6520 6521 6522 6523 6524
            else:
                arg = arg.default
            args.append(arg)
        super(DefaultsTupleNode, self).__init__(pos, args=args)


Stefan Behnel's avatar
Stefan Behnel committed
6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539
class LambdaNode(InnerFunctionNode):
    # Lambda expression node (only used as a function reference)
    #
    # args          [CArgDeclNode]         formal arguments
    # star_arg      PyArgDeclNode or None  * argument
    # starstar_arg  PyArgDeclNode or None  ** argument
    # lambda_name   string                 a module-globally unique lambda name
    # result_expr   ExprNode
    # def_node      DefNode                the underlying function 'def' node

    child_attrs = ['def_node']

    name = StringEncoding.EncodedString('<lambda>')

    def analyse_declarations(self, env):
6540 6541
        self.def_node.no_assignment_synthesis = True
        self.def_node.pymethdef_required = True
Stefan Behnel's avatar
Stefan Behnel committed
6542
        self.def_node.analyse_declarations(env)
6543
        self.def_node.is_cyfunction = True
Stefan Behnel's avatar
Stefan Behnel committed
6544 6545 6546
        self.pymethdef_cname = self.def_node.entry.pymethdef_cname
        env.add_lambda_def(self.def_node)

6547 6548 6549 6550 6551 6552 6553 6554
    def analyse_types(self, env):
        self.def_node.analyse_expressions(env)
        super(LambdaNode, self).analyse_types(env)

    def generate_result_code(self, code):
        self.def_node.generate_execution_code(code)
        super(LambdaNode, self).generate_result_code(code)

6555

6556 6557 6558 6559 6560 6561 6562 6563
class GeneratorExpressionNode(LambdaNode):
    # A generator expression, e.g.  (i for i in range(10))
    #
    # Result is a generator.
    #
    # loop      ForStatNode   the for-loop, containing a YieldExprNode
    # def_node  DefNode       the underlying generator 'def' node

6564
    name = StringEncoding.EncodedString('genexpr')
6565 6566 6567
    binding = False

    def analyse_declarations(self, env):
6568 6569 6570
        super(GeneratorExpressionNode, self).analyse_declarations(env)
        # No pymethdef required
        self.def_node.pymethdef_required = False
6571
        self.def_node.py_wrapper_required = False
6572
        self.def_node.is_cyfunction = False
6573 6574
        # Force genexpr signature
        self.def_node.entry.signature = TypeSlots.pyfunction_noargs
6575 6576 6577

    def generate_result_code(self, code):
        code.putln(
6578
            '%s = %s(%s); %s' % (
6579
                self.result(),
6580
                self.def_node.entry.pyfunc_cname,
6581 6582 6583 6584 6585
                self.self_result_code(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())


6586 6587 6588 6589 6590
class YieldExprNode(ExprNode):
    # Yield expression node
    #
    # arg         ExprNode   the value to return from the generator
    # label_name  string     name of the C label used for this yield
6591
    # label_num   integer    yield label number
6592
    # is_yield_from  boolean is a YieldFromExprNode to delegate to another generator
6593 6594 6595

    subexprs = ['arg']
    type = py_object_type
6596
    label_num = 0
6597
    is_yield_from = False
6598 6599

    def analyse_types(self, env):
6600 6601
        if not self.label_num:
            error(self.pos, "'yield' not supported here")
6602 6603 6604 6605
        self.is_temp = 1
        if self.arg is not None:
            self.arg.analyse_types(env)
            if not self.arg.type.is_pyobject:
6606 6607 6608 6609
                self.coerce_yield_argument(env)

    def coerce_yield_argument(self, env):
        self.arg = self.arg.coerce_to_pyobject(env)
6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622

    def generate_evaluation_code(self, code):
        if self.arg:
            self.arg.generate_evaluation_code(code)
            self.arg.make_owned_reference(code)
            code.putln(
                "%s = %s;" % (
                    Naming.retval_cname,
                    self.arg.result_as(py_object_type)))
            self.arg.generate_post_assignment_code(code)
            self.arg.free_temps(code)
        else:
            code.put_init_to_py_none(Naming.retval_cname, py_object_type)
6623 6624 6625 6626 6627 6628 6629 6630 6631 6632
        self.generate_yield_code(code)

    def generate_yield_code(self, code):
        """
        Generate the code to return the argument in 'Naming.retval_cname'
        and to continue at the yield label.
        """
        self.label_name = code.new_label('resume_from_yield')
        code.use_label(self.label_name)

6633
        saved = []
6634
        code.funcstate.closure_temps.reset()
6635
        for cname, type, manage_ref in code.funcstate.temps_in_use():
6636
            save_cname = code.funcstate.closure_temps.allocate_temp(type)
6637 6638 6639 6640
            saved.append((cname, save_cname, type))
            if type.is_pyobject:
                code.put_xgiveref(cname)
            code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname))
6641

6642
        code.put_xgiveref(Naming.retval_cname)
6643
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
6644
        code.putln("/* return from generator, yielding value */")
6645 6646
        code.putln("%s->resume_label = %d;" % (
            Naming.generator_cname, self.label_num))
6647
        code.putln("return %s;" % Naming.retval_cname);
6648

6649
        code.put_label(self.label_name)
6650 6651 6652 6653 6654 6655
        for cname, save_cname, type in saved:
            code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname))
            if type.is_pyobject:
                code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname))
            if type.is_pyobject:
                code.put_xgotref(cname)
6656 6657 6658 6659 6660 6661 6662 6663
        if self.result_is_used:
            self.allocate_temp_result(code)
            code.putln('%s = %s; %s' %
                       (self.result(), Naming.sent_value_cname,
                        code.error_goto_if_null(self.result(), self.pos)))
            code.put_incref(self.result(), py_object_type)
        else:
            code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
6664

Vitja Makarov's avatar
Vitja Makarov committed
6665

6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683
class YieldFromExprNode(YieldExprNode):
    # "yield from GEN" expression
    is_yield_from = True

    def coerce_yield_argument(self, env):
        if not self.arg.type.is_string:
            # FIXME: support C arrays and C++ iterators?
            error(self.pos, "yielding from non-Python object not supported")
        self.arg = self.arg.coerce_to_pyobject(env)

    def generate_evaluation_code(self, code):
        code.globalstate.use_utility_code(UtilityCode.load_cached("YieldFrom", "Generator.c"))

        self.arg.generate_evaluation_code(code)
        code.putln("%s = __Pyx_Generator_Yield_From(%s, %s);" % (
            Naming.retval_cname,
            Naming.generator_cname,
            self.arg.result_as(py_object_type)))
6684
        self.arg.generate_disposal_code(code)
6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705
        self.arg.free_temps(code)
        code.put_xgotref(Naming.retval_cname)

        code.putln("if (likely(%s)) {" % Naming.retval_cname)
        self.generate_yield_code(code)
        code.putln("} else {")
        # either error or sub-generator has normally terminated: return value => node result
        if self.result_is_used:
            # YieldExprNode has allocated the result temp for us
            code.putln("if (__Pyx_PyGen_FetchStopIterationValue(&%s) < 0) %s" % (
                self.result(),
                code.error_goto(self.pos)))
        else:
            code.putln("PyObject* exc_type = PyErr_Occurred();")
            code.putln("if (exc_type) {")
            code.putln("if (!PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) %s" %
                code.error_goto(self.pos))
            code.putln("PyErr_Clear();")
            code.putln("}")
        code.putln("}")

Vitja Makarov's avatar
Vitja Makarov committed
6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720
class GlobalsExprNode(AtomicExprNode):
    type = dict_type
    is_temp = 1

    def analyse_types(self, env):
        env.use_utility_code(Builtin.globals_utility_code)

    gil_message = "Constructing globals dict"

    def generate_result_code(self, code):
        code.putln('%s = __Pyx_Globals(); %s' % (
            self.result(),
            code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.result())

Vitja Makarov's avatar
Vitja Makarov committed
6721

6722
class FuncLocalsExprNode(DictNode):
Vitja Makarov's avatar
Vitja Makarov committed
6723 6724 6725 6726 6727 6728 6729 6730
    def __init__(self, pos, env):
        local_vars = [var.name for var in env.entries.values() if var.name]
        items = [DictItemNode(pos, key=IdentifierStringNode(pos, value=var),
                              value=NameNode(pos, name=var, allow_null=True))
                 for var in local_vars]
        DictNode.__init__(self, pos, key_value_pairs=items,
                          exclude_null_values=True)

6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755

class PyClassLocalsExprNode(AtomicExprNode):
    def __init__(self, pos, pyclass_dict):
        AtomicExprNode.__init__(self, pos)
        self.pyclass_dict = pyclass_dict

    def analyse_types(self, env):
        self.type = self.pyclass_dict.type
        self.is_tmep = 0

    def result(self):
        return self.pyclass_dict.result()

    def generate_result_code(self, code):
        pass


def LocalsExprNode(pos, scope_node, env):
    if env.is_module_scope:
        return GlobalsExprNode(pos)
    if env.is_py_class_scope:
        return PyClassLocalsExprNode(pos, scope_node.dict)
    return FuncLocalsExprNode(pos, env)


William Stein's avatar
William Stein committed
6756 6757 6758 6759 6760 6761
#-------------------------------------------------------------------
#
#  Unary operator nodes
#
#-------------------------------------------------------------------

6762 6763 6764 6765 6766 6767 6768
compile_time_unary_operators = {
    'not': operator.not_,
    '~': operator.inv,
    '-': operator.neg,
    '+': operator.pos,
}

6769
class UnopNode(ExprNode):
William Stein's avatar
William Stein committed
6770 6771 6772 6773 6774 6775 6776 6777 6778 6779
    #  operator     string
    #  operand      ExprNode
    #
    #  Processing during analyse_expressions phase:
    #
    #    analyse_c_operation
    #      Called when the operand is not a pyobject.
    #      - Check operand type and coerce if needed.
    #      - Determine result type and result code fragment.
    #      - Allocate temporary for result if needed.
6780

William Stein's avatar
William Stein committed
6781
    subexprs = ['operand']
Robert Bradshaw's avatar
Robert Bradshaw committed
6782
    infix = True
6783 6784 6785 6786

    def calculate_constant_result(self):
        func = compile_time_unary_operators[self.operator]
        self.constant_result = func(self.operand.constant_result)
6787

6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798
    def compile_time_value(self, denv):
        func = compile_time_unary_operators.get(self.operator)
        if not func:
            error(self.pos,
                "Unary '%s' not supported in compile-time expression"
                    % self.operator)
        operand = self.operand.compile_time_value(denv)
        try:
            return func(operand)
        except Exception, e:
            self.compile_time_value_error(e)
6799

6800
    def infer_type(self, env):
6801 6802 6803 6804 6805
        operand_type = self.operand.infer_type(env)
        if operand_type.is_pyobject:
            return py_object_type
        else:
            return operand_type
6806

William Stein's avatar
William Stein committed
6807 6808 6809 6810 6811 6812
    def analyse_types(self, env):
        self.operand.analyse_types(env)
        if self.is_py_operation():
            self.coerce_operand_to_pyobject(env)
            self.type = py_object_type
            self.is_temp = 1
6813 6814
        elif self.is_cpp_operation():
            self.analyse_cpp_operation(env)
William Stein's avatar
William Stein committed
6815 6816
        else:
            self.analyse_c_operation(env)
6817

William Stein's avatar
William Stein committed
6818
    def check_const(self):
6819
        return self.operand.check_const()
6820

William Stein's avatar
William Stein committed
6821 6822
    def is_py_operation(self):
        return self.operand.type.is_pyobject
6823

6824
    def nogil_check(self, env):
6825
        if self.is_py_operation():
6826
            self.gil_error()
6827

Danilo Freitas's avatar
Danilo Freitas committed
6828
    def is_cpp_operation(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
6829
        type = self.operand.type
Robert Bradshaw's avatar
Robert Bradshaw committed
6830
        return type.is_cpp_class
6831

William Stein's avatar
William Stein committed
6832 6833
    def coerce_operand_to_pyobject(self, env):
        self.operand = self.operand.coerce_to_pyobject(env)
6834

William Stein's avatar
William Stein committed
6835 6836 6837
    def generate_result_code(self, code):
        if self.operand.type.is_pyobject:
            self.generate_py_operation_code(code)
6838

William Stein's avatar
William Stein committed
6839 6840 6841
    def generate_py_operation_code(self, code):
        function = self.py_operation_function()
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
6842
            "%s = %s(%s); %s" % (
6843 6844
                self.result(),
                function,
William Stein's avatar
William Stein committed
6845
                self.operand.py_result(),
6846
                code.error_goto_if_null(self.result(), self.pos)))
6847
        code.put_gotref(self.py_result())
6848

William Stein's avatar
William Stein committed
6849 6850 6851 6852 6853 6854
    def type_error(self):
        if not self.operand.type.is_error:
            error(self.pos, "Invalid operand type for '%s' (%s)" %
                (self.operator, self.operand.type))
        self.type = PyrexTypes.error_type

Danilo Freitas's avatar
Danilo Freitas committed
6855
    def analyse_cpp_operation(self, env):
6856
        type = self.operand.type
Robert Bradshaw's avatar
Robert Bradshaw committed
6857
        if type.is_ptr:
Danilo Freitas's avatar
Danilo Freitas committed
6858
            type = type.base_type
Robert Bradshaw's avatar
Robert Bradshaw committed
6859
        function = type.scope.lookup("operator%s" % self.operator)
Danilo Freitas's avatar
Danilo Freitas committed
6860 6861
        if not function:
            error(self.pos, "'%s' operator not defined for %s"
6862
                % (self.operator, type))
Danilo Freitas's avatar
Danilo Freitas committed
6863 6864
            self.type_error()
            return
6865 6866 6867 6868
        func_type = function.type
        if func_type.is_ptr:
            func_type = func_type.base_type
        self.type = func_type.return_type
Danilo Freitas's avatar
Danilo Freitas committed
6869

William Stein's avatar
William Stein committed
6870

6871
class NotNode(ExprNode):
William Stein's avatar
William Stein committed
6872 6873 6874
    #  'not' operator
    #
    #  operand   ExprNode
6875

6876
    type = PyrexTypes.c_bint_type
6877

6878
    subexprs = ['operand']
6879

6880 6881 6882
    def calculate_constant_result(self):
        self.constant_result = not self.operand.constant_result

6883 6884 6885 6886 6887 6888 6889
    def compile_time_value(self, denv):
        operand = self.operand.compile_time_value(denv)
        try:
            return not operand
        except Exception, e:
            self.compile_time_value_error(e)

6890 6891
    def infer_type(self, env):
        return PyrexTypes.c_bint_type
6892

William Stein's avatar
William Stein committed
6893 6894 6895
    def analyse_types(self, env):
        self.operand.analyse_types(env)
        self.operand = self.operand.coerce_to_boolean(env)
6896

William Stein's avatar
William Stein committed
6897
    def calculate_result_code(self):
6898
        return "(!%s)" % self.operand.result()
6899

William Stein's avatar
William Stein committed
6900 6901 6902 6903 6904 6905
    def generate_result_code(self, code):
        pass


class UnaryPlusNode(UnopNode):
    #  unary '+' operator
6906

William Stein's avatar
William Stein committed
6907
    operator = '+'
6908

William Stein's avatar
William Stein committed
6909
    def analyse_c_operation(self, env):
Lisandro Dalcin's avatar
Lisandro Dalcin committed
6910
        self.type = PyrexTypes.widest_numeric_type(
Robert Bradshaw's avatar
Robert Bradshaw committed
6911
            self.operand.type, PyrexTypes.c_int_type)
6912

William Stein's avatar
William Stein committed
6913 6914
    def py_operation_function(self):
        return "PyNumber_Positive"
6915

William Stein's avatar
William Stein committed
6916
    def calculate_result_code(self):
6917 6918 6919 6920
        if self.is_cpp_operation():
            return "(+%s)" % self.operand.result()
        else:
            return self.operand.result()
William Stein's avatar
William Stein committed
6921 6922 6923 6924


class UnaryMinusNode(UnopNode):
    #  unary '-' operator
6925

William Stein's avatar
William Stein committed
6926
    operator = '-'
6927

William Stein's avatar
William Stein committed
6928 6929
    def analyse_c_operation(self, env):
        if self.operand.type.is_numeric:
6930 6931
            self.type = PyrexTypes.widest_numeric_type(
                self.operand.type, PyrexTypes.c_int_type)
6932 6933
        elif self.operand.type.is_enum:
            self.type = PyrexTypes.c_int_type
William Stein's avatar
William Stein committed
6934 6935
        else:
            self.type_error()
Robert Bradshaw's avatar
Robert Bradshaw committed
6936
        if self.type.is_complex:
6937
            self.infix = False
6938

William Stein's avatar
William Stein committed
6939 6940
    def py_operation_function(self):
        return "PyNumber_Negative"
6941

William Stein's avatar
William Stein committed
6942
    def calculate_result_code(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
6943 6944 6945 6946
        if self.infix:
            return "(-%s)" % self.operand.result()
        else:
            return "%s(%s)" % (self.operand.type.unary_op('-'), self.operand.result())
William Stein's avatar
William Stein committed
6947

6948 6949 6950 6951 6952
    def get_constant_c_result_code(self):
        value = self.operand.get_constant_c_result_code()
        if value:
            return "(-%s)" % (value)

William Stein's avatar
William Stein committed
6953 6954 6955 6956 6957
class TildeNode(UnopNode):
    #  unary '~' operator

    def analyse_c_operation(self, env):
        if self.operand.type.is_int:
6958 6959
            self.type = PyrexTypes.widest_numeric_type(
                self.operand.type, PyrexTypes.c_int_type)
6960 6961
        elif self.operand.type.is_enum:
            self.type = PyrexTypes.c_int_type
William Stein's avatar
William Stein committed
6962 6963 6964 6965 6966
        else:
            self.type_error()

    def py_operation_function(self):
        return "PyNumber_Invert"
6967

William Stein's avatar
William Stein committed
6968
    def calculate_result_code(self):
6969
        return "(~%s)" % self.operand.result()
William Stein's avatar
William Stein committed
6970 6971


6972 6973
class CUnopNode(UnopNode):

Robert Bradshaw's avatar
Robert Bradshaw committed
6974 6975 6976
    def is_py_operation(self):
        return False

6977 6978
class DereferenceNode(CUnopNode):
    #  unary * operator
6979 6980

    operator = '*'
6981

Robert Bradshaw's avatar
Robert Bradshaw committed
6982 6983 6984 6985 6986 6987 6988 6989
    def analyse_c_operation(self, env):
        if self.operand.type.is_ptr:
            self.type = self.operand.type.base_type
        else:
            self.type_error()

    def calculate_result_code(self):
        return "(*%s)" % self.operand.result()
William Stein's avatar
William Stein committed
6990 6991


6992 6993
class DecrementIncrementNode(CUnopNode):
    #  unary ++/-- operator
6994

6995
    def analyse_c_operation(self, env):
6996 6997 6998 6999
        if self.operand.type.is_numeric:
            self.type = PyrexTypes.widest_numeric_type(
                self.operand.type, PyrexTypes.c_int_type)
        elif self.operand.type.is_ptr:
7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013
            self.type = self.operand.type
        else:
            self.type_error()

    def calculate_result_code(self):
        if self.is_prefix:
            return "(%s%s)" % (self.operator, self.operand.result())
        else:
            return "(%s%s)" % (self.operand.result(), self.operator)

def inc_dec_constructor(is_prefix, operator):
    return lambda pos, **kwds: DecrementIncrementNode(pos, is_prefix=is_prefix, operator=operator, **kwds)


7014
class AmpersandNode(ExprNode):
William Stein's avatar
William Stein committed
7015 7016 7017
    #  The C address-of operator.
    #
    #  operand  ExprNode
7018

William Stein's avatar
William Stein committed
7019
    subexprs = ['operand']
7020

7021 7022
    def infer_type(self, env):
        return PyrexTypes.c_ptr_type(self.operand.infer_type(env))
William Stein's avatar
William Stein committed
7023 7024 7025 7026

    def analyse_types(self, env):
        self.operand.analyse_types(env)
        argtype = self.operand.type
7027
        if not (argtype.is_cfunction or self.operand.is_addressable()):
William Stein's avatar
William Stein committed
7028 7029 7030 7031 7032 7033
            self.error("Taking address of non-lvalue")
            return
        if argtype.is_pyobject:
            self.error("Cannot take address of Python variable")
            return
        self.type = PyrexTypes.c_ptr_type(argtype)
7034

William Stein's avatar
William Stein committed
7035
    def check_const(self):
7036
        return self.operand.check_const_addr()
7037

William Stein's avatar
William Stein committed
7038 7039 7040 7041
    def error(self, mess):
        error(self.pos, mess)
        self.type = PyrexTypes.error_type
        self.result_code = "<error>"
7042

William Stein's avatar
William Stein committed
7043
    def calculate_result_code(self):
7044
        return "(&%s)" % self.operand.result()
William Stein's avatar
William Stein committed
7045 7046 7047

    def generate_result_code(self, code):
        pass
7048

William Stein's avatar
William Stein committed
7049 7050 7051 7052 7053 7054 7055 7056

unop_node_classes = {
    "+":  UnaryPlusNode,
    "-":  UnaryMinusNode,
    "~":  TildeNode,
}

def unop_node(pos, operator, operand):
7057
    # Construct unnop node of appropriate class for
William Stein's avatar
William Stein committed
7058
    # given operator.
7059
    if isinstance(operand, IntNode) and operator == '-':
7060
        return IntNode(pos = operand.pos, value = str(-Utils.str_to_number(operand.value)))
Robert Bradshaw's avatar
Robert Bradshaw committed
7061 7062
    elif isinstance(operand, UnopNode) and operand.operator == operator:
        warning(pos, "Python has no increment/decrement operator: %s%sx = %s(%sx) = x" % ((operator,)*4), 5)
7063 7064
    return unop_node_classes[operator](pos,
        operator = operator,
William Stein's avatar
William Stein committed
7065 7066 7067
        operand = operand)


7068
class TypecastNode(ExprNode):
William Stein's avatar
William Stein committed
7069 7070
    #  C type cast
    #
7071
    #  operand      ExprNode
William Stein's avatar
William Stein committed
7072 7073
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
7074 7075 7076
    #
    #  If used from a transform, one can if wanted specify the attribute
    #  "type" directly and leave base_type and declarator to None
7077

William Stein's avatar
William Stein committed
7078
    subexprs = ['operand']
7079
    base_type = declarator = type = None
7080

Robert Bradshaw's avatar
Robert Bradshaw committed
7081
    def type_dependencies(self, env):
7082
        return ()
7083

Robert Bradshaw's avatar
Robert Bradshaw committed
7084
    def infer_type(self, env):
7085 7086 7087 7088
        if self.type is None:
            base_type = self.base_type.analyse(env)
            _, self.type = self.declarator.analyse(base_type, env)
        return self.type
7089

William Stein's avatar
William Stein committed
7090
    def analyse_types(self, env):
7091 7092 7093
        if self.type is None:
            base_type = self.base_type.analyse(env)
            _, self.type = self.declarator.analyse(base_type, env)
7094 7095 7096 7097
        if self.type.is_cfunction:
            error(self.pos,
                "Cannot cast to a function type")
            self.type = PyrexTypes.error_type
William Stein's avatar
William Stein committed
7098 7099 7100
        self.operand.analyse_types(env)
        to_py = self.type.is_pyobject
        from_py = self.operand.type.is_pyobject
7101 7102
        if from_py and not to_py and self.operand.is_ephemeral() and not self.type.is_numeric:
            error(self.pos, "Casting temporary Python object to non-numeric non-Python type")
William Stein's avatar
William Stein committed
7103
        if to_py and not from_py:
7104 7105 7106 7107 7108 7109
            if self.type is bytes_type and self.operand.type.is_int:
                # FIXME: the type cast node isn't needed in this case
                # and can be dropped once analyse_types() can return a
                # different node
                self.operand = CoerceIntToBytesNode(self.operand, env)
            elif self.operand.type.can_coerce_to_pyobject(env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7110
                self.result_ctype = py_object_type
7111
                self.operand = self.operand.coerce_to_pyobject(env)
7112
            else:
7113 7114 7115 7116
                if self.operand.type.is_ptr:
                    if not (self.operand.type.base_type.is_void or self.operand.type.base_type.is_struct):
                        error(self.pos, "Python objects cannot be cast from pointers of primitive types")
                else:
7117
                    # Should this be an error?
7118
                    warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.operand.type, self.type))
7119
                self.operand = self.operand.coerce_to_simple(env)
7120
        elif from_py and not to_py:
7121
            if self.type.create_from_py_utility_code(env):
7122
                self.operand = self.operand.coerce_to(self.type, env)
7123 7124 7125
            elif self.type.is_ptr:
                if not (self.type.base_type.is_void or self.type.base_type.is_struct):
                    error(self.pos, "Python objects cannot be cast to pointers of primitive types")
7126 7127
            else:
                warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.type, self.operand.type))
7128 7129
        elif from_py and to_py:
            if self.typecheck and self.type.is_extension_type:
7130
                self.operand = PyTypeTestNode(self.operand, self.type, env, notnone=True)
7131 7132
        elif self.type.is_complex and self.operand.type.is_complex:
            self.operand = self.operand.coerce_to_simple(env)
7133 7134
        elif self.operand.type.is_fused:
            self.operand = self.operand.coerce_to(self.type, env)
7135
            #self.type = self.operand.type
7136

Stefan Behnel's avatar
Stefan Behnel committed
7137
    def is_simple(self):
7138 7139
        # either temp or a C cast => no side effects other than the operand's
        return self.operand.is_simple()
Stefan Behnel's avatar
Stefan Behnel committed
7140

7141 7142 7143
    def nonlocally_immutable(self):
        return self.operand.nonlocally_immutable()

7144 7145 7146
    def nogil_check(self, env):
        if self.type and self.type.is_pyobject and self.is_temp:
            self.gil_error()
7147

William Stein's avatar
William Stein committed
7148
    def check_const(self):
7149
        return self.operand.check_const()
Stefan Behnel's avatar
Stefan Behnel committed
7150 7151

    def calculate_constant_result(self):
7152 7153 7154
        # we usually do not know the result of a type cast at code
        # generation time
        pass
7155

William Stein's avatar
William Stein committed
7156
    def calculate_result_code(self):
7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167
        if self.type.is_complex:
            operand_result = self.operand.result()
            if self.operand.type.is_complex:
                real_part = self.type.real_type.cast_code("__Pyx_CREAL(%s)" % operand_result)
                imag_part = self.type.real_type.cast_code("__Pyx_CIMAG(%s)" % operand_result)
            else:
                real_part = self.type.real_type.cast_code(operand_result)
                imag_part = "0"
            return "%s(%s, %s)" % (
                    self.type.from_parts,
                    real_part,
7168
                    imag_part)
7169 7170
        else:
            return self.type.cast_code(self.operand.result())
7171

7172 7173 7174 7175
    def get_constant_c_result_code(self):
        operand_result = self.operand.get_constant_c_result_code()
        if operand_result:
            return self.type.cast_code(operand_result)
7176

William Stein's avatar
William Stein committed
7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187
    def result_as(self, type):
        if self.type.is_pyobject and not self.is_temp:
            #  Optimise away some unnecessary casting
            return self.operand.result_as(type)
        else:
            return ExprNode.result_as(self, type)

    def generate_result_code(self, code):
        if self.is_temp:
            code.putln(
                "%s = (PyObject *)%s;" % (
7188 7189 7190
                    self.result(),
                    self.operand.result()))
            code.put_incref(self.result(), self.ctype())
William Stein's avatar
William Stein committed
7191 7192


7193 7194 7195 7196
ERR_START = "Start may not be given"
ERR_NOT_STOP = "Stop must be provided to indicate shape"
ERR_STEPS = ("Strides may only be given to indicate contiguity. "
             "Consider slicing it after conversion")
7197
ERR_NOT_POINTER = "Can only create cython.array from pointer or array"
7198 7199 7200 7201 7202 7203 7204
ERR_BASE_TYPE = "Pointer base type does not match cython.array base type"

class CythonArrayNode(ExprNode):
    """
    Used when a pointer of base_type is cast to a memoryviewslice with that
    base type. i.e.

7205
        <int[:M:1, :N]> p
7206 7207 7208 7209 7210 7211 7212

    creates a fortran-contiguous cython.array.

    We leave the type set to object so coercions to object are more efficient
    and less work. Acquiring a memoryviewslice from this will be just as
    efficient. ExprNode.coerce_to() will do the additional typecheck on
    self.compile_time_type
7213 7214 7215 7216 7217 7218

    This also handles <int[:, :]> my_c_array


    operand             ExprNode                 the thing we're casting
    base_type_node      MemoryViewSliceTypeNode  the cast expression node
7219 7220 7221 7222 7223 7224 7225
    """

    subexprs = ['operand', 'shapes']

    shapes = None
    is_temp = True
    mode = "c"
7226
    array_dtype = None
7227 7228 7229 7230 7231 7232

    shape_type = PyrexTypes.c_py_ssize_t_type

    def analyse_types(self, env):
        import MemoryView

7233 7234 7235 7236 7237 7238 7239 7240 7241
        self.operand.analyse_types(env)
        if self.array_dtype:
            array_dtype = self.array_dtype
        else:
            array_dtype = self.base_type_node.base_type_node.analyse(env)
        axes = self.base_type_node.axes

        MemoryView.validate_memslice_dtype(self.pos, array_dtype)

7242 7243
        self.type = error_type
        self.shapes = []
7244
        ndim = len(axes)
7245

7246 7247 7248
        # Base type of the pointer or C array we are converting
        base_type = self.operand.type

7249 7250 7251
        if not self.operand.type.is_ptr and not self.operand.type.is_array:
            return error(self.operand.pos, ERR_NOT_POINTER)

7252 7253 7254 7255 7256 7257
        # Dimension sizes of C array
        array_dimension_sizes = []
        if base_type.is_array:
            while base_type.is_array:
                array_dimension_sizes.append(base_type.size)
                base_type = base_type.base_type
7258
        elif base_type.is_ptr:
7259
            base_type = base_type.base_type
7260 7261
        else:
            return error()
7262

7263
        if not (base_type.same_as(array_dtype) or base_type.is_void):
7264 7265 7266 7267 7268 7269 7270 7271 7272 7273
            return error(self.operand.pos, ERR_BASE_TYPE)
        elif self.operand.type.is_array and len(array_dimension_sizes) != ndim:
            return error(self.operand.pos,
                         "Expected %d dimensions, array has %d dimensions" %
                                            (ndim, len(array_dimension_sizes)))

        # Verify the start, stop and step values
        # In case of a C array, use the size of C array in each dimension to
        # get an automatic cast
        for axis_no, axis in enumerate(axes):
7274 7275 7276 7277
            if not axis.start.is_none:
                return error(axis.start.pos, ERR_START)

            if axis.stop.is_none:
7278 7279 7280 7281 7282 7283 7284
                if array_dimension_sizes:
                    dimsize = array_dimension_sizes[axis_no]
                    axis.stop = IntNode(self.pos, value=dimsize,
                                        constant_result=dimsize,
                                        type=PyrexTypes.c_int_type)
                else:
                    return error(axis.pos, ERR_NOT_STOP)
7285 7286 7287 7288 7289 7290 7291 7292

            axis.stop.analyse_types(env)
            shape = axis.stop.coerce_to(self.shape_type, env)
            if not shape.is_literal:
                shape.coerce_to_temp(env)

            self.shapes.append(shape)

7293
            first_or_last = axis_no in (0, ndim - 1)
7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311
            if not axis.step.is_none and first_or_last:
                axis.step.analyse_types(env)
                if (not axis.step.type.is_int and axis.step.is_literal and not
                        axis.step.type.is_error):
                    return error(axis.step.pos, "Expected an integer literal")

                if axis.step.compile_time_value(env) != 1:
                    return error(axis.step.pos, ERR_STEPS)

                if axis_no == 0:
                    self.mode = "fortran"

            elif axis.step and not first_or_last:
                return error(axis.step.pos, ERR_STEPS)

        if not self.operand.is_name:
            self.operand = self.operand.coerce_to_temp(env)

7312
        axes = [('direct', 'follow')] * len(axes)
7313 7314 7315 7316 7317 7318
        if self.mode == "fortran":
            axes[0] = ('direct', 'contig')
        else:
            axes[-1] = ('direct', 'contig')

        self.coercion_type = PyrexTypes.MemoryViewSliceType(array_dtype, axes)
7319
        self.type = self.get_cython_array_type(env)
7320
        MemoryView.use_cython_array_utility_code(env)
7321 7322 7323 7324 7325 7326 7327 7328
        env.use_utility_code(MemoryView.typeinfo_to_format_code)

    def allocate_temp_result(self, code):
        if self.temp_code:
            raise RuntimeError("temp allocated mulitple times")

        self.temp_code = code.funcstate.allocate_temp(self.type, True)

7329 7330 7331 7332
    def infer_type(self, env):
        return self.get_cython_array_type(env)

    def get_cython_array_type(self, env):
7333
        return env.global_scope().context.cython_scope.viewscope.lookup("array").type
7334

7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347
    def generate_result_code(self, code):
        import Buffer

        shapes = [self.shape_type.cast_code(shape.result())
                      for shape in self.shapes]
        dtype = self.coercion_type.dtype

        shapes_temp = code.funcstate.allocate_temp(py_object_type, True)
        format_temp = code.funcstate.allocate_temp(py_object_type, True)

        itemsize = "sizeof(%s)" % dtype.declaration_code("")
        type_info = Buffer.get_type_information_cname(code, dtype)

7348 7349 7350 7351 7352 7353
        if self.operand.type.is_ptr:
            code.putln("if (!%s) {" % self.operand.result())
            code.putln(    'PyErr_SetString(PyExc_ValueError,'
                                '"Cannot create cython.array from NULL pointer");')
            code.putln(code.error_goto(self.operand.pos))
            code.putln("}")
7354 7355 7356

        code.putln("%s = __pyx_format_from_typeinfo(&%s);" %
                                                (format_temp, type_info))
7357 7358 7359 7360
        buildvalue_fmt = " __PYX_BUILD_PY_SSIZE_T " * len(shapes)
        code.putln('%s = Py_BuildValue("(" %s ")", %s);' % (shapes_temp,
                                                            buildvalue_fmt,
                                                            ", ".join(shapes)))
7361

7362 7363 7364
        err = "!%s || !%s || !PyBytes_AsString(%s)" % (format_temp,
                                                       shapes_temp,
                                                       format_temp)
7365 7366 7367 7368 7369 7370 7371 7372
        code.putln(code.error_goto_if(err, self.pos))
        code.put_gotref(format_temp)
        code.put_gotref(shapes_temp)

        tup = (self.result(), shapes_temp, itemsize, format_temp,
               self.mode, self.operand.result())
        code.putln('%s = __pyx_array_new('
                            '%s, %s, PyBytes_AS_STRING(%s), '
7373
                            '(char *) "%s", (char *) %s);' % tup)
7374 7375 7376 7377 7378 7379 7380 7381 7382 7383
        code.putln(code.error_goto_if_null(self.result(), self.pos))
        code.put_gotref(self.result())

        def dispose(temp):
            code.put_decref_clear(temp, py_object_type)
            code.funcstate.release_temp(temp)

        dispose(shapes_temp)
        dispose(format_temp)

7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406
    @classmethod
    def from_carray(cls, src_node, env):
        """
        Given a C array type, return a CythonArrayNode
        """
        pos = src_node.pos
        base_type = src_node.type

        none_node = NoneNode(pos)
        axes = []

        while base_type.is_array:
            axes.append(SliceNode(pos, start=none_node, stop=none_node,
                                       step=none_node))
            base_type = base_type.base_type
        axes[-1].step = IntNode(pos, value="1", is_c_literal=True)

        memslicenode = Nodes.MemoryViewSliceTypeNode(pos, axes=axes,
                                                     base_type_node=base_type)
        result = CythonArrayNode(pos, base_type_node=memslicenode,
                                 operand=src_node, array_dtype=base_type)
        result.analyse_types(env)
        return result
7407

7408
class SizeofNode(ExprNode):
William Stein's avatar
William Stein committed
7409
    #  Abstract base class for sizeof(x) expression nodes.
7410

7411
    type = PyrexTypes.c_size_t_type
William Stein's avatar
William Stein committed
7412 7413

    def check_const(self):
7414
        return True
William Stein's avatar
William Stein committed
7415 7416 7417 7418 7419 7420 7421 7422 7423 7424

    def generate_result_code(self, code):
        pass


class SizeofTypeNode(SizeofNode):
    #  C sizeof function applied to a type
    #
    #  base_type   CBaseTypeNode
    #  declarator  CDeclaratorNode
7425

William Stein's avatar
William Stein committed
7426
    subexprs = []
7427
    arg_type = None
7428

William Stein's avatar
William Stein committed
7429
    def analyse_types(self, env):
7430 7431
        # we may have incorrectly interpreted a dotted name as a type rather than an attribute
        # this could be better handled by more uniformly treating types as runtime-available objects
7432
        if 0 and self.base_type.module_path:
7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443
            path = self.base_type.module_path
            obj = env.lookup(path[0])
            if obj.as_module is None:
                operand = NameNode(pos=self.pos, name=path[0])
                for attr in path[1:]:
                    operand = AttributeNode(pos=self.pos, obj=operand, attribute=attr)
                operand = AttributeNode(pos=self.pos, obj=operand, attribute=self.base_type.name)
                self.operand = operand
                self.__class__ = SizeofVarNode
                self.analyse_types(env)
                return
7444 7445 7446 7447
        if self.arg_type is None:
            base_type = self.base_type.analyse(env)
            _, arg_type = self.declarator.analyse(base_type, env)
            self.arg_type = arg_type
7448
        self.check_type()
7449

7450 7451
    def check_type(self):
        arg_type = self.arg_type
7452
        if arg_type.is_pyobject and not arg_type.is_extension_type:
William Stein's avatar
William Stein committed
7453 7454 7455 7456 7457
            error(self.pos, "Cannot take sizeof Python object")
        elif arg_type.is_void:
            error(self.pos, "Cannot take sizeof void")
        elif not arg_type.is_complete():
            error(self.pos, "Cannot take sizeof incomplete type '%s'" % arg_type)
7458

William Stein's avatar
William Stein committed
7459
    def calculate_result_code(self):
7460 7461 7462 7463 7464 7465
        if self.arg_type.is_extension_type:
            # the size of the pointer is boring
            # we want the size of the actual struct
            arg_code = self.arg_type.declaration_code("", deref=1)
        else:
            arg_code = self.arg_type.declaration_code("")
William Stein's avatar
William Stein committed
7466
        return "(sizeof(%s))" % arg_code
7467

William Stein's avatar
William Stein committed
7468 7469 7470 7471 7472

class SizeofVarNode(SizeofNode):
    #  C sizeof function applied to a variable
    #
    #  operand   ExprNode
7473

William Stein's avatar
William Stein committed
7474
    subexprs = ['operand']
7475

William Stein's avatar
William Stein committed
7476
    def analyse_types(self, env):
7477 7478 7479 7480 7481
        # We may actually be looking at a type rather than a variable...
        # If we are, traditional analysis would fail...
        operand_as_type = self.operand.analyse_as_type(env)
        if operand_as_type:
            self.arg_type = operand_as_type
Mark Florisson's avatar
Mark Florisson committed
7482 7483
            if self.arg_type.is_fused:
                self.arg_type = self.arg_type.specialize(env.fused_to_specific)
7484 7485 7486 7487
            self.__class__ = SizeofTypeNode
            self.check_type()
        else:
            self.operand.analyse_types(env)
7488

William Stein's avatar
William Stein committed
7489
    def calculate_result_code(self):
7490
        return "(sizeof(%s))" % self.operand.result()
7491

William Stein's avatar
William Stein committed
7492 7493 7494
    def generate_result_code(self, code):
        pass

Robert Bradshaw's avatar
Robert Bradshaw committed
7495
class TypeofNode(ExprNode):
7496 7497 7498
    #  Compile-time type of an expression, as a string.
    #
    #  operand   ExprNode
Robert Bradshaw's avatar
Robert Bradshaw committed
7499
    #  literal   StringNode # internal
7500

Robert Bradshaw's avatar
Robert Bradshaw committed
7501 7502
    literal = None
    type = py_object_type
7503

Stefan Behnel's avatar
Stefan Behnel committed
7504
    subexprs = ['literal'] # 'operand' will be ignored after type analysis!
7505

7506 7507
    def analyse_types(self, env):
        self.operand.analyse_types(env)
7508
        value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name())
7509
        self.literal = StringNode(self.pos, value=value)
Robert Bradshaw's avatar
Robert Bradshaw committed
7510 7511
        self.literal.analyse_types(env)
        self.literal = self.literal.coerce_to_pyobject(env)
7512 7513 7514 7515

    def may_be_none(self):
        return False

7516
    def generate_evaluation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
7517
        self.literal.generate_evaluation_code(code)
7518

Robert Bradshaw's avatar
Robert Bradshaw committed
7519 7520
    def calculate_result_code(self):
        return self.literal.calculate_result_code()
William Stein's avatar
William Stein committed
7521 7522 7523 7524 7525 7526 7527

#-------------------------------------------------------------------
#
#  Binary operator nodes
#
#-------------------------------------------------------------------

Stefan Behnel's avatar
Stefan Behnel committed
7528 7529 7530
def _not_in(x, seq):
    return x not in seq

7531 7532 7533
compile_time_binary_operators = {
    '<': operator.lt,
    '<=': operator.le,
7534
    '==': operator.eq,
7535 7536 7537 7538 7539 7540 7541
    '!=': operator.ne,
    '>=': operator.ge,
    '>': operator.gt,
    'is': operator.is_,
    'is_not': operator.is_not,
    '+': operator.add,
    '&': operator.and_,
7542
    '/': operator.truediv,
7543 7544 7545 7546 7547 7548 7549 7550 7551
    '//': operator.floordiv,
    '<<': operator.lshift,
    '%': operator.mod,
    '*': operator.mul,
    '|': operator.or_,
    '**': operator.pow,
    '>>': operator.rshift,
    '-': operator.sub,
    '^': operator.xor,
Stefan Behnel's avatar
Stefan Behnel committed
7552 7553
    'in': operator.contains,
    'not_in': _not_in,
7554 7555 7556 7557 7558 7559 7560
}

def get_compile_time_binop(node):
    func = compile_time_binary_operators.get(node.operator)
    if not func:
        error(node.pos,
            "Binary '%s' not supported in compile-time expression"
7561
                % node.operator)
7562 7563
    return func

7564
class BinopNode(ExprNode):
William Stein's avatar
William Stein committed
7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575
    #  operator     string
    #  operand1     ExprNode
    #  operand2     ExprNode
    #
    #  Processing during analyse_expressions phase:
    #
    #    analyse_c_operation
    #      Called when neither operand is a pyobject.
    #      - Check operand types and coerce if needed.
    #      - Determine result type and result code fragment.
    #      - Allocate temporary for result if needed.
7576

William Stein's avatar
William Stein committed
7577
    subexprs = ['operand1', 'operand2']
7578
    inplace = False
7579 7580 7581 7582 7583 7584 7585

    def calculate_constant_result(self):
        func = compile_time_binary_operators[self.operator]
        self.constant_result = func(
            self.operand1.constant_result,
            self.operand2.constant_result)

7586 7587 7588 7589 7590 7591 7592 7593
    def compile_time_value(self, denv):
        func = get_compile_time_binop(self)
        operand1 = self.operand1.compile_time_value(denv)
        operand2 = self.operand2.compile_time_value(denv)
        try:
            return func(operand1, operand2)
        except Exception, e:
            self.compile_time_value_error(e)
7594

7595 7596
    def infer_type(self, env):
        return self.result_type(self.operand1.infer_type(env),
Robert Bradshaw's avatar
Robert Bradshaw committed
7597
                                self.operand2.infer_type(env))
7598

William Stein's avatar
William Stein committed
7599 7600 7601
    def analyse_types(self, env):
        self.operand1.analyse_types(env)
        self.operand2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
7602
        self.analyse_operation(env)
7603

Robert Bradshaw's avatar
Robert Bradshaw committed
7604
    def analyse_operation(self, env):
William Stein's avatar
William Stein committed
7605 7606
        if self.is_py_operation():
            self.coerce_operands_to_pyobjects(env)
7607 7608 7609
            self.type = self.result_type(self.operand1.type,
                                         self.operand2.type)
            assert self.type.is_pyobject
William Stein's avatar
William Stein committed
7610
            self.is_temp = 1
DaniloFreitas's avatar
DaniloFreitas committed
7611 7612
        elif self.is_cpp_operation():
            self.analyse_cpp_operation(env)
William Stein's avatar
William Stein committed
7613 7614
        else:
            self.analyse_c_operation(env)
7615

William Stein's avatar
William Stein committed
7616
    def is_py_operation(self):
7617
        return self.is_py_operation_types(self.operand1.type, self.operand2.type)
7618

7619 7620 7621
    def is_py_operation_types(self, type1, type2):
        return type1.is_pyobject or type2.is_pyobject

DaniloFreitas's avatar
DaniloFreitas committed
7622
    def is_cpp_operation(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7623 7624
        return (self.operand1.type.is_cpp_class
            or self.operand2.type.is_cpp_class)
7625

7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641
    def analyse_cpp_operation(self, env):
        type1 = self.operand1.type
        type2 = self.operand2.type
        entry = env.lookup_operator(self.operator, [self.operand1, self.operand2])
        if not entry:
            self.type_error()
            return
        func_type = entry.type
        if func_type.is_ptr:
            func_type = func_type.base_type
        if len(func_type.args) == 1:
            self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env)
        else:
            self.operand1 = self.operand1.coerce_to(func_type.args[0].type, env)
            self.operand2 = self.operand2.coerce_to(func_type.args[1].type, env)
        self.type = func_type.return_type
7642

7643 7644
    def result_type(self, type1, type2):
        if self.is_py_operation_types(type1, type2):
7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662
            if type2.is_string:
                type2 = Builtin.bytes_type
            if type1.is_string:
                type1 = Builtin.bytes_type
            elif self.operator == '%' \
                     and type1 in (Builtin.str_type, Builtin.unicode_type):
                # note that  b'%s' % b'abc'  doesn't work in Py3
                return type1
            if type1.is_builtin_type:
                if type1 is type2:
                    if self.operator in '**%+|&^':
                        # FIXME: at least these operators should be safe - others?
                        return type1
                elif self.operator == '*':
                    if type1 in (Builtin.bytes_type, Builtin.str_type, Builtin.unicode_type):
                        return type1
                    # multiplication of containers/numbers with an
                    # integer value always (?) returns the same type
7663
                    if type2.is_int:
7664
                        return type1
7665 7666 7667 7668
            elif type2.is_builtin_type and type1.is_int and self.operator == '*':
                # multiplication of containers/numbers with an
                # integer value always (?) returns the same type
                return type2
7669 7670 7671
            return py_object_type
        else:
            return self.compute_c_result_type(type1, type2)
7672

7673
    def nogil_check(self, env):
7674
        if self.is_py_operation():
7675
            self.gil_error()
7676

William Stein's avatar
William Stein committed
7677 7678 7679
    def coerce_operands_to_pyobjects(self, env):
        self.operand1 = self.operand1.coerce_to_pyobject(env)
        self.operand2 = self.operand2.coerce_to_pyobject(env)
7680

William Stein's avatar
William Stein committed
7681
    def check_const(self):
7682
        return self.operand1.check_const() and self.operand2.check_const()
7683

William Stein's avatar
William Stein committed
7684 7685 7686 7687
    def generate_result_code(self, code):
        #print "BinopNode.generate_result_code:", self.operand1, self.operand2 ###
        if self.operand1.type.is_pyobject:
            function = self.py_operation_function()
7688
            if self.operator == '**':
William Stein's avatar
William Stein committed
7689 7690 7691 7692
                extra_args = ", Py_None"
            else:
                extra_args = ""
            code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
7693
                "%s = %s(%s, %s%s); %s" % (
7694 7695
                    self.result(),
                    function,
William Stein's avatar
William Stein committed
7696 7697 7698
                    self.operand1.py_result(),
                    self.operand2.py_result(),
                    extra_args,
7699
                    code.error_goto_if_null(self.result(), self.pos)))
7700
            code.put_gotref(self.py_result())
7701

William Stein's avatar
William Stein committed
7702 7703 7704 7705
    def type_error(self):
        if not (self.operand1.type.is_error
                or self.operand2.type.is_error):
            error(self.pos, "Invalid operand types for '%s' (%s; %s)" %
7706
                (self.operator, self.operand1.type,
William Stein's avatar
William Stein committed
7707 7708 7709 7710
                    self.operand2.type))
        self.type = PyrexTypes.error_type


Robert Bradshaw's avatar
Robert Bradshaw committed
7711
class CBinopNode(BinopNode):
7712

Robert Bradshaw's avatar
Robert Bradshaw committed
7713 7714 7715 7716
    def analyse_types(self, env):
        BinopNode.analyse_types(self, env)
        if self.is_py_operation():
            self.type = PyrexTypes.error_type
7717

Stefan Behnel's avatar
Stefan Behnel committed
7718
    def py_operation_function(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7719
        return ""
7720

Robert Bradshaw's avatar
Robert Bradshaw committed
7721 7722
    def calculate_result_code(self):
        return "(%s %s %s)" % (
7723 7724
            self.operand1.result(),
            self.operator,
Robert Bradshaw's avatar
Robert Bradshaw committed
7725 7726 7727 7728 7729 7730 7731 7732
            self.operand2.result())


def c_binop_constructor(operator):
    def make_binop_node(pos, **operands):
        return CBinopNode(pos, operator=operator, **operands)
    return make_binop_node

William Stein's avatar
William Stein committed
7733 7734
class NumBinopNode(BinopNode):
    #  Binary operation taking numeric arguments.
7735

Robert Bradshaw's avatar
Robert Bradshaw committed
7736
    infix = True
7737

William Stein's avatar
William Stein committed
7738 7739 7740 7741 7742 7743
    def analyse_c_operation(self, env):
        type1 = self.operand1.type
        type2 = self.operand2.type
        self.type = self.compute_c_result_type(type1, type2)
        if not self.type:
            self.type_error()
7744
            return
7745
        if self.type.is_complex:
Robert Bradshaw's avatar
Robert Bradshaw committed
7746
            self.infix = False
7747
        if not self.infix or (type1.is_numeric and type2.is_numeric):
7748 7749
            self.operand1 = self.operand1.coerce_to(self.type, env)
            self.operand2 = self.operand2.coerce_to(self.type, env)
7750

William Stein's avatar
William Stein committed
7751 7752
    def compute_c_result_type(self, type1, type2):
        if self.c_types_okay(type1, type2):
7753 7754 7755 7756 7757
            widest_type = PyrexTypes.widest_numeric_type(type1, type2)
            if widest_type is PyrexTypes.c_bint_type:
                if self.operator not in '|^&':
                    # False + False == 0 # not False!
                    widest_type = PyrexTypes.c_int_type
7758 7759 7760
            else:
                widest_type = PyrexTypes.widest_numeric_type(
                    widest_type, PyrexTypes.c_int_type)
7761
            return widest_type
William Stein's avatar
William Stein committed
7762 7763
        else:
            return None
7764

7765 7766 7767 7768 7769 7770 7771 7772 7773 7774
    def may_be_none(self):
        type1 = self.operand1.type
        type2 = self.operand2.type
        if type1 and type1.is_builtin_type and type2 and type2.is_builtin_type:
            # XXX: I can't think of any case where a binary operation
            # on builtin types evaluates to None - add a special case
            # here if there is one.
            return False
        return super(NumBinopNode, self).may_be_none()

7775 7776 7777 7778 7779 7780 7781
    def get_constant_c_result_code(self):
        value1 = self.operand1.get_constant_c_result_code()
        value2 = self.operand2.get_constant_c_result_code()
        if value1 and value2:
            return "(%s %s %s)" % (value1, self.operator, value2)
        else:
            return None
7782

William Stein's avatar
William Stein committed
7783
    def c_types_okay(self, type1, type2):
7784 7785 7786
        #print "NumBinopNode.c_types_okay:", type1, type2 ###
        return (type1.is_numeric  or type1.is_enum) \
            and (type2.is_numeric  or type2.is_enum)
William Stein's avatar
William Stein committed
7787 7788

    def calculate_result_code(self):
7789 7790
        if self.infix:
            return "(%s %s %s)" % (
7791 7792
                self.operand1.result(),
                self.operator,
7793 7794
                self.operand2.result())
        else:
7795 7796 7797
            func = self.type.binary_op(self.operator)
            if func is None:
                error(self.pos, "binary operator %s not supported for %s" % (self.operator, self.type))
7798
            return "%s(%s, %s)" % (
7799
                func,
7800 7801
                self.operand1.result(),
                self.operand2.result())
7802

7803
    def is_py_operation_types(self, type1, type2):
Stefan Behnel's avatar
Stefan Behnel committed
7804 7805
        return (type1.is_unicode_char or
                type2.is_unicode_char or
7806
                BinopNode.is_py_operation_types(self, type1, type2))
7807

William Stein's avatar
William Stein committed
7808
    def py_operation_function(self):
7809 7810 7811 7812
        fuction = self.py_functions[self.operator]
        if self.inplace:
            fuction = fuction.replace('PyNumber_', 'PyNumber_InPlace')
        return fuction
William Stein's avatar
William Stein committed
7813 7814

    py_functions = {
Robert Bradshaw's avatar
Robert Bradshaw committed
7815 7816 7817
        "|":        "PyNumber_Or",
        "^":        "PyNumber_Xor",
        "&":        "PyNumber_And",
7818 7819
        "<<":       "PyNumber_Lshift",
        ">>":       "PyNumber_Rshift",
Robert Bradshaw's avatar
Robert Bradshaw committed
7820 7821 7822 7823
        "+":        "PyNumber_Add",
        "-":        "PyNumber_Subtract",
        "*":        "PyNumber_Multiply",
        "/":        "__Pyx_PyNumber_Divide",
7824
        "//":       "PyNumber_FloorDivide",
Robert Bradshaw's avatar
Robert Bradshaw committed
7825
        "%":        "PyNumber_Remainder",
7826
        "**":       "PyNumber_Power"
William Stein's avatar
William Stein committed
7827 7828 7829 7830
    }

class IntBinopNode(NumBinopNode):
    #  Binary operation taking integer arguments.
7831

William Stein's avatar
William Stein committed
7832
    def c_types_okay(self, type1, type2):
7833 7834 7835
        #print "IntBinopNode.c_types_okay:", type1, type2 ###
        return (type1.is_int or type1.is_enum) \
            and (type2.is_int or type2.is_enum)
William Stein's avatar
William Stein committed
7836

7837

William Stein's avatar
William Stein committed
7838 7839
class AddNode(NumBinopNode):
    #  '+' operator.
7840

7841 7842 7843
    def is_py_operation_types(self, type1, type2):
        if type1.is_string and type2.is_string:
            return 1
William Stein's avatar
William Stein committed
7844
        else:
7845
            return NumBinopNode.is_py_operation_types(self, type1, type2)
William Stein's avatar
William Stein committed
7846 7847

    def compute_c_result_type(self, type1, type2):
7848 7849
        #print "AddNode.compute_c_result_type:", type1, self.operator, type2 ###
        if (type1.is_ptr or type1.is_array) and (type2.is_int or type2.is_enum):
William Stein's avatar
William Stein committed
7850
            return type1
7851
        elif (type2.is_ptr or type2.is_array) and (type1.is_int or type1.is_enum):
William Stein's avatar
William Stein committed
7852 7853 7854 7855 7856 7857 7858 7859
            return type2
        else:
            return NumBinopNode.compute_c_result_type(
                self, type1, type2)


class SubNode(NumBinopNode):
    #  '-' operator.
7860

William Stein's avatar
William Stein committed
7861
    def compute_c_result_type(self, type1, type2):
7862
        if (type1.is_ptr or type1.is_array) and (type2.is_int or type2.is_enum):
William Stein's avatar
William Stein committed
7863
            return type1
7864
        elif (type1.is_ptr or type1.is_array) and (type2.is_ptr or type2.is_array):
William Stein's avatar
William Stein committed
7865 7866 7867 7868 7869 7870 7871 7872
            return PyrexTypes.c_int_type
        else:
            return NumBinopNode.compute_c_result_type(
                self, type1, type2)


class MulNode(NumBinopNode):
    #  '*' operator.
7873

7874
    def is_py_operation_types(self, type1, type2):
William Stein's avatar
William Stein committed
7875 7876 7877 7878
        if (type1.is_string and type2.is_int) \
            or (type2.is_string and type1.is_int):
                return 1
        else:
7879
            return NumBinopNode.is_py_operation_types(self, type1, type2)
William Stein's avatar
William Stein committed
7880 7881


7882 7883
class DivNode(NumBinopNode):
    #  '/' or '//' operator.
7884

7885
    cdivision = None
7886 7887
    truedivision = None   # == "unknown" if operator == '/'
    ctruedivision = False
Robert Bradshaw's avatar
Robert Bradshaw committed
7888
    cdivision_warnings = False
7889
    zerodivision_check = None
7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911

    def find_compile_time_binary_operator(self, op1, op2):
        func = compile_time_binary_operators[self.operator]
        if self.operator == '/' and self.truedivision is None:
            # => true div for floats, floor div for integers
            if isinstance(op1, (int,long)) and isinstance(op2, (int,long)):
                func = compile_time_binary_operators['//']
        return func

    def calculate_constant_result(self):
        op1 = self.operand1.constant_result
        op2 = self.operand2.constant_result
        func = self.find_compile_time_binary_operator(op1, op2)
        self.constant_result = func(
            self.operand1.constant_result,
            self.operand2.constant_result)

    def compile_time_value(self, denv):
        operand1 = self.operand1.compile_time_value(denv)
        operand2 = self.operand2.compile_time_value(denv)
        try:
            func = self.find_compile_time_binary_operator(
Robert Bradshaw's avatar
Robert Bradshaw committed
7912
                operand1, operand2)
7913 7914 7915 7916
            return func(operand1, operand2)
        except Exception, e:
            self.compile_time_value_error(e)

Robert Bradshaw's avatar
Robert Bradshaw committed
7917
    def analyse_operation(self, env):
7918 7919 7920 7921
        if self.cdivision or env.directives['cdivision']:
            self.ctruedivision = False
        else:
            self.ctruedivision = self.truedivision
Robert Bradshaw's avatar
Robert Bradshaw committed
7922
        NumBinopNode.analyse_operation(self, env)
7923 7924
        if self.is_cpp_operation():
            self.cdivision = True
7925
        if not self.type.is_pyobject:
7926 7927
            self.zerodivision_check = (
                self.cdivision is None and not env.directives['cdivision']
7928
                and (not self.operand2.has_constant_result() or
7929
                     self.operand2.constant_result == 0))
7930 7931 7932 7933
            if self.zerodivision_check or env.directives['cdivision_warnings']:
                # Need to check ahead of time to warn or raise zero division error
                self.operand1 = self.operand1.coerce_to_simple(env)
                self.operand2 = self.operand2.coerce_to_simple(env)
7934 7935
                if env.nogil:
                    error(self.pos, "Pythonic division not allowed without gil, consider using cython.cdivision(True)")
7936 7937 7938 7939 7940 7941 7942 7943 7944

    def compute_c_result_type(self, type1, type2):
        if self.operator == '/' and self.ctruedivision:
            if not type1.is_float and not type2.is_float:
                widest_type = PyrexTypes.widest_numeric_type(type1, PyrexTypes.c_double_type)
                widest_type = PyrexTypes.widest_numeric_type(type2, widest_type)
                return widest_type
        return NumBinopNode.compute_c_result_type(self, type1, type2)

7945 7946 7947 7948 7949
    def zero_division_message(self):
        if self.type.is_int:
            return "integer division or modulo by zero"
        else:
            return "float division"
Robert Bradshaw's avatar
Robert Bradshaw committed
7950

7951
    def generate_evaluation_code(self, code):
7952
        if not self.type.is_pyobject and not self.type.is_complex:
7953
            if self.cdivision is None:
7954
                self.cdivision = (code.globalstate.directives['cdivision']
7955 7956 7957
                                    or not self.type.signed
                                    or self.type.is_float)
            if not self.cdivision:
7958
                code.globalstate.use_utility_code(div_int_utility_code.specialize(self.type))
7959
        NumBinopNode.generate_evaluation_code(self, code)
7960
        self.generate_div_warning_code(code)
7961

7962
    def generate_div_warning_code(self, code):
7963 7964
        if not self.type.is_pyobject:
            if self.zerodivision_check:
7965 7966 7967 7968 7969
                if not self.infix:
                    zero_test = "%s(%s)" % (self.type.unary_op('zero'), self.operand2.result())
                else:
                    zero_test = "%s == 0" % self.operand2.result()
                code.putln("if (unlikely(%s)) {" % zero_test)
7970 7971 7972
                code.putln('PyErr_Format(PyExc_ZeroDivisionError, "%s");' % self.zero_division_message())
                code.putln(code.error_goto(self.pos))
                code.putln("}")
7973 7974 7975
                if self.type.is_int and self.type.signed and self.operator != '%':
                    code.globalstate.use_utility_code(division_overflow_test_code)
                    code.putln("else if (sizeof(%s) == sizeof(long) && unlikely(%s == -1) && unlikely(UNARY_NEG_WOULD_OVERFLOW(%s))) {" % (
7976
                                    self.type.declaration_code(''),
7977 7978 7979 7980 7981
                                    self.operand2.result(),
                                    self.operand1.result()))
                    code.putln('PyErr_Format(PyExc_OverflowError, "value too large to perform division");')
                    code.putln(code.error_goto(self.pos))
                    code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
7982
            if code.globalstate.directives['cdivision_warnings'] and self.operator != '/':
7983 7984 7985 7986 7987
                code.globalstate.use_utility_code(cdivision_warning_utility_code)
                code.putln("if ((%s < 0) ^ (%s < 0)) {" % (
                                self.operand1.result(),
                                self.operand2.result()))
                code.putln(code.set_error_info(self.pos));
7988 7989 7990 7991 7992 7993
                code.put("if (__Pyx_cdivision_warning(%(FILENAME)s, "
                                                     "%(LINENO)s)) " % {
                    'FILENAME': Naming.filename_cname,
                    'LINENO':  Naming.lineno_cname,
                    })

7994 7995
                code.put_goto(code.error_label)
                code.putln("}")
7996

Robert Bradshaw's avatar
Robert Bradshaw committed
7997
    def calculate_result_code(self):
7998 7999 8000
        if self.type.is_complex:
            return NumBinopNode.calculate_result_code(self)
        elif self.type.is_float and self.operator == '//':
8001
            return "floor(%s / %s)" % (
8002
                self.operand1.result(),
8003
                self.operand2.result())
8004 8005 8006 8007 8008 8009 8010 8011 8012
        elif self.truedivision or self.cdivision:
            op1 = self.operand1.result()
            op2 = self.operand2.result()
            if self.truedivision:
                if self.type != self.operand1.type:
                    op1 = self.type.cast_code(op1)
                if self.type != self.operand2.type:
                    op2 = self.type.cast_code(op2)
            return "(%s / %s)" % (op1, op2)
8013 8014
        else:
            return "__Pyx_div_%s(%s, %s)" % (
Craig Citro's avatar
Craig Citro committed
8015
                    self.type.specialization_name(),
8016
                    self.operand1.result(),
8017
                    self.operand2.result())
Robert Bradshaw's avatar
Robert Bradshaw committed
8018 8019


Robert Bradshaw's avatar
Robert Bradshaw committed
8020
class ModNode(DivNode):
William Stein's avatar
William Stein committed
8021
    #  '%' operator.
8022

8023 8024 8025 8026
    def is_py_operation_types(self, type1, type2):
        return (type1.is_string
            or type2.is_string
            or NumBinopNode.is_py_operation_types(self, type1, type2))
William Stein's avatar
William Stein committed
8027

8028 8029 8030 8031 8032
    def zero_division_message(self):
        if self.type.is_int:
            return "integer division or modulo by zero"
        else:
            return "float divmod()"
8033

8034
    def generate_evaluation_code(self, code):
8035 8036 8037 8038 8039
        if not self.type.is_pyobject:
            if self.cdivision is None:
                self.cdivision = code.globalstate.directives['cdivision'] or not self.type.signed
            if not self.cdivision:
                if self.type.is_int:
8040
                    code.globalstate.use_utility_code(mod_int_utility_code.specialize(self.type))
8041
                else:
8042 8043
                    code.globalstate.use_utility_code(
                        mod_float_utility_code.specialize(self.type, math_h_modifier=self.type.math_h_modifier))
8044
        NumBinopNode.generate_evaluation_code(self, code)
8045
        self.generate_div_warning_code(code)
8046

Robert Bradshaw's avatar
Robert Bradshaw committed
8047
    def calculate_result_code(self):
8048 8049 8050 8051
        if self.cdivision:
            if self.type.is_float:
                return "fmod%s(%s, %s)" % (
                    self.type.math_h_modifier,
8052
                    self.operand1.result(),
8053 8054 8055
                    self.operand2.result())
            else:
                return "(%s %% %s)" % (
8056
                    self.operand1.result(),
8057
                    self.operand2.result())
Robert Bradshaw's avatar
Robert Bradshaw committed
8058
        else:
8059
            return "__Pyx_mod_%s(%s, %s)" % (
Craig Citro's avatar
Craig Citro committed
8060
                    self.type.specialization_name(),
8061
                    self.operand1.result(),
8062
                    self.operand2.result())
William Stein's avatar
William Stein committed
8063 8064 8065

class PowNode(NumBinopNode):
    #  '**' operator.
8066

Robert Bradshaw's avatar
Robert Bradshaw committed
8067 8068
    def analyse_c_operation(self, env):
        NumBinopNode.analyse_c_operation(self, env)
8069
        if self.type.is_complex:
Robert Bradshaw's avatar
Robert Bradshaw committed
8070 8071 8072 8073 8074 8075 8076
            if self.type.real_type.is_float:
                self.operand1 = self.operand1.coerce_to(self.type, env)
                self.operand2 = self.operand2.coerce_to(self.type, env)
                self.pow_func = "__Pyx_c_pow" + self.type.real_type.math_h_modifier
            else:
                error(self.pos, "complex int powers not supported")
                self.pow_func = "<error>"
8077
        elif self.type.is_float:
8078
            self.pow_func = "pow" + self.type.math_h_modifier
William Stein's avatar
William Stein committed
8079
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
8080 8081
            self.pow_func = "__Pyx_pow_%s" % self.type.declaration_code('').replace(' ', '_')
            env.use_utility_code(
8082
                    int_pow_utility_code.specialize(func_name=self.pow_func,
Robert Bradshaw's avatar
Robert Bradshaw committed
8083
                                                type=self.type.declaration_code('')))
8084

William Stein's avatar
William Stein committed
8085
    def calculate_result_code(self):
8086 8087 8088 8089 8090 8091
        # Work around MSVC overloading ambiguity.
        def typecast(operand):
            if self.type == operand.type:
                return operand.result()
            else:
                return self.type.cast_code(operand.result())
Robert Bradshaw's avatar
Robert Bradshaw committed
8092
        return "%s(%s, %s)" % (
8093 8094
            self.pow_func,
            typecast(self.operand1),
8095
            typecast(self.operand2))
8096

William Stein's avatar
William Stein committed
8097

Craig Citro's avatar
Craig Citro committed
8098
# Note: This class is temporarily "shut down" into an ineffective temp
8099 8100
# allocation mode.
#
Craig Citro's avatar
Craig Citro committed
8101 8102 8103
# More sophisticated temp reuse was going on before, one could have a
# look at adding this again after /all/ classes are converted to the
# new temp scheme. (The temp juggling cannot work otherwise).
8104
class BoolBinopNode(ExprNode):
William Stein's avatar
William Stein committed
8105 8106 8107 8108 8109
    #  Short-circuiting boolean operation.
    #
    #  operator     string
    #  operand1     ExprNode
    #  operand2     ExprNode
8110

8111
    subexprs = ['operand1', 'operand2']
8112

8113
    def infer_type(self, env):
8114 8115
        type1 = self.operand1.infer_type(env)
        type2 = self.operand2.infer_type(env)
8116
        return PyrexTypes.independent_spanning_type(type1, type2)
8117

Stefan Behnel's avatar
Stefan Behnel committed
8118 8119 8120 8121 8122 8123
    def may_be_none(self):
        if self.operator == 'or':
            return self.operand2.may_be_none()
        else:
            return self.operand1.may_be_none() or self.operand2.may_be_none()

8124 8125 8126 8127 8128 8129 8130 8131 8132
    def calculate_constant_result(self):
        if self.operator == 'and':
            self.constant_result = \
                self.operand1.constant_result and \
                self.operand2.constant_result
        else:
            self.constant_result = \
                self.operand1.constant_result or \
                self.operand2.constant_result
8133

8134 8135 8136 8137 8138 8139 8140
    def compile_time_value(self, denv):
        if self.operator == 'and':
            return self.operand1.compile_time_value(denv) \
                and self.operand2.compile_time_value(denv)
        else:
            return self.operand1.compile_time_value(denv) \
                or self.operand2.compile_time_value(denv)
8141

8142
    def coerce_to_boolean(self, env):
8143 8144 8145 8146 8147 8148 8149
        return BoolBinopNode(
            self.pos,
            operator = self.operator,
            operand1 = self.operand1.coerce_to_boolean(env),
            operand2 = self.operand2.coerce_to_boolean(env),
            type = PyrexTypes.c_bint_type,
            is_temp = self.is_temp)
8150

William Stein's avatar
William Stein committed
8151 8152 8153
    def analyse_types(self, env):
        self.operand1.analyse_types(env)
        self.operand2.analyse_types(env)
8154
        self.type = PyrexTypes.independent_spanning_type(self.operand1.type, self.operand2.type)
8155 8156
        self.operand1 = self.operand1.coerce_to(self.type, env)
        self.operand2 = self.operand2.coerce_to(self.type, env)
8157

William Stein's avatar
William Stein committed
8158 8159
        # For what we're about to do, it's vital that
        # both operands be temp nodes.
8160 8161
        self.operand1 = self.operand1.coerce_to_simple(env)
        self.operand2 = self.operand2.coerce_to_simple(env)
William Stein's avatar
William Stein committed
8162
        self.is_temp = 1
8163 8164 8165

    gil_message = "Truth-testing Python object"

William Stein's avatar
William Stein committed
8166
    def check_const(self):
8167
        return self.operand1.check_const() and self.operand2.check_const()
8168

William Stein's avatar
William Stein committed
8169
    def generate_evaluation_code(self, code):
8170
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
8171
        self.operand1.generate_evaluation_code(code)
8172
        test_result, uses_temp = self.generate_operand1_test(code)
William Stein's avatar
William Stein committed
8173 8174 8175 8176 8177 8178 8179 8180
        if self.operator == 'and':
            sense = ""
        else:
            sense = "!"
        code.putln(
            "if (%s%s) {" % (
                sense,
                test_result))
8181 8182
        if uses_temp:
            code.funcstate.release_temp(test_result)
8183
        self.operand1.generate_disposal_code(code)
William Stein's avatar
William Stein committed
8184
        self.operand2.generate_evaluation_code(code)
8185
        self.allocate_temp_result(code)
8186
        self.operand2.make_owned_reference(code)
8187
        code.putln("%s = %s;" % (self.result(), self.operand2.result()))
8188 8189
        self.operand2.generate_post_assignment_code(code)
        self.operand2.free_temps(code)
8190
        code.putln("} else {")
8191
        self.operand1.make_owned_reference(code)
8192
        code.putln("%s = %s;" % (self.result(), self.operand1.result()))
8193 8194
        self.operand1.generate_post_assignment_code(code)
        self.operand1.free_temps(code)
8195
        code.putln("}")
8196

William Stein's avatar
William Stein committed
8197 8198 8199
    def generate_operand1_test(self, code):
        #  Generate code to test the truth of the first operand.
        if self.type.is_pyobject:
8200 8201
            test_result = code.funcstate.allocate_temp(PyrexTypes.c_bint_type,
                                                       manage_ref=False)
William Stein's avatar
William Stein committed
8202
            code.putln(
8203
                "%s = __Pyx_PyObject_IsTrue(%s); %s" % (
William Stein's avatar
William Stein committed
8204 8205
                    test_result,
                    self.operand1.py_result(),
Robert Bradshaw's avatar
Robert Bradshaw committed
8206
                    code.error_goto_if_neg(test_result, self.pos)))
William Stein's avatar
William Stein committed
8207
        else:
8208
            test_result = self.operand1.result()
8209
        return (test_result, self.type.is_pyobject)
William Stein's avatar
William Stein committed
8210 8211


8212
class CondExprNode(ExprNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
8213 8214 8215 8216 8217
    #  Short-circuiting conditional expression.
    #
    #  test        ExprNode
    #  true_val    ExprNode
    #  false_val   ExprNode
8218

8219 8220
    true_val = None
    false_val = None
8221

Robert Bradshaw's avatar
Robert Bradshaw committed
8222
    subexprs = ['test', 'true_val', 'false_val']
8223

Robert Bradshaw's avatar
Robert Bradshaw committed
8224 8225
    def type_dependencies(self, env):
        return self.true_val.type_dependencies(env) + self.false_val.type_dependencies(env)
8226

Robert Bradshaw's avatar
Robert Bradshaw committed
8227
    def infer_type(self, env):
8228 8229
        return PyrexTypes.independent_spanning_type(self.true_val.infer_type(env),
                                                    self.false_val.infer_type(env))
8230 8231 8232 8233 8234 8235 8236

    def calculate_constant_result(self):
        if self.test.constant_result:
            self.constant_result = self.true_val.constant_result
        else:
            self.constant_result = self.false_val.constant_result

Robert Bradshaw's avatar
Robert Bradshaw committed
8237 8238 8239 8240 8241
    def analyse_types(self, env):
        self.test.analyse_types(env)
        self.test = self.test.coerce_to_boolean(env)
        self.true_val.analyse_types(env)
        self.false_val.analyse_types(env)
8242
        self.type = PyrexTypes.independent_spanning_type(self.true_val.type, self.false_val.type)
8243 8244 8245 8246 8247
        if self.true_val.type.is_pyobject or self.false_val.type.is_pyobject:
            self.true_val = self.true_val.coerce_to(self.type, env)
            self.false_val = self.false_val.coerce_to(self.type, env)
        self.is_temp = 1
        if self.type == PyrexTypes.error_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
8248
            self.type_error()
8249

Robert Bradshaw's avatar
Robert Bradshaw committed
8250 8251 8252 8253 8254
    def type_error(self):
        if not (self.true_val.type.is_error or self.false_val.type.is_error):
            error(self.pos, "Incompatable types in conditional expression (%s; %s)" %
                (self.true_val.type, self.false_val.type))
        self.type = PyrexTypes.error_type
8255

Robert Bradshaw's avatar
Robert Bradshaw committed
8256
    def check_const(self):
8257
        return (self.test.check_const()
8258 8259
            and self.true_val.check_const()
            and self.false_val.check_const())
8260

Robert Bradshaw's avatar
Robert Bradshaw committed
8261
    def generate_evaluation_code(self, code):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8262 8263
        # Because subexprs may not be evaluated we can use a more optimal
        # subexpr allocation strategy than the default, so override evaluation_code.
8264

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8265
        code.mark_pos(self.pos)
8266
        self.allocate_temp_result(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
8267
        self.test.generate_evaluation_code(code)
8268
        code.putln("if (%s) {" % self.test.result() )
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8269
        self.eval_and_get(code, self.true_val)
Robert Bradshaw's avatar
Robert Bradshaw committed
8270
        code.putln("} else {")
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8271
        self.eval_and_get(code, self.false_val)
Robert Bradshaw's avatar
Robert Bradshaw committed
8272 8273
        code.putln("}")
        self.test.generate_disposal_code(code)
8274
        self.test.free_temps(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
8275

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8276 8277 8278
    def eval_and_get(self, code, expr):
        expr.generate_evaluation_code(code)
        expr.make_owned_reference(code)
8279
        code.putln('%s = %s;' % (self.result(), expr.result_as(self.ctype())))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8280 8281 8282
        expr.generate_post_assignment_code(code)
        expr.free_temps(code)

8283 8284 8285 8286 8287 8288 8289 8290 8291 8292
richcmp_constants = {
    "<" : "Py_LT",
    "<=": "Py_LE",
    "==": "Py_EQ",
    "!=": "Py_NE",
    "<>": "Py_NE",
    ">" : "Py_GT",
    ">=": "Py_GE",
}

8293
class CmpNode(object):
William Stein's avatar
William Stein committed
8294 8295
    #  Mixin class containing code common to PrimaryCmpNodes
    #  and CascadedCmpNodes.
8296 8297 8298

    special_bool_cmp_function = None

Stefan Behnel's avatar
typo  
Stefan Behnel committed
8299
    def infer_type(self, env):
8300 8301
        # TODO: Actually implement this (after merging with -unstable).
        return py_object_type
8302 8303 8304 8305 8306

    def calculate_cascaded_constant_result(self, operand1_result):
        func = compile_time_binary_operators[self.operator]
        operand2_result = self.operand2.constant_result
        result = func(operand1_result, operand2_result)
8307 8308 8309 8310 8311 8312 8313
        if self.cascade:
            self.cascade.calculate_cascaded_constant_result(operand2_result)
            if self.cascade.constant_result:
                self.constant_result = result and self.cascade.constant_result
        else:
            self.constant_result = result

8314 8315
    def cascaded_compile_time_value(self, operand1, denv):
        func = get_compile_time_binop(self)
8316
        operand2 = self.operand2.compile_time_value(denv)
8317 8318 8319 8320
        try:
            result = func(operand1, operand2)
        except Exception, e:
            self.compile_time_value_error(e)
8321
            result = None
8322 8323 8324
        if result:
            cascade = self.cascade
            if cascade:
8325
                # FIXME: I bet this must call cascaded_compile_time_value()
8326
                result = result and cascade.cascaded_compile_time_value(operand2, denv)
8327 8328
        return result

8329
    def is_cpp_comparison(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
8330
        return self.operand1.type.is_cpp_class or self.operand2.type.is_cpp_class
8331

8332
    def find_common_int_type(self, env, op, operand1, operand2):
8333 8334 8335 8336 8337 8338
        # type1 != type2 and at least one of the types is not a C int
        type1 = operand1.type
        type2 = operand2.type
        type1_can_be_int = False
        type2_can_be_int = False

8339
        if operand1.is_string_literal and operand1.can_coerce_to_char_literal():
8340
            type1_can_be_int = True
8341
        if operand2.is_string_literal and operand2.can_coerce_to_char_literal():
8342 8343 8344 8345
            type2_can_be_int = True

        if type1.is_int:
            if type2_can_be_int:
8346
                return type1
8347 8348
        elif type2.is_int:
            if type1_can_be_int:
8349
                return type2
8350 8351
        elif type1_can_be_int:
            if type2_can_be_int:
8352
                return PyrexTypes.c_uchar_type
William Stein's avatar
William Stein committed
8353

8354
        return None
8355

8356
    def find_common_type(self, env, op, operand1, common_type=None):
8357
        operand2 = self.operand2
William Stein's avatar
William Stein committed
8358 8359
        type1 = operand1.type
        type2 = operand2.type
8360

8361 8362
        new_common_type = None

Stefan Behnel's avatar
Stefan Behnel committed
8363
        # catch general errors
8364 8365 8366
        if type1 == str_type and (type2.is_string or type2 in (bytes_type, unicode_type)) or \
               type2 == str_type and (type1.is_string or type1 in (bytes_type, unicode_type)):
            error(self.pos, "Comparisons between bytes/unicode and str are not portable to Python 3")
8367
            new_common_type = error_type
Stefan Behnel's avatar
Stefan Behnel committed
8368 8369

        # try to use numeric comparisons where possible
8370
        elif type1.is_complex or type2.is_complex:
8371 8372 8373
            if op not in ('==', '!=') \
               and (type1.is_complex or type1.is_numeric) \
               and (type2.is_complex or type2.is_numeric):
8374 8375
                error(self.pos, "complex types are unordered")
                new_common_type = error_type
8376
            elif type1.is_pyobject:
8377 8378 8379
                new_common_type = type1
            elif type2.is_pyobject:
                new_common_type = type2
8380
            else:
8381
                new_common_type = PyrexTypes.widest_numeric_type(type1, type2)
8382 8383
        elif type1.is_numeric and type2.is_numeric:
            new_common_type = PyrexTypes.widest_numeric_type(type1, type2)
8384
        elif common_type is None or not common_type.is_pyobject:
8385
            new_common_type = self.find_common_int_type(env, op, operand1, operand2)
8386 8387

        if new_common_type is None:
Stefan Behnel's avatar
Stefan Behnel committed
8388
            # fall back to generic type compatibility tests
8389
            if type1 == type2:
8390 8391 8392 8393 8394 8395
                new_common_type = type1
            elif type1.is_pyobject or type2.is_pyobject:
                if type2.is_numeric or type2.is_string:
                    if operand2.check_for_coercion_error(type1):
                        new_common_type = error_type
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
8396
                        new_common_type = py_object_type
8397 8398 8399 8400
                elif type1.is_numeric or type1.is_string:
                    if operand1.check_for_coercion_error(type2):
                        new_common_type = error_type
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
8401 8402 8403
                        new_common_type = py_object_type
                elif py_object_type.assignable_from(type1) and py_object_type.assignable_from(type2):
                    new_common_type = py_object_type
8404 8405 8406 8407
                else:
                    # one Python type and one non-Python type, not assignable
                    self.invalid_types_error(operand1, op, operand2)
                    new_common_type = error_type
8408 8409 8410 8411
            elif type1.assignable_from(type2):
                new_common_type = type1
            elif type2.assignable_from(type1):
                new_common_type = type2
8412 8413 8414 8415
            else:
                # C types that we couldn't handle up to here are an error
                self.invalid_types_error(operand1, op, operand2)
                new_common_type = error_type
8416

8417 8418 8419 8420 8421 8422
        if new_common_type.is_string and (isinstance(operand1, BytesNode) or
                                          isinstance(operand2, BytesNode)):
            # special case when comparing char* to bytes literal: must
            # compare string values!
            new_common_type = bytes_type

Stefan Behnel's avatar
Stefan Behnel committed
8423
        # recursively merge types
8424
        if common_type is None or new_common_type.is_error:
8425
            common_type = new_common_type
William Stein's avatar
William Stein committed
8426
        else:
8427 8428 8429
            # we could do a lot better by splitting the comparison
            # into a non-Python part and a Python part, but this is
            # safer for now
8430
            common_type = PyrexTypes.spanning_type(common_type, new_common_type)
8431 8432

        if self.cascade:
8433
            common_type = self.cascade.find_common_type(env, self.operator, operand2, common_type)
8434

8435 8436
        return common_type

8437 8438 8439 8440
    def invalid_types_error(self, operand1, op, operand2):
        error(self.pos, "Invalid types for '%s' (%s, %s)" %
              (op, operand1.type, operand2.type))

Stefan Behnel's avatar
Stefan Behnel committed
8441
    def is_python_comparison(self):
8442 8443 8444 8445 8446
        return (not self.is_ptr_contains()
            and not self.is_c_string_contains()
            and (self.has_python_operands()
                 or (self.cascade and self.cascade.is_python_comparison())
                 or self.operator in ('in', 'not_in')))
Stefan Behnel's avatar
Stefan Behnel committed
8447

8448 8449 8450 8451 8452 8453
    def coerce_operands_to(self, dst_type, env):
        operand2 = self.operand2
        if operand2.type != dst_type:
            self.operand2 = operand2.coerce_to(dst_type, env)
        if self.cascade:
            self.cascade.coerce_operands_to(dst_type, env)
8454

8455
    def is_python_result(self):
8456
        return ((self.has_python_operands() and
8457
                 self.special_bool_cmp_function is None and
8458
                 self.operator not in ('is', 'is_not', 'in', 'not_in') and
8459 8460
                 not self.is_c_string_contains() and
                 not self.is_ptr_contains())
8461
            or (self.cascade and self.cascade.is_python_result()))
William Stein's avatar
William Stein committed
8462

8463 8464
    def is_c_string_contains(self):
        return self.operator in ('in', 'not_in') and \
8465 8466
               ((self.operand1.type.is_int
                 and (self.operand2.type.is_string or self.operand2.type is bytes_type)) or
Stefan Behnel's avatar
Stefan Behnel committed
8467
                (self.operand1.type.is_unicode_char
8468
                 and self.operand2.type is unicode_type))
8469

8470 8471
    def is_ptr_contains(self):
        if self.operator in ('in', 'not_in'):
8472 8473 8474
            container_type = self.operand2.type
            return (container_type.is_ptr or container_type.is_array) \
                and not container_type.is_string
8475

8476 8477 8478 8479 8480
    def find_special_bool_compare_function(self, env):
        if self.operator in ('==', '!='):
            type1, type2 = self.operand1.type, self.operand2.type
            if type1.is_pyobject and type2.is_pyobject:
                if type1 is Builtin.unicode_type or type2 is Builtin.unicode_type:
8481
                    env.use_utility_code(UtilityCode.load_cached("UnicodeEquals", "StringTools.c"))
8482 8483
                    self.special_bool_cmp_function = "__Pyx_PyUnicode_Equals"
                    return True
8484
                elif type1 is Builtin.bytes_type or type2 is Builtin.bytes_type:
8485
                    env.use_utility_code(UtilityCode.load_cached("BytesEquals", "StringTools.c"))
8486 8487 8488
                    self.special_bool_cmp_function = "__Pyx_PyBytes_Equals"
                    return True
                elif type1 is Builtin.str_type or type2 is Builtin.str_type:
8489
                    env.use_utility_code(UtilityCode.load_cached("StrEquals", "StringTools.c"))
8490 8491
                    self.special_bool_cmp_function = "__Pyx_PyString_Equals"
                    return True
8492 8493
        return False

8494
    def generate_operation_code(self, code, result_code,
William Stein's avatar
William Stein committed
8495
            operand1, op , operand2):
8496
        if self.type.is_pyobject:
8497 8498 8499
            coerce_result = "__Pyx_PyBool_FromLong"
        else:
            coerce_result = ""
8500
        if 'not' in op:
8501
            negation = "!"
8502
        else:
8503
            negation = ""
8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520
        if self.special_bool_cmp_function:
            if operand1.type.is_pyobject:
                result1 = operand1.py_result()
            else:
                result1 = operand1.result()
            if operand2.type.is_pyobject:
                result2 = operand2.py_result()
            else:
                result2 = operand2.result()
            code.putln("%s = %s(%s, %s, %s); %s" % (
                result_code,
                self.special_bool_cmp_function,
                result1,
                result2,
                richcmp_constants[op],
                code.error_goto_if_neg(result_code, self.pos)))
        elif op == 'in' or op == 'not_in':
Stefan Behnel's avatar
typo  
Stefan Behnel committed
8521
            code.globalstate.use_utility_code(contains_utility_code)
8522
            if self.type.is_pyobject:
8523
                coerce_result = "__Pyx_PyBoolOrNull_FromLong"
8524
            if op == 'not_in':
8525
                negation = "__Pyx_NegateNonNeg"
8526
            if operand2.type is dict_type:
8527
                method = "PyDict_Contains"
8528
            else:
8529
                method = "PySequence_Contains"
8530
            if self.type.is_pyobject:
8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541
                error_clause = code.error_goto_if_null
                got_ref = "__Pyx_XGOTREF(%s); " % result_code
            else:
                error_clause = code.error_goto_if_neg
                got_ref = ""
            code.putln(
                "%s = %s(%s(%s(%s, %s))); %s%s" % (
                    result_code,
                    coerce_result,
                    negation,
                    method,
8542 8543
                    operand2.py_result(),
                    operand1.py_result(),
8544 8545
                    got_ref,
                    error_clause(result_code, self.pos)))
William Stein's avatar
William Stein committed
8546 8547
        elif (operand1.type.is_pyobject
            and op not in ('is', 'is_not')):
8548
                code.putln("%s = PyObject_RichCompare(%s, %s, %s); %s" % (
8549 8550 8551
                        result_code,
                        operand1.py_result(),
                        operand2.py_result(),
8552 8553
                        richcmp_constants[op],
                        code.error_goto_if_null(result_code, self.pos)))
8554
                code.put_gotref(result_code)
8555
        elif operand1.type.is_complex:
8556
            if op == "!=":
8557
                negation = "!"
8558
            else:
8559
                negation = ""
8560
            code.putln("%s = %s(%s%s(%s, %s));" % (
8561
                result_code,
8562 8563
                coerce_result,
                negation,
8564 8565
                operand1.type.unary_op('eq'),
                operand1.result(),
8566
                operand2.result()))
William Stein's avatar
William Stein committed
8567
        else:
8568 8569 8570 8571 8572
            type1 = operand1.type
            type2 = operand2.type
            if (type1.is_extension_type or type2.is_extension_type) \
                    and not type1.same_as(type2):
                common_type = py_object_type
8573 8574
            elif type1.is_numeric:
                common_type = PyrexTypes.widest_numeric_type(type1, type2)
8575
            else:
8576 8577 8578
                common_type = type1
            code1 = operand1.result_as(common_type)
            code2 = operand2.result_as(common_type)
8579
            code.putln("%s = %s(%s %s %s);" % (
8580 8581 8582 8583
                result_code,
                coerce_result,
                code1,
                self.c_operator(op),
8584 8585
                code2))

William Stein's avatar
William Stein committed
8586 8587 8588 8589 8590 8591 8592
    def c_operator(self, op):
        if op == 'is':
            return "=="
        elif op == 'is_not':
            return "!="
        else:
            return op
8593

Stefan Behnel's avatar
typo  
Stefan Behnel committed
8594
contains_utility_code = UtilityCode(
8595
proto="""
8596 8597
static CYTHON_INLINE int __Pyx_NegateNonNeg(int b) {
    return unlikely(b < 0) ? b : !b;
Lisandro Dalcin's avatar
Lisandro Dalcin committed
8598
}
8599
static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) {
8600 8601 8602 8603
    return unlikely(b < 0) ? NULL : __Pyx_PyBool_FromLong(b);
}
""")

William Stein's avatar
William Stein committed
8604

8605
class PrimaryCmpNode(ExprNode, CmpNode):
William Stein's avatar
William Stein committed
8606 8607 8608 8609 8610 8611 8612
    #  Non-cascaded comparison or first comparison of
    #  a cascaded sequence.
    #
    #  operator      string
    #  operand1      ExprNode
    #  operand2      ExprNode
    #  cascade       CascadedCmpNode
8613

William Stein's avatar
William Stein committed
8614 8615 8616 8617
    #  We don't use the subexprs mechanism, because
    #  things here are too complicated for it to handle.
    #  Instead, we override all the framework methods
    #  which use it.
8618

Robert Bradshaw's avatar
Robert Bradshaw committed
8619
    child_attrs = ['operand1', 'operand2', 'cascade']
8620

William Stein's avatar
William Stein committed
8621
    cascade = None
8622

Robert Bradshaw's avatar
Robert Bradshaw committed
8623 8624 8625 8626 8627 8628 8629
    def infer_type(self, env):
        # TODO: Actually implement this (after merging with -unstable).
        return py_object_type

    def type_dependencies(self, env):
        return ()

8630
    def calculate_constant_result(self):
8631
        self.calculate_cascaded_constant_result(self.operand1.constant_result)
8632

8633
    def compile_time_value(self, denv):
8634
        operand1 = self.operand1.compile_time_value(denv)
8635 8636
        return self.cascaded_compile_time_value(operand1, denv)

William Stein's avatar
William Stein committed
8637 8638 8639
    def analyse_types(self, env):
        self.operand1.analyse_types(env)
        self.operand2.analyse_types(env)
8640 8641
        if self.is_cpp_comparison():
            self.analyse_cpp_comparison(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
8642 8643 8644
            if self.cascade:
                error(self.pos, "Cascading comparison not yet supported for cpp types.")
            return
William Stein's avatar
William Stein committed
8645
        if self.cascade:
8646 8647
            self.cascade.analyse_types(env)

8648
        if self.operator in ('in', 'not_in'):
8649 8650 8651 8652 8653 8654 8655
            if self.is_c_string_contains():
                self.is_pycmp = False
                common_type = None
                if self.cascade:
                    error(self.pos, "Cascading comparison not yet supported for 'int_val in string'.")
                    return
                if self.operand2.type is unicode_type:
8656
                    env.use_utility_code(UtilityCode.load_cached("PyUCS4InUnicode", "StringTools.c"))
8657 8658 8659 8660 8661
                else:
                    if self.operand1.type is PyrexTypes.c_uchar_type:
                        self.operand1 = self.operand1.coerce_to(PyrexTypes.c_char_type, env)
                    if self.operand2.type is not bytes_type:
                        self.operand2 = self.operand2.coerce_to(bytes_type, env)
8662
                    env.use_utility_code(UtilityCode.load_cached("BytesContains", "StringTools.c"))
Stefan Behnel's avatar
Stefan Behnel committed
8663 8664
                self.operand2 = self.operand2.as_none_safe_node(
                    "argument of type 'NoneType' is not iterable")
8665 8666 8667 8668 8669 8670
            elif self.is_ptr_contains():
                if self.cascade:
                    error(self.pos, "Cascading comparison not yet supported for 'val in sliced pointer'.")
                self.type = PyrexTypes.c_bint_type
                # Will be transformed by IterationTransform
                return
8671
            else:
8672 8673
                if self.operand2.type is dict_type:
                    self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable")
8674 8675
                common_type = py_object_type
                self.is_pycmp = True
8676 8677 8678 8679
        elif self.find_special_bool_compare_function(env):
            common_type = None # if coercion needed, the method call above has already done it
            self.is_pycmp = False # result is bint
            self.is_temp = True # must check for error return
8680 8681 8682 8683
        else:
            common_type = self.find_common_type(env, self.operator, self.operand1)
            self.is_pycmp = common_type.is_pyobject

8684
        if common_type is not None and not common_type.is_error:
8685 8686 8687
            if self.operand1.type != common_type:
                self.operand1 = self.operand1.coerce_to(common_type, env)
            self.coerce_operands_to(common_type, env)
8688

William Stein's avatar
William Stein committed
8689 8690 8691
        if self.cascade:
            self.operand2 = self.operand2.coerce_to_simple(env)
            self.cascade.coerce_cascaded_operands_to_temp(env)
8692 8693 8694 8695 8696 8697 8698 8699
        if self.is_python_result():
            self.type = PyrexTypes.py_object_type
        else:
            self.type = PyrexTypes.c_bint_type
        cdr = self.cascade
        while cdr:
            cdr.type = self.type
            cdr = cdr.cascade
William Stein's avatar
William Stein committed
8700 8701
        if self.is_pycmp or self.cascade:
            self.is_temp = 1
8702

8703 8704 8705
    def analyse_cpp_comparison(self, env):
        type1 = self.operand1.type
        type2 = self.operand2.type
8706 8707
        entry = env.lookup_operator(self.operator, [self.operand1, self.operand2])
        if entry is None:
8708 8709
            error(self.pos, "Invalid types for '%s' (%s, %s)" %
                (self.operator, type1, type2))
8710 8711 8712
            self.type = PyrexTypes.error_type
            self.result_code = "<error>"
            return
8713 8714 8715 8716 8717
        func_type = entry.type
        if func_type.is_ptr:
            func_type = func_type.base_type
        if len(func_type.args) == 1:
            self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env)
8718
        else:
8719 8720 8721
            self.operand1 = self.operand1.coerce_to(func_type.args[0].type, env)
            self.operand2 = self.operand2.coerce_to(func_type.args[1].type, env)
        self.type = func_type.return_type
8722

William Stein's avatar
William Stein committed
8723 8724 8725
    def has_python_operands(self):
        return (self.operand1.type.is_pyobject
            or self.operand2.type.is_pyobject)
8726

William Stein's avatar
William Stein committed
8727 8728 8729
    def check_const(self):
        if self.cascade:
            self.not_const()
8730 8731 8732
            return False
        else:
            return self.operand1.check_const() and self.operand2.check_const()
William Stein's avatar
William Stein committed
8733 8734

    def calculate_result_code(self):
8735 8736 8737 8738 8739 8740 8741
        if self.operand1.type.is_complex:
            if self.operator == "!=":
                negation = "!"
            else:
                negation = ""
            return "(%s%s(%s, %s))" % (
                negation,
8742 8743
                self.operand1.type.binary_op('=='),
                self.operand1.result(),
8744
                self.operand2.result())
8745
        elif self.is_c_string_contains():
8746
            if self.operand2.type is unicode_type:
8747
                method = "__Pyx_UnicodeContainsUCS4"
8748
            else:
8749
                method = "__Pyx_BytesContains"
8750 8751 8752 8753 8754 8755 8756
            if self.operator == "not_in":
                negation = "!"
            else:
                negation = ""
            return "(%s%s(%s, %s))" % (
                negation,
                method,
8757
                self.operand2.result(),
8758
                self.operand1.result())
8759 8760 8761 8762 8763
        else:
            return "(%s %s %s)" % (
                self.operand1.result(),
                self.c_operator(self.operator),
                self.operand2.result())
8764

William Stein's avatar
William Stein committed
8765 8766 8767 8768
    def generate_evaluation_code(self, code):
        self.operand1.generate_evaluation_code(code)
        self.operand2.generate_evaluation_code(code)
        if self.is_temp:
8769
            self.allocate_temp_result(code)
8770
            self.generate_operation_code(code, self.result(),
William Stein's avatar
William Stein committed
8771 8772 8773
                self.operand1, self.operator, self.operand2)
            if self.cascade:
                self.cascade.generate_evaluation_code(code,
8774
                    self.result(), self.operand2)
William Stein's avatar
William Stein committed
8775
            self.operand1.generate_disposal_code(code)
8776
            self.operand1.free_temps(code)
William Stein's avatar
William Stein committed
8777
            self.operand2.generate_disposal_code(code)
8778
            self.operand2.free_temps(code)
8779

William Stein's avatar
William Stein committed
8780 8781 8782 8783 8784
    def generate_subexpr_disposal_code(self, code):
        #  If this is called, it is a non-cascaded cmp,
        #  so only need to dispose of the two main operands.
        self.operand1.generate_disposal_code(code)
        self.operand2.generate_disposal_code(code)
8785

8786 8787 8788 8789 8790
    def free_subexpr_temps(self, code):
        #  If this is called, it is a non-cascaded cmp,
        #  so only need to dispose of the two main operands.
        self.operand1.free_temps(code)
        self.operand2.free_temps(code)
8791

8792 8793 8794 8795 8796
    def annotate(self, code):
        self.operand1.annotate(code)
        self.operand2.annotate(code)
        if self.cascade:
            self.cascade.annotate(code)
William Stein's avatar
William Stein committed
8797 8798 8799


class CascadedCmpNode(Node, CmpNode):
8800 8801 8802
    #  A CascadedCmpNode is not a complete expression node. It
    #  hangs off the side of another comparison node, shares
    #  its left operand with that node, and shares its result
William Stein's avatar
William Stein committed
8803 8804 8805 8806 8807 8808
    #  with the PrimaryCmpNode at the head of the chain.
    #
    #  operator      string
    #  operand2      ExprNode
    #  cascade       CascadedCmpNode

Robert Bradshaw's avatar
Robert Bradshaw committed
8809 8810
    child_attrs = ['operand2', 'cascade']

William Stein's avatar
William Stein committed
8811
    cascade = None
8812 8813
    constant_result = constant_value_not_set # FIXME: where to calculate this?

Robert Bradshaw's avatar
Robert Bradshaw committed
8814 8815 8816 8817 8818 8819 8820
    def infer_type(self, env):
        # TODO: Actually implement this (after merging with -unstable).
        return py_object_type

    def type_dependencies(self, env):
        return ()

8821 8822 8823 8824
    def has_constant_result(self):
        return self.constant_result is not constant_value_not_set and \
               self.constant_result is not not_a_constant

8825
    def analyse_types(self, env):
William Stein's avatar
William Stein committed
8826 8827
        self.operand2.analyse_types(env)
        if self.cascade:
8828
            self.cascade.analyse_types(env)
8829

William Stein's avatar
William Stein committed
8830 8831
    def has_python_operands(self):
        return self.operand2.type.is_pyobject
8832

William Stein's avatar
William Stein committed
8833 8834
    def coerce_operands_to_pyobjects(self, env):
        self.operand2 = self.operand2.coerce_to_pyobject(env)
8835 8836
        if self.operand2.type is dict_type and self.operator in ('in', 'not_in'):
            self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable")
William Stein's avatar
William Stein committed
8837 8838 8839 8840 8841 8842 8843 8844
        if self.cascade:
            self.cascade.coerce_operands_to_pyobjects(env)

    def coerce_cascaded_operands_to_temp(self, env):
        if self.cascade:
            #self.operand2 = self.operand2.coerce_to_temp(env) #CTT
            self.operand2 = self.operand2.coerce_to_simple(env)
            self.cascade.coerce_cascaded_operands_to_temp(env)
8845

William Stein's avatar
William Stein committed
8846
    def generate_evaluation_code(self, code, result, operand1):
8847 8848
        if self.type.is_pyobject:
            code.putln("if (__Pyx_PyObject_IsTrue(%s)) {" % result)
8849
            code.put_decref(result, self.type)
8850 8851
        else:
            code.putln("if (%s) {" % result)
William Stein's avatar
William Stein committed
8852
        self.operand2.generate_evaluation_code(code)
8853
        self.generate_operation_code(code, result,
William Stein's avatar
William Stein committed
8854 8855 8856 8857 8858 8859
            operand1, self.operator, self.operand2)
        if self.cascade:
            self.cascade.generate_evaluation_code(
                code, result, self.operand2)
        # Cascaded cmp result is always temp
        self.operand2.generate_disposal_code(code)
8860
        self.operand2.free_temps(code)
William Stein's avatar
William Stein committed
8861 8862
        code.putln("}")

8863 8864 8865 8866 8867
    def annotate(self, code):
        self.operand2.annotate(code)
        if self.cascade:
            self.cascade.annotate(code)

William Stein's avatar
William Stein committed
8868 8869

binop_node_classes = {
8870 8871
    "or":       BoolBinopNode,
    "and":      BoolBinopNode,
Robert Bradshaw's avatar
Robert Bradshaw committed
8872 8873 8874
    "|":        IntBinopNode,
    "^":        IntBinopNode,
    "&":        IntBinopNode,
8875 8876
    "<<":       IntBinopNode,
    ">>":       IntBinopNode,
Robert Bradshaw's avatar
Robert Bradshaw committed
8877 8878 8879
    "+":        AddNode,
    "-":        SubNode,
    "*":        MulNode,
8880 8881
    "/":        DivNode,
    "//":       DivNode,
Robert Bradshaw's avatar
Robert Bradshaw committed
8882
    "%":        ModNode,
8883
    "**":       PowNode
William Stein's avatar
William Stein committed
8884 8885
}

8886
def binop_node(pos, operator, operand1, operand2, inplace=False):
8887
    # Construct binop node of appropriate class for
William Stein's avatar
William Stein committed
8888
    # given operator.
8889 8890 8891
    return binop_node_classes[operator](pos,
        operator = operator,
        operand1 = operand1,
8892 8893
        operand2 = operand2,
        inplace = inplace)
William Stein's avatar
William Stein committed
8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905

#-------------------------------------------------------------------
#
#  Coercion nodes
#
#  Coercion nodes are special in that they are created during
#  the analyse_types phase of parse tree processing.
#  Their __init__ methods consequently incorporate some aspects
#  of that phase.
#
#-------------------------------------------------------------------

8906
class CoercionNode(ExprNode):
William Stein's avatar
William Stein committed
8907 8908 8909
    #  Abstract base class for coercion nodes.
    #
    #  arg       ExprNode       node being coerced
8910

William Stein's avatar
William Stein committed
8911
    subexprs = ['arg']
8912
    constant_result = not_a_constant
8913

William Stein's avatar
William Stein committed
8914 8915 8916 8917
    def __init__(self, arg):
        self.pos = arg.pos
        self.arg = arg
        if debug_coercion:
Stefan Behnel's avatar
Stefan Behnel committed
8918
            print("%s Coercing %s" % (self, self.arg))
8919 8920

    def calculate_constant_result(self):
8921 8922
        # constant folding can break type coercion, so this is disabled
        pass
8923

8924 8925 8926 8927 8928
    def annotate(self, code):
        self.arg.annotate(code)
        if self.arg.type != self.type:
            file, line, col = self.pos
            code.annotate((file, line, col-1), AnnotationItem(style='coerce', tag='coerce', text='[%s] to [%s]' % (self.arg.type, self.type)))
William Stein's avatar
William Stein committed
8929

8930
class CoerceToMemViewSliceNode(CoercionNode):
8931 8932 8933 8934
    """
    Coerce an object to a memoryview slice. This holds a new reference in
    a managed temp.
    """
8935 8936

    def __init__(self, arg, dst_type, env):
8937 8938
        assert dst_type.is_memoryviewslice
        assert not arg.type.is_memoryviewslice
8939 8940
        CoercionNode.__init__(self, arg)
        self.type = dst_type
8941
        self.is_temp = 1
8942 8943
        self.env = env
        self.use_managed_ref = True
8944
        self.arg = arg
8945 8946

    def generate_result_code(self, code):
8947 8948 8949 8950 8951
        self.type.create_from_py_utility_code(self.env)
        code.putln("%s = %s(%s);" % (self.result(),
                                     self.type.from_py_function,
                                     self.arg.py_result()))

8952 8953 8954
        error_cond = self.type.error_condition(self.result())
        code.putln(code.error_goto_if(error_cond, self.pos))

William Stein's avatar
William Stein committed
8955 8956 8957

class CastNode(CoercionNode):
    #  Wrap a node in a C type cast.
8958

William Stein's avatar
William Stein committed
8959 8960 8961
    def __init__(self, arg, new_type):
        CoercionNode.__init__(self, arg)
        self.type = new_type
Stefan Behnel's avatar
Stefan Behnel committed
8962 8963 8964

    def may_be_none(self):
        return self.arg.may_be_none()
8965

William Stein's avatar
William Stein committed
8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977
    def calculate_result_code(self):
        return self.arg.result_as(self.type)

    def generate_result_code(self, code):
        self.arg.generate_result_code(code)


class PyTypeTestNode(CoercionNode):
    #  This node is used to check that a generic Python
    #  object is an instance of a particular extension type.
    #  This node borrows the result of its argument node.

8978
    def __init__(self, arg, dst_type, env, notnone=False):
William Stein's avatar
William Stein committed
8979 8980
        #  The arg is know to be a Python object, and
        #  the dst_type is known to be an extension type.
Robert Bradshaw's avatar
Robert Bradshaw committed
8981
        assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type"
William Stein's avatar
William Stein committed
8982 8983 8984
        CoercionNode.__init__(self, arg)
        self.type = dst_type
        self.result_ctype = arg.ctype()
8985
        self.notnone = notnone
8986

8987
    nogil_check = Node.gil_error
8988
    gil_message = "Python type test"
8989

8990 8991
    def analyse_types(self, env):
        pass
Stefan Behnel's avatar
Stefan Behnel committed
8992 8993 8994 8995 8996

    def may_be_none(self):
        if self.notnone:
            return False
        return self.arg.may_be_none()
8997

8998 8999 9000
    def is_simple(self):
        return self.arg.is_simple()

William Stein's avatar
William Stein committed
9001 9002
    def result_in_temp(self):
        return self.arg.result_in_temp()
9003

William Stein's avatar
William Stein committed
9004 9005
    def is_ephemeral(self):
        return self.arg.is_ephemeral()
9006 9007 9008 9009 9010

    def calculate_constant_result(self):
        # FIXME
        pass

William Stein's avatar
William Stein committed
9011
    def calculate_result_code(self):
9012
        return self.arg.result()
9013

William Stein's avatar
William Stein committed
9014 9015
    def generate_result_code(self, code):
        if self.type.typeobj_is_available():
9016
            if not self.type.is_builtin_type:
9017
                code.globalstate.use_utility_code(type_test_utility_code)
William Stein's avatar
William Stein committed
9018
            code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
9019
                "if (!(%s)) %s" % (
9020
                    self.type.type_test_code(self.arg.py_result(), self.notnone),
William Stein's avatar
William Stein committed
9021 9022 9023 9024
                    code.error_goto(self.pos)))
        else:
            error(self.pos, "Cannot test type of extern C class "
                "without type object name specification")
9025

William Stein's avatar
William Stein committed
9026 9027
    def generate_post_assignment_code(self, code):
        self.arg.generate_post_assignment_code(code)
9028 9029 9030

    def free_temps(self, code):
        self.arg.free_temps(code)
9031 9032 9033 9034 9035 9036 9037


class NoneCheckNode(CoercionNode):
    # This node is used to check that a Python object is not None and
    # raises an appropriate exception (as specified by the creating
    # transform).

9038 9039
    def __init__(self, arg, exception_type_cname, exception_message,
                 exception_format_args):
9040 9041 9042 9043 9044
        CoercionNode.__init__(self, arg)
        self.type = arg.type
        self.result_ctype = arg.ctype()
        self.exception_type_cname = exception_type_cname
        self.exception_message = exception_message
9045
        self.exception_format_args = tuple(exception_format_args or ())
9046 9047 9048 9049

    def analyse_types(self, env):
        pass

9050 9051 9052
    def may_be_none(self):
        return False

9053 9054 9055
    def is_simple(self):
        return self.arg.is_simple()

9056 9057 9058 9059 9060
    def result_in_temp(self):
        return self.arg.result_in_temp()

    def calculate_result_code(self):
        return self.arg.result()
9061

9062 9063
    def generate_result_code(self, code):
        code.putln(
9064
            "if (unlikely(%s == Py_None)) {" % self.arg.py_result())
9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078
        escape = StringEncoding.escape_byte_string
        if self.exception_format_args:
            code.putln('PyErr_Format(%s, "%s", %s); %s ' % (
                self.exception_type_cname,
                StringEncoding.escape_byte_string(
                    self.exception_message.encode('UTF-8')),
                ', '.join([ '"%s"' % escape(str(arg).encode('UTF-8'))
                            for arg in self.exception_format_args ]),
                code.error_goto(self.pos)))
        else:
            code.putln('PyErr_SetString(%s, "%s"); %s ' % (
                self.exception_type_cname,
                escape(self.exception_message.encode('UTF-8')),
                code.error_goto(self.pos)))
9079 9080 9081 9082 9083 9084 9085 9086
        code.putln("}")

    def generate_post_assignment_code(self, code):
        self.arg.generate_post_assignment_code(code)

    def free_temps(self, code):
        self.arg.free_temps(code)

9087

William Stein's avatar
William Stein committed
9088 9089 9090
class CoerceToPyTypeNode(CoercionNode):
    #  This node is used to convert a C data type
    #  to a Python object.
9091

9092
    type = py_object_type
Robert Bradshaw's avatar
Robert Bradshaw committed
9093
    is_temp = 1
William Stein's avatar
William Stein committed
9094

9095
    def __init__(self, arg, env, type=py_object_type):
9096
        if not arg.type.create_to_py_utility_code(env):
9097 9098 9099 9100 9101 9102 9103
            error(arg.pos, "Cannot convert '%s' to Python object" % arg.type)
        elif arg.type.is_complex:
            # special case: complex coercion is so complex that it
            # uses a macro ("__pyx_PyComplex_FromComplex()"), for
            # which the argument must be simple
            arg = arg.coerce_to_simple(env)
        CoercionNode.__init__(self, arg)
9104 9105 9106 9107
        if type is py_object_type:
            # be specific about some known types
            if arg.type.is_string:
                self.type = bytes_type
Stefan Behnel's avatar
Stefan Behnel committed
9108
            elif arg.type.is_unicode_char:
9109 9110 9111 9112 9113 9114
                self.type = unicode_type
            elif arg.type.is_complex:
                self.type = Builtin.complex_type
        else:
            # FIXME: check that the target type and the resulting type are compatible
            pass
9115

9116 9117 9118 9119
        if arg.type.is_memoryviewslice:
            # Register utility codes at this point
            arg.type.get_to_py_function(env, arg)

9120 9121
        self.env = env

9122
    gil_message = "Converting to Python object"
9123

9124 9125 9126 9127
    def may_be_none(self):
        # FIXME: is this always safe?
        return False

9128
    def coerce_to_boolean(self, env):
9129 9130 9131 9132 9133 9134
        arg_type = self.arg.type
        if (arg_type == PyrexTypes.c_bint_type or
            (arg_type.is_pyobject and arg_type.name == 'bool')):
            return self.arg.coerce_to_temp(env)
        else:
            return CoerceToBooleanNode(self, env)
9135

9136 9137 9138 9139 9140 9141
    def coerce_to_integer(self, env):
        # If not already some C integer type, coerce to longint.
        if self.arg.type.is_int:
            return self.arg
        else:
            return self.arg.coerce_to(PyrexTypes.c_long_type, env)
9142

9143 9144 9145 9146
    def analyse_types(self, env):
        # The arg is always already analysed
        pass

William Stein's avatar
William Stein committed
9147
    def generate_result_code(self, code):
9148
        if self.arg.type.is_memoryviewslice:
9149
            funccall = self.arg.type.get_to_py_function(self.env, self.arg)
9150 9151 9152 9153 9154
        else:
            funccall = "%s(%s)" % (self.arg.type.to_py_function,
                                   self.arg.result())

        code.putln('%s = %s; %s' % (
9155
            self.result(),
9156
            funccall,
9157
            code.error_goto_if_null(self.result(), self.pos)))
9158

9159
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
9160 9161


9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201
class CoerceIntToBytesNode(CoerceToPyTypeNode):
    #  This node is used to convert a C int type to a Python bytes
    #  object.

    is_temp = 1

    def __init__(self, arg, env):
        arg = arg.coerce_to_simple(env)
        CoercionNode.__init__(self, arg)
        self.type = Builtin.bytes_type

    def generate_result_code(self, code):
        arg = self.arg
        arg_result = arg.result()
        if arg.type not in (PyrexTypes.c_char_type,
                            PyrexTypes.c_uchar_type,
                            PyrexTypes.c_schar_type):
            if arg.type.signed:
                code.putln("if ((%s < 0) || (%s > 255)) {" % (
                    arg_result, arg_result))
            else:
                code.putln("if (%s > 255) {" % arg_result)
            code.putln('PyErr_Format(PyExc_OverflowError, '
                       '"value too large to pack into a byte"); %s' % (
                           code.error_goto(self.pos)))
            code.putln('}')
        temp = None
        if arg.type is not PyrexTypes.c_char_type:
            temp = code.funcstate.allocate_temp(PyrexTypes.c_char_type, manage_ref=False)
            code.putln("%s = (char)%s;" % (temp, arg_result))
            arg_result = temp
        code.putln('%s = PyBytes_FromStringAndSize(&%s, 1); %s' % (
            self.result(),
            arg_result,
            code.error_goto_if_null(self.result(), self.pos)))
        if temp is not None:
            code.funcstate.release_temp(temp)
        code.put_gotref(self.py_result())


William Stein's avatar
William Stein committed
9202 9203 9204 9205 9206 9207 9208 9209
class CoerceFromPyTypeNode(CoercionNode):
    #  This node is used to convert a Python object
    #  to a C data type.

    def __init__(self, result_type, arg, env):
        CoercionNode.__init__(self, arg)
        self.type = result_type
        self.is_temp = 1
9210
        if not result_type.create_from_py_utility_code(env):
William Stein's avatar
William Stein committed
9211
            error(arg.pos,
Craig Citro's avatar
Craig Citro committed
9212
                  "Cannot convert Python object to '%s'" % result_type)
9213 9214 9215 9216 9217 9218 9219 9220
        if self.type.is_string:
            if self.arg.is_ephemeral():
                error(arg.pos,
                      "Obtaining char* from temporary Python value")
            elif self.arg.is_name and self.arg.entry and self.arg.entry.is_pyglobal:
                warning(arg.pos,
                        "Obtaining char* from externally modifiable global Python value",
                        level=1)
9221

9222 9223 9224 9225
    def analyse_types(self, env):
        # The arg is always already analysed
        pass

William Stein's avatar
William Stein committed
9226 9227
    def generate_result_code(self, code):
        function = self.type.from_py_function
9228 9229 9230 9231
        operand = self.arg.py_result()
        rhs = "%s(%s)" % (function, operand)
        if self.type.is_enum:
            rhs = typecast(self.type, c_long_type, rhs)
Robert Bradshaw's avatar
Robert Bradshaw committed
9232
        code.putln('%s = %s; %s' % (
9233
            self.result(),
9234
            rhs,
9235
            code.error_goto_if(self.type.error_condition(self.result()), self.pos)))
9236
        if self.type.is_pyobject:
9237
            code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
9238

9239 9240 9241
    def nogil_check(self, env):
        error(self.pos, "Coercion from Python not allowed without the GIL")

William Stein's avatar
William Stein committed
9242 9243 9244 9245

class CoerceToBooleanNode(CoercionNode):
    #  This node is used when a result needs to be used
    #  in a boolean context.
9246

9247
    type = PyrexTypes.c_bint_type
9248 9249 9250 9251

    _special_builtins = {
        Builtin.list_type    : 'PyList_GET_SIZE',
        Builtin.tuple_type   : 'PyTuple_GET_SIZE',
9252
        Builtin.bytes_type   : 'PyBytes_GET_SIZE',
9253 9254 9255
        Builtin.unicode_type : 'PyUnicode_GET_SIZE',
        }

William Stein's avatar
William Stein committed
9256 9257 9258 9259
    def __init__(self, arg, env):
        CoercionNode.__init__(self, arg)
        if arg.type.is_pyobject:
            self.is_temp = 1
9260

9261
    def nogil_check(self, env):
9262
        if self.arg.type.is_pyobject and self._special_builtins.get(self.arg.type) is None:
9263
            self.gil_error()
9264

9265
    gil_message = "Truth-testing Python object"
9266

William Stein's avatar
William Stein committed
9267 9268 9269
    def check_const(self):
        if self.is_temp:
            self.not_const()
9270 9271
            return False
        return self.arg.check_const()
9272

William Stein's avatar
William Stein committed
9273
    def calculate_result_code(self):
9274
        return "(%s != 0)" % self.arg.result()
William Stein's avatar
William Stein committed
9275 9276

    def generate_result_code(self, code):
9277 9278 9279 9280
        if not self.is_temp:
            return
        test_func = self._special_builtins.get(self.arg.type)
        if test_func is not None:
Stefan Behnel's avatar
Stefan Behnel committed
9281
            code.putln("%s = (%s != Py_None) && (%s(%s) != 0);" % (
9282 9283 9284 9285 9286
                       self.result(),
                       self.arg.py_result(),
                       test_func,
                       self.arg.py_result()))
        else:
William Stein's avatar
William Stein committed
9287
            code.putln(
9288
                "%s = __Pyx_PyObject_IsTrue(%s); %s" % (
9289 9290
                    self.result(),
                    self.arg.py_result(),
9291
                    code.error_goto_if_neg(self.result(), self.pos)))
William Stein's avatar
William Stein committed
9292

9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303
class CoerceToComplexNode(CoercionNode):

    def __init__(self, arg, dst_type, env):
        if arg.type.is_complex:
            arg = arg.coerce_to_simple(env)
        self.type = dst_type
        CoercionNode.__init__(self, arg)
        dst_type.create_declaration_utility_code(env)

    def calculate_result_code(self):
        if self.arg.type.is_complex:
9304 9305
            real_part = "__Pyx_CREAL(%s)" % self.arg.result()
            imag_part = "__Pyx_CIMAG(%s)" % self.arg.result()
9306 9307 9308 9309 9310 9311 9312
        else:
            real_part = self.arg.result()
            imag_part = "0"
        return "%s(%s, %s)" % (
                self.type.from_parts,
                real_part,
                imag_part)
9313

9314 9315
    def generate_result_code(self, code):
        pass
William Stein's avatar
William Stein committed
9316 9317 9318 9319 9320 9321 9322 9323 9324

class CoerceToTempNode(CoercionNode):
    #  This node is used to force the result of another node
    #  to be stored in a temporary. It is only used if the
    #  argument node's result is not already in a temporary.

    def __init__(self, arg, env):
        CoercionNode.__init__(self, arg)
        self.type = self.arg.type
9325
        self.constant_result = self.arg.constant_result
William Stein's avatar
William Stein committed
9326 9327 9328
        self.is_temp = 1
        if self.type.is_pyobject:
            self.result_ctype = py_object_type
9329 9330 9331

    gil_message = "Creating temporary Python reference"

9332 9333 9334
    def analyse_types(self, env):
        # The arg is always already analysed
        pass
9335

9336 9337
    def coerce_to_boolean(self, env):
        self.arg = self.arg.coerce_to_boolean(env)
9338 9339
        if self.arg.is_simple():
            return self.arg
9340 9341 9342
        self.type = self.arg.type
        self.result_ctype = self.type
        return self
9343

William Stein's avatar
William Stein committed
9344 9345 9346 9347
    def generate_result_code(self, code):
        #self.arg.generate_evaluation_code(code) # Already done
        # by generic generate_subexpr_evaluation_code!
        code.putln("%s = %s;" % (
9348
            self.result(), self.arg.result_as(self.ctype())))
9349 9350 9351 9352 9353 9354
        if self.use_managed_ref:
            if self.type.is_pyobject:
                code.put_incref(self.result(), self.ctype())
            elif self.type.is_memoryviewslice:
                code.put_incref_memoryviewslice(self.result(),
                                                not self.in_nogil_context)
William Stein's avatar
William Stein committed
9355

9356 9357 9358 9359 9360 9361 9362 9363 9364
class ProxyNode(CoercionNode):
    """
    A node that should not be replaced by transforms or other means,
    and hence can be useful to wrap the argument to a clone node

    MyNode    -> ProxyNode -> ArgNode
    CloneNode -^
    """

9365 9366
    nogil_check = None

9367 9368
    def __init__(self, arg):
        super(ProxyNode, self).__init__(arg)
9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380
        self._proxy_type()

    def analyse_expressions(self, env):
        self.arg.analyse_expressions(env)
        self._proxy_type()

    def _proxy_type(self):
        if hasattr(self.arg, 'type'):
            self.type = self.arg.type
            self.result_ctype = self.arg.result_ctype
        if hasattr(self.arg, 'entry'):
            self.entry = self.arg.entry
9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404

    def generate_result_code(self, code):
        self.arg.generate_result_code(code)

    def result(self):
        return self.arg.result()

    def is_simple(self):
        return self.arg.is_simple()

    def may_be_none(self):
        return self.arg.may_be_none()

    def generate_evaluation_code(self, code):
        self.arg.generate_evaluation_code(code)

    def generate_result_code(self, code):
        self.arg.generate_result_code(code)

    def generate_disposal_code(self, code):
        self.arg.generate_disposal_code(code)

    def free_temps(self, code):
        self.arg.free_temps(code)
William Stein's avatar
William Stein committed
9405 9406 9407 9408 9409 9410

class CloneNode(CoercionNode):
    #  This node is employed when the result of another node needs
    #  to be used multiple times. The argument node's result must
    #  be in a temporary. This node "borrows" the result from the
    #  argument node, and does not generate any evaluation or
9411
    #  disposal code for it. The original owner of the argument
William Stein's avatar
William Stein committed
9412
    #  node is responsible for doing those things.
9413

William Stein's avatar
William Stein committed
9414
    subexprs = [] # Arg is not considered a subexpr
9415
    nogil_check = None
9416

William Stein's avatar
William Stein committed
9417 9418
    def __init__(self, arg):
        CoercionNode.__init__(self, arg)
9419 9420 9421 9422 9423
        if hasattr(arg, 'type'):
            self.type = arg.type
            self.result_ctype = arg.result_ctype
        if hasattr(arg, 'entry'):
            self.entry = arg.entry
9424

9425
    def result(self):
9426
        return self.arg.result()
9427

9428 9429 9430
    def may_be_none(self):
        return self.arg.may_be_none()

Robert Bradshaw's avatar
Robert Bradshaw committed
9431 9432
    def type_dependencies(self, env):
        return self.arg.type_dependencies(env)
9433

9434 9435
    def infer_type(self, env):
        return self.arg.infer_type(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
9436

Robert Bradshaw's avatar
Robert Bradshaw committed
9437 9438 9439 9440
    def analyse_types(self, env):
        self.type = self.arg.type
        self.result_ctype = self.arg.result_ctype
        self.is_temp = 1
9441 9442
        if hasattr(self.arg, 'entry'):
            self.entry = self.arg.entry
9443

9444 9445 9446
    def is_simple(self):
        return True # result is always in a temp (or a name)

William Stein's avatar
William Stein committed
9447 9448 9449 9450 9451
    def generate_evaluation_code(self, code):
        pass

    def generate_result_code(self, code):
        pass
9452

9453
    def generate_disposal_code(self, code):
9454
        pass
9455

9456 9457
    def free_temps(self, code):
        pass
9458

9459

Stefan Behnel's avatar
Stefan Behnel committed
9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471
class CMethodSelfCloneNode(CloneNode):
    # Special CloneNode for the self argument of builtin C methods
    # that accepts subtypes of the builtin type.  This is safe only
    # for 'final' subtypes, as subtypes of the declared type may
    # override the C method.

    def coerce_to(self, dst_type, env):
        if dst_type.is_builtin_type and self.type.subtype_of(dst_type):
            return self
        return CloneNode.coerce_to(self, dst_type, env)


9472 9473
class ModuleRefNode(ExprNode):
    # Simple returns the module object
9474

9475 9476 9477
    type = py_object_type
    is_temp = False
    subexprs = []
9478

9479 9480 9481
    def analyse_types(self, env):
        pass

9482 9483 9484
    def may_be_none(self):
        return False

9485 9486 9487 9488 9489 9490 9491 9492
    def calculate_result_code(self):
        return Naming.module_cname

    def generate_result_code(self, code):
        pass

class DocstringRefNode(ExprNode):
    # Extracts the docstring of the body element
9493

9494 9495 9496
    subexprs = ['body']
    type = py_object_type
    is_temp = True
9497

9498 9499 9500 9501 9502 9503 9504 9505 9506
    def __init__(self, pos, body):
        ExprNode.__init__(self, pos)
        assert body.type.is_pyobject
        self.body = body

    def analyse_types(self, env):
        pass

    def generate_result_code(self, code):
9507 9508 9509
        code.putln('%s = __Pyx_GetAttrString(%s, "__doc__"); %s' % (
            self.result(), self.body.result(),
            code.error_goto_if_null(self.result(), self.pos)))
9510 9511 9512 9513
        code.put_gotref(self.result())



William Stein's avatar
William Stein committed
9514 9515 9516 9517 9518 9519
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

9520 9521
get_name_interned_utility_code = UtilityCode(
proto = """
9522
static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/
9523 9524
""",
impl = """
William Stein's avatar
William Stein committed
9525 9526 9527
static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) {
    PyObject *result;
    result = PyObject_GetAttr(dict, name);
9528 9529 9530 9531 9532 9533 9534 9535 9536
    if (!result) {
        if (dict != %(BUILTINS)s) {
            PyErr_Clear();
            result = PyObject_GetAttr(%(BUILTINS)s, name);
        }
        if (!result) {
            PyErr_SetObject(PyExc_NameError, name);
        }
    }
William Stein's avatar
William Stein committed
9537 9538
    return result;
}
9539
""" % {'BUILTINS' : Naming.builtins_cname})
William Stein's avatar
William Stein committed
9540 9541 9542

#------------------------------------------------------------------------------------

9543 9544
import_utility_code = UtilityCode(
proto = """
Haoyu Bai's avatar
Haoyu Bai committed
9545
static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level); /*proto*/
9546 9547
""",
impl = """
Haoyu Bai's avatar
Haoyu Bai committed
9548
static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level) {
9549
    PyObject *py_import = 0;
William Stein's avatar
William Stein committed
9550 9551 9552 9553 9554
    PyObject *empty_list = 0;
    PyObject *module = 0;
    PyObject *global_dict = 0;
    PyObject *empty_dict = 0;
    PyObject *list;
9555 9556
    py_import = __Pyx_GetAttrString(%(BUILTINS)s, "__import__");
    if (!py_import)
William Stein's avatar
William Stein committed
9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571
        goto bad;
    if (from_list)
        list = from_list;
    else {
        empty_list = PyList_New(0);
        if (!empty_list)
            goto bad;
        list = empty_list;
    }
    global_dict = PyModule_GetDict(%(GLOBALS)s);
    if (!global_dict)
        goto bad;
    empty_dict = PyDict_New();
    if (!empty_dict)
        goto bad;
Haoyu Bai's avatar
Haoyu Bai committed
9572 9573
    #if PY_VERSION_HEX >= 0x02050000
    {
9574
        #if PY_MAJOR_VERSION >= 3
9575
        if (level == -1) {
9576 9577 9578 9579
            if (strchr(__Pyx_MODULE_NAME, '.')) {
                /* try package relative import first */
                PyObject *py_level = PyInt_FromLong(1);
                if (!py_level)
9580
                    goto bad;
9581 9582 9583 9584 9585 9586 9587 9588
                module = PyObject_CallFunctionObjArgs(py_import,
                    name, global_dict, empty_dict, list, py_level, NULL);
                Py_DECREF(py_level);
                if (!module) {
                    if (!PyErr_ExceptionMatches(PyExc_ImportError))
                        goto bad;
                    PyErr_Clear();
                }
9589
            }
9590
            level = 0; /* try absolute import on failure */
9591 9592 9593 9594 9595 9596 9597 9598 9599 9600
        }
        #endif
        if (!module) {
            PyObject *py_level = PyInt_FromLong(level);
            if (!py_level)
                goto bad;
            module = PyObject_CallFunctionObjArgs(py_import,
                name, global_dict, empty_dict, list, py_level, NULL);
            Py_DECREF(py_level);
        }
Haoyu Bai's avatar
Haoyu Bai committed
9601 9602 9603 9604 9605 9606
    }
    #else
    if (level>0) {
        PyErr_SetString(PyExc_RuntimeError, "Relative import is not supported for Python <=2.4.");
        goto bad;
    }
9607
    module = PyObject_CallFunctionObjArgs(py_import,
9608
        name, global_dict, empty_dict, list, NULL);
Haoyu Bai's avatar
Haoyu Bai committed
9609
    #endif
William Stein's avatar
William Stein committed
9610 9611
bad:
    Py_XDECREF(empty_list);
9612
    Py_XDECREF(py_import);
William Stein's avatar
William Stein committed
9613 9614 9615 9616 9617 9618
    Py_XDECREF(empty_dict);
    return module;
}
""" % {
    "BUILTINS": Naming.builtins_cname,
    "GLOBALS":  Naming.module_cname,
9619
})
William Stein's avatar
William Stein committed
9620 9621 9622

#------------------------------------------------------------------------------------

9623 9624
get_exception_utility_code = UtilityCode(
proto = """
9625
static PyObject *__Pyx_GetExcValue(void); /*proto*/
9626 9627
""",
impl = """
William Stein's avatar
William Stein committed
9628 9629
static PyObject *__Pyx_GetExcValue(void) {
    PyObject *type = 0, *value = 0, *tb = 0;
9630
    PyObject *tmp_type, *tmp_value, *tmp_tb;
William Stein's avatar
William Stein committed
9631 9632 9633 9634 9635 9636 9637 9638 9639 9640
    PyObject *result = 0;
    PyThreadState *tstate = PyThreadState_Get();
    PyErr_Fetch(&type, &value, &tb);
    PyErr_NormalizeException(&type, &value, &tb);
    if (PyErr_Occurred())
        goto bad;
    if (!value) {
        value = Py_None;
        Py_INCREF(value);
    }
9641 9642 9643
    tmp_type = tstate->exc_type;
    tmp_value = tstate->exc_value;
    tmp_tb = tstate->exc_traceback;
William Stein's avatar
William Stein committed
9644 9645 9646
    tstate->exc_type = type;
    tstate->exc_value = value;
    tstate->exc_traceback = tb;
9647 9648 9649 9650 9651
    /* Make sure tstate is in a consistent state when we XDECREF
    these objects (XDECREF may run arbitrary code). */
    Py_XDECREF(tmp_type);
    Py_XDECREF(tmp_value);
    Py_XDECREF(tmp_tb);
William Stein's avatar
William Stein committed
9652 9653 9654 9655 9656 9657 9658 9659 9660 9661 9662
    result = value;
    Py_XINCREF(result);
    type = 0;
    value = 0;
    tb = 0;
bad:
    Py_XDECREF(type);
    Py_XDECREF(value);
    Py_XDECREF(tb);
    return result;
}
9663
""")
William Stein's avatar
William Stein committed
9664 9665 9666

#------------------------------------------------------------------------------------

9667 9668
type_test_utility_code = UtilityCode(
proto = """
9669
static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /*proto*/
9670 9671
""",
impl = """
9672
static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
9673
    if (unlikely(!type)) {
William Stein's avatar
William Stein committed
9674 9675 9676
        PyErr_Format(PyExc_SystemError, "Missing type object");
        return 0;
    }
9677
    if (likely(PyObject_TypeCheck(obj, type)))
William Stein's avatar
William Stein committed
9678
        return 1;
9679 9680
    PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s",
                 Py_TYPE(obj)->tp_name, type->tp_name);
William Stein's avatar
William Stein committed
9681 9682
    return 0;
}
9683
""")
William Stein's avatar
William Stein committed
9684 9685 9686

#------------------------------------------------------------------------------------

9687 9688 9689 9690 9691 9692 9693 9694 9695 9696 9697
find_py2_metaclass_utility_code = UtilityCode(
proto = '''
static PyObject *__Pyx_FindPy2Metaclass(PyObject *bases); /*proto*/
''',
impl = '''
static PyObject *__Pyx_FindPy2Metaclass(PyObject *bases) {
    PyObject *metaclass;
    /* Default metaclass */
#if PY_MAJOR_VERSION < 3
    if (PyTuple_Check(bases) && PyTuple_GET_SIZE(bases) > 0) {
        PyObject *base = PyTuple_GET_ITEM(bases, 0);
9698
        metaclass = PyObject_GetAttrString(base, (char *)"__class__");
9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718
        if (!metaclass) {
            PyErr_Clear();
            metaclass = (PyObject*) Py_TYPE(base);
        }
    } else {
        metaclass = (PyObject *) &PyClass_Type;
    }
#else
    if (PyTuple_Check(bases) && PyTuple_GET_SIZE(bases) > 0) {
        PyObject *base = PyTuple_GET_ITEM(bases, 0);
        metaclass = (PyObject*) Py_TYPE(base);
    } else {
        metaclass = (PyObject *) &PyType_Type;
    }
#endif
    Py_INCREF(metaclass);
    return metaclass;
}
''')

9719 9720
create_class_utility_code = UtilityCode(
proto = """
9721
static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name,
9722 9723 9724 9725 9726
                                   PyObject *modname); /*proto*/
""",
impl = """
static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name,
                                   PyObject *modname) {
9727 9728
    PyObject *result;
    PyObject *metaclass;
9729 9730 9731 9732 9733 9734

    if (PyDict_SetItemString(dict, "__module__", modname) < 0)
        return NULL;

    /* Python2 __metaclass__ */
    metaclass = PyDict_GetItemString(dict, "__metaclass__");
9735 9736 9737 9738
    if (metaclass) {
        Py_INCREF(metaclass);
    } else {
        metaclass = __Pyx_FindPy2Metaclass(bases);
9739 9740 9741 9742 9743
    }
    result = PyObject_CallFunctionObjArgs(metaclass, name, bases, dict, NULL);
    Py_DECREF(metaclass);
    return result;
}
9744 9745
""",
requires = [find_py2_metaclass_utility_code])
9746 9747 9748 9749 9750

#------------------------------------------------------------------------------------

create_py3class_utility_code = UtilityCode(
proto = """
Stefan Behnel's avatar
Stefan Behnel committed
9751 9752 9753
static PyObject *__Pyx_Py3MetaclassGet(PyObject *bases, PyObject *mkw); /*proto*/
static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *mkw, PyObject *modname, PyObject *doc); /*proto*/
static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw); /*proto*/
9754 9755
""",
impl = """
Stefan Behnel's avatar
Stefan Behnel committed
9756
PyObject *__Pyx_Py3MetaclassGet(PyObject *bases, PyObject *mkw) {
9757
    PyObject *metaclass = PyDict_GetItemString(mkw, "metaclass");
9758 9759 9760 9761 9762 9763 9764 9765
    if (metaclass) {
        Py_INCREF(metaclass);
        if (PyDict_DelItemString(mkw, "metaclass") < 0) {
            Py_DECREF(metaclass);
            return NULL;
        }
        return metaclass;
    }
9766
    return __Pyx_FindPy2Metaclass(bases);
9767 9768
}

Stefan Behnel's avatar
Stefan Behnel committed
9769 9770
PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *mkw,
                                    PyObject *modname, PyObject *doc) {
9771 9772 9773
    PyObject *prep;
    PyObject *pargs;
    PyObject *ns;
9774
    PyObject *str;
9775

9776
    prep = PyObject_GetAttrString(metaclass, (char *)"__prepare__");
Stefan Behnel's avatar
Stefan Behnel committed
9777
    if (!prep) {
9778
        if (!PyErr_ExceptionMatches(PyExc_AttributeError))
9779
            return NULL;
9780
        PyErr_Clear();
9781
        return PyDict_New();
9782
    }
Stefan Behnel's avatar
Stefan Behnel committed
9783 9784
    pargs = PyTuple_New(2);
    if (!pargs) {
9785
        Py_DECREF(prep);
9786
        return NULL;
9787
    }
9788

Stefan Behnel's avatar
Stefan Behnel committed
9789 9790 9791 9792
    Py_INCREF(name);
    Py_INCREF(bases);
    PyTuple_SET_ITEM(pargs, 0, name);
    PyTuple_SET_ITEM(pargs, 1, bases);
9793

Stefan Behnel's avatar
Stefan Behnel committed
9794
    ns = PyObject_Call(prep, pargs, mkw);
9795

9796
    Py_DECREF(prep);
9797 9798
    Py_DECREF(pargs);

9799
    if (ns == NULL)
9800 9801 9802 9803
        return NULL;

    /* Required here to emulate assignment order */
    /* XXX: use consts here */
Stefan Behnel's avatar
Stefan Behnel committed
9804 9805 9806
    #if PY_MAJOR_VERSION >= 3
    str = PyUnicode_FromString("__module__");
    #else
9807
    str = PyString_FromString("__module__");
Stefan Behnel's avatar
Stefan Behnel committed
9808
    #endif
9809
    if (!str) {
9810
        Py_DECREF(ns);
9811
        return NULL;
9812
    }
Vitja Makarov's avatar
Vitja Makarov committed
9813

9814 9815 9816
    if (PyObject_SetItem(ns, str, modname) < 0) {
        Py_DECREF(ns);
        Py_DECREF(str);
9817
        return NULL;
9818 9819 9820
    }
    Py_DECREF(str);
    if (doc) {
Stefan Behnel's avatar
Stefan Behnel committed
9821 9822 9823
        #if PY_MAJOR_VERSION >= 3
        str = PyUnicode_FromString("__doc__");
        #else
9824
        str = PyString_FromString("__doc__");
Stefan Behnel's avatar
Stefan Behnel committed
9825
        #endif
9826 9827
        if (!str) {
            Py_DECREF(ns);
9828
            return NULL;
Vitja Makarov's avatar
Vitja Makarov committed
9829
        }
9830 9831 9832 9833
        if (PyObject_SetItem(ns, str, doc) < 0) {
            Py_DECREF(ns);
            Py_DECREF(str);
            return NULL;
Stefan Behnel's avatar
Stefan Behnel committed
9834
        }
9835
        Py_DECREF(str);
Vitja Makarov's avatar
Vitja Makarov committed
9836
    }
9837 9838 9839
    return ns;
}

Stefan Behnel's avatar
Stefan Behnel committed
9840
PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw) {
9841
    PyObject *result;
9842
    PyObject *margs = PyTuple_New(3);
9843 9844
    if (!margs)
        return NULL;
9845 9846 9847 9848 9849 9850
    Py_INCREF(name);
    Py_INCREF(bases);
    Py_INCREF(dict);
    PyTuple_SET_ITEM(margs, 0, name);
    PyTuple_SET_ITEM(margs, 1, bases);
    PyTuple_SET_ITEM(margs, 2, dict);
Stefan Behnel's avatar
Stefan Behnel committed
9851
    result = PyObject_Call(metaclass, margs, mkw);
9852
    Py_DECREF(margs);
William Stein's avatar
William Stein committed
9853 9854
    return result;
}
9855 9856
""",
requires = [find_py2_metaclass_utility_code])
William Stein's avatar
William Stein committed
9857 9858

#------------------------------------------------------------------------------------
Robert Bradshaw's avatar
Robert Bradshaw committed
9859

9860 9861
cpp_exception_utility_code = UtilityCode(
proto = """
9862 9863
#ifndef __Pyx_CppExn2PyErr
static void __Pyx_CppExn2PyErr() {
9864 9865
  // Catch a handful of different errors here and turn them into the
  // equivalent Python errors.
Robert Bradshaw's avatar
Robert Bradshaw committed
9866 9867 9868 9869 9870
  try {
    if (PyErr_Occurred())
      ; // let the latest Python exn pass through and ignore the current one
    else
      throw;
9871 9872
  } catch (const std::bad_alloc& exn) {
    PyErr_SetString(PyExc_MemoryError, exn.what());
9873 9874
  } catch (const std::bad_cast& exn) {
    PyErr_SetString(PyExc_TypeError, exn.what());
9875 9876
  } catch (const std::domain_error& exn) {
    PyErr_SetString(PyExc_ValueError, exn.what());
9877 9878
  } catch (const std::invalid_argument& exn) {
    PyErr_SetString(PyExc_ValueError, exn.what());
9879 9880 9881 9882
  } catch (const std::ios_base::failure& exn) {
    // Unfortunately, in standard C++ we have no way of distinguishing EOF
    // from other errors here; be careful with the exception mask
    PyErr_SetString(PyExc_IOError, exn.what());
Robert Bradshaw's avatar
Robert Bradshaw committed
9883
  } catch (const std::out_of_range& exn) {
9884
    // Change out_of_range to IndexError
Robert Bradshaw's avatar
Robert Bradshaw committed
9885
    PyErr_SetString(PyExc_IndexError, exn.what());
9886 9887 9888 9889 9890 9891
  } catch (const std::overflow_error& exn) {
    PyErr_SetString(PyExc_OverflowError, exn.what());
  } catch (const std::range_error& exn) {
    PyErr_SetString(PyExc_ArithmeticError, exn.what());
  } catch (const std::underflow_error& exn) {
    PyErr_SetString(PyExc_ArithmeticError, exn.what());
Robert Bradshaw's avatar
Robert Bradshaw committed
9892 9893 9894 9895 9896 9897 9898 9899
  } catch (const std::exception& exn) {
    PyErr_SetString(PyExc_RuntimeError, exn.what());
  }
  catch (...)
  {
    PyErr_SetString(PyExc_RuntimeError, "Unknown exception");
  }
}
9900
#endif
9901 9902 9903
""",
impl = ""
)
Robert Bradshaw's avatar
Robert Bradshaw committed
9904

9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923
pyerr_occurred_withgil_utility_code= UtilityCode(
proto = """
static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */
""",
impl = """
static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) {
  int err;
  #ifdef WITH_THREAD
  PyGILState_STATE _save = PyGILState_Ensure();
  #endif
  err = !!PyErr_Occurred();
  #ifdef WITH_THREAD
  PyGILState_Release(_save);
  #endif
  return err;
}
"""
)

Robert Bradshaw's avatar
Robert Bradshaw committed
9924
#------------------------------------------------------------------------------------
Robert Bradshaw's avatar
Robert Bradshaw committed
9925

9926 9927 9928
raise_noneattr_error_utility_code = UtilityCode.load_cached("RaiseNoneAttrError", "ObjectHandling.c")
raise_noneindex_error_utility_code = UtilityCode.load_cached("RaiseNoneIndexingError", "ObjectHandling.c")
raise_none_iter_error_utility_code = UtilityCode.load_cached("RaiseNoneIterError", "ObjectHandling.c")
9929

9930 9931 9932 9933 9934 9935 9936 9937 9938 9939
raise_noneindex_memview_error_utility_code = UtilityCode(
    proto = """
static CYTHON_INLINE void __Pyx_RaiseNoneMemviewIndexingError(void);
""",
    impl = '''
static CYTHON_INLINE void __Pyx_RaiseNoneMemviewIndexingError(void) {
    PyErr_SetString(PyExc_TypeError, "Cannot index None memoryview slice");
}
''')

9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959
raise_unbound_local_error_utility_code = UtilityCode(
proto = """
static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);
""",
impl = """
static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) {
    PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname);
}
""")

raise_closure_name_error_utility_code = UtilityCode(
proto = """
static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);
""",
impl = """
static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) {
    PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname);
}
""")

9960 9961 9962 9963 9964 9965 9966 9967 9968 9969 9970
# Don't inline the function, it should really never be called in production
raise_unbound_memoryview_utility_code_nogil = UtilityCode(
proto = """
static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname);
""",
impl = """
static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) {
    #ifdef WITH_THREAD
    PyGILState_STATE gilstate = PyGILState_Ensure();
    #endif
    __Pyx_RaiseUnboundLocalError(varname);
9971
    #ifdef WITH_THREAD
9972 9973 9974 9975 9976 9977
    PyGILState_Release(gilstate);
    #endif
}
""",
requires = [raise_unbound_local_error_utility_code])

9978 9979 9980 9981
#------------------------------------------------------------------------------------

getitem_dict_utility_code = UtilityCode(
proto = """
9982
#if PY_MAJOR_VERSION >= 3
9983
static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) {
9984
    PyObject *value;
9985
    if (unlikely(d == Py_None)) {
9986 9987 9988
        __Pyx_RaiseNoneIndexingError();
        return NULL;
    }
9989 9990 9991
    value = PyDict_GetItemWithError(d, key);
    if (unlikely(!value)) {
        if (!PyErr_Occurred())
9992
            PyErr_SetObject(PyExc_KeyError, key);
9993
        return NULL;
9994
    }
9995 9996
    Py_INCREF(value);
    return value;
9997
}
9998 9999 10000
#else
    #define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key)
#endif
10001
""",
10002 10003 10004
requires = [raise_noneindex_error_utility_code])

#------------------------------------------------------------------------------------
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10005

10006 10007 10008 10009
getitem_int_pyunicode_utility_code = UtilityCode(
proto = '''
#define __Pyx_GetItemInt_Unicode(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \\
                                               __Pyx_GetItemInt_Unicode_Fast(o, i) : \\
Stefan Behnel's avatar
Stefan Behnel committed
10010
                                               __Pyx_GetItemInt_Unicode_Generic(o, to_py_func(i)))
10011

10012
static CYTHON_INLINE Py_UCS4 __Pyx_GetItemInt_Unicode_Fast(PyObject* ustring, Py_ssize_t i) {
10013 10014 10015
#if CYTHON_PEP393_ENABLED
    if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return (Py_UCS4)-1;
#endif
10016 10017 10018 10019 10020
    const Py_ssize_t length = __Pyx_PyUnicode_GET_LENGTH(ustring);
    if (likely((0 <= i) & (i < length))) {
        return __Pyx_PyUnicode_READ_CHAR(ustring, i);
    } else if ((-length <= i) & (i < 0)) {
        return __Pyx_PyUnicode_READ_CHAR(ustring, i + length);
10021 10022
    } else {
        PyErr_SetString(PyExc_IndexError, "string index out of range");
10023
        return (Py_UCS4)-1;
10024 10025 10026
    }
}

10027 10028
static CYTHON_INLINE Py_UCS4 __Pyx_GetItemInt_Unicode_Generic(PyObject* ustring, PyObject* j) {
    Py_UCS4 uchar;
10029
    PyObject *uchar_string;
10030
    if (!j) return (Py_UCS4)-1;
10031
    uchar_string = PyObject_GetItem(ustring, j);
10032
    Py_DECREF(j);
10033
    if (!uchar_string) return (Py_UCS4)-1;
10034 10035 10036 10037 10038 10039
#if CYTHON_PEP393_ENABLED
    if (unlikely(__Pyx_PyUnicode_READY(uchar_string) < 0)) {
        Py_DECREF(uchar_string);
        return (Py_UCS4)-1;
    }
#endif
10040
    uchar = __Pyx_PyUnicode_READ_CHAR(uchar_string, 0);
10041
    Py_DECREF(uchar_string);
10042 10043
    return uchar;
}
10044
''')
10045

10046 10047
getitem_int_utility_code = UtilityCode(
proto = """
10048

10049
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) {
10050
    PyObject *r;
10051
    if (!j) return NULL;
10052 10053 10054 10055
    r = PyObject_GetItem(o, j);
    Py_DECREF(j);
    return r;
}
10056

10057 10058
""" + ''.join([
"""
10059 10060
#define __Pyx_GetItemInt_%(type)s(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \\
                                                    __Pyx_GetItemInt_%(type)s_Fast(o, i) : \\
10061 10062
                                                    __Pyx_GetItemInt_Generic(o, to_py_func(i)))

10063
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_%(type)s_Fast(PyObject *o, Py_ssize_t i) {
10064 10065 10066 10067 10068 10069
    if (likely(o != Py_None)) {
        if (likely((0 <= i) & (i < Py%(type)s_GET_SIZE(o)))) {
            PyObject *r = Py%(type)s_GET_ITEM(o, i);
            Py_INCREF(r);
            return r;
        }
10070 10071
        else if ((-Py%(type)s_GET_SIZE(o) <= i) & (i < 0)) {
            PyObject *r = Py%(type)s_GET_ITEM(o, Py%(type)s_GET_SIZE(o) + i);
10072 10073 10074
            Py_INCREF(r);
            return r;
        }
10075
    }
10076
    return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i));
10077
}
10078 10079
""" % {'type' : type_name} for type_name in ('List', 'Tuple')
]) + """
10080

10081 10082
#define __Pyx_GetItemInt(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \\
                                                    __Pyx_GetItemInt_Fast(o, i) : \\
10083 10084
                                                    __Pyx_GetItemInt_Generic(o, to_py_func(i)))

10085
static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i) {
10086 10087 10088 10089 10090 10091 10092
    if (PyList_CheckExact(o)) {
        Py_ssize_t n = (likely(i >= 0)) ? i : i + PyList_GET_SIZE(o);
        if (likely((n >= 0) & (n < PyList_GET_SIZE(o)))) {
            PyObject *r = PyList_GET_ITEM(o, n);
            Py_INCREF(r);
            return r;
        }
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10093
    }
10094 10095 10096 10097 10098 10099 10100
    else if (PyTuple_CheckExact(o)) {
        Py_ssize_t n = (likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o);
        if (likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) {
            PyObject *r = PyTuple_GET_ITEM(o, n);
            Py_INCREF(r);
            return r;
        }
10101
    }
10102 10103 10104 10105 10106
    else if (likely(i >= 0)) {
        PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence;
        if (likely(m && m->sq_item)) {
            return m->sq_item(o, i);
        }
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10107
    }
10108
    return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i));
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10109 10110
}
""",
10111 10112
impl = """
""")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10113

10114 10115


Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10116 10117
#------------------------------------------------------------------------------------

10118 10119
setitem_int_utility_code = UtilityCode(
proto = """
10120 10121
#define __Pyx_SetItemInt(o, i, v, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \\
                                                    __Pyx_SetItemInt_Fast(o, i, v) : \\
10122 10123
                                                    __Pyx_SetItemInt_Generic(o, to_py_func(i), v))

10124
static CYTHON_INLINE int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) {
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10125
    int r;
10126 10127 10128 10129 10130 10131
    if (!j) return -1;
    r = PyObject_SetItem(o, j, v);
    Py_DECREF(j);
    return r;
}

10132
static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v) {
10133 10134 10135
    if (PyList_CheckExact(o)) {
        Py_ssize_t n = (likely(i >= 0)) ? i : i + PyList_GET_SIZE(o);
        if (likely((n >= 0) & (n < PyList_GET_SIZE(o)))) {
10136
            PyObject* old = PyList_GET_ITEM(o, n);
10137
            Py_INCREF(v);
10138
            PyList_SET_ITEM(o, n, v);
10139 10140 10141
            Py_DECREF(old);
            return 1;
        }
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10142
    }
10143 10144 10145 10146 10147
    else if (likely(i >= 0)) {
        PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence;
        if (likely(m && m->sq_ass_item)) {
            return m->sq_ass_item(o, i, v);
        }
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10148
    }
10149
    return __Pyx_SetItemInt_Generic(o, PyInt_FromSsize_t(i), v);
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
10150 10151
}
""",
10152 10153 10154
impl = """
""")

10155 10156
#------------------------------------------------------------------------------------

10157 10158
delitem_int_utility_code = UtilityCode(
proto = """
10159 10160
#define __Pyx_DelItemInt(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \\
                                                    __Pyx_DelItemInt_Fast(o, i) : \\
10161 10162
                                                    __Pyx_DelItem_Generic(o, to_py_func(i)))

10163
static CYTHON_INLINE int __Pyx_DelItem_Generic(PyObject *o, PyObject *j) {
10164
    int r;
10165 10166 10167 10168 10169 10170
    if (!j) return -1;
    r = PyObject_DelItem(o, j);
    Py_DECREF(j);
    return r;
}

10171
static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i) {
10172 10173 10174 10175 10176
    if (likely(i >= 0)) {
        PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence;
        if (likely(m && m->sq_ass_item)) {
            return m->sq_ass_item(o, i, (PyObject *)NULL);
        }
10177
    }
10178
    return __Pyx_DelItem_Generic(o, PyInt_FromSsize_t(i));
10179 10180 10181 10182 10183 10184 10185
}
""",
impl = """
""")

#------------------------------------------------------------------------------------

10186 10187
raise_too_many_values_to_unpack = UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c")
raise_need_more_values_to_unpack = UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")
10188 10189 10190

#------------------------------------------------------------------------------------

10191
tuple_unpacking_error_code = UtilityCode.load_cached("UnpackTupleError", "ObjectHandling.c")
Robert Bradshaw's avatar
Robert Bradshaw committed
10192

10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204
#------------------------------------------------------------------------------------

# CPython supports calling functions with non-dict kwargs by
# converting them to a dict first

kwargs_call_utility_code = UtilityCode(
proto = """
static PyObject* __Pyx_PyEval_CallObjectWithKeywords(PyObject*, PyObject*, PyObject*); /*proto*/
""",
impl = """
static PyObject* __Pyx_PyEval_CallObjectWithKeywords(PyObject *callable, PyObject *args, PyObject *kwargs) {
    PyObject* result;
10205
    if (likely(PyDict_Check(kwargs))) {
10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216
        return PyEval_CallObjectWithKeywords(callable, args, kwargs);
    } else {
        PyObject* real_dict;
        real_dict = PyObject_CallFunctionObjArgs((PyObject*)&PyDict_Type, kwargs, NULL);
        if (unlikely(!real_dict))
            return NULL;
        result = PyEval_CallObjectWithKeywords(callable, args, real_dict);
        Py_DECREF(real_dict);
        return result; /* may be NULL */
    }
}
10217
""",
10218 10219
)

Robert Bradshaw's avatar
Robert Bradshaw committed
10220 10221 10222 10223 10224

#------------------------------------------------------------------------------------

int_pow_utility_code = UtilityCode(
proto="""
10225
static CYTHON_INLINE %(type)s %(func_name)s(%(type)s, %(type)s); /* proto */
Robert Bradshaw's avatar
Robert Bradshaw committed
10226 10227
""",
impl="""
10228
static CYTHON_INLINE %(type)s %(func_name)s(%(type)s b, %(type)s e) {
Robert Bradshaw's avatar
Robert Bradshaw committed
10229 10230 10231 10232 10233 10234 10235 10236 10237 10238 10239 10240 10241 10242 10243 10244 10245 10246 10247 10248 10249
    %(type)s t = b;
    switch (e) {
        case 3:
            t *= b;
        case 2:
            t *= b;
        case 1:
            return t;
        case 0:
            return 1;
    }
    if (unlikely(e<0)) return 0;
    t = 1;
    while (likely(e)) {
        t *= (b * (e&1)) | ((~e)&1);    /* 1 or b */
        b *= b;
        e >>= 1;
    }
    return t;
}
""")
10250 10251 10252

# ------------------------------ Division ------------------------------------

10253 10254
div_int_utility_code = UtilityCode(
proto="""
10255
static CYTHON_INLINE %(type)s __Pyx_div_%(type_name)s(%(type)s, %(type)s); /* proto */
10256 10257
""",
impl="""
10258
static CYTHON_INLINE %(type)s __Pyx_div_%(type_name)s(%(type)s a, %(type)s b) {
10259 10260 10261 10262 10263
    %(type)s q = a / b;
    %(type)s r = a - q*b;
    q -= ((r != 0) & ((r ^ b) < 0));
    return q;
}
10264 10265
""")

10266
mod_int_utility_code = UtilityCode(
10267
proto="""
10268
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s, %(type)s); /* proto */
10269 10270
""",
impl="""
10271
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
10272 10273 10274
    %(type)s r = a %% b;
    r += ((r != 0) & ((r ^ b) < 0)) * b;
    return r;
10275 10276 10277
}
""")

10278
mod_float_utility_code = UtilityCode(
10279
proto="""
10280
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s, %(type)s); /* proto */
10281 10282
""",
impl="""
10283
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
10284 10285 10286
    %(type)s r = fmod%(math_h_modifier)s(a, b);
    r += ((r != 0) & ((r < 0) ^ (b < 0))) * b;
    return r;
10287 10288
}
""")
Robert Bradshaw's avatar
Robert Bradshaw committed
10289

10290
cdivision_warning_utility_code = UtilityCode(
Robert Bradshaw's avatar
Robert Bradshaw committed
10291
proto="""
10292
static int __Pyx_cdivision_warning(const char *, int); /* proto */
Robert Bradshaw's avatar
Robert Bradshaw committed
10293 10294
""",
impl="""
10295
static int __Pyx_cdivision_warning(const char *filename, int lineno) {
10296
    return PyErr_WarnExplicit(PyExc_RuntimeWarning,
10297
                              "division with oppositely signed operands, C and Python semantics differ",
10298 10299
                              filename,
                              lineno,
10300
                              __Pyx_MODULE_NAME,
10301
                              NULL);
Robert Bradshaw's avatar
Robert Bradshaw committed
10302
}
10303
""")
10304 10305 10306 10307

# from intobject.c
division_overflow_test_code = UtilityCode(
proto="""
Vitja Makarov's avatar
Vitja Makarov committed
10308 10309
#define UNARY_NEG_WOULD_OVERFLOW(x)    \
        (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x)))
10310
""")
Robert Bradshaw's avatar
Robert Bradshaw committed
10311

10312 10313 10314
binding_cfunc_utility_code = TempitaUtilityCode.load(
    "CythonFunction", context=vars(Naming))
fused_function_utility_code = TempitaUtilityCode.load(
10315 10316 10317 10318
        "FusedFunction",
        "CythonFunction.c",
        context=vars(Naming),
        requires=[binding_cfunc_utility_code])
10319 10320 10321 10322
cyfunction_class_cell_utility_code = UtilityCode.load(
    "CyFunctionClassCell",
    "CythonFunction.c",
    requires=[binding_cfunc_utility_code])
10323

10324 10325 10326 10327
generator_utility_code = UtilityCode.load(
    "Generator",
    "Generator.c",
    requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
Stefan Behnel's avatar
Stefan Behnel committed
10328
)