PyrexTypes.py 220 KB
Newer Older
1
#
2
#   Cython/Python language types
3 4
#

5 6
from __future__ import absolute_import

7
import copy
8
import re
9

10 11 12 13 14
try:
    reduce
except NameError:
    from functools import reduce

15
from Cython.Utils import cached_function
16 17 18 19
from .Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode
from . import StringEncoding
from . import Naming

20
from .Errors import error, warning
21

22

23
class BaseType(object):
24
    #
25
    #  Base class for all Cython types including pseudo-types.
26

27 28
    # List of attribute names of any subtypes
    subtypes = []
29
    _empty_declaration = None
30
    _specialization_name = None
31
    default_format_spec = None
32

33 34 35
    def can_coerce_to_pyobject(self, env):
        return False

36 37 38
    def can_coerce_from_pyobject(self, env):
        return False

39
    def can_coerce_to_pystring(self, env, format_spec=None):
40 41
        return False

42 43 44
    def convert_to_pystring(self, cvalue, code, format_spec=None):
        raise NotImplementedError("C types that support string formatting must override this method")

45
    def cast_code(self, expr_code):
46 47 48 49 50 51
        return "((%s)%s)" % (self.empty_declaration_code(), expr_code)

    def empty_declaration_code(self):
        if self._empty_declaration is None:
            self._empty_declaration = self.declaration_code('')
        return self._empty_declaration
52

Craig Citro's avatar
Craig Citro committed
53
    def specialization_name(self):
54 55 56 57 58 59 60 61 62
        if self._specialization_name is None:
            # This is not entirely robust.
            common_subs = (self.empty_declaration_code()
                           .replace("unsigned ", "unsigned_")
                           .replace("long long", "long_long")
                           .replace(" ", "__"))
            self._specialization_name = re.sub(
                '[^a-zA-Z0-9_]', lambda x: '_%x_' % ord(x.group(0)), common_subs)
        return self._specialization_name
63

64 65 66 67 68 69
    def base_declaration_code(self, base_code, entity_code):
        if entity_code:
            return "%s %s" % (base_code, entity_code)
        else:
            return base_code

70 71 72 73 74 75 76
    def __deepcopy__(self, memo):
        """
        Types never need to be copied, if we do copy, Unfortunate Things
        Will Happen!
        """
        return self

77 78
    def get_fused_types(self, result=None, seen=None, subtypes=None):
        subtypes = subtypes or self.subtypes
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
        if not subtypes:
            return None

        if result is None:
            result = []
            seen = set()

        for attr in subtypes:
            list_or_subtype = getattr(self, attr)
            if list_or_subtype:
                if isinstance(list_or_subtype, BaseType):
                    list_or_subtype.get_fused_types(result, seen)
                else:
                    for subtype in list_or_subtype:
                        subtype.get_fused_types(result, seen)

        return result
96

97 98 99 100 101 102
    def specialize_fused(self, env):
        if env.fused_to_specific:
            return self.specialize(env.fused_to_specific)

        return self

103 104
    @property
    def is_fused(self):
105
        """
106
        Whether this type or any of its subtypes is a fused type
107
        """
108 109
        # Add this indirection for the is_fused property to allow overriding
        # get_fused_types in subclasses.
110 111
        return self.get_fused_types()

112 113 114 115 116
    def deduce_template_params(self, actual):
        """
        Deduce any template params in this (argument) type given the actual
        argument type.

117
        https://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction
118
        """
119
        return {}
120

121 122 123 124
    def __lt__(self, other):
        """
        For sorting. The sorting order should correspond to the preference of
        conversion from Python types.
125 126 127

        Override to provide something sensible. This is only implemented so that
        python 3 doesn't trip
128
        """
129
        return id(type(self)) < id(type(other))
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146

    def py_type_name(self):
        """
        Return the name of the Python type that can coerce to this type.
        """

    def typeof_name(self):
        """
        Return the string with which fused python functions can be indexed.
        """
        if self.is_builtin_type or self.py_type_name() == 'object':
            index_name = self.py_type_name()
        else:
            index_name = str(self)

        return index_name

147 148 149 150 151 152 153 154
    def check_for_null_code(self, cname):
        """
        Return the code for a NULL-check in case an UnboundLocalError should
        be raised if an entry of this type is referenced before assignment.
        Returns None if no check should be performed.
        """
        return None

155
    def invalid_value(self):
156
        """
157 158
        Returns the most invalid value an object of this type can assume as a
        C expression string. Returns None if no such value exists.
159 160
        """

Mark Florisson's avatar
Mark Florisson committed
161

162
class PyrexType(BaseType):
163
    #
164
    #  Base class for all Cython types
165 166 167
    #
    #  is_pyobject           boolean     Is a Python object type
    #  is_extension_type     boolean     Is a Python extension type
168
    #  is_cyp_wrapper        boolean     Is a Python extension type that wraps a cypclass
169
    #  is_final_type         boolean     Is a final extension type
170 171 172
    #  is_numeric            boolean     Is a C numeric type
    #  is_int                boolean     Is a C integer type
    #  is_float              boolean     Is a C floating point type
173
    #  is_complex            boolean     Is a C complex type
174 175 176 177
    #  is_void               boolean     Is the C void type
    #  is_array              boolean     Is a C array type
    #  is_ptr                boolean     Is a C pointer type
    #  is_null_ptr           boolean     Is the type of NULL
Danilo Freitas's avatar
Danilo Freitas committed
178
    #  is_reference          boolean     Is a C reference type
179 180 181
    #  is_const              boolean     Is a C const type
    #  is_volatile           boolean     Is a C volatile type
    #  is_cv_qualified       boolean     Is a C const or volatile type
182
    #  is_cfunction          boolean     Is a C function type
183 184 185
    #  is_from_qualified     boolean     Is a method looked-up from a qualified type
    #  is_from_const         boolean     Is a method looked-up from a const type
    #  is_from_volatile      boolean     Is a method looked-up from a volatile type
186
    #  is_struct_or_union    boolean     Is a C struct or union type
187
    #  is_struct             boolean     Is a C struct type
188
    #  is_enum               boolean     Is a C enum type
189
    #  is_cpp_enum           boolean     Is a C++ scoped enum type
190
    #  is_typedef            boolean     Is a typedef type
191
    #  is_string             boolean     Is a C char * type
192
    #  is_pyunicode_ptr      boolean     Is a C PyUNICODE * type
193
    #  is_cpp_string         boolean     Is a C++ std::string type
Stefan Behnel's avatar
Stefan Behnel committed
194
    #  is_unicode_char       boolean     Is either Py_UCS4 or Py_UNICODE
195 196
    #  is_returncode         boolean     Is used only to signal exceptions
    #  is_error              boolean     Is the dummy error type
197
    #  is_buffer             boolean     Is buffer access type
198 199
    #  is_pythran_expr       boolean     Is Pythran expr
    #  is_numpy_buffer       boolean     Is Numpy array buffer
200
    #  has_attributes        boolean     Has C dot-selectable attributes
201 202
    #  needs_refcounting          boolean     Needs code to be generated similar to incref/gotref/decref.
    #                                    Largely used internally.
203 204
    #  default_value         string      Initial value that can be assigned before first user assignment.
    #  declaration_value     string      The value statically assigned on declaration (if any).
205
    #  entry                 Entry       The Entry for this type
206
    #
207
    #  declaration_code(entity_code,
208 209 210 211 212 213 214 215 216
    #      for_display = 0, dll_linkage = None, pyrex = 0)
    #    Returns a code fragment for the declaration of an entity
    #    of this type, given a code fragment for the entity.
    #    * If for_display, this is for reading by a human in an error
    #      message; otherwise it must be valid C code.
    #    * If dll_linkage is not None, it must be 'DL_EXPORT' or
    #      'DL_IMPORT', and will be added to the base type part of
    #      the declaration.
    #    * If pyrex = 1, this is for use in a 'cdef extern'
217
    #      statement of a Cython include file.
218 219 220 221 222 223 224 225 226 227
    #
    #  assignable_from(src_type)
    #    Tests whether a variable of this type can be
    #    assigned a value of type src_type.
    #
    #  same_as(other_type)
    #    Tests whether this type represents the same type
    #    as other_type.
    #
    #  as_argument_type():
228
    #    Coerces array and C function types into pointer type for use as
229 230
    #    a formal argument type.
    #
231 232 233
    #  as_returned_type():
    #    Potentially annotates function call return types as pure rvalues.
    #
234

235
    is_pyobject = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
236
    is_unspecified = 0
237
    is_extension_type = 0
238
    is_cyp_wrapper = 0
239
    is_final_type = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
240
    is_builtin_type = 0
241
    is_cython_builtin_type = 0
242 243 244
    is_numeric = 0
    is_int = 0
    is_float = 0
245
    is_complex = 0
246 247 248 249
    is_void = 0
    is_array = 0
    is_ptr = 0
    is_null_ptr = 0
250
    is_reference = 0
251
    is_const = 0
252 253
    is_volatile = 0
    is_cv_qualified = 0
254
    is_cfunction = 0
255 256 257
    is_from_qualified = 0
    is_from_const = 0
    is_from_volatile = 0
258
    is_struct_or_union = 0
259
    is_cpp_class = 0
260
    is_cyp_class = 0
261
    is_const_cyp_class = 0
262
    is_qualified_cyp_class = 0
263
    is_cpp_string = 0
264
    is_struct = 0
265
    is_enum = 0
266
    is_cpp_enum = False
267
    is_typedef = 0
268
    is_string = 0
269
    is_pyunicode_ptr = 0
Stefan Behnel's avatar
Stefan Behnel committed
270
    is_unicode_char = 0
271 272
    is_returncode = 0
    is_error = 0
273
    is_buffer = 0
274
    is_ctuple = 0
275
    is_memoryviewslice = 0
276 277
    is_pythran_expr = 0
    is_numpy_buffer = 0
278
    is_checked_result = 0
279
    is_template_typename = 0
280
    has_attributes = 0
281
    needs_refcounting = 0
282
    default_value = ""
283
    declaration_value = ""
284

285 286 287
    def resolve(self):
        # If a typedef, returns the base type.
        return self
288

289
    def specialize(self, values):
Robert Bradshaw's avatar
Robert Bradshaw committed
290
        # TODO(danilo): Override wherever it makes sense.
291
        return self
292

293 294 295 296
    def literal_code(self, value):
        # Returns a C code fragment representing a literal
        # value of this type.
        return str(value)
297

298
    def __str__(self):
299
        return self.declaration_code("", for_display = 1).strip()
300

301 302
    def same_as(self, other_type, **kwds):
        return self.same_as_resolved_type(other_type.resolve(), **kwds)
303

304
    def same_as_resolved_type(self, other_type):
305
        return self == other_type or other_type is error_type
306

307 308
    def subtype_of(self, other_type):
        return self.subtype_of_resolved_type(other_type.resolve())
309

310 311
    def subtype_of_resolved_type(self, other_type):
        return self.same_as(other_type)
312

313 314
    def assignable_from(self, src_type):
        return self.assignable_from_resolved_type(src_type.resolve())
315

316 317
    def assignable_from_resolved_type(self, src_type):
        return self.same_as(src_type)
318

319 320
    def as_argument_type(self):
        return self
321

322 323 324
    def as_returned_type(self):
        return self

325 326 327 328 329
    def is_complete(self):
        # A type is incomplete if it is an unsized array,
        # a struct whose attributes are not defined, etc.
        return 1

330
    def is_simple_buffer_dtype(self):
331
        return (self.is_int or self.is_float or self.is_complex or self.is_pyobject or
332 333
                self.is_extension_type or self.is_ptr)

334 335 336 337 338 339
    def struct_nesting_depth(self):
        # Returns the number levels of nested structs. This is
        # used for constructing a stack for walking the run-time
        # type information of the struct.
        return 1

340 341
    def global_init_code(self, entry, code):
        # abstract
342
        pass
343

344 345
    def needs_nonecheck(self):
        return 0
346

347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
    def _assign_from_py_code(self, source_code, result_code, error_pos, code,
                             from_py_function=None, error_condition=None, extra_args=None):
        args = ', ' + ', '.join('%s' % arg for arg in extra_args) if extra_args else ''
        convert_call = "%s(%s%s)" % (
            from_py_function or self.from_py_function,
            source_code,
            args,
        )
        if self.is_enum:
            convert_call = typecast(self, c_long_type, convert_call)
        return '%s = %s; %s' % (
            result_code,
            convert_call,
            code.error_goto_if(error_condition or self.error_condition(result_code), error_pos))

362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
    def _generate_dummy_refcounting(self, code, *ignored_args, **ignored_kwds):
        if self.needs_refcounting:
            raise NotImplementedError("Ref-counting operation not yet implemented for type %s" %
                                      self)

    def _generate_dummy_refcounting_assignment(self, code, cname, rhs_cname, *ignored_args, **ignored_kwds):
        if self.needs_refcounting:
            raise NotImplementedError("Ref-counting operation not yet implemented for type %s" %
                                      self)
        code.putln("%s = %s" % (cname, rhs_cname))

    generate_incref = generate_xincref = generate_decref = generate_xdecref \
        = generate_decref_clear = generate_xdecref_clear \
        = generate_gotref = generate_xgotref = generate_giveref = generate_xgiveref \
            = _generate_dummy_refcounting

    generate_decref_set = generate_xdecref_set = _generate_dummy_refcounting_assignment

    def nullcheck_string(self, code, cname):
        if self.needs_refcounting:
            raise NotImplementedError("Ref-counting operation not yet implemented for type %s" %
                                      self)
        code.putln("1")


387

388 389
def public_decl(base_code, dll_linkage):
    if dll_linkage:
390
        return "%s(%s)" % (dll_linkage, base_code.replace(',', ' __PYX_COMMA '))
391 392
    else:
        return base_code
393

394
def create_typedef_type(name, base_type, cname, is_external=0, namespace=None):
395 396
    is_fused = base_type.is_fused
    if base_type.is_complex or is_fused:
397
        if is_external:
398 399 400 401 402 403 404
            if is_fused:
                msg = "Fused"
            else:
                msg = "Complex"

            raise ValueError("%s external typedefs not supported" % msg)

405 406
        return base_type
    else:
407
        return CTypedefType(name, base_type, cname, is_external, namespace)
408

409

410
class CTypedefType(BaseType):
411
    #
412
    #  Pseudo-type defined with a ctypedef statement in a
413 414 415
    #  'cdef extern from' block.
    #  Delegates most attribute lookups to the base type.
    #  (Anything not defined here or in the BaseType is delegated.)
416
    #
417
    #  qualified_name      string
418
    #  typedef_name        string
419 420
    #  typedef_cname       string
    #  typedef_base_type   PyrexType
421
    #  typedef_is_external bool
422
    #  is_specialised      bool
423

424
    is_typedef = 1
425
    typedef_is_external = 0
426
    is_specialised = 0
427 428 429

    to_py_utility_code = None
    from_py_utility_code = None
430 431

    subtypes = ['typedef_base_type']
432

433
    def __init__(self, name, base_type, cname, is_external=0, namespace=None):
434
        assert not base_type.is_complex
435
        self.typedef_name = name
436 437
        self.typedef_cname = cname
        self.typedef_base_type = base_type
438
        self.typedef_is_external = is_external
439
        self.typedef_namespace = namespace
440

441 442 443
    def invalid_value(self):
        return self.typedef_base_type.invalid_value()

444 445
    def resolve(self):
        return self.typedef_base_type.resolve()
446 447

    def declaration_code(self, entity_code,
448
            for_display = 0, dll_linkage = None, pyrex = 0):
449
        if pyrex or for_display:
450
            base_code = self.typedef_name
451
        else:
452
            base_code = public_decl(self.typedef_cname, dll_linkage)
453
        if self.typedef_namespace is not None and not pyrex:
454
            base_code = "%s::%s" % (namespace_declaration_code(self.typedef_namespace), base_code)
455 456
        if self.is_specialised and self.is_cyp_class and entity_code:
            base_code = "Cy_Raw<%s>" % base_code
457
        return self.base_declaration_code(base_code, entity_code)
458

459 460 461
    def as_argument_type(self):
        return self

462 463
    def cast_code(self, expr_code):
        # If self is really an array (rather than pointer), we can't cast.
464
        # For example, the gmp mpz_t.
465 466 467
        if self.typedef_base_type.is_array:
            base_type = self.typedef_base_type.base_type
            return CPtrType(base_type).cast_code(expr_code)
468 469
        else:
            return BaseType.cast_code(self, expr_code)
470

Robert Bradshaw's avatar
Robert Bradshaw committed
471 472 473 474 475 476
    def specialize(self, values):
        base_type = self.typedef_base_type.specialize(values)
        namespace = self.typedef_namespace.specialize(values) if self.typedef_namespace else None
        if base_type is self.typedef_base_type and namespace is self.typedef_namespace:
            return self
        else:
477
            specialized_type = create_typedef_type(self.typedef_name, base_type, self.typedef_cname,
478
                                0, namespace)
479 480
            specialized_type.is_specialised = base_type is not self.typedef_base_type
            return specialized_type
Robert Bradshaw's avatar
Robert Bradshaw committed
481

482 483
    def __repr__(self):
        return "<CTypedefType %s>" % self.typedef_cname
484

485
    def __str__(self):
486
        return self.typedef_name
487 488 489

    def _create_utility_code(self, template_utility_code,
                             template_function_name):
490
        type_name = type_identifier(self.typedef_cname)
491 492 493 494 495 496 497 498 499 500
        utility_code = template_utility_code.specialize(
            type     = self.typedef_cname,
            TypeName = type_name)
        function_name = template_function_name % type_name
        return utility_code, function_name

    def create_to_py_utility_code(self, env):
        if self.typedef_is_external:
            if not self.to_py_utility_code:
                base_type = self.typedef_base_type
Robert Bradshaw's avatar
Robert Bradshaw committed
501
                if type(base_type) is CIntType:
502
                    self.to_py_function = "__Pyx_PyInt_From_" + self.specialization_name()
503
                    env.use_utility_code(TempitaUtilityCode.load_cached(
504
                        "CIntToPy", "TypeConversion.c",
505
                        context={"TYPE": self.empty_declaration_code(),
506
                                 "TO_PY_FUNCTION": self.to_py_function}))
507
                    return True
508
                elif base_type.is_float:
509
                    pass  # XXX implement!
510
                elif base_type.is_complex:
511
                    pass  # XXX implement!
512
                    pass
513 514 515 516 517 518 519 520 521 522 523
                elif base_type.is_cpp_string:
                    cname = "__pyx_convert_PyObject_string_to_py_%s" % type_identifier(self)
                    context = {
                        'cname': cname,
                        'type': self.typedef_cname,
                    }
                    from .UtilityCode import CythonUtilityCode
                    env.use_utility_code(CythonUtilityCode.load(
                        "string.to_py", "CppConvert.pyx", context=context))
                    self.to_py_function = cname
                    return True
524 525 526 527 528 529 530 531 532 533
            if self.to_py_utility_code:
                env.use_utility_code(self.to_py_utility_code)
                return True
        # delegation
        return self.typedef_base_type.create_to_py_utility_code(env)

    def create_from_py_utility_code(self, env):
        if self.typedef_is_external:
            if not self.from_py_utility_code:
                base_type = self.typedef_base_type
Robert Bradshaw's avatar
Robert Bradshaw committed
534
                if type(base_type) is CIntType:
535
                    self.from_py_function = "__Pyx_PyInt_As_" + self.specialization_name()
536
                    env.use_utility_code(TempitaUtilityCode.load_cached(
537
                        "CIntFromPy", "TypeConversion.c",
538
                        context={"TYPE": self.empty_declaration_code(),
539
                                 "FROM_PY_FUNCTION": self.from_py_function}))
540
                    return True
541
                elif base_type.is_float:
542
                    pass  # XXX implement!
543
                elif base_type.is_complex:
544
                    pass  # XXX implement!
545 546 547 548 549 550 551 552 553 554 555
                elif base_type.is_cpp_string:
                    cname = '__pyx_convert_string_from_py_%s' % type_identifier(self)
                    context = {
                        'cname': cname,
                        'type': self.typedef_cname,
                    }
                    from .UtilityCode import CythonUtilityCode
                    env.use_utility_code(CythonUtilityCode.load(
                        "string.from_py", "CppConvert.pyx", context=context))
                    self.from_py_function = cname
                    return True
556 557 558 559 560 561
            if self.from_py_utility_code:
                env.use_utility_code(self.from_py_utility_code)
                return True
        # delegation
        return self.typedef_base_type.create_from_py_utility_code(env)

562 563 564 565 566 567 568 569 570
    def to_py_call_code(self, source_code, result_code, result_type, to_py_function=None):
        if to_py_function is None:
            to_py_function = self.to_py_function
        return self.typedef_base_type.to_py_call_code(
            source_code, result_code, result_type, to_py_function)

    def from_py_call_code(self, source_code, result_code, error_pos, code,
                          from_py_function=None, error_condition=None):
        return self.typedef_base_type.from_py_call_code(
571 572 573 574
            source_code, result_code, error_pos, code,
            from_py_function or self.from_py_function,
            error_condition or self.error_condition(result_code)
        )
575

576
    def overflow_check_binop(self, binop, env, const_rhs=False):
577
        env.use_utility_code(UtilityCode.load("Common", "Overflow.c"))
578
        type = self.empty_declaration_code()
579
        name = self.specialization_name()
580
        if binop == "lshift":
581
            env.use_utility_code(TempitaUtilityCode.load_cached(
582 583
                "LeftShift", "Overflow.c",
                context={'TYPE': type, 'NAME': name, 'SIGNED': self.signed}))
584 585 586 587
        else:
            if const_rhs:
                binop += "_const"
            _load_overflow_base(env)
588
            env.use_utility_code(TempitaUtilityCode.load_cached(
589 590
                "SizeCheck", "Overflow.c",
                context={'TYPE': type, 'NAME': name}))
591
            env.use_utility_code(TempitaUtilityCode.load_cached(
592 593
                "Binop", "Overflow.c",
                context={'TYPE': type, 'NAME': name, 'BINOP': binop}))
594 595
        return "__Pyx_%s_%s_checking_overflow" % (binop, name)

596 597 598
    def error_condition(self, result_code):
        if self.typedef_is_external:
            if self.exception_value:
599 600
                condition = "(%s == %s)" % (
                    result_code, self.cast_code(self.exception_value))
601 602 603 604 605 606
                if self.exception_check:
                    condition += " && PyErr_Occurred()"
                return condition
        # delegation
        return self.typedef_base_type.error_condition(result_code)

607 608 609
    def __getattr__(self, name):
        return getattr(self.typedef_base_type, name)

610 611
    def py_type_name(self):
        return self.typedef_base_type.py_type_name()
612

613 614 615
    def can_coerce_to_pyobject(self, env):
        return self.typedef_base_type.can_coerce_to_pyobject(env)

616 617 618
    def can_coerce_from_pyobject(self, env):
        return self.typedef_base_type.can_coerce_from_pyobject(env)

619

620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
class CheckedResultType(BaseType):
    # Pseudo-type to wrap the return value of c functions in a "CheckedResult"

    is_checked_result = 1
    is_void = 0

    to_py_utility_code = None
    from_py_utility_code = None

    subtypes = ['checked_base_type']

    def __init__(self, pos, base_type):
        if base_type.is_reference:
            error(pos, "Cannot use 'except ~' exception signaling with functions that return by reference")
        elif base_type.is_cpp_class:
            base_type.check_nullary_constructor(pos)
636
        self.pos = pos
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
        self.checked_base_type = base_type

    def invalid_value(self):
        return self.checked_base_type.invalid_value()

    def resolve(self):
        return self.checked_base_type.resolve()

    def declaration_code(self, entity_code,
            for_display = 0, dll_linkage = None, pyrex = 0):
        checked_code = "CheckedResult<%s>" % self.checked_base_type.declaration_code('')
        if pyrex or for_display:
            base_code = checked_code
        else:
            base_code = public_decl(checked_code, dll_linkage)
        return self.base_declaration_code(base_code, entity_code)

    def as_argument_type(self):
        return self.checked_base_type.as_argument_type()

657 658 659 660 661 662
    def as_returned_type(self):
        returned_base_type = self.checked_base_type.as_returned_type()
        if returned_base_type.same_as(self.checked_base_type):
            return self
        return CheckedResultType(self.pos, returned_base_type)

663 664 665 666
    def cast_code(self, expr_code):
        return self.checked_base_type.cast_code(expr_code)

    def specialize(self, values):
667
        return CheckedResultType(self.pos, self.checked_base_type.specialize(values))
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713

    def __repr__(self):
        return "<CheckedResult[%s]>" % repr(self.checked_base_type)

    def __str__(self):
        return str(self.checked_base_type)

    def create_to_py_utility_code(self, env):
        return self.checked_base_type.create_to_py_utility_code(env)

    def create_from_py_utility_code(self, env):
        return self.checked_base_type.create_from_py_utility_code(env)

    def to_py_call_code(self, source_code, result_code, result_type, to_py_function=None):
        return self.checked_base_type.to_py_call_code(
            source_code, result_code, result_type, to_py_function)

    def from_py_call_code(self, source_code, result_code, error_pos, code,
                          from_py_function=None, error_condition=None):
        return self.checked_base_type.from_py_call_code(
            source_code, result_code, error_pos, code,
            from_py_function or self.from_py_function,
            error_condition or self.error_condition(result_code)
        )

    def overflow_check_binop(self, binop, env, const_rhs=False):
        return self.checked_base_type.overflow_check_binop(binop, env, const_rhs)

    def error_condition(self, result_code):
        return "%s.is_error()" % result_code

    def set_error(self, result_code):
        return "%s.set_error();" % result_code

    def __getattr__(self, name):
        return getattr(self.checked_base_type, name)

    def py_type_name(self):
        return self.checked_base_type.py_type_name()

    def can_coerce_to_pyobject(self, env):
        return self.checked_base_type.can_coerce_to_pyobject(env)

    def can_coerce_from_pyobject(self, env):
        return self.checked_base_type.can_coerce_from_pyobject(env)

714 715
    def generate_dummy_refcount(self, code, *args, **kwargs):
        pass
716

717 718 719 720 721 722
    generate_incref = generate_xincref = generate_dummy_refcount
    generate_decref = generate_decref_clear = generate_dummy_refcount
    generate_xdecref = generate_xdecref_clear = generate_dummy_refcount
    generate_gotref = generate_xgotref = generate_dummy_refcount
    generate_giveref = generate_xgiveref = generate_dummy_refcount
    generate_decref_set = generate_xdecref_set = generate_dummy_refcount
723 724


725
class MemoryViewSliceType(PyrexType):
726

727
    is_memoryviewslice = 1
728
    default_value = "{ 0, 0, { 0 }, { 0 }, { 0 } }"
Kurt Smith's avatar
Kurt Smith committed
729

730
    has_attributes = 1
731 732 733 734
    needs_refcounting = 1  # Ideally this would be true and reference counting for
        # memoryview and pyobject code could be generated in the same way.
        # However, memoryviews are sufficiently specialized that this doesn't
        # seem practical. Implement a limited version of it for now
735 736
    scope = None

Stefan Behnel's avatar
Stefan Behnel committed
737
    # These are special cased in Defnode
738 739 740 741
    from_py_function = None
    to_py_function = None

    exception_value = None
742
    exception_check = True
743

744 745
    subtypes = ['dtype']

746
    def __init__(self, base_dtype, axes):
Stefan Behnel's avatar
Stefan Behnel committed
747
        """
748
        MemoryViewSliceType(base, axes)
749 750 751 752 753 754 755 756 757 758 759 760 761 762

        Base is the C base type; axes is a list of (access, packing) strings,
        where access is one of 'full', 'direct' or 'ptr' and packing is one of
        'contig', 'strided' or 'follow'.  There is one (access, packing) tuple
        for each dimension.

        the access specifiers determine whether the array data contains
        pointers that need to be dereferenced along that axis when
        retrieving/setting:

        'direct' -- No pointers stored in this dimension.
        'ptr' -- Pointer stored in this dimension.
        'full' -- Check along this dimension, don't assume either.

Brian Wignall's avatar
Brian Wignall committed
763
        the packing specifiers specify how the array elements are laid-out
764 765
        in memory.

luz.paz's avatar
luz.paz committed
766
        'contig' -- The data is contiguous in memory along this dimension.
767
                At most one dimension may be specified as 'contig'.
luz.paz's avatar
luz.paz committed
768
        'strided' -- The data isn't contiguous along this dimension.
769 770 771 772 773 774 775 776 777 778
        'follow' -- Used for C/Fortran contiguous arrays, a 'follow' dimension
            has its stride automatically computed from extents of the other
            dimensions to ensure C or Fortran memory layout.

        C-contiguous memory has 'direct' as the access spec, 'contig' as the
        *last* axis' packing spec and 'follow' for all other packing specs.

        Fortran-contiguous memory has 'direct' as the access spec, 'contig' as
        the *first* axis' packing spec and 'follow' for all other packing
        specs.
Stefan Behnel's avatar
Stefan Behnel committed
779
        """
780
        from . import Buffer, MemoryView
781

Kurt Smith's avatar
Kurt Smith committed
782
        self.dtype = base_dtype
783
        self.axes = axes
784
        self.ndim = len(axes)
785
        self.flags = MemoryView.get_buf_flags(self.axes)
786

787 788 789
        self.is_c_contig, self.is_f_contig = MemoryView.is_cf_contig(self.axes)
        assert not (self.is_c_contig and self.is_f_contig)

790
        self.mode = MemoryView.get_mode(axes)
791
        self.writable_needed = False
792

793
        if not self.dtype.is_fused:
794
            self.dtype_name = Buffer.mangle_dtype_name(self.dtype)
795

796 797 798
    def __hash__(self):
        return hash(self.__class__) ^ hash(self.dtype) ^ hash(tuple(self.axes))

799 800 801 802 803
    def __eq__(self, other):
        if isinstance(other, BaseType):
            return self.same_as_resolved_type(other)
        else:
            return False
804

805 806 807 808
    def __ne__(self, other):
        # TODO drop when Python2 is dropped
        return not (self == other)

809 810
    def same_as_resolved_type(self, other_type):
        return ((other_type.is_memoryviewslice and
811
            #self.writable_needed == other_type.writable_needed and  # FIXME: should be only uni-directional
812 813 814 815
            self.dtype.same_as(other_type.dtype) and
            self.axes == other_type.axes) or
            other_type is error_type)

816 817 818
    def needs_nonecheck(self):
        return True

Kurt Smith's avatar
Kurt Smith committed
819
    def is_complete(self):
820
        # incomplete since the underlying struct doesn't have a cython.memoryview object.
Kurt Smith's avatar
Kurt Smith committed
821 822 823 824
        return 0

    def declaration_code(self, entity_code,
            for_display = 0, dll_linkage = None, pyrex = 0):
825 826 827
        # XXX: we put these guards in for now...
        assert not pyrex
        assert not dll_linkage
828
        from . import MemoryView
829 830
        base_code = StringEncoding.EncodedString(
                        (str(self)) if for_display else MemoryView.memviewslice_cname)
831
        return self.base_declaration_code(
832
                base_code,
833
                entity_code)
Kurt Smith's avatar
Kurt Smith committed
834

835 836
    def attributes_known(self):
        if self.scope is None:
837
            from . import Symtab
838

839 840
            self.scope = scope = Symtab.CClassScope(
                    'mvs_class_'+self.specialization_suffix(),
841 842
                    None,
                    visibility='extern')
843 844

            scope.parent_type = self
845
            scope.directives = {}
846

847 848 849 850 851
            scope.declare_var('_data', c_char_ptr_type, None,
                              cname='data', is_cdef=1)

        return True

852
    def declare_attribute(self, attribute, env, pos):
853
        from . import MemoryView, Options
854 855

        scope = self.scope
856

857
        if attribute == 'shape':
858 859
            scope.declare_var('shape',
                    c_array_type(c_py_ssize_t_type,
860 861
                                 Options.buffer_max_dims),
                    pos,
862 863 864
                    cname='shape',
                    is_cdef=1)

865
        elif attribute == 'strides':
866 867
            scope.declare_var('strides',
                    c_array_type(c_py_ssize_t_type,
868 869
                                 Options.buffer_max_dims),
                    pos,
870 871 872
                    cname='strides',
                    is_cdef=1)

873
        elif attribute == 'suboffsets':
874 875
            scope.declare_var('suboffsets',
                    c_array_type(c_py_ssize_t_type,
876 877
                                 Options.buffer_max_dims),
                    pos,
878 879 880
                    cname='suboffsets',
                    is_cdef=1)

881
        elif attribute in ("copy", "copy_fortran"):
882
            ndim = len(self.axes)
883

884 885 886 887
            follow_dim = [('direct', 'follow')]
            contig_dim = [('direct', 'contig')]
            to_axes_c = follow_dim * (ndim - 1) + contig_dim
            to_axes_f = contig_dim + follow_dim * (ndim -1)
888

889
            dtype = self.dtype
890 891
            if dtype.is_cv_qualified:
                dtype = dtype.cv_base_type
892 893 894

            to_memview_c = MemoryViewSliceType(dtype, to_axes_c)
            to_memview_f = MemoryViewSliceType(dtype, to_axes_f)
895

896 897
            for to_memview, cython_name in [(to_memview_c, "copy"),
                                            (to_memview_f, "copy_fortran")]:
898 899 900 901 902 903 904 905 906
                copy_func_type = CFuncType(
                    to_memview,
                    [CFuncTypeArg("memviewslice", self, None)])
                copy_cname = MemoryView.copy_c_or_fortran_cname(to_memview)

                entry = scope.declare_cfunction(
                    cython_name,
                    copy_func_type, pos=pos, defining=1,
                    cname=copy_cname)
907

908 909
                utility = MemoryView.get_copy_new_utility(pos, self, to_memview)
                env.use_utility_code(utility)
910

911 912
            MemoryView.use_cython_array_utility_code(env)

913
        elif attribute in ("is_c_contig", "is_f_contig"):
914
            # is_c_contig and is_f_contig functions
915
            for (c_or_f, cython_name) in (('C', 'is_c_contig'), ('F', 'is_f_contig')):
916

917
                is_contig_name = MemoryView.get_is_contig_func_name(c_or_f, self.ndim)
918

919
                cfunctype = CFuncType(
920
                        return_type=c_bint_type,
921 922 923 924
                        args=[CFuncTypeArg("memviewslice", self, None)],
                        exception_value="-1",
                )

925
                entry = scope.declare_cfunction(cython_name,
926
                            cfunctype,
927 928 929
                            pos=pos,
                            defining=1,
                            cname=is_contig_name)
930

931
                entry.utility_code_definition = MemoryView.get_is_contig_utility(c_or_f, self.ndim)
932

933
        return True
934

935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
    def get_entry(self, node, cname=None, type=None):
        from . import MemoryView, Symtab

        if cname is None:
            assert node.is_simple() or node.is_temp or node.is_elemental
            cname = node.result()

        if type is None:
            type = node.type

        entry = Symtab.Entry(cname, cname, type, node.pos)
        return MemoryView.MemoryViewSliceBufferEntry(entry)

    def conforms_to(self, dst, broadcast=False, copying=False):
        """
        Returns True if src conforms to dst, False otherwise.

        If conformable, the types are the same, the ndims are equal, and each axis spec is conformable.

        Any packing/access spec is conformable to itself.

        'direct' and 'ptr' are conformable to 'full'.
        'contig' and 'follow' are conformable to 'strided'.
        Any other combo is not conformable.
        """
        from . import MemoryView

        src = self

964 965
        #if not copying and self.writable_needed and not dst.writable_needed:
        #    return False
966 967

        src_dtype, dst_dtype = src.dtype, dst.dtype
968 969 970 971 972 973 974 975 976 977 978 979
        # We can add but not remove const/volatile modifiers
        # (except if we are copying by value, then anything is fine)
        if not copying:
            if src_dtype.is_const and not dst_dtype.is_const:
                return False
            if src_dtype.is_volatile and not dst_dtype.is_volatile:
                return False
        # const/volatile checks are done, remove those qualifiers
        if src_dtype.is_cv_qualified:
            src_dtype = src_dtype.cv_base_type
        if dst_dtype.is_cv_qualified:
            dst_dtype = dst_dtype.cv_base_type
980 981

        if src_dtype != dst_dtype:
982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
            return False

        if src.ndim != dst.ndim:
            if broadcast:
                src, dst = MemoryView.broadcast_types(src, dst)
            else:
                return False

        for src_spec, dst_spec in zip(src.axes, dst.axes):
            src_access, src_packing = src_spec
            dst_access, dst_packing = dst_spec
            if src_access != dst_access and dst_access != 'full':
                return False
            if src_packing != dst_packing and dst_packing != 'strided' and not copying:
                return False

        return True

    def valid_dtype(self, dtype, i=0):
        """
        Return whether type dtype can be used as the base type of a
        memoryview slice.

        We support structs, numeric types and objects
        """
        if dtype.is_complex and dtype.real_type.is_int:
            return False

        if dtype.is_struct and dtype.kind == 'struct':
            for member in dtype.scope.var_entries:
                if not self.valid_dtype(member.type):
                    return False

            return True

        return (
            dtype.is_error or
            # Pointers are not valid (yet)
            # (dtype.is_ptr and valid_memslice_dtype(dtype.base_type)) or
            (dtype.is_array and i < 8 and self.valid_dtype(dtype.base_type, i + 1)) or
            dtype.is_numeric or
            dtype.is_pyobject or
            dtype.is_fused or  # accept this as it will be replaced by specializations later
            (dtype.is_typedef and self.valid_dtype(dtype.typedef_base_type))
        )

    def validate_memslice_dtype(self, pos):
        if not self.valid_dtype(self.dtype):
            error(pos, "Invalid base type for memoryview slice: %s" % self.dtype)

    def assert_direct_dims(self, pos):
        for access, packing in self.axes:
            if access != 'direct':
                error(pos, "All dimensions must be direct")
                return False
        return True

    def transpose(self, pos):
        if not self.assert_direct_dims(pos):
            return error_type
        return MemoryViewSliceType(self.dtype, self.axes[::-1])

1044
    def specialization_name(self):
1045 1046 1047
        return '%s_%s' % (
            super(MemoryViewSliceType,self).specialization_name(),
            self.specialization_suffix())
1048

1049
    def specialization_suffix(self):
1050
        return "%s_%s" % (self.axes_to_name(), self.dtype_name)
1051

1052 1053 1054
    def can_coerce_to_pyobject(self, env):
        return True

1055 1056 1057
    def can_coerce_from_pyobject(self, env):
        return True

1058 1059 1060
    def check_for_null_code(self, cname):
        return cname + '.memview'

1061
    def create_from_py_utility_code(self, env):
1062
        from . import MemoryView, Buffer
1063 1064 1065

        # We don't have 'code', so use a LazyUtilityCode with a callback.
        def lazy_utility_callback(code):
1066
            context['dtype_typeinfo'] = Buffer.get_type_information_cname(code, self.dtype)
1067
            return TempitaUtilityCode.load(
1068
                "ObjectToMemviewSlice", "MemoryView_C.c", context=context)
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079

        env.use_utility_code(MemoryView.memviewslice_init_code)
        env.use_utility_code(LazyUtilityCode(lazy_utility_callback))

        if self.is_c_contig:
            c_or_f_flag = "__Pyx_IS_C_CONTIG"
        elif self.is_f_contig:
            c_or_f_flag = "__Pyx_IS_F_CONTIG"
        else:
            c_or_f_flag = "0"

1080 1081
        suffix = self.specialization_suffix()
        funcname = "__Pyx_PyObject_to_MemoryviewSlice_" + suffix
1082 1083 1084

        context = dict(
            MemoryView.context,
1085
            buf_flag = self.flags,
1086
            ndim = self.ndim,
1087
            axes_specs = ', '.join(self.axes_to_code()),
1088
            dtype_typedecl = self.dtype.empty_declaration_code(),
1089 1090 1091 1092 1093 1094 1095 1096
            struct_nesting_depth = self.dtype.struct_nesting_depth(),
            c_or_f_flag = c_or_f_flag,
            funcname = funcname,
        )

        self.from_py_function = funcname
        return True

1097 1098
    def from_py_call_code(self, source_code, result_code, error_pos, code,
                          from_py_function=None, error_condition=None):
1099 1100 1101
        # NOTE: auto-detection of readonly buffers is disabled:
        # writable = self.writable_needed or not self.dtype.is_const
        writable = not self.dtype.is_const
1102 1103
        return self._assign_from_py_code(
            source_code, result_code, error_pos, code, from_py_function, error_condition,
1104
            extra_args=['PyBUF_WRITABLE' if writable else '0'])
1105

1106
    def create_to_py_utility_code(self, env):
1107
        self._dtype_to_py_func, self._dtype_from_py_func = self.dtype_object_conversion_funcs(env)
1108 1109
        return True

1110 1111 1112 1113 1114 1115
    def to_py_call_code(self, source_code, result_code, result_type, to_py_function=None):
        assert self._dtype_to_py_func
        assert self._dtype_from_py_func

        to_py_func = "(PyObject *(*)(char *)) " + self._dtype_to_py_func
        from_py_func = "(int (*)(char *, PyObject *)) " + self._dtype_from_py_func
1116

1117 1118
        tup = (result_code, source_code, self.ndim, to_py_func, from_py_func, self.dtype.is_pyobject)
        return "%s = __pyx_memoryview_fromslice(%s, %s, %s, %s, %d);" % tup
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131

    def dtype_object_conversion_funcs(self, env):
        get_function = "__pyx_memview_get_%s" % self.dtype_name
        set_function = "__pyx_memview_set_%s" % self.dtype_name

        context = dict(
            get_function = get_function,
            set_function = set_function,
        )

        if self.dtype.is_pyobject:
            utility_name = "MemviewObjectToObject"
        else:
1132 1133
            self.dtype.create_to_py_utility_code(env)
            to_py_function = self.dtype.to_py_function
1134

1135 1136 1137 1138
            from_py_function = None
            if not self.dtype.is_const:
                self.dtype.create_from_py_utility_code(env)
                from_py_function = self.dtype.from_py_function
1139

1140 1141 1142 1143 1144
            if not (to_py_function or from_py_function):
                return "NULL", "NULL"
            if not to_py_function:
                get_function = "NULL"
            if not from_py_function:
1145 1146
                set_function = "NULL"

1147 1148 1149 1150
            utility_name = "MemviewDtypeToObject"
            error_condition = (self.dtype.error_condition('value') or
                               'PyErr_Occurred()')
            context.update(
1151 1152 1153 1154
                to_py_function=to_py_function,
                from_py_function=from_py_function,
                dtype=self.dtype.empty_declaration_code(),
                error_condition=error_condition,
1155 1156
            )

1157 1158
        utility = TempitaUtilityCode.load_cached(
            utility_name, "MemoryView_C.c", context=context)
1159 1160
        env.use_utility_code(utility)
        return get_function, set_function
1161

1162
    def axes_to_code(self):
Stefan Behnel's avatar
Stefan Behnel committed
1163
        """Return a list of code constants for each axis"""
1164
        from . import MemoryView
1165 1166 1167
        d = MemoryView._spec_to_const
        return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes]

1168
    def axes_to_name(self):
Stefan Behnel's avatar
Stefan Behnel committed
1169
        """Return an abbreviated name for our axes"""
1170
        from . import MemoryView
1171 1172 1173 1174 1175 1176
        d = MemoryView._spec_to_abbrev
        return "".join(["%s%s" % (d[a], d[p]) for a, p in self.axes])

    def error_condition(self, result_code):
        return "!%s.memview" % result_code

1177
    def __str__(self):
1178
        from . import MemoryView
1179 1180

        axes_code_list = []
1181
        for idx, (access, packing) in enumerate(self.axes):
1182 1183 1184 1185
            flag = MemoryView.get_memoryview_flag(access, packing)
            if flag == "strided":
                axes_code_list.append(":")
            else:
1186
                if flag == 'contiguous':
1187
                    have_follow = [p for a, p in self.axes[idx - 1:idx + 2]
1188 1189 1190 1191
                                         if p == 'follow']
                    if have_follow or self.ndim == 1:
                        flag = '1'

1192 1193 1194 1195 1196 1197 1198 1199 1200
                axes_code_list.append("::" + flag)

        if self.dtype.is_pyobject:
            dtype_name = self.dtype.name
        else:
            dtype_name = self.dtype

        return "%s[%s]" % (dtype_name, ", ".join(axes_code_list))

1201
    def specialize(self, values):
Stefan Behnel's avatar
Stefan Behnel committed
1202
        """This does not validate the base type!!"""
1203 1204 1205 1206
        dtype = self.dtype.specialize(values)
        if dtype is not self.dtype:
            return MemoryViewSliceType(dtype, self.axes)

1207 1208
        return self

1209 1210
    def cast_code(self, expr_code):
        return expr_code
1211

1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
    # When memoryviews are increfed currently seems heavily special-cased.
    # Therefore, use our own function for now
    def generate_incref(self, code, name, **kwds):
        pass

    def generate_incref_memoryviewslice(self, code, slice_cname, have_gil):
        # TODO ideally would be done separately
        code.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    # decref however did look to always apply for memoryview slices
    # with "have_gil" set to True by default
    def generate_xdecref(self, code, cname, nanny, have_gil):
        code.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (cname, int(have_gil)))

    def generate_decref(self, code, cname, nanny, have_gil):
        # Fall back to xdecref since we don't care to have a separate decref version for this.
        self.generate_xdecref(code, cname, nanny, have_gil)

    def generate_xdecref_clear(self, code, cname, clear_before_decref, **kwds):
        self.generate_xdecref(code, cname, **kwds)
        code.putln("%s.memview = NULL; %s.data = NULL;" % (cname, cname))

    def generate_decref_clear(self, code, cname, **kwds):
        # memoryviews don't currently distinguish between xdecref and decref
        self.generate_xdecref_clear(code, cname, **kwds)

    # memoryviews don't participate in giveref/gotref
    generate_gotref = generate_xgotref = generate_xgiveref = generate_giveref = lambda *args: None


1242

1243 1244
class BufferType(BaseType):
    #
1245 1246 1247
    #  Delegates most attribute lookups to the base type.
    #  (Anything not defined here or in the BaseType is delegated.)
    #
1248 1249 1250 1251 1252 1253 1254
    # dtype            PyrexType
    # ndim             int
    # mode             str
    # negative_indices bool
    # cast             bool
    # is_buffer        bool
    # writable         bool
1255 1256

    is_buffer = 1
1257
    writable = True
1258 1259 1260

    subtypes = ['dtype']

1261
    def __init__(self, base, dtype, ndim, mode, negative_indices, cast):
1262 1263 1264
        self.base = base
        self.dtype = dtype
        self.ndim = ndim
1265
        self.buffer_ptr_type = CPtrType(dtype)
1266
        self.mode = mode
1267
        self.negative_indices = negative_indices
1268
        self.cast = cast
1269
        self.is_numpy_buffer = self.base.name == "ndarray"
1270

da-woods's avatar