Code.py 80.6 KB
Newer Older
1
# cython: language_level = 2
William Stein's avatar
William Stein committed
2
#
3
#   Code output module
William Stein's avatar
William Stein committed
4 5
#

6 7
from __future__ import absolute_import

Stefan Behnel's avatar
Stefan Behnel committed
8
import cython
9 10
cython.declare(os=object, re=object, operator=object,
               Naming=object, Options=object, StringEncoding=object,
Stefan Behnel's avatar
Stefan Behnel committed
11
               Utils=object, SourceDescriptor=object, StringIOTree=object,
12
               DebugFlags=object, basestring=object)
Stefan Behnel's avatar
Stefan Behnel committed
13

14
import os
15
import re
Stefan Behnel's avatar
Stefan Behnel committed
16
import sys
17
import operator
18
import textwrap
19 20
from string import Template
from functools import partial
Stefan Behnel's avatar
Stefan Behnel committed
21 22
from contextlib import closing
from collections import defaultdict
23

Robert Bradshaw's avatar
Robert Bradshaw committed
24 25 26 27 28
try:
    import hashlib
except ImportError:
    import md5 as hashlib

29 30 31 32 33 34 35
from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree
36

Stefan Behnel's avatar
Stefan Behnel committed
37
try:
Stefan Behnel's avatar
Stefan Behnel committed
38 39 40
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring
41

42
KEYWORDS_MUST_BE_BYTES = sys.version_info < (2,7)
Stefan Behnel's avatar
Stefan Behnel committed
43

44 45

non_portable_builtins_map = {
46
    # builtins that have different names in different Python versions
47 48
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
49
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
50
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
51
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
52 53
    }

54 55
basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
56 57
    'PyTypeObject': 'PyHeapTypeObject',
}
58

59 60 61 62
uncachable_builtins = [
    # builtin names that cannot be cached because they may or may not
    # be available at import time
    'WindowsError',
63 64
    '_',  # e.g. gettext
]
65

66 67 68 69
modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get

70 71
is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match

Stefan Behnel's avatar
Stefan Behnel committed
72

73 74 75 76 77
def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")
Mark Florisson's avatar
Mark Florisson committed
78

Stefan Behnel's avatar
Stefan Behnel committed
79

80
class UtilityCodeBase(object):
81 82 83 84 85 86
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####
87

88
        [proto declarations]
89

90
        ##### MyUtility.init #####
91

92
        [code run at module initialization]
93 94 95

        ##### MyUtility #####
        #@requires: MyOtherUtility
96
        #@substitute: naming
97

98
        [definitions]
99 100 101 102 103 104 105 106 107

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """
108 109

    is_cython_utility = False
110
    requires = None
111 112
    _utility_cache = {}

113
    @classmethod
114
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
115 116
        if utility is None:
            return
117

118 119 120 121 122 123 124 125 126 127 128
        code = '\n'.join(lines)
        if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
            del tags['substitute']
            try:
                code = Template(code).substitute(vars(Naming))
            except (KeyError, ValueError), e:
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code
129

130
        if type == 'proto':
131
            utility[0] = code
132
        elif type == 'impl':
133
            utility[1] = code
134 135
        else:
            all_tags = utility[2]
Stefan Behnel's avatar
Stefan Behnel committed
136 137
            if KEYWORDS_MUST_BE_BYTES:
                type = type.encode('ASCII')
138
            all_tags[type] = code
139

140 141
        if tags:
            all_tags = utility[2]
Stefan Behnel's avatar
Stefan Behnel committed
142
            for name, values in tags.items():
Stefan Behnel's avatar
Stefan Behnel committed
143 144
                if KEYWORDS_MUST_BE_BYTES:
                    name = name.encode('ASCII')
145
                all_tags.setdefault(name, set()).update(values)
146

147
    @classmethod
148 149 150 151 152
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

153
        filename = os.path.join(get_utility_dir(), path)
154 155 156
        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
Stefan Behnel's avatar
Stefan Behnel committed
157
            strip_comments = partial(re.compile(r'^\s*#.*').sub, '')
158
            rstrip = unicode.rstrip
159
        else:
160
            comment = '/'
Stefan Behnel's avatar
Stefan Behnel committed
161
            strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '')
162
            rstrip = partial(re.compile(r'\s*(\\?)$').sub, r'\1')
163
        match_special = re.compile(
164
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
Stefan Behnel's avatar
Stefan Behnel committed
165 166
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') %
            {'C': comment}).match
167
        match_type = re.compile('(.+)[.](proto|impl|init|cleanup)$').match
168

Stefan Behnel's avatar
Stefan Behnel committed
169
        with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f:
170
            all_lines = f.readlines()
171

Stefan Behnel's avatar
Stefan Behnel committed
172
        utilities = defaultdict(lambda: [None, None, {}])
173
        lines = []
Stefan Behnel's avatar
Stefan Behnel committed
174
        tags = defaultdict(set)
175 176 177
        utility = type = None
        begin_lineno = 0

Mark Florisson's avatar
Mark Florisson committed
178
        for lineno, line in enumerate(all_lines):
179
            m = match_special(line)
Mark Florisson's avatar
Mark Florisson committed
180
            if m:
181
                if m.group('name'):
182 183 184 185 186 187
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1
                    del lines[:]
                    tags.clear()

188 189 190 191
                    name = m.group('name')
                    mtype = match_type(name)
                    if mtype:
                        name, type = mtype.groups()
192
                    else:
193
                        type = 'impl'
Stefan Behnel's avatar
Stefan Behnel committed
194
                    utility = utilities[name]
Mark Florisson's avatar
Mark Florisson committed
195
                else:
Stefan Behnel's avatar
Stefan Behnel committed
196 197
                    tags[m.group('tag')].add(m.group('value'))
                    lines.append('')  # keep line number correct
Mark Florisson's avatar
Mark Florisson committed
198
            else:
Stefan Behnel's avatar
Stefan Behnel committed
199
                lines.append(rstrip(strip_comments(line)))
Mark Florisson's avatar
Mark Florisson committed
200

201
        if utility is None:
202 203 204
            raise ValueError("Empty utility code file")

        # Don't forget to add the last utility code
205
        cls._add_utility(utility, type, lines, begin_lineno, tags)
206

Stefan Behnel's avatar
Stefan Behnel committed
207
        utilities = dict(utilities)  # un-defaultdict-ify
208 209 210
        cls._utility_cache[path] = utilities
        return utilities

211
    @classmethod
212
    def load(cls, util_code_name, from_file=None, **kwargs):
213
        """
214 215 216 217
        Load utility code from a file specified by from_file (relative to
        Cython/Utility) and name util_code_name.  If from_file is not given,
        load it from the file util_code_name.*.  There should be only one
        file matched by this pattern.
218
        """
219 220 221
        if '::' in util_code_name:
            from_file, util_code_name = util_code_name.rsplit('::', 1)
        if not from_file:
222 223
            utility_dir = get_utility_dir()
            prefix = util_code_name + '.'
224 225 226 227 228 229 230 231 232
            try:
                listing = os.listdir(utility_dir)
            except OSError:
                # XXX the code below assumes as 'zipimport.zipimporter' instance
                # XXX should be easy to generalize, but too lazy right now to write it
                import zipfile
                global __loader__
                loader = __loader__
                archive = loader.archive
Stefan Behnel's avatar
Stefan Behnel committed
233 234 235 236
                with closing(zipfile.ZipFile(archive)) as fileobj:
                    listing = [ os.path.basename(name)
                                for name in fileobj.namelist()
                                if os.path.join(archive, name).startswith(utility_dir)]
237
            files = [ os.path.join(utility_dir, filename)
238
                      for filename in listing
239 240 241 242 243 244
                      if filename.startswith(prefix) ]
            if not files:
                raise ValueError("No match found for utility code " + util_code_name)
            if len(files) > 1:
                raise ValueError("More than one filename match found for utility code " + util_code_name)
            from_file = files[0]
245 246

        utilities = cls.load_utilities_from_file(from_file)
247 248 249 250
        proto, impl, tags = utilities[util_code_name]

        if tags:
            orig_kwargs = kwargs.copy()
Stefan Behnel's avatar
Stefan Behnel committed
251
            for name, values in tags.items():
252 253 254 255
                if name in kwargs:
                    continue
                # only pass lists when we have to: most argument expect one value or None
                if name == 'requires':
256 257 258 259 260 261 262
                    if orig_kwargs:
                        values = [cls.load(dep, from_file, **orig_kwargs)
                                  for dep in sorted(values)]
                    else:
                        # dependencies are rarely unique, so use load_cached() when we can
                        values = [cls.load_cached(dep, from_file)
                                  for dep in sorted(values)]
263 264 265 266 267
                elif not values:
                    values = None
                elif len(values) == 1:
                    values = values[0]
                kwargs[name] = values
Mark Florisson's avatar
Mark Florisson committed
268

269 270 271 272
        if proto is not None:
            kwargs['proto'] = proto
        if impl is not None:
            kwargs['impl'] = impl
Mark Florisson's avatar
Mark Florisson committed
273

274 275
        if 'name' not in kwargs:
            kwargs['name'] = util_code_name
Mark Florisson's avatar
Mark Florisson committed
276

277 278 279 280 281
        if 'file' not in kwargs and from_file:
            kwargs['file'] = from_file
        return cls(**kwargs)

    @classmethod
282
    def load_cached(cls, utility_code_name, from_file=None, __cache={}):
283
        """
284
        Calls .load(), but using a per-type cache based on utility name and file name.
285
        """
286 287
        key = (cls, from_file, utility_code_name)
        try:
288
            return __cache[key]
289 290
        except KeyError:
            pass
291
        code = __cache[key] = cls.load(utility_code_name, from_file)
292
        return code
293

294 295 296 297 298 299 300
    @classmethod
    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
        """
        Load a utility code as a string. Returns (proto, implementation)
        """
        util = cls.load(util_code_name, from_file, **kwargs)
        proto, impl = util.proto, util.impl
301
        return util.format_code(proto), util.format_code(impl)
302

303
    def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
304 305 306 307 308 309 310
        """
        Format a code section for output.
        """
        if code_string:
            code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'
        return code_string

311 312 313
    def __str__(self):
        return "<%s(%s)" % (type(self).__name__, self.name)

314 315
    def get_tree(self):
        pass
316

Stefan Behnel's avatar
Stefan Behnel committed
317

318
class UtilityCode(UtilityCodeBase):
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
    """
    Stores utility code to add during code generation.

    See GlobalState.put_utility_code.

    hashes/equals by instance

    proto           C prototypes
    impl            implemenation code
    init            code to call on module initialization
    requires        utility code dependencies
    proto_block     the place in the resulting file where the prototype should
                    end up
    name            name of the utility code (or None)
    file            filename of the utility code file this utility was loaded
                    from (or None)
    """
336

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
337
    def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
338
                 proto_block='utility_code_proto', name=None, file=None):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
339
        # proto_block: Which code block to dump prototype in. See GlobalState.
340 341 342 343 344 345 346
        self.proto = proto
        self.impl = impl
        self.init = init
        self.cleanup = cleanup
        self.requires = requires
        self._cache = {}
        self.specialize_list = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
347
        self.proto_block = proto_block
348
        self.name = name
349
        self.file = file
350

351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
    def __hash__(self):
        return hash((self.proto, self.impl))

    def __eq__(self, other):
        if self is other:
            return True
        if not isinstance(other, type(self)):
            return False

        self_proto = getattr(self, 'proto', None)
        other_proto = getattr(other, 'proto', None)
        return (self_proto, self.impl) == (other_proto, other.impl)

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return s % context

    def specialize(self, pyrex_type=None, **data):
373 374
        # Dicts aren't hashable...
        if pyrex_type is not None:
375
            data['type'] = pyrex_type.empty_declaration_code()
Craig Citro's avatar
Craig Citro committed
376
            data['type_name'] = pyrex_type.specialization_name()
Stefan Behnel's avatar
Stefan Behnel committed
377
        key = tuple(sorted(data.items()))
378 379 380 381 382 383 384
        try:
            return self._cache[key]
        except KeyError:
            if self.requires is None:
                requires = None
            else:
                requires = [r.specialize(data) for r in self.requires]
385

386
            s = self._cache[key] = UtilityCode(
387 388 389 390
                    self.none_or_sub(self.proto, data),
                    self.none_or_sub(self.impl, data),
                    self.none_or_sub(self.init, data),
                    self.none_or_sub(self.cleanup, data),
391 392 393
                    requires,
                    self.proto_block)

394 395 396
            self.specialize_list.append(s)
            return s

397 398 399
    def inject_string_constants(self, impl, output):
        """Replace 'PYIDENT("xyz")' by a constant Python identifier cname.
        """
400 401 402 403 404 405 406 407 408 409 410 411
        replacements = {}
        def externalise(matchobj):
            name = matchobj.group(1)
            try:
                cname = replacements[name]
            except KeyError:
                cname = replacements[name] = output.get_interned_identifier(
                    StringEncoding.EncodedString(name)).cname
            return cname

        impl = re.sub('PYIDENT\("([^"]+)"\)', externalise, impl)
        return bool(replacements), impl
412

413 414 415 416 417
    def put_code(self, output):
        if self.requires:
            for dependency in self.requires:
                output.use_utility_code(dependency)
        if self.proto:
418 419 420
            output[self.proto_block].put_or_include(
                self.format_code(self.proto),
                '%s_proto' % self.name)
421
        if self.impl:
422 423 424 425 426 427 428 429
            impl = self.format_code(self.impl)
            is_specialised, impl = self.inject_string_constants(impl, output)
            if not is_specialised:
                # no module specific adaptations => can be reused
                output['utility_code_def'].put_or_include(
                    impl, '%s_impl' % self.name)
            else:
                output['utility_code_def'].put(impl)
430 431
        if self.init:
            writer = output['init_globals']
432
            writer.putln("/* %s.init */" % self.name)
433
            if isinstance(self.init, basestring):
434
                writer.put(self.format_code(self.init))
435 436
            else:
                self.init(writer, output.module_pos)
437 438
            writer.putln(writer.error_goto_if_PyErr(output.module_pos))
            writer.putln()
439 440 441
        if self.cleanup and Options.generate_cleanup_code:
            writer = output['cleanup_globals']
            if isinstance(self.cleanup, basestring):
442 443 444
                writer.put_or_include(
                    self.format_code(self.cleanup),
                    '%s_cleanup' % self.name)
445 446
            else:
                self.cleanup(writer, output.module_pos)
447 448


449
def sub_tempita(s, context, file=None, name=None):
450 451 452
    "Run tempita on string s with given context."
    if not s:
        return None
453

454 455 456 457
    if file:
        context['__name'] = "%s:%s" % (file, name)
    elif name:
        context['__name'] = name
458

459
    from ..Tempita import sub
460
    return sub(s, **context)
461

462
class TempitaUtilityCode(UtilityCode):
463
    def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
464 465
        if context is None:
            context = {}
466 467
        proto = sub_tempita(proto, context, file, name)
        impl = sub_tempita(impl, context, file, name)
468
        init = sub_tempita(init, context, file, name)
469
        super(TempitaUtilityCode, self).__init__(
470
            proto, impl, init=init, name=name, file=file, **kwargs)
471 472 473 474 475 476 477 478

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return sub_tempita(s, context, self.file, self.name)
479 480 481 482 483 484 485 486 487 488 489 490 491 492 493


class LazyUtilityCode(UtilityCodeBase):
    """
    Utility code that calls a callback with the root code writer when
    available. Useful when you only have 'env' but not 'code'.
    """

    def __init__(self, callback):
        self.callback = callback

    def put_code(self, globalstate):
        utility = self.callback(globalstate.rootwriter)
        globalstate.use_utility_code(utility)

494

495 496 497 498 499 500 501 502 503
class FunctionState(object):
    # return_label     string          function return point label
    # error_label      string          error catch point label
    # continue_label   string          loop continue point label
    # break_label      string          loop break point label
    # return_from_error_cleanup_label string
    # label_counter    integer         counter for naming labels
    # in_try_finally   boolean         inside try of try...finally
    # exc_vars         (string * 3)    exception variables for reraise, or None
504
    # can_trace        boolean         line tracing is supported in the current context
505

506
    # Not used for now, perhaps later
Robert Bradshaw's avatar
Robert Bradshaw committed
507
    def __init__(self, owner, names_taken=set()):
508
        self.names_taken = names_taken
509
        self.owner = owner
510

511 512
        self.error_label = None
        self.label_counter = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
513
        self.labels_used = set()
514 515 516 517
        self.return_label = self.new_label()
        self.new_error_label()
        self.continue_label = None
        self.break_label = None
518
        self.yield_labels = []
519

520 521
        self.in_try_finally = 0
        self.exc_vars = None
522
        self.can_trace = False
523
        self.gil_owned = True
524

525
        self.temps_allocated = [] # of (name, type, manage_ref, static)
526 527
        self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
        self.temps_used_type = {} # name -> (type, manage_ref)
528
        self.temp_counter = 0
529
        self.closure_temps = None
530

531 532 533 534
        # This is used to collect temporaries, useful to find out which temps
        # need to be privatized in parallel sections
        self.collect_temps_stack = []

535 536 537 538 539
        # This is used for the error indicator, which needs to be local to the
        # function. It used to be global, which relies on the GIL being held.
        # However, exceptions may need to be propagated through 'nogil'
        # sections, in which case we introduce a race condition.
        self.should_declare_error_indicator = False
540
        self.uses_error_indicator = False
541

542 543
    # labels

544
    def new_label(self, name=None):
545 546
        n = self.label_counter
        self.label_counter = n + 1
547 548 549 550
        label = "%s%d" % (Naming.label_prefix, n)
        if name is not None:
            label += '_' + name
        return label
551

552 553 554 555 556 557
    def new_yield_label(self):
        label = self.new_label('resume_from_yield')
        num_and_label = (len(self.yield_labels) + 1, label)
        self.yield_labels.append(num_and_label)
        return num_and_label

558 559
    def new_error_label(self):
        old_err_lbl = self.error_label
560
        self.error_label = self.new_label('error')
561
        return old_err_lbl
562

563 564 565 566
    def get_loop_labels(self):
        return (
            self.continue_label,
            self.break_label)
567

568 569 570
    def set_loop_labels(self, labels):
        (self.continue_label,
         self.break_label) = labels
571

572 573 574
    def new_loop_labels(self):
        old_labels = self.get_loop_labels()
        self.set_loop_labels(
575
            (self.new_label("continue"),
Robert Bradshaw's avatar
Robert Bradshaw committed
576
             self.new_label("break")))
577
        return old_labels
578

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
    def get_all_labels(self):
        return (
            self.continue_label,
            self.break_label,
            self.return_label,
            self.error_label)

    def set_all_labels(self, labels):
        (self.continue_label,
         self.break_label,
         self.return_label,
         self.error_label) = labels

    def all_new_labels(self):
        old_labels = self.get_all_labels()
        new_labels = []
595
        for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
596
            if old_label:
597
                new_labels.append(self.new_label(name))
598 599 600 601
            else:
                new_labels.append(old_label)
        self.set_all_labels(new_labels)
        return old_labels
602

603
    def use_label(self, lbl):
Stefan Behnel's avatar
Stefan Behnel committed
604
        self.labels_used.add(lbl)
605

606 607 608
    def label_used(self, lbl):
        return lbl in self.labels_used

609 610
    # temp handling

611
    def allocate_temp(self, type, manage_ref, static=False):
612 613 614 615 616
        """
        Allocates a temporary (which may create a new one or get a previously
        allocated and released one of the same type). Type is simply registered
        and handed back, but will usually be a PyrexType.

617 618 619 620 621
        If type.is_pyobject, manage_ref comes into play. If manage_ref is set to
        True, the temp will be decref-ed on return statements and in exception
        handling clauses. Otherwise the caller has to deal with any reference
        counting of the variable.

622 623 624 625
        If not type.is_pyobject, then manage_ref will be ignored, but it
        still has to be passed. It is recommended to pass False by convention
        if it is known that type will never be a Python object.

626 627 628 629
        static=True marks the temporary declaration with "static".
        This is only used when allocating backing store for a module-level
        C array literals.

630 631
        A C string referring to the variable is returned.
        """
632 633
        if type.is_const:
            type = type.const_base_type
634
        if not type.is_pyobject and not type.is_memoryviewslice:
635 636 637
            # Make manage_ref canonical, so that manage_ref will always mean
            # a decref is needed.
            manage_ref = False
638

639
        freelist = self.temps_free.get((type, manage_ref))
640
        if freelist is not None and len(freelist) > 0:
641
            result = freelist.pop()
642
        else:
643 644 645 646
            while True:
                self.temp_counter += 1
                result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
                if not result in self.names_taken: break
647
            self.temps_allocated.append((result, type, manage_ref, static))
648
        self.temps_used_type[result] = (type, manage_ref)
649
        if DebugFlags.debug_temp_code_comments:
650
            self.owner.putln("/* %s allocated (%s) */" % (result, type))
651 652 653 654

        if self.collect_temps_stack:
            self.collect_temps_stack[-1].add((result, type))

655 656 657 658 659 660 661
        return result

    def release_temp(self, name):
        """
        Releases a temporary so that it can be reused by other code needing
        a temp of the same type.
        """
662 663
        type, manage_ref = self.temps_used_type[name]
        freelist = self.temps_free.get((type, manage_ref))
664 665
        if freelist is None:
            freelist = []
666
            self.temps_free[(type, manage_ref)] = freelist
667 668
        if name in freelist:
            raise RuntimeError("Temp %s freed twice!" % name)
669
        freelist.append(name)
670 671
        if DebugFlags.debug_temp_code_comments:
            self.owner.putln("/* %s released */" % name)
672

673
    def temps_in_use(self):
674
        """Return a list of (cname,type,manage_ref) tuples of temp names and their type
675 676 677
        that are currently in use.
        """
        used = []
678
        for name, type, manage_ref, static in self.temps_allocated:
679
            freelist = self.temps_free.get((type, manage_ref))
680
            if freelist is None or name not in freelist:
681
                used.append((name, type, manage_ref and type.is_pyobject))
682 683
        return used

684
    def temps_holding_reference(self):
685
        """Return a list of (cname,type) tuples of temp names and their type
686 687
        that are currently in use. This includes only temps of a
        Python object type which owns its reference.
688 689
        """
        return [(name, type)
690
                for name, type, manage_ref in self.temps_in_use()
691
                if manage_ref  and type.is_pyobject]
692 693 694 695 696

    def all_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python objects.
        """
        return [(cname, type)
697
                    for cname, type, manage_ref, static in self.temps_allocated
698
                        if manage_ref]
699

700 701 702 703 704 705 706
    def all_free_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python
        objects that are not currently in use.  This is used by
        try-except and try-finally blocks to clean up temps in the
        error case.
        """
        return [(cname, type)
707 708 709
                    for (type, manage_ref), freelist in self.temps_free.items()
                        if manage_ref
                            for cname in freelist]
710

711 712 713 714
    def start_collecting_temps(self):
        """
        Useful to find out which temps were used in a code block
        """
Robert Bradshaw's avatar
Robert Bradshaw committed
715
        self.collect_temps_stack.append(set())
716 717 718

    def stop_collecting_temps(self):
        return self.collect_temps_stack.pop()
719

720 721 722
    def init_closure_temps(self, scope):
        self.closure_temps = ClosureTempAllocator(scope)

723

Stefan Behnel's avatar
Stefan Behnel committed
724 725 726 727 728 729 730
class NumConst(object):
    """Global info about a Python number constant held by GlobalState.

    cname       string
    value       string
    py_type     string     int, long, float
    value_code  string     evaluation code if different from value
731 732
    """

Stefan Behnel's avatar
Stefan Behnel committed
733
    def __init__(self, cname, value, py_type, value_code=None):
734 735
        self.cname = cname
        self.value = value
Stefan Behnel's avatar
Stefan Behnel committed
736 737 738
        self.py_type = py_type
        self.value_code = value_code or value

739

740 741 742 743 744 745 746 747 748 749
class PyObjectConst(object):
    """Global info about a generic constant held by GlobalState.
    """
    # cname       string
    # type        PyrexType

    def __init__(self, cname, type):
        self.cname = cname
        self.type = type

Stefan Behnel's avatar
Stefan Behnel committed
750

751
cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
752
               replace_identifier=object, find_alphanums=object)
Stefan Behnel's avatar
Stefan Behnel committed
753 754
possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
755
replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
756
find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
757 758 759 760 761 762 763 764 765 766 767 768 769

class StringConst(object):
    """Global info about a C string constant held by GlobalState.
    """
    # cname            string
    # text             EncodedString or BytesLiteral
    # py_strings       {(identifier, encoding) : PyStringConst}

    def __init__(self, cname, text, byte_string):
        self.cname = cname
        self.text = text
        self.escaped_value = StringEncoding.escape_byte_string(byte_string)
        self.py_strings = None
770 771 772
        self.py_versions = []

    def add_py_version(self, version):
773 774 775
        if not version:
            self.py_versions = [2,3]
        elif version not in self.py_versions:
776
            self.py_versions.append(version)
777

778 779
    def get_py_string_const(self, encoding, identifier=None,
                            is_str=False, py3str_cstring=None):
780 781 782
        py_strings = self.py_strings
        text = self.text

Stefan Behnel's avatar
Stefan Behnel committed
783
        is_str = bool(identifier or is_str)
784
        is_unicode = encoding is None and not is_str
785

786 787 788 789 790 791 792 793 794 795 796 797
        if encoding is None:
            # unicode string
            encoding_key = None
        else:
            # bytes or str
            encoding = encoding.lower()
            if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
                encoding = None
                encoding_key = None
            else:
                encoding_key = ''.join(find_alphanums(encoding))

798 799 800 801 802 803
        key = (is_str, is_unicode, encoding_key, py3str_cstring)
        if py_strings is not None:
            try:
                return py_strings[key]
            except KeyError:
                pass
804
        else:
805
            self.py_strings = {}
806

807 808 809 810 811 812 813 814 815 816
        if identifier:
            intern = True
        elif identifier is None:
            if isinstance(text, unicode):
                intern = bool(possible_unicode_identifier(text))
            else:
                intern = bool(possible_bytes_identifier(text))
        else:
            intern = False
        if intern:
817
            prefix = Naming.interned_prefixes['str']
818 819
        else:
            prefix = Naming.py_const_prefix
820 821 822 823 824 825 826

        if encoding_key:
            encoding_prefix = '_%s' % encoding_key
        else:
            encoding_prefix = ''

        pystring_cname = "%s%s%s_%s" % (
827 828
            prefix,
            (is_str and 's') or (is_unicode and 'u') or 'b',
829
            encoding_prefix,
830 831 832 833 834
            self.cname[len(Naming.const_prefix):])

        py_string = PyStringConst(
            pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
        self.py_strings[key] = py_string
835 836 837 838 839 840
        return py_string

class PyStringConst(object):
    """Global info about a Python string constant held by GlobalState.
    """
    # cname       string
841
    # py3str_cstring string
842
    # encoding    string
843
    # intern      boolean
844 845
    # is_unicode  boolean
    # is_str      boolean
846

847 848
    def __init__(self, cname, encoding, is_unicode, is_str=False,
                 py3str_cstring=None, intern=False):
849
        self.cname = cname
850
        self.py3str_cstring = py3str_cstring
851 852 853
        self.encoding = encoding
        self.is_str = is_str
        self.is_unicode = is_unicode
854 855
        self.intern = intern

Stefan Behnel's avatar
Stefan Behnel committed
856 857 858
    def __lt__(self, other):
        return self.cname < other.cname

859

860 861 862 863 864 865
class GlobalState(object):
    # filename_table   {string : int}  for finding filename table indexes
    # filename_list    [string]        filenames in filename table order
    # input_file_contents dict         contents (=list of lines) of any file that was used as input
    #                                  to create this output C code.  This is
    #                                  used to annotate the comments.
866
    #
867
    # utility_codes   set                IDs of used utility code (to avoid reinsertion)
868 869 870 871 872 873
    #
    # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
    #                                  constants etc. into the pyx-declared ones (i.e,
    #                                  check if constants are already added).
    #                                  In time, hopefully the literals etc. will be
    #                                  supplied directly instead.
874
    #
875
    # const_cnames_used  dict          global counter for unique constant identifiers
876
    #
877

878 879
    # parts            {string:CCodeWriter}

880

881 882 883 884
    # interned_strings
    # consts
    # interned_nums

885 886 887 888 889 890
    # directives       set             Temporary variable used to track
    #                                  the current set of directives in the code generation
    #                                  process.

    directives = {}

891 892
    code_layout = [
        'h_code',
Robert Bradshaw's avatar
Robert Bradshaw committed
893
        'filename_table',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
894
        'utility_code_proto_before_types',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
895 896 897 898
        'numeric_typedefs',          # Let these detailed individual parts stay!,
        'complex_type_declarations', # as the proper solution is to make a full DAG...
        'type_declarations',         # More coarse-grained blocks would simply hide
        'utility_code_proto',        # the ugliness, not fix it
899 900
        'module_declarations',
        'typeinfo',
901 902
        'before_global_var',
        'global_var',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
903
        'decls',
904
        'all_the_rest',
905 906
        'pystring_table',
        'cached_builtins',
907
        'cached_constants',
908 909 910 911 912
        'init_globals',
        'init_module',
        'cleanup_globals',
        'cleanup_module',
        'main_method',
913 914
        'utility_code_def',
        'end'
915
    ]
916

917

918
    def __init__(self, writer, module_node, emit_linenums=False, common_utility_include_dir=None):
919 920 921
        self.filename_table = {}
        self.filename_list = []
        self.input_file_contents = {}
Robert Bradshaw's avatar
Robert Bradshaw committed
922
        self.utility_codes = set()
923
        self.declared_cnames = {}
924
        self.in_utility_code_generation = False
925
        self.emit_linenums = emit_linenums
926
        self.common_utility_include_dir = common_utility_include_dir
927
        self.parts = {}
928 929
        self.module_node = module_node # because some utility code generation needs it
                                       # (generating backwards-compatible Get/ReleaseBuffer
930

931
        self.const_cnames_used = {}
932
        self.string_const_index = {}
Nikita Nemkin's avatar
Nikita Nemkin committed
933
        self.pyunicode_ptr_const_index = {}
Stefan Behnel's avatar
Stefan Behnel committed
934
        self.num_const_index = {}
935
        self.py_constants = []
936

937 938
        assert writer.globalstate is None
        writer.globalstate = self
939
        self.rootwriter = writer
940

941 942 943 944
    def initialize_main_c_code(self):
        rootwriter = self.rootwriter
        for part in self.code_layout:
            self.parts[part] = rootwriter.insertion_point()
945

946 947 948 949 950 951
        if not Options.cache_builtins:
            del self.parts['cached_builtins']
        else:
            w = self.parts['cached_builtins']
            w.enter_cfunc_scope()
            w.putln("static int __Pyx_InitCachedBuiltins(void) {")
952

953 954 955 956
        w = self.parts['cached_constants']
        w.enter_cfunc_scope()
        w.putln("")
        w.putln("static int __Pyx_InitCachedConstants(void) {")
957
        w.put_declare_refcount_context()
958 959
        w.put_setup_refcount_context("__Pyx_InitCachedConstants")

960 961 962 963
        w = self.parts['init_globals']
        w.enter_cfunc_scope()
        w.putln("")
        w.putln("static int __Pyx_InitGlobals(void) {")
964

965 966 967 968 969 970 971
        if not Options.generate_cleanup_code:
            del self.parts['cleanup_globals']
        else:
            w = self.parts['cleanup_globals']
            w.enter_cfunc_scope()
            w.putln("")
            w.putln("static void __Pyx_CleanupGlobals(void) {")
972

973 974 975 976
        code = self.parts['utility_code_proto']
        code.putln("")
        code.putln("/* --- Runtime support code (head) --- */")

977 978 979 980
        code = self.parts['utility_code_def']
        if self.emit_linenums:
            code.write('\n#line 1 "cython_utility"\n')
        code.putln("")
981
        code.putln("/* --- Runtime support code --- */")
982

983
    def finalize_main_c_code(self):
984 985 986 987 988 989
        self.close_global_decls()

        #
        # utility_code_def
        #
        code = self.parts['utility_code_def']
990
        code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[1])
991 992
        code.putln("")

993 994 995
    def __getitem__(self, key):
        return self.parts[key]

996 997 998
    #
    # Global constants, interned objects, etc.
    #
999 1000
    def close_global_decls(self):
        # This is called when it is known that no more global declarations will
1001
        # declared.
1002
        self.generate_const_declarations()
1003
        if Options.cache_builtins:
1004
            w = self.parts['cached_builtins']
1005
            w.putln("return 0;")
1006 1007 1008
            if w.label_used(w.error_label):
                w.put_label(w.error_label)
                w.putln("return -1;")
1009 1010 1011
            w.putln("}")
            w.exit_cfunc_scope()

1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
        w = self.parts['cached_constants']
        w.put_finish_refcount_context()
        w.putln("return 0;")
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.put_finish_refcount_context()
            w.putln("return -1;")
        w.putln("}")
        w.exit_cfunc_scope()

1022
        w = self.parts['init_globals']
1023
        w.putln("return 0;")
1024 1025 1026
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.putln("return -1;")
1027 1028
        w.putln("}")
        w.exit_cfunc_scope()
1029

1030 1031 1032 1033 1034
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_globals']
            w.putln("}")
            w.exit_cfunc_scope()

1035 1036 1037 1038
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_module']
            w.putln("}")
            w.exit_cfunc_scope()
1039

1040
    def put_pyobject_decl(self, entry):
1041
        self['global_var'].putln("static PyObject *%s;" % entry.cname)
1042

1043 1044
    # constant handling at code generation time

1045 1046 1047
    def get_cached_constants_writer(self):
        return self.parts['cached_constants']

1048
    def get_int_const(self, str_value, longness=False):
Stefan Behnel's avatar
Stefan Behnel committed
1049
        py_type = longness and 'long' or 'int'
1050
        try:
Stefan Behnel's avatar
Stefan Behnel committed
1051
            c = self.num_const_index[(str_value, py_type)]
1052
        except KeyError:
Stefan Behnel's avatar
Stefan Behnel committed
1053 1054 1055 1056 1057 1058 1059 1060
            c = self.new_num_const(str_value, py_type)
        return c

    def get_float_const(self, str_value, value_code):
        try:
            c = self.num_const_index[(str_value, 'float')]
        except KeyError:
            c = self.new_num_const(str_value, 'float', value_code)
1061 1062
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1063
    def get_py_const(self, type, prefix='', cleanup_level=None):
1064
        # create a new Python object constant
Stefan Behnel's avatar
Stefan Behnel committed
1065
        const = self.new_py_const(type, prefix)
1066
        if cleanup_level is not None \
Stefan Behnel's avatar
Stefan Behnel committed
1067
                and cleanup_level <= Options.generate_cleanup_code:
1068
            cleanup_writer = self.parts['cleanup_globals']
1069
            cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
1070
        return const
1071

1072
    def get_string_const(self, text, py_version=None):
1073 1074
        # return a C string constant, creating a new one if necessary
        if text.is_unicode:
1075
            byte_string = text.utf8encode()
1076 1077 1078 1079 1080 1081
        else:
            byte_string = text.byteencode()
        try:
            c = self.string_const_index[byte_string]
        except KeyError:
            c = self.new_string_const(text, byte_string)
1082
        c.add_py_version(py_version)
1083 1084
        return c

1085
    def get_pyunicode_ptr_const(self, text):
1086 1087 1088
        # return a Py_UNICODE[] constant, creating a new one if necessary
        assert text.is_unicode
        try:
Nikita Nemkin's avatar
Nikita Nemkin committed
1089
            c = self.pyunicode_ptr_const_index[text]
1090
        except KeyError:
Nikita Nemkin's avatar
Nikita Nemkin committed
1091
            c = self.pyunicode_ptr_const_index[text] = self.new_const_cname()
1092 1093
        return c

1094 1095
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
1096
        # return a Python string constant, creating a new one if necessary
1097 1098
        py3str_cstring = None
        if is_str and unicode_value is not None \
1099
               and unicode_value.utf8encode() != text.byteencode():
1100 1101 1102 1103
            py3str_cstring = self.get_string_const(unicode_value, py_version=3)
            c_string = self.get_string_const(text, py_version=2)
        else:
            c_string = self.get_string_const(text)
1104 1105
        py_string = c_string.get_py_string_const(
            text.encoding, identifier, is_str, py3str_cstring)
1106 1107
        return py_string

1108 1109 1110
    def get_interned_identifier(self, text):
        return self.get_py_string_const(text, identifier=True)

1111
    def new_string_const(self, text, byte_string):
Stefan Behnel's avatar
Stefan Behnel committed
1112
        cname = self.new_string_const_cname(byte_string)
1113 1114 1115 1116
        c = StringConst(cname, text, byte_string)
        self.string_const_index[byte_string] = c
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1117 1118 1119 1120
    def new_num_const(self, value, py_type, value_code=None):
        cname = self.new_num_const_cname(value, py_type)
        c = NumConst(cname, value, py_type, value_code)
        self.num_const_index[(value, py_type)] = c
1121 1122
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1123 1124
    def new_py_const(self, type, prefix=''):
        cname = self.new_const_cname(prefix)
1125 1126 1127 1128
        c = PyObjectConst(cname, type)
        self.py_constants.append(c)
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1129
    def new_string_const_cname(self, bytes_value):
1130
        # Create a new globally-unique nice name for a C string constant.
Stefan Behnel's avatar
Stefan Behnel committed
1131
        value = bytes_value.decode('ASCII', 'ignore')
1132
        return self.new_const_cname(value=value)
1133

Stefan Behnel's avatar
Stefan Behnel committed
1134 1135
    def new_num_const_cname(self, value, py_type):
        if py_type == 'long':
1136
            value += 'L'
1137 1138
            py_type = 'int'
        prefix = Naming.interned_prefixes[py_type]
Stefan Behnel's avatar
Stefan Behnel committed
1139
        cname = "%s%s" % (prefix, value)
1140
        cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_')
1141 1142
        return cname

1143 1144
    def new_const_cname(self, prefix='', value=''):
        value = replace_identifier('_', value)[:32].strip('_')
1145 1146 1147
        used = self.const_cnames_used
        name_suffix = value
        while name_suffix in used:
1148
            counter = used[value] = used[value] + 1
1149
            name_suffix = '%s_%d' % (value, counter)
1150
        used[name_suffix] = 1
1151 1152 1153 1154 1155
        if prefix:
            prefix = Naming.interned_prefixes[prefix]
        else:
            prefix = Naming.const_prefix
        return "%s%s" % (prefix, name_suffix)
1156 1157

    def add_cached_builtin_decl(self, entry):
1158
        if entry.is_builtin and entry.is_const:
1159 1160
            if self.should_declare(entry.cname, entry):
                self.put_pyobject_decl(entry)
1161
                w = self.parts['cached_builtins']
1162 1163 1164 1165
                condition = None
                if entry.name in non_portable_builtins_map:
                    condition, replacement = non_portable_builtins_map[entry.name]
                    w.putln('#if %s' % condition)
1166
                    self.put_cached_builtin_init(
1167
                        entry.pos, StringEncoding.EncodedString(replacement),
1168
                        entry.cname)
1169 1170 1171 1172
                    w.putln('#else')
                self.put_cached_builtin_init(
                    entry.pos, StringEncoding.EncodedString(entry.name),
                    entry.cname)
1173
                if condition:
1174 1175 1176 1177 1178
                    w.putln('#endif')

    def put_cached_builtin_init(self, pos, name, cname):
        w = self.parts['cached_builtins']
        interned_cname = self.get_interned_identifier(name).cname
1179 1180 1181
        self.use_utility_code(
            UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c"))
        w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % (
1182 1183 1184 1185
            cname,
            interned_cname,
            cname,
            w.error_goto(pos)))
1186 1187 1188

    def generate_const_declarations(self):
        self.generate_string_constants()
Stefan Behnel's avatar
Stefan Behnel committed
1189
        self.generate_num_constants()
1190 1191 1192 1193 1194 1195
        self.generate_object_constant_decls()

    def generate_object_constant_decls(self):
        consts = [ (len(c.cname), c.cname, c)
                   for c in self.py_constants ]
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1196
        decls_writer = self.parts['decls']
1197
        for _, cname, c in consts:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1198
            decls_writer.putln(
1199
                "static %s;" % c.type.declaration_code(cname))
1200 1201 1202

    def generate_string_constants(self):
        c_consts = [ (len(c.cname), c.cname, c)
Stefan Behnel's avatar
Stefan Behnel committed
1203
                     for c in self.string_const_index.values() ]
1204 1205
        c_consts.sort()
        py_strings = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1206 1207

        decls_writer = self.parts['decls']
1208
        for _, cname, c in c_consts:
1209 1210 1211 1212 1213
            conditional = False
            if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
                conditional = True
                decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
                    (2 in c.py_versions) and '<' or '>='))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1214
            decls_writer.putln('static char %s[] = "%s";' % (
1215
                cname, StringEncoding.split_string_literal(c.escaped_value)))
1216 1217
            if conditional:
                decls_writer.putln("#endif")
1218
            if c.py_strings is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1219
                for py_string in c.py_strings.values():
1220 1221
                    py_strings.append((c.cname, len(py_string.cname), py_string))

Nikita Nemkin's avatar
Nikita Nemkin committed
1222
        for c, cname in self.pyunicode_ptr_const_index.items():
1223
            utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
1224 1225
            if utf16_array:
                # Narrow and wide representations differ
Nikita Nemkin's avatar
Nikita Nemkin committed
1226
                decls_writer.putln("#ifdef Py_UNICODE_WIDE")
1227 1228 1229 1230 1231
            decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array))
            if utf16_array:
                decls_writer.putln("#else")
                decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
                decls_writer.putln("#endif")
1232

1233
        if py_strings:
1234
            self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
1235
            py_strings.sort()
1236 1237 1238
            w = self.parts['pystring_table']
            w.putln("")
            w.putln("static __Pyx_StringTabEntry %s[] = {" %
1239 1240
                                      Naming.stringtab_cname)
            for c_cname, _, py_string in py_strings:
1241 1242 1243 1244 1245 1246 1247
                if not py_string.is_str or not py_string.encoding or \
                       py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
                                              'UTF8', 'UTF-8'):
                    encoding = '0'
                else:
                    encoding = '"%s"' % py_string.encoding.lower()

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1248
                decls_writer.putln(
1249
                    "static PyObject *%s;" % py_string.cname)
1250 1251 1252 1253 1254 1255 1256
                if py_string.py3str_cstring:
                    w.putln("#if PY_MAJOR_VERSION >= 3")
                    w.putln(
                        "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
                        py_string.cname,
                        py_string.py3str_cstring.cname,
                        py_string.py3str_cstring.cname,
1257 1258
                        '0', 1, 0,
                        py_string.intern
1259 1260
                        ))
                    w.putln("#else")
1261
                w.putln(
1262
                    "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1263 1264 1265
                    py_string.cname,
                    c_cname,
                    c_cname,
1266 1267 1268 1269
                    encoding,
                    py_string.is_unicode,
                    py_string.is_str,
                    py_string.intern
1270
                    ))
1271 1272
                if py_string.py3str_cstring:
                    w.putln("#endif")
1273
            w.putln("{0, 0, 0, 0, 0, 0, 0}")
1274
            w.putln("};")
1275

1276 1277
            init_globals = self.parts['init_globals']
            init_globals.putln(
1278 1279
                "if (__Pyx_InitStrings(%s) < 0) %s;" % (
                    Naming.stringtab_cname,
1280
                    init_globals.error_goto(self.module_pos)))
1281

Stefan Behnel's avatar
Stefan Behnel committed
1282
    def generate_num_constants(self):
1283
        consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
Stefan Behnel's avatar
Stefan Behnel committed
1284
                  for c in self.num_const_index.values()]
1285
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1286
        decls_writer = self.parts['decls']
Stefan Behnel's avatar
Stefan Behnel committed
1287
        init_globals = self.parts['init_globals']
1288
        for py_type, _, _, value, value_code, c in consts:
1289
            cname = c.cname
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1290
            decls_writer.putln("static PyObject *%s;" % cname)
Stefan Behnel's avatar
Stefan Behnel committed
1291
            if py_type == 'float':
Stefan Behnel's avatar
Stefan Behnel committed
1292
                function = 'PyFloat_FromDouble(%s)'
Stefan Behnel's avatar
Stefan Behnel committed
1293
            elif py_type == 'long':
Stefan Behnel's avatar
Stefan Behnel committed
1294
                function = 'PyLong_FromString((char *)"%s", 0, 0)'
1295
            elif Utils.long_literal(value):
Stefan Behnel's avatar
Stefan Behnel committed
1296
                function = 'PyInt_FromString((char *)"%s", 0, 0)'
1297 1298
            elif len(value.lstrip('-')) > 4:
                function = "PyInt_FromLong(%sL)"
1299
            else:
Stefan Behnel's avatar
Stefan Behnel committed
1300 1301 1302
                function = "PyInt_FromLong(%s)"
            init_globals.putln('%s = %s; %s' % (
                cname, function % value_code,
1303
                init_globals.error_goto_if_null(cname, self.module_pos)))
1304

1305 1306 1307
    # The functions below are there in a transition phase only
    # and will be deprecated. They are called from Nodes.BlockNode.
    # The copy&paste duplication is intentional in order to be able
1308
    # to see quickly how BlockNode worked, until this is replaced.
1309

1310 1311 1312
    def should_declare(self, cname, entry):
        if cname in self.declared_cnames:
            other = self.declared_cnames[cname]
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1313
            assert str(entry.type) == str(other.type)
1314 1315 1316 1317 1318 1319 1320 1321 1322
            assert entry.init == other.init
            return False
        else:
            self.declared_cnames[cname] = entry
            return True

    #
    # File name state
    #
1323

1324
    def lookup_filename(self, source_desc):
1325
        try:
1326
            index = self.filename_table[source_desc.get_filenametable_entry()]
1327 1328
        except KeyError:
            index = len(self.filename_list)
1329 1330
            self.filename_list.append(source_desc)
            self.filename_table[source_desc.get_filenametable_entry()] = index
1331 1332 1333 1334 1335 1336
        return index

    def commented_file_contents(self, source_desc):
        try:
            return self.input_file_contents[source_desc]
        except KeyError:
1337 1338 1339 1340
            pass
        source_file = source_desc.get_lines(encoding='ASCII',
                                            error_handling='ignore')
        try:
1341 1342
            F = [u' * ' + line.rstrip().replace(
                    u'*/', u'*[inserted by cython to avoid comment closer]/'
1343 1344
                    ).replace(
                    u'/*', u'/[inserted by cython to avoid comment start]*'
1345
                    )
1346 1347 1348 1349 1350 1351 1352
                 for line in source_file]
        finally:
            if hasattr(source_file, 'close'):
                source_file.close()
        if not F: F.append(u'')
        self.input_file_contents[source_desc] = F
        return F
1353

1354 1355 1356
    #
    # Utility code state
    #
1357

1358
    def use_utility_code(self, utility_code):
1359
        """
1360 1361 1362 1363
        Adds code to the C file. utility_code should
        a) implement __eq__/__hash__ for the purpose of knowing whether the same
           code has already been included
        b) implement put_code, which takes a globalstate instance
1364

1365
        See UtilityCode.
1366
        """
1367 1368 1369
        if utility_code not in self.utility_codes:
            self.utility_codes.add(utility_code)
            utility_code.put_code(self)
1370

1371

1372
def funccontext_property(name):
1373
    attribute_of = operator.attrgetter(name)
1374
    def get(self):
1375
        return attribute_of(self.funcstate)
1376
    def set(self, value):
1377
        setattr(self.funcstate, name, value)
1378 1379
    return property(get, set)

1380

1381
class CCodeWriter(object):
1382
    """
1383
    Utility class to output C code.
1384

1385
    When creating an insertion point one must care about the state that is
1386
    kept:
1387 1388
    - formatting state (level, bol) is cloned and used in insertion points
      as well
1389 1390
    - labels, temps, exc_vars: One must construct a scope in which these can
      exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
1391 1392 1393 1394 1395
      sanity checking and forward compatabilty). Created insertion points
      looses this scope and cannot access it.
    - marker: Not copied to insertion point
    - filename_table, filename_list, input_file_contents: All codewriters
      coming from the same root share the same instances simultaneously.
1396
    """
1397

1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
    # f                   file            output file
    # buffer              StringIOTree

    # level               int             indentation level
    # bol                 bool            beginning of line?
    # marker              string          comment to emit before next line
    # funcstate           FunctionState   contains state local to a C function used for code
    #                                     generation (labels and temps state etc.)
    # globalstate         GlobalState     contains state global for a C file (input file info,
    #                                     utility code, declared constants etc.)
    # emit_linenums       boolean         whether or not to write #line pragmas
1409
    #
1410 1411 1412 1413
    # c_line_in_traceback boolean         append the c file and line number to the traceback for exceptions
    #
    # pyclass_stack       list            used during recursive code generation to pass information
    #                                     about the current class one is in
1414

1415
    globalstate = None
1416

1417
    def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None, c_line_in_traceback=True):
1418 1419
        if buffer is None: buffer = StringIOTree()
        self.buffer = buffer
1420 1421
        self.last_pos = None
        self.last_marked_pos = None
1422
        self.pyclass_stack = []
1423

1424 1425
        self.funcstate = None
        self.level = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
1426
        self.call_level = 0
1427
        self.bol = 1
1428 1429

        if create_from is not None:
1430 1431
            # Use same global state
            self.globalstate = create_from.globalstate
1432
            self.funcstate = create_from.funcstate
1433
            # Clone formatting state
1434 1435 1436
            if copy_formatting:
                self.level = create_from.level
                self.bol = create_from.bol
Robert Bradshaw's avatar
Robert Bradshaw committed
1437
                self.call_level = create_from.call_level
1438 1439
            self.last_pos = create_from.last_pos
            self.last_marked_pos = create_from.last_marked_pos
1440

1441
        if emit_linenums is None and self.globalstate:
1442 1443 1444
            self.emit_linenums = self.globalstate.emit_linenums
        else:
            self.emit_linenums = emit_linenums
1445
        self.c_line_in_traceback = c_line_in_traceback
1446

1447
    def create_new(self, create_from, buffer, copy_formatting):
1448 1449
        # polymorphic constructor -- very slightly more versatile
        # than using __class__
1450 1451
        result = CCodeWriter(create_from, buffer, copy_formatting,
                             c_line_in_traceback=self.c_line_in_traceback)
1452
        return result
1453 1454 1455 1456

    def copyto(self, f):
        self.buffer.copyto(f)

1457 1458 1459 1460
    def getvalue(self):
        return self.buffer.getvalue()

    def write(self, s):
1461
        # also put invalid markers (lineno 0), to indicate that those lines
Mark Florisson's avatar
Mark Florisson committed
1462
        # have no Cython source code correspondence
1463
        cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0
Mark Florisson's avatar
Mark Florisson committed
1464
        self.buffer.markers.extend([cython_lineno] * s.count('\n'))
1465
        self.buffer.write(s)
1466 1467

    def insertion_point(self):
1468
        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
1469 1470
        return other

1471 1472 1473 1474 1475
    def new_writer(self):
        """
        Creates a new CCodeWriter connected to the same global state, which
        can later be inserted using insert.
        """
1476
        return CCodeWriter(create_from=self, c_line_in_traceback=self.c_line_in_traceback)
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486

    def insert(self, writer):
        """
        Inserts the contents of another code writer (created with
        the same global state) in the current location.

        It is ok to write to the inserted writer also after insertion.
        """
        assert writer.globalstate is self.globalstate
        self.buffer.insert(writer.buffer)
1487 1488 1489 1490 1491 1492 1493 1494

    # Properties delegated to function scope
    label_counter = funccontext_property("label_counter")
    return_label = funccontext_property("return_label")
    error_label = funccontext_property("error_label")
    labels_used = funccontext_property("labels_used")
    continue_label = funccontext_property("continue_label")
    break_label = funccontext_property("break_label")
1495
    return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label")
1496
    yield_labels = funccontext_property("yield_labels")
1497 1498

    # Functions delegated to function scope
Dag Sverre Seljebotn's avatar
merge  
Dag Sverre Seljebotn committed
1499
    def new_label(self, name=None):    return self.funcstate.new_label(name)
1500
    def new_error_label(self):         return self.funcstate.new_error_label()
1501
    def new_yield_label(self):         return self.funcstate.new_yield_label()
1502 1503 1504 1505 1506 1507 1508 1509
    def get_loop_labels(self):         return self.funcstate.get_loop_labels()
    def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
    def new_loop_labels(self):         return self.funcstate.new_loop_labels()
    def get_all_labels(self):          return self.funcstate.get_all_labels()
    def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
    def all_new_labels(self):          return self.funcstate.all_new_labels()
    def use_label(self, lbl):          return self.funcstate.use_label(lbl)
    def label_used(self, lbl):         return self.funcstate.label_used(lbl)
1510 1511


1512
    def enter_cfunc_scope(self):
1513
        self.funcstate = FunctionState(self)
1514

1515
    def exit_cfunc_scope(self):
1516
        self.funcstate = None
1517

1518 1519
    # constant handling

Stefan Behnel's avatar
Stefan Behnel committed
1520
    def get_py_int(self, str_value, longness):
1521 1522
        return self.globalstate.get_int_const(str_value, longness).cname

Stefan Behnel's avatar
Stefan Behnel committed
1523 1524 1525
    def get_py_float(self, str_value, value_code):
        return self.globalstate.get_float_const(str_value, value_code).cname

1526 1527
    def get_py_const(self, type, prefix='', cleanup_level=None):
        return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
1528

1529 1530 1531
    def get_string_const(self, text):
        return self.globalstate.get_string_const(text).cname

1532 1533
    def get_pyunicode_ptr_const(self, text):
        return self.globalstate.get_pyunicode_ptr_const(text)
1534

1535 1536 1537 1538
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
        return self.globalstate.get_py_string_const(
            text, identifier, is_str, unicode_value).cname
1539

1540 1541 1542
    def get_argument_default_const(self, type):
        return self.globalstate.get_py_const(type).cname

1543 1544 1545 1546
    def intern(self, text):
        return self.get_py_string_const(text)

    def intern_identifier(self, text):
1547
        return self.get_py_string_const(text, identifier=True)
1548

1549 1550 1551
    def get_cached_constants_writer(self):
        return self.globalstate.get_cached_constants_writer()

1552 1553
    # code generation

Stefan Behnel's avatar
Stefan Behnel committed
1554
    def putln(self, code="", safe=False):
1555
        if self.last_pos and self.bol:
William Stein's avatar
William Stein committed
1556
            self.emit_marker()
1557 1558 1559
        if self.emit_linenums and self.last_marked_pos:
            source_desc, line, _ = self.last_marked_pos
            self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description()))
William Stein's avatar
William Stein committed
1560
        if code:
1561 1562 1563 1564
            if safe:
                self.put_safe(code)
            else:
                self.put(code)
1565
        self.write("\n")
William Stein's avatar
William Stein committed
1566
        self.bol = 1
1567

1568
    def mark_pos(self, pos, trace=True):
1569 1570 1571 1572
        if pos is None:
            return
        if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]:
            return
1573
        self.last_pos = (pos, trace)
1574

William Stein's avatar
William Stein committed
1575
    def emit_marker(self):
1576 1577
        pos, trace = self.last_pos
        self.last_marked_pos = pos
1578
        self.last_pos = None
1579
        self.write("\n")
William Stein's avatar
William Stein committed
1580
        self.indent()
1581
        self.write("/* %s */\n" % self._build_marker(pos))
1582
        if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']:
1583
            self.indent()
1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
            self.write('__Pyx_TraceLine(%d,%d,%s)\n' % (
                pos[1], not self.funcstate.gil_owned, self.error_goto(pos)))

    def _build_marker(self, pos):
        source_desc, line, col = pos
        assert isinstance(source_desc, SourceDescriptor)
        contents = self.globalstate.commented_file_contents(source_desc)
        lines = contents[max(0, line-3):line]  # line numbers start at 1
        lines[-1] += u'             # <<<<<<<<<<<<<<'
        lines += contents[line:line+2]
        return u'"%s":%d\n%s\n' % (source_desc.get_escaped_description(), line, u'\n'.join(lines))
William Stein's avatar
William Stein committed
1595

1596 1597 1598 1599 1600
    def put_safe(self, code):
        # put code, but ignore {}
        self.write(code)
        self.bol = 0

1601
    def put_or_include(self, code, name):
Stefan Behnel's avatar
Stefan Behnel committed
1602
        include_dir = self.globalstate.common_utility_include_dir
1603
        if include_dir and len(code) > 1024:
Stefan Behnel's avatar
Stefan Behnel committed
1604
            include_file = "%s_%s.h" % (
1605
                name, hashlib.md5(code.encode('utf8')).hexdigest())
Stefan Behnel's avatar
Stefan Behnel committed
1606 1607 1608
            path = os.path.join(include_dir, include_file)
            if not os.path.exists(path):
                tmp_path = '%s.tmp%s' % (path, os.getpid())
Stefan Behnel's avatar
Stefan Behnel committed
1609
                with closing(Utils.open_new_file(tmp_path)) as f:
Stefan Behnel's avatar
Stefan Behnel committed
1610 1611 1612 1613
                    f.write(code)
                os.rename(tmp_path, path)
            code = '#include "%s"\n' % path
        self.put(code)
1614

William Stein's avatar
William Stein committed
1615
    def put(self, code):
1616 1617
        if is_self_assignment(code):
            return
1618
        fix_indent = False
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
        if "{" in code:
            dl = code.count("{")
        else:
            dl = 0
        if "}" in code:
            dl -= code.count("}")
            if dl < 0:
                self.level += dl
            elif dl == 0 and code[0] == "}":
                # special cases like "} else {" need a temporary dedent
                fix_indent = True
                self.level -= 1
William Stein's avatar
William Stein committed
1631 1632
        if self.bol:
            self.indent()
1633
        self.write(code)
William Stein's avatar
William Stein committed
1634 1635 1636
        self.bol = 0
        if dl > 0:
            self.level += dl
1637
        elif fix_indent:
1638 1639
            self.level += 1

Mark Florisson's avatar
Mark Florisson committed
1640
    def putln_tempita(self, code, **context):
1641
        from ..Tempita import sub
1642
        self.putln(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1643 1644

    def put_tempita(self, code, **context):
1645
        from ..Tempita import sub
1646
        self.put(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1647

William Stein's avatar
William Stein committed
1648
    def increase_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1649
        self.level += 1
1650

William Stein's avatar
William Stein committed
1651
    def decrease_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1652
        self.level -= 1
1653

William Stein's avatar
William Stein committed
1654 1655 1656
    def begin_block(self):
        self.putln("{")
        self.increase_indent()
1657

William Stein's avatar
William Stein committed
1658 1659 1660
    def end_block(self):
        self.decrease_indent()
        self.putln("}")
1661

William Stein's avatar
William Stein committed
1662
    def indent(self):
1663
        self.write("  " * self.level)
1664

1665 1666 1667
    def get_py_version_hex(self, pyversion):
        return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]

William Stein's avatar
William Stein committed
1668
    def put_label(self, lbl):
1669
        if lbl in self.funcstate.labels_used:
1670
            self.putln("%s:;" % lbl)
1671

1672
    def put_goto(self, lbl):
1673
        self.funcstate.use_label(lbl)
1674
        self.putln("goto %s;" % lbl)
1675

1676
    def put_var_declaration(self, entry, storage_class="",
Stefan Behnel's avatar
Stefan Behnel committed
1677
                            dll_linkage=None, definition=True):
1678
        #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
1679 1680
        if entry.visibility == 'private' and not (definition or entry.defined_in_pxd):
            #print "...private and not definition, skipping", entry.cname ###
1681
            return
1682 1683
        if entry.visibility == "private" and not entry.used:
            #print "...private and not used, skipping", entry.cname ###
William Stein's avatar
William Stein committed
1684 1685 1686
            return
        if storage_class:
            self.put("%s " % storage_class)
1687 1688
        if not entry.cf_used:
            self.put('CYTHON_UNUSED ')
1689
        self.put(entry.type.declaration_code(
Stefan Behnel's avatar
Stefan Behnel committed
1690
            entry.cname, dll_linkage=dll_linkage))
William Stein's avatar
William Stein committed
1691
        if entry.init is not None:
1692
            self.put_safe(" = %s" % entry.type.literal_code(entry.init))
1693
        elif entry.type.is_pyobject:
Stefan Behnel's avatar
Stefan Behnel committed
1694
            self.put(" = NULL")
William Stein's avatar
William Stein committed
1695
        self.putln(";")
1696 1697

    def put_temp_declarations(self, func_context):
1698
        for name, type, manage_ref, static in func_context.temps_allocated:
1699 1700 1701
            decl = type.declaration_code(name)
            if type.is_pyobject:
                self.putln("%s = NULL;" % decl)
1702
            elif type.is_memoryviewslice:
1703
                from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
1704
                self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init))
1705
            else:
1706
                self.putln("%s%s;" % (static and "static " or "", decl))
1707

1708
        if func_context.should_declare_error_indicator:
1709 1710 1711 1712
            if self.funcstate.uses_error_indicator:
                unused = ''
            else:
                unused = 'CYTHON_UNUSED '
1713
            # Initialize these variables to silence compiler warnings
1714 1715 1716
            self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname))
            self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname))
            self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))
1717

1718 1719 1720
    def put_h_guard(self, guard):
        self.putln("#ifndef %s" % guard)
        self.putln("#define %s" % guard)
1721

1722 1723 1724 1725 1726 1727
    def unlikely(self, cond):
        if Options.gcc_branch_hints:
            return 'unlikely(%s)' % cond
        else:
            return cond

1728 1729 1730 1731 1732
    def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper):
        if not modifiers:
            return ''
        return '%s ' % ' '.join([mapper(m,m) for m in modifiers])

1733 1734
    # Python objects and reference counting

William Stein's avatar
William Stein committed
1735 1736
    def entry_as_pyobject(self, entry):
        type = entry.type
1737 1738
        if (not entry.is_self_arg and not entry.type.is_complete()
            or entry.type.is_extension_type):
William Stein's avatar
William Stein committed
1739 1740 1741
            return "(PyObject *)" + entry.cname
        else:
            return entry.cname
1742

William Stein's avatar
William Stein committed
1743
    def as_pyobject(self, cname, type):
1744
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
1745
        return typecast(py_object_type, type, cname)
1746

1747
    def put_gotref(self, cname):
1748
        self.putln("__Pyx_GOTREF(%s);" % cname)
1749

1750 1751
    def put_giveref(self, cname):
        self.putln("__Pyx_GIVEREF(%s);" % cname)
1752

1753 1754 1755
    def put_xgiveref(self, cname):
        self.putln("__Pyx_XGIVEREF(%s);" % cname)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1756 1757 1758
    def put_xgotref(self, cname):
        self.putln("__Pyx_XGOTREF(%s);" % cname)

1759 1760 1761 1762 1763
    def put_incref(self, cname, type, nanny=True):
        if nanny:
            self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
        else:
            self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
1764

1765
    def put_decref(self, cname, type, nanny=True):
1766
        self._put_decref(cname, type, nanny, null_check=False, clear=False)
1767 1768 1769 1770

    def put_var_gotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
1771

1772 1773 1774 1775
    def put_var_giveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))

1776 1777 1778 1779
    def put_var_xgotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))

1780 1781 1782 1783
    def put_var_xgiveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))

William Stein's avatar
William Stein committed
1784 1785
    def put_var_incref(self, entry):
        if entry.type.is_pyobject:
1786
            self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
1787

1788 1789 1790
    def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=False,
                         clear=True, clear_before_decref=clear_before_decref)
1791

1792
    def put_xdecref(self, cname, type, nanny=True, have_gil=True):
1793 1794
        self._put_decref(cname, type, nanny, null_check=True,
                         have_gil=have_gil, clear=False)
1795

1796 1797 1798
    def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=True,
                         clear=True, clear_before_decref=clear_before_decref)
1799

1800 1801
    def _put_decref(self, cname, type, nanny=True, null_check=False,
                    have_gil=True, clear=False, clear_before_decref=False):
1802
        if type.is_memoryviewslice:
1803
            self.put_xdecref_memoryviewslice(cname, have_gil=have_gil)
1804 1805
            return

1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
        prefix = nanny and '__Pyx' or 'Py'
        X = null_check and 'X' or ''

        if clear:
            if clear_before_decref:
                if not nanny:
                    X = ''  # CPython doesn't have a Py_XCLEAR()
                self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname))
            else:
                self.putln("%s_%sDECREF(%s); %s = 0;" % (
                    prefix, X, self.as_pyobject(cname, type), cname))
1817
        else:
1818 1819
            self.putln("%s_%sDECREF(%s);" % (
                prefix, X, self.as_pyobject(cname, type)))
William Stein's avatar
William Stein committed
1820

1821 1822 1823 1824 1825 1826
    def put_decref_set(self, cname, rhs_cname):
        self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname))

    def put_xdecref_set(self, cname, rhs_cname):
        self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname))

William Stein's avatar
William Stein committed
1827 1828
    def put_var_decref(self, entry):
        if entry.type.is_pyobject:
1829
            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1830

William Stein's avatar
William Stein committed
1831 1832
    def put_var_xdecref(self, entry):
        if entry.type.is_pyobject:
1833
            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1834

1835 1836 1837
    def put_var_decref_clear(self, entry):
        self._put_var_decref_clear(entry, null_check=False)

William Stein's avatar
William Stein committed
1838
    def put_var_xdecref_clear(self, entry):
1839 1840 1841
        self._put_var_decref_clear(entry, null_check=True)

    def _put_var_decref_clear(self, entry, null_check):
William Stein's avatar
William Stein committed
1842
        if entry.type.is_pyobject:
1843 1844 1845 1846 1847 1848 1849 1850 1851
            if entry.in_closure:
                # reset before DECREF to make sure closure state is
                # consistent during call to DECREF()
                self.putln("__Pyx_%sCLEAR(%s);" % (
                    null_check and 'X' or '',
                    entry.cname))
            else:
                self.putln("__Pyx_%sDECREF(%s); %s = 0;" % (
                    null_check and 'X' or '',
1852
                    self.entry_as_pyobject(entry),
1853
                    entry.cname))
1854

1855
    def put_var_decrefs(self, entries, used_only = 0):
William Stein's avatar
William Stein committed
1856
        for entry in entries:
1857 1858 1859 1860 1861
            if not used_only or entry.used:
                if entry.xdecref_cleanup:
                    self.put_var_xdecref(entry)
                else:
                    self.put_var_decref(entry)
1862

William Stein's avatar
William Stein committed
1863 1864 1865
    def put_var_xdecrefs(self, entries):
        for entry in entries:
            self.put_var_xdecref(entry)
1866

William Stein's avatar
William Stein committed
1867 1868 1869
    def put_var_xdecrefs_clear(self, entries):
        for entry in entries:
            self.put_var_xdecref_clear(entry)
1870

1871
    def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
1872
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
1873
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
1874 1875 1876
        self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
1877
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
1878
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
1879 1880 1881 1882 1883
        self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xgiveref_memoryviewslice(self, slice_cname):
        self.put_xgiveref("%s.memview" % slice_cname)

1884
    def put_init_to_py_none(self, cname, type, nanny=True):
1885
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
1886
        py_none = typecast(type, py_object_type, "Py_None")
1887 1888 1889 1890
        if nanny:
            self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
        else:
            self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
1891

1892
    def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
William Stein's avatar
William Stein committed
1893 1894
        code = template % entry.cname
        #if entry.type.is_extension_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
1895
        #    code = "((PyObject*)%s)" % code
1896
        self.put_init_to_py_none(code, entry.type, nanny)
1897
        if entry.in_closure:
Stefan Behnel's avatar
Stefan Behnel committed
1898
            self.put_giveref('Py_None')
William Stein's avatar
William Stein committed
1899

1900
    def put_pymethoddef(self, entry, term, allow_skip=True):
1901
        if entry.is_special or entry.name == '__getattribute__':
1902 1903 1904
            if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcharbuffer__', '__getbuffer__', '__releasebuffer__']:
                if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
                    pass
1905 1906 1907
                # Python's typeobject.c will automatically fill in our slot
                # in add_operators() (called by PyType_Ready) with a value
                # that's better than ours.
1908
                elif allow_skip:
1909
                    return
1910
        from .TypeSlots import method_coexist
William Stein's avatar
William Stein committed
1911 1912 1913 1914
        if entry.doc:
            doc_code = entry.doc_cname
        else:
            doc_code = 0
1915 1916
        method_flags = entry.signature.method_flags()
        if method_flags:
1917
            if entry.is_special:
1918
                method_flags += [method_coexist]
1919
            self.putln(
1920
                '{"%s", (PyCFunction)%s, %s, %s}%s' % (
1921
                    entry.name,
1922
                    entry.func_cname,
1923
                    "|".join(method_flags),
1924 1925
                    doc_code,
                    term))
1926

1927 1928
    # GIL methods

1929
    def put_ensure_gil(self, declare_gilstate=True, variable=None):
1930 1931 1932 1933 1934 1935
        """
        Acquire the GIL. The generated code is safe even when no PyThreadState
        has been allocated for this thread (for threads not initialized by
        using the Python API). Additionally, the code generated by this method
        may be called recursively.
        """
1936 1937
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
1938
        self.putln("#ifdef WITH_THREAD")
1939 1940 1941 1942 1943
        if not variable:
            variable = '__pyx_gilstate_save'
            if declare_gilstate:
                self.put("PyGILState_STATE ")
        self.putln("%s = PyGILState_Ensure();" % variable)
1944 1945
        self.putln("#endif")

1946
    def put_release_ensured_gil(self, variable=None):
1947 1948 1949
        """
        Releases the GIL, corresponds to `put_ensure_gil`.
        """
1950 1951
        if not variable:
            variable = '__pyx_gilstate_save'
1952
        self.putln("#ifdef WITH_THREAD")
1953
        self.putln("PyGILState_Release(%s);" % variable)
1954 1955
        self.putln("#endif")

1956
    def put_acquire_gil(self, variable=None):
1957 1958 1959 1960
        """
        Acquire the GIL. The thread's thread state must have been initialized
        by a previous `put_release_gil`
        """
1961 1962 1963
        self.putln("#ifdef WITH_THREAD")
        if variable:
            self.putln('_save = %s;' % variable)
1964
        self.putln("Py_BLOCK_THREADS")
1965
        self.putln("#endif")
1966

1967
    def put_release_gil(self, variable=None):
1968 1969
        "Release the GIL, corresponds to `put_acquire_gil`."
        self.putln("#ifdef WITH_THREAD")
1970
        self.putln("PyThreadState *_save;")
1971
        self.putln("Py_UNBLOCK_THREADS")
1972 1973 1974
        if variable:
            self.putln('%s = _save;' % variable)
        self.putln("#endif")
1975

1976 1977
    def declare_gilstate(self):
        self.putln("#ifdef WITH_THREAD")
1978
        self.putln("PyGILState_STATE __pyx_gilstate_save;")
1979 1980
        self.putln("#endif")

1981 1982
    # error handling

Robert Bradshaw's avatar
Robert Bradshaw committed
1983 1984 1985 1986
    def put_error_if_neg(self, pos, value):
#        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
        return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))

1987
    def put_error_if_unbound(self, pos, entry, in_nogil_context=False):
1988
        from . import ExprNodes
1989
        if entry.from_closure:
1990 1991 1992
            func = '__Pyx_RaiseClosureNameError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_closure_name_error_utility_code)
1993 1994 1995 1996
        elif entry.type.is_memoryviewslice and in_nogil_context:
            func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_memoryview_utility_code_nogil)
1997
        else:
1998 1999 2000
            func = '__Pyx_RaiseUnboundLocalError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_local_error_utility_code)
Mark Florisson's avatar
Mark Florisson committed
2001

2002
        self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (
Mark Florisson's avatar
Mark Florisson committed
2003 2004 2005 2006
                                entry.type.check_for_null_code(entry.cname),
                                func,
                                entry.name,
                                self.error_goto(pos)))
2007

2008
    def set_error_info(self, pos, used=False):
2009
        self.funcstate.should_declare_error_indicator = True
2010 2011
        if used:
            self.funcstate.uses_error_indicator = True
2012
        if self.c_line_in_traceback:
2013
            cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
Robert Bradshaw's avatar
Robert Bradshaw committed
2014 2015
        else:
            cinfo = ""
2016

2017
        return "%s = %s[%s]; %s = %s;%s" % (
William Stein's avatar
William Stein committed
2018 2019 2020 2021 2022
            Naming.filename_cname,
            Naming.filetable_cname,
            self.lookup_filename(pos[0]),
            Naming.lineno_cname,
            pos[1],
2023
            cinfo)
2024

2025 2026 2027 2028 2029
    def error_goto(self, pos):
        lbl = self.funcstate.error_label
        self.funcstate.use_label(lbl)
        return "{%s goto %s;}" % (
            self.set_error_info(pos),
2030
            lbl)
2031 2032 2033

    def error_goto_if(self, cond, pos):
        return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
2034

Robert Bradshaw's avatar
Robert Bradshaw committed
2035 2036
    def error_goto_if_null(self, cname, pos):
        return self.error_goto_if("!%s" % cname, pos)
2037

Robert Bradshaw's avatar
Robert Bradshaw committed
2038 2039
    def error_goto_if_neg(self, cname, pos):
        return self.error_goto_if("%s < 0" % cname, pos)
2040

Robert Bradshaw's avatar
Robert Bradshaw committed
2041 2042
    def error_goto_if_PyErr(self, pos):
        return self.error_goto_if("PyErr_Occurred()", pos)
2043

William Stein's avatar
William Stein committed
2044
    def lookup_filename(self, filename):
2045
        return self.globalstate.lookup_filename(filename)
William Stein's avatar
William Stein committed
2046

2047
    def put_declare_refcount_context(self):
2048
        self.putln('__Pyx_RefNannyDeclarations')
2049

2050 2051
    def put_setup_refcount_context(self, name, acquire_gil=False):
        if acquire_gil:
2052 2053
            self.globalstate.use_utility_code(
                UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2054
        self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0))
2055

2056
    def put_finish_refcount_context(self):
2057
        self.putln("__Pyx_RefNannyFinishContext();")
2058

2059 2060 2061 2062
    def put_add_traceback(self, qualified_name):
        """
        Build a Python traceback for propagating exceptions.

2063
        qualified_name should be the qualified name of the function.
2064 2065 2066 2067 2068 2069 2070
        """
        format_tuple = (
            qualified_name,
            Naming.clineno_cname,
            Naming.lineno_cname,
            Naming.filename_cname,
        )
2071
        self.funcstate.uses_error_indicator = True
2072 2073
        self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)

2074
    def put_unraisable(self, qualified_name, nogil=False):
2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
        """
        Generate code to print a Python warning for an unraisable exception.

        qualified_name should be the qualified name of the function.
        """
        format_tuple = (
            qualified_name,
            Naming.clineno_cname,
            Naming.lineno_cname,
            Naming.filename_cname,
2085 2086
            self.globalstate.directives['unraisable_tracebacks'],
            nogil,
2087 2088
        )
        self.funcstate.uses_error_indicator = True
2089
        self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %d, %d);' % format_tuple)
2090 2091 2092
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))

2093 2094
    def put_trace_declarations(self, codeobj=None, nogil=False):
        self.putln('__Pyx_TraceDeclarations(%s, %d)' % (codeobj or 'NULL', nogil))
2095

2096
    def put_trace_call(self, name, pos, nogil=False):
2097 2098
        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d, %s);' % (
            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil, self.error_goto(pos)))
2099

Robert Bradshaw's avatar
Robert Bradshaw committed
2100 2101
    def put_trace_exception(self):
        self.putln("__Pyx_TraceException();")
2102

2103 2104
    def put_trace_return(self, retvalue_cname, nogil=False):
        self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
Robert Bradshaw's avatar
Robert Bradshaw committed
2105

Mark Florisson's avatar
Mark Florisson committed
2106 2107 2108
    def putln_openmp(self, string):
        self.putln("#ifdef _OPENMP")
        self.putln(string)
2109
        self.putln("#endif /* _OPENMP */")
William Stein's avatar
William Stein committed
2110

2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
    def undef_builtin_expect(self, cond):
        """
        Redefine the macros likely() and unlikely to no-ops, depending on
        condition 'cond'
        """
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   (x)")
        self.putln("    #define unlikely(x) (x)")
        self.putln("#endif")

    def redef_builtin_expect(self, cond):
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   __builtin_expect(!!(x), 1)")
        self.putln("    #define unlikely(x) __builtin_expect(!!(x), 0)")
        self.putln("#endif")

2131
class PyrexCodeWriter(object):
William Stein's avatar
William Stein committed
2132 2133 2134 2135
    # f                file      output file
    # level            int       indentation level

    def __init__(self, outfile_name):
2136
        self.f = Utils.open_new_file(outfile_name)
William Stein's avatar
William Stein committed
2137
        self.level = 0
2138

William Stein's avatar
William Stein committed
2139 2140
    def putln(self, code):
        self.f.write("%s%s\n" % (" " * self.level, code))
2141

William Stein's avatar
William Stein committed
2142 2143
    def indent(self):
        self.level += 1
2144

William Stein's avatar
William Stein committed
2145 2146 2147
    def dedent(self):
        self.level -= 1

2148 2149
class PyxCodeWriter(object):
    """
2150 2151 2152
    Can be used for writing out some Cython code. To use the indenter
    functionality, the Cython.Compiler.Importer module will have to be used
    to load the code to support python 2.4
2153 2154
    """

2155
    def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'):
2156 2157 2158
        self.buffer = buffer or StringIOTree()
        self.level = indent_level
        self.context = context
2159
        self.encoding = encoding
2160 2161 2162

    def indent(self, levels=1):
        self.level += levels
2163
        return True
2164 2165 2166 2167 2168 2169

    def dedent(self, levels=1):
        self.level -= levels

    def indenter(self, line):
        """
2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
        Instead of

            with pyx_code.indenter("for i in range(10):"):
                pyx_code.putln("print i")

        write

            if pyx_code.indenter("for i in range(10);"):
                pyx_code.putln("print i")
                pyx_code.dedent()
2180 2181
        """
        self.putln(line)
2182 2183
        self.indent()
        return True
2184 2185

    def getvalue(self):
2186 2187 2188 2189 2190
        result = self.buffer.getvalue()
        if not isinstance(result, unicode):
            result = result.decode(self.encoding)

        return result
2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216

    def putln(self, line, context=None):
        context = context or self.context
        if context:
            line = sub_tempita(line, context)
        self._putln(line)

    def _putln(self, line):
        self.buffer.write("%s%s\n" % (self.level * "    ", line))

    def put_chunk(self, chunk, context=None):
        context = context or self.context
        if context:
            chunk = sub_tempita(chunk, context)

        chunk = textwrap.dedent(chunk)
        for line in chunk.splitlines():
            self._putln(line)

    def insertion_point(self):
        return PyxCodeWriter(self.buffer.insertion_point(), self.level,
                             self.context)

    def named_insertion_point(self, name):
        setattr(self, name, self.insertion_point())

2217 2218

class ClosureTempAllocator(object):
2219
    def __init__(self, klass):
2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239
        self.klass = klass
        self.temps_allocated = {}
        self.temps_free = {}
        self.temps_count = 0

    def reset(self):
        for type, cnames in self.temps_allocated.items():
            self.temps_free[type] = list(cnames)

    def allocate_temp(self, type):
        if not type in self.temps_allocated:
            self.temps_allocated[type] = []
            self.temps_free[type] = []
        elif self.temps_free[type]:
            return self.temps_free[type].pop(0)
        cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
        self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
        self.temps_allocated[type].append(cname)
        self.temps_count += 1
        return cname