merged in latest cython-devel

Cython/Compiler/Builtin.py

@@ -162,24 +162,42 @@ pyexec_utility_code = UtilityCode(
 proto = """
 static PyObject* __Pyx_PyRun(PyObject*, PyObject*, PyObject*);
 """,
-impl = """
+impl = '''
 static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
     PyObject* result;
     PyObject* s = 0;
     char *code = 0;
 
-    if (!locals && !globals) {
-        globals = PyModule_GetDict(%s);""" % Naming.module_cname + """
+    if (!globals || globals == Py_None) {
+        globals = PyModule_GetDict(%s);''' % Naming.module_cname + '''
         if (!globals)
             goto bad;
+    } else if (!PyDict_Check(globals)) {
+        PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.100s",
+                     globals->ob_type->tp_name);
+        goto bad;
+    }
+    if (!locals || locals == Py_None) {
         locals = globals;
-    } else if (!locals) {
-        locals = globals;
-    } else if (!globals) {
-        globals = locals;
     }
 
+
+    if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
+	PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins());
+    }
+
+    if (PyCode_Check(o)) {
+        if (PyCode_GetNumFree((PyCodeObject *)o) > 0) {
+            PyErr_SetString(PyExc_TypeError,
+                "code object passed to exec() may not contain free variables");
+            goto bad;
+        }
+	result = PyEval_EvalCode((PyCodeObject *)o, globals, locals);
+    } else {
+        PyCompilerFlags cf;
+        cf.cf_flags = 0;
 	if (PyUnicode_Check(o)) {
+            cf.cf_flags = PyCF_SOURCE_IS_UTF8;
     	    s = PyUnicode_AsUTF8String(o);
     	    if (!s) goto bad;
     	    o = s;
@@ -188,26 +206,29 @@ static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
 	#else
 	} else if (!PyString_Check(o)) {
 	#endif
-        /* FIXME: support file objects and code objects */
     	    PyErr_SetString(PyExc_TypeError,
-            "exec currently requires a string as code input.");
+        	"exec: arg 1 must be string, bytes or code object");
     	    goto bad;
 	}
-
 	#if PY_MAJOR_VERSION >= 3
 	code = PyBytes_AS_STRING(o);
 	#else
 	code = PyString_AS_STRING(o);
 	#endif
+	if (PyEval_MergeCompilerFlags(&cf)) {
+	    result = PyRun_StringFlags(code, Py_file_input, globals, locals, &cf);
+        } else {
 	    result = PyRun_String(code, Py_file_input, globals, locals);
-
+        }
         Py_XDECREF(s);
+    }
+
     return result;
 bad:
     Py_XDECREF(s);
     return 0;
 }
-""")
+''')
 
 intern_utility_code = UtilityCode(
 proto = """

Cython/Compiler/Code.py

@@ -1244,6 +1244,15 @@ class CCodeWriter(object):
     def put_finish_refcount_context(self):
         self.putln("__Pyx_FinishRefcountContext();")
 
+    def put_trace_call(self, name, pos):
+        self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]));
+    
+    def put_trace_exception(self):
+        self.putln("__Pyx_TraceException();")
+    
+    def put_trace_return(self, retvalue_cname):
+        self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname)
+
 
 class PyrexCodeWriter(object):
     # f                file      output file

Cython/Compiler/ExprNodes.py

@@ -2487,6 +2487,9 @@ class GeneralCallNode(CallNode):
         if self.starstar_arg:
             self.starstar_arg.analyse_types(env)
         if not self.function.type.is_pyobject:
+            if self.function.type.is_error:
+                self.type = error_type
+                return error_type
             if hasattr(self.function, 'entry') and not self.function.entry.as_variable:
                 error(self.pos, "Keyword arguments not allowed in cdef functions.")
             else:
@@ -2507,6 +2510,7 @@ class GeneralCallNode(CallNode):
         self.is_temp = 1
         
     def generate_result_code(self, code):
+        if self.type.is_error: return
         if self.keyword_args and self.starstar_arg:
             code.put_error_if_neg(self.pos, 
                 "PyDict_Update(%s, %s)" % (

Cython/Compiler/ModuleNode.py

@@ -566,6 +566,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln('static PyObject *%s;' % env.module_cname)
         code.putln('static PyObject *%s;' % Naming.builtins_cname)
         code.putln('static PyObject *%s;' % Naming.empty_tuple)
+        code.putln('static PyObject *%s;' % Naming.empty_bytes)
         if Options.pre_import is not None:
             code.putln('static PyObject *%s;' % Naming.preimport_cname)
         code.putln('static int %s;' % Naming.lineno_cname)
@@ -1637,6 +1638,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#endif")
 
         code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)));
+        code.putln("#if PY_MAJOR_VERSION < 3");
+        code.putln("%s = PyString_FromStringAndSize(\"\", 0); %s" % (Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos)));
+        code.putln("#else");
+        code.putln("%s = PyBytes_FromStringAndSize(\"\", 0); %s" % (Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos)));
+        code.putln("#endif");
 
         code.putln("/*--- Library function declarations ---*/")
         env.generate_library_function_declarations(code)
@@ -2463,6 +2469,7 @@ static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL;
 #define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r)
 """)
 
+
 main_method = UtilityCode(
 impl = """
 #if PY_MAJOR_VERSION < 3 || (!defined(WIN32) && !defined(MS_WINDOWS))

Cython/Compiler/Naming.py

@@ -75,6 +75,7 @@ c_api_tab_cname  = pyrex_prefix + "c_api_tab"
 gilstate_cname   = pyrex_prefix + "state"
 skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
 empty_tuple      = pyrex_prefix + "empty_tuple"
+empty_bytes      = pyrex_prefix + "empty_bytes"
 print_function   = pyrex_prefix + "print"
 print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
@@ -84,6 +85,8 @@ import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
 cur_scope_cname  = pyrex_prefix + "cur_scope"
 enc_scope_cname  = pyrex_prefix + "enc_scope"
+frame_cname      = pyrex_prefix + "frame"
+frame_code_cname = pyrex_prefix + "frame_code"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Optimize.py

@@ -18,13 +18,9 @@ try:
 except NameError:
     from functools import reduce
 
-#def unwrap_node(node):
-#    while isinstance(node, ExprNodes.PersistentNode):
-#        node = node.arg
-#    return node
-
-# Temporary hack while PersistentNode is out of order
 def unwrap_node(node):
+    while isinstance(node, UtilNodes.ResultRefNode):
+        node = node.expression
     return node
 
 def is_common_value(a, b):
@@ -391,12 +387,16 @@ class SwitchTransform(Visitor.VisitorTransform):
     is common among all clauses and both var and value are ints. 
     """
     def extract_conditions(self, cond):
-    
+        while True:
             if isinstance(cond, ExprNodes.CoerceToTempNode):
                 cond = cond.arg
-
-        if isinstance(cond, ExprNodes.TypecastNode):
+            elif isinstance(cond, UtilNodes.EvalWithTempExprNode):
+                # this is what we get from the FlattenInListTransform
+                cond = cond.subexpression
+            elif isinstance(cond, ExprNodes.TypecastNode):
                 cond = cond.operand
+            else:
+                break
 
         if (isinstance(cond, ExprNodes.PrimaryCmpNode) 
                 and cond.cascade is None 

Cython/Compiler/Options.py

@@ -67,10 +67,11 @@ option_defaults = {
     'wraparound' : True,
     'c99_complex' : False, # Don't use macro wrappers for complex arith, not sure what to name this...
     'callspec' : "",
+    'profile': None,
 }
 
 # Override types possibilities above, if needed
-option_types = { }
+option_types = { 'profile': bool }
 
 for key, val in option_defaults.items():
     if key not in option_types:

Cython/Compiler/PyrexTypes.py

@@ -836,13 +836,14 @@ static %(type)s __pyx_PyObject_As_%(type_name)s(PyObject* o); /* proto */
 """, 
 impl="""
 static %(type)s __pyx_PyObject_As_%(type_name)s(PyObject* o) {
-    if (PyComplex_Check(o)) {
+    if (PyComplex_CheckExact(o)) {
         return %(type_name)s_from_parts(
             (%(real_type)s)((PyComplexObject *)o)->cval.real,
             (%(real_type)s)((PyComplexObject *)o)->cval.imag);
     }
     else {
-        return %(type_name)s_from_parts(%(type_convert)s(o), 0);
+        Py_complex cval = PyComplex_AsCComplex(o);
+        return %(type_name)s_from_parts((%(real_type)s)cval.real, (%(real_type)s)cval.imag);
     }
 }
 """)

Demos/freeze/Makefile

+CC = gcc
+CYTHON = ./../bin/cython
+CYTHON_FREEZE = ../../bin/cython_freeze.py
+
+CFLAGS = -fPIC -g -O2 -Wall -Wextra
+CPPFLAGS = -I /usr/include/python2.6
+LDFLAGS = -Xlinker -export-dynamic -Wl,-O1 -Wl,-Bsymbolic-functions
+LDLIBS = /usr/lib/python2.6/config/libpython2.6.a \
+	-lm -ldl -pthread -lutil -lz
+
+
+# Name of executable
+TARGET = nCr
+
+# List of Cython source files, with main module first.
+CYTHON_SOURCE = combinatorics.pyx cmath.pyx
+
+
+all : $(TARGET)
+
+$(TARGET) : $(TARGET).o $(CYTHON_SOURCE:.pyx=.o)
+
+$(TARGET).c :
+	$(CYTHON_FREEZE) $(CYTHON_SOURCE:.pyx=) > $@
+
+%.c : %.pyx
+	$(CYTHON) $(CYTHONFLAGS) $^
+
+clean:
+	$(RM) *.o *.c $(TARGET)
+
+.PHONY: clean
+.SECONDARY: $(CYTHON_SOURCE:.pyx=.c)

Demos/freeze/README.rst

+NAME
+====
+
+cython_freeze.py - create a C file for embedding Cython modules
+
+
+SYNOPSIS
+========
+
+cython_freeze.py module [...]
+
+
+DESCRIPTION
+===========
+
+**cython_freeze.py** generates a C source file to embed a Python interpreter
+with one or more Cython modules built in.  This allows one to create a single
+executable from Cython code, without having to have separate shared objects
+for each Cython module.
+
+A major advantage of this approach is that it allows debuging with gprof(1),
+which does not work with shared objects.
+
+Note that this method differs from ``cython --embed``.  The ``--embed`` options
+modifies the resulting C source file to include a ``main()`` function, so it
+can only be used on a single Cython module.  The advantage ``--embed`` is
+simplicity.  This module, on the other hand, can be used with multiple
+modules, but it requires another C source file to be created.
+
+
+EXAMPLE
+=======
+
+In the example directory, there exist two Cython modules:
+
+cmath.pyx
+    A module that interfaces with the -lm library.
+
+combinatorics.pyx
+    A module that implements n-choose-r using cmath.
+
+Both modules have the Python idiom ``if __name__ == "__main__"``, which only
+execute if that module is the "main" module.  If run as main, cmath prints the
+factorial of the argument, while combinatorics prints n-choose-r.
+
+The provided Makefile creates an executable, *nCr*, using combinatorics as the
+"main" module.  It basically performs the following (ignoring the compiler
+flags)::
+
+    $ cython_freeze.py combintorics cmath > nCr.c
+    $ cython combinatorics.pyx
+    $ cython cmath.pyx
+    $ gcc nCr.c -o nCr.o
+    $ gcc combinatorics.c -o combinatorics.o
+    $ gcc cmath.c -o cmath.o
+    $ gcc nCr.o combinatorics.o cmath.o -o nCr
+
+Because the combinatorics module was listed first, its ``__name__`` is set
+to ``"__main__"``, while cmath's is set to ``"cmath"``.  The executable now
+contains a Python interpreter and both Cython modules. ::
+
+    $ ./nCr
+    USAGE: ./nCr n r
+    Prints n-choose-r.
+    $ ./nCr 15812351235 12
+    5.10028093999e+113
+
+
+
+
+PREREQUISITES
+=============
+
+Cython 0.11.2 (or newer, assuming the API does not change)
+
+
+SEE ALSO
+========
+
+* `Python <http://www.python.org>`_
+* `Cython <http://www.cython.org>`_
+* `freeze.py <http://wiki.python.org/moin/Freeze>`_

Demos/freeze/cmath.pyx

+cdef extern from "math.h":
+    double c_lgamma "lgamma" (double)
+    double c_exp "exp" (double)
+
+def exp(n):
+    """Return e**n."""
+    return c_exp(n)
+
+def lfactorial(n):
+    """Return an estimate of the log factorial of n."""
+    return c_lgamma(n+1)
+
+def factorial(n):
+    """Return an estimate of the factorial of n."""
+    return c_exp( c_lgamma(n+1) )
+
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 2:
+        sys.stderr.write("USAGE: %s n\nPrints n!.\n" % sys.argv[0])
+        sys.exit(1)
+    n = map(float, sys.argv[1:])
+    print factorial(n)

Demos/freeze/combinatorics.pyx

+import cmath
+
+def nCr(n, r):
+    """Return the number of ways to choose r elements of a set of n."""
+    return cmath.exp( cmath.lfactorial(n) - cmath.lfactorial(r)
+                      - cmath.lfactorial(n-r) )
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 3:
+        sys.stderr.write("USAGE: %s n r\nPrints n-choose-r.\n" % sys.argv[0])
+        sys.exit(1)
+    n, r = map(float, sys.argv[1:])
+    print nCr(n, r)

Demos/setup.py

@@ -4,6 +4,12 @@ from distutils.core import setup
 from distutils.extension import Extension
 from Cython.Distutils import build_ext
 
+try:
+    from numpy.distutils.misc_util import get_numpy_include_dirs
+    numpy_include_dirs = get_numpy_include_dirs()
+except:
+    numpy_include_dirs = []
+
 ext_modules=[ 
     Extension("primes",       ["primes.pyx"]),
     Extension("spam",         ["spam.pyx"]),
@@ -11,7 +17,7 @@ ext_modules=[
 
 for file in glob.glob("*.pyx"):
     if file != "numeric_demo.pyx":
-        ext_modules.append(Extension(file[:-4], [file]))
+        ext_modules.append(Extension(file[:-4], [file], include_dirs = numpy_include_dirs))
 
 setup(
   name = 'Demos',

bin/cython_freeze.py

+#!/usr/bin/env python
+"""
+Create a C file for embedding one or more Cython source files.
+Requires Cython 0.11.2 (or perhaps newer).
+
+See README.rst for more details.
+"""
+
+import sys
+
+if len(sys.argv) < 2:
+    print >>sys.stderr, "USAGE: %s module [module ...]" % sys.argv[0]
+    sys.exit(1)
+
+def format_modname(name):
+    if name.endswith('.pyx'):
+        name = name[:-4]
+    return name.replace('.','_')
+
+modules = [format_modname(x) for x in sys.argv[1:]]
+
+print """
+#include <Python.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#if PY_MAJOR_VERSION < 3
+# define MODINIT(name)  init ## name
+#else
+# define MODINIT(name)  PyInit_ ## name
+#endif
+"""
+
+for name in modules:
+    print "PyMODINIT_FUNC MODINIT(%s) (void);" % name
+
+print """
+static struct _inittab inittab[] = {"""
+
+for name in modules:
+    print '    {"%(name)s", MODINIT(%(name)s)},' % {'name' : name}
+
+print """    {NULL, NULL}
+};
+
+extern int __pyx_module_is_main_%(main)s;
+
+#if PY_MAJOR_VERSION < 3 || (!defined(WIN32) && !defined(MS_WINDOWS))
+int main(int argc, char** argv) {
+#else
+int wmain(int argc, wchar_t **argv) {
+#endif
+    int r = 0;
+    PyObject *m = NULL;
+    if (PyImport_ExtendInittab(inittab)) {
+        fprintf(stderr, "No memory\\n");
+        exit(1);
+    }
+    Py_SetProgramName(argv[0]);
+    Py_Initialize();
+    PySys_SetArgv(argc, argv);
+    __pyx_module_is_main_%(main)s = 1;
+    m = PyImport_ImportModule(inittab[0].name);
+    if (!m) {
+        r = 1;
+        PyErr_Print(); /* This exits with the right code if SystemExit. */
+        if (Py_FlushLine()); PyErr_Clear();
+    }
+    Py_XDECREF(m);
+    Py_Finalize();
+    return r;
+}
+""" % {'main' : modules[0]}

tests/bugs.txt

@@ -14,3 +14,6 @@ large_consts_T237
 bad_c_struct_T252
 missing_baseclass_in_predecl_T262
 ifelseexpr_T267
+
+# Not yet enabled
+profile_test

tests/errors/callingnonexisting_T307.pyx

+nonexisting(3, with_kw_arg=4)
+
+_ERRORS = u"""
+1:11: undeclared name not builtin: nonexisting
+"""

tests/run/delete.pyx

@@ -4,6 +4,21 @@
 [2, 1]
 >>> a.g()
 (False, True)
+>>> del_item({1: 'a', 2: 'b'}, 1)
+{2: 'b'}
+>>> del_item(range(10), 2)
+[0, 1, 3, 4, 5, 6, 7, 8, 9]
+
+>>> del_dict({1: 'a', 2: 'b'}, 1)
+{2: 'b'}
+>>> del_list(range(5), 3)
+[0, 1, 2, 4]
+>>> del_int(range(5), 3)
+[0, 1, 2, 4]
+>>> del_list_int(range(5), 3)
+[0, 1, 2, 4]
+>>> del_int({-1: 'neg', 1: 'pos'}, -1)
+{1: 'pos'}
 """
 
 class A:
@@ -16,3 +31,23 @@ class A:
         self.a = 3
         del self.a
         return (hasattr(self, u"a"), hasattr(self, u"g"))
+
+def del_item(L, o):
+    del L[o]
+    return L
+
+def del_dict(dict D, o):
+    del D[o]
+    return D
+
+def del_list(list L, o):
+    del L[o]
+    return L
+
+def del_int(L, int i):
+    del L[i]
+    return L
+
+def del_list_int(L, int i):
+    del L[i]
+    return L

tests/run/exectest.pyx

+# -*- coding: utf-8 -*-
+
 __doc__ = u"""
 #>>> a
 #Traceback (most recent call last):
@@ -43,6 +45,21 @@ __doc__ = u"""
 >>> list(add_iter())
 [2, 3, 4, 5]
 
+>>> d = {}
+>>> test_encoding(d, None)
+>>> print(d['b'])
+üöä
+
+>>> d = {}
+>>> test_encoding_unicode(d, None)
+>>> print(d['b'])
+üöä
+
+>>> d = dict(a=1, c=3)
+>>> test_compile(d)
+>>> d['b']
+4
+
 >>> # errors
 
 >>> d1, d2 = {}, {}
@@ -76,3 +93,23 @@ def test():
         yield x+1
 """ % varref in d
     return d[u'test']
+
+import sys
+
+def test_encoding(d1, d2):
+    if sys.version_info[0] >= 3:
+        s = "b = 'üöä'"
+    else:
+        s = "# -*- coding: utf-8 -*-" + "\n" + "b = u'üöä'"
+    exec s in d1, d2
+
+def test_encoding_unicode(d1, d2):
+    if sys.version_info[0] >= 3:
+        s = u"b = 'üöä'"
+    else:
+        s = u"b = u'üöä'"
+    exec s in d1, d2
+
+def test_compile(d):
+    c = compile(u"b = a+c", u"<string>", u"exec")
+    exec c in d

tests/run/numpy_test.pyx

@@ -195,6 +195,11 @@ try:
         ...
     ValueError: Item size of buffer (1 byte) does not match size of 'int' (4 bytes)
 
+    >>> test_complextypes()
+    1,1
+    1,1
+    8,16
+    
 """
 except:
     __doc__ = u""
@@ -376,3 +381,15 @@ def test_unpacked_align(np.ndarray[UnpackedStruct] arr):
     arr[0].a = 22
     arr[0].b = 23    
     return repr(arr).replace('<', '!').replace('>', '!')
+
+def test_complextypes():
+    cdef np.complex64_t x64 = 1, y64 = 1j
+    cdef np.complex128_t x128 = 1, y128 = 1j
+    x64 = x64 + y64
+    print "%.0f,%.0f" % (x64.real, x64.imag)
+    x128 = x128 + y128
+    print "%.0f,%.0f" % (x128.real, x128.imag)
+    print "%d,%d" % (sizeof(x64), sizeof(x128))
+
+
+    

tests/run/profile_test.pyx

+__doc__ = u"""
+    >>> import os, tempfile, cProfile as profile, pstats
+    >>> statsfile = tempfile.mkstemp()[1]
+    >>> profile.runctx("test_profile(100)", locals(), globals(), statsfile)
+    >>> s = pstats.Stats(statsfile)
+    >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()])
+    >>> short_stats['f_def']
+    100
+    >>> short_stats['f_cdef']
+    100
+    >>> short_stats['f_inline']
+    Traceback (most recent call last):
+    ...
+    KeyError: 'f_inline'
+    >>> short_stats['f_inline_prof']
+    100
+    >>> short_stats['f_noprof']
+    Traceback (most recent call last):
+    ...
+    KeyError: 'f_noprof'
+    >>> short_stats['f_raise']
+    100
+    >>> os.unlink(statsfile)
+"""
+
+cimport cython
+
+def test_profile(long N):
+    cdef long i, n = 0
+    for i from 0 <= i < N:
+        n += f_def(i)
+        n += f_cdef(i)
+        n += f_inline(i)
+        n += f_inline_prof(i)
+        n += f_noprof(i)
+        try:
+            n += f_raise(i+2)
+        except RuntimeError:
+            pass
+    return n
+
+def f_def(long a):
+    return a
+
+cdef long f_cdef(long a):
+    return a
+
+cdef inline long f_inline(long a):
+    return a
+
+@cython.profile(True)
+cdef inline long f_inline_prof(long a):
+    return a
+
+@cython.profile(False)
+cdef int f_noprof(long a):
+    return a
+
+cdef long f_raise(long) except -2:
+    raise RuntimeError

tests/run/switch.pyx

@@ -74,6 +74,17 @@ __doc__ = u"""
 >>> switch_or(4)
 0
 
+>>> switch_in(0)
+0
+>>> switch_in(1)
+1
+>>> switch_in(2)
+0
+>>> switch_in(7)
+1
+>>> switch_in(8)
+0
+
 >>> switch_short(0)
 0
 >>> switch_short(1)
@@ -161,6 +172,11 @@ def switch_or(int x):
         return 0
     return -1
 
+def switch_in(int X):
+    if X in (1,3,5,7):
+        return 1
+    return 0
+
 def switch_short(int x):
     if x == 1:
         return 1