merge cython-devel

Cython/Compiler/CmdLine.py

@@ -38,6 +38,7 @@ Options:
   -a, --annotate                 Produce a colorized HTML version of the source.
   --line-directives              Produce #line directives pointing to the .pyx source
   --cplus                        Output a c++ rather than c file.
+  --embed                        Embed the Python interpreter in a main() method.
   --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
 """
 
@@ -89,6 +90,8 @@ def parse_command_line(args):
                 options.obj_only = 0
             elif option in ("-+", "--cplus"):
                 options.cplus = 1
+            elif option == "--embed":
+                Options.embed = True
             elif option.startswith("-I"):
                 options.include_path.append(get_param(option))
             elif option == "--include-dir":

Cython/Compiler/ExprNodes.py

@@ -1642,6 +1642,7 @@ class IndexNode(ExprNode):
                     error(self.pos,
                         "Invalid index type '%s'" %
                             self.index.type)
+        self.gil_check(env)
 
     def gil_check(self, env):
         if not self.is_buffer_access:
@@ -1650,6 +1651,17 @@ class IndexNode(ExprNode):
 
     gil_message = "Indexing Python object"
 
+    def gil_check(self, env):
+        if self.is_buffer_access and env.nogil:
+            if env.directives['boundscheck']:
+                error(self.pos, "Cannot check buffer index bounds without gil; use boundscheck(False) directive")
+                return
+            elif self.type.is_pyobject:
+                error(self.pos, "Cannot access buffer with object dtype without gil")
+                return
+        super(IndexNode, self).gil_check(env)
+
+
     def check_const_addr(self):
         self.base.check_const_addr()
         self.index.check_const()
@@ -1741,14 +1753,13 @@ class IndexNode(ExprNode):
             index_code = self.index.py_result()
             if self.base.type is dict_type:
                 function = "PyDict_SetItem"
-            elif self.base.type is list_type:
-                function = "PyList_SetItem"
-            # don't use PyTuple_SetItem(), as we'd normally get a
-            # TypeError when changing a tuple, while PyTuple_SetItem()
-            # would allow updates
-            #
-            #elif self.base.type is tuple_type:
-            #    function = "PyTuple_SetItem"
+            # It would seem that we could specalized lists/tuples, but that
+            # shouldn't happen here. 
+            # Both PyList_SetItem PyTuple_SetItem and a Py_ssize_t as input, 
+            # not a PyObject*, and bad conversion here would give the wrong 
+            # exception. Also, tuples are supposed to be immutable, and raise 
+            # TypeErrors when trying to set their entries (PyTuple_SetItem 
+            # is for creating new tuples from). 
             else:
                 function = "PyObject_SetItem"
         code.putln(
@@ -5786,12 +5797,11 @@ static int __Pyx_cdivision_warning(void) {
                               "division with oppositely signed operands, C and Python semantics differ",
                               %(FILENAME)s, 
                               %(LINENO)s,
-                              %(MODULENAME)s,
+                              __Pyx_MODULE_NAME,
                               NULL);
 }
 """ % {
     'FILENAME': Naming.filename_cname,
-    'MODULENAME': Naming.modulename_cname,
     'LINENO':  Naming.lineno_cname,
 })
 

Cython/Compiler/ModuleNode.py

@@ -55,6 +55,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         else:
             env.doc = self.doc
         env.directives = self.directives
+        if Options.embed:
+            self.__main__cname = env.intern_identifier(EncodedString("__main__"))
         self.body.analyse_declarations(env)
     
     def process_implementation(self, options, result):
@@ -249,7 +251,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.globalstate.use_utility_code(refcount_utility_code)
 
-        code.putln('const char *%s = "%s";' % (Naming.modulename_cname, self.full_module_name))
+        code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name)
         code.putln("")
         code.putln("/* Implementation of %s */" % env.qualified_name)
 
@@ -267,6 +269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_module_init_func(modules[:-1], env, code)
         code.mark_pos(None)
         self.generate_module_cleanup_func(env, code)
+        if Options.embed:
+            self.generate_main_method(env, code)
         self.generate_filename_table(code)
         self.generate_utility_functions(env, code, h_code)
 
@@ -1723,6 +1727,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("Py_INCREF(Py_None); return Py_None;")
         code.putln('}')
 
+    def generate_main_method(self, env, code):
+        code.globalstate.use_utility_code(main_method.specialize(module_name=env.module_name))
+
     def generate_filename_init_call(self, code):
         code.putln("%s();" % Naming.fileinit_cname)
 
@@ -1787,6 +1794,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 env.module_cname,
                 Naming.builtins_cname,
                 code.error_goto(self.pos)))
+        if Options.embed:
+            code.putln(
+                'if (__Pyx_SetAttrString(%s, "__name__", %s) < 0) %s;' % (
+                    env.module_cname,
+                    self.__main__cname,
+                    code.error_goto(self.pos)))
         if Options.pre_import is not None:
             code.putln(
                 '%s = PyImport_AddModule(__Pyx_NAMESTR("%s"));' % (
@@ -2431,3 +2444,27 @@ static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL;
 #define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r)
 #define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r)
 """)
+
+main_method = UtilityCode(
+impl = """
+int main(int argc, char** argv) {
+    int r = 0;
+    PyObject* m = NULL;
+    Py_SetProgramName(argv[0]);
+    Py_Initialize();
+    PySys_SetArgv(argc, argv);
+#if PY_MAJOR_VERSION < 3
+        init%(module_name)s();
+#else
+        m = PyInit_%(module_name)s(name);
+#endif
+    if (PyErr_Occurred() != NULL) {
+        r = 1;
+        PyErr_Print(); /* This exits with the right code if SystemExit. */
+        if (Py_FlushLine()) PyErr_Clear();
+    }
+    Py_XDECREF(m);
+    Py_Finalize();
+    return r;
+}
+""")

Cython/Compiler/Options.py

@@ -53,6 +53,11 @@ optimize_simple_methods = 1
 # Append the c file and line number to the traceback for exceptions. 
 c_line_in_traceback = 1
 
+# Whether or not to embed the Python interpreter, for use in making a 
+# standalone executable. This will provide a main() method which simply 
+# executes the body of this module. 
+embed = False
+
 
 # Declare pragmas
 option_defaults = {

Demos/embed/Makefile

-PYVERSION = 2.2
-PYHOME = $(HOME)/pkg/python/$(PYVERSION)
-PYARCH = $(PYHOME)/$(ARCH)
-PYINCLUDE = \
-	-I$(PYHOME)/include/python$(PYVERSION) \
-	-I$(PYARCH)/include/python$(PYVERSION)
-PYLIB = -L$(PYARCH)/lib/python$(PYVERSION)/config \
-	-lpython$(PYVERSION) \
-	-ldl -lpthread -lutil -lm
+# Makefile for creating our standalone Cython program
+PYVERSION=2.3
+PYPREFIX=/usr
+INCLUDES=-I$(PYPREFIX)/include/python$(PYVERSION)
+
+embedded: embedded.o
+	gcc -o $@ $^ -lpython$(PYVERSION)
 
-%.c:	%.pyx
-	../../bin/cython $<
+embedded.o: embedded.c
+	gcc -c $^ $(INCLUDES)
 
-%.o:	%.c
-	gcc -c -fPIC $(PYINCLUDE) $<
+embedded.c: embedded.pyx
+	@python ../../cython.py --embed embedded.pyx
 
-#%.so:	%.o
-#	gcc -shared $< -lm -o $@
-
-all:	main
-
-main:	main.o embedded.o
-	gcc main.o embedded.o $(PYLIB) -o main
+all: embedded
 
 clean:
 	@echo Cleaning Demos/embed
-	@rm -f *~ *.o *.so core core.* embedded.h embedded.c main
-
-embedded.h:	embedded.c
-main.o:		embedded.h
+	@rm -f *~ *.o *.so core core.* *.c embedded

Demos/embed/Makefile.unix

-PYVERSION = 2.2
-PYHOME = $(HOME)/pkg/python/$(PYVERSION)
-PYARCH = $(PYHOME)/$(ARCH)
-PYINCLUDE = \
-	-I$(PYHOME)/include/python$(PYVERSION) \
-	-I$(PYARCH)/include/python$(PYVERSION)
-PYLIB = -L$(PYARCH)/lib/python$(PYVERSION)/config \
-	-lpython$(PYVERSION) \
-	-ldl -lpthread -lutil -lm
+# Makefile for creating our standalone Cython program
+PYVERSION=2.3
+PYPREFIX=/usr
+INCLUDES=-I$(PYPREFIX)/include/python$(PYVERSION)
+
+embedded: embedded.o
+	gcc -o $@ $^ -lpython$(PYVERSION)
 
-%.c:	%.pyx
-	../../bin/cython $<
+embedded.o: embedded.c
+	gcc -c $^ $(INCLUDES)
 
-%.o:	%.c
-	gcc -c -fPIC $(PYINCLUDE) $<
+embedded.c: embedded.pyx
+	@python ../../cython.py --embed embedded.pyx
 
-#%.so:	%.o
-#	gcc -shared $< -lm -o $@
-
-all:	main
-
-main:	main.o embedded.o
-	gcc main.o embedded.o $(PYLIB) -o main
+all: embedded
 
 clean:
 	@echo Cleaning Demos/embed
-	@rm -f *~ *.o *.so core core.* embedded.h embedded.c main
-
-embedded.h:	embedded.c
-main.o:		embedded.h
+	@rm -f *~ *.o *.so core core.* *.c embedded

Demos/embed/embedded.pyx

-cdef public void spam():
-  praise()
 
-def praise():
-  print "Spam, glorious spam!"
+print __name__
+
+if __name__ == "__main__":
+    print "Hi, I'm embedded."
+else:
+    print "I'm being imported."

INSTALL.txt

-Pyrex - Installation Instructions
-=================================
+Cython - Installation Instructions
+==================================
 
 You have two installation options:
 
@@ -8,7 +8,7 @@ You have two installation options:
 
         python setup.py install
 
-    This will install the Pyrex package
+    This will install the Cython package
     into your Python system.
 
 OR

tests/bugs.txt

@@ -14,4 +14,3 @@ large_consts_T237
 bad_c_struct_T252
 missing_baseclass_in_predecl_T262
 ifelseexpr_T267
-cdef_setitem_T284

tests/errors/e_bufaccess2.pyx

 cimport e_bufaccess_pxd # was needed to provoke a bug involving ErrorType
+import cython
 
 def f():
     cdef object[e_bufaccess_pxd.T] buf
 
+def withnogil_access_fail():
+    cdef object[int] buf = None
+    with nogil:
+        buf[2] = 2
+
+@cython.boundscheck(False)
+def withnogil_access_ok():
+    cdef object[int] buf = None
+    with nogil:
+        buf[2] = 2 # No error should be triggered here
+
+@cython.boundscheck(False)
+def withnogil_access_fail_2():
+    cdef object[object] buf = None
+    with nogil:
+        buf[2] = 2 # Not OK as dtype is object
+
+def withnogil_acquire(x):
+    cdef object[int] buf
+    with nogil:
+        buf = x
+
 _ERRORS = u"""
 3:9: 'nothing' is not a type identifier
+10:11: Cannot check buffer index bounds without gil; use boundscheck(False) directive
+22:11: Cannot access buffer with object dtype without gil
+22:11: Assignment of Python object not allowed without gil
+27:12: Assignment of Python object not allowed without gil
 """

tests/run/bufaccess.pyx

@@ -1449,3 +1449,18 @@ def complex_struct_inplace(object[LongComplex] buf):
     buf[0].imag += 2
     print buf[0].real, buf[0].imag
     
+#
+# Nogil
+#
+@testcase
+@cython.boundscheck(False)
+def buffer_nogil():
+    """
+    >>> buffer_nogil()
+    10
+    """
+    cdef object[int] buf = IntMockBuffer(None, [1,2,3])
+    with nogil:
+        buf[1] = 10
+    return buf[1]
+    

tests/run/cdef_setitem_T284.pyx

 __doc__ = u'''
 >>> no_cdef()
 >>> with_cdef()
+>>> test_list(range(11), -2, None)
+[0, 1, 2, 3, 4, 5, 6, 7, 8, None, 10]
+>>> test_list(range(11), "invalid index", None)
+Traceback (most recent call last):
+...
+TypeError: list indices must be integers
 '''
 def no_cdef():
     lst = range(11)
@@ -15,3 +21,7 @@ def with_cdef():
     lst[ob] = -10
     cdef dict dd = {}
     dd[ob] = -10
+
+def test_list(list L, object i, object a):
+    L[i] = a
+    return L