Merge remote-tracking branch 'origin'

Cython/Compiler/Code.py

@@ -1758,6 +1758,12 @@ class CCodeWriter(object):
             self.putln("%s_%sDECREF(%s);" % (
                 prefix, X, self.as_pyobject(cname, type)))
 
+    def put_decref_set(self, cname, rhs_cname):
+        self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname))
+
+    def put_xdecref_set(self, cname, rhs_cname):
+        self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname))
+
     def put_var_decref(self, entry):
         if entry.type.is_pyobject:
             self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))

Cython/Compiler/ExprNodes.py

@@ -1895,7 +1895,7 @@ class NameNode(AtomicExprNode):
                 # variables that the acquired buffer info is stored to is allocated
                 # per entry and coupled with it.
                 self.generate_acquire_buffer(rhs, code)
-
+            assigned = False
             if self.type.is_pyobject:
                 #print "NameNode.generate_assignment_code: to", self.name ###
                 #print "...from", rhs ###
@@ -1910,18 +1910,26 @@ class NameNode(AtomicExprNode):
                                 code.put_xgotref(self.py_result())
                             else:
                                 code.put_gotref(self.py_result())
+                    assigned = True
                     if entry.is_cglobal:
-                        code.put_decref(self.result(), self.ctype())
+                        code.put_decref_set(
+                            self.result(), rhs.result_as(self.ctype()))
                     else:
                         if not self.cf_is_null:
                             if self.cf_maybe_null:
-                                code.put_xdecref(self.result(), self.ctype())
+                                code.put_xdecref_set(
+                                    self.result(), rhs.result_as(self.ctype()))
                             else:
-                                code.put_decref(self.result(), self.ctype())
+                                code.put_decref_set(
+                                    self.result(), rhs.result_as(self.ctype()))
+                        else:
+                            assigned = False
                     if is_external_ref:
                         code.put_giveref(rhs.py_result())
             if not self.type.is_memoryviewslice:
-                code.putln('%s = %s;' % (self.result(), rhs.result_as(self.ctype())))
+                if not assigned:
+                    code.putln('%s = %s;' % (
+                        self.result(), rhs.result_as(self.ctype())))
                 if debug_disposal_code:
                     print("NameNode.generate_assignment_code:")
                     print("...generating post-assignment code for %s" % rhs)

Cython/Compiler/FlowControl.py

@@ -9,6 +9,7 @@ cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object,
 import Builtin
 import ExprNodes
 import Nodes
+import Options
 from PyrexTypes import py_object_type, unspecified_type
 import PyrexTypes
 
@@ -574,8 +575,9 @@ def check_definitions(flow, compiler_directives):
             if node.allow_null or entry.from_closure or entry.is_pyclass_attr:
                 pass # Can be uninitialized here
             elif node.cf_is_null:
-                if (entry.type.is_pyobject or entry.type.is_unspecified or
-                        entry.error_on_uninitialized):
+                if entry.error_on_uninitialized or (
+                        Options.error_on_uninitialized and (
+                        entry.type.is_pyobject or entry.type.is_unspecified)):
                     messages.error(
                         node.pos,
                         "local variable '%s' referenced before assignment"

Cython/Compiler/Nodes.py

@@ -4848,7 +4848,8 @@ class DelStatNode(StatNode):
             arg = self.args[i] = arg.analyse_target_expression(env, None)
             if arg.type.is_pyobject or (arg.is_name and
                                         arg.type.is_memoryviewslice):
-                pass
+                if arg.is_name and arg.entry.is_cglobal:
+                    error(arg.pos, "Deletion of global C variable")
             elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
                 self.cpp_check(env)
             elif arg.type.is_cpp_class:

Cython/Compiler/Options.py

@@ -34,6 +34,12 @@ warning_errors = False
 # you should disable this option and also 'cache_builtins'.
 error_on_unknown_names = True
 
+# Make uninitialized local variable reference a compile time error.
+# Python raises UnboundLocalError at runtime, whereas this option makes
+# them a compile time error. Note that this option affects only variables
+# of "python object" type.
+error_on_uninitialized = True
+
 # This will convert statements of the form "for i in range(...)"
 # to "for i from ..." when i is a cdef'd integer type, and the direction
 # (i.e. sign of step) can be determined.

Cython/Debugger/Cygdb.py

@@ -17,8 +17,7 @@ import glob
 import tempfile
 import textwrap
 import subprocess
-
-usage = "Usage: cygdb [PATH [GDB_ARGUMENTS]]"
+import optparse
 
 def make_command_file(path_to_debug_info, prefix_code='', no_import=False):
     if not no_import:
@@ -63,6 +62,8 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False):
 
     return tempfilename
 
+usage = "Usage: cygdb [options] [PATH [GDB_ARGUMENTS]]"
+
 def main(path_to_debug_info=None, gdb_argv=None, no_import=False):
     """
     Start the Cython debugger. This tells gdb to import the Cython and Python
@@ -73,20 +74,26 @@ def main(path_to_debug_info=None, gdb_argv=None, no_import=False):
     gdb_argv is the list of options to gdb
     no_import tells cygdb whether it should import debug information
     """
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option("--gdb-executable",
+        dest="gdb", default='gdb',
+        help="gdb executable to use [default: gdb]")
+
+    (options, args) = parser.parse_args()
     if path_to_debug_info is None:
-        if len(sys.argv) > 1:
-            path_to_debug_info = sys.argv[1]
+        if len(args) > 1:
+            path_to_debug_info = args[1]
         else:
             path_to_debug_info = os.curdir
 
     if gdb_argv is None:
-        gdb_argv = sys.argv[2:]
+        gdb_argv = args[2:]
 
     if path_to_debug_info == '--':
         no_import = True
 
     tempfilename = make_command_file(path_to_debug_info, no_import=no_import)
-    p = subprocess.Popen(['gdb', '-command', tempfilename] + gdb_argv)
+    p = subprocess.Popen([options.gdb, '-command', tempfilename] + gdb_argv)
     while True:
         try:
             p.wait()

Cython/Includes/libc/stdio.pxd

@@ -22,6 +22,9 @@ cdef extern from "stdio.h" nogil:
     int  rename   (const char *oldname, const char *newname)
     FILE *tmpfile ()
 
+    int remove (const char *pathname)
+    int rename (const char *oldpath, const char *newpath)
+
     enum: _IOFBF
     enum: _IOLBF
     enum: _IONBF
@@ -34,8 +37,9 @@ cdef extern from "stdio.h" nogil:
     int    fflush (FILE *stream)
 
     enum: EOF
-    int feof   (FILE *stream)
-    int ferror (FILE *stream)
+    void clearerr (FILE *stream)
+    int feof      (FILE *stream)
+    int ferror    (FILE *stream)
 
     enum: SEEK_SET
     enum: SEEK_CUR
@@ -62,6 +66,15 @@ cdef extern from "stdio.h" nogil:
 
     char *gets  (char *s)
     char *fgets (char *s, int count, FILE *stream)
+    int getchar ()
+    int fgetc   (FILE *stream)
+    int getc    (FILE *stream)
+    int ungetc  (int c, FILE *stream)
+
+    int puts    (const char *s)
+    int fputs   (const char *s, FILE *stream)
+    int putchar (int c)
+    int fputc   (int c, FILE *stream)
+    int putc    (int c, FILE *stream)
+
 
-    int  puts   (const char *s)
-    int  fputs  (const char *s, FILE *stream)

Cython/Includes/numpy.pxd

@@ -488,8 +488,8 @@ cdef extern from "numpy/arrayobject.h":
 
     object PyArray_FROM_O(object)
     object PyArray_FROM_OF(object m, int flags)
-    bint PyArray_FROM_OT(object m, int type)
-    bint PyArray_FROM_OTF(object m, int type, int flags)
+    object PyArray_FROM_OT(object m, int type)
+    object PyArray_FROM_OTF(object m, int type, int flags)
     object PyArray_FROMANY(object m, int type, int min, int max, int flags)
     object PyArray_ZEROS(int nd, npy_intp* dims, int type, int fortran)
     object PyArray_EMPTY(int nd, npy_intp* dims, int type, int fortran)

Cython/Utility/ModuleSetupCode.c

@@ -531,6 +531,15 @@ static int __Pyx_check_binary_version(void) {
   #define __Pyx_XGIVEREF(r)
 #endif /* CYTHON_REFNANNY */
 
+#define __Pyx_XDECREF_SET(r, v) do {                            \
+        PyObject *tmp = (PyObject *) r;                         \
+        r = v; __Pyx_XDECREF(tmp);                              \
+    } while (0)
+#define __Pyx_DECREF_SET(r, v) do {                             \
+        PyObject *tmp = (PyObject *) r;                         \
+        r = v; __Pyx_DECREF(tmp);                               \
+    } while (0)
+
 #define __Pyx_CLEAR(r)    do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0)
 #define __Pyx_XCLEAR(r)   do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0)
 

docs/src/tutorial/strings.rst

@@ -3,14 +3,43 @@
 Unicode and passing strings
 ===========================
 
-Similar to the string semantics in Python 3, Cython also strictly
-separates byte strings and unicode strings.  Above all, this means
-that by default there is no automatic conversion between byte strings
-and unicode strings (except for what Python 2 does in string operations).
-All encoding and decoding must pass through an explicit encoding/decoding
-step.  For simple cases, the  module-level ``c_string_type`` and
-``c_string_encoding`` directives can be used to implicitly insert these
-encoding/decoding steps to ease conversion between Python and C strings.
+Similar to the string semantics in Python 3, Cython strictly separates
+byte strings and unicode strings.  Above all, this means that by default
+there is no automatic conversion between byte strings and unicode strings
+(except for what Python 2 does in string operations).  All encoding and
+decoding must pass through an explicit encoding/decoding step.  To ease
+conversion between Python and C strings in simple cases, the module-level
+``c_string_type`` and ``c_string_encoding`` directives can be used to
+implicitly insert these encoding/decoding steps.
+
+
+Python string types in Cython code
+----------------------------------
+
+Cython supports three Python string types: ``bytes``, ``str``
+and ``unicode``.  The ``str`` type is special in that it is the
+byte string in Python 2 and the Unicode string in Python 3 (for Cython
+code compiled with language level 2, i.e. the default).  Thus, in Python
+2, both ``bytes`` and ``str`` represent the byte string type,
+whereas in Python 3, ``str`` and ``unicode`` represent the Python
+Unicode string type.  The switch is made at C compile time, the Python
+version that is used to run Cython is not relevant.
+
+When compiling Cython code with language level 3, the ``str`` type
+is identified with exactly the Unicode string type at Cython compile time,
+i.e. it no does not identify with ``bytes`` when running in Python 2.
+
+Note that the ``str`` type is not compatible with the ``unicode``
+type in Python 2, i.e. you cannot assign a Unicode string to a variable
+or argument that is typed ``str``.  The attempt will result in either
+a compile time error (if detectable) or a ``TypeError`` exception at
+runtime.  You should therefore be careful when you statically type a
+string variable in code that must be compatible with Python 2, as this
+Python version allows a mix of byte strings and unicode strings for data
+and users normally expect code to be able to work with both.  Code that
+only targets Python 3 can safely type variables and arguments as either
+``bytes`` or ``unicode``.
+
 
 General notes about C strings
 -----------------------------
@@ -38,7 +67,9 @@ using C strings where possible and use Python string objects instead.
 The obvious exception to this is when passing them back and forth
 from and to external C code.  Also, C++ strings remember their length
 as well, so they can provide a suitable alternative to Python bytes
-objects in some cases.
+objects in some cases, e.g. when reference counting is not needed
+within a well defined context.
+
 
 Passing byte strings
 --------------------

runtests.py

@@ -520,7 +520,9 @@ class CythonCompileTestCase(unittest.TestCase):
     def setUp(self):
         from Cython.Compiler import Options
         self._saved_options = [ (name, getattr(Options, name))
-                                for name in ('warning_errors', 'error_on_unknown_names') ]
+                                for name in ('warning_errors',
+                                             'error_on_unknown_names',
+                                             'error_on_uninitialized') ]
         self._saved_default_directives = Options.directive_defaults.items()
         Options.warning_errors = self.warning_errors
 
@@ -989,6 +991,7 @@ class CythonPyregrTestCase(CythonRunTestCase):
         CythonRunTestCase.setUp(self)
         from Cython.Compiler import Options
         Options.error_on_unknown_names = False
+        Options.error_on_uninitialized = False
         Options.directive_defaults.update(dict(
             binding=True, always_allow_keywords=True,
             set_initial_path="SOURCEFILE"))

tests/compile/libc_all.pyx

@@ -36,6 +36,3 @@ from libc.stdio  cimport *
 from libc.stdlib cimport *
 from libc.string cimport *
 
-libc.stdio.printf("hello %s\n", b"world")
-stdio.printf("hello %s\n", b"world")
-printf("hello %s\n", b"world")

tests/compile/libc_stdio.pyx

+# mode: compile
+
+cimport libc
+from libc cimport stdio
+from libc.stdio cimport printf, puts, fputs, putchar, fputc, putc, stdout
+
+
+libc.stdio.printf("hello %s\n", b"world")
+stdio.printf("hello %s\n", b"world")
+printf("hello %s\n", b"world")
+printf("printf_output %d %d\n", 1, 2)
+puts("puts_output")
+fputs("fputs_output", stdout)
+putchar(b'z')
+fputc(b'x', stdout)
+putc(b'c', stdout)

tests/errors/e_del.pyx

@@ -19,6 +19,9 @@ def outer(a):
     del a
     return inner()
 
+cdef object g
+del g
+
 
 _ERRORS = u"""
 10:9: Cannot assign to or delete this
@@ -26,4 +29,5 @@ _ERRORS = u"""
 13:9: Deletion of non-Python, non-C++ object
 14:9: Deletion of non-Python, non-C++ object
 19:9: can not delete variable 'a' referenced in nested scope
+23:5: Deletion of global C variable
 """

tests/run/external_ref_reassignment.pyx

+# Test that variable visible outside of the local scope (e.g. closure, cglobals)
+# is set before original value is decrefed.
+cdef object g
+
+def test_cglobals_reassignment():
+    """
+    >>> test_cglobals_reassignment()
+    1234
+    """
+    global g
+    class Special:
+        def __del__(self):
+            print g
+    g = (Special(),)
+    g = 1234
+
+def test_closure_reassignment():
+    """
+    >>> test_closure_reassignment()
+    4321
+    """
+    class Special:
+        def __del__(self):
+            print c
+    c = (Special(),)
+    c = 4321