merge

Cython/Compiler/Builtin.py

@@ -392,13 +392,15 @@ def init_builtins():
     init_builtin_funcs()
     init_builtin_types()
     init_builtin_structs()
-    global list_type, tuple_type, dict_type, set_type, bytes_type, unicode_type, type_type
+    global list_type, tuple_type, dict_type, set_type, type_type
+    global bytes_type, str_type, unicode_type
     type_type  = builtin_scope.lookup('type').type
     list_type  = builtin_scope.lookup('list').type
     tuple_type = builtin_scope.lookup('tuple').type
     dict_type  = builtin_scope.lookup('dict').type
     set_type   = builtin_scope.lookup('set').type
     bytes_type = builtin_scope.lookup('bytes').type
+    str_type   = builtin_scope.lookup('str').type
     unicode_type = builtin_scope.lookup('unicode').type
 
 init_builtins()

Cython/Compiler/Code.py

@@ -289,7 +289,8 @@ class PyObjectConst(object):
 
 possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
-nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
+nice_identifier = re.compile('^[a-zA-Z0-9_]+$').match
+find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
 
 class StringConst(object):
     """Global info about a C string constant held by GlobalState.
@@ -304,19 +305,31 @@ class StringConst(object):
         self.escaped_value = StringEncoding.escape_byte_string(byte_string)
         self.py_strings = None
 
-    def get_py_string_const(self, encoding, identifier=None):
+    def get_py_string_const(self, encoding, identifier=None, is_str=False):
         py_strings = self.py_strings
         text = self.text
-        if encoding is not None:
-            encoding = encoding.upper()
 
-        key = (bool(identifier), encoding)
+        is_str = bool(identifier or is_str)
+        is_unicode = encoding is None and not is_str
+
+        if encoding is None:
+            # unicode string
+            encoding_key = None
+        else:
+            # bytes or str
+            encoding = encoding.lower()
+            if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
+                encoding = None
+                encoding_key = None
+            else:
+                encoding_key = ''.join(find_alphanums(encoding))
+
+        key = (is_str, is_unicode, encoding_key)
         if py_strings is not None and key in py_strings:
             py_string = py_strings[key]
         else:
             if py_strings is None:
                 self.py_strings = {}
-            is_unicode = encoding is None
             if identifier:
                 intern = True
             elif identifier is None:
@@ -330,14 +343,13 @@ class StringConst(object):
                 prefix = Naming.interned_str_prefix
             else:
                 prefix = Naming.py_const_prefix
-            pystring_cname = "%s%s%s_%s" % (
+            pystring_cname = "%s%s_%s" % (
                 prefix,
-                is_unicode and 'u' or 'b',
-                identifier and 'i' or '',
+                (is_str and 's') or (is_unicode and 'u') or 'b',
                 self.cname[len(Naming.const_prefix):])
 
             py_string = PyStringConst(
-                pystring_cname, is_unicode, bool(identifier), intern)
+                pystring_cname, encoding, is_unicode, is_str, intern)
             self.py_strings[key] = py_string
 
         return py_string
@@ -346,14 +358,16 @@ class PyStringConst(object):
     """Global info about a Python string constant held by GlobalState.
     """
     # cname       string
-    # unicode     boolean
+    # encoding    string
     # intern      boolean
-    # identifier  boolean
+    # is_unicode  boolean
+    # is_str      boolean
 
-    def __init__(self, cname, is_unicode, identifier=False, intern=False):
+    def __init__(self, cname, encoding, is_unicode, is_str=False, intern=False):
         self.cname = cname
-        self.identifier = identifier
-        self.unicode = is_unicode
+        self.encoding = encoding
+        self.is_str = is_str
+        self.is_unicode = is_unicode
         self.intern = intern
 
     def __lt__(self, other):
@@ -393,7 +407,6 @@ class GlobalState(object):
 
     code_layout = [
         'h_code',
-        'complex_numbers_utility_code',
         'utility_code_proto_before_types',
         'type_declarations',
         'utility_code_proto',
@@ -550,12 +563,15 @@ class GlobalState(object):
             c = self.new_string_const(text, byte_string)
         return c
 
-    def get_py_string_const(self, text, identifier=None):
+    def get_py_string_const(self, text, identifier=None, is_str=False):
         # return a Python string constant, creating a new one if necessary
         c_string = self.get_string_const(text)
-        py_string = c_string.get_py_string_const(text.encoding, identifier)
+        py_string = c_string.get_py_string_const(text.encoding, identifier, is_str)
         return py_string
 
+    def get_interned_identifier(self, text):
+        return self.get_py_string_const(text, identifier=True)
+
     def new_string_const(self, text, byte_string):
         cname = self.new_string_const_cname(byte_string)
         c = StringConst(cname, text, byte_string)
@@ -582,7 +598,7 @@ class GlobalState(object):
             return self.new_const_cname()
 
         if len(value) < 20 and nice_identifier(value):
-            return "%s%s" % (Naming.const_prefix, value)
+            return "%s_%s" % (Naming.const_prefix, value)
         else:
             return self.new_const_cname()
 
@@ -601,7 +617,7 @@ class GlobalState(object):
     def add_cached_builtin_decl(self, entry):
         if Options.cache_builtins:
             if self.should_declare(entry.cname, entry):
-                interned_cname = self.get_py_string_const(entry.name, True).cname
+                interned_cname = self.get_interned_identifier(entry.name).cname
                 self.put_pyobject_decl(entry)
                 w = self.parts['cached_builtins']
                 w.putln('%s = __Pyx_GetName(%s, %s); if (!%s) %s' % (
@@ -649,18 +665,26 @@ class GlobalState(object):
             w.putln("static __Pyx_StringTabEntry %s[] = {" %
                                       Naming.stringtab_cname)
             for c_cname, _, py_string in py_strings:
+                if not py_string.is_str or not py_string.encoding or \
+                       py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
+                                              'UTF8', 'UTF-8'):
+                    encoding = '0'
+                else:
+                    encoding = '"%s"' % py_string.encoding.lower()
+
                 decls_writer.putln(
                     "static PyObject *%s;" % py_string.cname)
                 w.putln(
-                    "{&%s, %s, sizeof(%s), %d, %d, %d}," % (
+                    "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
                     py_string.cname,
                     c_cname,
                     c_cname,
-                    py_string.unicode,
-                    py_string.intern,
-                    py_string.identifier
+                    encoding,
+                    py_string.is_unicode,
+                    py_string.is_str,
+                    py_string.intern
                     ))
-            w.putln("{0, 0, 0, 0, 0, 0}")
+            w.putln("{0, 0, 0, 0, 0, 0, 0}")
             w.putln("};")
 
             init_globals = self.parts['init_globals']
@@ -894,8 +918,8 @@ class CCodeWriter(object):
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname
 
-    def get_py_string_const(self, text, identifier=None):
-        return self.globalstate.get_py_string_const(text, identifier).cname
+    def get_py_string_const(self, text, identifier=None, is_str=False):
+        return self.globalstate.get_py_string_const(text, identifier, is_str).cname
 
     def get_argument_default_const(self, type):
         return self.globalstate.get_py_const(type).cname
@@ -904,7 +928,7 @@ class CCodeWriter(object):
         return self.get_py_string_const(text)
 
     def intern_identifier(self, text):
-        return self.get_py_string_const(text, True)
+        return self.get_py_string_const(text, identifier=True)
 
     # code generation
 
@@ -1249,10 +1273,10 @@ class CCodeWriter(object):
         return self.globalstate.lookup_filename(filename)
 
     def put_setup_refcount_context(self, name):
-        self.putln('__Pyx_SetupRefcountContext("%s");' % name)
+        self.putln('__Pyx_RefNannySetupContext("%s");' % name)
 
     def put_finish_refcount_context(self):
-        self.putln("__Pyx_FinishRefcountContext();")
+        self.putln("__Pyx_RefNannyFinishContext();")
 
     def put_trace_call(self, name, pos):
         self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]));

Cython/Compiler/ModuleNode.py

@@ -260,7 +260,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         globalstate.module_pos = self.pos
         globalstate.directives = self.directives
 
-        globalstate.use_utility_code(refcount_utility_code)
+        globalstate.use_utility_code(refnanny_utility_code)
 
         code = globalstate['before_global_var']
         code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name)
@@ -454,6 +454,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyVarObject_HEAD_INIT(type, size) \\")
         code.putln("          PyObject_HEAD_INIT(type) size,")
         code.putln("  #define PyType_Modified(t)")
+        code.putln("  #define PyBytes_CheckExact PyString_CheckExact")
         code.putln("")
         code.putln("  typedef struct {")
         code.putln("     void *buf;")
@@ -495,7 +496,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#if PY_MAJOR_VERSION >= 3")
         code.putln("  #define PyBaseString_Type            PyUnicode_Type")
         code.putln("  #define PyString_Type                PyUnicode_Type")
-        code.putln("  #define PyString_CheckExact          PyBytes_CheckExact")
+        code.putln("  #define PyString_CheckExact          PyUnicode_CheckExact")
         code.putln("  #define PyInt_Type                   PyLong_Type")
         code.putln("  #define PyInt_Check(op)              PyLong_Check(op)")
         code.putln("  #define PyInt_CheckExact(op)         PyLong_CheckExact(op)")
@@ -558,12 +559,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#include <math.h>")
         code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env))
         self.generate_includes(env, cimported_modules, code)
-        if env.directives['c99_complex']:
-            code.putln("#ifndef _Complex_I")
-            code.putln("#include <complex.h>")
+        if env.directives['ccomplex']:
+            code.putln("")
+            code.putln("#if !defined(CYTHON_CCOMPLEX)")
+            code.putln("#define CYTHON_CCOMPLEX 1")
             code.putln("#endif")
-        code.putln("#define __PYX_USE_C99_COMPLEX defined(_Complex_I)")
-        code.putln('')
+            code.putln("")
         code.put(Nodes.utility_function_predeclarations)
         code.put(PyrexTypes.type_conversion_predeclarations)
         code.put(Nodes.branch_prediction_macros)
@@ -1633,19 +1634,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("{")
         tempdecl_code = code.insertion_point()
 
-        self.generate_filename_init_call(code)
-        code.putln("#ifdef CYTHON_REFNANNY")
-        code.putln("void* __pyx_refchk = NULL;")
-        code.putln("__Pyx_Refnanny = __Pyx_ImportRefcountAPI(\"refnanny\");")
-        code.putln("if (!__Pyx_Refnanny) {")
+        code.putln("#if CYTHON_REFNANNY")
+        code.putln("void* __pyx_refnanny = NULL;")
+        code.putln("__Pyx_RefNanny = __Pyx_RefNannyImportAPI(\"refnanny\");")
+        code.putln("if (!__Pyx_RefNanny) {")
         code.putln("  PyErr_Clear();")
-        code.putln("  __Pyx_Refnanny = __Pyx_ImportRefcountAPI(\"Cython.Runtime.refnanny\");")
-        code.putln("  if (!__Pyx_Refnanny)")
-        code.putln("      Py_FatalError(\"failed to import refnanny module\");")
+        code.putln("  __Pyx_RefNanny = __Pyx_RefNannyImportAPI(\"Cython.Runtime.refnanny\");")
+        code.putln("  if (!__Pyx_RefNanny)")
+        code.putln("      Py_FatalError(\"failed to import 'refnanny' module\");")
         code.putln("}")
-        code.putln("__pyx_refchk = __Pyx_Refnanny->NewContext(\"%s\", __LINE__, __FILE__);"% header3)
+        code.putln("__pyx_refnanny = __Pyx_RefNanny->SetupContext(\"%s\", __LINE__, __FILE__);"% header3)
         code.putln("#endif")
 
+        self.generate_filename_init_call(code)
+
         code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)));
         code.putln("#if PY_MAJOR_VERSION < 3");
         code.putln("%s = PyString_FromStringAndSize(\"\", 0); %s" % (Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos)));
@@ -2443,36 +2445,51 @@ bad:
 """ % {'IMPORT_STAR'     : Naming.import_star,
        'IMPORT_STAR_SET' : Naming.import_star_set }
         
-refcount_utility_code = UtilityCode(proto="""
-#ifdef CYTHON_REFNANNY
-typedef struct {
+refnanny_utility_code = UtilityCode(proto="""
+#ifndef CYTHON_REFNANNY
+  #define CYTHON_REFNANNY 0
+#endif
+
+#if CYTHON_REFNANNY
+  typedef struct {
     void (*INCREF)(void*, PyObject*, int);
     void (*DECREF)(void*, PyObject*, int);
     void (*GOTREF)(void*, PyObject*, int);
     void (*GIVEREF)(void*, PyObject*, int);
-  void* (*NewContext)(const char*, int, const char*);
+    void* (*SetupContext)(const char*, int, const char*);
     void (*FinishContext)(void**);
-} __Pyx_RefnannyAPIStruct;
-static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL;
-#define __Pyx_ImportRefcountAPI(name) \
-  (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)\"RefnannyAPI\")
-#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__)
-#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__)
-#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__)
-#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__)
-#define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r);} } while(0)
-#define __Pyx_SetupRefcountContext(name) \
-  void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__)
-#define __Pyx_FinishRefcountContext() \
-  __Pyx_Refnanny->FinishContext(&__pyx_refchk)
+  } __Pyx_RefNannyAPIStruct;
+  static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL;
+  static __Pyx_RefNannyAPIStruct * __Pyx_RefNannyImportAPI(const char *modname) {
+    PyObject *m = NULL, *p = NULL;
+    void *r = NULL;
+    m = PyImport_ImportModule((char *)modname);
+    if (!m) goto end;
+    p = PyObject_GetAttrString(m, (char *)\"RefNannyAPI\");
+    if (!p) goto end;
+    r = PyLong_AsVoidPtr(p);
+  end:
+    Py_XDECREF(p);
+    Py_XDECREF(m);
+    return (__Pyx_RefNannyAPIStruct *)r;
+  }
+  #define __Pyx_RefNannySetupContext(name) \
+          void *__pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
+  #define __Pyx_RefNannyFinishContext() \
+          __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
+  #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
+  #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r);} } while(0)
 #else
-#define __Pyx_INCREF(r) Py_INCREF(r)
-#define __Pyx_DECREF(r) Py_DECREF(r)
-#define __Pyx_GOTREF(r)
-#define __Pyx_GIVEREF(r)
-#define __Pyx_XDECREF(r) Py_XDECREF(r)
-#define __Pyx_SetupRefcountContext(name)
-#define __Pyx_FinishRefcountContext()
+  #define __Pyx_RefNannySetupContext(name)
+  #define __Pyx_RefNannyFinishContext()
+  #define __Pyx_INCREF(r) Py_INCREF(r)
+  #define __Pyx_DECREF(r) Py_DECREF(r)
+  #define __Pyx_GOTREF(r)
+  #define __Pyx_GIVEREF(r)
+  #define __Pyx_XDECREF(r) Py_XDECREF(r)
 #endif /* CYTHON_REFNANNY */
 #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);} } while(0)
 #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r);} } while(0)

Cython/Compiler/Nodes.py

@@ -3134,7 +3134,7 @@ class InPlaceAssignmentNode(AssignmentNode):
                 c_op = "/"
             elif c_op == "**":
                 error(self.pos, "No C inplace power operator")
-            elif self.lhs.type.is_complex and not code.globalstate.directives['c99_complex']:
+            elif self.lhs.type.is_complex:
                 error(self.pos, "Inplace operators not implemented for complex types.")
                 
             # have to do assignment directly to avoid side-effects
@@ -3479,15 +3479,11 @@ class RaiseStatNode(StatNode):
             tb_code = self.exc_tb.py_result()
         else:
             tb_code = "0"
-        if self.exc_type or self.exc_value or self.exc_tb:
         code.putln(
             "__Pyx_Raise(%s, %s, %s);" % (
                 type_code,
                 value_code,
                 tb_code))
-        else:
-            code.putln(
-                "__Pyx_ReRaise();")
         for obj in (self.exc_type, self.exc_value, self.exc_tb):
             if obj:
                 obj.generate_disposal_code(code)
@@ -4141,30 +4137,21 @@ class TryExceptStatNode(StatNode):
         if error_label_used or not self.has_default_clause:
             if error_label_used:
                 code.put_label(except_error_label)
-            for var in Naming.exc_save_vars:
-                code.put_xdecref(var, py_object_type)
-            code.put_goto(old_error_label)
-            
-        if code.label_used(try_break_label):
-            code.put_label(try_break_label)
             for var in Naming.exc_save_vars: code.put_xgiveref(var)
             code.putln("__Pyx_ExceptionReset(%s);" %
                        ', '.join(Naming.exc_save_vars))
-            code.put_goto(old_break_label)
+            code.put_goto(old_error_label)
 
-        if code.label_used(try_continue_label):
-            code.put_label(try_continue_label)
-            for var in Naming.exc_save_vars: code.put_xgiveref(var)
-            code.putln("__Pyx_ExceptionReset(%s);" %
-                       ', '.join(Naming.exc_save_vars))
-            code.put_goto(old_continue_label)
+        for exit_label, old_label in zip(
+            [try_break_label, try_continue_label, except_return_label],
+            [old_break_label, old_continue_label, old_return_label]):
 
-        if code.label_used(except_return_label):
-            code.put_label(except_return_label)
+            if code.label_used(exit_label):
+                code.put_label(exit_label)
                 for var in Naming.exc_save_vars: code.put_xgiveref(var)
                 code.putln("__Pyx_ExceptionReset(%s);" %
                            ', '.join(Naming.exc_save_vars))
-            code.put_goto(old_return_label)
+                code.put_goto(old_label)
 
         if code.label_used(except_end_label):
             code.put_label(except_end_label)
@@ -4757,7 +4744,7 @@ utility_function_predeclarations = \
 #define INLINE 
 #endif
 
-typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/
+typedef struct {PyObject **p; char *s; const long n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/
 
 """
 
@@ -5006,30 +4993,6 @@ raise_error:
 
 #------------------------------------------------------------------------------------
 
-reraise_utility_code = UtilityCode(
-proto = """
-static void __Pyx_ReRaise(void); /*proto*/
-""",
-impl = """
-static void __Pyx_ReRaise(void) {
-    PyThreadState *tstate = PyThreadState_GET();
-    PyObject* tmp_type = tstate->curexc_type;
-    PyObject* tmp_value = tstate->curexc_value;
-    PyObject* tmp_tb = tstate->curexc_traceback;
-    tstate->curexc_type = tstate->exc_type;
-    tstate->curexc_value = tstate->exc_value;
-    tstate->curexc_traceback = tstate->exc_traceback;
-    tstate->exc_type = 0;
-    tstate->exc_value = 0;
-    tstate->exc_traceback = 0;
-    Py_XDECREF(tmp_type);
-    Py_XDECREF(tmp_value);
-    Py_XDECREF(tmp_tb);
-}
-""")
-
-#------------------------------------------------------------------------------------
-
 arg_type_test_utility_code = UtilityCode(
 proto = """
 static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed,
@@ -5518,7 +5481,7 @@ impl = """
 static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
     while (t->p) {
         #if PY_MAJOR_VERSION < 3
-        if (t->is_unicode && (!t->is_identifier)) {
+        if (t->is_unicode) {
             *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
         } else if (t->intern) {
             *t->p = PyString_InternFromString(t->s);
@@ -5526,10 +5489,14 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
             *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
         }
         #else  /* Python 3+ has unicode identifiers */
-        if (t->is_identifier || (t->is_unicode && t->intern)) {
+        if (t->is_unicode | t->is_str) {
+            if (t->intern) {
                 *t->p = PyUnicode_InternFromString(t->s);
-        } else if (t->is_unicode) {
+            } else if (t->encoding) {
+                *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL);
+            } else {
                 *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
+            }
         } else {
             *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1);
         }

Cython/Compiler/Options.py

@@ -65,7 +65,7 @@ option_defaults = {
     'cdivision_warnings': False,
     'always_allow_keywords': False,
     'wraparound' : True,
-    'c99_complex' : False, # Don't use macro wrappers for complex arith, not sure what to name this...
+    'ccomplex' : False, # use C99/C++ for complex types and arith
     'callspec' : "",
     'profile': False,
     'infer_types': False,

Cython/Compiler/Parsing.py

@@ -348,7 +348,7 @@ def p_call(s, function):
                     s.error("Expected an identifier before '='",
                         pos = arg.pos)
                 encoded_name = EncodedString(arg.name)
-                keyword = ExprNodes.StringNode(arg.pos, value = encoded_name)
+                keyword = ExprNodes.IdentifierStringNode(arg.pos, value = encoded_name)
                 arg = p_simple_expr(s)
                 keyword_args.append((keyword, arg))
             else:
@@ -1128,14 +1128,14 @@ def p_import_statement(s):
         else:
             if as_name and "." in dotted_name:
                 name_list = ExprNodes.ListNode(pos, args = [
-                        ExprNodes.StringNode(pos, value = EncodedString("*"))])
+                        ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])
             else:
                 name_list = None
             stat = Nodes.SingleAssignmentNode(pos,
                 lhs = ExprNodes.NameNode(pos, 
                     name = as_name or target_name),
                 rhs = ExprNodes.ImportNode(pos, 
-                    module_name = ExprNodes.StringNode(
+                    module_name = ExprNodes.IdentifierStringNode(
                         pos, value = dotted_name),
                     name_list = name_list))
         stats.append(stat)
@@ -1193,7 +1193,7 @@ def p_from_import_statement(s, first_statement = 0):
         for (name_pos, name, as_name, kind) in imported_names:
             encoded_name = EncodedString(name)
             imported_name_strings.append(
-                ExprNodes.StringNode(name_pos, value = encoded_name))
+                ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
             items.append(
                 (name,
                  ExprNodes.NameNode(name_pos, 
@@ -1203,7 +1203,7 @@ def p_from_import_statement(s, first_statement = 0):
         dotted_name = EncodedString(dotted_name)
         return Nodes.FromImportStatNode(pos,
             module = ExprNodes.ImportNode(dotted_name_pos,
-                module_name = ExprNodes.StringNode(pos, value = dotted_name),
+                module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
                 name_list = import_list),
             items = items)
 
@@ -1713,7 +1713,7 @@ def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keyword
                     parsed_type = True
                 else:
                     arg = p_simple_expr(s)
-                keyword_node = ExprNodes.StringNode(
+                keyword_node = ExprNodes.IdentifierStringNode(
                     arg.pos, value = EncodedString(ident))
                 keyword_args.append((keyword_node, arg))
                 was_keyword = True

Cython/Compiler/Scanning.py

@@ -454,7 +454,6 @@ class PyrexScanner(Scanner):
                 sy = systring
             else:
                 systring = EncodedString(systring)
-                systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
         if False: # debug_scanner:

Cython/Compiler/Symtab.py

@@ -1157,7 +1157,7 @@ class StructOrUnionScope(Scope):
     def declare_cfunction(self, name, type, pos, 
                           cname = None, visibility = 'private', defining = 0,
                           api = 0, in_pxd = 0, modifiers = ()):
-        self.declare_var(name, type, pos, cname, visibility)
+        return self.declare_var(name, type, pos, cname, visibility)
 
 class ClassScope(Scope):
     #  Abstract base class for namespace of
@@ -1450,7 +1450,12 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) {
     }
     if (PyObject_TypeCheck(method, methoddescr_type)) { /* cdef classes */
         PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
-        return PyDescr_NewClassMethod(descr->d_type, descr->d_method);
+        #if PY_VERSION_HEX < 0x03020000
+        PyTypeObject *d_type = descr->d_type;
+        #else
+        PyTypeObject *d_type = descr->d_common.d_type;
+        #endif
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
     }
     else if (PyMethod_Check(method)) { /* python classes */
         return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
@@ -1458,7 +1463,9 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) {
     else if (PyCFunction_Check(method)) {
         return PyClassMethod_New(method);
     }
-    PyErr_Format(PyExc_TypeError, "Class-level classmethod() can only be called on a method_descriptor or instance method.");
+    PyErr_Format(PyExc_TypeError,
+                 "Class-level classmethod() can only be called on"
+                 "a method_descriptor or instance method.");
     return NULL;
 }
 """)

Cython/Includes/python.pxd

@@ -125,6 +125,7 @@ from python_mem cimport *
 from python_tuple cimport *
 from python_list cimport *
 from python_object cimport *
+from python_cobject cimport *
 from python_sequence cimport *
 from python_mapping cimport *
 from python_iterator cimport *

Cython/Includes/python_cobject.pxd

+cdef extern from "Python.h":
+    ctypedef void PyObject
+
+    ###########################################################################
+    # Warning:
+    #
+    # The CObject API is deprecated as of Python 3.1. Please switch to
+    # the new Capsules API.
+    ###########################################################################
+
+    int PyCObject_Check(object p)
+    #     Return true if its argument is a PyCObject.
+
+    object PyCObject_FromVoidPtr(void* cobj, void (*destr)(void *))
+    #     Return value: New reference.
+    #
+    #     Create a PyCObject from the void * cobj. The destr function will
+    #     be called when the object is reclaimed, unless it is NULL.
+
+    object PyCObject_FromVoidPtrAndDesc(void* cobj, void* desc, void (*destr)(void *, void *))
+    #     Return value: New reference.
+    #
+    #     Create a PyCObject from the void * cobj. The destr function will
+    #     be called when the object is reclaimed. The desc argument can be
+    #     used to pass extra callback data for the destructor function.
+
+    void* PyCObject_AsVoidPtr(object self)
+    #     Return the object void * that the PyCObject self was created with.
+
+    void* PyCObject_GetDesc(object self)
+    #     Return the description void * that the PyCObject self was created with.
+
+    int PyCObject_SetVoidPtr(object self, void* cobj)
+    #     Set the void pointer inside self to cobj. The PyCObject must not
+    #     have an associated destructor. Return true on success, false on
+    #     failure.

Cython/Runtime/refnanny.pyx

@@ -69,7 +69,7 @@ cdef void report_unraisable(object e):
 # exception has been fetched, in case we are called from
 # exception-handling code.
 
-cdef PyObject* NewContext(char* funcname, int lineno, char* filename) except NULL:
+cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except NULL:
     if Context is None:
         # Context may be None during finalize phase.
         # In that case, we don't want to be doing anything fancy
@@ -143,23 +143,23 @@ cdef void FinishContext(PyObject** ctx):
     ctx[0] = NULL
     PyErr_Restore(type, value, tb)
 
-cdef extern from "Python.h":
-    object PyCObject_FromVoidPtr(void*, void (*)(void*))
-
-ctypedef struct RefnannyAPIStruct:
+ctypedef struct RefNannyAPIStruct:
   void (*INCREF)(PyObject*, PyObject*, int)
   void (*DECREF)(PyObject*, PyObject*, int)
   void (*GOTREF)(PyObject*, PyObject*, int)
   void (*GIVEREF)(PyObject*, PyObject*, int)
-  PyObject* (*NewContext)(char*, int, char*) except NULL
+  PyObject* (*SetupContext)(char*, int, char*) except NULL
   void (*FinishContext)(PyObject**)
 
-cdef RefnannyAPIStruct api
+cdef RefNannyAPIStruct api
 api.INCREF = INCREF
 api.DECREF =  DECREF
 api.GOTREF =  GOTREF
 api.GIVEREF = GIVEREF
-api.NewContext = NewContext
+api.SetupContext = SetupContext
 api.FinishContext = FinishContext
 
-RefnannyAPI = PyCObject_FromVoidPtr(<void*>&api, NULL)
+cdef extern from "Python.h":
+    object PyLong_FromVoidPtr(void*)
+
+RefNannyAPI = PyLong_FromVoidPtr(<void*>&api)

runtests.py

@@ -59,7 +59,11 @@ class build_ext(_build_ext):
     def build_extension(self, ext):
         if ext.language == 'c++':
             try:
-                self.compiler.compiler_so.remove('-Wstrict-prototypes')
+                try: # Py2.7+ & Py3.2+ 
+                    compiler_obj = self.compiler_obj
+                except AttributeError:
+                    compiler_obj = self.compiler
+                compiler_obj.compiler_so.remove('-Wstrict-prototypes')
             except Exception:
                 pass
         _build_ext.build_extension(self, ext)
@@ -427,9 +431,14 @@ class _FakeClass(object):
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
 
-class PartialTestResult(unittest._TextTestResult):
+try: # Py2.7+ and Py3.2+
+    from unittest.runner import _TextTestResult
+except ImportError:
+    from unittest import _TextTestResult
+
+class PartialTestResult(_TextTestResult):
     def __init__(self, base_result):
-        unittest._TextTestResult.__init__(
+        _TextTestResult.__init__(
             self, self._StringIO(), True,
             base_result.dots + base_result.showAll*2)
 
@@ -730,7 +739,7 @@ if __name__ == '__main__':
                              build_in_temp=True,
                              pyxbuild_dir=os.path.join(WORKDIR, "support"))
         sys.path.insert(0, os.path.split(libpath)[0])
-        CFLAGS.append("-DCYTHON_REFNANNY")
+        CFLAGS.append("-DCYTHON_REFNANNY=1")
 
     test_bugs = False
     if options.tickets:

tests/compile/cnamespec.pyx

@@ -12,6 +12,9 @@ cdef double spam "c_spam" (int i, float f):
     cdef foo *p
     global b
     d = spam(a, f)
+    cdef foo q
+    q.i = 7
+    p = &q
     b = p.i
     p.i = x
     p.i = y

tests/errors/string_assignments.pyx

+# coding: ASCII
+
+# ok:
+cdef char* c1   =  "abc"
+cdef str s1     =  "abc"
+
+cdef unicode u1 = u"abc"
+
+cdef bytes b1 = b"abc"
+cdef char* c2 = b"abc"
+
+cdef bytes b2 = c1
+cdef char* c3 = b1
+
+cdef object o1  =  "abc"
+cdef object o2  = b"abc"
+cdef object o3  = u"abc"
+
+o4 = c1
+o5 = b1
+o6 = s1
+o7 = u1
+
+# errors:
+cdef char* c_f1   = u"abc"
+cdef char* c_f2   = u1
+cdef char* c_f3   = s1
+
+cdef bytes b_f1   = u"abc"
+cdef bytes b_f2   = u1
+cdef bytes b_f3   = s1
+
+cdef str s_f1  = b"abc"
+cdef str s_f2  = b1
+cdef str s_f3  = u"abc"
+cdef str s_f4  = u1
+
+cdef unicode u_f1 = "abc"
+cdef unicode u_f2 = s1
+cdef unicode u_f3 = b"abc"
+cdef unicode u_f4 = b1
+cdef unicode u_f5 = c1
+
+cdef tuple t_f1 =  "abc"
+cdef tuple t_f2 = u"abc"
+cdef tuple t_f3 = b"abc"
+
+cdef list  l_f1 = s1
+cdef list  l_f2 = b1
+cdef list  l_f3 = u1
+
+_ERRORS = u"""
+25:20: Unicode objects do not support coercion to C types.
+26:22: Unicode objects do not support coercion to C types.
+27:22: 'str' objects do not support coercion to C types.
+
+29:20: Cannot convert Unicode string to 'bytes' implicitly, encoding required.
+30:22: Cannot convert Unicode string to 'bytes' implicitly, encoding required.
+31:22: Cannot convert 'str' to 'bytes' implicitly. This is not portable.
+
+33:17: Cannot assign type 'char *' to 'str object'
+34:19: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
+35:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
+36:19: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
+
+38:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
+39:22: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
+40:20: Cannot assign type 'char *' to 'unicode object'
+41:22: Cannot convert 'bytes' object to unicode implicitly, decoding required
+42:22: Cannot convert 'char*' to unicode implicitly, decoding required
+
+44:19: Cannot assign type 'str object' to 'tuple object'
+45:18: Cannot assign type 'unicode object' to 'tuple object'
+46:18: Cannot assign type 'char *' to 'tuple object'
+"""

tests/run/complex_numbers_T305.pyx

 __doc__ = u"""
     >>> test_object_conversion(2)
-    ((2+0j), (2+0j))
+    ((2+0j), (2+0j), (2+0j))
     >>> test_object_conversion(2j - 0.5)
-    ((-0.5+2j), (-0.5+2j))
+    ((-0.5+2j), (-0.5+2j), (-0.5+2j))
     
     >>> test_arithmetic(2j, 4j)
-    (-2j, 6j, -2j, (-8+0j), (0.5+0j))
+    (2j, -2j, 6j, -2j, (-8+0j), (0.5+0j))
     >>> test_arithmetic(6+12j, 3j)
-    ((-6-12j), (6+15j), (6+9j), (-36+18j), (4-2j))
+    ((6+12j), (-6-12j), (6+15j), (6+9j), (-36+18j), (4-2j))
     >>> test_arithmetic(5-10j, 3+4j)
-    ((-5+10j), (8-6j), (2-14j), (55-10j), (-1-2j))
+    ((5-10j), (-5+10j), (8-6j), (2-14j), (55-10j), (-1-2j))
 
-    >>> test_div_by_zero(4j)
-    -0.25j
-    >>> test_div_by_zero(0)
-    Traceback (most recent call last):
-    ...
-    ZeroDivisionError: float division
+## XXX this is not working
+## >>> test_div_by_zero(4j)
+## -0.25j
+## >>> test_div_by_zero(0)
+## Traceback (most recent call last):
+## ...
+## ZeroDivisionError: float division
 
     >>> test_coercion(1, 1.5, 2.5, 4+1j, 10j)
     (1+0j)
@@ -56,6 +57,10 @@ __doc__ = u"""
     (1+2j)
     >>> test_real_imag_assignment(1.5, -3.5)
     (1.5-3.5j)
+
+## XXX not implemented yet!
+## >>> test_conjugate(1+2j)
+## (1-2j)
 """
 
 #cdef extern from "complex.h":
@@ -65,15 +70,17 @@ cimport cython
 
 def test_object_conversion(o):
     cdef float complex a = o
-    cdef double complex z = o
-    return (a, z)
+    cdef double complex b = o
+    cdef long double complex c = o
+    return (a, b, c)
 
 def test_arithmetic(double complex z, double complex w):
-    return -z, z+w, z-w, z*w, z/w
+    return +z, -z, z+w, z-w, z*w, z/w
 
-@cython.cdivision(False)
-def test_div_by_zero(double complex z):
-    return 1/z
+## XXX this is not working
+## @cython.cdivision(False)
+## def test_div_by_zero(double complex z):
+##    return 1/z
 
 def test_coercion(int a, float b, double c, float complex d, double complex e):
     cdef double complex z
@@ -102,3 +109,6 @@ def test_real_imag_assignment(object a, double b):
     z.imag = b
     return z
 
+## XXX not implemented yet!
+## def test_conjugate(float complex z):
+##     return z.conjugate()

tests/run/complex_numbers_c89_T398.h

+#define CYTHON_CCOMPLEX 0

tests/run/complex_numbers_c89_T398.pyx

+cdef extern from "complex_numbers_c89_T398.h": pass
+include "complex_numbers_T305.pyx"

tests/run/complex_numbers_c99_T398.h

+#if !defined(__cplusplus)
+#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) \
+  || defined(__GNUC__)						 \
+  || defined(__INTEL_COMPILER)					 \
+  || defined(__IBMC__)						 \
+
+#include <complex.h>
+#if !defined(_Complex_I)
+#error The "complex.h" header does not define the '_Complex_I' macro.
+#error Please report this to Cython developers <cython-dev@codespeak.net>
+#endif
+
+#endif
+#endif

tests/run/complex_numbers_c99_T398.pyx

+cdef extern from "complex_numbers_c99_T398.h": pass
+include "complex_numbers_T305.pyx"

tests/run/complex_numbers_cxx_T398.h

+#if defined(__cplusplus)
+#define CYTHON_CCOMPLEX 1
+#else
+#define CYTHON_CCOMPLEX 0
+#endif

tests/run/complex_numbers_cxx_T398.pyx

+cdef extern from "complex_numbers_cxx_T398.h": pass
+include "complex_numbers_T305.pyx"

tests/run/embedsignatures.pyx

@@ -68,10 +68,10 @@ __doc__ = ur"""
     'with_doc_4(int a, str b, list c) -> str\n\n    Existing string\n    '
 
     >>> f_sd.__doc__
-    "f_sd(str s=u'spam')"
+    "f_sd(str s='spam')"
 
     >>> cf_sd.__doc__
-    "cf_sd(str s=u'spam') -> str"
+    "cf_sd(str s='spam') -> str"
 
     >>> types.__doc__
     'types(Ext a, int b, unsigned short c, float d, e)'
@@ -219,10 +219,10 @@ cpdef str with_doc_4(int a, str b, list c):
     """
     return b
 
-def f_sd(str s=u'spam'):
+def f_sd(str s='spam'):
     return s
 
-cpdef str cf_sd(str s=u'spam'):
+cpdef str cf_sd(str s='spam'):
     return s
 
 cpdef char f_c(char c):

tests/run/funcexc_iter_T228.pyx

@@ -46,8 +46,6 @@ True
 
 import sys
 
-IS_PY3 = sys.version_info[0] >= 3
-
 cdef class cy_iterator(object):
     def __iter__(self):
         return self

tests/run/funcexceptraise.pyx

+__doc__ = u"""
+>>> def bar():
+...     try:
+...         foo()
+...     except ValueError:
+...         pass
+
+>>> bar()
+>>> print(sys.exc_info())
+(None, None, None)
+"""
+
+import sys
+
+def foo():
+    try:
+        raise TypeError
+    except TypeError:
+        raise ValueError

tests/run/literals.pyx

@@ -37,7 +37,7 @@ double quoted string."""
     i = 'This string\
  has an ignored newline.'
     j = 'One-char escapes: \'\"\\\a\b\f\n\r\t\v'
-    k = 'Oct and hex escapes: \1 \12 \123 \x45 \xaf \xAF'
+    k = b'Oct and hex escapes: \1 \12 \123 \x45 \xaf \xAF'
     l = r'''This is\
 a \three \line
 raw string with some backslashes.'''

tests/run/str_encoding_latin1.pyx

+#  -*- coding: latin-1 -*-
+
+__doc__ = (u"""
+>>> a == 'abc'
+True
+>>> isinstance(a, str)
+True
+
+>>> isinstance(s, str)
+True
+>>> len(s)
+6
+>>> s == 'ao'
+True
+
+>>> isinstance(add(), str)
+True
+>>> len(add())
+9
+>>> add() == 'abcao'
+True
+
+>>> isinstance(add_literal(), str)
+True
+>>> len(add_literal())
+9
+>>> add_literal() == 'abcao'
+True
+
+>>> isinstance(typed(), str)
+True
+>>> len(typed())
+6
+>>> typed() == ''
+True
+
+"""
+# recoding/escaping is required to properly pass the literals to doctest
+).encode('unicode_escape').decode('ASCII')
+
+
+a = 'abc'
+s = 'ao'
+u = u'ao'
+
+cdef str S = ''
+
+def add():
+    return a+s
+
+def add_literal():
+    return 'abc' + 'ao'
+
+def typed():
+    return S

tests/run/type_inference.pyx

@@ -23,7 +23,7 @@ def simple():
     b = b"abc"
     assert typeof(b) == "char *", typeof(b)
     s = "abc"
-    assert typeof(s) == "Python object", typeof(s)
+    assert typeof(s) == "str object", typeof(s)
     u = u"xyz"
     assert typeof(u) == "unicode object", typeof(u)
     L = [1,2,3]

tests/run/typed_slice.pyx

@@ -52,13 +52,13 @@ def slice_list_assign(list l, value):
 
 
 def slice_charp(py_string_arg):
-    cdef str py_string = py_string_arg.encode(u'ASCII')
+    cdef bytes py_string = py_string_arg.encode(u'ASCII')
     cdef char* s = py_string
     return s[1:3].decode(u'ASCII')
 
 def slice_charp_repeat(py_string_arg):
-    cdef str py_string = py_string_arg.encode(u'ASCII')
+    cdef bytes py_string = py_string_arg.encode(u'ASCII')
     cdef char* s = py_string
-    cdef str slice_val = s[1:6]
+    cdef bytes slice_val = s[1:6]
     s = slice_val
     return s[1:3].decode(u'ASCII')