Renamed Py_UNICODE* entities to use "pyunicode_ptr" prefix; fixed small issues...

Renamed Py_UNICODE* entities to use "pyunicode_ptr" prefix; fixed small issues in Py_UNICODE* support.

Cython/Compiler/Code.py

@@ -778,7 +778,6 @@ class StringConst(object):
         self.py_strings[key] = py_string
         return py_string
 
-
 class PyStringConst(object):
     """Global info about a Python string constant held by GlobalState.
     """
@@ -1018,7 +1017,7 @@ class GlobalState(object):
         c.add_py_version(py_version)
         return c
 
-    def get_unicode_const(self, text):
+    def get_pyunicode_ptr_const(self, text):
         # return a Py_UNICODE[] constant, creating a new one if necessary
         assert text.is_unicode
         try:
@@ -1153,7 +1152,7 @@ class GlobalState(object):
                     py_strings.append((c.cname, len(py_string.cname), py_string))
 
         for c, cname in self.unicode_const_index.items():
-            utf16_array, utf32_array = StringEncoding.encode_py_unicode_string(c)
+            utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
             if utf16_array:
                 # Narrow and wide representations differ
                 decls_writer.putln("#ifdef Py_UNICODE_WIDE")
@@ -1457,8 +1456,8 @@ class CCodeWriter(object):
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname
 
-    def get_unicode_const(self, text):
-        return self.globalstate.get_unicode_const(text)
+    def get_pyunicode_ptr_const(self, text):
+        return self.globalstate.get_pyunicode_ptr_const(text)
 
     def get_py_string_const(self, text, identifier=None,
                             is_str=False, unicode_value=None):

Cython/Compiler/ExprNodes.py

@@ -1215,7 +1215,7 @@ class UnicodeNode(ConstNode):
             if dst_type.is_string and self.bytes_value is not None:
                 # special case: '-3' enforced unicode literal used in a C char* context
                 return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env)
-            if dst_type.is_unicode:
+            if dst_type.is_pyunicode_ptr:
                 node = UnicodeNode(self.pos, value=self.value)
                 node.type = dst_type
                 return node
@@ -1242,7 +1242,7 @@ class UnicodeNode(ConstNode):
         if self.type.is_pyobject:
             self.result_code = code.get_py_string_const(self.value)
         else:
-            self.result_code = code.get_unicode_const(self.value)
+            self.result_code = code.get_pyunicode_ptr_const(self.value)
 
     def calculate_result_code(self):
         return self.result_code
@@ -2646,7 +2646,7 @@ class IndexNode(ExprNode):
             if base_type.is_string:
                 # sliced C strings must coerce to Python
                 return bytes_type
-            elif base_type.is_unicode:
+            elif base_type.is_pyunicode_ptr:
                 # sliced Py_UNICODE* strings must coerce to Python
                 return unicode_type
             elif base_type in (unicode_type, bytes_type, str_type, list_type, tuple_type):
@@ -3462,7 +3462,7 @@ class SliceIndexNode(ExprNode):
         base_type = self.base.infer_type(env)
         if base_type.is_string or base_type.is_cpp_class:
             return bytes_type
-        elif base_type.is_unicode:
+        elif base_type.is_pyunicode_ptr:
             return unicode_type
         elif base_type in (bytes_type, str_type, unicode_type,
                            list_type, tuple_type):
@@ -3528,7 +3528,7 @@ class SliceIndexNode(ExprNode):
         base_type = self.base.type
         if base_type.is_string or base_type.is_cpp_string:
             self.type = default_str_type(env)
-        elif base_type.is_unicode:
+        elif base_type.is_pyunicode_ptr:
             self.type = unicode_type
         elif base_type.is_ptr:
             self.type = base_type
@@ -3598,7 +3598,7 @@ class SliceIndexNode(ExprNode):
                         stop_code,
                         start_code,
                         code.error_goto_if_null(result, self.pos)))
-        elif self.base.type.is_unicode:
+        elif self.base.type.is_pyunicode_ptr:
             base_result = self.base.result()
             if self.base.type != PyrexTypes.c_py_unicode_ptr_type:
                 base_result = '((const Py_UNICODE*)%s)' % base_result
@@ -4944,11 +4944,11 @@ class AttributeNode(ExprNode):
         self.is_py_attr = 0
         self.member = self.attribute
         if obj_type is None:
-            if self.obj.type.is_string or self.obj.type.is_unicode:
+            if self.obj.type.is_string or self.obj.type.is_pyunicode_ptr:
                 self.obj = self.obj.coerce_to_pyobject(env)
             obj_type = self.obj.type
         else:
-            if obj_type.is_string or obj_type.is_unicode:
+            if obj_type.is_string or obj_type.is_pyunicode_ptr:
                 obj_type = py_object_type
         if obj_type.is_ptr or obj_type.is_array:
             obj_type = obj_type.base_type
@@ -8378,11 +8378,11 @@ class BinopNode(ExprNode):
         if self.is_py_operation_types(type1, type2):
             if type2.is_string:
                 type2 = Builtin.bytes_type
-            elif type2.is_unicode:
+            elif type2.is_pyunicode_ptr:
                 type2 = Builtin.unicode_type
             if type1.is_string:
                 type1 = Builtin.bytes_type
-            elif type1.is_unicode:
+            elif type1.is_pyunicode_ptr:
                 type1 = Builtin.unicode_type
             elif self.operator == '%' \
                      and type1 in (Builtin.str_type, Builtin.unicode_type):
@@ -8632,7 +8632,7 @@ class AddNode(NumBinopNode):
     #  '+' operator.
 
     def is_py_operation_types(self, type1, type2):
-        if type1.is_string and type2.is_string or type1.is_unicode and type2.is_unicode:
+        if type1.is_string and type2.is_string or type1.is_pyunicode_ptr and type2.is_pyunicode_ptr:
             return 1
         else:
             return NumBinopNode.is_py_operation_types(self, type1, type2)
@@ -9995,7 +9995,7 @@ class CoerceToPyTypeNode(CoercionNode):
             # be specific about some known types
             if arg.type.is_string or arg.type.is_cpp_string:
                 self.type = default_str_type(env)
-            elif arg.type.is_unicode or arg.type.is_unicode_char:
+            elif arg.type.is_pyunicode_ptr or arg.type.is_unicode_char:
                 self.type = unicode_type
             elif arg.type.is_complex:
                 self.type = Builtin.complex_type
@@ -10110,7 +10110,7 @@ class CoerceFromPyTypeNode(CoercionNode):
         if not result_type.create_from_py_utility_code(env):
             error(arg.pos,
                   "Cannot convert Python object to '%s'" % result_type)
-        if self.type.is_string or self.type.is_unicode:
+        if self.type.is_string or self.type.is_pyunicode_ptr:
             if self.arg.is_ephemeral():
                 error(arg.pos,
                       "Obtaining '%s' from temporary Python value" % result_type)

Cython/Compiler/Optimize.py

@@ -2017,12 +2017,11 @@ class OptimizeBuiltinCalls(Visitor.MethodDispatcherTransform):
                 args = [arg],
                 is_temp = node.is_temp,
                 utility_code = UtilityCode.load_cached("IncludeStringH", "StringTools.c"))
-        elif arg.type.is_unicode:
+        elif arg.type.is_pyunicode_ptr:
             new_node = ExprNodes.PythonCapiCallNode(
                 node.pos, "__Pyx_Py_UNICODE_strlen", self.Pyx_Py_UNICODE_strlen_func_type,
                 args = [arg],
-                is_temp = node.is_temp,
-                utility_code = UtilityCode.load_cached("py_unicode_strlen", "StringTools.c"))
+                is_temp = node.is_temp)
         elif arg.type.is_pyobject:
             cfunc_name = self._map_to_capi_len_function(arg.type)
             if cfunc_name is None:

Cython/Compiler/PyrexTypes.py

@@ -145,7 +145,7 @@ class PyrexType(BaseType):
     #  is_enum               boolean     Is a C enum type
     #  is_typedef            boolean     Is a typedef type
     #  is_string             boolean     Is a C char * type
-    #  is_unicode            boolean     Is a C PyUNICODE * type
+    #  is_pyunicode_ptr      boolean     Is a C PyUNICODE * type
     #  is_cpp_string         boolean     Is a C++ std::string type
     #  is_unicode_char       boolean     Is either Py_UCS4 or Py_UNICODE
     #  is_returncode         boolean     Is used only to signal exceptions
@@ -203,7 +203,7 @@ class PyrexType(BaseType):
     is_enum = 0
     is_typedef = 0
     is_string = 0
-    is_unicode = 0
+    is_pyunicode_ptr = 0
     is_unicode_char = 0
     is_returncode = 0
     is_error = 0
@@ -873,7 +873,7 @@ class PyObjectType(PyrexType):
 
     def assignable_from(self, src_type):
         # except for pointers, conversion will be attempted
-        return not src_type.is_ptr or src_type.is_string or src_type.is_unicode
+        return not src_type.is_ptr or src_type.is_string or src_type.is_pyunicode_ptr
 
     def declaration_code(self, entity_code,
             for_display = 0, dll_linkage = None, pyrex = 0):
@@ -1163,7 +1163,7 @@ class CType(PyrexType):
 
     def error_condition(self, result_code):
         conds = []
-        if self.is_string or self.is_unicode:
+        if self.is_string or self.is_pyunicode_ptr:
             conds.append("(!%s)" % result_code)
         elif self.exception_value is not None:
             conds.append("(%s == (%s)%s)" % (result_code, self.sign_and_name(), self.exception_value))
@@ -2182,7 +2182,7 @@ class CPointerBaseType(CType):
                 break
         else:
             if base_type.same_as(c_py_unicode_type):
-                self.is_unicode = 1
+                self.is_pyunicode_ptr = 1
 
         if self.is_string and not base_type.is_error:
             if base_type.signed:
@@ -2194,7 +2194,7 @@ class CPointerBaseType(CType):
                 if self.is_ptr:
                     self.from_py_function = "__Pyx_PyObject_AsUString"
             self.exception_value = "NULL"
-        elif self.is_unicode and not base_type.is_error:
+        elif self.is_pyunicode_ptr and not base_type.is_error:
             self.to_py_function = "__Pyx_PyUnicode_FromUnicode"
             if self.is_ptr:
                 self.from_py_function = "__Pyx_PyUnicode_AsUnicode"
@@ -2203,7 +2203,7 @@ class CPointerBaseType(CType):
     def py_type_name(self):
         if self.is_string:
             return "bytes"
-        elif self.is_unicode:
+        elif self.is_pyunicode_ptr:
             return "unicode"
         else:
             return super(CPointerBaseType, self).py_type_name()

Cython/Compiler/StringEncoding.py

@@ -264,7 +264,7 @@ def split_string_literal(s, limit=2000):
             start = end
         return '""'.join(chunks)
 
-def encode_py_unicode_string(s):
+def encode_pyunicode_string(s):
     """Create Py_UNICODE[] representation of a given unicode string.
     """
     utf32_array = array.array('i', s.encode('UTF-32'))

Cython/Utility/StringTools.c

@@ -604,17 +604,3 @@ static CYTHON_INLINE char __Pyx_PyBytes_GetItemInt(PyObject* bytes, Py_ssize_t i
         index += PyBytes_GET_SIZE(bytes);
     return PyBytes_AS_STRING(bytes)[index];
 }
-
-
-/////////////// py_unicode_strlen.proto ///////////////
-
-#if PY_VERSION_HEX < 0x03000000
-static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u)
-{
-    const Py_UNICODE *u_end = u;
-    while (*u_end++) ;
-    return u_end - u - 1;
-}
-#else
-#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen
-#endif

Cython/Utility/TypeConversion.c

 /////////////// TypeConversions.proto ///////////////
-// @requires: py_unicode_strlen
 
 /* Type Conversion Predeclarations */
 
@@ -25,14 +24,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char*);
 #define __Pyx_PyStr_FromUString(s)     __Pyx_PyStr_FromString((char*)s)
 #define __Pyx_PyUnicode_FromUString(s) __Pyx_PyUnicode_FromString((char*)s)
 
-#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u))
-#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode
-#if CYTHON_PEP393_ENABLED
-#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode
+#if PY_MAJOR_VERSION < 3
+static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u)
+{
+    const Py_UNICODE *u_end = u;
+    while (*u_end++) ;
+    return u_end - u - 1;
+}
 #else
-#define __Pyx_PyUnicode_AsUnicode PyUnicode_AS_UNICODE
+#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen
 #endif
 
+#define __Pyx_PyUnicode_FromUnicode(u)       PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u))
+#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode
+#define __Pyx_PyUnicode_AsUnicode            PyUnicode_AsUnicode
+
 #define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None)
 #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
 static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*);

tests/run/py_unicode_strings.pyx

 # tag: py_unicode_strings
 
+import sys
+
 cimport cython
 from libc.string cimport memcpy, strcpy
 
@@ -63,6 +65,10 @@ def test_c_to_python():
     else:
         assert len(c_pu_wide_literal) == 4
 
+    if sys.version_info >= (3, 3):
+        # Make sure len(unicode) is not reverted to pre-3.3 behavior
+        assert len(uwide_literal) == 2
+
     assert u'unicode'
     assert not u''
     assert c_pu_str