byte encode docstrings correctly

Cython/Compiler/PyrexTypes.py

@@ -2,6 +2,7 @@
 #   Pyrex - Types
 #
 
+from Cython import Utils
 import Naming
 
 class BaseType:
@@ -922,23 +923,6 @@ class CEnumType(CType):
             return self.base_declaration_code(public_decl(base, dll_linkage), entity_code)
 
 
-def _escape_byte_string(s):
-    s = s.replace('\0', r'\x00')
-    try:
-        s.decode("ASCII")
-        return s
-    except UnicodeDecodeError:
-        pass
-    l = []
-    append = l.append
-    for c in s:
-        o = ord(c)
-        if o >= 128:
-            append('\\x%X' % o)
-        else:
-            append(c)
-    return ''.join(l)
-
 class CStringType:
     #  Mixin class for C string types.
 
@@ -951,7 +935,7 @@ class CStringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CUTF8StringType:
@@ -965,7 +949,7 @@ class CUTF8StringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CCharArrayType(CStringType, CArrayType):

Cython/Compiler/TypeSlots.py

@@ -3,6 +3,7 @@
 #           and associated know-how.
 #
 
+from Cython import Utils
 import Naming
 import PyrexTypes
 import sys
@@ -298,7 +299,11 @@ class DocStringSlot(SlotDescriptor):
     
     def slot_code(self, scope):
         if scope.doc is not None:
-            return '"%s"' % scope.doc
+            if scope.doc.is_unicode:
+                doc = scope.doc.utf8encode()
+            else:
+                doc = scope.doc.byteencode()
+            return '"%s"' % Utils.escape_byte_string(doc)
         else:
             return "0"
 

Cython/Utils.py

@@ -91,3 +91,20 @@ class EncodedString(unicode):
 #    def __eq__(self, other):
 #        return unicode.__eq__(self, other) and \
 #            getattr(other, 'encoding', '') == self.encoding
+
+def escape_byte_string(s):
+    s = s.replace('\0', r'\x00')
+    try:
+        s.decode("ASCII")
+        return s
+    except UnicodeDecodeError:
+        pass
+    l = []
+    append = l.append
+    for c in s:
+        o = ord(c)
+        if o >= 128:
+            append('\\x%X' % o)
+        else:
+            append(c)
+    return ''.join(l)