Issue #8670: ctypes.c_wchar supports non-BMP characters with 32 bits wchar_t

Lib/ctypes/test/test_strings.py

@@ -74,6 +74,13 @@ else:
             buf[1] = "Z"
             self.assertEqual(buf.value, "xZCD")
 
+        @unittest.skipIf(sizeof(c_wchar) < 4,
+                         "sizeof(wchar_t) is smaller than 4 bytes")
+        def test_nonbmp(self):
+            u = chr(0x10ffff)
+            w = c_wchar(u)
+            self.assertEqual(w.value, u)
+
 class StringTestCase(unittest.TestCase):
     def XX_test_basic_strings(self):
         cs = c_string("abcdef")

Lib/test/test_unicode.py

@@ -1396,26 +1396,26 @@ class UnicodeTest(string_tests.CommonTest,
 
     # Test PyUnicode_AsWideChar()
     def test_aswidechar(self):
-        from _testcapi import test_aswidechar
+        from _testcapi import unicode_aswidechar
         from ctypes import c_wchar, sizeof
 
-        wchar, size = test_aswidechar('abcdef', 2)
+        wchar, size = unicode_aswidechar('abcdef', 2)
         self.assertEquals(size, 2)
         self.assertEquals(wchar, 'ab')
 
-        wchar, size = test_aswidechar('abc', 3)
+        wchar, size = unicode_aswidechar('abc', 3)
         self.assertEquals(size, 3)
         self.assertEquals(wchar, 'abc')
 
-        wchar, size = test_aswidechar('abc', 4)
+        wchar, size = unicode_aswidechar('abc', 4)
         self.assertEquals(size, 3)
         self.assertEquals(wchar, 'abc\0')
 
-        wchar, size = test_aswidechar('abc', 10)
+        wchar, size = unicode_aswidechar('abc', 10)
         self.assertEquals(size, 3)
         self.assertEquals(wchar, 'abc\0')
 
-        wchar, size = test_aswidechar('abc\0def', 20)
+        wchar, size = unicode_aswidechar('abc\0def', 20)
         self.assertEquals(size, 7)
         self.assertEquals(wchar, 'abc\0def\0')
 
@@ -1426,20 +1426,20 @@ class UnicodeTest(string_tests.CommonTest,
         else: # sizeof(c_wchar) == 4
             buflen = 2
             nchar = 1
-        wchar, size = test_aswidechar(nonbmp, buflen)
+        wchar, size = unicode_aswidechar(nonbmp, buflen)
         self.assertEquals(size, nchar)
         self.assertEquals(wchar, nonbmp + '\0')
 
     # Test PyUnicode_AsWideCharString()
     def test_aswidecharstring(self):
-        from _testcapi import test_aswidecharstring
+        from _testcapi import unicode_aswidecharstring
         from ctypes import c_wchar, sizeof
 
-        wchar, size = test_aswidecharstring('abc')
+        wchar, size = unicode_aswidecharstring('abc')
         self.assertEquals(size, 3)
         self.assertEquals(wchar, 'abc\0')
 
-        wchar, size = test_aswidecharstring('abc\0def')
+        wchar, size = unicode_aswidecharstring('abc\0def')
         self.assertEquals(size, 7)
         self.assertEquals(wchar, 'abc\0def\0')
 
@@ -1448,7 +1448,7 @@ class UnicodeTest(string_tests.CommonTest,
             nchar = 2
         else: # sizeof(c_wchar) == 4
             nchar = 1
-        wchar, size = test_aswidecharstring(nonbmp)
+        wchar, size = unicode_aswidecharstring(nonbmp)
         self.assertEquals(size, nchar)
         self.assertEquals(wchar, nonbmp + '\0')
 

Modules/_ctypes/cfield.c

@@ -1203,6 +1203,7 @@ static PyObject *
 u_set(void *ptr, PyObject *value, Py_ssize_t size)
 {
     Py_ssize_t len;
+    wchar_t chars[2];
     if (!PyUnicode_Check(value)) {
         PyErr_Format(PyExc_TypeError,
                         "unicode string expected instead of %s instance",
@@ -1211,7 +1212,7 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size)
     } else
         Py_INCREF(value);
 
-    len = PyUnicode_GET_SIZE(value);
+    len = PyUnicode_AsWideChar((PyUnicodeObject *)value, chars, 2);
     if (len != 1) {
         Py_DECREF(value);
         PyErr_SetString(PyExc_TypeError,
@@ -1219,7 +1220,7 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size)
         return NULL;
     }
 
-    *(wchar_t *)ptr = PyUnicode_AS_UNICODE(value)[0];
+    *(wchar_t *)ptr = chars[0];
     Py_DECREF(value);
 
     _RET(value);

Modules/_testcapimodule.c

@@ -1386,7 +1386,7 @@ test_widechar(PyObject *self)
 }
 
 static PyObject *
-test_aswidechar(PyObject *self, PyObject *args)
+unicode_aswidechar(PyObject *self, PyObject *args)
 {
     PyObject *unicode, *result;
     Py_ssize_t buflen, size;
@@ -1417,7 +1417,7 @@ test_aswidechar(PyObject *self, PyObject *args)
 }
 
 static PyObject *
-test_aswidecharstring(PyObject *self, PyObject *args)
+unicode_aswidecharstring(PyObject *self, PyObject *args)
 {
     PyObject *unicode, *result;
     Py_ssize_t size;
@@ -2321,8 +2321,8 @@ static PyMethodDef TestMethods[] = {
     {"test_u_code",             (PyCFunction)test_u_code,        METH_NOARGS},
     {"test_Z_code",             (PyCFunction)test_Z_code,        METH_NOARGS},
     {"test_widechar",           (PyCFunction)test_widechar,      METH_NOARGS},
-    {"test_aswidechar",         test_aswidechar,                 METH_VARARGS},
-    {"test_aswidecharstring",   test_aswidecharstring,           METH_VARARGS},
+    {"unicode_aswidechar",      unicode_aswidechar,                 METH_VARARGS},
+    {"unicode_aswidecharstring",unicode_aswidecharstring,           METH_VARARGS},
 #ifdef WITH_THREAD
     {"_test_thread_state",      test_thread_state,               METH_VARARGS},
     {"_pending_threadfunc",     pending_threadfunc,              METH_VARARGS},