PyInterpreterState_New(), PyThreadState_New(): use malloc/free directly.

This appears to finish repairs for SF bug 1041645.

This is a critical bugfix.

Misc/NEWS

@@ -136,6 +136,16 @@ Build
 C API
 -----
 
+- The C API calls ``PyInterpreterState_New()`` and ``PyThreadState_New()``
+  are two of the very few advertised as being safe to call without holding
+  the GIL.  However, this wasn't true in a debug build, as bug 1041645
+  demonstrated.  In a debug build, Python redirects the ``PyMem`` family
+  of calls to Python's small-object allocator, to get the benefit of
+  its extra debugging capabilities.  But Python's small-object allocator
+  isn't threadsafe, relying on the GIL to avoid the expense of doing its
+  own locking.  ``PyInterpreterState_New()`` and ``PyThreadState_New()``
+  call the platform ``malloc()`` directly now, regardless of build type.
+
 - PyLong_AsUnsignedLong[Mask] now support int objects as well.
 
 - SF patch #998993: ``PyUnicode_DecodeUTF8Stateful`` and

Python/pystate.c

@@ -3,6 +3,16 @@
 
 #include "Python.h"
 
+/* --------------------------------------------------------------------------
+CAUTION
+
+Always use malloc() and free() directly in this file.  A number of these
+functions are advertised as safe to call when the GIL isn't held, and in
+a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's debugging
+obmalloc functions.  Those aren't thread-safe (they rely on the GIL to avoid
+the expense of doing their own locking).
+-------------------------------------------------------------------------- */
+
 #ifdef HAVE_DLOPEN
 #ifdef HAVE_DLFCN_H
 #include <dlfcn.h>
@@ -41,7 +51,8 @@ PyThreadFrameGetter _PyThreadState_GetFrame = NULL;
 PyInterpreterState *
 PyInterpreterState_New(void)
 {
-	PyInterpreterState *interp = PyMem_NEW(PyInterpreterState, 1);
+	PyInterpreterState *interp = (PyInterpreterState *)
+				     malloc(sizeof(PyInterpreterState));
 
 	if (interp != NULL) {
 		HEAD_INIT();
@@ -119,7 +130,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
 		Py_FatalError("PyInterpreterState_Delete: remaining threads");
 	*p = interp->next;
 	HEAD_UNLOCK();
-	PyMem_DEL(interp);
+	free(interp);
 }
 
 
@@ -133,7 +144,8 @@ threadstate_getframe(PyThreadState *self)
 PyThreadState *
 PyThreadState_New(PyInterpreterState *interp)
 {
-	PyThreadState *tstate = PyMem_NEW(PyThreadState, 1);
+	PyThreadState *tstate = (PyThreadState *)malloc(sizeof(PyThreadState));
+
 	if (_PyThreadState_GetFrame == NULL)
 		_PyThreadState_GetFrame = threadstate_getframe;
 
@@ -226,7 +238,7 @@ tstate_delete_common(PyThreadState *tstate)
 	}
 	*p = tstate->next;
 	HEAD_UNLOCK();
-	PyMem_DEL(tstate);
+	free(tstate);
 }