New PyGILState_ API - implements pep 311, from patch 684256.

Include/pystate.h

@@ -72,6 +72,7 @@ typedef struct _ts {
     PyObject *dict;
 
     int tick_counter;
+    int gilstate_counter;
 
     /* XXX signal handlers should also be here */
 
@@ -104,6 +105,51 @@ PyAPI_DATA(PyThreadState *) _PyThreadState_Current;
 #define PyThreadState_GET() (_PyThreadState_Current)
 #endif
 
+typedef 
+    enum {PyGILState_LOCKED, PyGILState_UNLOCKED}
+        PyGILState_STATE;
+
+/* Ensure that the current thread is ready to call the Python
+   C API, regardless of the current state of Python, or of its
+   thread lock.  This may be called as many times as desired
+   by a thread so long as each call is matched with a call to 
+   PyGILState_Release().  In general, other thread-state APIs may 
+   be used between _Ensure() and _Release() calls, so long as the 
+   thread-state is restored to its previous state before the Release().
+   For example, normal use of the Py_BEGIN_ALLOW_THREADS/
+   Py_END_ALLOW_THREADS macros are acceptable.
+
+   The return value is an opaque "handle" to the thread state when
+   PyGILState_Acquire() was called, and must be passed to
+   PyGILState_Release() to ensure Python is left in the same state. Even
+   though recursive calls are allowed, these handles can *not* be shared - 
+   each unique call to PyGILState_Ensure must save the handle for its 
+   call to PyGILState_Release.
+
+   When the function returns, the current thread will hold the GIL.
+
+   Failure is a fatal error.
+*/
+PyAPI_FUNC(PyGILState_STATE) PyGILState_Ensure(void);
+
+/* Release any resources previously acquired.  After this call, Python's
+   state will be the same as it was prior to the corresponding
+   PyGILState_Acquire call (but generally this state will be unknown to 
+   the caller, hence the use of the GILState API.)
+
+   Every call to PyGILState_Ensure must be matched by a call to 
+   PyGILState_Release on the same thread.
+*/
+PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE);
+
+/* Helper/diagnostic function - get the current thread state for
+   this thread.  May return NULL if no GILState API has been used 
+   on the current thread.  Note the main thread always has such a 
+   thread-state, even if no auto-thread-state call has been made 
+   on the main thread.
+*/
+PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void);
+
 /* Routines for advanced debuggers, requested by David Beazley.
    Don't use unless you know what you are doing! */
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Head(void);

Include/pythread.h

@@ -30,10 +30,12 @@ PyAPI_FUNC(void) PyThread_exit_prog(int);
 PyAPI_FUNC(void) PyThread__PyThread_exit_prog(int);
 #endif
 
+/* Thread Local Storage (TLS) API */
 PyAPI_FUNC(int) PyThread_create_key(void);
 PyAPI_FUNC(void) PyThread_delete_key(int);
 PyAPI_FUNC(int) PyThread_set_key_value(int, void *);
 PyAPI_FUNC(void *) PyThread_get_key_value(int);
+PyAPI_FUNC(void) PyThread_delete_key_value(int key);
 
 #ifdef __cplusplus
 }

Lib/test/test_capi.py

@@ -14,3 +14,32 @@ for name in dir(_testcapi):
             test()
         except _testcapi.error:
             raise test_support.TestFailed, sys.exc_info()[1]
+
+# some extra thread-state tests driven via _testcapi
+def TestThreadState():
+    import thread
+    import time
+
+    if test_support.verbose:
+        print "auto-thread-state"
+
+    idents = []
+
+    def callback():
+        idents.append(thread.get_ident())
+    
+    _testcapi._test_thread_state(callback)
+    time.sleep(1)
+    # Check our main thread is in the list exactly 3 times.
+    if idents.count(thread.get_ident()) != 3:
+        raise test_support.TestFailed, \
+              "Couldn't find main thread correctly in the list"
+
+try:
+    _testcapi._test_thread_state
+    have_thread_state = True
+except AttributeError:
+    have_thread_state = False
+    
+if have_thread_state:
+    TestThreadState()

Modules/_testcapimodule.c

@@ -7,6 +7,10 @@
 
 #include "Python.h"
 
+#ifdef WITH_THREAD
+#include "pythread.h"
+#endif /* WITH_THREAD */
+
 static PyObject *TestError;	/* set to exception object in init */
 
 /* Raise TestError with test_name + ": " + msg, and return NULL. */
@@ -535,6 +539,46 @@ raise_exception(PyObject *self, PyObject *args)
 	return NULL;
 }
 
+#ifdef WITH_THREAD
+
+void _make_call(void *callable)
+{
+	PyObject *rc;
+	PyGILState_STATE s = PyGILState_Ensure();
+	rc = PyObject_CallFunction(callable, "");
+	Py_XDECREF(rc);
+	PyGILState_Release(s);
+}
+
+static PyObject *
+test_thread_state(PyObject *self, PyObject *args)
+{
+	PyObject *fn;
+	if (!PyArg_ParseTuple(args, "O:test_thread_state", &fn))
+		return NULL;
+	/* Ensure Python is setup for threading */
+	PyEval_InitThreads();
+	/* Start a new thread for our callback. */
+	PyThread_start_new_thread( _make_call, fn);
+	/* Make the callback with the thread lock held by this thread */
+	_make_call(fn);
+	/* Do it all again, but this time with the thread-lock released */
+	Py_BEGIN_ALLOW_THREADS
+	_make_call(fn);
+	Py_END_ALLOW_THREADS
+	/* And once more with and without a thread
+	   XXX - should use a lock and work out exactly what we are trying 
+	   to test <wink> 
+	*/
+	Py_BEGIN_ALLOW_THREADS
+	PyThread_start_new_thread( _make_call, fn);
+	_make_call(fn);
+	Py_END_ALLOW_THREADS
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+#endif
+
 static PyMethodDef TestMethods[] = {
 	{"raise_exception",	raise_exception,		 METH_VARARGS},
 	{"test_config",		(PyCFunction)test_config,	 METH_NOARGS},
@@ -553,6 +597,9 @@ static PyMethodDef TestMethods[] = {
 #endif
 #ifdef Py_USING_UNICODE
 	{"test_u_code",		(PyCFunction)test_u_code,	 METH_NOARGS},
+#endif
+#ifdef WITH_THREAD
+	{"_test_thread_state", (PyCFunction)test_thread_state, METH_VARARGS},
 #endif
 	{NULL, NULL} /* sentinel */
 };

Modules/posixmodule.c

@@ -4354,22 +4354,11 @@ _PyPopen(char *cmdstring, int mode, int n)
  * exit code as the result of the close() operation.  This permits the
  * files to be closed in any order - it is always the close() of the
  * final handle that will return the exit code.
+ *
+ * NOTE: This function is currently called with the GIL released.
+ * hence we use the GILState API to manage our state.
  */
 
- /* RED_FLAG 31-Aug-2000 Tim
-  * This is always called (today!) between a pair of
-  * Py_BEGIN_ALLOW_THREADS/ Py_END_ALLOW_THREADS
-  * macros.  So the thread running this has no valid thread state, as
-  * far as Python is concerned.  However, this calls some Python API
-  * functions that cannot be called safely without a valid thread
-  * state, in particular PyDict_GetItem.
-  * As a temporary hack (although it may last for years ...), we
-  * *rely* on not having a valid thread state in this function, in
-  * order to create our own "from scratch".
-  * This will deadlock if _PyPclose is ever called by a thread
-  * holding the global lock.
-  */
-
 static int _PyPclose(FILE *file)
 {
 	int result;
@@ -4378,40 +4367,16 @@ static int _PyPclose(FILE *file)
 	PyObject *procObj, *hProcessObj, *intObj, *fileObj;
 	long file_count;
 #ifdef WITH_THREAD
-	PyInterpreterState* pInterpreterState;
-	PyThreadState* pThreadState;
+	PyGILState_STATE state;
 #endif
 
 	/* Close the file handle first, to ensure it can't block the
 	 * child from exiting if it's the last handle.
 	 */
 	result = fclose(file);
-
 #ifdef WITH_THREAD
-	/* Bootstrap a valid thread state into existence. */
-	pInterpreterState = PyInterpreterState_New();
-	if (!pInterpreterState) {
-		/* Well, we're hosed now!  We don't have a thread
-		 * state, so can't call a nice error routine, or raise
-		 * an exception.  Just die.
-		 */
-		 Py_FatalError("unable to allocate interpreter state "
-		 	       "when closing popen object");
-		 return -1;  /* unreachable */
-	}
-	pThreadState = PyThreadState_New(pInterpreterState);
-	if (!pThreadState) {
-		 Py_FatalError("unable to allocate thread state "
-		 	       "when closing popen object");
-		 return -1;  /* unreachable */
-	}
-	/* Grab the global lock.  Note that this will deadlock if the
-	 * current thread already has the lock! (see RED_FLAG comments
-	 * before this function)
-	 */
-	PyEval_RestoreThread(pThreadState);
+	state = PyGILState_Ensure();
 #endif
-
 	if (_PyPopenProcs) {
 		if ((fileObj = PyLong_FromVoidPtr(file)) != NULL &&
 		    (procObj = PyDict_GetItem(_PyPopenProcs,
@@ -4470,17 +4435,8 @@ static int _PyPclose(FILE *file)
 	} /* if _PyPopenProcs */
 
 #ifdef WITH_THREAD
-	/* Tear down the thread & interpreter states.
-	 * Note that interpreter state clear & delete functions automatically
-	 * call the thread clear & delete functions, and indeed insist on
-	 * doing that themselves.  The lock must be held during the clear, but
-	 * need not be held during the delete.
-	 */
-	PyInterpreterState_Clear(pInterpreterState);
-	PyEval_ReleaseThread(pThreadState);
-	PyInterpreterState_Delete(pInterpreterState);
+	PyGILState_Release(state);
 #endif
-
 	return result;
 }
 

Python/ceval.c

@@ -321,6 +321,8 @@ PyEval_AcquireThread(PyThreadState *tstate)
 {
 	if (tstate == NULL)
 		Py_FatalError("PyEval_AcquireThread: NULL new thread state");
+	/* Check someone has called PyEval_InitThreads() to create the lock */
+	assert(interpreter_lock);
 	PyThread_acquire_lock(interpreter_lock, 1);
 	if (PyThreadState_Swap(tstate) != NULL)
 		Py_FatalError(

Python/pystate.c

@@ -142,6 +142,7 @@ PyThreadState_New(PyInterpreterState *interp)
 		tstate->tracing = 0;
 		tstate->use_tracing = 0;
 		tstate->tick_counter = 0;
+		tstate->gilstate_counter = 0;
 
 		tstate->dict = NULL;
 
@@ -259,7 +260,17 @@ PyThreadState_Swap(PyThreadState *new)
 	PyThreadState *old = _PyThreadState_Current;
 
 	_PyThreadState_Current = new;
-
+	/* It should not be possible for more than one thread state
+	   to be used for a thread.  Check this the best we can in debug 
+	   builds.
+	*/
+#if defined(Py_DEBUG)
+	if (new) {
+		PyThreadState *check = PyGILState_GetThisThreadState();
+		if (check && check != new)
+			Py_FatalError("Invalid thread state for this thread");
+	}
+#endif
 	return old;
 }
 
@@ -308,3 +319,131 @@ PyThreadState *
 PyThreadState_Next(PyThreadState *tstate) {
 	return tstate->next;
 }
+
+/* Python "auto thread state" API. */
+#ifdef WITH_THREAD
+
+/* Keep this as a static, as it is not reliable!  It can only
+   ever be compared to the state for the *current* thread.
+   * If not equal, then it doesn't matter that the actual
+     value may change immediately after comparison, as it can't
+     possibly change to the current thread's state.
+   * If equal, then the current thread holds the lock, so the value can't
+     change until we yield the lock.
+*/
+static int
+PyThreadState_IsCurrent(PyThreadState *tstate)
+{
+	/* Must be the tstate for this thread */
+	assert(PyGILState_GetThisThreadState()==tstate);
+	/* On Windows at least, simple reads and writes to 32 bit values
+	   are atomic.
+	*/
+	return tstate == _PyThreadState_Current;
+}
+
+/* The single PyInterpreterState used by this process'
+   GILState implementation
+*/
+static PyInterpreterState *autoInterpreterState = NULL;
+static int autoTLSkey = 0;
+
+/* Internal initialization/finalization functions called by 
+   Py_Initialize/Py_Finalize 
+*/
+void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t)
+{
+	assert(i && t); /* must init with a valid states */
+	autoTLSkey = PyThread_create_key();
+	autoInterpreterState = i;
+	/* Now stash the thread state for this thread in TLS */
+	PyThread_set_key_value(autoTLSkey, (void *)t);
+	assert(t->gilstate_counter==0); /* must be a new thread state */
+	t->gilstate_counter = 1;
+}
+
+void _PyGILState_Fini(void)
+{
+	PyThread_delete_key(autoTLSkey);
+	autoTLSkey = 0;
+	autoInterpreterState = NULL;;
+}
+
+/* The public functions */
+PyThreadState *PyGILState_GetThisThreadState(void)
+{
+	if (autoInterpreterState==NULL || autoTLSkey==0)
+		return NULL;
+	return (PyThreadState *) PyThread_get_key_value(autoTLSkey);
+}
+
+PyGILState_STATE PyGILState_Ensure(void)
+{
+	int current;
+	PyThreadState *tcur;
+	/* Note that we do not auto-init Python here - apart from 
+	   potential races with 2 threads auto-initializing, pep-311 
+	   spells out other issues.  Embedders are expected to have
+	   called Py_Initialize() and usually PyEval_InitThreads().
+	*/
+	assert(autoInterpreterState); /* Py_Initialize() hasn't been called! */
+	tcur = PyThread_get_key_value(autoTLSkey);
+	if (tcur==NULL) {
+		/* Create a new thread state for this thread */
+		tcur = PyThreadState_New(autoInterpreterState);
+		if (tcur==NULL)
+			Py_FatalError("Couldn't create thread-state for new thread");
+		PyThread_set_key_value(autoTLSkey, (void *)tcur);
+		current = 0; /* new thread state is never current */
+	} else
+		current = PyThreadState_IsCurrent(tcur);
+	if (!current)
+		PyEval_RestoreThread(tcur);
+	/* Update our counter in the thread-state - no need for locks:
+	   - tcur will remain valid as we hold the GIL.
+	   - the counter is safe as we are the only thread "allowed" 
+	     to modify this value
+	*/
+	tcur->gilstate_counter++;
+	return current ? PyGILState_LOCKED : PyGILState_UNLOCKED;
+}
+
+void PyGILState_Release(PyGILState_STATE oldstate)
+{
+	PyThreadState *tcur = PyThread_get_key_value(autoTLSkey);
+	if (tcur==NULL)
+		Py_FatalError("auto-releasing thread-state, "
+		              "but no thread-state for this thread");
+	/* We must hold the GIL and have our thread state current */
+	/* XXX - remove the check - the assert should be fine,
+	   but while this is very new (April 2003), the extra check 
+	   by release-only users can't hurt.
+	*/
+	if (!PyThreadState_IsCurrent(tcur))
+		Py_FatalError("This thread state must be current when releasing");
+	assert (PyThreadState_IsCurrent(tcur));
+	tcur->gilstate_counter -= 1;
+	assert (tcur->gilstate_counter >= 0); /* illegal counter value */
+
+	/* If we are about to destroy this thread-state, we must 
+	   clear it while the lock is held, as destructors may run
+	*/
+	if (tcur->gilstate_counter==0) {
+		/* can't have been locked when we created it */
+		assert(oldstate==PyGILState_UNLOCKED);
+		PyThreadState_Clear(tcur);
+	}
+
+	/* Release the lock if necessary */
+	if (oldstate==PyGILState_UNLOCKED)
+		PyEval_ReleaseThread(tcur);
+
+	/* Now complete destruction of the thread if necessary */
+	if (tcur->gilstate_counter==0) {
+		/* Delete this thread from our TLS */
+		PyThread_delete_key_value(autoTLSkey);
+		/* Delete the thread-state */
+		PyThreadState_Delete(tcur);
+	}
+}
+#endif /* WITH_THREAD */

Python/pythonrun.c

@@ -50,6 +50,11 @@ static void call_ll_exitfuncs(void);
 extern void _PyUnicode_Init(void);
 extern void _PyUnicode_Fini(void);
 
+#ifdef WITH_THREAD
+extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *);
+extern void _PyGILState_Fini(void);
+#endif /* WITH_THREAD */
+
 int Py_DebugFlag; /* Needed by parser.c */
 int Py_VerboseFlag; /* Needed by import.c */
 int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */
@@ -180,6 +185,11 @@ Py_Initialize(void)
 	if (!Py_NoSiteFlag)
 		initsite(); /* Module site */
 
+	/* auto-thread-state API, if available */
+#ifdef WITH_THREAD
+	_PyGILState_Init(interp, tstate);
+#endif /* WITH_THREAD */
+
 	PyModule_WarningsModule = PyImport_ImportModule("warnings");
 
 #if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET)
@@ -244,6 +254,11 @@ Py_Finalize(void)
 	call_sys_exitfunc();
 	initialized = 0;
 
+	/* Cleanup auto-thread-state */
+#ifdef WITH_THREAD
+	_PyGILState_Fini();
+#endif /* WITH_THREAD */
+
 	/* Get current thread state and interpreter pointer */
 	tstate = PyThreadState_Get();
 	interp = tstate->interp;

Python/thread.c

@@ -137,3 +137,111 @@ void PyThread_init_thread(void)
 #include "thread_foobar.h"
 #endif
 */
+
+#ifndef Py_HAVE_NATIVE_TLS
+/* If the platform has not supplied a platform specific
+   TLS implementation, provide our own.
+
+   This code stolen from "thread_sgi.h", where it was the only
+   implementation of an existing Python TLS API.
+*/
+/*
+ * Per-thread data ("key") support.
+ */
+
+struct key {
+	struct key *next;
+	long id;
+	int key;
+	void *value;
+};
+
+static struct key *keyhead = NULL;
+static int nkeys = 0;
+static PyThread_type_lock keymutex = NULL;
+
+static struct key *find_key(int key, void *value)
+{
+	struct key *p;
+	long id = PyThread_get_thread_ident();
+	for (p = keyhead; p != NULL; p = p->next) {
+		if (p->id == id && p->key == key)
+			return p;
+	}
+	if (value == NULL)
+		return NULL;
+	p = (struct key *)malloc(sizeof(struct key));
+	if (p != NULL) {
+		p->id = id;
+		p->key = key;
+		p->value = value;
+		PyThread_acquire_lock(keymutex, 1);
+		p->next = keyhead;
+		keyhead = p;
+		PyThread_release_lock(keymutex);
+	}
+	return p;
+}
+
+int PyThread_create_key(void)
+{
+	if (keymutex == NULL)
+		keymutex = PyThread_allocate_lock();
+	return ++nkeys;
+}
+
+void PyThread_delete_key(int key)
+{
+	struct key *p, **q;
+	PyThread_acquire_lock(keymutex, 1);
+	q = &keyhead;
+	while ((p = *q) != NULL) {
+		if (p->key == key) {
+			*q = p->next;
+			free((void *)p);
+			/* NB This does *not* free p->value! */
+		}
+		else
+			q = &p->next;
+	}
+	PyThread_release_lock(keymutex);
+}
+
+int PyThread_set_key_value(int key, void *value)
+{
+	struct key *p = find_key(key, value);
+	if (p == NULL)
+		return -1;
+	else
+		return 0;
+}
+
+void *PyThread_get_key_value(int key)
+{
+	struct key *p = find_key(key, NULL);
+	if (p == NULL)
+		return NULL;
+	else
+		return p->value;
+}
+
+void PyThread_delete_key_value(int key)
+{
+	long id = PyThread_get_thread_ident();
+	struct key *p, **q;
+	PyThread_acquire_lock(keymutex, 1);
+	q = &keyhead;
+	while ((p = *q) != NULL) {
+		if (p->key == key && p->id == id) {
+			*q = p->next;
+			free((void *)p);
+			/* NB This does *not* free p->value! */
+			break;
+		}
+		else
+			q = &p->next;
+	}
+	PyThread_release_lock(keymutex);
+}
+
+#endif /* Py_HAVE_NATIVE_TLS */

Python/thread_sgi.h

@@ -377,83 +377,3 @@ void PyThread_release_lock(PyThread_type_lock lock)
 	if (usunsetlock((ulock_t) lock) < 0)
 		perror("usunsetlock");
 }
-
-/*
- * Per-thread data ("key") support.
- */
-
-struct key {
-	struct key *next;
-	long id;
-	int key;
-	void *value;
-};
-
-static struct key *keyhead = NULL;
-static int nkeys = 0;
-static PyThread_type_lock keymutex = NULL;
-
-static struct key *find_key(int key, void *value)
-{
-	struct key *p;
-	long id = PyThread_get_thread_ident();
-	for (p = keyhead; p != NULL; p = p->next) {
-		if (p->id == id && p->key == key)
-			return p;
-	}
-	if (value == NULL)
-		return NULL;
-	p = (struct key *)malloc(sizeof(struct key));
-	if (p != NULL) {
-		p->id = id;
-		p->key = key;
-		p->value = value;
-		PyThread_acquire_lock(keymutex, 1);
-		p->next = keyhead;
-		keyhead = p;
-		PyThread_release_lock(keymutex);
-	}
-	return p;
-}
-
-int PyThread_create_key(void)
-{
-	if (keymutex == NULL)
-		keymutex = PyThread_allocate_lock();
-	return ++nkeys;
-}
-
-void PyThread_delete_key(int key)
-{
-	struct key *p, **q;
-	PyThread_acquire_lock(keymutex, 1);
-	q = &keyhead;
-	while ((p = *q) != NULL) {
-		if (p->key == key) {
-			*q = p->next;
-			free((void *)p);
-			/* NB This does *not* free p->value! */
-		}
-		else
-			q = &p->next;
-	}
-	PyThread_release_lock(keymutex);
-}
-
-int PyThread_set_key_value(int key, void *value)
-{
-	struct key *p = find_key(key, value);
-	if (p == NULL)
-		return -1;
-	else
-		return 0;
-}
-
-void *PyThread_get_key_value(int key)
-{
-	struct key *p = find_key(key, NULL);
-	if (p == NULL)
-		return NULL;
-	else
-		return p->value;
-}