bpo-36356: Destroy the GIL at exit (GH-12453)

* Add _PyEval_FiniThreads2(). _PyEval_FiniThreads() now only clears
  the pending lock, whereas _PyEval_FiniThreads2() destroys the GIL.
* pymain_free() now calls _PyEval_FiniThreads2().
* Py_FinalizeEx() now calls _PyEval_FiniThreads().

Include/ceval.h

@@ -192,9 +192,6 @@ PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *);
 
 PyAPI_FUNC(int)  PyEval_ThreadsInitialized(void);
 PyAPI_FUNC(void) PyEval_InitThreads(void);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _PyEval_FiniThreads(void);
-#endif /* !Py_LIMITED_API */
 PyAPI_FUNC(void) PyEval_AcquireLock(void) Py_DEPRECATED(3.2);
 PyAPI_FUNC(void) PyEval_ReleaseLock(void) /* Py_DEPRECATED(3.2) */;
 PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate);

Include/internal/pycore_ceval.h

@@ -54,6 +54,9 @@ struct _ceval_runtime_state {
 
 PyAPI_FUNC(void) _PyEval_Initialize(struct _ceval_runtime_state *);
 
+PyAPI_FUNC(void) _PyEval_FiniThreads(void);
+PyAPI_FUNC(void) _PyEval_FiniThreads2(void);
+
 #ifdef __cplusplus
 }
 #endif

Modules/main.c

 /* Python interpreter main program */
 
 #include "Python.h"
+#include "pycore_ceval.h"   /* _PyEval_FiniThreads2() */
 #include "pycore_coreconfig.h"
 #include "pycore_pylifecycle.h"
 #include "pycore_pymem.h"
@@ -525,15 +526,15 @@ done:
 
 /* --- pymain_main() ---------------------------------------------- */
 
+/* Free global variables which cannot be freed in Py_Finalize():
+   configuration options set before Py_Initialize() which should
+   remain valid after Py_Finalize(), since
+   Py_Initialize()-Py_Finalize() can be called multiple times. */
 static void
 pymain_free(void)
 {
     _PyImport_Fini2();
-
-    /* Free global variables which cannot be freed in Py_Finalize():
-       configuration options set before Py_Initialize() which should
-       remain valid after Py_Finalize(), since
-       Py_Initialize()-Py_Finalize() can be called multiple times. */
+    _PyEval_FiniThreads2();
     _PyPathConfig_ClearGlobal();
     _Py_ClearStandardStreamEncoding();
     _Py_ClearArgcArgv();

Python/ceval.c

@@ -188,8 +188,19 @@ PyEval_InitThreads(void)
     }
 }
 
+
 void
 _PyEval_FiniThreads(void)
+{
+    if (_PyRuntime.ceval.pending.lock != NULL) {
+        PyThread_free_lock(_PyRuntime.ceval.pending.lock);
+        _PyRuntime.ceval.pending.lock = NULL;
+    }
+}
+
+
+void
+_PyEval_FiniThreads2(void)
 {
     if (!gil_created()) {
         return;
@@ -197,11 +208,6 @@ _PyEval_FiniThreads(void)
 
     destroy_gil();
     assert(!gil_created());
-
-    if (_PyRuntime.ceval.pending.lock != NULL) {
-        PyThread_free_lock(_PyRuntime.ceval.pending.lock);
-        _PyRuntime.ceval.pending.lock = NULL;
-    }
 }
 
 static inline void

Python/pylifecycle.c

@@ -4,8 +4,9 @@
 
 #include "Python-ast.h"
 #undef Yield   /* undefine macro conflicting with <winbase.h> */
-#include "pycore_coreconfig.h"
+#include "pycore_ceval.h"   /* _PyEval_FiniThreads() */
 #include "pycore_context.h"
+#include "pycore_coreconfig.h"
 #include "pycore_fileutils.h"
 #include "pycore_hamt.h"
 #include "pycore_pathconfig.h"
@@ -555,12 +556,11 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
         return _Py_INIT_ERR("can't make first thread");
     (void) PyThreadState_Swap(tstate);
 
-    /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because
-       destroying the GIL might fail when it is being referenced from
-       another running thread (see issue #9901).
+    /* Destroying the GIL in Py_FinalizeEx might fail when it is being
+       referenced from another running thread (see bpo-9901).
        Instead we destroy the previously created GIL here, which ensures
        that we can call Py_Initialize / Py_FinalizeEx multiple times. */
-    _PyEval_FiniThreads();
+    _PyEval_FiniThreads2();
 
     /* Auto-thread-state API */
     _PyGILState_Init(runtime, interp, tstate);
@@ -1357,6 +1357,7 @@ Py_FinalizeEx(void)
 
     call_ll_exitfuncs(runtime);
 
+    _PyEval_FiniThreads();
     _PyRuntime_Finalize();
     return status;
 }