bpo-36737: Use the module state C-API for warnings. (gh-13159)

Include/internal/pycore_pylifecycle.h

@@ -81,7 +81,7 @@ extern void PyLong_Fini(void);
 extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
 extern int _PyTraceMalloc_Fini(void);
-extern void _PyWarnings_Fini(_PyRuntimeState *runtime);
+extern void _PyWarnings_Fini(PyInterpreterState *interp);
 
 extern void _PyGILState_Init(
     _PyRuntimeState *runtime,

Include/internal/pycore_pystate.h

@@ -90,6 +90,8 @@ struct _is {
     PyObject *pyexitmodule;
 
     uint64_t tstate_next_unique_id;
+
+    struct _warnings_runtime_state warnings;
 };
 
 PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T);
@@ -179,7 +181,6 @@ typedef struct pyruntimestate {
     int nexitfuncs;
 
     struct _gc_runtime_state gc;
-    struct _warnings_runtime_state warnings;
     struct _ceval_runtime_state ceval;
     struct _gilstate_runtime_state gilstate;
 

Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst

+Move PyRuntimeState.warnings into per-interpreter state (via "module
+state").

Python/_warnings.c

@@ -15,6 +15,134 @@ _Py_IDENTIFIER(default);
 _Py_IDENTIFIER(ignore);
 #endif
 
+
+/*************************************************************************/
+
+typedef struct _warnings_runtime_state WarningsState;
+
+/* Forward declaration of the _warnings module definition. */
+static struct PyModuleDef warningsmodule;
+
+/* Given a module object, get its per-module state. */
+static WarningsState *
+_Warnings_GetState()
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    if (tstate == NULL) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "_Warnings_GetState: could not identify current interpreter");
+        return NULL;
+    }
+    return &tstate->interp->warnings;
+}
+
+/* Clear the given warnings module state. */
+static void
+_Warnings_ClearState(WarningsState *st)
+{
+    Py_CLEAR(st->filters);
+    Py_CLEAR(st->once_registry);
+    Py_CLEAR(st->default_action);
+}
+
+#ifndef Py_DEBUG
+static PyObject *
+create_filter(PyObject *category, _Py_Identifier *id, const char *modname)
+{
+    PyObject *modname_obj = NULL;
+    PyObject *action_str = _PyUnicode_FromId(id);
+    if (action_str == NULL) {
+        return NULL;
+    }
+
+    /* Default to "no module name" for initial filter set */
+    if (modname != NULL) {
+        modname_obj = PyUnicode_InternFromString(modname);
+        if (modname_obj == NULL) {
+            return NULL;
+        }
+    } else {
+        modname_obj = Py_None;
+    }
+
+    /* This assumes the line number is zero for now. */
+    return PyTuple_Pack(5, action_str, Py_None,
+                        category, modname_obj, _PyLong_Zero);
+}
+#endif
+
+static PyObject *
+init_filters(void)
+{
+#ifdef Py_DEBUG
+    /* Py_DEBUG builds show all warnings by default */
+    return PyList_New(0);
+#else
+    /* Other builds ignore a number of warning categories by default */
+    PyObject *filters = PyList_New(5);
+    if (filters == NULL) {
+        return NULL;
+    }
+
+    size_t pos = 0;  /* Post-incremented in each use. */
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__"));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_ImportWarning, &PyId_ignore, NULL));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL));
+
+    for (size_t x = 0; x < pos; x++) {
+        if (PyList_GET_ITEM(filters, x) == NULL) {
+            Py_DECREF(filters);
+            return NULL;
+        }
+    }
+    return filters;
+#endif
+}
+
+/* Initialize the given warnings module state. */
+static int
+_Warnings_InitState(WarningsState *st)
+{
+    if (st->filters == NULL) {
+        st->filters = init_filters();
+        if (st->filters == NULL) {
+            goto error;
+        }
+    }
+
+    if (st->once_registry == NULL) {
+        st->once_registry = PyDict_New();
+        if (st->once_registry == NULL) {
+            goto error;
+        }
+    }
+
+    if (st->default_action == NULL) {
+        st->default_action = PyUnicode_FromString("default");
+        if (st->default_action == NULL) {
+            goto error;
+        }
+    }
+
+    st->filters_version = 0;
+
+    return 0;
+
+error:
+    _Warnings_ClearState(st);
+    return -1;
+}
+
+
+/*************************************************************************/
+
 static int
 check_matched(PyObject *obj, PyObject *arg)
 {
@@ -93,7 +221,7 @@ get_warnings_attr(_Py_Identifier *attr_id, int try_import)
 
 
 static PyObject *
-get_once_registry(void)
+get_once_registry(WarningsState *st)
 {
     PyObject *registry;
     _Py_IDENTIFIER(onceregistry);
@@ -102,8 +230,8 @@ get_once_registry(void)
     if (registry == NULL) {
         if (PyErr_Occurred())
             return NULL;
-        assert(_PyRuntime.warnings.once_registry);
-        return _PyRuntime.warnings.once_registry;
+        assert(st->once_registry);
+        return st->once_registry;
     }
     if (!PyDict_Check(registry)) {
         PyErr_Format(PyExc_TypeError,
@@ -113,13 +241,13 @@ get_once_registry(void)
         Py_DECREF(registry);
         return NULL;
     }
-    Py_SETREF(_PyRuntime.warnings.once_registry, registry);
+    Py_SETREF(st->once_registry, registry);
     return registry;
 }
 
 
 static PyObject *
-get_default_action(void)
+get_default_action(WarningsState *st)
 {
     PyObject *default_action;
     _Py_IDENTIFIER(defaultaction);
@@ -129,8 +257,8 @@ get_default_action(void)
         if (PyErr_Occurred()) {
             return NULL;
         }
-        assert(_PyRuntime.warnings.default_action);
-        return _PyRuntime.warnings.default_action;
+        assert(st->default_action);
+        return st->default_action;
     }
     if (!PyUnicode_Check(default_action)) {
         PyErr_Format(PyExc_TypeError,
@@ -140,7 +268,7 @@ get_default_action(void)
         Py_DECREF(default_action);
         return NULL;
     }
-    Py_SETREF(_PyRuntime.warnings.default_action, default_action);
+    Py_SETREF(st->default_action, default_action);
     return default_action;
 }
 
@@ -154,6 +282,10 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno,
     Py_ssize_t i;
     PyObject *warnings_filters;
     _Py_IDENTIFIER(filters);
+    WarningsState *st = _Warnings_GetState();
+    if (st == NULL) {
+        return NULL;
+    }
 
     warnings_filters = get_warnings_attr(&PyId_filters, 0);
     if (warnings_filters == NULL) {
@@ -161,17 +293,17 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno,
             return NULL;
     }
     else {
-        Py_SETREF(_PyRuntime.warnings.filters, warnings_filters);
+        Py_SETREF(st->filters, warnings_filters);
     }
 
-    PyObject *filters = _PyRuntime.warnings.filters;
+    PyObject *filters = st->filters;
     if (filters == NULL || !PyList_Check(filters)) {
         PyErr_SetString(PyExc_ValueError,
                         MODULE_NAME ".filters must be a list");
         return NULL;
     }
 
-    /* _PyRuntime.warnings.filters could change while we are iterating over it. */
+    /* WarningsState.filters could change while we are iterating over it. */
     for (i = 0; i < PyList_GET_SIZE(filters); i++) {
         PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj;
         Py_ssize_t ln;
@@ -232,7 +364,7 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno,
         Py_DECREF(tmp_item);
     }
 
-    action = get_default_action();
+    action = get_default_action(st);
     if (action != NULL) {
         Py_INCREF(Py_None);
         *item = Py_None;
@@ -252,16 +384,20 @@ already_warned(PyObject *registry, PyObject *key, int should_set)
     if (key == NULL)
         return -1;
 
+    WarningsState *st = _Warnings_GetState();
+    if (st == NULL) {
+        return -1;
+    }
     version_obj = _PyDict_GetItemIdWithError(registry, &PyId_version);
     if (version_obj == NULL
         || !PyLong_CheckExact(version_obj)
-        || PyLong_AsLong(version_obj) != _PyRuntime.warnings.filters_version)
+        || PyLong_AsLong(version_obj) != st->filters_version)
     {
         if (PyErr_Occurred()) {
             return -1;
         }
         PyDict_Clear(registry);
-        version_obj = PyLong_FromLong(_PyRuntime.warnings.filters_version);
+        version_obj = PyLong_FromLong(st->filters_version);
         if (version_obj == NULL)
             return -1;
         if (_PyDict_SetItemId(registry, &PyId_version, version_obj) < 0) {
@@ -567,11 +703,15 @@ warn_explicit(PyObject *category, PyObject *message,
 
         if (_PyUnicode_EqualToASCIIString(action, "once")) {
             if (registry == NULL || registry == Py_None) {
-                registry = get_once_registry();
+                WarningsState *st = _Warnings_GetState();
+                if (st == NULL) {
+                    goto cleanup;
+                }
+                registry = get_once_registry(st);
                 if (registry == NULL)
                     goto cleanup;
             }
-            /* _PyRuntime.warnings.once_registry[(text, category)] = 1 */
+            /* WarningsState.once_registry[(text, category)] = 1 */
             rc = update_registry(registry, text, category, 0);
         }
         else if (_PyUnicode_EqualToASCIIString(action, "module")) {
@@ -925,7 +1065,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds)
 static PyObject *
 warnings_filters_mutated(PyObject *self, PyObject *args)
 {
-    _PyRuntime.warnings.filters_version++;
+    WarningsState *st = _Warnings_GetState();
+    if (st == NULL) {
+        return NULL;
+    }
+    st->filters_version++;
     Py_RETURN_NONE;
 }
 
@@ -1175,78 +1319,16 @@ static PyMethodDef warnings_functions[] = {
 };
 
 
-#ifndef Py_DEBUG
-static PyObject *
-create_filter(PyObject *category, _Py_Identifier *id, const char *modname)
-{
-    PyObject *modname_obj = NULL;
-    PyObject *action_str = _PyUnicode_FromId(id);
-    if (action_str == NULL) {
-        return NULL;
-    }
-
-    /* Default to "no module name" for initial filter set */
-    if (modname != NULL) {
-        modname_obj = PyUnicode_InternFromString(modname);
-        if (modname_obj == NULL) {
-            return NULL;
-        }
-    } else {
-        modname_obj = Py_None;
-    }
-
-    /* This assumes the line number is zero for now. */
-    return PyTuple_Pack(5, action_str, Py_None,
-                        category, modname_obj, _PyLong_Zero);
-}
-#endif
-
-
-static PyObject *
-init_filters(void)
-{
-#ifdef Py_DEBUG
-    /* Py_DEBUG builds show all warnings by default */
-    return PyList_New(0);
-#else
-    /* Other builds ignore a number of warning categories by default */
-    PyObject *filters = PyList_New(5);
-    if (filters == NULL) {
-        return NULL;
-    }
-
-    size_t pos = 0;  /* Post-incremented in each use. */
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__"));
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL));
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL));
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_ImportWarning, &PyId_ignore, NULL));
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL));
-
-    for (size_t x = 0; x < pos; x++) {
-        if (PyList_GET_ITEM(filters, x) == NULL) {
-            Py_DECREF(filters);
-            return NULL;
-        }
-    }
-    return filters;
-#endif
-}
-
 static struct PyModuleDef warningsmodule = {
         PyModuleDef_HEAD_INIT,
-        MODULE_NAME,
-        warnings__doc__,
-        0,
-        warnings_functions,
-        NULL,
-        NULL,
-        NULL,
-        NULL
+        MODULE_NAME,            /* m_name */
+        warnings__doc__,        /* m_doc */
+        0,                      /* m_size */
+        warnings_functions,     /* m_methods */
+        NULL,                   /* m_reload */
+        NULL,                   /* m_traverse */
+        NULL,                   /* m_clear */
+        NULL                    /* m_free */
 };
 
 
@@ -1256,49 +1338,46 @@ _PyWarnings_Init(void)
     PyObject *m;
 
     m = PyModule_Create(&warningsmodule);
-    if (m == NULL)
+    if (m == NULL) {
         return NULL;
+    }
 
-    struct _warnings_runtime_state *state = &_PyRuntime.warnings;
-    if (state->filters == NULL) {
-        state->filters = init_filters();
-        if (state->filters == NULL)
-            return NULL;
+    WarningsState *st = _Warnings_GetState();
+    if (st == NULL) {
+        goto error;
+    }
+    if (_Warnings_InitState(st) < 0) {
+        goto error;
     }
-    Py_INCREF(state->filters);
-    if (PyModule_AddObject(m, "filters", state->filters) < 0)
-        return NULL;
 
-    if (state->once_registry == NULL) {
-        state->once_registry = PyDict_New();
-        if (state->once_registry == NULL)
-            return NULL;
+    Py_INCREF(st->filters);
+    if (PyModule_AddObject(m, "filters", st->filters) < 0) {
+        goto error;
     }
-    Py_INCREF(state->once_registry);
-    if (PyModule_AddObject(m, "_onceregistry",
-                           state->once_registry) < 0)
-        return NULL;
 
-    if (state->default_action == NULL) {
-        state->default_action = PyUnicode_FromString("default");
-        if (state->default_action == NULL)
-            return NULL;
+    Py_INCREF(st->once_registry);
+    if (PyModule_AddObject(m, "_onceregistry", st->once_registry) < 0) {
+        goto error;
+    }
+
+    Py_INCREF(st->default_action);
+    if (PyModule_AddObject(m, "_defaultaction", st->default_action) < 0) {
+        goto error;
     }
-    Py_INCREF(state->default_action);
-    if (PyModule_AddObject(m, "_defaultaction",
-                           state->default_action) < 0)
-        return NULL;
 
-    state->filters_version = 0;
     return m;
-}
 
+error:
+    if (st != NULL) {
+        _Warnings_ClearState(st);
+    }
+    Py_DECREF(m);
+    return NULL;
+}
 
+// We need this to ensure that warnings still work until late in finalization.
 void
-_PyWarnings_Fini(_PyRuntimeState *runtime)
+_PyWarnings_Fini(PyInterpreterState *interp)
 {
-    struct _warnings_runtime_state *state = &runtime->warnings;
-    Py_CLEAR(state->filters);
-    Py_CLEAR(state->once_registry);
-    Py_CLEAR(state->default_action);
+    _Warnings_ClearState(&interp->warnings);
 }

Python/pylifecycle.c

@@ -1288,7 +1288,7 @@ Py_FinalizeEx(void)
     PyDict_Fini();
     PySlice_Fini();
     _PyGC_Fini(runtime);
-    _PyWarnings_Fini(runtime);
+    _PyWarnings_Fini(interp);
     _Py_HashRandomization_Fini();
     _PyArg_Fini();
     PyAsyncGen_Fini();

Python/pystate.c

@@ -5,6 +5,7 @@
 #include "pycore_coreconfig.h"
 #include "pycore_pymem.h"
 #include "pycore_pystate.h"
+#include "pycore_pylifecycle.h"
 
 /* --------------------------------------------------------------------------
 CAUTION
@@ -257,6 +258,9 @@ _PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp)
     Py_CLEAR(interp->after_forkers_parent);
     Py_CLEAR(interp->after_forkers_child);
 #endif
+    if (runtime->finalizing == NULL) {
+        _PyWarnings_Fini(interp);
+    }
     // XXX Once we have one allocator per interpreter (i.e.
     // per-interpreter GC) we must ensure that all of the interpreter's
     // objects have been cleaned up at the point.