bpo-36829: Add sys.unraisablehook() (GH-13187)

Add new sys.unraisablehook() function which can be overridden to
control how "unraisable exceptions" are handled. It is called when an
exception has occurred but there is no way for Python to handle it.
For example, when a destructor raises an exception or during garbage
collection (gc.collect()).

Changes:

* Add an internal UnraisableHookArgs type used to pass arguments to
  sys.unraisablehook.
* Add _PyErr_WriteUnraisableDefaultHook().
* The default hook now ignores exception on writing the traceback.
* test_sys now uses unittest.main() to automatically discover tests:
  remove test_main().
* Add _PyErr_Init().
* Fix PyErr_WriteUnraisable(): hold a strong reference to sys.stderr
  while using it

Doc/c-api/exceptions.rst

@@ -72,6 +72,9 @@ Printing and clearing
 
 .. c:function:: void PyErr_WriteUnraisable(PyObject *obj)
 
+   Call :func:`sys.unraisablehook` using the current exception and *obj*
+   argument.
+
    This utility function prints a warning message to ``sys.stderr`` when an
    exception has been set but it is impossible for the interpreter to actually
    raise the exception.  It is used, for example, when an exception occurs in an
@@ -81,6 +84,8 @@ Printing and clearing
    in which the unraisable exception occurred. If possible,
    the repr of *obj* will be printed in the warning message.
 
+   An exception must be set when calling this function.
+
 
 Raising exceptions
 ==================

Doc/library/sys.rst

@@ -248,16 +248,19 @@ always available.
    before the program exits.  The handling of such top-level exceptions can be
    customized by assigning another three-argument function to ``sys.excepthook``.
 
+   See also :func:`unraisablehook` which handles unraisable exceptions.
+
 
 .. data:: __breakpointhook__
           __displayhook__
           __excepthook__
+          __unraisablehook__
 
    These objects contain the original values of ``breakpointhook``,
-   ``displayhook``, and ``excepthook`` at the start of the program.  They are
-   saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be
-   restored in case they happen to get replaced with broken or alternative
-   objects.
+   ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the
+   program.  They are saved so that ``breakpointhook``, ``displayhook`` and
+   ``excepthook``, ``unraisablehook`` can be restored in case they happen to
+   get replaced with broken or alternative objects.
 
    .. versionadded:: 3.7
       __breakpointhook__
@@ -1487,6 +1490,28 @@ always available.
    is suppressed and only the exception type and value are printed.
 
 
+.. function:: unraisablehook(unraisable, /)
+
+   Handle an unraisable exception.
+
+   Called when an exception has occurred but there is no way for Python to
+   handle it. For example, when a destructor raises an exception or during
+   garbage collection (:func:`gc.collect`).
+
+   The *unraisable* argument has the following attributes:
+
+   * *exc_type*: Exception type.
+   * *exc_value*: Exception value, can be ``None``.
+   * *exc_traceback*: Exception traceback, can be ``None``.
+   * *object*: Object causing the exception, can be ``None``.
+
+   :func:`sys.unraisablehook` can be overridden to control how unraisable
+   exceptions are handled.
+
+   See also :func:`excepthook` which handles uncaught exceptions.
+
+   .. versionadded:: 3.8
+
 .. data:: version
 
    A string containing the version number of the Python interpreter plus additional

Doc/whatsnew/3.8.rst

@@ -481,6 +481,16 @@ and manipulating normal distributions of a random variable.
     [7.672102882379219, 12.000027119750287, 4.647488369766392]
 
 
+sys
+---
+
+Add new :func:`sys.unraisablehook` function which can be overridden to control
+how "unraisable exceptions" are handled. It is called when an exception has
+occurred but there is no way for Python to handle it. For example, when a
+destructor raises an exception or during garbage collection
+(:func:`gc.collect`).
+
+
 tarfile
 -------
 

Include/internal/pycore_pylifecycle.h

@@ -48,6 +48,7 @@ extern int _PySys_InitMain(
     PyInterpreterState *interp);
 extern _PyInitError _PyImport_Init(PyInterpreterState *interp);
 extern _PyInitError _PyExc_Init(void);
+extern _PyInitError _PyErr_Init(void);
 extern _PyInitError _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern _PyInitError _PyImportHooks_Init(void);
 extern int _PyFloat_Init(void);
@@ -100,8 +101,11 @@ PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromCoreConfig(
     const _PyCoreConfig *coreconfig,
     const _PyArgv *args);
 
+
 PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
 
+PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_sys.py

@@ -876,6 +876,81 @@ class SysModuleTest(unittest.TestCase):
         self.assertEqual(out, 'mbcs replace')
 
 
+@test.support.cpython_only
+class UnraisableHookTest(unittest.TestCase):
+    def write_unraisable_exc(self, exc, obj):
+        import _testcapi
+        import types
+        try:
+            # raise the exception to get a traceback in the except block
+            try:
+                raise exc
+            except Exception as exc2:
+                _testcapi.write_unraisable_exc(exc2, obj)
+                return types.SimpleNamespace(exc_type=type(exc2),
+                                             exc_value=exc2,
+                                             exc_traceback=exc2.__traceback__,
+                                             object=obj)
+        finally:
+            # Explicitly break any reference cycle
+            exc = None
+            exc2 = None
+
+    def test_original_unraisablehook(self):
+        obj = "an object"
+
+        with test.support.captured_output("stderr") as stderr:
+            with test.support.swap_attr(sys, 'unraisablehook',
+                                        sys.__unraisablehook__):
+                self.write_unraisable_exc(ValueError(42), obj)
+
+        err = stderr.getvalue()
+        self.assertIn(f'Exception ignored in: {obj!r}\n', err)
+        self.assertIn('Traceback (most recent call last):\n', err)
+        self.assertIn('ValueError: 42\n', err)
+
+    def test_original_unraisablehook_wrong_type(self):
+        exc = ValueError(42)
+        with test.support.swap_attr(sys, 'unraisablehook',
+                                    sys.__unraisablehook__):
+            with self.assertRaises(TypeError):
+                sys.unraisablehook(exc)
+
+    def test_custom_unraisablehook(self):
+        hook_args = None
+
+        def hook_func(args):
+            nonlocal hook_args
+            hook_args = args
+
+        obj = object()
+        try:
+            with test.support.swap_attr(sys, 'unraisablehook', hook_func):
+                expected = self.write_unraisable_exc(ValueError(42), obj)
+                for attr in "exc_type exc_value exc_traceback object".split():
+                    self.assertEqual(getattr(hook_args, attr),
+                                     getattr(expected, attr),
+                                     (hook_args, expected))
+        finally:
+            # expected and hook_args contain an exception: break reference cycle
+            expected = None
+            hook_args = None
+
+    def test_custom_unraisablehook_fail(self):
+        def hook_func(*args):
+            raise Exception("hook_func failed")
+
+        with test.support.captured_output("stderr") as stderr:
+            with test.support.swap_attr(sys, 'unraisablehook', hook_func):
+                self.write_unraisable_exc(ValueError(42), None)
+
+        err = stderr.getvalue()
+        self.assertIn(f'Exception ignored in: {hook_func!r}\n',
+                      err)
+        self.assertIn('Traceback (most recent call last):\n', err)
+        self.assertIn('Exception: hook_func failed\n', err)
+
+
 @test.support.cpython_only
 class SizeofTest(unittest.TestCase):
 
@@ -1277,8 +1352,5 @@ class SizeofTest(unittest.TestCase):
         self.assertIsNone(cur.finalizer)
 
 
-def test_main():
-    test.support.run_unittest(SysModuleTest, SizeofTest)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()

Misc/NEWS.d/next/Library/2019-05-08-12-51-37.bpo-36829.8enFMA.rst

+Add new :func:`sys.unraisablehook` function which can be overridden to
+control how "unraisable exceptions" are handled. It is called when an
+exception has occurred but there is no way for Python to handle it. For
+example, when a destructor raises an exception or during garbage collection
+(:func:`gc.collect`).

Modules/_testcapimodule.c

@@ -4982,6 +4982,20 @@ negative_refcount(PyObject *self, PyObject *Py_UNUSED(args))
 #endif
 
 
+static PyObject*
+test_write_unraisable_exc(PyObject *self, PyObject *args)
+{
+    PyObject *exc, *obj;
+    if (!PyArg_ParseTuple(args, "OO", &exc, &obj)) {
+        return NULL;
+    }
+
+    PyErr_SetObject((PyObject *)Py_TYPE(exc), exc);
+    PyErr_WriteUnraisable(obj);
+    Py_RETURN_NONE;
+}
+
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",       raise_memoryerror,               METH_NOARGS},
@@ -5221,6 +5235,7 @@ static PyMethodDef TestMethods[] = {
 #ifdef Py_REF_DEBUG
     {"negative_refcount", negative_refcount, METH_NOARGS},
 #endif
+    {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS},
     {NULL, NULL} /* sentinel */
 };
 

Python/clinic/sysmodule.c.h

@@ -65,6 +65,22 @@ sys_exc_info(PyObject *module, PyObject *Py_UNUSED(ignored))
     return sys_exc_info_impl(module);
 }
 
+PyDoc_STRVAR(sys_unraisablehook__doc__,
+"unraisablehook($module, unraisable, /)\n"
+"--\n"
+"\n"
+"Handle an unraisable exception.\n"
+"\n"
+"The unraisable argument has the following attributes:\n"
+"\n"
+"* exc_type: Exception type.\n"
+"* exc_value: Exception value.\n"
+"* exc_tb: Exception traceback, can be None.\n"
+"* obj: Object causing the exception, can be None.");
+
+#define SYS_UNRAISABLEHOOK_METHODDEF    \
+    {"unraisablehook", (PyCFunction)sys_unraisablehook, METH_O, sys_unraisablehook__doc__},
+
 PyDoc_STRVAR(sys_exit__doc__,
 "exit($module, status=None, /)\n"
 "--\n"
@@ -1060,4 +1076,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=3ba4c194d00f1866 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=603e4d5a453dc769 input=a9049054013a1b77]*/

Python/errors.c

@@ -2,6 +2,7 @@
 /* Error handling */
 
 #include "Python.h"
+#include "pycore_coreconfig.h"
 #include "pycore_pystate.h"
 
 #ifndef __STDC__
@@ -944,90 +945,271 @@ PyErr_NewExceptionWithDoc(const char *name, const char *doc,
 }
 
 
-/* Call when an exception has occurred but there is no way for Python
-   to handle it.  Examples: exception in __del__ or during GC. */
-void
-PyErr_WriteUnraisable(PyObject *obj)
+PyDoc_STRVAR(UnraisableHookArgs__doc__,
+"UnraisableHookArgs\n\
+\n\
+Type used to pass arguments to sys.unraisablehook.");
+
+static PyTypeObject UnraisableHookArgsType;
+
+static PyStructSequence_Field UnraisableHookArgs_fields[] = {
+    {"exc_type", "Exception type"},
+    {"exc_value", "Exception value"},
+    {"exc_traceback", "Exception traceback"},
+    {"object", "Object causing the exception"},
+    {0}
+};
+
+static PyStructSequence_Desc UnraisableHookArgs_desc = {
+    .name = "UnraisableHookArgs",
+    .doc = UnraisableHookArgs__doc__,
+    .fields = UnraisableHookArgs_fields,
+    .n_in_sequence = 4
+};
+
+
+_PyInitError
+_PyErr_Init(void)
 {
-    _Py_IDENTIFIER(__module__);
-    PyObject *f, *t, *v, *tb;
-    PyObject *moduleName = NULL;
-    const char *className;
+    if (UnraisableHookArgsType.tp_name == NULL) {
+        if (PyStructSequence_InitType2(&UnraisableHookArgsType,
+                                       &UnraisableHookArgs_desc) < 0) {
+            return _Py_INIT_ERR("failed to initialize UnraisableHookArgs type");
+        }
+    }
+    return _Py_INIT_OK();
+}
 
-    PyErr_Fetch(&t, &v, &tb);
 
-    f = _PySys_GetObjectId(&PyId_stderr);
-    if (f == NULL || f == Py_None)
-        goto done;
+static PyObject *
+make_unraisable_hook_args(PyObject *exc_type, PyObject *exc_value,
+                          PyObject *exc_tb, PyObject *obj)
+{
+    PyObject *args = PyStructSequence_New(&UnraisableHookArgsType);
+    if (args == NULL) {
+        return NULL;
+    }
+
+    Py_ssize_t pos = 0;
+#define ADD_ITEM(exc_type) \
+        do { \
+            if (exc_type == NULL) { \
+                exc_type = Py_None; \
+            } \
+            Py_INCREF(exc_type); \
+            PyStructSequence_SET_ITEM(args, pos++, exc_type); \
+        } while (0)
+
+
+    ADD_ITEM(exc_type);
+    ADD_ITEM(exc_value);
+    ADD_ITEM(exc_tb);
+    ADD_ITEM(obj);
+#undef ADD_ITEM
+
+    if (PyErr_Occurred()) {
+        Py_DECREF(args);
+        return NULL;
+    }
+    return args;
+}
+
+
+
+/* Default implementation of sys.unraisablehook.
+
+   It can be called to log the exception of a custom sys.unraisablehook.
 
-    if (obj) {
-        if (PyFile_WriteString("Exception ignored in: ", f) < 0)
-            goto done;
-        if (PyFile_WriteObject(obj, f, 0) < 0) {
+   Do nothing if sys.stderr attribute doesn't exist or is set to None. */
+static int
+write_unraisable_exc_file(PyObject *exc_type, PyObject *exc_value,
+                          PyObject *exc_tb, PyObject *obj, PyObject *file)
+{
+    if (obj != NULL && obj != Py_None) {
+        if (PyFile_WriteString("Exception ignored in: ", file) < 0) {
+            return -1;
+        }
+
+        if (PyFile_WriteObject(obj, file, 0) < 0) {
             PyErr_Clear();
-            if (PyFile_WriteString("<object repr() failed>", f) < 0) {
-                goto done;
+            if (PyFile_WriteString("<object repr() failed>", file) < 0) {
+                return -1;
             }
         }
-        if (PyFile_WriteString("\n", f) < 0)
-            goto done;
+        if (PyFile_WriteString("\n", file) < 0) {
+            return -1;
+        }
     }
 
-    if (PyTraceBack_Print(tb, f) < 0)
-        goto done;
+    if (exc_tb != NULL && exc_tb != Py_None) {
+        if (PyTraceBack_Print(exc_tb, file) < 0) {
+            /* continue even if writing the traceback failed */
+            PyErr_Clear();
+        }
+    }
 
-    if (!t)
-        goto done;
+    if (!exc_type) {
+        return -1;
+    }
 
-    assert(PyExceptionClass_Check(t));
-    className = PyExceptionClass_Name(t);
+    assert(PyExceptionClass_Check(exc_type));
+    const char *className = PyExceptionClass_Name(exc_type);
     if (className != NULL) {
         const char *dot = strrchr(className, '.');
         if (dot != NULL)
             className = dot+1;
     }
 
-    moduleName = _PyObject_GetAttrId(t, &PyId___module__);
+    _Py_IDENTIFIER(__module__);
+    PyObject *moduleName = _PyObject_GetAttrId(exc_type, &PyId___module__);
     if (moduleName == NULL || !PyUnicode_Check(moduleName)) {
+        Py_XDECREF(moduleName);
         PyErr_Clear();
-        if (PyFile_WriteString("<unknown>", f) < 0)
-            goto done;
+        if (PyFile_WriteString("<unknown>", file) < 0) {
+            return -1;
+        }
     }
     else {
         if (!_PyUnicode_EqualToASCIIId(moduleName, &PyId_builtins)) {
-            if (PyFile_WriteObject(moduleName, f, Py_PRINT_RAW) < 0)
-                goto done;
-            if (PyFile_WriteString(".", f) < 0)
-                goto done;
+            if (PyFile_WriteObject(moduleName, file, Py_PRINT_RAW) < 0) {
+                Py_DECREF(moduleName);
+                return -1;
+            }
+            Py_DECREF(moduleName);
+            if (PyFile_WriteString(".", file) < 0) {
+                return -1;
+            }
+        }
+        else {
+            Py_DECREF(moduleName);
         }
     }
     if (className == NULL) {
-        if (PyFile_WriteString("<unknown>", f) < 0)
-            goto done;
+        if (PyFile_WriteString("<unknown>", file) < 0) {
+            return -1;
+        }
     }
     else {
-        if (PyFile_WriteString(className, f) < 0)
-            goto done;
+        if (PyFile_WriteString(className, file) < 0) {
+            return -1;
+        }
     }
 
-    if (v && v != Py_None) {
-        if (PyFile_WriteString(": ", f) < 0)
-            goto done;
-        if (PyFile_WriteObject(v, f, Py_PRINT_RAW) < 0) {
+    if (exc_value && exc_value != Py_None) {
+        if (PyFile_WriteString(": ", file) < 0) {
+            return -1;
+        }
+        if (PyFile_WriteObject(exc_value, file, Py_PRINT_RAW) < 0) {
             PyErr_Clear();
-            if (PyFile_WriteString("<exception str() failed>", f) < 0) {
+            if (PyFile_WriteString("<exception str() failed>", file) < 0) {
+                return -1;
+            }
+        }
+    }
+    if (PyFile_WriteString("\n", file) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+
+static int
+write_unraisable_exc(PyObject *exc_type, PyObject *exc_value,
+                     PyObject *exc_tb, PyObject *obj)
+{
+    PyObject *file = _PySys_GetObjectId(&PyId_stderr);
+    if (file == NULL || file == Py_None) {
+        return 0;
+    }
+
+    /* Hold a strong reference to ensure that sys.stderr doesn't go away
+       while we use it */
+    Py_INCREF(file);
+    int res = write_unraisable_exc_file(exc_type, exc_value, exc_tb,
+                                        obj, file);
+    Py_DECREF(file);
+
+    return res;
+}
+
+
+PyObject*
+_PyErr_WriteUnraisableDefaultHook(PyObject *args)
+{
+    if (Py_TYPE(args) != &UnraisableHookArgsType) {
+        PyErr_SetString(PyExc_TypeError,
+                        "sys.unraisablehook argument type "
+                        "must be UnraisableHookArgs");
+        return NULL;
+    }
+
+    /* Borrowed references */
+    PyObject *exc_type = PyStructSequence_GET_ITEM(args, 0);
+    PyObject *exc_value = PyStructSequence_GET_ITEM(args, 1);
+    PyObject *exc_tb = PyStructSequence_GET_ITEM(args, 2);
+    PyObject *obj = PyStructSequence_GET_ITEM(args, 3);
+
+    if (write_unraisable_exc(exc_type, exc_value, exc_tb, obj) < 0) {
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+
+
+/* Call sys.unraisablehook().
+
+   This function can be used when an exception has occurred but there is no way
+   for Python to handle it. For example, when a destructor raises an exception
+   or during garbage collection (gc.collect()).
+
+   An exception must be set when calling this function. */
+void
+PyErr_WriteUnraisable(PyObject *obj)
+{
+    PyObject *exc_type, *exc_value, *exc_tb;
+
+    PyErr_Fetch(&exc_type, &exc_value, &exc_tb);
+
+    assert(exc_type != NULL);
+
+    if (exc_type == NULL) {
+        /* sys.unraisablehook requires that at least exc_type is set */
+        goto default_hook;
+    }
+
+    _Py_IDENTIFIER(unraisablehook);
+    PyObject *hook = _PySys_GetObjectId(&PyId_unraisablehook);
+    if (hook != NULL && hook != Py_None) {
+        PyObject *hook_args;
+
+        hook_args = make_unraisable_hook_args(exc_type, exc_value, exc_tb, obj);
+        if (hook_args != NULL) {
+            PyObject *args[1] = {hook_args};
+            PyObject *res = _PyObject_FastCall(hook, args, 1);
+            Py_DECREF(hook_args);
+            if (res != NULL) {
+                Py_DECREF(res);
                 goto done;
             }
         }
+
+        /* sys.unraisablehook failed: log its error using default hook */
+        Py_XDECREF(exc_type);
+        Py_XDECREF(exc_value);
+        Py_XDECREF(exc_tb);
+        PyErr_Fetch(&exc_type, &exc_value, &exc_tb);
+
+        obj = hook;
     }
-    if (PyFile_WriteString("\n", f) < 0)
-        goto done;
+
+default_hook:
+    /* Call the default unraisable hook (ignore failure) */
+    (void)write_unraisable_exc(exc_type, exc_value, exc_tb, obj);
 
 done:
-    Py_XDECREF(moduleName);
-    Py_XDECREF(t);
-    Py_XDECREF(v);
-    Py_XDECREF(tb);
+    Py_XDECREF(exc_type);
+    Py_XDECREF(exc_value);
+    Py_XDECREF(exc_tb);
     PyErr_Clear(); /* Just in case */
 }
 

Python/pylifecycle.c

@@ -587,6 +587,12 @@ pycore_init_types(void)
     if (!_PyContext_Init()) {
         return _Py_INIT_ERR("can't init context");
     }
+
+    err = _PyErr_Init();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
     return _Py_INIT_OK();
 }
 
@@ -1462,6 +1468,12 @@ new_interpreter(PyThreadState **tstate_p)
         return err;
     }
 
+    err = _PyErr_Init();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+
     /* XXX The following is lax in error checking */
     PyObject *modules = PyDict_New();
     if (modules == NULL) {

Python/sysmodule.c

@@ -374,6 +374,30 @@ sys_exc_info_impl(PyObject *module)
 }
 
 
+/*[clinic input]
+sys.unraisablehook
+
+    unraisable: object
+    /
+
+Handle an unraisable exception.
+
+The unraisable argument has the following attributes:
+
+* exc_type: Exception type.
+* exc_value: Exception value.
+* exc_tb: Exception traceback, can be None.
+* obj: Object causing the exception, can be None.
+[clinic start generated code]*/
+
+static PyObject *
+sys_unraisablehook(PyObject *module, PyObject *unraisable)
+/*[clinic end generated code: output=bb92838b32abaa14 input=fdbdb47fdd0bee06]*/
+{
+    return _PyErr_WriteUnraisableDefaultHook(unraisable);
+}
+
+
 /*[clinic input]
 sys.exit
 
@@ -1672,6 +1696,7 @@ static PyMethodDef sys_methods[] = {
      METH_VARARGS | METH_KEYWORDS, set_asyncgen_hooks_doc},
     SYS_GET_ASYNCGEN_HOOKS_METHODDEF
     SYS_GETANDROIDAPILEVEL_METHODDEF
+    SYS_UNRAISABLEHOOK_METHODDEF
     {NULL,              NULL}           /* sentinel */
 };
 
@@ -2369,6 +2394,9 @@ _PySys_InitCore(_PyRuntimeState *runtime, PyInterpreterState *interp,
     SET_SYS_FROM_STRING_BORROW(
         "__breakpointhook__",
         PyDict_GetItemString(sysdict, "breakpointhook"));
+    SET_SYS_FROM_STRING_BORROW("__unraisablehook__",
+                               PyDict_GetItemString(sysdict, "unraisablehook"));
+
     SET_SYS_FROM_STRING("version",
                          PyUnicode_FromString(Py_GetVersion()));
     SET_SYS_FROM_STRING("hexversion",