bpo-32089: Fix warnings filters in dev mode (#4482)

The developer mode (-X dev) now creates all default warnings filters
to order filters in the correct order to always show ResourceWarning
and make BytesWarning depend on the -b option.

Write a functional test to make sure that ResourceWarning is logged
twice at the same location in the developer mode.

Add a new 'dev_mode' field to _PyCoreConfig.

Doc/using/cmdline.rst

@@ -412,13 +412,13 @@ Miscellaneous options
      application.  Typical usage is ``python3 -X importtime -c 'import
      asyncio'``.  See also :envvar:`PYTHONPROFILEIMPORTTIME`.
    * ``-X dev`` enables the "developer mode": enable debug checks at runtime.
-     In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug PYTHONASYNCIODEBUG=1 python3
-     -W default -X faulthandler ...``, except that the :envvar:`PYTHONMALLOC`
-     and :envvar:`PYTHONASYNCIODEBUG` environment variables are not set in
-     practice. Developer mode:
+     Developer mode:
 
-     * Add ``default`` warnings option. For example, display
-       :exc:`DeprecationWarning` and :exc:`ResourceWarning` warnings.
+     * Warning filters: add a filter to display all warnings (``"default"``
+       action), except of :exc:`BytesWarning` which still depends on the
+       :option:`-b` option, and use ``"always"`` action for
+       :exc:`ResourceWarning` warnings. For example, display
+       :exc:`DeprecationWarning` warnings.
      * Install debug hooks on memory allocators: see the
        :c:func:`PyMem_SetupDebugHooks` C function.
      * Enable the :mod:`faulthandler` module to dump the Python traceback

Include/pystate.h

@@ -33,6 +33,7 @@ typedef struct {
     int faulthandler;
     int tracemalloc;        /* Number of saved frames, 0=don't trace */
     int importtime;         /* -X importtime */
+    int dev_mode;           /* -X dev */
 } _PyCoreConfig;
 
 #define _PyCoreConfig_INIT \
@@ -43,7 +44,8 @@ typedef struct {
      .allocator = NULL, \
      .faulthandler = 0, \
      .tracemalloc = 0, \
-     .importtime = 0}
+     .importtime = 0, \
+     .dev_mode = 0}
 
 /* Placeholders while working on the new configuration API
  *

Lib/subprocess.py

@@ -262,15 +262,11 @@ def _args_from_interpreter_flags():
             args.append('-' + opt * v)
 
     # -W options
-    warnoptions = sys.warnoptions
-    xoptions = getattr(sys, '_xoptions', {})
-    if 'dev' in xoptions and warnoptions and warnoptions[-1] == 'default':
-        # special case: -X dev adds 'default' to sys.warnoptions
-        warnoptions = warnoptions[:-1]
-    for opt in warnoptions:
+    for opt in sys.warnoptions:
         args.append('-W' + opt)
 
     # -X options
+    xoptions = getattr(sys, '_xoptions', {})
     if 'dev' in xoptions:
         args.extend(('-X', 'dev'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',

Lib/test/test_cmd_line.py

@@ -507,14 +507,14 @@ class CmdLineTest(unittest.TestCase):
             with self.subTest(envar_value=value):
                 assert_python_ok('-c', code, **env_vars)
 
-    def run_xdev(self, code, check_exitcode=True):
+    def run_xdev(self, *args, check_exitcode=True):
         env = dict(os.environ)
         env.pop('PYTHONWARNINGS', None)
         # Force malloc() to disable the debug hooks which are enabled
         # by default for Python compiled in debug mode
         env['PYTHONMALLOC'] = 'malloc'
 
-        args = (sys.executable, '-X', 'dev', '-c', code)
+        args = (sys.executable, '-X', 'dev', *args)
         proc = subprocess.run(args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
@@ -525,8 +525,34 @@ class CmdLineTest(unittest.TestCase):
         return proc.stdout.rstrip()
 
     def test_xdev(self):
-        out = self.run_xdev("import sys; print(sys.warnoptions)")
-        self.assertEqual(out, "['default']")
+        code = ("import sys, warnings; "
+                "print(' '.join('%s::%s' % (f[0], f[2].__name__) "
+                                "for f in warnings.filters))")
+
+        out = self.run_xdev("-c", code)
+        self.assertEqual(out,
+                         "ignore::BytesWarning "
+                         "always::ResourceWarning "
+                         "default::Warning")
+
+        out = self.run_xdev("-b", "-c", code)
+        self.assertEqual(out,
+                         "default::BytesWarning "
+                         "always::ResourceWarning "
+                         "default::Warning")
+
+        out = self.run_xdev("-bb", "-c", code)
+        self.assertEqual(out,
+                         "error::BytesWarning "
+                         "always::ResourceWarning "
+                         "default::Warning")
+
+        out = self.run_xdev("-Werror", "-c", code)
+        self.assertEqual(out,
+                         "error::Warning "
+                         "ignore::BytesWarning "
+                         "always::ResourceWarning "
+                         "default::Warning")
 
         try:
             import _testcapi
@@ -535,7 +561,7 @@ class CmdLineTest(unittest.TestCase):
         else:
             code = "import _testcapi; _testcapi.pymem_api_misuse()"
             with support.SuppressCrashReport():
-                out = self.run_xdev(code, check_exitcode=False)
+                out = self.run_xdev("-c", code, check_exitcode=False)
             self.assertIn("Debug memory block at address p=", out)
 
         try:
@@ -544,9 +570,23 @@ class CmdLineTest(unittest.TestCase):
             pass
         else:
             code = "import faulthandler; print(faulthandler.is_enabled())"
-            out = self.run_xdev(code)
+            out = self.run_xdev("-c", code)
             self.assertEqual(out, "True")
 
+        # Make sure that ResourceWarning emitted twice at the same line number
+        # is logged twice
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
+        with open(filename, "w", encoding="utf8") as fp:
+            print("def func(): open(__file__)", file=fp)
+            print("func()", file=fp)
+            print("func()", file=fp)
+            fp.flush()
+
+        out = self.run_xdev(filename)
+        self.assertEqual(out.count(':1: ResourceWarning: '), 2, out)
+
+
 class IgnoreEnvironmentTest(unittest.TestCase):
 
     def run_ignoring_vars(self, predicate, **env_vars):

Lib/warnings.py

@@ -486,7 +486,6 @@ class catch_warnings(object):
 # - a compiled regex that must match the module that is being warned
 # - a line number for the line being warning, or 0 to mean any line
 # If either if the compiled regexs are None, match anything.
-_warnings_defaults = False
 try:
     from _warnings import (filters, _defaultaction, _onceregistry,
                            warn, warn_explicit, _filters_mutated)
@@ -504,12 +503,16 @@ except ImportError:
         global _filters_version
         _filters_version += 1
 
+    _warnings_defaults = False
+
 
 # Module initialization
 _processoptions(sys.warnoptions)
 if not _warnings_defaults:
+    dev_mode = ('dev' in getattr(sys, '_xoptions', {}))
     py_debug = hasattr(sys, 'gettotalrefcount')
-    if not py_debug:
+
+    if not(dev_mode or py_debug):
         silence = [ImportWarning, PendingDeprecationWarning]
         silence.append(DeprecationWarning)
         for cls in silence:
@@ -525,10 +528,15 @@ if not _warnings_defaults:
     simplefilter(bytes_action, category=BytesWarning, append=1)
 
     # resource usage warnings are enabled by default in pydebug mode
-    if py_debug:
+    if dev_mode or py_debug:
         resource_action = "always"
     else:
         resource_action = "ignore"
     simplefilter(resource_action, category=ResourceWarning, append=1)
 
+    if dev_mode:
+        simplefilter("default", category=Warning, append=1)
+
+    del py_debug, dev_mode
+
 del _warnings_defaults

Modules/main.c

@@ -1397,14 +1397,9 @@ pymain_parse_envvars(_PyMain *pymain)
         return -1;
     }
     if (pymain_get_xoption(pymain, L"dev")) {
-        /* "python3 -X dev ..." behaves
-           as "PYTHONMALLOC=debug python3 -Wd -X faulthandler ..." */
-        core_config->allocator = "debug";
-        if (pymain_optlist_append(pymain, &pymain->cmdline.warning_options,
-                                  L"default") < 0) {
-            return -1;
-        }
+        core_config->dev_mode = 1;
         core_config->faulthandler = 1;
+        core_config->allocator = "debug";
     }
     return 0;
 }

Python/_warnings.c

@@ -1196,11 +1196,19 @@ create_filter(PyObject *category, const char *action)
 static PyObject *
 init_filters(void)
 {
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
+    int dev_mode = interp->core_config.dev_mode;
+
+    Py_ssize_t count = 2;
+    if (dev_mode) {
+        count++;
+    }
 #ifndef Py_DEBUG
-    PyObject *filters = PyList_New(5);
-#else
-    PyObject *filters = PyList_New(2);
+    if (!dev_mode) {
+        count += 3;
+    }
 #endif
+    PyObject *filters = PyList_New(count);
     unsigned int pos = 0;  /* Post-incremented in each use. */
     unsigned int x;
     const char *bytes_action, *resource_action;
@@ -1209,12 +1217,14 @@ init_filters(void)
         return NULL;
 
 #ifndef Py_DEBUG
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_DeprecationWarning, "ignore"));
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_PendingDeprecationWarning, "ignore"));
-    PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_ImportWarning, "ignore"));
+    if (!dev_mode) {
+        PyList_SET_ITEM(filters, pos++,
+                        create_filter(PyExc_DeprecationWarning, "ignore"));
+        PyList_SET_ITEM(filters, pos++,
+                        create_filter(PyExc_PendingDeprecationWarning, "ignore"));
+        PyList_SET_ITEM(filters, pos++,
+                        create_filter(PyExc_ImportWarning, "ignore"));
+    }
 #endif
 
     if (Py_BytesWarningFlag > 1)
@@ -1225,14 +1235,21 @@ init_filters(void)
         bytes_action = "ignore";
     PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning,
                     bytes_action));
+
     /* resource usage warnings are enabled by default in pydebug mode */
 #ifdef Py_DEBUG
     resource_action = "always";
 #else
-    resource_action = "ignore";
+    resource_action = (dev_mode ? "always" : "ignore");
 #endif
     PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning,
                     resource_action));
+
+    if (dev_mode) {
+        PyList_SET_ITEM(filters, pos++,
+                        create_filter(PyExc_Warning, "default"));
+    }
+
     for (x = 0; x < pos; x += 1) {
         if (PyList_GET_ITEM(filters, x) == NULL) {
             Py_DECREF(filters);