bpo-31975 (PEP 565): Show DeprecationWarning in __main__ (GH-4458)

- primary change is to add a new default filter entry for
  'default::DeprecationWarning:__main__'
- secondary change is an internal one to cope with plain
  strings in the warning module's internal filter list
  (this avoids the need to create a compiled regex object
  early on during interpreter startup)
- assorted documentation updates, including many more
  examples of configuring the warnings settings
- additional tests to ensure that both the pure Python and
  the C accelerated warnings modules have the expected
  default configuration

Doc/library/exceptions.rst

@@ -661,11 +661,13 @@ depending on the system error code.
    :pep:`3151` - Reworking the OS and IO exception hierarchy
 
 
+.. _warning-categories-as-exceptions:
+
 Warnings
 --------
 
-The following exceptions are used as warning categories; see the :mod:`warnings`
-module for more information.
+The following exceptions are used as warning categories; see the
+:ref:`warning-categories` documentation for more details.
 
 .. exception:: Warning
 
@@ -679,12 +681,14 @@ module for more information.
 
 .. exception:: DeprecationWarning
 
-   Base class for warnings about deprecated features.
+   Base class for warnings about deprecated features when those warnings are
+   intended for other Python developers.
 
 
 .. exception:: PendingDeprecationWarning
 
-   Base class for warnings about features which will be deprecated in the future.
+   Base class for warnings about features which will be deprecated in the
+   future.
 
 
 .. exception:: SyntaxWarning
@@ -699,8 +703,8 @@ module for more information.
 
 .. exception:: FutureWarning
 
-   Base class for warnings about constructs that will change semantically in the
-   future.
+   Base class for warnings about deprecated features when those warnings are
+   intended for end users of applications that are written in Python.
 
 
 .. exception:: ImportWarning
@@ -720,7 +724,8 @@ module for more information.
 
 .. exception:: ResourceWarning
 
-   Base class for warnings related to resource usage.
+   Base class for warnings related to resource usage. Ignored by the default
+   warning filters.
 
    .. versionadded:: 3.2
 

Doc/tools/susp-ignored.csv

@@ -259,12 +259,8 @@ tutorial/stdlib2,,:start,extra = data[start:start+extra_size]
 tutorial/stdlib2,,:start,"fields = struct.unpack('<IIIHH', data[start:start+16])"
 tutorial/stdlib2,,:start,filename = data[start:start+filenamesize]
 tutorial/stdlib2,,:Warning,WARNING:root:Warning:config file server.conf not found
-using/cmdline,,:category,action:message:category:module:line
 using/cmdline,,:errorhandler,:errorhandler
-using/cmdline,,:line,action:message:category:module:line
 using/cmdline,,:line,file:line: category: message
-using/cmdline,,:message,action:message:category:module:line
-using/cmdline,,:module,action:message:category:module:line
 using/unix,,:Packaging,https://en.opensuse.org/Portal:Packaging
 whatsnew/2.0,,:len,
 whatsnew/2.3,,::,
@@ -302,6 +298,20 @@ whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf
 library/re,,`,!#$%&'*+-.^_`|~:
 library/re,,`,!\#\$%\&'\*\+\-\.\^_`\|\~:
 library/tarfile,,:xz,'x:xz'
+library/warnings,,:message,action:message:category:module:line
+library/warnings,,:category,action:message:category:module:line
+library/warnings,,:module,action:message:category:module:line
+library/warnings,,:line,action:message:category:module:line
+library/warnings,,::,error::ResourceWarning
+library/warnings,,::,default::DeprecationWarning
+library/warnings,,::,default:::mymodule
+library/warnings,,:mymodule,default:::mymodule
+library/warnings,,::,error:::mymodule
+library/warnings,,:mymodule,error:::mymodule
+library/warnings,,::,ignore::DeprecationWarning
+library/warnings,,::,ignore::PendingDeprecationWarning
+library/warnings,,::,ignore::ImportWarning
+library/warnings,,::,ignore::ResourceWarning
 library/xml.etree.elementtree,,:sometag,prefix:sometag
 library/xml.etree.elementtree,,:fictional,"<actors xmlns:fictional=""http://characters.example.com"""
 library/xml.etree.elementtree,,:character,<fictional:character>Lancelot</fictional:character>
@@ -330,3 +340,4 @@ whatsnew/3.7,,`,'`'
 whatsnew/3.7,,::,error::BytesWarning
 whatsnew/changelog,,::,error::BytesWarning
 whatsnew/changelog,,::,default::BytesWarning
+whatsnew/changelog,,::,default::DeprecationWarning

Doc/using/cmdline.rst

@@ -356,49 +356,27 @@ Miscellaneous options
    :option:`-W` options are ignored (though, a warning message is printed about
    invalid options when the first warning is issued).
 
-   Warnings can also be controlled from within a Python program using the
+   Warnings can also be controlled using the :envvar:`PYTHONWARNINGS`
+   environment variable and from within a Python program using the
    :mod:`warnings` module.
 
-   The simplest form of argument is one of the following action strings (or a
-   unique abbreviation):
-
-   ``ignore``
-      Ignore all warnings.
-   ``default``
-      Explicitly request the default behavior (printing each warning once per
-      source line).
-   ``all``
-      Print a warning each time it occurs (this may generate many messages if a
-      warning is triggered repeatedly for the same source line, such as inside a
-      loop).
-   ``module``
-      Print each warning only the first time it occurs in each module.
-   ``once``
-      Print each warning only the first time it occurs in the program.
-   ``error``
-      Raise an exception instead of printing a warning message.
-
-   The full form of argument is::
-
-       action:message:category:module:line
-
-   Here, *action* is as explained above but only applies to messages that match
-   the remaining fields.  Empty fields match all values; trailing empty fields
-   may be omitted.  The *message* field matches the start of the warning message
-   printed; this match is case-insensitive.  The *category* field matches the
-   warning category.  This must be a class name; the match tests whether the
-   actual warning category of the message is a subclass of the specified warning
-   category.  The full class name must be given.  The *module* field matches the
-   (fully-qualified) module name; this match is case-sensitive.  The *line*
-   field matches the line number, where zero matches all line numbers and is
-   thus equivalent to an omitted line number.
+   The simplest settings apply a particular action unconditionally to all
+   warnings emitted by a process (even those that are otherwise ignored by
+   default)::
 
-   .. seealso::
-      :mod:`warnings` -- the warnings module
+       -Wdefault  # Warn once per call location
+       -Werror    # Convert to exceptions
+       -Walways   # Warn every time
+       -Wmodule   # Warn once per calling module
+       -Wonce     # Warn once per Python process
+       -Wignore   # Never warn
 
-      :pep:`230` -- Warning framework
+   The action names can be abbreviated as desired (e.g. ``-Wi``, ``-Wd``,
+   ``-Wa``, ``-We``) and the interpreter will resolve them to the appropriate
+   action name.
 
-      :envvar:`PYTHONWARNINGS`
+   See :ref:`warning-filter` and :ref:`describing-warning-filters` for more
+   details.
 
 
 .. cmdoption:: -x
@@ -659,7 +637,23 @@ conflict.
 
    This is equivalent to the :option:`-W` option. If set to a comma
    separated string, it is equivalent to specifying :option:`-W` multiple
-   times.
+   times, with filters later in the list taking precedence over those earlier
+   in the list.
+
+   The simplest settings apply a particular action unconditionally to all
+   warnings emitted by a process (even those that are otherwise ignored by
+   default)::
+
+       PYTHONWARNINGS=default  # Warn once per call location
+       PYTHONWARNINGS=error    # Convert to exceptions
+       PYTHONWARNINGS=always   # Warn every time
+       PYTHONWARNINGS=module   # Warn once per calling module
+       PYTHONWARNINGS=once     # Warn once per Python process
+       PYTHONWARNINGS=ignore   # Never warn
+
+   See :ref:`warning-filter` and :ref:`describing-warning-filters` for more
+   details.
+
 
 .. envvar:: PYTHONFAULTHANDLER
 

Doc/whatsnew/3.7.rst

@@ -70,6 +70,7 @@ Summary -- Release highlights
 New Features
 ============
 
+
 .. _whatsnew37-pep538:
 
 PEP 538: Legacy C Locale Coercion
@@ -107,6 +108,7 @@ locale remains active when the core interpreter is initialized.
     :pep:`538` -- Coercing the legacy C locale to a UTF-8 based locale
        PEP written and implemented by Nick Coghlan.
 
+
 .. _whatsnew37-pep553:
 
 PEP 553: Built-in breakpoint()
@@ -203,6 +205,44 @@ resolution on Linux and Windows.
        PEP written and implemented by Victor Stinner
 
 
+.. _whatsnew37-pep565:
+
+PEP 565: Show DeprecationWarning in ``__main__``
+------------------------------------------------
+
+The default handling of :exc:`DeprecationWarning` has been changed such that
+these warnings are once more shown by default, but only when the code
+triggering them is running directly in the ``__main__`` module. As a result,
+developers of single file scripts and those using Python interactively should
+once again start seeing deprecation warnings for the APIs they use, but
+deprecation warnings triggered by imported application, library and framework
+modules will continue to be hidden by default.
+
+As a result of this change, the standard library now allows developers to choose
+between three different deprecation warning behaviours:
+
+* :exc:`FutureWarning`: always displayed by default, recommended for warnings
+  intended to be seen by application end users (e.g. for deprecated application
+  configuration settings).
+* :exc:`DeprecationWarning`: displayed by default only in ``__main__`` and when
+  running tests, recommended for warnings intended to be seen by other Python
+  developers where a version upgrade may result in changed behaviour or an
+  error.
+* :exc:`PendingDeprecationWarning`: displayed by default only when running
+  tests, intended for cases where a future version upgrade will change the
+  warning category to :exc:`DeprecationWarning` or :exc:`FutureWarning`.
+
+Previously both :exc:`DeprecationWarning` and :exc:`PendingDeprecationWarning`
+were only visible when running tests, which meant that developers primarily
+writing single file scripts or using Python interactively could be surprised
+by breaking changes in the APIs they used.
+
+.. seealso::
+
+    :pep:`565` -- Show DeprecationWarning in ``__main__``
+      PEP written and implemented by Nick Coghlan
+
+
 PEP 540: Add a new UTF-8 mode
 -----------------------------
 

Lib/test/test_cmd_line.py

@@ -558,6 +558,7 @@ class CmdLineTest(unittest.TestCase):
             expected_filters = "default::Warning"
         else:
             expected_filters = ("default::Warning "
+                                "default::DeprecationWarning "
                                 "ignore::DeprecationWarning "
                                 "ignore::PendingDeprecationWarning "
                                 "ignore::ImportWarning "
@@ -626,6 +627,7 @@ class CmdLineTest(unittest.TestCase):
                             "always::UserWarning")
         if not Py_DEBUG:
             expected_filters += (" "
+                                 "default::DeprecationWarning "
                                  "ignore::DeprecationWarning "
                                  "ignore::PendingDeprecationWarning "
                                  "ignore::ImportWarning "

Lib/test/test_warnings/__init__.py

@@ -16,6 +16,8 @@ import warnings as original_warnings
 py_warnings = support.import_fresh_module('warnings', blocked=['_warnings'])
 c_warnings = support.import_fresh_module('warnings', fresh=['_warnings'])
 
+Py_DEBUG = hasattr(sys, 'gettotalrefcount')
+
 @contextmanager
 def warnings_state(module):
     """Use a specific warnings implementation in warning_tests."""
@@ -320,6 +322,7 @@ class FilterTests(BaseTest):
                 self.module.filters[0][0], "error",
                 "simplefilter did not promote filter to the beginning of list"
             )
+
     def test_append_duplicate(self):
         with original_warnings.catch_warnings(module=self.module,
                 record=True) as w:
@@ -1143,6 +1146,37 @@ class EnvironmentVariableTests(BaseTest):
              b"  File \"<string>\", line 1, in <module>",
              b"DeprecationWarning: Message"])
 
+    def test_default_filter_configuration(self):
+        pure_python_api = self.module is py_warnings
+        if Py_DEBUG:
+            expected_default_filters = []
+        else:
+            if pure_python_api:
+                main_module_filter = re.compile("__main__")
+            else:
+                main_module_filter = "__main__"
+            expected_default_filters = [
+                ('default', None, DeprecationWarning, main_module_filter, 0),
+                ('ignore', None, DeprecationWarning, None, 0),
+                ('ignore', None, PendingDeprecationWarning, None, 0),
+                ('ignore', None, ImportWarning, None, 0),
+                ('ignore', None, ResourceWarning, None, 0),
+            ]
+        expected_output = [str(f).encode() for f in expected_default_filters]
+
+        if pure_python_api:
+            # Disable the warnings acceleration module in the subprocess
+            code = "import sys; sys.modules.pop('warnings', None); sys.modules['_warnings'] = None; "
+        else:
+            code = ""
+        code += "import warnings; [print(f) for f in warnings.filters]"
+
+        rc, stdout, stderr = assert_python_ok("-c", code, __isolated=True)
+        stdout_lines = [line.strip() for line in stdout.splitlines()]
+        self.maxDiff = None
+        self.assertEqual(stdout_lines, expected_output)
+
+
     @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii',
                          'requires non-ascii filesystemencoding')
     def test_nonascii(self):
@@ -1192,7 +1226,7 @@ a=A()
         rc, out, err = assert_python_ok("-c", code)
         # note: "__main__" filename is not correct, it should be the name
         # of the script
-        self.assertEqual(err, b'__main__:7: UserWarning: test')
+        self.assertEqual(err.decode(), '__main__:7: UserWarning: test')
 
     def test_late_resource_warning(self):
         # Issue #21925: Emitting a ResourceWarning late during the Python

Lib/warnings.py

@@ -519,8 +519,10 @@ except ImportError:
 # Module initialization
 _processoptions(sys.warnoptions)
 if not _warnings_defaults:
-    # Several warning categories are ignored by default in Py_DEBUG builds
+    # Several warning categories are ignored by default in regular builds
     if not hasattr(sys, 'gettotalrefcount'):
+        filterwarnings("default", category=DeprecationWarning,
+                       module="__main__", append=1)
         simplefilter("ignore", category=DeprecationWarning, append=1)
         simplefilter("ignore", category=PendingDeprecationWarning, append=1)
         simplefilter("ignore", category=ImportWarning, append=1)

Misc/NEWS.d/next/Core and Builtins/2018-01-05-20-54-27.bpo-31975.AmftlU.rst

+The default warning filter list now starts with a
+"default::DeprecationWarning:__main__" entry, so deprecation warnings are
+once again shown by default in single-file scripts and at the interactive
+prompt.

Python/_warnings.c

@@ -12,6 +12,7 @@ MODULE_NAME " provides basic warning filtering support.\n"
 _Py_IDENTIFIER(argv);
 _Py_IDENTIFIER(stderr);
 #ifndef Py_DEBUG
+_Py_IDENTIFIER(default);
 _Py_IDENTIFIER(ignore);
 #endif
 
@@ -22,8 +23,20 @@ check_matched(PyObject *obj, PyObject *arg)
     _Py_IDENTIFIER(match);
     int rc;
 
+    /* A 'None' filter always matches */
     if (obj == Py_None)
         return 1;
+
+    /* An internal plain text default filter must match exactly */
+    if (PyUnicode_CheckExact(obj)) {
+        int cmp_result = PyUnicode_Compare(obj, arg);
+        if (cmp_result == -1 && PyErr_Occurred()) {
+            return -1;
+        }
+        return !cmp_result;
+    }
+
+    /* Otherwise assume a regex filter and call its match() method */
     result = _PyObject_CallMethodIdObjArgs(obj, &PyId_match, arg, NULL);
     if (result == NULL)
         return -1;
@@ -1158,16 +1171,27 @@ static PyMethodDef warnings_functions[] = {
 
 #ifndef Py_DEBUG
 static PyObject *
-create_filter(PyObject *category, _Py_Identifier *id)
+create_filter(PyObject *category, _Py_Identifier *id, const char *modname)
 {
+    PyObject *modname_obj = NULL;
     PyObject *action_str = _PyUnicode_FromId(id);
     if (action_str == NULL) {
         return NULL;
     }
 
+    /* Default to "no module name" for initial filter set */
+    if (modname != NULL) {
+        modname_obj = PyUnicode_InternFromString(modname);
+        if (modname_obj == NULL) {
+            return NULL;
+        }
+    } else {
+        modname_obj = Py_None;
+    }
+
     /* This assumes the line number is zero for now. */
     return PyTuple_Pack(5, action_str, Py_None,
-                        category, Py_None, _PyLong_Zero);
+                        category, modname_obj, _PyLong_Zero);
 }
 #endif
 
@@ -1180,20 +1204,22 @@ init_filters(void)
     return PyList_New(0);
 #else
     /* Other builds ignore a number of warning categories by default */
-    PyObject *filters = PyList_New(4);
+    PyObject *filters = PyList_New(5);
     if (filters == NULL) {
         return NULL;
     }
 
     size_t pos = 0;  /* Post-incremented in each use. */
     PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_DeprecationWarning, &PyId_ignore));
+                    create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__"));
+    PyList_SET_ITEM(filters, pos++,
+                    create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL));
     PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore));
+                    create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL));
     PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_ImportWarning, &PyId_ignore));
+                    create_filter(PyExc_ImportWarning, &PyId_ignore, NULL));
     PyList_SET_ITEM(filters, pos++,
-                    create_filter(PyExc_ResourceWarning, &PyId_ignore));
+                    create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL));
 
     for (size_t x = 0; x < pos; x++) {
         if (PyList_GET_ITEM(filters, x) == NULL) {