bpo-33053: -m now adds *starting* directory to sys.path (GH-6231)

Historically, -m added the empty string as sys.path
zero, meaning it resolved imports against the current
working directory, the same way -c and the interactive
prompt do.

This changes the sys.path initialisation to add the
*starting* working directory as sys.path[0] instead,
such that changes to the working directory while the
program is running will have no effect on imports
when using the -m switch.

Doc/library/test.rst

@@ -1332,8 +1332,8 @@ script execution tests.
 .. function:: run_python_until_end(*args, **env_vars)
 
    Set up the environment based on *env_vars* for running the interpreter
-   in a subprocess.  The values can include ``__isolated``, ``__cleavenv``,
-   and ``TERM``.
+   in a subprocess.  The values can include ``__isolated``, ``__cleanenv``,
+   ``__cwd``, and ``TERM``.
 
 
 .. function:: assert_python_ok(*args, **env_vars)

Doc/whatsnew/3.7.rst

@@ -421,6 +421,12 @@ Other Language Changes
   writable.
   (Contributed by Nathaniel J. Smith in :issue:`30579`.)
 
+* When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded
+  to the full starting directory path, rather than being left as the empty
+  directory (which allows imports from the *current* working directory at the
+  time when an import occurs)
+  (Contributed by Nick Coghlan in :issue:`33053`.)
+
 
 New Modules
 ===========
@@ -1138,6 +1144,11 @@ Changes in Python behavior
   parentheses can be omitted only on calls.
   (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.)
 
+* When using the ``-m`` switch, the starting directory is now added to sys.path,
+  rather than the current working directory. Any programs that are found to be
+  relying on the previous behaviour will need to be updated to manipulate
+  :data:`sys.path` appropriately.
+
 
 Changes in the Python API
 -------------------------

Lib/test/support/script_helper.py

@@ -87,6 +87,7 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult",
 # Executing the interpreter in a subprocess
 def run_python_until_end(*args, **env_vars):
     env_required = interpreter_requires_environment()
+    cwd = env_vars.pop('__cwd', None)
     if '__isolated' in env_vars:
         isolated = env_vars.pop('__isolated')
     else:
@@ -125,7 +126,7 @@ def run_python_until_end(*args, **env_vars):
     cmd_line.extend(args)
     proc = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                         env=env)
+                         env=env, cwd=cwd)
     with proc:
         try:
             out, err = proc.communicate()

Lib/test/test_bdb.py

@@ -524,13 +524,13 @@ def run_test(modules, set_list, skip=None):
     test.id = lambda : None
     test.expect_set = list(gen(repeat(()), iter(sl)))
     with create_modules(modules):
-        sys.path.append(os.getcwd())
         with TracerRun(test, skip=skip) as tracer:
             tracer.runcall(tfunc_import)
 
 @contextmanager
 def create_modules(modules):
     with test.support.temp_cwd():
+        sys.path.append(os.getcwd())
         try:
             for m in modules:
                 fname = m + '.py'
@@ -542,6 +542,7 @@ def create_modules(modules):
         finally:
             for m in modules:
                 test.support.forget(m)
+            sys.path.pop()
 
 def break_in_func(funcname, fname=__file__, temporary=False, cond=None):
     return 'break', (fname, None, temporary, cond, funcname)

Lib/test/test_doctest.py

@@ -9,7 +9,7 @@ import os
 import sys
 import importlib
 import unittest
-
+import tempfile
 
 # NOTE: There are some additional tests relating to interaction with
 #       zipimport in the test_zipimport_support test module.
@@ -688,10 +688,16 @@ class TestDocTestFinder(unittest.TestCase):
 
     def test_empty_namespace_package(self):
         pkg_name = 'doctest_empty_pkg'
-        os.mkdir(pkg_name)
-        mod = importlib.import_module(pkg_name)
-        assert doctest.DocTestFinder().find(mod) == []
-        os.rmdir(pkg_name)
+        with tempfile.TemporaryDirectory() as parent_dir:
+            pkg_dir = os.path.join(parent_dir, pkg_name)
+            os.mkdir(pkg_dir)
+            sys.path.append(parent_dir)
+            try:
+                mod = importlib.import_module(pkg_name)
+            finally:
+                support.forget(pkg_name)
+                sys.path.pop()
+            assert doctest.DocTestFinder().find(mod) == []
 
 
 def test_DocTestParser(): r"""

Lib/test/test_import/__init__.py

@@ -118,7 +118,7 @@ class ImportTests(unittest.TestCase):
                 f.write("__all__ = [b'invalid_type']")
             globals = {}
             with self.assertRaisesRegex(
-                TypeError, f"{re.escape(name)}\.__all__ must be str"
+                TypeError, f"{re.escape(name)}\\.__all__ must be str"
             ):
                 exec(f"from {name} import *", globals)
             self.assertNotIn(b"invalid_type", globals)
@@ -127,7 +127,7 @@ class ImportTests(unittest.TestCase):
                 f.write("globals()[b'invalid_type'] = object()")
             globals = {}
             with self.assertRaisesRegex(
-                TypeError, f"{re.escape(name)}\.__dict__ must be str"
+                TypeError, f"{re.escape(name)}\\.__dict__ must be str"
             ):
                 exec(f"from {name} import *", globals)
             self.assertNotIn(b"invalid_type", globals)
@@ -847,8 +847,11 @@ class PycacheTests(unittest.TestCase):
         unload(TESTFN)
         importlib.invalidate_caches()
         m = __import__(TESTFN)
-        self.assertEqual(m.__file__,
-                         os.path.join(os.curdir, os.path.relpath(pyc_file)))
+        try:
+            self.assertEqual(m.__file__,
+                             os.path.join(os.curdir, os.path.relpath(pyc_file)))
+        finally:
+            os.remove(pyc_file)
 
     def test___cached__(self):
         # Modules now also have an __cached__ that points to the pyc file.

Misc/NEWS.d/next/Core and Builtins/2018-03-25-19-49-06.bpo-33053.V3xlsH.rst

+When using the -m switch, sys.path[0] is now explicitly expanded as the
+*starting* working directory, rather than being left as the empty path
+(which allows imports from the current working directory at the time of the
+import)

Python/pathconfig.c

@@ -3,6 +3,7 @@
 #include "Python.h"
 #include "osdefs.h"
 #include "internal/pystate.h"
+#include <wchar.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -255,11 +256,6 @@ Py_GetProgramName(void)
     return _Py_path_config.program_name;
 }
 
-
-#define _HAVE_SCRIPT_ARGUMENT(argc, argv) \
-  (argc > 0 && argv0 != NULL && \
-   wcscmp(argv0, L"-c") != 0 && wcscmp(argv0, L"-m") != 0)
-
 /* Compute argv[0] which will be prepended to sys.argv */
 PyObject*
 _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
@@ -267,6 +263,8 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
     wchar_t *argv0;
     wchar_t *p = NULL;
     Py_ssize_t n = 0;
+    int have_script_arg = 0;
+    int have_module_arg = 0;
 #ifdef HAVE_READLINK
     wchar_t link[MAXPATHLEN+1];
     wchar_t argv0copy[2*MAXPATHLEN+1];
@@ -278,11 +276,25 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
     wchar_t fullpath[MAX_PATH];
 #endif
 
-
     argv0 = argv[0];
+    if (argc > 0 && argv0 != NULL) {
+        have_module_arg = (wcscmp(argv0, L"-m") == 0);
+        have_script_arg = !have_module_arg && (wcscmp(argv0, L"-c") != 0);
+    }
+
+    if (have_module_arg) {
+        #if defined(HAVE_REALPATH) || defined(MS_WINDOWS)
+            _Py_wgetcwd(fullpath, Py_ARRAY_LENGTH(fullpath));
+            argv0 = fullpath;
+            n = wcslen(argv0);
+        #else
+            argv0 = L".";
+            n = 1;
+        #endif
+    }
 
 #ifdef HAVE_READLINK
-    if (_HAVE_SCRIPT_ARGUMENT(argc, argv))
+    if (have_script_arg)
         nr = _Py_wreadlink(argv0, link, MAXPATHLEN);
     if (nr > 0) {
         /* It's a symlink */
@@ -310,7 +322,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
 
 #if SEP == '\\'
     /* Special case for Microsoft filename syntax */
-    if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) {
+    if (have_script_arg) {
         wchar_t *q;
 #if defined(MS_WINDOWS)
         /* Replace the first element in argv with the full path. */
@@ -334,7 +346,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
         }
     }
 #else /* All other filename syntaxes */
-    if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) {
+    if (have_script_arg) {
 #if defined(HAVE_REALPATH)
         if (_Py_wrealpath(argv0, fullpath, Py_ARRAY_LENGTH(fullpath))) {
             argv0 = fullpath;