Commit 9048c493 authored by Steve Dower's avatar Steve Dower Committed by GitHub

bpo-37369: Fix initialization of sys members when launched via an app container (GH-14428)

sys._base_executable is now always defined on all platforms, and can be overridden through configuration.
Also adds test.support.PythonSymlink to encapsulate platform-specific logic for symlinking sys.executable
parent 80097e08
......@@ -373,10 +373,11 @@ typedef struct {
module_search_paths_set is equal
to zero. */
wchar_t *executable; /* sys.executable */
wchar_t *prefix; /* sys.prefix */
wchar_t *base_prefix; /* sys.base_prefix */
wchar_t *exec_prefix; /* sys.exec_prefix */
wchar_t *executable; /* sys.executable */
wchar_t *base_executable; /* sys._base_executable */
wchar_t *prefix; /* sys.prefix */
wchar_t *base_prefix; /* sys.base_prefix */
wchar_t *exec_prefix; /* sys.exec_prefix */
wchar_t *base_exec_prefix; /* sys.base_exec_prefix */
/* --- Parameter only used by Py_Main() ---------- */
......
......@@ -27,6 +27,8 @@ typedef struct _PyPathConfig {
are ignored when their value are equal to -1 (unset). */
int isolated;
int site_import;
/* Set when a venv is detected */
wchar_t *base_executable;
} _PyPathConfig;
#define _PyPathConfig_INIT \
......
......@@ -22,8 +22,7 @@ WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
def _path_eq(p1, p2):
return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
WINENV = (hasattr(sys, '_base_executable') and
not _path_eq(sys.executable, sys._base_executable))
WINENV = not _path_eq(sys.executable, sys._base_executable)
def _close_handles(*handles):
......
......@@ -459,13 +459,6 @@ def venv(known_paths):
env = os.environ
if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__']
elif sys.platform == 'win32' and '__PYVENV_LAUNCHER__' in env:
executable = sys.executable
import _winapi
sys._base_executable = _winapi.GetModuleFileName(0)
# bpo-35873: Clear the environment variable to avoid it being
# inherited by child processes.
del os.environ['__PYVENV_LAUNCHER__']
else:
executable = sys.executable
exe_dir, _ = os.path.split(os.path.abspath(executable))
......
......@@ -12,6 +12,7 @@ import faulthandler
import fnmatch
import functools
import gc
import glob
import importlib
import importlib.util
import io
......@@ -2500,6 +2501,84 @@ def skip_unless_symlink(test):
msg = "Requires functional symlink implementation"
return test if ok else unittest.skip(msg)(test)
class PythonSymlink:
"""Creates a symlink for the current Python executable"""
def __init__(self, link=None):
self.link = link or os.path.abspath(TESTFN)
self._linked = []
self.real = os.path.realpath(sys.executable)
self._also_link = []
self._env = None
self._platform_specific()
def _platform_specific(self):
pass
if sys.platform == "win32":
def _platform_specific(self):
import _winapi
if os.path.lexists(self.real) and not os.path.exists(self.real):
# App symlink appears to not exist, but we want the
# real executable here anyway
self.real = _winapi.GetModuleFileName(0)
dll = _winapi.GetModuleFileName(sys.dllhandle)
src_dir = os.path.dirname(dll)
dest_dir = os.path.dirname(self.link)
self._also_link.append((
dll,
os.path.join(dest_dir, os.path.basename(dll))
))
for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")):
self._also_link.append((
runtime,
os.path.join(dest_dir, os.path.basename(runtime))
))
self._env = {k.upper(): os.getenv(k) for k in os.environ}
self._env["PYTHONHOME"] = os.path.dirname(self.real)
if sysconfig.is_python_build(True):
self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
def __enter__(self):
os.symlink(self.real, self.link)
self._linked.append(self.link)
for real, link in self._also_link:
os.symlink(real, link)
self._linked.append(link)
return self
def __exit__(self, exc_type, exc_value, exc_tb):
for link in self._linked:
try:
os.remove(link)
except IOError as ex:
if verbose:
print("failed to clean up {}: {}".format(link, ex))
def _call(self, python, args, env, returncode):
cmd = [python, *args]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
r = p.communicate()
if p.returncode != returncode:
if verbose:
print(repr(r[0]))
print(repr(r[1]), file=sys.stderr)
raise RuntimeError(
'unexpected return code: {0} (0x{0:08X})'.format(p.returncode))
return r
def call_real(self, *args, returncode=0):
return self._call(self.real, args, None, returncode)
def call_link(self, *args, returncode=0):
return self._call(self.link, args, self._env, returncode)
_can_xattr = None
def can_xattr():
global _can_xattr
......
......@@ -362,6 +362,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'pythonpath_env': None,
'home': None,
'executable': GET_DEFAULT_CONFIG,
'base_executable': GET_DEFAULT_CONFIG,
'prefix': GET_DEFAULT_CONFIG,
'base_prefix': GET_DEFAULT_CONFIG,
......@@ -534,14 +535,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
expected['stdio_errors'] = 'surrogateescape'
if sys.platform == 'win32':
default_executable = self.test_exe
elif expected['program_name'] is not self.GET_DEFAULT_CONFIG:
default_executable = os.path.abspath(expected['program_name'])
else:
default_executable = os.path.join(os.getcwd(), '_testembed')
if expected['executable'] is self.GET_DEFAULT_CONFIG:
if sys.platform == 'win32':
expected['executable'] = self.test_exe
else:
if expected['program_name'] is not self.GET_DEFAULT_CONFIG:
expected['executable'] = os.path.abspath(expected['program_name'])
else:
expected['executable'] = os.path.join(os.getcwd(), '_testembed')
expected['executable'] = default_executable
if expected['base_executable'] is self.GET_DEFAULT_CONFIG:
expected['base_executable'] = default_executable
if expected['program_name'] is self.GET_DEFAULT_CONFIG:
expected['program_name'] = './_testembed'
......
......@@ -610,9 +610,10 @@ class CGIHTTPServerTestCase(BaseTestCase):
# The shebang line should be pure ASCII: use symlink if possible.
# See issue #7668.
self._pythonexe_symlink = None
if support.can_symlink():
self.pythonexe = os.path.join(self.parent_dir, 'python')
os.symlink(sys.executable, self.pythonexe)
self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__()
else:
self.pythonexe = sys.executable
......@@ -655,8 +656,8 @@ class CGIHTTPServerTestCase(BaseTestCase):
def tearDown(self):
try:
os.chdir(self.cwd)
if self.pythonexe != sys.executable:
os.remove(self.pythonexe)
if self._pythonexe_symlink:
self._pythonexe_symlink.__exit__(None, None, None)
if self.nocgi_path:
os.remove(self.nocgi_path)
if self.file1_path:
......
......@@ -20,37 +20,9 @@ class PlatformTest(unittest.TestCase):
@support.skip_unless_symlink
def test_architecture_via_symlink(self): # issue3762
# On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
# so we add the directory to the path, PYTHONHOME and PYTHONPATH.
env = None
if sys.platform == "win32":
env = {k.upper(): os.environ[k] for k in os.environ}
env["PATH"] = "{};{}".format(
os.path.dirname(sys.executable), env.get("PATH", ""))
env["PYTHONHOME"] = os.path.dirname(sys.executable)
if sysconfig.is_python_build(True):
env["PYTHONPATH"] = os.path.dirname(os.__file__)
def get(python, env=None):
cmd = [python, '-c',
'import platform; print(platform.architecture())']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
r = p.communicate()
if p.returncode:
print(repr(r[0]))
print(repr(r[1]), file=sys.stderr)
self.fail('unexpected return code: {0} (0x{0:08X})'
.format(p.returncode))
return r
real = os.path.realpath(sys.executable)
link = os.path.abspath(support.TESTFN)
os.symlink(real, link)
try:
self.assertEqual(get(real), get(link, env=env))
finally:
os.remove(link)
with support.PythonSymlink() as py:
cmd = "-c", "import platform; print(platform.architecture())"
self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
def test_platform(self):
for aliased in (False, True):
......@@ -275,6 +247,11 @@ class PlatformTest(unittest.TestCase):
os.path.exists(sys.executable+'.exe'):
# Cygwin horror
executable = sys.executable + '.exe'
elif sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
executable = _winapi.GetModuleFileName(0)
else:
executable = sys.executable
platform.libc_ver(executable)
......
......@@ -6,7 +6,8 @@ import shutil
from copy import copy
from test.support import (import_module, TESTFN, unlink, check_warnings,
captured_stdout, skip_unless_symlink, change_cwd)
captured_stdout, skip_unless_symlink, change_cwd,
PythonSymlink)
import sysconfig
from sysconfig import (get_paths, get_platform, get_config_vars,
......@@ -232,39 +233,10 @@ class TestSysConfig(unittest.TestCase):
self.assertEqual(get_scheme_names(), wanted)
@skip_unless_symlink
def test_symlink(self):
# On Windows, the EXE needs to know where pythonXY.dll is at so we have
# to add the directory to the path.
env = None
if sys.platform == "win32":
env = {k.upper(): os.environ[k] for k in os.environ}
env["PATH"] = "{};{}".format(
os.path.dirname(sys.executable), env.get("PATH", ""))
# Requires PYTHONHOME as well since we locate stdlib from the
# EXE path and not the DLL path (which should be fixed)
env["PYTHONHOME"] = os.path.dirname(sys.executable)
if sysconfig.is_python_build(True):
env["PYTHONPATH"] = os.path.dirname(os.__file__)
# Issue 7880
def get(python, env=None):
cmd = [python, '-c',
'import sysconfig; print(sysconfig.get_platform())']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env)
out, err = p.communicate()
if p.returncode:
print((out, err))
self.fail('Non-zero return code {0} (0x{0:08X})'
.format(p.returncode))
return out, err
real = os.path.realpath(sys.executable)
link = os.path.abspath(TESTFN)
os.symlink(real, link)
try:
self.assertEqual(get(real), get(link, env))
finally:
unlink(link)
def test_symlink(self): # Issue 7880
with PythonSymlink() as py:
cmd = "-c", "import sysconfig; print(sysconfig.get_platform())"
self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
def test_user_similar(self):
# Issue #8759: make sure the posix scheme for the users
......
......@@ -28,8 +28,8 @@ except ImportError:
# Platforms that set sys._base_executable can create venvs from within
# another venv, so no need to skip tests that require venv.create().
requireVenvCreate = unittest.skipUnless(
hasattr(sys, '_base_executable')
or sys.prefix == sys.base_prefix,
sys.prefix == sys.base_prefix
or sys._base_executable != sys.executable,
'cannot run venv.create from within a venv on this platform')
def check_output(cmd, encoding=None):
......@@ -57,8 +57,14 @@ class BaseTest(unittest.TestCase):
self.bindir = 'bin'
self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
self.include = 'include'
executable = getattr(sys, '_base_executable', sys.executable)
executable = sys._base_executable
self.exe = os.path.split(executable)[-1]
if (sys.platform == 'win32'
and os.path.lexists(executable)
and not os.path.exists(executable)):
self.cannot_link_exe = True
else:
self.cannot_link_exe = False
def tearDown(self):
rmtree(self.env_dir)
......@@ -102,7 +108,7 @@ class BasicTest(BaseTest):
else:
self.assertFalse(os.path.exists(p))
data = self.get_text_file_contents('pyvenv.cfg')
executable = getattr(sys, '_base_executable', sys.executable)
executable = sys._base_executable
path = os.path.dirname(executable)
self.assertIn('home = %s' % path, data)
fn = self.get_env_file(self.bindir, self.exe)
......@@ -158,10 +164,6 @@ class BasicTest(BaseTest):
"""
Test that the prefix values are as expected.
"""
#check our prefixes
self.assertEqual(sys.base_prefix, sys.prefix)
self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)
# check a venv's prefixes
rmtree(self.env_dir)
self.run_with_capture(venv.create, self.env_dir)
......@@ -169,9 +171,9 @@ class BasicTest(BaseTest):
cmd = [envpy, '-c', None]
for prefix, expected in (
('prefix', self.env_dir),
('prefix', self.env_dir),
('base_prefix', sys.prefix),
('base_exec_prefix', sys.exec_prefix)):
('exec_prefix', self.env_dir),
('base_prefix', sys.base_prefix),
('base_exec_prefix', sys.base_exec_prefix)):
cmd[2] = 'import sys; print(sys.%s)' % prefix
out, err = check_output(cmd)
self.assertEqual(out.strip(), expected.encode())
......@@ -283,7 +285,12 @@ class BasicTest(BaseTest):
# symlinked to 'python3.3' in the env, even when symlinking in
# general isn't wanted.
if usl:
self.assertTrue(os.path.islink(fn))
if self.cannot_link_exe:
# Symlinking is skipped when our executable is already a
# special app symlink
self.assertFalse(os.path.islink(fn))
else:
self.assertTrue(os.path.islink(fn))
# If a venv is created from a source build and that venv is used to
# run the test, the pyvenv.cfg in the venv created in the test will
......
......@@ -112,7 +112,7 @@ class EnvBuilder:
prompt = self.prompt if self.prompt is not None else context.env_name
context.prompt = '(%s) ' % prompt
create_if_needed(env_dir)
executable = getattr(sys, '_base_executable', sys.executable)
executable = sys._base_executable
dirname, exename = os.path.split(os.path.abspath(executable))
context.executable = executable
context.python_dir = dirname
......@@ -163,47 +163,66 @@ class EnvBuilder:
if self.prompt is not None:
f.write(f'prompt = {self.prompt!r}\n')
def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
"""
force_copy = not self.symlinks
if not force_copy:
try:
if not os.path.islink(dst): # can't link to itself!
if os.name != 'nt':
def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
"""
force_copy = not self.symlinks
if not force_copy:
try:
if not os.path.islink(dst): # can't link to itself!
if relative_symlinks_ok:
assert os.path.dirname(src) == os.path.dirname(dst)
os.symlink(os.path.basename(src), dst)
else:
os.symlink(src, dst)
except Exception: # may need to use a more specific exception
logger.warning('Unable to symlink %r to %r', src, dst)
force_copy = True
if force_copy:
shutil.copyfile(src, dst)
else:
def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
"""
bad_src = os.path.lexists(src) and not os.path.exists(src)
if self.symlinks and not bad_src and not os.path.islink(dst):
try:
if relative_symlinks_ok:
assert os.path.dirname(src) == os.path.dirname(dst)
os.symlink(os.path.basename(src), dst)
else:
os.symlink(src, dst)
except Exception: # may need to use a more specific exception
logger.warning('Unable to symlink %r to %r', src, dst)
force_copy = True
if force_copy:
if os.name == 'nt':
# On Windows, we rewrite symlinks to our base python.exe into
# copies of venvlauncher.exe
basename, ext = os.path.splitext(os.path.basename(src))
srcfn = os.path.join(os.path.dirname(__file__),
"scripts",
"nt",
basename + ext)
# Builds or venv's from builds need to remap source file
# locations, as we do not put them into Lib/venv/scripts
if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
if basename.endswith('_d'):
ext = '_d' + ext
basename = basename[:-2]
if basename == 'python':
basename = 'venvlauncher'
elif basename == 'pythonw':
basename = 'venvwlauncher'
src = os.path.join(os.path.dirname(src), basename + ext)
else:
src = srcfn
if not os.path.exists(src):
logger.warning('Unable to copy %r', src)
return
except Exception: # may need to use a more specific exception
logger.warning('Unable to symlink %r to %r', src, dst)
# On Windows, we rewrite symlinks to our base python.exe into
# copies of venvlauncher.exe
basename, ext = os.path.splitext(os.path.basename(src))
srcfn = os.path.join(os.path.dirname(__file__),
"scripts",
"nt",
basename + ext)
# Builds or venv's from builds need to remap source file
# locations, as we do not put them into Lib/venv/scripts
if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
if basename.endswith('_d'):
ext = '_d' + ext
basename = basename[:-2]
if basename == 'python':
basename = 'venvlauncher'
elif basename == 'pythonw':
basename = 'venvwlauncher'
src = os.path.join(os.path.dirname(src), basename + ext)
else:
src = srcfn
if not os.path.exists(src):
if not bad_src:
logger.warning('Unable to copy %r', src)
return
shutil.copyfile(src, dst)
......@@ -251,7 +270,7 @@ class EnvBuilder:
for suffix in suffixes:
src = os.path.join(dirname, suffix)
if os.path.exists(src):
if os.path.lexists(src):
copier(src, os.path.join(binpath, suffix))
if sysconfig.is_python_build(True):
......
Fixes path for :data:`sys.executable` when running from the Microsoft Store.
......@@ -537,14 +537,28 @@ get_program_full_path(const PyConfig *config,
wchar_t program_full_path[MAXPATHLEN+1];
memset(program_full_path, 0, sizeof(program_full_path));
if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
/* GetModuleFileName should never fail when passed NULL */
return _PyStatus_ERR("Cannot determine program path");
}
/* The launcher may need to force the executable path to a
* different environment, so override it here. */
pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
if (pyvenv_launcher && pyvenv_launcher[0]) {
/* If overridden, preserve the original full path */
pathconfig->base_executable = PyMem_RawMalloc(
sizeof(wchar_t) * (MAXPATHLEN + 1));
PyStatus status = canonicalize(pathconfig->base_executable,
program_full_path);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
} else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
/* GetModuleFileName should never fail when passed NULL */
return _PyStatus_ERR("Cannot determine program path");
/* bpo-35873: Clear the environment variable to avoid it being
* inherited by child processes. */
_wputenv_s(L"__PYVENV_LAUNCHER__", L"");
}
pathconfig->program_full_path = PyMem_RawMalloc(
......
This diff is collapsed.
......@@ -528,6 +528,7 @@ PyConfig_Clear(PyConfig *config)
config->module_search_paths_set = 0;
CLEAR(config->executable);
CLEAR(config->base_executable);
CLEAR(config->prefix);
CLEAR(config->base_prefix);
CLEAR(config->exec_prefix);
......@@ -765,6 +766,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
COPY_ATTR(module_search_paths_set);
COPY_WSTR_ATTR(executable);
COPY_WSTR_ATTR(base_executable);
COPY_WSTR_ATTR(prefix);
COPY_WSTR_ATTR(base_prefix);
COPY_WSTR_ATTR(exec_prefix);
......@@ -865,6 +867,7 @@ config_as_dict(const PyConfig *config)
SET_ITEM_WSTR(home);
SET_ITEM_WSTRLIST(module_search_paths);
SET_ITEM_WSTR(executable);
SET_ITEM_WSTR(base_executable);
SET_ITEM_WSTR(prefix);
SET_ITEM_WSTR(base_prefix);
SET_ITEM_WSTR(exec_prefix);
......@@ -2404,6 +2407,7 @@ PyConfig_Read(PyConfig *config)
assert(config->module_search_paths_set != 0);
/* don't check config->module_search_paths */
assert(config->executable != NULL);
assert(config->base_executable != NULL);
assert(config->prefix != NULL);
assert(config->base_prefix != NULL);
assert(config->exec_prefix != NULL);
......
......@@ -57,6 +57,7 @@ pathconfig_clear(_PyPathConfig *config)
CLEAR(config->module_search_path);
CLEAR(config->home);
CLEAR(config->program_name);
CLEAR(config->base_executable);
#undef CLEAR
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
......@@ -89,6 +90,14 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
status = _PyStatus_NO_MEMORY();
goto error;
}
if (config->base_executable) {
PyMem_RawFree(new_config.base_executable);
if (copy_wstr(&new_config.base_executable,
config->base_executable) < 0) {
status = _PyStatus_NO_MEMORY();
goto error;
}
}
pathconfig_clear(pathconfig);
*pathconfig = new_config;
......@@ -132,6 +141,7 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config)
COPY_ATTR(module_search_path);
COPY_ATTR(program_name);
COPY_ATTR(home);
COPY_ATTR(base_executable);
pathconfig_clear(&_Py_path_config);
/* Steal new_config strings; don't clear new_config */
......@@ -224,6 +234,9 @@ _PyConfig_SetPathConfig(const PyConfig *config)
if (copy_wstr(&pathconfig.home, config->home) < 0) {
goto no_memory;
}
if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) {
goto no_memory;
}
status = _PyPathConfig_SetGlobal(&pathconfig);
if (_PyStatus_EXCEPTION(status)) {
......@@ -321,6 +334,13 @@ config_calculate_pathconfig(PyConfig *config)
}
}
if (config->base_executable == NULL) {
if (copy_wstr(&config->base_executable,
pathconfig.base_executable) < 0) {
goto no_memory;
}
}
if (pathconfig.isolated != -1) {
config->isolated = pathconfig.isolated;
}
......@@ -367,6 +387,14 @@ _PyConfig_InitPathConfig(PyConfig *config)
return _PyStatus_NO_MEMORY();
}
}
if (config->base_executable == NULL) {
if (copy_wstr(&config->base_executable,
config->executable) < 0) {
return _PyStatus_NO_MEMORY();
}
}
return _PyStatus_OK();
}
......@@ -434,6 +462,8 @@ Py_SetPath(const wchar_t *path)
_Py_path_config.home = NULL;
new_config.program_name = _Py_path_config.program_name;
_Py_path_config.program_name = NULL;
new_config.base_executable = _Py_path_config.base_executable;
_Py_path_config.base_executable = NULL;
pathconfig_clear(&_Py_path_config);
_Py_path_config = new_config;
......
......@@ -2850,6 +2850,7 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyThreadState *tstate)
COPY_LIST("path", config->module_search_paths);
SET_SYS_FROM_WSTR("executable", config->executable);
SET_SYS_FROM_WSTR("_base_executable", config->base_executable);
SET_SYS_FROM_WSTR("prefix", config->prefix);
SET_SYS_FROM_WSTR("base_prefix", config->base_prefix);
SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment