Skip to content

C
cpython
Commit 1c3de541 authored by Steve Dower's avatar Steve Dower Committed by GitHub
Browse files
Options

bpo-34977: Use venv redirector instead of original python.exe on Windows (GH-11029)

parent b6ef6f69
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 452 additions and 53 deletions
Doc/library/venv.rst
View file @ 1c3de541
......@@ -130,7 +130,6 @@ creation according to their needs, the :class:`EnvBuilder` class.
.. versionadded:: 3.6
Added the ``prompt`` parameter
Creators of third-party virtual environment tools will be free to use the
provided ``EnvBuilder`` class as a base class.
......@@ -177,16 +176,15 @@ creation according to their needs, the :class:`EnvBuilder` class.
.. method:: setup_python(context)
Creates a copy of the Python executable (and, under Windows, DLLs) in
the environment. On a POSIX system, if a specific executable
``python3.x`` was used, symlinks to ``python`` and ``python3`` will be
created pointing to that executable, unless files with those names
already exist.
Creates a copy of the Python executable in the environment on POSIX
systems. If a specific executable ``python3.x`` was used, symlinks to
``python`` and ``python3`` will be created pointing to that executable,
unless files with those names already exist.
.. method:: setup_scripts(context)
Installs activation scripts appropriate to the platform into the virtual
environment.
environment. On Windows, also installs the ``python[w].exe`` scripts.
.. method:: post_setup(context)
......@@ -194,6 +192,11 @@ creation according to their needs, the :class:`EnvBuilder` class.
implementations to pre-install packages in the virtual environment or
perform other post-creation steps.
.. versionchanged:: 3.7.2
Windows now uses redirector scripts for ``python[w].exe`` instead of
copying the actual binaries, and so :meth:`setup_python` does nothing
unless running from a build in the source tree.
In addition, :class:`EnvBuilder` provides this utility method that can be
called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to
assist in installing custom scripts into the virtual environment.
......
Doc/whatsnew/3.7.rst
View file @ 1c3de541
......@@ -2512,3 +2512,13 @@ In 3.7.1 the :mod:`tokenize` module now implicitly emits a ``NEWLINE`` token
when provided with input that does not have a trailing new line. This behavior
now matches what the C tokenizer does internally.
(Contributed by Ammar Askar in :issue:`33899`.)
Notable changes in Python 3.7.2
===============================
In 3.7.2, :mod:`venv` on Windows no longer copies the original binaries, but
creates redirector scripts named ``python.exe`` and ``pythonw.exe`` instead.
This resolves a long standing issue where all virtual environments would have
to be upgraded or recreated with each Python update. However, note that this
release will still require recreation of virtual environments in order to get
the new scripts.
Lib/test/test_venv.py
View file @ 1c3de541
......@@ -243,6 +243,7 @@ class BasicTest(BaseTest):
self.assertIn('include-system-site-packages = %s\n' % s, data)
@unittest.skipUnless(can_symlink(), 'Needs symlinks')
@unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows')
def test_symlinking(self):
"""
Test symlinking works as expected
......
Lib/venv/__init__.py
View file @ 1c3de541
......@@ -64,10 +64,11 @@ class EnvBuilder:
self.system_site_packages = False
self.create_configuration(context)
self.setup_python(context)
if not self.upgrade:
self.setup_scripts(context)
if self.with_pip:
self._setup_pip(context)
if not self.upgrade:
self.setup_scripts(context)
self.post_setup(context)
if true_system_site_packages:
# We had set it to False before, now
......@@ -158,14 +159,6 @@ class EnvBuilder:
f.write('include-system-site-packages = %s\n' % incl)
f.write('version = %d.%d.%d\n' % sys.version_info[:3])
if os.name == 'nt':
def include_binary(self, f):
if f.endswith(('.pyd', '.dll')):
result = True
else:
result = f.startswith('python') and f.endswith('.exe')
return result
def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
......@@ -195,9 +188,9 @@ class EnvBuilder:
binpath = context.bin_path
path = context.env_exe
copier = self.symlink_or_copy
copier(context.executable, path)
dirname = context.python_dir
if os.name != 'nt':
copier(context.executable, path)
if not os.path.islink(path):
os.chmod(path, 0o755)
for suffix in ('python', 'python3'):
......@@ -209,26 +202,22 @@ class EnvBuilder:
if not os.path.islink(path):
os.chmod(path, 0o755)
else:
# See bpo-34011. When using a proper install, we should only need to
# copy the top-level of DLLs.
include = self.include_binary
files = [f for f in os.listdir(dirname) if include(f)]
for f in files:
src = os.path.join(dirname, f)
dst = os.path.join(binpath, f)
if dst != context.env_exe: # already done, above
copier(src, dst)
# When creating from a build directory, we continue to copy all files.
# For normal cases, the venvlauncher will be copied from
# our scripts folder. For builds, we need to copy it
# manually.
if sysconfig.is_python_build(True):
subdir = 'DLLs'
dirname = os.path.join(dirname, subdir)
if os.path.isdir(dirname):
files = [f for f in os.listdir(dirname) if include(f)]
for f in files:
src = os.path.join(dirname, f)
dst = os.path.join(binpath, f)
copier(src, dst)
suffix = '.exe'
if context.python_exe.lower().endswith('_d.exe'):
suffix = '_d.exe'
src = os.path.join(dirname, "venvlauncher" + suffix)
dst = os.path.join(binpath, context.python_exe)
copier(src, dst)
src = os.path.join(dirname, "venvwlauncher" + suffix)
dst = os.path.join(binpath, "pythonw" + suffix)
copier(src, dst)
# copy init.tcl over
for root, dirs, files in os.walk(context.python_dir):
if 'init.tcl' in files:
......@@ -326,7 +315,7 @@ class EnvBuilder:
dstfile = os.path.join(dstdir, f)
with open(srcfile, 'rb') as f:
data = f.read()
if not srcfile.endswith('.exe'):
if not srcfile.endswith(('.exe', '.pdb')):
try:
data = data.decode('utf-8')
data = self.replace_variables(data, context)
......
Misc/NEWS.d/next/Windows/2018-12-07-10-00-38.bpo-34977.agQJbD.rst 0 → 100644
View file @ 1c3de541
venv on Windows will now use a python.exe redirector rather than copying the
actual binaries from the base environment.
PC/getpathp.c
View file @ 1c3de541
......@@ -536,10 +536,16 @@ static _PyInitError
get_program_full_path(const _PyCoreConfig *core_config,
PyCalculatePath *calculate, _PyPathConfig *config)
{
const wchar_t *pyvenv_launcher;
wchar_t program_full_path[MAXPATHLEN+1];
memset(program_full_path, 0, sizeof(program_full_path));
if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
/* The launcher may need to force the executable path to a
* different environment, so override it here. */
pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
if (pyvenv_launcher && pyvenv_launcher[0]) {
wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
} else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
/* GetModuleFileName should never fail when passed NULL */
return _Py_INIT_ERR("Cannot determine program path");
}
......
PC/launcher.c
View file @ 1c3de541
This diff is collapsed.
PCbuild/pcbuild.proj
View file @ 1c3de541
......@@ -70,6 +70,8 @@
<Projects2 Include="_freeze_importlib.vcxproj" />
<!-- python[w].exe -->
<Projects2 Include="python.vcxproj;pythonw.vcxproj" />
<!-- venv[w]launcher.exe -->
<Projects2 Include="venvlauncher.vcxproj;venvwlauncher.vcxproj" />
</ItemGroup>
<Target Name="Build">
......
PCbuild/venvlauncher.vcxproj 0 → 100644
View file @ 1c3de541
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|Win32">
<Configuration>PGInstrument</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|x64">
<Configuration>PGInstrument</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|Win32">
<Configuration>PGUpdate</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|x64">
<Configuration>PGUpdate</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{494BAC80-A60C-43A9-99E7-ACB691CE2C4D}</ProjectGuid>
<RootNamespace>venvlauncher</RootNamespace>
<TargetName>venvlauncher</TargetName>
<SupportPGO>false</SupportPGO>
</PropertyGroup>
<Import Project="python.props" />
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<PropertyGroup>
<MakeVersionInfoBeforeTarget>ClCompile</MakeVersionInfoBeforeTarget>
</PropertyGroup>
<ImportGroup Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="pyproject.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>_CONSOLE;VENV_REDIRECT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>PY_ICON;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ResourceCompile>
<Link>
<AdditionalDependencies>version.lib;%(AdditionalDependencies)</AdditionalDependencies>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\PC\launcher.c" />
</ItemGroup>
<ItemGroup>
<None Include="..\PC\launcher.ico" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\PC\pylauncher.rc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
PCbuild/venvwlauncher.vcxproj 0 → 100644
View file @ 1c3de541
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|Win32">
<Configuration>PGInstrument</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|x64">
<Configuration>PGInstrument</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|Win32">
<Configuration>PGUpdate</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|x64">
<Configuration>PGUpdate</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{FDB84CBB-2FB6-47C8-A2D6-091E0833239D}</ProjectGuid>
<RootNamespace>venvwlauncher</RootNamespace>
<TargetName>venvwlauncher</TargetName>
<SupportPGO>false</SupportPGO>
</PropertyGroup>
<Import Project="python.props" />
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<PropertyGroup>
<MakeVersionInfoBeforeTarget>ClCompile</MakeVersionInfoBeforeTarget>
</PropertyGroup>
<ImportGroup Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="pyproject.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>_WINDOWS;VENV_REDIRECT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>PYW_ICON;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ResourceCompile>
<Link>
<AdditionalDependencies>version.lib;%(AdditionalDependencies)</AdditionalDependencies>
<SubSystem>Windows</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\PC\launcher.c" />
</ItemGroup>
<ItemGroup>
<None Include="..\PC\launcher.ico" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\PC\pylauncher.rc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
Tools/msi/lib/lib_files.wxs
View file @ 1c3de541
......@@ -2,6 +2,8 @@
<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
<?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_msi;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_contextvars ?>
<Fragment>
<DirectoryRef Id="Lib_venv_scripts_nt" />
<ComponentGroup Id="lib_extensions">
<?foreach ext in $(var.exts)?>
......@@ -20,10 +22,25 @@
<Component Id="libssl.dll" Directory="DLLs" Guid="*">
<File Name="libssl$(var.ssltag).dll" KeyPath="yes" />
</Component>
<Component Id="venvlauncher.exe" Directory="Lib_venv_scripts_nt" Guid="*">
<File Name="python.exe" Source="venvlauncher.exe" KeyPath="yes" />
</Component>
<Component Id="venvwlauncher.exe" Directory="Lib_venv_scripts_nt" Guid="*">
<File Name="pythonw.exe" Source="venvwlauncher.exe" KeyPath="yes" />
</Component>
</ComponentGroup>
</Fragment>
<Fragment>
<!-- The auto-generated directory is not available when building symbols -->
<DirectoryRef Id="Lib">
<Directory Id="Lib_venv__pdbs" Name="venv">
<Directory Id="Lib_venv_scripts__pdbs" Name="scripts">
<Directory Id="Lib_venv_scripts_nt__pdbs" Name="nt" />
</Directory>
</Directory>
</DirectoryRef>
<ComponentGroup Id="lib_extensions_symbols">
<?foreach ext in $(var.exts)?>
......@@ -42,6 +59,12 @@
<Component Id="libssl.pdb" Directory="DLLs" Guid="*">
<File Name="libssl$(var.ssltag).pdb" KeyPath="yes" />
</Component>
<Component Id="venvlauncher.pdb" Directory="Lib_venv_scripts_nt__pdbs" Guid="*">
<File Name="python.pdb" Source="venvlauncher.pdb" KeyPath="yes" />
</Component>
<Component Id="venvwlauncher.pdb" Directory="Lib_venv_scripts_nt__pdbs" Guid="*">
<File Name="pythonw.pdb" Source="venvwlauncher.pdb" KeyPath="yes" />
</Component>
</ComponentGroup>
</Fragment>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7