[2.7] bpo-30450: Pull Windows dependencies from GitHub rather than SVN (GH-1783) (GH-3306)

The Windows build now depends on Python 3.6 to fetch externals, but it
will be downloaded via NuGet (which is downloaded via PowerShell) if it
is not available via `py -3.6`.  This means the only thing that must be
installed on a modern Windows box to do a full build of CPython with all
extensions is Visual Studio.

Cherry-picked from 51599e2b, parts of 40a23e88, parts of 68d663cf, d5cd21d7, and possibly others that I've missed.

Also:

* Rename db -> bsddb for disambiguity

* Update sqlite3 to 3.14.2.0 since it's the version we use on 3.x, and it's simpler to just use it than to also upload the old version to cpython-source-deps

* Add PCbuild/*.ilk to .gitignore

.github/appveyor.yml

@@ -10,6 +10,8 @@ build_script:
 - cmd: PCbuild\python.exe -m test.pythoninfo
 test_script:
 - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest -j2
+environment:
+  HOST_PYTHON: C:\Python36\python.exe
 
 # Only trigger AppVeyor if actual code or its configuration changes
 only_commits:

.gitignore

@@ -44,6 +44,7 @@ PC/*/*.suo
 PC/*/Win32-temp-*
 PC/*/x64-temp-*
 PC/*/amd64
+PCbuild/*.ilk
 PCbuild/*.user
 PCbuild/*.suo
 PCbuild/*.*sdf

Doc/make.bat

@@ -5,8 +5,19 @@ pushd %~dp0
 
 set this=%~n0
 
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
-if "%PYTHON%" EQU "" set PYTHON=py
+call ..\PCBuild\find_python.bat %PYTHON%
+if not defined SPHINXBUILD if defined PYTHON (
+    %PYTHON% -c "import sphinx" > nul 2> nul
+    if errorlevel 1 (
+        echo Installing sphinx with %PYTHON%
+        %PYTHON% -m pip install sphinx
+        if errorlevel 1 exit /B
+    )
+    set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
+)
+
+if not defined PYTHON set PYTHON=py
+if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
 
 if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)%
 if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles%
@@ -73,7 +84,7 @@ goto end
 if NOT "%PAPER%" == "" (
     set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
 )
-cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
+cmd /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*"
 
 if "%1" EQU "htmlhelp" (
     if  not exist "%HTMLHELP%" (

Misc/NEWS.d/next/Windows/2017-09-04-14-00-37.bpo-30450.YwitaJ.rst

+Pull build dependencies from GitHub rather than svn.python.org.

PC/VS9.0/pyproject.vsprops

@@ -50,7 +50,7 @@
 	/>
 	<UserMacro
 		Name="bsddb47Dir"
-		Value="$(externalsDir)\db-4.7.25.0\build_windows"
+		Value="$(externalsDir)\bsddb-4.7.25.0\build_windows"
 	/>
 	<UserMacro
 		Name="bsddb47DepLibs"
@@ -74,7 +74,7 @@
 	/>
 	<UserMacro
 		Name="sqlite3Dir"
-		Value="$(externalsDir)\sqlite-3.8.11.0"
+		Value="$(externalsDir)\sqlite-3.14.2.0"
 	/>
 	<UserMacro
 		Name="bz2Dir"

PCbuild/build.bat

@@ -5,12 +5,15 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
 echo.
 echo.Build CPython from the command line.  Requires the appropriate
 echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
-echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
-echo.given.
 echo.
 echo.After the flags recognized by this script, up to 9 arguments to be passed
 echo.directly to MSBuild may be passed.  If the argument contains an '=', the
-echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`)
+echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`).
+echo.Alternatively you can put extra flags for MSBuild in a file named 
+echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file
+echo.will be picked automatically by MSBuild. Flags put in this file does not
+echo.need to be quoted. You can still use environment variables inside the 
+echo.response file.
 echo.
 echo.Available flags:
 echo.  -h  Display this help message
@@ -47,7 +50,6 @@ exit /b 127
 :Run
 setlocal
 set platf=Win32
-set vs_platf=x86
 set conf=Release
 set target=Build
 set dir=%~dp0
@@ -56,10 +58,6 @@ set verbose=/nologo /v:m
 set kill=
 set do_pgo=
 set pgo_job=-m test.regrtest --pgo
-set on_64_bit=true
-
-rem This may not be 100% accurate, but close enough.
-if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)
 
 :CheckOpts
 if "%~1"=="-h" goto Usage
@@ -89,19 +87,13 @@ if "%IncludeBsddb%"=="" set IncludeBsddb=true
 
 if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
 
-if "%platf%"=="x64" (
-    if "%on_64_bit%"=="true" (
-        rem This ought to always be correct these days...
-        set vs_platf=amd64
-    ) else (
-        if "%do_pgo%"=="true" (
+if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
+    if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
         echo.ERROR: Cannot cross-compile with PGO 
-            echo.    32bit operating system detected, if this is incorrect,
-            echo.    make sure the ProgramFiles(x86^) environment variable is set
+        echo.       32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
+        echo.       and PROCESSOR_ARCHITEW6432 environment variables are correct.
         exit /b 1 
     )
-        set vs_platf=x86_amd64
-    )
 )
 
 if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc"
@@ -109,13 +101,14 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%"
 if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=
 
 rem Setup the environment
-call "%dir%env.bat" %vs_platf% >nul
+call "%dir%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if "%kill%"=="true" call :Kill
 
 if "%do_pgo%"=="true" (
     set conf=PGInstrument
-    call :Build
+    call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
     del /s "%dir%\*.pgc"
     del /s "%dir%\..\Lib\*.pyc"
     echo on
@@ -123,12 +116,13 @@ if "%do_pgo%"=="true" (
     @echo off
     call :Kill
     set conf=PGUpdate
+    set target=Build
 )
 goto Build
 
 :Kill
 echo on
-msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
+%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:KillPython=true
 
@@ -140,7 +134,7 @@ rem Call on MSBuild to do the work, echo the command.
 rem Passing %1-9 is not the preferred option, but argument parsing in
 rem batch is, shall we say, "lackluster"
 echo on
-msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
+%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:IncludeExternals=%IncludeExternals%^
  /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^

PCbuild/find_msbuild.bat

+@rem
+@rem Searches for MSBuild.exe. This is the only tool we need to initiate
+@rem a build, so we no longer search for the full VC toolset.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
+
+:begin_search
+@set MSBUILD=
+
+@rem If msbuild.exe is on the PATH, assume that the user wants that one.
+@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
+@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
+
+@rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
+@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+    @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
+
+@exit /b 1
+
+:found
+@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
+@set _Py_MSBuild_Source=

PCbuild/find_python.bat

+@rem
+@rem Searches for python.exe and may download a private copy from nuget.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
+
+:begin_search
+@set PYTHON=
+
+@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
+@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
+
+@rem If we have Python in externals, use that one
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
+
+@rem If HOST_PYTHON is recent enough, use that
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+
+@rem If py.exe finds a recent enough version, use that one
+@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
+
+@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
+@set _Py_NUGET=%NUGET%
+@set _Py_NUGET_URL=%NUGET_URL%
+@set _Py_HOST_PYTHON=%HOST_PYTHON%
+@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
+@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
+@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
+@if NOT exist "%_Py_NUGET%" (
+    @echo Downloading nuget...
+    @rem NB: Must use single quotes around NUGET here, NOT double!
+    @rem Otherwise, a space in the path would break things
+    @rem If it fails, retry with any available copy of Python
+    @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
+    @if errorlevel 1 (
+        @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
+    )
+)
+@echo Installing Python via nuget...
+@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
+@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
+
+
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=
+@exit /b 1
+
+:found
+@echo Using %PYTHON% (%_Py_Python_Source%)
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=

PCbuild/get_external.py

+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import zipfile
+from urllib.request import urlretrieve
+
+
+def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
+    repo = f'cpython-{"bin" if binary else "source"}-deps'
+    url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
+    reporthook = None
+    if verbose:
+        reporthook = print
+    zip_dir.mkdir(parents=True, exist_ok=True)
+    filename, headers = urlretrieve(
+        url,
+        zip_dir / f'{commit_hash}.zip',
+        reporthook=reporthook,
+    )
+    return filename
+
+
+def extract_zip(externals_dir, zip_path):
+    with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+        zf.extractall(os.fspath(externals_dir))
+        return externals_dir / zf.namelist()[0].split('/')[0]
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument('-v', '--verbose', action='store_true')
+    p.add_argument('-b', '--binary', action='store_true',
+                   help='Is the dependency in the binary repo?')
+    p.add_argument('-O', '--organization',
+                   help='Organization owning the deps repos', default='python')
+    p.add_argument('-e', '--externals-dir', type=pathlib.Path,
+                   help='Directory in which to store dependencies',
+                   default=pathlib.Path(__file__).parent.parent / 'externals')
+    p.add_argument('tag',
+                   help='tag of the dependency')
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    zip_path = fetch_zip(
+        args.tag,
+        args.externals_dir / 'zips',
+        org=args.organization,
+        binary=args.binary,
+        verbose=args.verbose,
+    )
+    final_name = args.externals_dir / args.tag
+    extract_zip(args.externals_dir, zip_path).replace(final_name)
+
+
+if __name__ == '__main__':
+    main()

PCbuild/get_externals.bat

@@ -2,51 +2,38 @@
 setlocal
 rem Simple script to fetch source for external libraries
 
-if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
-pushd "%~dp0..\externals"
+if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
+if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
 
-if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+set DO_FETCH=true
+set DO_CLEAN=false
 
-rem Optionally clean up first.  Be warned that this can be very destructive!
-if not "%1"=="" (
-    for %%c in (-c --clean --clean-only) do (
-        if "%1"=="%%c" goto clean
-    )
-    goto usage
-)
-goto fetch
+:CheckOpts
+if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
+if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
+if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
+if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
+if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
+if "x%~1" NEQ "x" goto usage
 
+if "%DO_CLEAN%"=="false" goto fetch
 :clean
 echo.Cleaning up external libraries.
-for /D %%d in (
-               bzip2-*
-               db-*
-               nasm-*
-               openssl-*
-               tcl-*
-               tcltk*
-               tk-*
-               tix-*
-               sqlite-*
-               xz-*
-               ) do (
-    echo.Removing %%d
-    rmdir /s /q %%d
-)
-if "%1"=="--clean-only" (
-    goto end
+if exist "%EXTERNALS_DIR%" (
+    rem Sometimes this fails the first time; try it twice
+    rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
 )
 
+if "%DO_FETCH%"=="false" goto end
 :fetch
-rem Fetch current versions
-
-svn --version > nul 2>&1
-if ERRORLEVEL 9009 (
-    echo.svn.exe must be on your PATH.
-    echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
-    echo.command line tools option.
-    popd
-    exit /b 1
+
+if "%ORG%"=="" (set ORG=python)
+call "%PCBUILD%find_python.bat" "%PYTHON%"
+
+if "%PYTHON%"=="" (
+    where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1
 )
 
 echo.Fetching external libraries...
@@ -56,52 +43,62 @@ rem files in both this dir and PC\VS9.0
 
 set libraries=
 set libraries=%libraries%                                    bzip2-1.0.6
-if NOT "%IncludeBsddb%"=="false" set libraries=%libraries%   db-4.7.25.0
-if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     nasm-2.11.06
+if NOT "%IncludeBsddb%"=="false" set libraries=%libraries%   bsddb-4.7.25.0
 if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     openssl-1.0.2k
-set libraries=%libraries%                                    sqlite-3.8.11.0
+set libraries=%libraries%                                    sqlite-3.14.2.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-8.5.15.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.5.15.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.5
 
 for %%e in (%libraries%) do (
-    if exist %%e (
+    if exist "%EXTERNALS_DIR%\%%e" (
         echo.%%e already exists, skipping.
+    ) else if "%PYTHON%"=="" (
+        echo.Fetching %%e with git...
+        git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e"
     ) else (
         echo.Fetching %%e...
-        svn export -q %SVNROOT%%%e
+        %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e
+    )
+)
+
+echo.Fetching external binaries...
+
+set binaries=
+set binaries=%binaries%
+if NOT "%IncludeSSL%"=="false" set binaries=%binaries%     nasm-2.11.06
+
+for %%b in (%binaries%) do (
+    if exist "%EXTERNALS_DIR%\%%b" (
+        echo.%%b already exists, skipping.
+    ) else if "%PYTHON%"=="" (
+        echo.Fetching %%b with git...
+        git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b"
+    ) else (
+        echo.Fetching %%b...
+        %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b
     )
 )
 
+echo Finished.
 goto end
 
 :usage
-echo.invalid argument: %1
-echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
+echo.Valid options: -c, --clean, --clean-only, --organization, --python,
+echo.--no-tkinter, --no-openssl
 echo.
-echo.Pull all sources necessary for compiling optional extension modules
-echo.that rely on external libraries.  Requires svn.exe to be on your PATH
-echo.and pulls sources from %SVNROOT%.
+echo.Pull all sources and binaries necessary for compiling optional extension
+echo.modules that rely on external libraries.
 echo.
-echo.Use the -c or --clean option to clean up all external library sources
-echo.before pulling in the current versions.
+echo.The --organization option determines which github organization to download
+echo.from, the --python option determines which Python 3.6+ interpreter to use
+echo.with PCbuild\get_external.py.
+echo.
+echo.Use the -c or --clean option to remove the entire externals directory.
 echo.
 echo.Use the --clean-only option to do the same cleaning, without pulling in
 echo.anything new.
 echo.
-echo.Only the first argument is checked, all others are ignored.
-echo.
-echo.**WARNING**: the cleaning options unconditionally remove any directory
-echo.that is a child of
-echo.   %CD%
-echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
-echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
-echo.to be very destructive if you are not aware of what it is doing.  Use with
-echo.caution!
-popd
 exit /b -1
 
-
 :end
-echo Finished.
-popd

PCbuild/python.props

@@ -32,9 +32,9 @@
 
     <!-- Directories of external projects. tcltk is handled in tcltk.props -->
     <ExternalsDir>$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`))</ExternalsDir>
-    <sqlite3Dir>$(ExternalsDir)sqlite-3.8.11.0\</sqlite3Dir>
+    <sqlite3Dir>$(ExternalsDir)sqlite-3.14.2.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
-    <bsddbDir>$(ExternalsDir)db-4.7.25.0</bsddbDir>
+    <bsddbDir>$(ExternalsDir)bsddb-4.7.25.0</bsddbDir>
     <opensslDir>$(ExternalsDir)openssl-1.0.2k\</opensslDir>
     <opensslIncludeDir>$(opensslDir)include32</opensslIncludeDir>
     <opensslIncludeDir Condition="'$(ArchName)' == 'amd64'">$(opensslDir)include64</opensslIncludeDir>

PCbuild/python.vcxproj

@@ -93,6 +93,8 @@
   </ImportGroup>
   <Target Name="GeneratePythonBat" AfterTargets="AfterBuild">
     <PropertyGroup>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin</_PGOPath>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64</_PGOPath>
       <_Content>@rem This script invokes the most recently built Python with all arguments
 @rem passed through to the interpreter.  This file is generated by the
 @rem build process and any changes *will* be thrown away by the next
@@ -100,6 +102,9 @@
 @rem This is only meant as a convenience for developing CPython
 @rem and using it outside of that context is ill-advised.
 @echo Running $(Configuration)^|$(Platform) interpreter...
+@setlocal
+@set PYTHONHOME=$(PySourcePath)
+$(_PGOPath)
 @"$(OutDir)python$(PyDebugExt).exe" %*
 </_Content>
       <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat'))</_ExistingContent>

PCbuild/readme.txt

@@ -4,9 +4,11 @@ Quick Start Guide
 1.  Install Microsoft Visual Studio 2008, any edition.
 2.  Install Microsoft Visual Studio 2010, any edition, or Windows SDK 7.1
     and any version of Microsoft Visual Studio newer than 2010.
-3.  Install Subversion, and make sure 'svn.exe' is on your PATH.
-4.  Run "build.bat -e" to build Python in 32-bit Release configuration.
-5.  (Optional, but recommended) Run the test suite with "rt.bat -q".
+2a. Optionally install Python 3.6 or later.  If not installed,
+    get_externals.bat (build.bat -e) will download and use Python via
+    NuGet.
+3.  Run "build.bat -e" to build Python in 32-bit Release configuration.
+4.  (Optional, but recommended) Run the test suite with "rt.bat -q".
 
 
 Building Python using MSVC 9.0 via MSBuild
@@ -250,9 +252,16 @@ order to download the relevant source files for each project before they
 can be built.  However, a simple script is provided to make this as
 painless as possible, called "get_externals.bat" and located in this
 directory.  This script extracts all the external sub-projects from
-    http://svn.python.org/projects/external
-via Subversion (so you'll need svn.exe on your PATH) and places them
-in ..\externals (relative to this directory).
+    https://github.com/python/cpython-source-deps
+and
+    https://github.com/python/cpython-bin-deps
+via a Python script called "get_external.py", located in this directory.
+If Python 3.6 or later is not available via the "py.exe" launcher, the
+path or command to use for Python can be provided in the PYTHON_FOR_BUILD
+environment variable, or get_externals.bat will download the latest
+version of NuGet and use it to download the latest "pythonx86" package
+for use with get_external.py.  Everything downloaded by these scripts is
+stored in ..\externals (relative to this directory).
 
 It is also possible to download sources from each project's homepage,
 though you may have to change folder names or pass the names to MSBuild

Tools/nuget/build.bat

@@ -21,6 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO
 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1)
 
 if not defined NUGET where nuget -q || echo Cannot find nuget.exe on PATH and NUGET is not set. && exit /B 1
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 if not defined PYTHON set PYTHON=py -3
 
 @%PYTHON% -c "" >nul 2>nul
@@ -29,7 +31,6 @@ if not defined PYTHON set PYTHON=py -3
     set PYTHON="%D%obj\python\tools\python.exe"
 )
 
-call "%PCBUILD%env.bat" x86
 
 if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%"
 
@@ -38,7 +39,7 @@ if defined BUILDX86 (
     ) else if not exist "%PCBUILD%python.exe" call "%PCBUILD%build.bat" -e
     if errorlevel 1 goto :eof
 
-    msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
+    %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
     if errorlevel 1 goto :eof
 )
 
@@ -47,7 +48,7 @@ if defined BUILDX64 (
     ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e
     if errorlevel 1 goto :eof
 
-    msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
+    %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
     if errorlevel 1 goto :eof
 )