Issue #21907: Improved the batch scripts provided for building Python.

The user-facing scripts in PCbuild have been updated to be easier to use
and the buildbot scripts in Tools\buildbot have been updated to use the
user-facing scripts in PCbuild wherever possible.

Misc/NEWS

@@ -746,6 +746,8 @@ Tools/Demos
 Windows
 -------
 
+- Issue #21907: Improved the batch scripts provided for building Python.
+
 - Issue #21671, CVE-2014-0224: The bundled version of OpenSSL has been
   updated to 1.0.1h.
 

PCbuild/build.bat

 @echo off
-rem A batch program to build or rebuild a particular configuration.
+rem A batch program to build or rebuild a particular configuration,
 rem just for convenience.
 
+rem Arguments:
+rem  -c  Set the configuration (default: Release)
+rem  -p  Set the platform (x64 or Win32, default: Win32)
+rem  -r  Target Rebuild instead of Build
+rem  -d  Set the configuration to Debug
+rem  -e  Pull in external libraries using get_externals.bat
+
 setlocal
 set platf=Win32
 set conf=Release
-set target=build
+set target=Build
 set dir=%~dp0
 
 :CheckOpts
 if "%1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts
 if "%1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts
-if "%1"=="-r" (set target=rebuild) & shift & goto CheckOpts
+if "%1"=="-r" (set target=Rebuild) & shift & goto CheckOpts
 if "%1"=="-d" (set conf=Debug) & shift & goto CheckOpts
+if "%1"=="-e" call "%dir%get_externals.bat" & shift & goto CheckOpts
+
+if "%platf%"=="x64" (set vs_platf=x86_amd64)
+
+rem Setup the environment
+call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %vs_platf%
 
-set cmd=msbuild /p:useenv=true %dir%pcbuild.sln /t:%target% /p:Configuration=%conf% /p:Platform=%platf%
-echo %cmd%
-%cmd%
+rem Call on MSBuild to do the work, echo the command.
+rem Passing %1-9 is not the preferred option, but argument parsing in
+rem batch is, shall we say, "lackluster"
+echo on
+msbuild "%dir%pcbuild.sln" /t:%target% /p:Configuration=%conf% /p:Platform=%platf% %1 %2 %3 %4 %5 %6 %7 %8 %9

PCbuild/build_pgo.bat

@@ -9,12 +9,12 @@ setlocal
 set platf=Win32
 
 rem use the performance testsuite.  This is quick and simple
-set job1=..\tools\pybench\pybench.py -n 1 -C 1 --with-gc
-set path1=..\tools\pybench
+set job1=%~dp0..\tools\pybench\pybench.py -n 1 -C 1 --with-gc
+set path1=%~dp0..\tools\pybench
 
 rem or the whole testsuite for more thorough testing
-set job2=..\lib\test\regrtest.py
-set path2=..\lib
+set job2=%~dp0..\lib\test\regrtest.py
+set path2=%~dp0..\lib
 
 set job=%job1%
 set clrpath=%path1%
@@ -31,9 +31,9 @@ rem build the instrumented version
 call build -p %platf% -c PGInstrument
 
 rem remove .pyc files, .pgc files and execute the job
-%PGI%\python.exe rmpyc.py %clrpath%
+%PGI%\python.exe rmpyc.py "%clrpath%"
 del %PGI%\*.pgc
-%PGI%\python.exe %job%
+%PGI%\python.exe "%job%"
 
 rem finally build the optimized version
 if exist %PGO% del /s /q %PGO%

PCbuild/env.bat

 @echo off
-set VS10=%ProgramFiles(x86)%\Microsoft Visual Studio 10.0
-IF EXIST "%VS10%" GOTO ok
-set VS10=%ProgramFiles%\Microsoft Visual Studio 10.0
-:ok
 
 echo Build environments: x86, ia64, amd64, x86_amd64, x86_ia64
 echo.
-call "%VS10%\VC\vcvarsall.bat" %1
+call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %1

PCbuild/get_externals.bat

+@echo off
+setlocal
+rem Simple script to fetch source for external libraries
+
+pushd "%~dp0..\.."
+
+if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+
+rem Optionally clean up first.  Be warned that this can be very destructive!
+if not "%1"=="" (
+    for %%c in (-c --clean --clean-only) do (
+        if "%1"=="%%c" goto clean
+    )
+    goto usage
+)
+goto fetch
+
+:clean
+echo.Cleaning up external libraries.
+for /D %%d in (
+               bzip2-*
+               db-*
+               openssl-*
+               tcl-*
+               tcltk*
+               tk-*
+               tix-*
+               sqlite-*
+               xz-*
+               ) do (
+    echo.Removing %%d
+    rmdir /s /q %%d
+)
+if "%1"=="--clean-only" (
+    goto end
+)
+
+:fetch
+rem Fetch current versions
+
+svn --version > nul 2>&1
+if ERRORLEVEL 9009 (
+    echo.svn.exe must be on your PATH.
+    echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
+    echo.command line tools option.
+    popd
+    exit /b 1
+)
+
+echo.Fetching external libraries...
+
+for %%e in (
+            bzip2-1.0.6
+            openssl-1.0.1h
+            tcl-8.6.1.0
+            tk-8.6.1.0
+            tix-8.4.3.4
+            sqlite-3.8.3.1
+            xz-5.0.5
+            ) do (
+    if exist %%e (
+        echo.%%e already exists, skipping.
+    ) else (
+        echo.Fetching %%e...
+        svn export %SVNROOT%%%e
+    )
+)
+
+goto end
+
+:usage
+echo.invalid argument: %1
+echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
+echo.
+echo.Pull all sources necessary for compiling optional extension modules
+echo.that rely on external libraries.  Requires svn.exe to be on your PATH
+echo.and pulls sources from %SVNROOT%.
+echo.
+echo.Use the -c or --clean option to clean up all external library sources
+echo.before pulling in the current versions.
+echo.
+echo.Use the --clean-only option to do the same cleaning, without pulling in
+echo.anything new.
+echo.
+echo.Only the first argument is checked, all others are ignored.
+echo.
+echo.**WARNING**: the cleaning options unconditionally remove any directory
+echo.that is a child of
+echo.   %CD%
+echo.and matches wildcard patterns beginning with bzip2-, db-, openssl-, tcl-,
+echo.tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential to be
+echo.very destructive if you are not aware of what it is doing.  Use with
+echo.caution!
+popd
+exit /b -1
+
+
+:end
+echo Finished.
+popd

PCbuild/readme.txt

+Quick Start Guide
+-----------------
+
+1.  Install Microsoft Visual C++ 2010 SP1, any edition.
+2.  Install Subversion, and make sure 'svn.exe' is on your PATH.
+3.  Install NASM, and make sure 'nasm.exe' is on your PATH.
+4.  Run "build.bat -e" to build Python in 32-bit Release configuration.
+5.  (Optional, but recommended) Run the test suite with "rt.bat -q".
+
+
 Building Python using Microsoft Visual C++
 ------------------------------------------
 
@@ -24,8 +34,8 @@ All you need to do to build is open the solution "pcbuild.sln" in Visual
 Studio, select the desired combination of configuration and platform,
 then build with "Build Solution" or the F7 keyboard shortcut.  You can
 also build from the command line using the "build.bat" script in this
-directory.  The solution is configured to build the projects in the
-correct order.
+directory; see below for details.  The solution is configured to build
+the projects in the correct order.
 
 The solution currently supports two platforms.  The Win32 platform is
 used to build standard x86-compatible 32-bit binaries, output into this
@@ -56,6 +66,26 @@ Release
     settings, though without PGO.
 
 
+Building Python using the build.bat script
+----------------------------------------------
+
+In this directory you can find build.bat, a script designed to make
+building Python on Windows simpler.  The only absolute requirement for
+using this script is for the VS100COMNTOOLS environment variable to be
+properly set, which should be done by Microsoft Visual C++ 2010
+installation.
+
+By default, build.bat will build Python in Release configuration for
+the 32-bit Win32 platform.  It accepts several arguments to change
+this behavior:
+
+   -c <configuration>  Set the configuration (see above)
+   -d                  Shortcut for "-c Debug"
+   -p <platform>       Set the platform to build for ("Win32" or "x64")
+   -r                  Rebuild instead of just building
+   -e                  Use get_externals.bat to fetch external sources
+
+
 Legacy support
 --------------
 
@@ -227,25 +257,18 @@ Getting External Sources
 The last category of sub-projects listed above wrap external projects
 Python doesn't control, and as such a little more work is required in
 order to download the relevant source files for each project before they
-can be built.  The buildbots must ensure that all libraries are present
-before building, so the easiest approach is to run either external.bat
-or external-amd64.bat (depending on platform) in the ..\Tools\buildbot
-directory from ..\, i.e.:
-
-    C:\python\cpython\PCbuild>cd ..
-    C:\python\cpython>Tools\buildbot\external.bat
-
-This extracts all the external sub-projects from
+can be built.  However, a simple script is provided to make this as
+painless as possible, called "get_externals.bat" and located in this
+directory.  This script extracts all the external sub-projects from
     http://svn.python.org/projects/external
-via Subversion (so you'll need an svn.exe on your PATH) and places them
+via Subversion (so you'll need svn.exe on your PATH) and places them
 in ..\.. (relative to this directory).
 
 It is also possible to download sources from each project's homepage,
-though you may have to change the names of some folders in order to make
-things work.  For instance, if you were to download a version 5.0.7 of
-XZ Utils, you would need to extract the archive into ..\..\xz-5.0.5
-anyway, since that is where the solution is set to look for xz.  The
-same is true for all other external projects.
+though you may have to change folder names or pass the names to MSBuild
+as the values of certain properties in order for the build solution to
+find them.  This is an advanced topic and not necessarily fully
+supported.
 
 
 Building for AMD64

Tools/buildbot/build-amd64.bat

 @rem Used by the buildbot "compile" step.
-cmd /c Tools\buildbot\external-amd64.bat
-call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
-cmd /c Tools\buildbot\clean-amd64.bat
-
-msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64
+call "%~dp0build.bat" -p x64 %*

Tools/buildbot/build.bat

 @rem Used by the buildbot "compile" step.
-cmd /c Tools\buildbot\external.bat
-call "%VS100COMNTOOLS%vsvars32.bat"
-cmd /c Tools\buildbot\clean.bat
 
-msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32
+@rem Clean up
+call "%~dp0clean.bat"
 
+@rem If you need the buildbots to start fresh (such as when upgrading to
+@rem a new version of an external library, especially Tcl/Tk):
+@rem 1) uncomment the following line:
+
+@rem    call "%~dp0..\..\PCbuild\get_externals.bat" --clean-only
+
+@rem 2) commit and push
+@rem 3) wait for all Windows bots to start a build with that changeset
+@rem 4) re-comment, commit and push again
+
+@rem Do the build
+call "%~dp0..\..\PCbuild\build.bat" -e -d %*

Tools/buildbot/buildmsi.bat

 @rem Used by the buildbot "buildmsi" step.
+setlocal
 
-cmd /c Tools\buildbot\external.bat
+set cwd=%CD%
 @rem build release versions of things
-call "%VS100COMNTOOLS%vsvars32.bat"
-
-@rem build Python
-msbuild /p:useenv=true PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32
+call "%~dp0build.bat" -c Release
 
 @rem build the documentation
-bash.exe -c 'cd Doc;make PYTHON=python2.5 update htmlhelp'
-"%ProgramFiles%\HTML Help Workshop\hhc.exe" Doc\build\htmlhelp\python26a3.hhp
+call "%~dp0..\..\Doc\make.bat" htmlhelp
 
 @rem build the MSI file
-cd PC
+call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" x86
+cd "%~dp0..\..\PC"
 nmake /f icons.mak
 cd ..\Tools\msi
 del *.msi
 nmake /f msisupport.mak
 %HOST_PYTHON% msi.py
 
+cd "%cwd%"

Tools/buildbot/clean-amd64.bat

 @rem Used by the buildbot "clean" step.
-call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
-@echo Deleting .pyc/.pyo files ...
-del /s Lib\*.pyc Lib\*.pyo
-@echo Deleting test leftovers ...
-rmdir /s /q build
-cd PCbuild
-msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64
-msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64
-cd ..
+call "%~dp0clean.bat"

Tools/buildbot/clean.bat

-@rem Used by the buildbot "clean" step.
-call "%VS100COMNTOOLS%vsvars32.bat"
-@echo Deleting test leftovers ...
-rmdir /s /q build
-cd PCbuild
-msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x86
-msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x86
-cd ..
+@echo off
+rem Used by the buildbot "clean" step.
+
+setlocal
+set root=%~dp0..\..
+set pcbuild=%root%\PCbuild
+
+echo.Attempting to kill Pythons...
+for %%k in (kill_python.exe
+            kill_python_d.exe
+            amd64\kill_python.exe
+            amd64\kill_python_d.exe
+            ) do (
+    if exist "%pcbuild%\%%k" (
+        echo.Calling %pcbuild%\%%k...
+        "%pcbuild%\%%k"
+    )
+)
+
+echo Purging all non-tracked files with `hg purge`
+echo on
+hg -R "%root%" --config extensions.purge= purge --all -X "%root%\Lib\test\data"

Tools/buildbot/external-amd64.bat

-@rem Fetches (and builds if necessary) external dependencies
-
-@rem Assume we start inside the Python source directory
-call "Tools\buildbot\external-common.bat"
-
-
+@echo Please use PCbuild\get_externals.bat instead.
+@"%~dp0..\..\PCbuild\get_externals.bat" %*

Tools/buildbot/external-common.bat

-@rem Common file shared between external.bat and external-amd64.bat.  Responsible for
-@rem fetching external components into the root\.. buildbot directories.
-
-cd ..
-@rem XXX: If you need to force the buildbots to start from a fresh environment, uncomment
-@rem the following, check it in, then check it out, comment it out, then check it back in.
-@rem if exist bzip2-1.0.6 rd /s/q bzip2-1.0.6
-@rem if exist tcltk rd /s/q tcltk
-@rem if exist tcltk64 rd /s/q tcltk64
-@rem if exist tcl-8.6.1.0 rd /s/q tcl-8.6.1.0
-@rem if exist tk-8.6.1.0 rd /s/q tk-8.6.1.0
-@rem if exist tix-8.4.3.4 rd /s/q tix-8.4.3.4
-@rem if exist db-4.4.20 rd /s/q db-4.4.20
-@rem if exist openssl-1.0.1h rd /s/q openssl-1.0.1h
-@rem if exist sqlite-3.7.12 rd /s/q sqlite-3.7.12
-
-@rem bzip
-if not exist bzip2-1.0.6 (
-   rd /s/q bzip2-1.0.5
-  svn export http://svn.python.org/projects/external/bzip2-1.0.6
-)
-
-@rem OpenSSL
-if not exist openssl-1.0.1h (
-    rd /s/q openssl-1.0.1g
-    svn export http://svn.python.org/projects/external/openssl-1.0.1h
-)
-
-@rem tcl/tk/tix
-if not exist tcl-8.6.1.0 (
-   rd /s/q tcltk tcltk64 tcl-8.5.11.0 tk-8.5.11.0 tix-8.4.3.3
-   svn export http://svn.python.org/projects/external/tcl-8.6.1.0
-)
-if not exist tk-8.6.1.0 svn export http://svn.python.org/projects/external/tk-8.6.1.0
-if not exist tix-8.4.3.4 svn export http://svn.python.org/projects/external/tix-8.4.3.4
-
-@rem sqlite3
-if not exist sqlite-3.8.3.1 (
-  rd /s/q sqlite-source-3.8.1
-  svn export http://svn.python.org/projects/external/sqlite-3.8.3.1
-)
-
-@rem lzma
-if not exist xz-5.0.5 (
-  rd /s/q xz-5.0.3
-  svn export http://svn.python.org/projects/external/xz-5.0.5
-)

Tools/buildbot/external.bat

-@rem Fetches (and builds if necessary) external dependencies
-
-@rem Assume we start inside the Python source directory
-call "Tools\buildbot\external-common.bat"
-
+@echo Please use PCbuild\get_externals.bat instead.
+@"%~dp0..\..\PCbuild\get_externals.bat" %*

Tools/buildbot/test-amd64.bat

 @rem Used by the buildbot "test" step.
 
 rem The following line should be removed before #20035 is closed
-set TCL_LIBRARY=%CD%\..\tcltk64\lib\tcl8.6
-cd PCbuild
-call rt.bat -d -q -x64 -uall -rwW -n --timeout=3600 %1 %2 %3 %4 %5 %6 %7 %8 %9
+set TCL_LIBRARY=%~dp0..\..\..\tcltk64\lib\tcl8.6
+
+"%~dp0..\..\PCbuild\amd64\python_d.exe" "%~dp0..\scripts\run_tests.py" -j 1 -u all -W --timeout=3600 %*

Tools/buildbot/test.bat

 @rem Used by the buildbot "test" step.
 
 rem The following line should be removed before #20035 is closed
-set TCL_LIBRARY=%CD%\..\tcltk\lib\tcl8.6
-cd PCbuild
-call rt.bat -d -q -uall -rwW -n --timeout=3600 %1 %2 %3 %4 %5 %6 %7 %8 %9
+set TCL_LIBRARY=%~dp0..\..\..\tcltk\lib\tcl8.6
+
+"%~dp0..\..\PCbuild\python_d.exe" "%~dp0..\scripts\run_tests.py" -j 1 -u all -W --timeout=3600 %*

Tools/scripts/run_tests.py

@@ -48,7 +48,11 @@ def main(regrtest_args):
         args.extend(['-u', 'all,-largefile,-audio,-gui'])
     args.extend(regrtest_args)
     print(' '.join(args))
-    os.execv(sys.executable, args)
+    if sys.platform == 'win32':
+        from subprocess import call
+        call(args)
+    else:
+        os.execv(sys.executable, args)
 
 
 if __name__ == '__main__':