tests now pass in Windows (and still in Linux)

bootstrap/bootstrap.py

@@ -110,8 +110,10 @@ try:
     import setuptools # A flag.  Sometimes pkg_resources is installed alone.
     import pkg_resources
 except ImportError:
+    ez_code = urllib2.urlopen(
+        configuration['--ez_setup-source']).read().replace('\r\n', '\n')
     ez = {}
-    exec urllib2.urlopen(configuration['--ez_setup-source']).read() in ez
+    exec ez_code in ez
     setuptools_args = dict(to_dir=configuration['--eggs'], download_delay=0)
     if configuration['--download-base']:
         setuptools_args['download_base'] = configuration['--download-base']

src/zc/buildout/easy_install.py

@@ -1300,13 +1300,16 @@ def _get_system_paths(executable):
         cmd.extend([
             "-c", "import sys, os;"
             "print repr([os.path.normpath(p) for p in sys.path if p])"])
+        # Windows needs some (as yet to be determined) part of the real env.
+        env = os.environ.copy()
+        env.update(kwargs)
         _proc = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=kwargs)
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         stdout, stderr = _proc.communicate();
         if _proc.returncode:
             raise RuntimeError(
                 'error trying to get system packages:\n%s' % (stderr,))
-        res = eval(stdout)
+        res = eval(stdout.strip())
         try:
             res.remove('.')
         except ValueError:
@@ -1506,10 +1509,52 @@ def _generate_interpreter(name, dest, executable, site_py_dest,
     full_name = os.path.join(dest, name)
     site_py_dest_string, rpsetup = _relative_path_and_setup(
         full_name, [site_py_dest], relative_paths)
+    if sys.platform == 'win32':
+        windows_import = '\nimport subprocess'
+        # os.exec* is a mess on Windows, particularly if the path
+        # to the executable has spaces and the Python is using MSVCRT.
+        # The standard fix is to surround the executable's path with quotes,
+        # but that has been unreliable in testing.
+        #
+        # Here's a demonstration of the problem.  Given a Python
+        # compiled with a MSVCRT-based compiler, such as the free Visual
+        # C++ 2008 Express Edition, and an executable path with spaces
+        # in it such as the below, we see the following.
+        #
+        # >>> import os
+        # >>> p0 = 'C:\\Documents and Settings\\Administrator\\My Documents\\Downloads\\Python-2.6.4\\PCbuild\\python.exe'
+        # >>> os.path.exists(p0)
+        # True
+        # >>> os.execv(p0, [])
+        # Traceback (most recent call last):
+        #  File "<stdin>", line 1, in <module>
+        # OSError: [Errno 22] Invalid argument
+        #
+        # That seems like a standard problem.  The standard solution is
+        # to quote the path (see, for instance
+        # http://bugs.python.org/issue436259).  However, this solution,
+        # and other variations, fail:
+        #
+        # >>> p1 = '"C:\\Documents and Settings\\Administrator\\My Documents\\Downloads\\Python-2.6.4\\PCbuild\\python.exe"'
+        # >>> os.execv(p1, [])
+        # Traceback (most recent call last):
+        #   File "<stdin>", line 1, in <module>
+        # OSError: [Errno 22] Invalid argument
+        #
+        # We simply use subprocess instead, since it handles everything
+        # nicely, and the transparency of exec* (that is, not running,
+        # perhaps unexpectedly, in a subprocess) is arguably not a
+        # necessity, at least for many use cases.
+        execute = 'subprocess.call(argv, env=environ)'
+    else:
+        windows_import = ''
+        execute = 'os.execve(sys.executable, argv, environ)'
     contents = interpreter_template % dict(
-        python = _safe_arg(executable),
-        site_dest = site_py_dest_string,
-        relative_paths_setup = rpsetup,
+        python=_safe_arg(executable),
+        site_dest=site_py_dest_string,
+        relative_paths_setup=rpsetup,
+        windows_import=windows_import,
+        execute=execute,
         )
     return _write_script(full_name, contents, 'interpreter')
 
@@ -1517,7 +1562,7 @@ interpreter_template = script_header + '''\
 
 %(relative_paths_setup)s
 import os
-import sys
+import sys%(windows_import)s
 
 argv = [sys.executable] + sys.argv[1:]
 environ = os.environ.copy()
@@ -1525,7 +1570,7 @@ path = %(site_dest)s
 if environ.get('PYTHONPATH'):
     path = os.pathsep.join([path, environ['PYTHONPATH']])
 environ['PYTHONPATH'] = path
-os.execve(sys.executable, argv, environ)
+%(execute)s
 '''
 
 # End of script generation code.

src/zc/buildout/easy_install.txt

@@ -1094,12 +1094,11 @@ in both site packages and eggs.
 
 Here are some examples of the interpreter in use.
 
-    >>> print system(interpreter_path + ' -c "print 16+26"')
+    >>> print call_py(interpreter_path, "print 16+26")
     42
     <BLANKLINE>
-    >>> res = system(interpreter_path +
-    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
-    >>> print res # doctest: +ELLIPSIS
+    >>> res = call_py(interpreter_path, "import sys; print sys.path")
+    >>> print res # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     ['',
      '/interpreter/eggs/demo-0.3-pyN.N.egg',
      '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
@@ -1129,8 +1128,7 @@ If you provide initialization, it goes in sitecustomize.py.
     >>> cat(sitecustomize_path)
     import os
     os.environ['FOO'] = 'bar baz bing shazam'
-    >>> print system(interpreter_path +
-    ...              """ -c 'import os; print os.environ["FOO"]'""")
+    >>> print call_py(interpreter_path, "import os; print os.environ['FOO']")
     bar baz bing shazam
     <BLANKLINE>
 
@@ -1182,8 +1180,8 @@ paths join a base to a path, as with the use of this argument in the
 
 The paths resolve in practice as you would expect.
 
-    >>> print system(interpreter_path +
-    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+    >>> print call_py(interpreter_path,
+    ...               "import sys, pprint; pprint.pprint(sys.path)")
     ... # doctest: +ELLIPSIS
     ['',
      '/interpreter/eggs/demo-0.3-py2.4.egg',
@@ -1210,8 +1208,8 @@ The ``extra_paths`` argument affects the path in site.py.  Notice that
             '/interpreter/other'
             ]...
 
-    >>> print system(interpreter_path +
-    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+    >>> print call_py(interpreter_path,
+    ...               "import sys, pprint; pprint.pprint(sys.path)")
     ... # doctest: +ELLIPSIS
     ['',
      '/interpreter/eggs/demo-0.3-pyN.N.egg',
@@ -1355,9 +1353,8 @@ Here's an example of the new script in use.  Other documents and tests in
 this package give the feature a more thorough workout, but this should
 give you an idea of the feature.
 
-    >>> res = system(join(interpreter_dir, 'bin', 'py') +
-    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
-    >>> print res # doctest: +ELLIPSIS
+    >>> res = call_py(interpreter_path, "import sys; print sys.path")
+    >>> print res # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     ['',
      '/interpreter/eggs/demo-0.3-py2.4.egg',
      '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
@@ -1434,7 +1431,10 @@ The demo script runs the entry point defined in the demo egg:
     if __name__ == '__main__':
         eggrecipedemo.main()
 
-    >>> print system(join(interpreter_bin_dir, 'demo'))
+    >>> demo_call = join(interpreter_bin_dir, 'demo')
+    >>> if sys.platform == 'win32':
+    ...     demo_call = '"%s"' % demo_call
+    >>> print system(demo_call)
     3 1
     <BLANKLINE>
 

src/zc/buildout/testing.py

@@ -106,6 +106,16 @@ def system(command, input=''):
     e.close()
     return result
 
+def call_py(interpreter, cmd, flags=None):
+    if sys.platform == 'win32':
+        args = ['"%s"' % arg for arg in (interpreter, flags, cmd) if arg]
+        args.insert(-1, '"-c"')
+        return system('"%s"' % ' '.join(args))
+    else:
+        cmd = repr(cmd)
+        return system(
+            ' '.join(arg for arg in (interpreter, flags, '-c', cmd) if arg))
+
 def get(url):
     return urllib2.urlopen(url).read()
 
@@ -336,6 +346,7 @@ def buildoutSetUp(test):
         tmpdir = tmpdir,
         write = write,
         system = system,
+        call_py = call_py,
         get = get,
         cd = (lambda *path: os.chdir(os.path.join(*path))),
         join = os.path.join,
@@ -536,10 +547,14 @@ def _normalize_path(match):
             path = path[1:]
     return '/' + path.replace(os.path.sep, '/')
 
+if sys.platform == 'win32':
+    sep = r'[\\/]' # Windows uses both sometimes.
+else:
+    sep = re.escape(os.path.sep)
 normalize_path = (
     re.compile(
-        r'''[^'" \t\n\r]+\%(sep)s_[Tt][Ee][Ss][Tt]_\%(sep)s([^"' \t\n\r]+)'''
-        % dict(sep=os.path.sep)),
+        r'''[^'" \t\n\r]+%(sep)s_[Tt][Ee][Ss][Tt]_%(sep)s([^"' \t\n\r]+)'''
+        % dict(sep=sep)),
     _normalize_path,
     )
 

src/zc/buildout/tests.py

@@ -1794,11 +1794,9 @@ package, so this demonstrates that our Python does in fact have demo
 version 0.3 and demoneeded version 1.1.
 
     >>> py_path = make_py_with_system_install(make_py, sample_eggs)
-    >>> print system(
-    ...     py_path + " -c '" +
-    ...     "import tellmy.version\n" +
-    ...     "print tellmy.version.__version__\n" +
-    ...     "'"),
+    >>> print call_py(
+    ...     py_path,
+    ...     "import tellmy.version; print tellmy.version.__version__"),
     1.1
 
 Now here's a setup that would expose the bug, using the
@@ -1832,11 +1830,9 @@ To demonstrate this, we will create three packages: tellmy.version 1.0,
 tellmy.version 1.1, and tellmy.fortune 1.0.  tellmy.version 1.1 is installed.
 
     >>> py_path = make_py_with_system_install(make_py, sample_eggs)
-    >>> print system(
-    ...     py_path + " -c '" +
-    ...     "import tellmy.version\n" +
-    ...     "print tellmy.version.__version__\n" +
-    ...     "'")
+    >>> print call_py(
+    ...     py_path,
+    ...     "import tellmy.version; print tellmy.version.__version__")
     1.1
     <BLANKLINE>
 
@@ -1895,13 +1891,12 @@ In other words, we got the site-packages version of tellmy.version, and
 we could not import tellmy.fortune at all.  The following are the correct
 results for the interpreter and for the script.
 
-    >>> print system(
-    ...     join('bin', 'py') + " -c '" +
-    ...     "import tellmy.version\n" +
-    ...     "print tellmy.version.__version__\n" +
-    ...     "import tellmy.fortune\n" +
-    ...     "print tellmy.fortune.__version__\n" +
-    ...     "'") # doctest: +ELLIPSIS
+    >>> print call_py(
+    ...     join('bin', 'py'),
+    ...     "import tellmy.version; " +
+    ...     "print tellmy.version.__version__; " +
+    ...     "import tellmy.fortune; " +
+    ...     "print tellmy.fortune.__version__") # doctest: +ELLIPSIS
     1.0
     1.0...
 
@@ -1961,11 +1956,9 @@ these unpleasant tricks, and a Python that has an older version installed.
     ...             zc.buildout.testing.sys_install(tmp, site_packages_path)
     ...     finally:
     ...         shutil.rmtree(tmp)
-    >>> print system(
-    ...     py_path + " -c '" +
-    ...     "import tellmy.version\n" +
-    ...     "print tellmy.version.__version__\n" +
-    ...     "'")
+    >>> print call_py(
+    ...     py_path,
+    ...     "import tellmy.version; print tellmy.version.__version__")
     1.0
     <BLANKLINE>
     >>> write('buildout.cfg',
@@ -3161,7 +3154,7 @@ def test_suite():
                 'We have a develop egg: zc.buildout X.X.'),
                (re.compile(r'\\[\\]?'), '/'),
                (re.compile('WindowsError'), 'OSError'),
-               (re.compile(r'\[Error 17\] Cannot create a file '
+               (re.compile(r'\[Error \d+\] Cannot create a file '
                            r'when that file already exists: '),
                 '[Errno 17] File exists: '
                 ),
@@ -3215,6 +3208,10 @@ def test_suite():
                (re.compile('[-d]  setuptools-\S+[.]egg'), 'setuptools.egg'),
                (re.compile(r'\\[\\]?'), '/'),
                (re.compile(r'\#!\S+\bpython\S*'), '#!/usr/bin/python'),
+               # Normalize generate_script's Windows interpreter to UNIX:
+               (re.compile(r'\nimport subprocess\n'), '\n'),
+               (re.compile('subprocess\\.call\\(argv, env=environ\\)'),
+                'os.execve(sys.executable, argv, environ)'),
                ]+(sys.version_info < (2, 5) and [
                   (re.compile('.*No module named runpy.*', re.S), ''),
                   (re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),

z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt

@@ -138,7 +138,8 @@ You can also generate an interpreter alone with the ``interpreter`` recipe.
     Generated interpreter '/sample-buildout/bin/py'.
 
 In both cases, the bin/py script works by restarting Python after
-specifying a special path in PYTHONPATH.
+specifying a special path in PYTHONPATH.  This example shows the UNIX version;
+the Windows version actually uses subprocess instead.
 
     >>> cat(sample_buildout, 'bin', 'py') # doctest: +NORMALIZE_WHITESPACE
     #!/usr/bin/python2.4 -S

z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py

@@ -270,6 +270,10 @@ def test_suite():
                (re.compile(r'eggs\\\\demo'), 'eggs/demo'),
                (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
                (re.compile(r'\#!\S+\bpython\S*'), '#!/usr/bin/python'),
+               # Normalize generate_script's Windows interpreter to UNIX:
+               (re.compile(r'\nimport subprocess\n'), '\n'),
+               (re.compile('subprocess\\.call\\(argv, env=environ\\)'),
+                'os.execve(sys.executable, argv, environ)'),
                ])
             ),
         doctest.DocTestSuite(
@@ -279,6 +283,7 @@ def test_suite():
                 zc.buildout.testing.normalize_path,
                 zc.buildout.testing.normalize_endings,
                 zc.buildout.testing.normalize_egg_py,
+                (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
                 ]),
             ),