Merge pull request #624 from gevent/appveyor-test

Fix some Windows failures and enable AppVeyor

Makefile

@@ -8,7 +8,6 @@ all: gevent/gevent.corecext.c gevent/gevent.ares.c gevent/gevent._semaphore.c ge
 
 gevent/gevent.corecext.c: gevent/core.ppyx gevent/libev.pxd
 	$(PYTHON) util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx
-	echo                          >> gevent.corecext.c
 	echo '#include "callbacks.c"' >> gevent.corecext.c
 	mv gevent.corecext.* gevent/
 

appveyor.yml

+environment:
+  global:
+    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+    # /E:ON and /V:ON options are not enabled in the batch script intepreter
+    # See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
+
+  matrix:
+
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7.x" # currently 2.7.9
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x" # currently 2.7.9
+      PYTHON_ARCH: "64"
+
+    #- PYTHON: "C:\\Python33"
+    #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
+    #  PYTHON_ARCH: "32"
+
+    #- PYTHON: "C:\\Python33-x64"
+    #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
+    #  PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python34"
+      PYTHON_VERSION: "3.4.x" # currently 3.4.3
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python34-x64"
+      PYTHON_VERSION: "3.4.x" # currently 3.4.3
+      PYTHON_ARCH: "64"
+
+    # Also test a Python version not pre-installed
+    # See: https://github.com/ogrisel/python-appveyor-demo/issues/10
+
+    #- PYTHON: "C:\\Python266"
+    #  PYTHON_VERSION: "2.6.6"
+    #  PYTHON_ARCH: "32"
+
+install:
+  - ECHO "Filesystem root:"
+  - ps: "ls \"C:/\""
+
+  - ECHO "Installed SDKs:"
+  - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
+
+  # Install Python (from the official .msi of http://python.org) and pip when
+  # not already installed.
+  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Install the build dependencies of the project. If some dependencies contain
+  # compiled extensions and are not provided as pre-built wheel packages,
+  # pip will build them from source using the MSVC compiler matching the
+  # target Python version and architecture
+  - "%CMD_IN_ENV% pip install cython greenlet"
+
+  - "python util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx"
+  - "type gevent\\callbacks.c >> gevent.corecext.c"
+  - "move gevent.corecext.* gevent"
+  - "cython -o gevent.ares.c gevent/ares.pyx"
+  - "move gevent.ares.* gevent"
+  - "move gevent\\_semaphore.pyx gevent\\_semaphore.py"
+  - "cython -o gevent._semaphore.c gevent/_semaphore.py"
+  - "move gevent._semaphore.* gevent"
+  - "del gevent\\_semaphore.py"
+  - "cython -o gevent._util.c gevent/_util.pyx"
+  - "move gevent._util.* gevent"
+
+build: false  # Not a C# project, build stuff at the test step instead.
+
+test_script:
+  # Build the compiled extension and run the project tests
+  - "%CMD_IN_ENV% python setup.py develop"
+  - "cd greentest && python testrunner.py --config ../known_failures.py && cd .."
+
+after_test:
+  # If tests are successful, create a whl package for the project.
+  - "%CMD_IN_ENV% pip install wheel"
+  - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst"
+  - ps: "ls dist"
+
+artifacts:
+  # Archive the generated wheel package in the ci.appveyor.com build report.
+  - path: dist\*
+
+#on_success:
+#  - TODO: upload the content of dist/*.whl to a public wheelhouse
+#

appveyor/install.ps1

+# Sample script to install Python and pip under Windows
+# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner
+# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+
+$MINICONDA_URL = "http://repo.continuum.io/miniconda/"
+$BASE_URL = "https://www.python.org/ftp/python/"
+$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
+$GET_PIP_PATH = "C:\get-pip.py"
+
+
+function DownloadPython ($python_version, $platform_suffix) {
+    $webclient = New-Object System.Net.WebClient
+    $filename = "python-" + $python_version + $platform_suffix + ".msi"
+    $url = $BASE_URL + $python_version + "/" + $filename
+
+    $basedir = $pwd.Path + "\"
+    $filepath = $basedir + $filename
+    if (Test-Path $filename) {
+        Write-Host "Reusing" $filepath
+        return $filepath
+    }
+
+    # Download and retry up to 3 times in case of network transient errors.
+    Write-Host "Downloading" $filename "from" $url
+    $retry_attempts = 2
+    for($i=0; $i -lt $retry_attempts; $i++){
+        try {
+            $webclient.DownloadFile($url, $filepath)
+            break
+        }
+        Catch [Exception]{
+            Start-Sleep 1
+        }
+   }
+   if (Test-Path $filepath) {
+       Write-Host "File saved at" $filepath
+   } else {
+       # Retry once to get the error message if any at the last try
+       $webclient.DownloadFile($url, $filepath)
+   }
+   return $filepath
+}
+
+
+function InstallPython ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = ""
+    } else {
+        $platform_suffix = ".amd64"
+    }
+    $msipath = DownloadPython $python_version $platform_suffix
+    Write-Host "Installing" $msipath "to" $python_home
+    $install_log = $python_home + ".log"
+    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
+    $uninstall_args = "/qn /x $msipath"
+    RunCommand "msiexec.exe" $install_args
+    if (-not(Test-Path $python_home)) {
+        Write-Host "Python seems to be installed else-where, reinstalling."
+        RunCommand "msiexec.exe" $uninstall_args
+        RunCommand "msiexec.exe" $install_args
+    }
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+function RunCommand ($command, $command_args) {
+    Write-Host $command $command_args
+    Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
+}
+
+
+function InstallPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $python_path = $python_home + "\python.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $webclient = New-Object System.Net.WebClient
+        $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
+        Write-Host "Executing:" $python_path $GET_PIP_PATH
+        Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+
+function DownloadMiniconda ($python_version, $platform_suffix) {
+    $webclient = New-Object System.Net.WebClient
+    if ($python_version -eq "3.4") {
+        $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
+    } else {
+        $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
+    }
+    $url = $MINICONDA_URL + $filename
+
+    $basedir = $pwd.Path + "\"
+    $filepath = $basedir + $filename
+    if (Test-Path $filename) {
+        Write-Host "Reusing" $filepath
+        return $filepath
+    }
+
+    # Download and retry up to 3 times in case of network transient errors.
+    Write-Host "Downloading" $filename "from" $url
+    $retry_attempts = 2
+    for($i=0; $i -lt $retry_attempts; $i++){
+        try {
+            $webclient.DownloadFile($url, $filepath)
+            break
+        }
+        Catch [Exception]{
+            Start-Sleep 1
+        }
+   }
+   if (Test-Path $filepath) {
+       Write-Host "File saved at" $filepath
+   } else {
+       # Retry once to get the error message if any at the last try
+       $webclient.DownloadFile($url, $filepath)
+   }
+   return $filepath
+}
+
+
+function InstallMiniconda ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = "x86"
+    } else {
+        $platform_suffix = "x86_64"
+    }
+    $filepath = DownloadMiniconda $python_version $platform_suffix
+    Write-Host "Installing" $filepath "to" $python_home
+    $install_log = $python_home + ".log"
+    $args = "/S /D=$python_home"
+    Write-Host $filepath $args
+    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallMinicondaPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $conda_path = $python_home + "\Scripts\conda.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $args = "install --yes pip"
+        Write-Host $conda_path $args
+        Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+function main () {
+    InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
+    InstallPip $env:PYTHON
+}
+
+main

appveyor/run_with_env.cmd

+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
+::
+:: To build extensions for 64 bit Python 2, we need to configure environment
+:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
+::
+:: 32 bit builds do not require specific environment configurations.
+::
+:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
+:: cmd interpreter, at least for (SDK v7.0)
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+:: http://stackoverflow.com/a/13751649/163740
+::
+:: Author: Olivier Grisel
+:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+@ECHO OFF
+
+SET COMMAND_TO_RUN=%*
+SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
+
+SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%"
+IF %MAJOR_PYTHON_VERSION% == "2" (
+    SET WINDOWS_SDK_VERSION="v7.0"
+) ELSE IF %MAJOR_PYTHON_VERSION% == "3" (
+    SET WINDOWS_SDK_VERSION="v7.1"
+) ELSE (
+    ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
+    EXIT 1
+)
+
+IF "%PYTHON_ARCH%"=="64" (
+    ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+    SET DISTUTILS_USE_SDK=1
+    SET MSSdk=1
+    "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+    "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+) ELSE (
+    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+)

gevent/_socket3.py

@@ -447,6 +447,9 @@ if hasattr(_socket, "socketpair"):
         b = socket(family, type, proto, b.detach())
         return a, b
 
+else:
+    __implements__.remove('socketpair')
+
 
 # PyPy needs drop and reuse
 def _do_reuse_or_drop(socket, methname):

gevent/win32util.py

@@ -86,7 +86,7 @@ class _ErrorFormatter(object):
         @rtype: C{str}
         """
         if self.winError is not None:
-            return self.winError(errorcode)[1]
+            return str(self.winError(errorcode))
         if self.formatMessage is not None:
             return self.formatMessage(errorcode)
         if self.errorTab is not None:

known_failures.py

 # This is a list of known failures (=bugs).
 # The tests listed there must fail (or testrunner.py will report error) unless they are prefixed with FLAKY
 # in which cases the result of them is simply ignored
+from __future__ import print_function
 import os
 import sys
+import struct
 
 
 LEAKTEST = os.getenv('GEVENTTEST_LEAKCHECK')
@@ -44,12 +46,43 @@ if sys.platform == 'win32':
     FAILING_TESTS += ['test__core_stat.py']
 
     # other Windows-related issues (need investigating)
-    FAILING_TESTS += [
-        'monkey_test test_threading.py',
-        'monkey_test --Event test_threading.py',
-        'monkey_test test_subprocess.py',
-        'monkey_test --Event test_subprocess.py'
-    ]
+    FAILING_TESTS += '''
+test__all__.py
+test__core_fork.py
+test__issues461_471.py
+test__execmodules.py
+test__socketpair.py
+test__makefile_ref.py
+FLAKY test__greenletset.py
+FLAKY test_hub_join_timeout.py
+'''.split('\n')
+
+    if struct.calcsize('P') * 8 == 64:
+        # could be a problem of appveyor - not sure
+        #  ======================================================================
+        #   ERROR: test_af (__main__.TestIPv6Environment)
+        #  ----------------------------------------------------------------------
+        #   File "C:\Python27-x64\lib\ftplib.py", line 135, in connect
+        #     self.sock = socket.create_connection((self.host, self.port), self.timeout)
+        #   File "c:\projects\gevent\gevent\socket.py", line 73, in create_connection
+        #     raise err
+        #   error: [Errno 10049] [Error 10049] The requested address is not valid in its context.
+        FAILING_TESTS.append('test_ftplib.py')
+
+    if PY3:
+        # XXX need investigating
+        FAILING_TESTS += '''
+            test__example_portforwarder.py
+            test__socket_ex.py
+            test__examples.py
+            test_subprocess.py
+            test__issue600.py
+            test__subprocess.py
+            test_threading_2.py
+            test__api_timeout.py
+            test__subprocess_poll.py
+            test__example_udp_client.py
+        '''.split('\n')
 
 
 if LEAKTEST:
@@ -86,14 +119,14 @@ if PY3:
     # No idea / TODO
     FAILING_TESTS += '''
 FLAKY test__socket_dns.py
-'''.strip().split('\n')
+'''.split('\n')
 
     if LEAKTEST:
         FAILING_TESTS += ['FLAKY test__threadpool.py']
         # refcount problems:
         FAILING_TESTS += '''
             test__timeout.py
-            test__greenletset.py
+            FLAKY test__greenletset.py
             test__core.py
             test__systemerror.py
             test__exc_info.py
@@ -106,9 +139,10 @@ FLAKY test__socket_dns.py
             test__select.py
             test__greenlet.py
             FLAKY test__socket.py
-'''.strip().split()
+'''.split('\n')
+
+FAILING_TESTS = [x.strip() for x in FAILING_TESTS if x.strip()]
 
 
 if __name__ == '__main__':
-    import pprint
-    pprint.pprint(FAILING_TESTS)
+    print ('known_failures:\n', FAILING_TESTS)

libev/ev.c

@@ -1809,7 +1809,7 @@ array_nextsize (int elem, int cur, int cnt)
   return ncur;
 }
 
-static void * noinline ecb_cold
+static void * ecb_cold
 array_realloc (int elem, void *base, int *cur, int cnt)
 {
   *cur = array_nextsize (elem, *cur, cnt);

setup.py

@@ -359,7 +359,7 @@ else:
 
 
 def run_setup(ext_modules, run_make):
-    if run_make:
+    if run_make and not os.environ.get('APPVEYOR'):
         if isinstance(run_make, str):
             make(run_make)
         else: