Use libev_vfd in corecffi for better Win32 compat (still not perfect). Fix a...

Use libev_vfd in corecffi for better Win32 compat (still not perfect). Fix a bug changing the fd and events of a io watcher and test it. Test pypy/windows.

appveyor.yml

@@ -10,44 +10,59 @@ environment:
     # Pre-installed Python versions, which Appveyor may upgrade to
     # a later point release.
 
+    - PYTHON: "C:\\pypy-4.0.1-win32"
+      PYTHON_ID: "pypy"
+      PYTHON_EXE: pypy
+      PYTHON_VERSION: "2.7.x"
+      PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5.x" # currently 3.5.0
       PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x" # currently 2.7.9
-      PYTHON_ARCH: "64"
+    # - PYTHON: "C:\\Python27-x64"
+    #   PYTHON_VERSION: "2.7.x" # currently 2.7.9
+    #   PYTHON_ARCH: "64"
+    #   PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.x" # currently 3.5.0
-      PYTHON_ARCH: "64"
+    # - PYTHON: "C:\\Python35-x64"
+    #   PYTHON_VERSION: "3.5.x" # currently 3.5.0
+    #   PYTHON_ARCH: "64"
+    #   PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7.x" # currently 2.7.9
-      PYTHON_ARCH: "32"
+    # - PYTHON: "C:\\Python27"
+    #   PYTHON_VERSION: "2.7.x" # currently 2.7.9
+    #   PYTHON_ARCH: "32"
+    #   PYTHON_EXE: python
 
     #- PYTHON: "C:\\Python33"
     #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
     #  PYTHON_ARCH: "32"
+    #   PYTHON_EXE: python
 
     #- PYTHON: "C:\\Python33-x64"
     #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
     #  PYTHON_ARCH: "64"
+    #   PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python34"
-      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-      PYTHON_ARCH: "32"
+    # - PYTHON: "C:\\Python34"
+    #   PYTHON_VERSION: "3.4.x" # currently 3.4.3
+    #   PYTHON_ARCH: "32"
+    #   PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python34-x64"
-      PYTHON_VERSION: "3.4.x" # currently 3.4.3
-      PYTHON_ARCH: "64"
+    # - PYTHON: "C:\\Python34-x64"
+    #   PYTHON_VERSION: "3.4.x" # currently 3.4.3
+    #   PYTHON_ARCH: "64"
+    #   PYTHON_EXE: python
 
     # Also test a Python version not pre-installed
     # See: https://github.com/ogrisel/python-appveyor-demo/issues/10
 
-    #- PYTHON: "C:\\Python266"
-    #  PYTHON_VERSION: "2.6.6"
-    #  PYTHON_ARCH: "32"
+    # - PYTHON: "C:\\Python266"
+    #   PYTHON_VERSION: "2.6.6"
+    #   PYTHON_ARCH: "32"
+    #   PYTHON_EXE: python
 
 install:
   - ECHO "Filesystem root:"
@@ -58,25 +73,48 @@ install:
 
   # Install Python (from the official .msi of http://python.org) and pip when
   # not already installed.
-  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+  # PyPy portion based on https://github.com/wbond/asn1crypto/blob/master/appveyor.yml
+  - ps:
+      $env:PYTMP = "${env:TMP}\py";
+      if (!(Test-Path "$env:PYTMP")) {
+        New-Item -ItemType directory -Path "$env:PYTMP" | Out-Null;
+      }
+      if ("${env:PYTHON_ID}" -eq "pypy") {
+        if (!(Test-Path "${env:PYTMP}\pypy-4.0.1-win32.zip")) {
+          (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy-4.0.1-win32.zip', "${env:PYTMP}\pypy-4.0.1-win32.zip");
+        }
+        7z x -y "${env:PYTMP}\pypy-4.0.1-win32.zip" -oC:\ | Out-Null;
+        if (!(Test-Path "${env:PYTMP}\get-pip.py")) {
+          (New-Object Net.WebClient).DownloadFile('https://bootstrap.pypa.io/get-pip.py', "${env:PYTMP}\get-pip.py");
+        }
+        & "${env:PYTHON}\pypy.exe" "${env:PYTMP}\get-pip.py";
+
+      }
+      elseif (-not(Test-Path($env:PYTHON))) {
+        & appveyor\install.ps1;
+      }
 
   # Prepend newly installed Python to the PATH of this build (this cannot be
   # done from inside the powershell script as it would require to restart
   # the parent CMD process).
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
+  - "SET PYEXE=%PYTHON%\\%PYTHON_EXE%.exe"
 
   # Check that we have the expected version and architecture for Python
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+  - "%PYEXE% --version"
+  - "%PYEXE% -c \"import struct; print(struct.calcsize('P') * 8)\""
 
   # Install the build dependencies of the project. If some dependencies contain
   # compiled extensions and are not provided as pre-built wheel packages,
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
-  - "%CMD_IN_ENV% pip install -U cython greenlet psutil"
+  - "%CMD_IN_ENV% pip install -U cython greenlet"
+
+  - ps: "if(Test-Path(\"${env:PYTHON}\\bin\")) {ls ${env:PYTHON}\\bin;}"
+  - ps: "if(Test-Path(\"${env:PYTHON}\\Scripts\")) {ls ${env:PYTHON}\\Scripts;}"
 
   # XXX: Most of this is a copy of the Makefile. Remember to update.
-  - "python util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx"
+  - "%PYEXE% util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx"
   - "type gevent\\callbacks.c >> gevent.corecext.c"
   - "move gevent.corecext.* gevent"
   - "cython -o gevent.ares.c gevent/ares.pyx"
@@ -88,17 +126,20 @@ install:
   - "cython -o gevent._util.c gevent/_util.pyx"
   - "move gevent._util.* gevent"
 
+cache:
+  - "%TMP%\\py\\"
+
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # Build the compiled extension and run the project tests
   - "%CMD_IN_ENV% python setup.py develop"
-  - "cd greentest && python testrunner.py --config ../known_failures.py && cd .."
+  - "cd greentest && %PYEXE% testrunner.py --config ../known_failures.py && cd .."
 
 after_test:
   # If tests are successful, create a whl package for the project.
   - "%CMD_IN_ENV% pip install -U wheel"
-  - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst"
+  - "%CMD_IN_ENV% %PYEXE% setup.py bdist_wheel bdist_wininst"
   - ps: "ls dist"
 
 artifacts:

gevent/_corecffi_build.py

@@ -203,8 +203,16 @@ void gevent_install_sigchld_handler();
 
 void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
 void ev_sleep (ev_tstamp delay); /* sleep for a while */
+
+typedef int... vfd_socket_t;
+int vfd_open(vfd_socket_t);
+vfd_socket_t vfd_get(int);
+void vfd_free(int);
 """
 
+# Note that we do not cdef the vfd_* family of functions,
+# nor do we include libev_vfd.h in the _source; we'd need to get
+# the correct parameter types to do so (which )
 
 _watcher_types = [
     'ev_async',
@@ -221,6 +229,7 @@ _watcher_types = [
 
 _source = """   // passed to the real C compiler
 #define LIBEV_EMBED 1
+#include "libev_vfd.h"
 #include "libev.h"
 
 static void

gevent/corecffi.py

@@ -31,7 +31,6 @@ libev = gevent._corecffi.lib
 # The C implementation does several things specially for Windows;
 # a possibly incomplete list is:
 #
-# - special handle mapping through libev_vfd.h when LIBEV_EMBED is defined;
 # - the loop runs a periodic signal checker;
 # - the io watcher constructor is different and it has a destructor;
 # - the child watcher is not defined
@@ -40,23 +39,6 @@ libev = gevent._corecffi.lib
 # is possibly NOT FUNCTIONALLY CORRECT on Win32
 #####
 
-# The C implementation defines special versions of these functions on
-# Windows and includes them (libev_vfd.h), making them a part of libev
-# itself (effectively). Our implementation does not. Prior to CFFI
-# 1.0, we could let attributes on the libev FFI library, but with
-# set_source/compile, we cannot do that anymore, so instead of using
-# 'libev.vfd_open(...)', we make them global functions
-def vfd_open(fd):
-    return fd
-
-
-def vfd_get(fd):
-    return fd
-
-
-def vfd_free(fd):
-    return
-
 #####
 ## Note on CFFI objects, callbacks and the lifecycle of watcher objects
 #
@@ -875,6 +857,8 @@ class io(watcher):
     _watcher_type = 'ev_io'
 
     def __init__(self, loop, fd, events, ref=True, priority=None):
+        # XXX: Win32: Need to vfd_open the fd and free the old one?
+        # XXX: Win32: Need a destructor to free the old fd?
         if fd < 0:
             raise ValueError('fd must be non-negative: %r' % fd)
         if events & ~(libev.EV__IOFDSET | libev.EV_READ | libev.EV_WRITE):
@@ -888,24 +872,24 @@ class io(watcher):
         watcher.start(self, callback, *args)
 
     def _get_fd(self):
-        return vfd_get(self._watcher.fd)
+        return libev.vfd_get(self._watcher.fd)
 
     def _set_fd(self, fd):
         if libev.ev_is_active(self._watcher):
             raise AttributeError("'io' watcher attribute 'fd' is read-only while watcher is active")
-        vfd = vfd_open(fd)
-        vfd_free(self._watcher.fd)
-        libev.ev_io_init(self._watcher, self._cb, vfd, self._watcher.events)
+        vfd = libev.vfd_open(fd)
+        libev.vfd_free(self._watcher.fd)
+        self._watcher_init(self._watcher, self._watcher_callback, vfd, self._watcher.events)
 
     fd = property(_get_fd, _set_fd)
 
     def _get_events(self):
-        return vfd_get(self._watcher.fd)
+        return libev.vfd_get(self._watcher.fd)
 
     def _set_events(self, events):
         if libev.ev_is_active(self._watcher):
             raise AttributeError("'io' watcher attribute 'events' is read-only while watcher is active")
-        libev.ev_io_init(self._watcher, self._cb, self._watcher.fd, events)
+        self._watcher_init(self._watcher, self._watcher_callback, self._watcher.fd, events)
 
     events = property(_get_events, _set_events)
 

greentest/test__core.py

@@ -36,10 +36,21 @@ class Test(TestCase):
     def test_io(self):
         if sys.platform == 'win32':
             Error = IOError
+            win32 = True
         else:
             Error = ValueError
+            win32 = False
         self.assertRaises(Error, core.loop().io, -1, 1)
         self.assertRaises(ValueError, core.loop().io, 1, core.TIMER)
+        # Test we can set events and io before it's started
+        if not win32:
+            # We can't do this with arbitrary FDs on windows;
+            # see libev_vfd.h
+            io = core.loop().io(1, core.READ)
+            io.fd = 2
+            self.assertEqual(io.fd, 2)
+            io.events = core.WRITE
+            self.assertEqual(io.events, core.WRITE)
 
     def test_timer(self):
         self.assertRaises(ValueError, core.loop().timer, 1, -1)