Give up on PyPy/Win for now due to the linking issues. Don't include libev_vfd...

Give up on PyPy/Win for now due to the linking issues. Don't include libev_vfd if we don't need it. Update install.ps1.

.gitignore

 *.py[co]
 build/
 *.so
+*.pyd
 gevent.*.[ch]
 gevent/core.pyx
 gevent/__pycache__

README.rst

@@ -25,8 +25,10 @@ Get gevent
 ==========
 
 Install Python 2.6, 2.7, 3.3 or 3.4 along with the greenlet_ extension
-(Python 3.5 has preliminary support). Or install PyPy 2.6 or above
-(but not PyPy3).
+(Python 3.5 has preliminary support). Or install PyPy 4.0.1 or above
+(but not PyPy3) (*note*: PyPy is not supported in Windows). On all
+platforms, installing setuptools is recommended (this is done
+automatically if working in a virtual environment).
 
 Download the latest release from `Python Package Index`_ or clone `the repository`_.
 
@@ -41,7 +43,7 @@ Development
 
 To install the latest development version::
 
-  pip install 'cython>=0.23.4' git+git://github.com/gevent/gevent.git#egg=gevent
+  pip install setuptools 'cython>=0.23.4' git+git://github.com/gevent/gevent.git#egg=gevent
 
 .. note:: You must have Cython installed to build a checkout.
 
@@ -51,7 +53,7 @@ Running Tests
 There are a few different ways to run the tests. To simply run the
 tests on one version of Python during development, try this::
 
-  pip install cython>=0.23.4
+  pip install setuptools cython>=0.23.4
   python setup.py build
   cd greentest
   PYTHONPATH=.. python testrunner.py --config ../known_failures.py

appveyor.yml

@@ -10,21 +10,22 @@ environment:
     # Pre-installed Python versions, which Appveyor may upgrade to
     # a later point release.
 
-    - PYTHON: "C:\\pypy-4.0.1-win32"
-      PYTHON_ID: "pypy"
-      PYTHON_EXE: pypy
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "32"
+    # XXX: Sadly, PyPy won't link CFFI on Win32
+    # - PYTHON: "C:\\pypy-4.0.1-win32"
+    #   PYTHON_ID: "pypy"
+    #   PYTHON_EXE: pypy
+    #   PYTHON_VERSION: "2.7.x"
+    #   PYTHON_ARCH: "32"
 
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5.x" # currently 3.5.0
       PYTHON_ARCH: "32"
       PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python27-x64"
-    #   PYTHON_VERSION: "2.7.x" # currently 2.7.9
-    #   PYTHON_ARCH: "64"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x" # currently 2.7.9
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     # - PYTHON: "C:\\Python35-x64"
     #   PYTHON_VERSION: "3.5.x" # currently 3.5.0
@@ -36,12 +37,12 @@ environment:
     #   PYTHON_ARCH: "32"
     #   PYTHON_EXE: python
 
-    #- PYTHON: "C:\\Python33"
+    # - PYTHON: "C:\\Python33"
     #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
     #  PYTHON_ARCH: "32"
     #   PYTHON_EXE: python
 
-    #- PYTHON: "C:\\Python33-x64"
+    # - PYTHON: "C:\\Python33-x64"
     #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
     #  PYTHON_ARCH: "64"
     #   PYTHON_EXE: python
@@ -92,6 +93,7 @@ install:
       }
       elseif (-not(Test-Path($env:PYTHON))) {
         & appveyor\install.ps1;
+        & "${env:CMD_IN_ENV}" pip install -U psutil;
       }
 
   # Prepend newly installed Python to the PATH of this build (this cannot be
@@ -113,9 +115,6 @@ install:
   - ps: "if(Test-Path(\"${env:PYTHON}\\bin\")) {ls ${env:PYTHON}\\bin;}"
   - ps: "if(Test-Path(\"${env:PYTHON}\\Scripts\")) {ls ${env:PYTHON}\\Scripts;}"
 
-  # XXX: Most of this is a copy of the Makefile. Remember to update.
-  - cmd: .\appveyor\make.cmd
-
 cache:
   - "%TMP%\\py\\"
 

appveyor/install.ps1

 # Sample script to install Python and pip under Windows
-# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner
+# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
 # License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
 
 $MINICONDA_URL = "http://repo.continuum.io/miniconda/"
@@ -7,11 +7,19 @@ $BASE_URL = "https://www.python.org/ftp/python/"
 $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
 $GET_PIP_PATH = "C:\get-pip.py"
 
+$PYTHON_PRERELEASE_REGEX = @"
+(?x)
+(?<major>\d+)
+\.
+(?<minor>\d+)
+\.
+(?<micro>\d+)
+(?<prerelease>[a-z]{1,2}\d+)
+"@
 
-function DownloadPython ($python_version, $platform_suffix) {
+
+function Download ($filename, $url) {
     $webclient = New-Object System.Net.WebClient
-    $filename = "python-" + $python_version + $platform_suffix + ".msi"
-    $url = $BASE_URL + $python_version + "/" + $filename
 
     $basedir = $pwd.Path + "\"
     $filepath = $basedir + $filename
@@ -23,7 +31,7 @@ function DownloadPython ($python_version, $platform_suffix) {
     # Download and retry up to 3 times in case of network transient errors.
     Write-Host "Downloading" $filename "from" $url
     $retry_attempts = 2
-    for($i=0; $i -lt $retry_attempts; $i++){
+    for ($i = 0; $i -lt $retry_attempts; $i++) {
         try {
             $webclient.DownloadFile($url, $filepath)
             break
@@ -42,6 +50,57 @@ function DownloadPython ($python_version, $platform_suffix) {
 }
 
 
+function ParsePythonVersion ($python_version) {
+    if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
+        return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
+                $matches.prerelease)
+    }
+    $version_obj = [version]$python_version
+    return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
+}
+
+
+function DownloadPython ($python_version, $platform_suffix) {
+    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
+
+    if (($major -le 2 -and $micro -eq 0) `
+        -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
+        ) {
+        $dir = "$major.$minor"
+        $python_version = "$major.$minor$prerelease"
+    } else {
+        $dir = "$major.$minor.$micro"
+    }
+
+    if ($prerelease) {
+        if (($major -le 2) `
+            -or ($major -eq 3 -and $minor -eq 1) `
+            -or ($major -eq 3 -and $minor -eq 2) `
+            -or ($major -eq 3 -and $minor -eq 3) `
+            ) {
+            $dir = "$dir/prev"
+        }
+    }
+
+    if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
+        $ext = "msi"
+        if ($platform_suffix) {
+            $platform_suffix = ".$platform_suffix"
+        }
+    } else {
+        $ext = "exe"
+        if ($platform_suffix) {
+            $platform_suffix = "-$platform_suffix"
+        }
+    }
+
+    $filename = "python-$python_version$platform_suffix.$ext"
+    $url = "$BASE_URL$dir/$filename"
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
 function InstallPython ($python_version, $architecture, $python_home) {
     Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
     if (Test-Path $python_home) {
@@ -51,18 +110,16 @@ function InstallPython ($python_version, $architecture, $python_home) {
     if ($architecture -eq "32") {
         $platform_suffix = ""
     } else {
-        $platform_suffix = ".amd64"
+        $platform_suffix = "amd64"
     }
-    $msipath = DownloadPython $python_version $platform_suffix
-    Write-Host "Installing" $msipath "to" $python_home
+    $installer_path = DownloadPython $python_version $platform_suffix
+    $installer_ext = [System.IO.Path]::GetExtension($installer_path)
+    Write-Host "Installing $installer_path to $python_home"
     $install_log = $python_home + ".log"
-    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
-    $uninstall_args = "/qn /x $msipath"
-    RunCommand "msiexec.exe" $install_args
-    if (-not(Test-Path $python_home)) {
-        Write-Host "Python seems to be installed else-where, reinstalling."
-        RunCommand "msiexec.exe" $uninstall_args
-        RunCommand "msiexec.exe" $install_args
+    if ($installer_ext -eq '.msi') {
+        InstallPythonMSI $installer_path $python_home $install_log
+    } else {
+        InstallPythonEXE $installer_path $python_home $install_log
     }
     if (Test-Path $python_home) {
         Write-Host "Python $python_version ($architecture) installation complete"
@@ -73,6 +130,24 @@ function InstallPython ($python_version, $architecture, $python_home) {
     }
 }
 
+
+function InstallPythonEXE ($exepath, $python_home, $install_log) {
+    $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
+    RunCommand $exepath $install_args
+}
+
+
+function InstallPythonMSI ($msipath, $python_home, $install_log) {
+    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
+    $uninstall_args = "/qn /x $msipath"
+    RunCommand "msiexec.exe" $install_args
+    if (-not(Test-Path $python_home)) {
+        Write-Host "Python seems to be installed else-where, reinstalling."
+        RunCommand "msiexec.exe" $uninstall_args
+        RunCommand "msiexec.exe" $install_args
+    }
+}
+
 function RunCommand ($command, $command_args) {
     Write-Host $command $command_args
     Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
@@ -87,7 +162,7 @@ function InstallPip ($python_home) {
         $webclient = New-Object System.Net.WebClient
         $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
         Write-Host "Executing:" $python_path $GET_PIP_PATH
-        Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru
+        & $python_path $GET_PIP_PATH
     } else {
         Write-Host "pip already installed."
     }
@@ -95,39 +170,13 @@ function InstallPip ($python_home) {
 
 
 function DownloadMiniconda ($python_version, $platform_suffix) {
-    $webclient = New-Object System.Net.WebClient
     if ($python_version -eq "3.4") {
         $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
     } else {
         $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
     }
     $url = $MINICONDA_URL + $filename
-
-    $basedir = $pwd.Path + "\"
-    $filepath = $basedir + $filename
-    if (Test-Path $filename) {
-        Write-Host "Reusing" $filepath
-        return $filepath
-    }
-
-    # Download and retry up to 3 times in case of network transient errors.
-    Write-Host "Downloading" $filename "from" $url
-    $retry_attempts = 2
-    for($i=0; $i -lt $retry_attempts; $i++){
-        try {
-            $webclient.DownloadFile($url, $filepath)
-            break
-        }
-        Catch [Exception]{
-            Start-Sleep 1
-        }
-   }
-   if (Test-Path $filepath) {
-       Write-Host "File saved at" $filepath
-   } else {
-       # Retry once to get the error message if any at the last try
-       $webclient.DownloadFile($url, $filepath)
-   }
+    $filepath = Download $filename $url
     return $filepath
 }
 

doc/whatsnew_1_1.rst

@@ -34,8 +34,10 @@ distributed as pre-compiled binary wheels, in addition to source code.
 PyPy Notes
 ----------
 
-PyPy has been tested on OS X and 64-bit Linux from version 2.5.0
-through 2.5.1, 2.6.0, 2.6.1, 4.0.0.
+PyPy has been tested on OS X and 64-bit Linux from version 2.6.1
+through 4.0.0 and 4.0.1.
+
+.. note:: PyPy is not supported on Windows.
 
 - Version 2.6.1 or above is required for the most robust signal
   handling. Prior to 2.6.1 and its inclusion of `cffi 1.3.0`_, signals
@@ -62,7 +64,7 @@ through 2.5.1, 2.6.0, 2.6.1, 4.0.0.
           when not acquired (which should be the typical case). The
           ``c-ares`` package has not been audited for this issue.
 
-.. note:: PyPy 4.0.0 on Linux is known to *rarely* (once per 24 hours)
+.. note:: PyPy 4.0.x on Linux is known to *rarely* (once per 24 hours)
           encounter crashes when running heavily loaded, heavily
           networked gevent programs. The exact cause is unknown and is
           being tracked in :issue:`677`.

gevent/_corecffi_build.py

@@ -203,11 +203,20 @@ void gevent_install_sigchld_handler();
 
 void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
 void ev_sleep (ev_tstamp delay); /* sleep for a while */
+"""
 
+if sys.platform.startswith('win'):
+    # We must have the vfd_open, etc, functions on
+    # Windows. But on other platforms, going through
+    # CFFI to just return the file-descriptor is slower
+    # than just doing it in Python, so we check for and
+    # workaround their absence in corecffi.py
+    _cdef += """
 typedef int... vfd_socket_t;
 int vfd_open(vfd_socket_t);
 vfd_socket_t vfd_get(int);
 void vfd_free(int);
+#endif
 """
 
 # Note that we do not cdef the vfd_* family of functions,
@@ -232,9 +241,10 @@ _source = """
 #define LIBEV_EMBED 1
 #ifdef _WIN32
 #define EV_STANDALONE 1
+#include "libev_vfd.h"
 #endif
 
-#include "libev_vfd.h"
+
 #include "libev.h"
 
 static void

gevent/corecffi.py

@@ -25,6 +25,14 @@ except ImportError:
 ffi = gevent._corecffi.ffi
 libev = gevent._corecffi.lib
 
+if hasattr(libev, 'vfd_open'):
+    # Must be on windows
+    assert sys.platform.startswith("win"), "vfd functions only needed on windows"
+    vfd_open = libev.vfd_open
+    vfd_free = libev.vfd_free
+    vfd_get = libev.vfd_get
+else:
+    vfd_open = vfd_free = vfd_get = lambda fd: fd
 
 #####
 ## NOTE on Windows:
@@ -872,19 +880,19 @@ class io(watcher):
         watcher.start(self, callback, *args)
 
     def _get_fd(self):
-        return libev.vfd_get(self._watcher.fd)
+        return vfd_get(self._watcher.fd)
 
     def _set_fd(self, fd):
         if libev.ev_is_active(self._watcher):
             raise AttributeError("'io' watcher attribute 'fd' is read-only while watcher is active")
-        vfd = libev.vfd_open(fd)
-        libev.vfd_free(self._watcher.fd)
+        vfd = vfd_open(fd)
+        vfd_free(self._watcher.fd)
         self._watcher_init(self._watcher, self._watcher_callback, vfd, self._watcher.events)
 
     fd = property(_get_fd, _set_fd)
 
     def _get_events(self):
-        return libev.vfd_get(self._watcher.fd)
+        return self._watcher.events
 
     def _set_events(self, events):
         if libev.ev_is_active(self._watcher):

greentest/greentest.py

@@ -479,7 +479,8 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
             x = fn[:-3]
             if x.endswith('_d'):
                 x = x[:-2]
-            if x in ['__init__', 'core', 'ares', '_util', '_semaphore', 'corecffi']:
+            if x in ['__init__', 'core', 'ares', '_util', '_semaphore',
+                     'corecffi', '_corecffi', '_corecffi_build']:
                 continue
             if x in OPTIONAL_MODULES:
                 try:
@@ -488,6 +489,8 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
                     continue
             yield path, modpath + x
         elif include_so and fn.endswith('.so'):
+            if '.pypy-' in fn:
+                continue
             if fn.endswith('_d.so'):
                 yield path, modpath + fn[:-5]
             else:

greentest/test__core.py

@@ -50,7 +50,7 @@ class Test(TestCase):
             io.fd = 2
             self.assertEqual(io.fd, 2)
             io.events = core.WRITE
-            self.assertEqual(io.events, core.WRITE)
+            self.assertEqual(core._events_to_str(io.events), 'WRITE|_IOFDSET')
 
     def test_timer(self):
         self.assertRaises(ValueError, core.loop().timer, 1, -1)

setup.py

@@ -11,13 +11,22 @@ from subprocess import check_call
 from glob import glob
 
 PYPY = hasattr(sys, 'pypy_version_info')
+WIN = sys.platform.startswith('win')
 
-try:
+if PYPY and WIN and not os.environ.get("PYPY_WIN_BUILD_ANYWAY"):
+    # The MS VC Compiler for Python (VC9.0) fails to properly
+    # link CFFI extensions (many unresolved symbols). Fail early
+    # so the user doesn't have to sit through all the cython compiles
+    raise Exception("Unable to install on PyPy/Windows")
+
+if WIN:
     # https://bugs.python.org/issue23246
-    import setuptools
-except ImportError:
-    if sys.platform.startswith("win"):
-        raise
+    # We must have setuptools on windows
+    __import__('setuptools')
+
+    # Make sure the env vars that make.cmd needs are set
+    if not os.environ.get('PYTHON_EXE'):
+        os.environ['PYTHON_EXE'] = 'pypy' if PYPY else 'python'
 
 
 import distutils
@@ -31,6 +40,7 @@ except ImportError:
         # need setuptools for include_package_data to work
         raise
     from distutils.core import Extension, setup
+
 from distutils.command.build_ext import build_ext
 from distutils.command.sdist import sdist as _sdist
 from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError
@@ -92,7 +102,7 @@ ares_configure_command = ' '.join(["(cd ", _quoted_abspath('c-ares/'),
                                    "> configure-output.txt"])
 
 
-if sys.platform == 'win32':
+if WIN:
     libraries += ['ws2_32']
     define_macros += [('FD_SETSIZE', '1024'), ('_WIN32', '1')]
 
@@ -152,7 +162,7 @@ def system(cmd):
 
 
 def configure_libev(bext, ext):
-    if sys.platform == "win32":
+    if WIN:
         CORE.define_macros.append(('EV_STANDALONE', '1'))
         return
 
@@ -181,7 +191,7 @@ def configure_ares(bext, ext):
     if not os.path.isdir(bdir):
         os.makedirs(bdir)
 
-    if sys.platform == "win32":
+    if WIN:
         shutil.copy("c-ares\\ares_build.h.dist", os.path.join(bdir, "ares_build.h"))
         return
 
@@ -233,7 +243,14 @@ _ran_make = []
 
 
 def make(targets=''):
+    # NOTE: We have two copies of the makefile, one
+    # for posix, one for windows
     if not _ran_make:
+        if WIN:
+            # make.cmd handles checking for PyPy and only making the
+            # right things, so we can ignore the targets
+            system("appveyor\\make.cmd")
+        else:
             if os.path.exists('Makefile'):
                 if "PYTHON" not in os.environ:
                     os.environ["PYTHON"] = sys.executable
@@ -346,11 +363,17 @@ if ((len(sys.argv) >= 2
 elif PYPY:
     sys.path.insert(0, '.')
     # XXX ugly - need to find a better way
-    system('cp -r libev gevent/libev')
+    cp_cmd = 'cp -r'
+    if WIN:
+        cp_cmd = "copy"
+    system(cp_cmd + ' libev gevent/libev')
+    if WIN:
+        system('echo > gevent/libev/__init__.py')
+    else:
         system('touch gevent/libev/__init__.py')
     if sys.platform != 'win32':
         system('cd gevent/libev && ./configure > configure_output.txt')
-    # XXX: Note that we're NOT adding the distutils extension module, but
+    # NOTE that we're NOT adding the distutils extension module, as
     # doing so compiles the module already: import gevent._corecffi_build
     # imports gevent, which imports the hub, which imports the core,
     # which compiles the module in-place. Instead we use the setup-time
@@ -391,7 +414,7 @@ else:
 
 
 def run_setup(ext_modules, run_make):
-    if run_make and not os.environ.get('APPVEYOR'):
+    if run_make:
         if isinstance(run_make, str):
             make(run_make)
         else: