Merge pull request #693 from gevent/modern-cffi

Use modern CFFI methods; drop unsupported PyPy versions < 2.6.1

.gitignore

 *.py[co]
 build/
 *.so
+*.pyd
+.runtimes
 gevent.*.[ch]
 gevent/core.pyx
 gevent/__pycache__
 gevent/libev
+gevent/_corecffi.c
+gevent/_corecffi.o
 *.egg-info
 Makefile.ext
 MANIFEST

.travis.yml

-language: python
+# .travis.yml based on https://github.com/DRMacIver/hypothesis/blob/master/.travis.yml
+language: c
 sudo: false
-python:
-    - 2.6
-    - 2.7
-    - pypy
-    - 3.3
-    - 3.4
-    - 3.5
+
 env:
-    - LINT=true
-    - LINT=false
-install:
-    # First install a newer pip so that it can use the wheel cache
-    # (only needed until travis upgrades pip to 7.x; note that the 3.5
-    # environment uses pip 7.1 by default)
-    - travis_retry pip install -U pip
-    # Then start installing our deps so they can be cached. Note that use of --build-options / --global-options / --install-options
-    # disables the cache.
-    # We need wheel>=0.26 on Python 3.5. See previous revisions.
-    - travis_retry pip install -U wheel
-    - travis_retry pip install -U tox cython greenlet pep8 pyflakes "coverage>=4.0" "coveralls>=1.0"
+    global:
+        - BUILD_RUNTIMES=$HOME/.runtimes
+
+    matrix:
+      # These are ordered to get as much diversity in the
+      # first group of parallel runs (4) as posible
+      - TASK=test-pypy
+      - TASK=lint-py27
+      - TASK=test-py35
+      - TASK=test-py26
+      - TASK=test-py27
+      - TASK=test-py33
+      - TASK=test-py34
+
+
+matrix:
+  fast_finish: true
+
 script:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $LINT == true ]]; then python setup.py develop && make travis_test_linters; elif [[ $LINT == false && $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then python setup.py develop && make fulltoxtest; elif [[ $LINT == false ]]; then python setup.py develop && make fulltoxtest; fi
+    - make $TASK
+
 notifications:
   email: false
+
 # cache: pip seems not to work if `install` is replaced (https://github.com/travis-ci/travis-ci/issues/3239)
 cache:
   directories:
     - $HOME/.cache/pip
+    - $HOME/.venv
+    - $HOME/.runtimes
+    - $HOME/.wheelhouse
+
 before_cache:
     - rm -f $HOME/.cache/pip/log/debug.log

MANIFEST.in

@@ -22,6 +22,7 @@ include .pep8
 recursive-include benchmarks *.sh
 recursive-include appveyor *.cmd
 recursive-include appveyor *.ps1
+recursive-include scripts *.sh
 
 ### Artifacts of configuring/building in place
 # These we want, they come from the Makefile step

Makefile

 # This file is renamed to "Makefile.ext" in release tarballs so that setup.py won't try to
 # run it.  If you want setup.py to run "make" automatically, rename it back to "Makefile".
 
-PYTHON ?= python${TRAVIS_PYTHON_VERSION}
-CYTHON ?= cython
+# The pyvenv multiple runtime support is based on https://github.com/DRMacIver/hypothesis/blob/master/Makefile
+
+PYTHON?=python${TRAVIS_PYTHON_VERSION}
+CYTHON?=cython
+
+
+
+export PATH:=$(BUILD_RUNTIMES)/snakepit:$(TOOLS):$(PATH)
+export LC_ALL=C.UTF-8
+
 
 all: gevent/gevent.corecext.c gevent/gevent.ares.c gevent/gevent._semaphore.c gevent/gevent._util.c
 
@@ -71,9 +79,11 @@ toxtest:
 	cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
 
 fulltoxtest:
-	cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
-	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 python testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
-	cd greentest && GEVENT_FILE=thread python testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
+	which ${PYTHON}
+	${PYTHON} --version
+	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
+	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
+	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
 
 leaktest:
 	GEVENTSETUP_EV_VERIFY=3 GEVENTTEST_LEAKCHECK=1 make travistest
@@ -100,3 +110,78 @@ travis_test_linters:
 
 
 .PHONY: clean all doc pep8 whitespace pyflakes lint travistest travis
+
+# Managing runtimes
+
+BUILD_RUNTIMES?=$(PWD)/.runtimes
+
+PY26=$(BUILD_RUNTIMES)/snakepit/python2.6
+PY27=$(BUILD_RUNTIMES)/snakepit/python2.7
+PY33=$(BUILD_RUNTIMES)/snakepit/python3.3
+PY34=$(BUILD_RUNTIMES)/snakepit/python3.4
+PY35=$(BUILD_RUNTIMES)/snakepit/python3.5
+PYPY=$(BUILD_RUNTIMES)/snakepit/pypy
+
+TOOLS=$(BUILD_RUNTIMES)/tools
+
+TOX=$(TOOLS)/tox
+
+TOOL_VIRTUALENV=$(BUILD_RUNTIMES)/virtualenvs/tools
+ISORT_VIRTUALENV=$(BUILD_RUNTIMES)/virtualenvs/isort
+TOOL_PYTHON=$(TOOL_VIRTUALENV)/bin/python
+TOOL_PIP=$(TOOL_VIRTUALENV)/bin/pip
+TOOL_INSTALL=$(TOOL_PIP) install --upgrade
+
+$(PY26):
+	scripts/install.sh 2.6
+
+$(PY27):
+	scripts/install.sh 2.7
+
+$(PY33):
+	scripts/install.sh 3.3
+
+$(PY34):
+	scripts/install.sh 3.4
+
+$(PY35):
+	scripts/install.sh 3.5
+
+$(PYPY):
+	scripts/install.sh pypy
+
+PIP?=$(BUILD_RUNTIMES)/versions/$(PYTHON)/bin/pip
+
+develop:
+	echo $(PIP) $(PYTHON)
+# First install a newer pip so that it can use the wheel cache
+# (only needed until travis upgrades pip to 7.x; note that the 3.5
+# environment uses pip 7.1 by default)
+	${PIP} install -U pip
+# Then start installing our deps so they can be cached. Note that use of --build-options / --global-options / --install-options
+# disables the cache.
+# We need wheel>=0.26 on Python 3.5. See previous revisions.
+	${PIP} install -U wheel
+	${PIP} install -U tox cython greenlet pep8 pyflakes "coverage>=4.0" "coveralls>=1.0"
+	${PYTHON} setup.py develop
+
+lint-py27: $(PY27)
+	PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop travis_test_linters
+
+test-py27: $(PY27)
+	PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop fulltoxtest
+
+test-py26: $(PY26)
+	PYTHON=python2.6 PATH=$(BUILD_RUNTIMES)/versions/python2.6/bin:$(PATH) make develop fulltoxtest
+
+test-py33: $(PY33)
+	PYTHON=python3.3 PATH=$(BUILD_RUNTIMES)/versions/python3.3/bin:$(PATH) make develop fulltoxtest
+
+test-py34: $(PY34)
+	PYTHON=python3.4 PATH=$(BUILD_RUNTIMES)/versions/python3.4/bin:$(PATH) make develop fulltoxtest
+
+test-py35: $(PY35)
+	PYTHON=python3.5 PATH=$(BUILD_RUNTIMES)/versions/python3.5/bin:$(PATH) make develop fulltoxtest
+
+test-pypy: $(PYPY)
+	PYTHON=pypy PATH=$(BUILD_RUNTIMES)/versions/pypy/bin:$(PATH) make develop toxtest

README.rst

@@ -25,8 +25,10 @@ Get gevent
 ==========
 
 Install Python 2.6, 2.7, 3.3 or 3.4 along with the greenlet_ extension
-(Python 3.5 has preliminary support). Or install PyPy 2.6 or above
-(but not PyPy3).
+(Python 3.5 has preliminary support). Or install PyPy 4.0.1 or above
+(but not PyPy3) (*note*: PyPy is not supported in Windows). On all
+platforms, installing setuptools is recommended (this is done
+automatically if working in a virtual environment).
 
 Download the latest release from `Python Package Index`_ or clone `the repository`_.
 
@@ -41,7 +43,7 @@ Development
 
 To install the latest development version::
 
-  pip install 'cython>=0.23.4' git+git://github.com/gevent/gevent.git#egg=gevent
+  pip install setuptools 'cython>=0.23.4' git+git://github.com/gevent/gevent.git#egg=gevent
 
 .. note:: You must have Cython installed to build a checkout.
 
@@ -51,7 +53,7 @@ Running Tests
 There are a few different ways to run the tests. To simply run the
 tests on one version of Python during development, try this::
 
-  pip install cython>=0.23.4
+  pip install setuptools cython>=0.23.4
   python setup.py build
   cd greentest
   PYTHONPATH=.. python testrunner.py --config ../known_failures.py

appveyor.yml

@@ -10,44 +10,60 @@ environment:
     # Pre-installed Python versions, which Appveyor may upgrade to
     # a later point release.
 
+    # XXX: Sadly, PyPy won't link CFFI on Win32
+    # - PYTHON: "C:\\pypy-4.0.1-win32"
+    #   PYTHON_ID: "pypy"
+    #   PYTHON_EXE: pypy
+    #   PYTHON_VERSION: "2.7.x"
+    #   PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5.x" # currently 3.5.0
       PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "2.7.x" # currently 2.7.9
       PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5.x" # currently 3.5.0
       PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7.x" # currently 2.7.9
       PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
-    #- PYTHON: "C:\\Python33"
-    #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
-    #  PYTHON_ARCH: "32"
+    - PYTHON: "C:\\Python33"
+      PYTHON_VERSION: "3.3.x" # currently 3.3.5
+      PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
-    #- PYTHON: "C:\\Python33-x64"
-    #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
-    #  PYTHON_ARCH: "64"
+    - PYTHON: "C:\\Python33-x64"
+      PYTHON_VERSION: "3.3.x" # currently 3.3.5
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python34"
       PYTHON_VERSION: "3.4.x" # currently 3.4.3
       PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4.x" # currently 3.4.3
       PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     # Also test a Python version not pre-installed
     # See: https://github.com/ogrisel/python-appveyor-demo/issues/10
 
-    #- PYTHON: "C:\\Python266"
-    #  PYTHON_VERSION: "2.6.6"
-    #  PYTHON_ARCH: "32"
+    # - PYTHON: "C:\\Python266"
+    #   PYTHON_VERSION: "2.6.6"
+    #   PYTHON_ARCH: "32"
+    #   PYTHON_EXE: python
 
 install:
   - ECHO "Filesystem root:"
@@ -58,47 +74,61 @@ install:
 
   # Install Python (from the official .msi of http://python.org) and pip when
   # not already installed.
-  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+  # PyPy portion based on https://github.com/wbond/asn1crypto/blob/master/appveyor.yml
+  - ps:
+      $env:PYTMP = "${env:TMP}\py";
+      if (!(Test-Path "$env:PYTMP")) {
+        New-Item -ItemType directory -Path "$env:PYTMP" | Out-Null;
+      }
+      if ("${env:PYTHON_ID}" -eq "pypy") {
+        if (!(Test-Path "${env:PYTMP}\pypy-4.0.1-win32.zip")) {
+          (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy-4.0.1-win32.zip', "${env:PYTMP}\pypy-4.0.1-win32.zip");
+        }
+        7z x -y "${env:PYTMP}\pypy-4.0.1-win32.zip" -oC:\ | Out-Null;
+        if (!(Test-Path "${env:PYTMP}\get-pip.py")) {
+          (New-Object Net.WebClient).DownloadFile('https://bootstrap.pypa.io/get-pip.py', "${env:PYTMP}\get-pip.py");
+        }
+        & "${env:PYTHON}\pypy.exe" "${env:PYTMP}\get-pip.py";
+
+      }
+      elseif (-not(Test-Path($env:PYTHON))) {
+        & appveyor\install.ps1;
+      }
 
   # Prepend newly installed Python to the PATH of this build (this cannot be
   # done from inside the powershell script as it would require to restart
   # the parent CMD process).
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
+  - "SET PYEXE=%PYTHON%\\%PYTHON_EXE%.exe"
 
   # Check that we have the expected version and architecture for Python
-  - "python --version"
-  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+  - "%PYEXE% --version"
+  - "%PYEXE% -c \"import struct; print(struct.calcsize('P') * 8)\""
 
   # Install the build dependencies of the project. If some dependencies contain
   # compiled extensions and are not provided as pre-built wheel packages,
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
+  # NOTE: psutil won't install under PyPy.
   - "%CMD_IN_ENV% pip install -U cython greenlet psutil"
 
-  # XXX: Most of this is a copy of the Makefile. Remember to update.
-  - "python util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx"
-  - "type gevent\\callbacks.c >> gevent.corecext.c"
-  - "move gevent.corecext.* gevent"
-  - "cython -o gevent.ares.c gevent/ares.pyx"
-  - "move gevent.ares.* gevent"
-  - "move gevent\\_semaphore.pyx gevent\\_semaphore.py"
-  - "cython -o gevent._semaphore.c gevent/_semaphore.py"
-  - "move gevent._semaphore.* gevent"
-  - "del gevent\\_semaphore.py"
-  - "cython -o gevent._util.c gevent/_util.pyx"
-  - "move gevent._util.* gevent"
+  - ps: "if(Test-Path(\"${env:PYTHON}\\bin\")) {ls ${env:PYTHON}\\bin;}"
+  - ps: "if(Test-Path(\"${env:PYTHON}\\Scripts\")) {ls ${env:PYTHON}\\Scripts;}"
+
+cache:
+  - "%TMP%\\py\\"
 
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # Build the compiled extension and run the project tests
-  - "%CMD_IN_ENV% python setup.py develop"
-  - "cd greentest && python testrunner.py --config ../known_failures.py && cd .."
+  - "%CMD_IN_ENV% %PYEXE% setup.py develop"
+  - "cd greentest && %PYEXE% testrunner.py --config ../known_failures.py && cd .."
 
 after_test:
   # If tests are successful, create a whl package for the project.
   - "%CMD_IN_ENV% pip install -U wheel"
-  - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst"
+  - "%CMD_IN_ENV% %PYEXE% setup.py bdist_wheel bdist_wininst"
   - ps: "ls dist"
 
 artifacts:

appveyor/install.ps1

 # Sample script to install Python and pip under Windows
-# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner
+# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
 # License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
 
 $MINICONDA_URL = "http://repo.continuum.io/miniconda/"
@@ -7,11 +7,19 @@ $BASE_URL = "https://www.python.org/ftp/python/"
 $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
 $GET_PIP_PATH = "C:\get-pip.py"
 
+$PYTHON_PRERELEASE_REGEX = @"
+(?x)
+(?<major>\d+)
+\.
+(?<minor>\d+)
+\.
+(?<micro>\d+)
+(?<prerelease>[a-z]{1,2}\d+)
+"@
 
-function DownloadPython ($python_version, $platform_suffix) {
+
+function Download ($filename, $url) {
     $webclient = New-Object System.Net.WebClient
-    $filename = "python-" + $python_version + $platform_suffix + ".msi"
-    $url = $BASE_URL + $python_version + "/" + $filename
 
     $basedir = $pwd.Path + "\"
     $filepath = $basedir + $filename
@@ -23,7 +31,7 @@ function DownloadPython ($python_version, $platform_suffix) {
     # Download and retry up to 3 times in case of network transient errors.
     Write-Host "Downloading" $filename "from" $url
     $retry_attempts = 2
-    for($i=0; $i -lt $retry_attempts; $i++){
+    for ($i = 0; $i -lt $retry_attempts; $i++) {
         try {
             $webclient.DownloadFile($url, $filepath)
             break
@@ -31,14 +39,65 @@ function DownloadPython ($python_version, $platform_suffix) {
         Catch [Exception]{
             Start-Sleep 1
         }
-   }
-   if (Test-Path $filepath) {
-       Write-Host "File saved at" $filepath
-   } else {
-       # Retry once to get the error message if any at the last try
-       $webclient.DownloadFile($url, $filepath)
-   }
-   return $filepath
+    }
+    if (Test-Path $filepath) {
+        Write-Host "File saved at" $filepath
+    } else {
+        # Retry once to get the error message if any at the last try
+        $webclient.DownloadFile($url, $filepath)
+    }
+    return $filepath
+}
+
+
+function ParsePythonVersion ($python_version) {
+    if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
+        return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
+                $matches.prerelease)
+    }
+    $version_obj = [version]$python_version
+    return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
+}
+
+
+function DownloadPython ($python_version, $platform_suffix) {
+    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
+
+    if (($major -le 2 -and $micro -eq 0) `
+        -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
+        ) {
+        $dir = "$major.$minor"
+        $python_version = "$major.$minor$prerelease"
+    } else {
+        $dir = "$major.$minor.$micro"
+    }
+
+    if ($prerelease) {
+        if (($major -le 2) `
+            -or ($major -eq 3 -and $minor -eq 1) `
+            -or ($major -eq 3 -and $minor -eq 2) `
+            -or ($major -eq 3 -and $minor -eq 3) `
+            ) {
+            $dir = "$dir/prev"
+        }
+    }
+
+    if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
+        $ext = "msi"
+        if ($platform_suffix) {
+            $platform_suffix = ".$platform_suffix"
+        }
+    } else {
+        $ext = "exe"
+        if ($platform_suffix) {
+            $platform_suffix = "-$platform_suffix"
+        }
+    }
+
+    $filename = "python-$python_version$platform_suffix.$ext"
+    $url = "$BASE_URL$dir/$filename"
+    $filepath = Download $filename $url
+    return $filepath
 }
 
 
@@ -51,18 +110,16 @@ function InstallPython ($python_version, $architecture, $python_home) {
     if ($architecture -eq "32") {
         $platform_suffix = ""
     } else {
-        $platform_suffix = ".amd64"
+        $platform_suffix = "amd64"
     }
-    $msipath = DownloadPython $python_version $platform_suffix
-    Write-Host "Installing" $msipath "to" $python_home
+    $installer_path = DownloadPython $python_version $platform_suffix
+    $installer_ext = [System.IO.Path]::GetExtension($installer_path)
+    Write-Host "Installing $installer_path to $python_home"
     $install_log = $python_home + ".log"
-    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
-    $uninstall_args = "/qn /x $msipath"
-    RunCommand "msiexec.exe" $install_args
-    if (-not(Test-Path $python_home)) {
-        Write-Host "Python seems to be installed else-where, reinstalling."
-        RunCommand "msiexec.exe" $uninstall_args
-        RunCommand "msiexec.exe" $install_args
+    if ($installer_ext -eq '.msi') {
+        InstallPythonMSI $installer_path $python_home $install_log
+    } else {
+        InstallPythonEXE $installer_path $python_home $install_log
     }
     if (Test-Path $python_home) {
         Write-Host "Python $python_version ($architecture) installation complete"
@@ -73,6 +130,24 @@ function InstallPython ($python_version, $architecture, $python_home) {
     }
 }
 
+
+function InstallPythonEXE ($exepath, $python_home, $install_log) {
+    $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
+    RunCommand $exepath $install_args
+}
+
+
+function InstallPythonMSI ($msipath, $python_home, $install_log) {
+    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
+    $uninstall_args = "/qn /x $msipath"
+    RunCommand "msiexec.exe" $install_args
+    if (-not(Test-Path $python_home)) {
+        Write-Host "Python seems to be installed else-where, reinstalling."
+        RunCommand "msiexec.exe" $uninstall_args
+        RunCommand "msiexec.exe" $install_args
+    }
+}
+
 function RunCommand ($command, $command_args) {
     Write-Host $command $command_args
     Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
@@ -87,7 +162,7 @@ function InstallPip ($python_home) {
         $webclient = New-Object System.Net.WebClient
         $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
         Write-Host "Executing:" $python_path $GET_PIP_PATH
-        Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru
+        & $python_path $GET_PIP_PATH
     } else {
         Write-Host "pip already installed."
     }
@@ -95,40 +170,14 @@ function InstallPip ($python_home) {
 
 
 function DownloadMiniconda ($python_version, $platform_suffix) {
-    $webclient = New-Object System.Net.WebClient
     if ($python_version -eq "3.4") {
         $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
     } else {
         $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
     }
     $url = $MINICONDA_URL + $filename
-
-    $basedir = $pwd.Path + "\"
-    $filepath = $basedir + $filename
-    if (Test-Path $filename) {
-        Write-Host "Reusing" $filepath
-        return $filepath
-    }
-
-    # Download and retry up to 3 times in case of network transient errors.
-    Write-Host "Downloading" $filename "from" $url
-    $retry_attempts = 2
-    for($i=0; $i -lt $retry_attempts; $i++){
-        try {
-            $webclient.DownloadFile($url, $filepath)
-            break
-        }
-        Catch [Exception]{
-            Start-Sleep 1
-        }
-   }
-   if (Test-Path $filepath) {
-       Write-Host "File saved at" $filepath
-   } else {
-       # Retry once to get the error message if any at the last try
-       $webclient.DownloadFile($url, $filepath)
-   }
-   return $filepath
+    $filepath = Download $filename $url
+    return $filepath
 }
 
 

appveyor/make.cmd

+IF "%PYTHON_EXE%" == "python" (
+    %PYEXE% util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx
+	type gevent\\callbacks.c >> gevent.corecext.c
+    move gevent.corecext.* gevent
+)
+cython -o gevent.ares.c gevent/ares.pyx
+move gevent.ares.* gevent
+move gevent\\_semaphore.pyx gevent\\_semaphore.py
+cython -o gevent._semaphore.c gevent/_semaphore.py
+move gevent._semaphore.* gevent
+del gevent\\_semaphore.py
+cython -o gevent._util.c gevent/_util.pyx
+move gevent._util.* gevent

doc/whatsnew_1_1.rst

@@ -34,8 +34,10 @@ distributed as pre-compiled binary wheels, in addition to source code.
 PyPy Notes
 ----------
 
-PyPy has been tested on OS X and 64-bit Linux from version 2.5.0
-through 2.5.1, 2.6.0, 2.6.1, 4.0.0.
+PyPy has been tested on OS X and 64-bit Linux from version 2.6.1
+through 4.0.0 and 4.0.1.
+
+.. note:: PyPy is not supported on Windows.
 
 - Version 2.6.1 or above is required for the most robust signal
   handling. Prior to 2.6.1 and its inclusion of `cffi 1.3.0`_, signals
@@ -62,7 +64,7 @@ through 2.5.1, 2.6.0, 2.6.1, 4.0.0.
           when not acquired (which should be the typical case). The
           ``c-ares`` package has not been audited for this issue.
 
-.. note:: PyPy 4.0.0 on Linux is known to *rarely* (once per 24 hours)
+.. note:: PyPy 4.0.x on Linux is known to *rarely* (once per 24 hours)
           encounter crashes when running heavily loaded, heavily
           networked gevent programs. The exact cause is unknown and is
           being tracked in :issue:`677`.

gevent/_corecffi_build.py

+# pylint: disable=too-many-lines, protected-access, redefined-outer-name
+
+# This module is only used to create and compile the gevent._corecffi module;
+# nothing should be directly imported from it except `ffi`, which should only be
+# used for `ffi.compile()`; programs should import gevent._corecfffi.
+# However, because we are using "out-of-line" mode, it is necessary to examine
+# this file to know what functions are created and available on the generated
+# module.
+from __future__ import absolute_import, print_function
+import sys
+import os
+import struct
+
+__all__ = []
+
+
+def system_bits():
+    return struct.calcsize('P') * 8
+
+
+def st_nlink_type():
+    if sys.platform == "darwin":
+        return "short"
+    elif system_bits() == 32:
+        return "unsigned long"
+    return "long long"
+
+
+from cffi import FFI
+ffi = FFI()
+_cdef = """
+#define EV_MINPRI ...
+#define EV_MAXPRI ...
+
+#define EV_VERSION_MAJOR ...
+#define EV_VERSION_MINOR ...
+
+#define EV_UNDEF ...
+#define EV_NONE ...
+#define EV_READ ...
+#define EV_WRITE ...
+#define EV__IOFDSET ...
+#define EV_TIMER ...
+#define EV_PERIODIC ...
+#define EV_SIGNAL ...
+#define EV_CHILD ...
+#define EV_STAT ...
+#define EV_IDLE ...
+#define EV_PREPARE ...
+#define EV_CHECK ...
+#define EV_EMBED ...
+#define EV_FORK ...
+#define EV_CLEANUP ...
+#define EV_ASYNC ...
+#define EV_CUSTOM ...
+#define EV_ERROR ...
+
+#define EVFLAG_AUTO ...
+#define EVFLAG_NOENV ...
+#define EVFLAG_FORKCHECK ...
+#define EVFLAG_NOINOTIFY ...
+#define EVFLAG_SIGNALFD ...
+#define EVFLAG_NOSIGMASK ...
+
+#define EVBACKEND_SELECT ...
+#define EVBACKEND_POLL ...
+#define EVBACKEND_EPOLL ...
+#define EVBACKEND_KQUEUE ...
+#define EVBACKEND_DEVPOLL ...
+#define EVBACKEND_PORT ...
+/* #define EVBACKEND_IOCP ... */
+
+#define EVBACKEND_ALL ...
+#define EVBACKEND_MASK ...
+
+#define EVRUN_NOWAIT ...
+#define EVRUN_ONCE ...
+
+#define EVBREAK_CANCEL ...
+#define EVBREAK_ONE ...
+#define EVBREAK_ALL ...
+
+struct ev_loop {
+    int backend_fd;
+    int activecnt;
+    ...;
+};
+struct ev_io {
+    int fd;
+    int events;
+    ...;
+};
+struct ev_timer {
+    double at;
+    ...;
+};
+struct ev_signal {...;};
+struct ev_idle  {...;};
+struct ev_prepare {...;};
+struct ev_check {...;};
+struct ev_fork  {...;};
+struct ev_async  {...;};
+struct ev_child {
+    int pid;
+    int rpid;
+    int rstatus;
+    ...;
+};
+struct stat {
+    """ + st_nlink_type() + """ st_nlink;
+    ...;
+};
+
+struct ev_stat {
+    struct stat attr;
+    const char* path;
+    struct stat prev;
+    double interval;
+    ...;
+};
+
+typedef double ev_tstamp;
+
+int ev_version_major();
+int ev_version_minor();
+
+unsigned int ev_supported_backends (void);
+unsigned int ev_recommended_backends (void);
+unsigned int ev_embeddable_backends (void);
+
+ev_tstamp ev_time (void);
+void ev_set_syserr_cb(void *);
+
+int ev_priority(void*);
+void ev_set_priority(void*, int);
+
+int ev_is_pending(void*);
+int ev_is_active(void*);
+void ev_io_init(struct ev_io*, void* callback, int fd, int events);
+void ev_io_start(struct ev_loop*, struct ev_io*);
+void ev_io_stop(struct ev_loop*, struct ev_io*);
+void ev_feed_event(struct ev_loop*, void*, int);
+
+void ev_timer_init(struct ev_timer*, void (*callback)(struct ev_loop *_loop, struct ev_timer *w, int revents), double, double);
+void ev_timer_start(struct ev_loop*, struct ev_timer*);
+void ev_timer_stop(struct ev_loop*, struct ev_timer*);
+void ev_timer_again(struct ev_loop*, struct ev_timer*);
+
+void ev_signal_init(struct ev_signal*, void* callback, int);
+void ev_signal_start(struct ev_loop*, struct ev_signal*);
+void ev_signal_stop(struct ev_loop*, struct ev_signal*);
+
+void ev_idle_init(struct ev_idle*, void* callback);
+void ev_idle_start(struct ev_loop*, struct ev_idle*);
+void ev_idle_stop(struct ev_loop*, struct ev_idle*);
+
+void ev_prepare_init(struct ev_prepare*, void* callback);
+void ev_prepare_start(struct ev_loop*, struct ev_prepare*);
+void ev_prepare_stop(struct ev_loop*, struct ev_prepare*);
+
+void ev_check_init(struct ev_check*, void* callback);
+void ev_check_start(struct ev_loop*, struct ev_check*);
+void ev_check_stop(struct ev_loop*, struct ev_check*);
+
+void ev_fork_init(struct ev_fork*, void* callback);
+void ev_fork_start(struct ev_loop*, struct ev_fork*);
+void ev_fork_stop(struct ev_loop*, struct ev_fork*);
+
+void ev_async_init(struct ev_async*, void* callback);
+void ev_async_start(struct ev_loop*, struct ev_async*);
+void ev_async_stop(struct ev_loop*, struct ev_async*);
+void ev_async_send(struct ev_loop*, struct ev_async*);
+int ev_async_pending(struct ev_async*);
+
+void ev_child_init(struct ev_child*, void* callback, int, int);
+void ev_child_start(struct ev_loop*, struct ev_child*);
+void ev_child_stop(struct ev_loop*, struct ev_child*);
+
+void ev_stat_init(struct ev_stat*, void* callback, char*, double);
+void ev_stat_start(struct ev_loop*, struct ev_stat*);
+void ev_stat_stop(struct ev_loop*, struct ev_stat*);
+
+struct ev_loop *ev_default_loop (unsigned int flags);
+struct ev_loop* ev_loop_new(unsigned int flags);
+void ev_loop_destroy(struct ev_loop*);
+void ev_loop_fork(struct ev_loop*);
+int ev_is_default_loop (struct ev_loop *);
+unsigned int ev_iteration(struct ev_loop*);
+unsigned int ev_depth(struct ev_loop*);
+unsigned int ev_backend(struct ev_loop*);
+void ev_verify(struct ev_loop*);
+void ev_run(struct ev_loop*, int flags);
+
+ev_tstamp ev_now (struct ev_loop *);
+void ev_now_update (struct ev_loop *); /* update event loop time */
+void ev_ref(struct ev_loop*);
+void ev_unref(struct ev_loop*);
+void ev_break(struct ev_loop*, int);
+unsigned int ev_pending_count(struct ev_loop*);
+
+struct ev_loop* gevent_ev_default_loop(unsigned int flags);
+void gevent_install_sigchld_handler();
+
+void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
+void ev_sleep (ev_tstamp delay); /* sleep for a while */
+"""
+
+if sys.platform.startswith('win'):
+    # We must have the vfd_open, etc, functions on
+    # Windows. But on other platforms, going through
+    # CFFI to just return the file-descriptor is slower
+    # than just doing it in Python, so we check for and
+    # workaround their absence in corecffi.py
+    _cdef += """
+typedef int... vfd_socket_t;
+int vfd_open(vfd_socket_t);
+vfd_socket_t vfd_get(int);
+void vfd_free(int);
+"""
+
+_watcher_types = [
+    'ev_async',
+    'ev_check',
+    'ev_child',
+    'ev_fork',
+    'ev_idle',
+    'ev_io',
+    'ev_prepare',
+    'ev_signal',
+    'ev_stat',
+    'ev_timer',
+]
+
+_source = """
+// passed to the real C compiler
+#define LIBEV_EMBED 1
+#ifdef _WIN32
+#define EV_STANDALONE 1
+#include "libev_vfd.h"
+#endif
+
+
+#include "libev.h"
+
+static void
+_gevent_noop(struct ev_loop *_loop, struct ev_timer *w, int revents) { }
+
+void (*gevent_noop)(struct ev_loop *, struct ev_timer *, int) = &_gevent_noop;
+"""
+
+# Setup the watcher callbacks
+_cbs = """
+static int (*python_callback)(void* handle, int revents);
+static void (*python_handle_error)(void* handle, int revents);
+static void (*python_stop)(void* handle);
+"""
+_cdef += _cbs
+_source += _cbs
+_watcher_type = None
+
+for _watcher_type in _watcher_types:
+    _cdef += """
+   struct gevent_%s {
+        // recall that the address of a struct is the
+        // same as the address of its first member, so
+        // this struct is interchangable with the ev_XX
+        // that is its first member.
+        struct %s watcher;
+        // the CFFI handle to the Python watcher object
+        void* handle;
+        ...;
+    };
+    static void _gevent_%s_callback(struct ev_loop* loop, struct %s* watcher, int revents);
+    """ % (_watcher_type, _watcher_type, _watcher_type, _watcher_type)
+
+    _source += """
+    struct gevent_%s {
+        struct %s watcher;
+        void* handle;
+    };
+    """ % (_watcher_type, _watcher_type)
+
+    _source += """
+    static void _gevent_%s_callback(struct ev_loop* loop, struct %s* watcher, int revents)
+    {
+        // invoke self.callback()
+        void* handle = ((struct gevent_%s *)watcher)->handle;
+        int cb_result = python_callback(handle, revents);
+        switch(cb_result) {
+            case -1:
+                // in case of exception, call self.loop.handle_error;
+                // this function is also responsible for stopping the watcher
+                // and allowing memory to be freed
+                python_handle_error(handle, revents);
+                break;
+            case 0:
+                // Code to stop the event. Note that if python_callback
+                // has disposed of the last reference to the handle,
+                // `watcher` could now be invalid/disposed memory!
+                if (!ev_is_active(watcher)) {
+                    python_stop(handle);
+                }
+                break;
+            default:
+                assert(cb_result == 1);
+                // watcher is already stopped and dead, nothing to do.
+        }
+    }
+    """ % (_watcher_type, _watcher_type, _watcher_type)
+
+thisdir = os.path.dirname(os.path.realpath(__file__))
+include_dirs = [thisdir, os.path.join(thisdir, 'libev')]
+ffi.cdef(_cdef)
+ffi.set_source('gevent._corecffi', _source, include_dirs=include_dirs)
+
+if __name__ == '__main__':
+    # XXX: Note, on Windows, we would need to specify the external libraries
+    # that should be linked in, such as ws2_32 and (because libev_vfd.h makes
+    # Python.h calls) the proper Python library---at least for PyPy. I never got
+    # that to work though, and calling python functions is strongly discouraged
+    # from CFFI code.
+    ffi.compile()

gevent/core.py

@@ -3,6 +3,12 @@ from gevent.hub import PYPY
 if PYPY:
     from gevent import corecffi as _core
 else:
+    # NOTE: On CPython, this file is never imported (and there is no
+    # corecext module). Instead, the core.so file that should be build
+    # is imported in preference.
+    # NOTE: CFFI is now usable on CPython, and the performance is
+    # mostly comparable, so this could be refactored to allow that
+    # (along with the makefile, etc)
     from gevent import corecext as _core
 
 

gevent/corecffi.py

@@ -4,7 +4,6 @@ import sys
 import os
 import traceback
 import signal as signalmodule
-import struct
 
 
 __all__ = ['get_version',
@@ -15,324 +14,38 @@ __all__ = ['get_version',
            'time',
            'loop']
 
-
-def system_bits():
-    return struct.calcsize('P') * 8
-
-
-def st_nlink_type():
-    if sys.platform == "darwin":
-        return "short"
-    elif system_bits() == 32:
-        return "unsigned long"
-    return "long long"
-
-import cffi
-if cffi.__version_info__ >= (1, 2, 0):
-    # See https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in.
-    # With this version, bundled with PyPy 2.6.1 and above, we can more reliably
-    # handle signals and other exceptions. With this support, we could simplify
-    # _python_callback and _python_handle_error in addition to the simplifications for
-    # signals and KeyboardInterrupt. However, because we need to support PyPy 2.5.0+,
-    # we keep as much as practical shared.
-    _cffi_supports_on_error = True
+try:
+    import gevent._corecffi
+except ImportError:
+    # Not built yet
+    import gevent._corecffi_build
+    gevent._corecffi_build.ffi.compile()
+    import gevent._corecffi
+
+ffi = gevent._corecffi.ffi
+libev = gevent._corecffi.lib
+
+if hasattr(libev, 'vfd_open'):
+    # Must be on windows
+    assert sys.platform.startswith("win"), "vfd functions only needed on windows"
+    vfd_open = libev.vfd_open
+    vfd_free = libev.vfd_free
+    vfd_get = libev.vfd_get
 else:
-    # In older versions, including the 2.5.0 currently on Travis CI, we
-    # have to use a kludge
-    _cffi_supports_on_error = False
-
-from cffi import FFI
-ffi = FFI()
-_cdef = """
-#define EV_MINPRI ...
-#define EV_MAXPRI ...
-
-#define EV_VERSION_MAJOR ...
-#define EV_VERSION_MINOR ...
-
-#define EV_UNDEF ...
-#define EV_NONE ...
-#define EV_READ ...
-#define EV_WRITE ...
-#define EV__IOFDSET ...
-#define EV_TIMER ...
-#define EV_PERIODIC ...
-#define EV_SIGNAL ...
-#define EV_CHILD ...
-#define EV_STAT ...
-#define EV_IDLE ...
-#define EV_PREPARE ...
-#define EV_CHECK ...
-#define EV_EMBED ...
-#define EV_FORK ...
-#define EV_CLEANUP ...
-#define EV_ASYNC ...
-#define EV_CUSTOM ...
-#define EV_ERROR ...
-
-#define EVFLAG_AUTO ...
-#define EVFLAG_NOENV ...
-#define EVFLAG_FORKCHECK ...
-#define EVFLAG_NOINOTIFY ...
-#define EVFLAG_SIGNALFD ...
-#define EVFLAG_NOSIGMASK ...
-
-#define EVBACKEND_SELECT ...
-#define EVBACKEND_POLL ...
-#define EVBACKEND_EPOLL ...
-#define EVBACKEND_KQUEUE ...
-#define EVBACKEND_DEVPOLL ...
-#define EVBACKEND_PORT ...
-/* #define EVBACKEND_IOCP ... */
-
-#define EVBACKEND_ALL ...
-#define EVBACKEND_MASK ...
-
-#define EVRUN_NOWAIT ...
-#define EVRUN_ONCE ...
-
-#define EVBREAK_CANCEL ...
-#define EVBREAK_ONE ...
-#define EVBREAK_ALL ...
-
-struct ev_loop {
-    int backend_fd;
-    int activecnt;
-    ...;
-};
-struct ev_io {
-    int fd;
-    int events;
-    ...;
-};
-struct ev_timer {
-    double at;
-    ...;
-};
-struct ev_signal {...;};
-struct ev_idle  {...;};
-struct ev_prepare {...;};
-struct ev_check {...;};
-struct ev_fork  {...;};
-struct ev_async  {...;};
-struct ev_child {
-    int pid;
-    int rpid;
-    int rstatus;
-    ...;
-};
-struct stat {
-    """ + st_nlink_type() + """ st_nlink;
-    ...;
-};
-
-struct ev_stat {
-    struct stat attr;
-    const char* path;
-    struct stat prev;
-    double interval;
-    ...;
-};
-
-typedef double ev_tstamp;
-
-int ev_version_major();
-int ev_version_minor();
-
-unsigned int ev_supported_backends (void);
-unsigned int ev_recommended_backends (void);
-unsigned int ev_embeddable_backends (void);
-
-ev_tstamp ev_time (void);
-void ev_set_syserr_cb(void *);
-
-int ev_priority(void*);
-void ev_set_priority(void*, int);
-
-int ev_is_pending(void*);
-int ev_is_active(void*);
-void ev_io_init(struct ev_io*, void* callback, int fd, int events);
-void ev_io_start(struct ev_loop*, struct ev_io*);
-void ev_io_stop(struct ev_loop*, struct ev_io*);
-void ev_feed_event(struct ev_loop*, void*, int);
-
-void ev_timer_init(struct ev_timer*, void (*callback)(struct ev_loop *_loop, struct ev_timer *w, int revents), double, double);
-void ev_timer_start(struct ev_loop*, struct ev_timer*);
-void ev_timer_stop(struct ev_loop*, struct ev_timer*);
-void ev_timer_again(struct ev_loop*, struct ev_timer*);
-
-void ev_signal_init(struct ev_signal*, void* callback, int);
-void ev_signal_start(struct ev_loop*, struct ev_signal*);
-void ev_signal_stop(struct ev_loop*, struct ev_signal*);
-
-void ev_idle_init(struct ev_idle*, void* callback);
-void ev_idle_start(struct ev_loop*, struct ev_idle*);
-void ev_idle_stop(struct ev_loop*, struct ev_idle*);
-
-void ev_prepare_init(struct ev_prepare*, void* callback);
-void ev_prepare_start(struct ev_loop*, struct ev_prepare*);
-void ev_prepare_stop(struct ev_loop*, struct ev_prepare*);
-
-void ev_check_init(struct ev_check*, void* callback);
-void ev_check_start(struct ev_loop*, struct ev_check*);
-void ev_check_stop(struct ev_loop*, struct ev_check*);
-
-void ev_fork_init(struct ev_fork*, void* callback);
-void ev_fork_start(struct ev_loop*, struct ev_fork*);
-void ev_fork_stop(struct ev_loop*, struct ev_fork*);
-
-void ev_async_init(struct ev_async*, void* callback);
-void ev_async_start(struct ev_loop*, struct ev_async*);
-void ev_async_stop(struct ev_loop*, struct ev_async*);
-void ev_async_send(struct ev_loop*, struct ev_async*);
-int ev_async_pending(struct ev_async*);
-
-void ev_child_init(struct ev_child*, void* callback, int, int);
-void ev_child_start(struct ev_loop*, struct ev_child*);
-void ev_child_stop(struct ev_loop*, struct ev_child*);
-
-void ev_stat_init(struct ev_stat*, void* callback, char*, double);
-void ev_stat_start(struct ev_loop*, struct ev_stat*);
-void ev_stat_stop(struct ev_loop*, struct ev_stat*);
-
-struct ev_loop *ev_default_loop (unsigned int flags);
-struct ev_loop* ev_loop_new(unsigned int flags);
-void ev_loop_destroy(struct ev_loop*);
-void ev_loop_fork(struct ev_loop*);
-int ev_is_default_loop (struct ev_loop *);
-unsigned int ev_iteration(struct ev_loop*);
-unsigned int ev_depth(struct ev_loop*);
-unsigned int ev_backend(struct ev_loop*);
-void ev_verify(struct ev_loop*);
-void ev_run(struct ev_loop*, int flags);
-
-ev_tstamp ev_now (struct ev_loop *);
-void ev_now_update (struct ev_loop *); /* update event loop time */
-void ev_ref(struct ev_loop*);
-void ev_unref(struct ev_loop*);
-void ev_break(struct ev_loop*, int);
-unsigned int ev_pending_count(struct ev_loop*);
-
-struct ev_loop* gevent_ev_default_loop(unsigned int flags);
-void gevent_install_sigchld_handler();
-
-void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
-void ev_sleep (ev_tstamp delay); /* sleep for a while */
-"""
-
-
-_watcher_types = [
-    'ev_async',
-    'ev_check',
-    'ev_child',
-    'ev_fork',
-    'ev_idle',
-    'ev_io',
-    'ev_prepare',
-    'ev_signal',
-    'ev_stat',
-    'ev_timer',
-]
-
-_source = """   // passed to the real C compiler
-#define LIBEV_EMBED 1
-#include "libev.h"
-
-static void
-_gevent_noop(struct ev_loop *_loop, struct ev_timer *w, int revents) { }
-
-void (*gevent_noop)(struct ev_loop *, struct ev_timer *, int) = &_gevent_noop;
-"""
-
-# Setup the watcher callbacks
-_cbs = """
-static int (*python_callback)(void* handle, int revents);
-static void (*python_handle_error)(void* handle, int revents);
-static void (*python_stop)(void* handle);
-"""
-_cdef += _cbs
-_source += _cbs
-_watcher_type = None
-
-for _watcher_type in _watcher_types:
-    _cdef += """
-   struct gevent_%s {
-        // recall that the address of a struct is the
-        // same as the address of its first member, so
-        // this struct is interchangable with the ev_XX
-        // that is its first member.
-        struct %s watcher;
-        // the CFFI handle to the Python watcher object
-        void* handle;
-        ...;
-    };
-    static void _gevent_%s_callback(struct ev_loop* loop, struct %s* watcher, int revents);
-    """ % (_watcher_type, _watcher_type, _watcher_type, _watcher_type)
-
-    _source += """
-    struct gevent_%s {
-        struct %s watcher;
-        void* handle;
-    };
-    """ % (_watcher_type, _watcher_type)
-
-    _source += """
-    static void _gevent_%s_callback(struct ev_loop* loop, struct %s* watcher, int revents)
-    {
-        // invoke self.callback()
-        void* handle = ((struct gevent_%s *)watcher)->handle;
-        int cb_result = python_callback(handle, revents);
-        switch(cb_result) {
-            case -1:
-                // in case of exception, call self.loop.handle_error;
-                // this function is also responsible for stopping the watcher
-                // and allowing memory to be freed
-                python_handle_error(handle, revents);
-                break;
-            case 0:
-                // Code to stop the event. Note that if python_callback
-                // has disposed of the last reference to the handle,
-                // `watcher` could now be invalid/disposed memory!
-                if (!ev_is_active(watcher)) {
-                    python_stop(handle);
-                }
-                break;
-            default:
-                assert(cb_result == 1);
-                // watcher is already stopped and dead, nothing to do.
-        }
-    }
-    """ % (_watcher_type, _watcher_type, _watcher_type)
-
-thisdir = os.path.dirname(os.path.realpath(__file__))
-include_dirs = [thisdir, os.path.join(thisdir, 'libev')]
+    vfd_open = vfd_free = vfd_get = lambda fd: fd
+
 #####
-## XXX NOTE:
+## NOTE on Windows:
+# The C implementation does several things specially for Windows;
+# a possibly incomplete list is:
 #
-# In recent versions of CFFI on CPython, doing something like
-# `libev._gevent_ev_io_callback` returns <built-in function
-# _gevent_ev_io_callback> ("for performance"). These have the
-# important property of not being able to pass them to any other libev
-# functions without generating `TypeError: Expected cdata`.
-# On PyPy, the same expression returns a cdata, and we rely on this below
-# (see watcher._init_subclasses).
+# - the loop runs a periodic signal checker;
+# - the io watcher constructor is different and it has a destructor;
+# - the child watcher is not defined
 #
-# The behaviour is the same though (they both return <built-in function>)
-# when you use `ffi.set_source` instead of the deprecated `ffi.verify` function.
-# In order to pass such attributes to other libev functions, you have to do
-# `ffi.addressof(libev, '_gevent_ev_io_callback)`... Unfortunately, this fails
-# when you use `ffi.verify`. So:
-#
-# - Using FFI on CPython cannot use ffi.verify();
-# - Using set_source() and not verify() requires PyPy >= 2.6.0
-# - Upgrading to set_source() will require moderate source changes;
+# The CFFI implementation does none of these things, and so
+# is possibly NOT FUNCTIONALLY CORRECT on Win32
 #####
-ffi.cdef(_cdef)
-libev = C = ffi.verify(_source, include_dirs=include_dirs)
-del thisdir, include_dirs, _watcher_type, _watcher_types
-
-libev.vfd_open = libev.vfd_get = lambda fd: fd
-libev.vfd_free = lambda fd: None
 
 #####
 ## Note on CFFI objects, callbacks and the lifecycle of watcher objects
@@ -390,7 +103,7 @@ def _python_callback(handle, revents):
     try:
         # Even dereferencing the handle needs to be inside the try/except;
         # if we don't return normally (e.g., a signal) then we wind up going
-        # to the 'onerror' handler if _cffi_supports_on_error is True, which
+        # to the 'onerror' handler, which
         # is not what we want; that can permanently wedge the loop depending
         # on which callback was executing
         watcher = ffi.from_handle(handle)
@@ -483,21 +196,21 @@ EVENTS = GEVENT_CORE_EVENTS = _EVENTSType()
 
 
 def get_version():
-    return 'libev-%d.%02d' % (C.ev_version_major(), C.ev_version_minor())
+    return 'libev-%d.%02d' % (libev.ev_version_major(), libev.ev_version_minor())
 
 
 def get_header_version():
-    return 'libev-%d.%02d' % (C.EV_VERSION_MAJOR, C.EV_VERSION_MINOR)
-
-_flags = [(C.EVBACKEND_PORT, 'port'),
-          (C.EVBACKEND_KQUEUE, 'kqueue'),
-          (C.EVBACKEND_EPOLL, 'epoll'),
-          (C.EVBACKEND_POLL, 'poll'),
-          (C.EVBACKEND_SELECT, 'select'),
-          (C.EVFLAG_NOENV, 'noenv'),
-          (C.EVFLAG_FORKCHECK, 'forkcheck'),
-          (C.EVFLAG_SIGNALFD, 'signalfd'),
-          (C.EVFLAG_NOSIGMASK, 'nosigmask')]
+    return 'libev-%d.%02d' % (libev.EV_VERSION_MAJOR, libev.EV_VERSION_MINOR)
+
+_flags = [(libev.EVBACKEND_PORT, 'port'),
+          (libev.EVBACKEND_KQUEUE, 'kqueue'),
+          (libev.EVBACKEND_EPOLL, 'epoll'),
+          (libev.EVBACKEND_POLL, 'poll'),
+          (libev.EVBACKEND_SELECT, 'select'),
+          (libev.EVFLAG_NOENV, 'noenv'),
+          (libev.EVFLAG_FORKCHECK, 'forkcheck'),
+          (libev.EVFLAG_SIGNALFD, 'signalfd'),
+          (libev.EVFLAG_NOSIGMASK, 'nosigmask')]
 
 _flags_str2int = dict((string, flag) for (flag, string) in _flags)
 
@@ -604,15 +317,12 @@ def embeddable_backends():
 
 
 def time():
-    return C.ev_time()
+    return libev.ev_time()
 
 _default_loop_destroyed = False
 
 
 def _loop_callback(*args, **kwargs):
-    if _cffi_supports_on_error:
-        return ffi.callback(*args, **kwargs)
-    kwargs.pop('onerror')
     return ffi.callback(*args, **kwargs)
 
 
@@ -680,32 +390,15 @@ class loop(object):
 
         self._keepaliveset = set()
 
-        if not _cffi_supports_on_error:
-            if default:
-                signalmodule.signal(2, self.int_handler)
-            self.ate_keyboard_interrupt = False
-            self.keyboard_interrupt_allowed = True
-
     def _check_callback_handle_error(self, t, v, tb):
         # None as the context argument causes the exception to be raised
         # in the main greenlet.
         self.handle_error(None, t, v, tb)
 
-    if _cffi_supports_on_error:
-        def _check_callback(self, *args):
-            # If we have the onerror callback, this is a no-op; all the real
-            # work to rethrow the exception is done by the onerror callback
-            pass
-    else:
-        def _check_callback(self, *args):
-            if self.ate_keyboard_interrupt:
-                self.handle_error(self, KeyboardInterrupt, KeyboardInterrupt(), None)
-                self.ate_keyboard_interrupt = False
-
-        def int_handler(self, *args):
-            if self.keyboard_interrupt_allowed:
-                raise KeyboardInterrupt
-            self.ate_keyboard_interrupt = True
+    def _check_callback(self, *args):
+        # If we have the onerror callback, this is a no-op; all the real
+        # work to rethrow the exception is done by the onerror callback
+        pass
 
     def _run_callbacks(self, evloop, _, revents):
         count = 1000
@@ -724,7 +417,6 @@ class loop(object):
                 cb.callback = None
 
                 try:
-                    self.keyboard_interrupt_allowed = True
                     callback(*args)
                 except:
                     # If we allow an exception to escape this method (while we are running the ev callback),
@@ -751,7 +443,6 @@ class loop(object):
                         except:
                             pass # Nothing we can do here
                 finally:
-                    self.keyboard_interrupt_allowed = False
                     # Note, this must be reset here, because cb.args is used as a flag in callback class,
                     cb.args = None
                     count -= 1
@@ -776,7 +467,6 @@ class loop(object):
                 _default_loop_destroyed = True
             libev.ev_loop_destroy(self._ptr)
             self._ptr = ffi.NULL
-        # XXX restore default_int_signal handler if we set it (_cffi_supports_on_error is False)
 
     @property
     def ptr(self):
@@ -1059,7 +749,8 @@ class watcher(object):
         for subclass in cls.__subclasses__():
             watcher_type = subclass._watcher_type
             subclass._watcher_struct_pointer_type = ffi.typeof('struct gevent_' + watcher_type + '*')
-            subclass._watcher_callback = getattr(libev, '_gevent_' + watcher_type + '_callback')
+            subclass._watcher_callback = ffi.addressof(libev,
+                                                       '_gevent_' + watcher_type + '_callback')
             for name in 'start', 'stop', 'init':
                 ev_name = watcher_type + '_' + name
                 watcher_name = '_watcher' + '_' + name
@@ -1174,6 +865,8 @@ class io(watcher):
     _watcher_type = 'ev_io'
 
     def __init__(self, loop, fd, events, ref=True, priority=None):
+        # XXX: Win32: Need to vfd_open the fd and free the old one?
+        # XXX: Win32: Need a destructor to free the old fd?
         if fd < 0:
             raise ValueError('fd must be non-negative: %r' % fd)
         if events & ~(libev.EV__IOFDSET | libev.EV_READ | libev.EV_WRITE):
@@ -1187,24 +880,24 @@ class io(watcher):
         watcher.start(self, callback, *args)
 
     def _get_fd(self):
-        return libev.vfd_get(self._watcher.fd)
+        return vfd_get(self._watcher.fd)
 
     def _set_fd(self, fd):
         if libev.ev_is_active(self._watcher):
             raise AttributeError("'io' watcher attribute 'fd' is read-only while watcher is active")
-        vfd = libev.vfd_open(fd)
-        libev.vfd_free(self._watcher.fd)
-        libev.ev_io_init(self._watcher, self._cb, vfd, self._watcher.events)
+        vfd = vfd_open(fd)
+        vfd_free(self._watcher.fd)
+        self._watcher_init(self._watcher, self._watcher_callback, vfd, self._watcher.events)
 
     fd = property(_get_fd, _set_fd)
 
     def _get_events(self):
-        return libev.vfd_get(self._watcher.fd)
+        return self._watcher.events
 
     def _set_events(self, events):
         if libev.ev_is_active(self._watcher):
             raise AttributeError("'io' watcher attribute 'events' is read-only while watcher is active")
-        libev.ev_io_init(self._watcher, self._cb, self._watcher.fd, events)
+        self._watcher_init(self._watcher, self._watcher_callback, self._watcher.fd, events)
 
     events = property(_get_events, _set_events)
 

greentest/greentest.py

@@ -479,7 +479,8 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
             x = fn[:-3]
             if x.endswith('_d'):
                 x = x[:-2]
-            if x in ['__init__', 'core', 'ares', '_util', '_semaphore', 'corecffi']:
+            if x in ['__init__', 'core', 'ares', '_util', '_semaphore',
+                     'corecffi', '_corecffi', '_corecffi_build']:
                 continue
             if x in OPTIONAL_MODULES:
                 try:
@@ -488,6 +489,8 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
                     continue
             yield path, modpath + x
         elif include_so and fn.endswith('.so'):
+            if '.pypy-' in fn:
+                continue
             if fn.endswith('_d.so'):
                 yield path, modpath + fn[:-5]
             else:

greentest/test__core.py

@@ -36,10 +36,21 @@ class Test(TestCase):
     def test_io(self):
         if sys.platform == 'win32':
             Error = IOError
+            win32 = True
         else:
             Error = ValueError
+            win32 = False
         self.assertRaises(Error, core.loop().io, -1, 1)
         self.assertRaises(ValueError, core.loop().io, 1, core.TIMER)
+        # Test we can set events and io before it's started
+        if not win32:
+            # We can't do this with arbitrary FDs on windows;
+            # see libev_vfd.h
+            io = core.loop().io(1, core.READ)
+            io.fd = 2
+            self.assertEqual(io.fd, 2)
+            io.events = core.WRITE
+            self.assertEqual(core._events_to_str(io.events), 'WRITE|_IOFDSET')
 
     def test_timer(self):
         self.assertRaises(ValueError, core.loop().timer, 1, -1)

greentest/test__execmodules.py

@@ -12,7 +12,7 @@ def make_exec_test(path, module):
         #sys.stderr.write('%s %s\n' % (module, path))
         with open(path, 'rb') as f:
             src = f.read()
-        six.exec_(src, {})
+        six.exec_(src, {'__file__': path})
 
     name = "test_" + module.replace(".", "_")
     test.__name__ = name

scripts/install.sh

+#!/usr/bin/env bash
+# GEVENT: Taken from https://raw.githubusercontent.com/DRMacIver/hypothesis/master/scripts/install.sh
+
+# Special license: Take literally anything you want out of this file. I don't
+# care. Consider it WTFPL licensed if you like.
+# Basically there's a lot of suffering encoded here that I don't want you to
+# have to go through and you should feel free to use this to avoid some of
+# that suffering in advance.
+
+set -e
+set -x
+
+# This is to guard against multiple builds in parallel. The various installers will tend
+# to stomp all over eachother if you do this and they haven't previously successfully
+# succeeded. We use a lock file to block progress so only one install runs at a time.
+# This script should be pretty fast once files are cached, so the lost of concurrency
+# is not a major problem.
+# This should be using the lockfile command, but that's not available on the
+# containerized travis and we can't install it without sudo.
+# Is is unclear if this is actually useful. I was seeing behaviour that suggested
+# concurrent runs of the installer, but I can't seem to find any evidence of this lock
+# ever not being acquired.
+
+BASE=${BUILD_RUNTIMES-$PWD/.runtimes}
+echo $BASE
+mkdir -p $BASE
+
+LOCKFILE="$BASE/.install-lockfile"
+while true; do
+  if mkdir $LOCKFILE 2>/dev/null; then
+    echo "Successfully acquired installer."
+    break
+  else
+    echo "Failed to acquire lock. Is another installer running? Waiting a bit."
+  fi
+
+  sleep $[ ( $RANDOM % 10)  + 1 ].$[ ( $RANDOM % 100) ]s
+
+  if (( $(date '+%s') > 300 + $(stat --format=%X $LOCKFILE) )); then
+    echo "We've waited long enough"
+    rm -rf $LOCKFILE
+  fi
+done
+trap "rm -rf $LOCKFILE" EXIT
+
+
+PYENV=$BASE/pyenv
+
+
+if [ ! -d "$PYENV/.git" ]; then
+  rm -rf $PYENV
+  git clone https://github.com/yyuu/pyenv.git $BASE/pyenv
+else
+  back=$PWD
+  cd $PYENV
+  git fetch || echo "Update failed to complete. Ignoring"
+  git reset --hard origin/master
+  cd $back
+fi
+
+SNAKEPIT=$BASE/snakepit
+
+install () {
+
+  VERSION="$1"
+  ALIAS="$2"
+  mkdir -p $BASE/versions
+  SOURCE=$BASE/versions/$ALIAS
+
+  if [ ! -e "$SOURCE" ]; then
+    mkdir -p $SNAKEPIT
+    mkdir -p $BASE/versions
+    $BASE/pyenv/plugins/python-build/bin/python-build $VERSION $SOURCE
+  fi
+ rm -f $SNAKEPIT/$ALIAS
+ mkdir -p $SNAKEPIT
+ $SOURCE/bin/python -m pip.__main__ install --upgrade pip wheel virtualenv
+ ln -s $SOURCE/bin/python $SNAKEPIT/$ALIAS
+}
+
+
+for var in "$@"; do
+  case "${var}" in
+    2.6)
+      install 2.6.9 python2.6
+      ;;
+    2.7)
+      install 2.7.9 python2.7
+      ;;
+    3.2)
+      install 3.2.6 python3.2
+      ;;
+    3.3)
+      install 3.3.6 python3.3
+      ;;
+    3.4)
+      install 3.4.3 python3.4
+      ;;
+    3.5)
+      install 3.5.0 python3.5
+      ;;
+    pypy)
+      install pypy-4.0.1 pypy
+      ;;
+  esac
+done

setup.py

@@ -10,12 +10,32 @@ from os.path import join, abspath, basename, dirname
 from subprocess import check_call
 from glob import glob
 
+PYPY = hasattr(sys, 'pypy_version_info')
+WIN = sys.platform.startswith('win')
+
+if PYPY and WIN and not os.environ.get("PYPY_WIN_BUILD_ANYWAY"):
+    # We can't properly handle (hah!) file-descriptors and
+    # handle mapping on Windows/CFFI, because the file needed,
+    # libev_vfd.h, can't be included, linked, and used: it uses
+    # Python API functions, and you're not supposed to do that from
+    # CFFI code. Plus I could never get the libraries= line to ffi.compile()
+    # correct to make linking work.
+    raise Exception("Unable to install on PyPy/Windows")
+
+if WIN:
+    # https://bugs.python.org/issue23246
+    # We must have setuptools on windows
+    __import__('setuptools')
+
+    # Make sure the env vars that make.cmd needs are set
+    if not os.environ.get('PYTHON_EXE'):
+        os.environ['PYTHON_EXE'] = 'pypy' if PYPY else 'python'
+
+
 import distutils
 import distutils.sysconfig  # to get CFLAGS to pass into c-ares configure script
 
 
-PYPY = hasattr(sys, 'pypy_version_info')
-
 try:
     from setuptools import Extension, setup
 except ImportError:
@@ -23,6 +43,7 @@ except ImportError:
         # need setuptools for include_package_data to work
         raise
     from distutils.core import Extension, setup
+
 from distutils.command.build_ext import build_ext
 from distutils.command.sdist import sdist as _sdist
 from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError
@@ -84,7 +105,7 @@ ares_configure_command = ' '.join(["(cd ", _quoted_abspath('c-ares/'),
                                    "> configure-output.txt"])
 
 
-if sys.platform == 'win32':
+if WIN:
     libraries += ['ws2_32']
     define_macros += [('FD_SETSIZE', '1024'), ('_WIN32', '1')]
 
@@ -144,7 +165,7 @@ def system(cmd):
 
 
 def configure_libev(bext, ext):
-    if sys.platform == "win32":
+    if WIN:
         CORE.define_macros.append(('EV_STANDALONE', '1'))
         return
 
@@ -173,7 +194,7 @@ def configure_ares(bext, ext):
     if not os.path.isdir(bdir):
         os.makedirs(bdir)
 
-    if sys.platform == "win32":
+    if WIN:
         shutil.copy("c-ares\\ares_build.h.dist", os.path.join(bdir, "ares_build.h"))
         return
 
@@ -209,7 +230,7 @@ else:
 if CARES_EMBED:
     ARES.sources += expand('c-ares/*.c')
     ARES.configure = configure_ares
-    if sys.platform == 'win32':
+    if WIN:
         ARES.libraries += ['advapi32']
         ARES.define_macros += [('CARES_STATICLIB', '')]
     else:
@@ -225,11 +246,18 @@ _ran_make = []
 
 
 def make(targets=''):
+    # NOTE: We have two copies of the makefile, one
+    # for posix, one for windows
     if not _ran_make:
-        if os.path.exists('Makefile'):
-            if "PYTHON" not in os.environ:
-                os.environ["PYTHON"] = sys.executable
-            system('make ' + targets)
+        if WIN:
+            # make.cmd handles checking for PyPy and only making the
+            # right things, so we can ignore the targets
+            system("appveyor\\make.cmd")
+        else:
+            if os.path.exists('Makefile'):
+                if "PYTHON" not in os.environ:
+                    os.environ["PYTHON"] = sys.executable
+                system('make ' + targets)
         _ran_make.append(1)
 
 
@@ -315,8 +343,12 @@ def read(name, *args):
 
 if PYPY:
     install_requires = []
+    cffi_modules = ['gevent/_corecffi_build.py:ffi']
+    setup_kwds = {'cffi_modules': cffi_modules}
 else:
     install_requires = ['greenlet >= 0.4.9']
+    setup_kwds = {}
+
 
 # If we are running info / help commands, or we're being imported by
 # tools like pyroma, we don't need to build anything
@@ -334,21 +366,35 @@ if ((len(sys.argv) >= 2
 elif PYPY:
     sys.path.insert(0, '.')
     # XXX ugly - need to find a better way
-    system('cp -r libev gevent/libev')
-    system('touch gevent/libev/__init__.py')
-    system('cd gevent/libev && ./configure > configure_output.txt')
-    from gevent import corecffi
-    ext_modules = [corecffi.ffi.verifier.get_extension(),
-                   ARES,
-                   # By building the semaphore with Cython under PyPy, we get
-                   # atomic operations (specifically, exiting/releasing), at the
-                   # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
-                   # at around 1s and the compiled version at around 4s). Some clever subclassing
-                   # and having only the bare minimum be in cython might help reduce that penalty.
-                   # NOTE: You must use version 0.23.4 or later to avoid a memory leak.
-                   # https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
-                   Extension(name="gevent._semaphore",
-                             sources=["gevent/gevent._semaphore.c"])]
+    cp_cmd = 'cp -r'
+    if WIN:
+        cp_cmd = "copy"
+    system(cp_cmd + ' libev gevent/libev')
+    if WIN:
+        system('echo > gevent/libev/__init__.py')
+    else:
+        system('touch gevent/libev/__init__.py')
+    if sys.platform != 'win32':
+        system('cd gevent/libev && ./configure > configure_output.txt')
+    # NOTE that we're NOT adding the distutils extension module, as
+    # doing so compiles the module already: import gevent._corecffi_build
+    # imports gevent, which imports the hub, which imports the core,
+    # which compiles the module in-place. Instead we use the setup-time
+    # support of cffi_modules
+    #from gevent import _corecffi_build
+    ext_modules = [
+        #_corecffi_build.ffi.distutils_extension(),
+        ARES,
+        # By building the semaphore with Cython under PyPy, we get
+        # atomic operations (specifically, exiting/releasing), at the
+        # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
+        # at around 1s and the compiled version at around 4s). Some clever subclassing
+        # and having only the bare minimum be in cython might help reduce that penalty.
+        # NOTE: You must use version 0.23.4 or later to avoid a memory leak.
+        # https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
+        Extension(name="gevent._semaphore",
+                  sources=["gevent/gevent._semaphore.c"]),
+    ]
     include_package_data = True
     run_make = 'gevent/gevent._semaphore.c gevent/gevent.ares.c'
 else:
@@ -371,7 +417,7 @@ else:
 
 
 def run_setup(ext_modules, run_make):
-    if run_make and not os.environ.get('APPVEYOR'):
+    if run_make:
         if isinstance(run_make, str):
             make(run_make)
         else:
@@ -409,7 +455,8 @@ def run_setup(ext_modules, run_make):
             "Topic :: Internet",
             "Topic :: Software Development :: Libraries :: Python Modules",
             "Intended Audience :: Developers",
-            "Development Status :: 4 - Beta"]
+            "Development Status :: 4 - Beta"],
+        **setup_kwds
     )
 
 # Tools like pyroma expect the actual call to `setup` to be performed

util/cythonpp.py

@@ -526,11 +526,11 @@ class Str_sourceline(str):
 
 def atomic_write(filename, data):
     tmpname = filename + '.tmp.%s' % os.getpid()
-    f = open(tmpname, 'w')
-    f.write(data)
-    f.flush()
-    os.fsync(f.fileno())
-    f.close()
+    with open(tmpname, 'w') as f:
+        f.write(data)
+        f.flush()
+        os.fsync(f.fileno())
+
     if os.path.exists(filename):
         os.unlink(filename)
     os.rename(tmpname, filename)
@@ -571,35 +571,35 @@ def postprocess_cython_output(filename, banner):
     #    confuse merger
     result = ['/* %s */\n' % (banner)]
 
-    input = open(filename)
-    firstline = input.readline()
+    with open(filename) as finput:
+        firstline = finput.readline()
 
-    m = cython_header_re.match(firstline.strip())
-    if m:
-        result.append('/* %s */' % m.group(1))
-    else:
-        result.append(firstline)
+        m = cython_header_re.match(firstline.strip())
+        if m:
+            result.append('/* %s */' % m.group(1))
+        else:
+            result.append(firstline)
 
-    in_comment = False
-    for line in input:
+        in_comment = False
+        for line in finput:
 
-        if line.endswith('\n'):
-            line = line[:-1].rstrip() + '\n'
+            if line.endswith('\n'):
+                line = line[:-1].rstrip() + '\n'
 
-        if in_comment:
-            if '*/' in line:
-                in_comment = False
-                result.append(line)
-            else:
-                result.append(line.replace('\n', newline_token))
-        else:
-            if line.lstrip().startswith('/* ') and '*/' not in line:
-                line = line.lstrip()  # cython adds space before /* for some reason
-                line = line.replace('\n', newline_token)
-                result.append(line)
-                in_comment = True
+            if in_comment:
+                if '*/' in line:
+                    in_comment = False
+                    result.append(line)
+                else:
+                    result.append(line.replace('\n', newline_token))
             else:
-                result.append(line)
+                if line.lstrip().startswith('/* ') and '*/' not in line:
+                    line = line.lstrip()  # cython adds space before /* for some reason
+                    line = line.replace('\n', newline_token)
+                    result.append(line)
+                    in_comment = True
+                else:
+                    result.append(line)
     return ''.join(result)