Trying the pyvenv approach to a travis matrix

.gitignore

@@ -2,6 +2,7 @@
 build/
 *.so
 *.pyd
+.runtimes
 gevent.*.[ch]
 gevent/core.pyx
 gevent/__pycache__

.travis.yml

-language: python
+# .travis.yml based on https://github.com/DRMacIver/hypothesis/blob/master/.travis.yml
+language: c
 sudo: false
-python:
-    - 2.6
-    - 2.7
-    - pypy
-    - 3.3
-    - 3.4
-    - 3.5
+
 env:
-    - LINT=true
-    - LINT=false
-install:
-    # First install a newer pip so that it can use the wheel cache
-    # (only needed until travis upgrades pip to 7.x; note that the 3.5
-    # environment uses pip 7.1 by default)
-    - travis_retry pip install -U pip
-    # Then start installing our deps so they can be cached. Note that use of --build-options / --global-options / --install-options
-    # disables the cache.
-    # We need wheel>=0.26 on Python 3.5. See previous revisions.
-    - travis_retry pip install -U wheel
-    - travis_retry pip install -U tox cython greenlet pep8 pyflakes "coverage>=4.0" "coveralls>=1.0"
+    global:
+        - BUILD_RUNTIMES=$HOME/.runtimes
+
+    matrix:
+      - TASK=lint-py27
+      - TASK=test-py27
+      - TASK=test-py26
+      - TASK=test-py33
+      - TASK=test-py34
+      - TASK=test-py35
+      - TASK=test-pypy
+
+matrix:
+  fast_finish: true
+
 script:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $LINT == true ]]; then python setup.py develop && make travis_test_linters; elif [[ $LINT == false && $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then python setup.py develop && make fulltoxtest; elif [[ $LINT == false ]]; then python setup.py develop && make fulltoxtest; fi
+    - make $TASK
+
 notifications:
   email: false
+
 # cache: pip seems not to work if `install` is replaced (https://github.com/travis-ci/travis-ci/issues/3239)
 cache:
   directories:
     - $HOME/.cache/pip
+    - $HOME/.venv
+    - $HOME/.runtimes
+    - $HOME/.wheelhouse
+
 before_cache:
     - rm -f $HOME/.cache/pip/log/debug.log

Makefile

 # This file is renamed to "Makefile.ext" in release tarballs so that setup.py won't try to
 # run it.  If you want setup.py to run "make" automatically, rename it back to "Makefile".
 
-PYTHON ?= python${TRAVIS_PYTHON_VERSION}
-CYTHON ?= cython
+# The pyvenv multiple runtime support is based on https://github.com/DRMacIver/hypothesis/blob/master/Makefile
+
+PYTHON?=python${TRAVIS_PYTHON_VERSION}
+CYTHON?=cython
+
+
+
+export PATH:=$(BUILD_RUNTIMES)/snakepit:$(TOOLS):$(PATH)
+export LC_ALL=C.UTF-8
+
 
 all: gevent/gevent.corecext.c gevent/gevent.ares.c gevent/gevent._semaphore.c gevent/gevent._util.c
 
@@ -71,9 +79,11 @@ toxtest:
 	cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
 
 fulltoxtest:
-	cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
-	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 python testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
-	cd greentest && GEVENT_FILE=thread python testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
+	which ${PYTHON}
+	${PYTHON} --version
+	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
+	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
+	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
 
 leaktest:
 	GEVENTSETUP_EV_VERIFY=3 GEVENTTEST_LEAKCHECK=1 make travistest
@@ -100,3 +110,78 @@ travis_test_linters:
 
 
 .PHONY: clean all doc pep8 whitespace pyflakes lint travistest travis
+
+# Managing runtimes
+
+BUILD_RUNTIMES?=$(PWD)/.runtimes
+
+PY26=$(BUILD_RUNTIMES)/snakepit/python2.6
+PY27=$(BUILD_RUNTIMES)/snakepit/python2.7
+PY33=$(BUILD_RUNTIMES)/snakepit/python3.3
+PY34=$(BUILD_RUNTIMES)/snakepit/python3.4
+PY35=$(BUILD_RUNTIMES)/snakepit/python3.5
+PYPY=$(BUILD_RUNTIMES)/snakepit/pypy
+
+TOOLS=$(BUILD_RUNTIMES)/tools
+
+TOX=$(TOOLS)/tox
+
+TOOL_VIRTUALENV=$(BUILD_RUNTIMES)/virtualenvs/tools
+ISORT_VIRTUALENV=$(BUILD_RUNTIMES)/virtualenvs/isort
+TOOL_PYTHON=$(TOOL_VIRTUALENV)/bin/python
+TOOL_PIP=$(TOOL_VIRTUALENV)/bin/pip
+TOOL_INSTALL=$(TOOL_PIP) install --upgrade
+
+$(PY26):
+	scripts/install.sh 2.6
+
+$(PY27):
+	scripts/install.sh 2.7
+
+$(PY33):
+	scripts/install.sh 3.3
+
+$(PY34):
+	scripts/install.sh 3.4
+
+$(PY35):
+	scripts/install.sh 3.5
+
+$(PYPY):
+	scripts/install.sh pypy
+
+PIP?=$(BUILD_RUNTIMES)/versions/$(PYTHON)/bin/pip
+
+develop:
+	echo $(PIP) $(PYTHON)
+# First install a newer pip so that it can use the wheel cache
+# (only needed until travis upgrades pip to 7.x; note that the 3.5
+# environment uses pip 7.1 by default)
+	${PIP} install -U pip
+# Then start installing our deps so they can be cached. Note that use of --build-options / --global-options / --install-options
+# disables the cache.
+# We need wheel>=0.26 on Python 3.5. See previous revisions.
+	${PIP} install -U wheel
+	${PIP} install -U tox cython greenlet pep8 pyflakes "coverage>=4.0" "coveralls>=1.0"
+	${PYTHON} setup.py develop
+
+lint-py27: $(PY27)
+	PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop travis_test_linters
+
+test-py27: $(PY27)
+	PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop fulltoxtest
+
+test-py26: $(PY26)
+	PYTHON=python2.6 PATH=$(BUILD_RUNTIMES)/versions/python2.6/bin:$(PATH) make develop fulltoxtest
+
+test-py33: $(PY33)
+	PYTHON=python3.3 PATH=$(BUILD_RUNTIMES)/versions/python3.3/bin:$(PATH) make develop fulltoxtest
+
+test-py34: $(PY34)
+	PYTHON=python3.4 PATH=$(BUILD_RUNTIMES)/versions/python3.4/bin:$(PATH) make develop fulltoxtest
+
+test-py35: $(PY35)
+	PYTHON=python3.5 PATH=$(BUILD_RUNTIMES)/versions/python3.5/bin:$(PATH) make develop fulltoxtest
+
+test-pypy: $(PYPY)
+	PYTHON=pypy PATH=$(BUILD_RUNTIMES)/versions/pypy/bin:$(PATH) make develop fulltoxtest

appveyor.yml

@@ -27,35 +27,35 @@ environment:
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python35-x64"
-    #   PYTHON_VERSION: "3.5.x" # currently 3.5.0
-    #   PYTHON_ARCH: "64"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python35-x64"
+      PYTHON_VERSION: "3.5.x" # currently 3.5.0
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python27"
-    #   PYTHON_VERSION: "2.7.x" # currently 2.7.9
-    #   PYTHON_ARCH: "32"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7.x" # currently 2.7.9
+      PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python33"
-    #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
-    #  PYTHON_ARCH: "32"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python33"
+      PYTHON_VERSION: "3.3.x" # currently 3.3.5
+      PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python33-x64"
-    #  PYTHON_VERSION: "3.3.x" # currently 3.3.5
-    #  PYTHON_ARCH: "64"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python33-x64"
+      PYTHON_VERSION: "3.3.x" # currently 3.3.5
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python34"
-    #   PYTHON_VERSION: "3.4.x" # currently 3.4.3
-    #   PYTHON_ARCH: "32"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python34"
+      PYTHON_VERSION: "3.4.x" # currently 3.4.3
+      PYTHON_ARCH: "32"
+      PYTHON_EXE: python
 
-    # - PYTHON: "C:\\Python34-x64"
-    #   PYTHON_VERSION: "3.4.x" # currently 3.4.3
-    #   PYTHON_ARCH: "64"
-    #   PYTHON_EXE: python
+    - PYTHON: "C:\\Python34-x64"
+      PYTHON_VERSION: "3.4.x" # currently 3.4.3
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     # Also test a Python version not pre-installed
     # See: https://github.com/ogrisel/python-appveyor-demo/issues/10
@@ -93,7 +93,6 @@ install:
       }
       elseif (-not(Test-Path($env:PYTHON))) {
         & appveyor\install.ps1;
-        & "${env:CMD_IN_ENV}" pip install -U psutil;
       }
 
   # Prepend newly installed Python to the PATH of this build (this cannot be
@@ -110,7 +109,8 @@ install:
   # compiled extensions and are not provided as pre-built wheel packages,
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
-  - "%CMD_IN_ENV% pip install -U cython greenlet"
+  # NOTE: psutil won't install under PyPy.
+  - "%CMD_IN_ENV% pip install -U cython greenlet psutil"
 
   - ps: "if(Test-Path(\"${env:PYTHON}\\bin\")) {ls ${env:PYTHON}\\bin;}"
   - ps: "if(Test-Path(\"${env:PYTHON}\\Scripts\")) {ls ${env:PYTHON}\\Scripts;}"

gevent/_corecffi_build.py

@@ -216,7 +216,6 @@ typedef int... vfd_socket_t;
 int vfd_open(vfd_socket_t);
 vfd_socket_t vfd_get(int);
 void vfd_free(int);
-#endif
 """
 
 # Note that we do not cdef the vfd_* family of functions,
@@ -319,4 +318,9 @@ ffi.cdef(_cdef)
 ffi.set_source('gevent._corecffi', _source, include_dirs=include_dirs)
 
 if __name__ == '__main__':
+    # XXX: Note, on Windows, we would need to specify the external libraries
+    # that should be linked in, such as ws2_32 and (because libev_vfd.h makes
+    # Python.h calls) the proper Python library---at least for PyPy. I never got
+    # that to work though, and calling python functions is strongly discouraged
+    # from CFFI code.
     ffi.compile()

scripts/install.sh

+#!/usr/bin/env bash
+# GEVENT: Taken from https://raw.githubusercontent.com/DRMacIver/hypothesis/master/scripts/install.sh
+
+# Special license: Take literally anything you want out of this file. I don't
+# care. Consider it WTFPL licensed if you like.
+# Basically there's a lot of suffering encoded here that I don't want you to
+# have to go through and you should feel free to use this to avoid some of
+# that suffering in advance.
+
+set -e
+set -x
+
+# This is to guard against multiple builds in parallel. The various installers will tend
+# to stomp all over eachother if you do this and they haven't previously successfully
+# succeeded. We use a lock file to block progress so only one install runs at a time.
+# This script should be pretty fast once files are cached, so the lost of concurrency
+# is not a major problem.
+# This should be using the lockfile command, but that's not available on the
+# containerized travis and we can't install it without sudo.
+# Is is unclear if this is actually useful. I was seeing behaviour that suggested
+# concurrent runs of the installer, but I can't seem to find any evidence of this lock
+# ever not being acquired.
+
+BASE=${BUILD_RUNTIMES-$PWD/.runtimes}
+echo $BASE
+mkdir -p $BASE
+
+LOCKFILE="$BASE/.install-lockfile"
+while true; do
+  if mkdir $LOCKFILE 2>/dev/null; then
+    echo "Successfully acquired installer."
+    break
+  else
+    echo "Failed to acquire lock. Is another installer running? Waiting a bit."
+  fi
+
+  sleep $[ ( $RANDOM % 10)  + 1 ].$[ ( $RANDOM % 100) ]s
+
+  if (( $(date '+%s') > 300 + $(stat --format=%X $LOCKFILE) )); then
+    echo "We've waited long enough"
+    rm -rf $LOCKFILE
+  fi
+done
+trap "rm -rf $LOCKFILE" EXIT
+
+
+PYENV=$BASE/pyenv
+
+
+if [ ! -d "$PYENV/.git" ]; then
+  rm -rf $PYENV
+  git clone https://github.com/yyuu/pyenv.git $BASE/pyenv
+else
+  back=$PWD
+  cd $PYENV
+  git fetch || echo "Update failed to complete. Ignoring"
+  git reset --hard origin/master
+  cd $back
+fi
+
+SNAKEPIT=$BASE/snakepit
+
+install () {
+
+  VERSION="$1"
+  ALIAS="$2"
+  mkdir -p $BASE/versions
+  SOURCE=$BASE/versions/$ALIAS
+
+  if [ ! -e "$SOURCE" ]; then
+    mkdir -p $SNAKEPIT
+    mkdir -p $BASE/versions
+    $BASE/pyenv/plugins/python-build/bin/python-build $VERSION $SOURCE
+  fi
+ rm -f $SNAKEPIT/$ALIAS
+ mkdir -p $SNAKEPIT
+ $SOURCE/bin/python -m pip.__main__ install --upgrade pip wheel virtualenv
+ ln -s $SOURCE/bin/python $SNAKEPIT/$ALIAS
+}
+
+
+for var in "$@"; do
+  case "${var}" in
+    2.6)
+      install 2.6.9 python2.6
+      ;;
+    2.7)
+      install 2.7.9 python2.7
+      ;;
+    3.2)
+      install 3.2.6 python3.2
+      ;;
+    3.3)
+      install 3.3.6 python3.3
+      ;;
+    3.4)
+      install 3.4.3 python3.4
+      ;;
+    3.5)
+      install 3.5.0 python3.5
+      ;;
+    pypy)
+      install pypy-4.0.1 pypy
+      ;;
+  esac
+done

setup.py

@@ -14,9 +14,12 @@ PYPY = hasattr(sys, 'pypy_version_info')
 WIN = sys.platform.startswith('win')
 
 if PYPY and WIN and not os.environ.get("PYPY_WIN_BUILD_ANYWAY"):
-    # The MS VC Compiler for Python (VC9.0) fails to properly
-    # link CFFI extensions (many unresolved symbols). Fail early
-    # so the user doesn't have to sit through all the cython compiles
+    # We can't properly handle (hah!) file-descriptors and
+    # handle mapping on Windows/CFFI, because the file needed,
+    # libev_vfd.h, can't be included, linked, and used: it uses
+    # Python API functions, and you're not supposed to do that from
+    # CFFI code. Plus I could never get the libraries= line to ffi.compile()
+    # correct to make linking work.
     raise Exception("Unable to install on PyPy/Windows")
 
 if WIN:
@@ -227,7 +230,7 @@ else:
 if CARES_EMBED:
     ARES.sources += expand('c-ares/*.c')
     ARES.configure = configure_ares
-    if sys.platform == 'win32':
+    if WIN:
         ARES.libraries += ['advapi32']
         ARES.define_macros += [('CARES_STATICLIB', '')]
     else: