Commit cd6c49ad authored by Jason Madden's avatar Jason Madden

Speed up emulated manylinux builds

- don't compile Cython as recommended at https://cython.readthedocs.io/en/latest/src/quickstart/install.html
- don't install test extras we will not need
parent 7253335b
...@@ -45,6 +45,11 @@ if [ -n "$GITHUB_ACTIONS" ]; then ...@@ -45,6 +45,11 @@ if [ -n "$GITHUB_ACTIONS" ]; then
SLOW_BUILD=1 SLOW_BUILD=1
GEVENTTEST_SKIP_ALL=1 GEVENTTEST_SKIP_ALL=1
export GEVENTSETUP_DISABLE_ARES=1 export GEVENTSETUP_DISABLE_ARES=1
# ccache has been seen to have some issues here with too many open files?
unset CC
unset LDSHARED
unset LDCCSHARED
unset LDCXXSHARED
fi fi
else else
OPTIMIZATION_FLAGS="-pipe -O3" OPTIMIZATION_FLAGS="-pipe -O3"
...@@ -63,6 +68,8 @@ export LDFLAGS="$LIBS" ...@@ -63,6 +68,8 @@ export LDFLAGS="$LIBS"
# a fallback loop. # a fallback loop.
export GEVENT_LOOP="libev-cext" export GEVENT_LOOP="libev-cext"
SEP="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
if [ -d /gevent -a -d /opt/python ]; then if [ -d /gevent -a -d /opt/python ]; then
# Running inside docker # Running inside docker
# Set a cache directory for pip. This was # Set a cache directory for pip. This was
...@@ -75,13 +82,16 @@ if [ -d /gevent -a -d /opt/python ]; then ...@@ -75,13 +82,16 @@ if [ -d /gevent -a -d /opt/python ]; then
# pip to disable the cache when it detects that the owner doesn't match. # pip to disable the cache when it detects that the owner doesn't match.
# The below is an attempt to fix that, taken frob bcrypt. It seems to work on # The below is an attempt to fix that, taken frob bcrypt. It seems to work on
# Github Actions. # Github Actions.
echo $SEP
if [ -n "$GITHUB_ACTIONS" ]; then if [ -n "$GITHUB_ACTIONS" ]; then
echo Adjusting pip cache permissions echo Adjusting pip cache permissions: $(whoami)
mkdir -p $XDG_CACHE_HOME/pip mkdir -p $XDG_CACHE_HOME/pip
chown -R $(whoami) $XDG_CACHE_HOME chown -R $(whoami) $XDG_CACHE_HOME
fi fi
ls -ld /cache ls -ld /cache
ls -ld /cache/pip ls -ld /cache/pip
echo $SEP
echo "Installing Build Deps"
if [ -e /usr/bin/yum ]; then if [ -e /usr/bin/yum ]; then
yum -y install libffi-devel yum -y install libffi-devel
# Some images/archs (specificaly 2014_aarch64) don't have ccache; # Some images/archs (specificaly 2014_aarch64) don't have ccache;
...@@ -100,9 +110,12 @@ if [ -d /gevent -a -d /opt/python ]; then ...@@ -100,9 +110,12 @@ if [ -d /gevent -a -d /opt/python ]; then
# the muslinux image # the muslinux image
apk add --no-cache build-base libffi-dev ccache apk add --no-cache build-base libffi-dev ccache
fi fi
echo $SEP
echo Current environment echo Current environment
echo $SEP
env | sort env | sort
echo $SEP
mkdir /tmp/build mkdir /tmp/build
cd /tmp/build cd /tmp/build
...@@ -114,12 +127,35 @@ if [ -d /gevent -a -d /opt/python ]; then ...@@ -114,12 +127,35 @@ if [ -d /gevent -a -d /opt/python ]; then
else else
echo Not configuring c-ares because it is disabled echo Not configuring c-ares because it is disabled
fi fi
echo $SEP
rm -rf /gevent/wheelhouse rm -rf /gevent/wheelhouse
mkdir /gevent/wheelhouse mkdir /gevent/wheelhouse
OPATH="$PATH" OPATH="$PATH"
which auditwheel which auditwheel
for variant in `ls -d /opt/python/cp{310,38,39,311,312}*`; do # June 2023: 3.8, 3.9, and 3.10 are in security-fix only mode, ending support in
# 2024, 2025, and 2026, respectively. Only 3.11+ are in active support mode.
# Building a variant in emulation takes at least 9 minutes on Github Actions,
# plus many minutes to build the deps (Cython) and then many more minutes to build and
# install test dependencies; a complete build for ppc64le with 3.8, 9, 10, 11, and 12
# took 1.5 hours. Multiply that times all the SLOW_BUILD images, and you've got a lot of time.
#
# So, for SLOW_BUILD environments:
# - skip security-fix only versions; users still on those versions are the least likely to
# upgrade dependencies
# - skip running most tests and installing test extras (which we don't need because we only run
# a tiny subset of tests)
echo Possible Builds
ls -l /opt/python/
for variant in /opt/python/cp{38,39,310,311,312}*; do
echo $SEP
export PATH="$variant/bin:$OPATH" export PATH="$variant/bin:$OPATH"
if [ -n "$SLOW_BUILD" ]; then
is_security_fix_only=$(python -c 'import sys; print(sys.version_info[:2] < (3, 10))')
if [ "$is_security_fix_only" == "True" ]; then
echo "Skipping build of $variant"
continue
fi
fi
echo "Building $variant $(python --version)" echo "Building $variant $(python --version)"
python -mpip install -U pip python -mpip install -U pip
...@@ -127,14 +163,19 @@ if [ -d /gevent -a -d /opt/python ]; then ...@@ -127,14 +163,19 @@ if [ -d /gevent -a -d /opt/python ]; then
# The downside is that we must install dependencies manually. # The downside is that we must install dependencies manually.
# NOTE: We can't upgrade ``wheel`` because ``auditwheel`` depends on # NOTE: We can't upgrade ``wheel`` because ``auditwheel`` depends on
# it, and auditwheel is installed in one of these environments. # it, and auditwheel is installed in one of these environments.
python -m pip install -U 'cython>=3.0b3; python_version < "3.12"' "Cython @ https://github.com/cython/cython/archive/37f4dcdc04547875e2836fda076f5707ec50e579.zip; python_version >= '3.12'" # If there is no Cython binary wheel available, don't try to build one; it takes
python -mpip install -U cffi 'greenlet >= 2.0.0; python_version < "3.12"' 'greenlet >= 3.0a1; python_version >= "3.12"' setuptools # forever!
time python -m pip install --config-settings='--install-option="--no-cython-compile"' -U 'cython>=3.0b3; python_version < "3.12"' "Cython @ https://github.com/cython/cython/archive/37f4dcdc04547875e2836fda076f5707ec50e579.zip; python_version >= '3.12'"
time python -mpip install -U cffi 'greenlet >= 2.0.0; python_version < "3.12"' 'greenlet >= 3.0a1; python_version >= "3.12"' setuptools
echo "$variant: Building wheel"
time (python setup.py bdist_wheel) time (python setup.py bdist_wheel)
PATH="$OPATH" auditwheel repair dist/gevent*.whl PATH="$OPATH" auditwheel repair dist/gevent*.whl
cp wheelhouse/gevent*.whl /gevent/wheelhouse cp wheelhouse/gevent*.whl /gevent/wheelhouse
# Install it and its deps to be sure that it can be done; no sense
python -mpip install -U --no-compile `ls dist/gevent*whl`[test] # trying to publish a wheel that can't be installed.
echo 'Installation details' time python -mpip install -U --no-compile $(ls dist/gevent*whl)
# Basic sanity checks
echo "$variant: Installation details"
python -c 'from __future__ import print_function; import gevent; print(gevent, gevent.__version__)' python -c 'from __future__ import print_function; import gevent; print(gevent, gevent.__version__)'
python -c 'from __future__ import print_function; from gevent._compat import get_clock_info; print("clock info", get_clock_info("perf_counter"))' python -c 'from __future__ import print_function; from gevent._compat import get_clock_info; print("clock info", get_clock_info("perf_counter"))'
python -c 'from __future__ import print_function; import greenlet; print(greenlet, greenlet.__version__)' python -c 'from __future__ import print_function; import greenlet; print(greenlet, greenlet.__version__)'
...@@ -147,6 +188,8 @@ if [ -d /gevent -a -d /opt/python ]; then ...@@ -147,6 +188,8 @@ if [ -d /gevent -a -d /opt/python ]; then
fi fi
if [ -z "$GEVENTTEST_SKIP_ALL" ]; then if [ -z "$GEVENTTEST_SKIP_ALL" ]; then
# With the test extra
time python -mpip install -U --no-compile $(ls dist/gevent*whl)[test]
python -mgevent.tests --second-chance python -mgevent.tests --second-chance
else else
# Allow skipping the bulk of the tests. If we're emulating Arm, # Allow skipping the bulk of the tests. If we're emulating Arm,
...@@ -165,15 +208,17 @@ fi ...@@ -165,15 +208,17 @@ fi
# Mount the current directory as /gevent # Mount the current directory as /gevent
# Mount the pip cache directory as /cache # Mount the pip cache directory as /cache
# `pip cache` requires pip 20.1 # `pip cache` requires pip 20.1
echo $SEP
echo Setting up caching echo Setting up caching
python --version python --version
python -mpip --version python -mpip --version
LCACHE="$(dirname `python -mpip cache dir`)" LCACHE="$(dirname `python -mpip cache dir`)"
echo Sharing pip cache at $LCACHE $(ls -ld $LCACHE) echo Sharing pip cache at $LCACHE $(ls -ld $LCACHE)
echo Sharing ccache dir at $HOME/.ccache echo Sharing ccache dir at $HOME/.ccache
if [ ! -d $HOME/.ccache ]; then if [ ! -d "$HOME/.ccache" ]; then
mkdir $HOME/.ccache mkdir "$HOME/.ccache"
fi fi
echo $SEP
# Travis CI and locally we want `-ti`, but github actions doesn't have a TTY, so one # Travis CI and locally we want `-ti`, but github actions doesn't have a TTY, so one
# or the other of the arguments causes this to fail with 'input device is not a TTY' # or the other of the arguments causes this to fail with 'input device is not a TTY'
# Pas through whether we're running on github or not to help with caching. # Pas through whether we're running on github or not to help with caching.
......
from __future__ import absolute_import, print_function, division from __future__ import absolute_import, print_function, division
# Important: This file should have no dependencies that are part of the
# ``test`` extra, because it is sometimes run for quick checks without those
# installed.
import unittest import unittest
import sys import sys
import gevent.testing as greentest import gevent.testing as greentest
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment