Commit cd6c49ad authored by Jason Madden's avatar Jason Madden

Speed up emulated manylinux builds

- don't compile Cython as recommended at https://cython.readthedocs.io/en/latest/src/quickstart/install.html
- don't install test extras we will not need
parent 7253335b
......@@ -45,6 +45,11 @@ if [ -n "$GITHUB_ACTIONS" ]; then
SLOW_BUILD=1
GEVENTTEST_SKIP_ALL=1
export GEVENTSETUP_DISABLE_ARES=1
# ccache has been seen to have some issues here with too many open files?
unset CC
unset LDSHARED
unset LDCCSHARED
unset LDCXXSHARED
fi
else
OPTIMIZATION_FLAGS="-pipe -O3"
......@@ -63,6 +68,8 @@ export LDFLAGS="$LIBS"
# a fallback loop.
export GEVENT_LOOP="libev-cext"
SEP="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
if [ -d /gevent -a -d /opt/python ]; then
# Running inside docker
# Set a cache directory for pip. This was
......@@ -75,13 +82,16 @@ if [ -d /gevent -a -d /opt/python ]; then
# pip to disable the cache when it detects that the owner doesn't match.
# The below is an attempt to fix that, taken frob bcrypt. It seems to work on
# Github Actions.
echo $SEP
if [ -n "$GITHUB_ACTIONS" ]; then
echo Adjusting pip cache permissions
echo Adjusting pip cache permissions: $(whoami)
mkdir -p $XDG_CACHE_HOME/pip
chown -R $(whoami) $XDG_CACHE_HOME
fi
ls -ld /cache
ls -ld /cache/pip
echo $SEP
echo "Installing Build Deps"
if [ -e /usr/bin/yum ]; then
yum -y install libffi-devel
# Some images/archs (specificaly 2014_aarch64) don't have ccache;
......@@ -100,9 +110,12 @@ if [ -d /gevent -a -d /opt/python ]; then
# the muslinux image
apk add --no-cache build-base libffi-dev ccache
fi
echo $SEP
echo Current environment
echo Current environment
echo $SEP
env | sort
echo $SEP
mkdir /tmp/build
cd /tmp/build
......@@ -114,12 +127,35 @@ if [ -d /gevent -a -d /opt/python ]; then
else
echo Not configuring c-ares because it is disabled
fi
echo $SEP
rm -rf /gevent/wheelhouse
mkdir /gevent/wheelhouse
OPATH="$PATH"
which auditwheel
for variant in `ls -d /opt/python/cp{310,38,39,311,312}*`; do
# June 2023: 3.8, 3.9, and 3.10 are in security-fix only mode, ending support in
# 2024, 2025, and 2026, respectively. Only 3.11+ are in active support mode.
# Building a variant in emulation takes at least 9 minutes on Github Actions,
# plus many minutes to build the deps (Cython) and then many more minutes to build and
# install test dependencies; a complete build for ppc64le with 3.8, 9, 10, 11, and 12
# took 1.5 hours. Multiply that times all the SLOW_BUILD images, and you've got a lot of time.
#
# So, for SLOW_BUILD environments:
# - skip security-fix only versions; users still on those versions are the least likely to
# upgrade dependencies
# - skip running most tests and installing test extras (which we don't need because we only run
# a tiny subset of tests)
echo Possible Builds
ls -l /opt/python/
for variant in /opt/python/cp{38,39,310,311,312}*; do
echo $SEP
export PATH="$variant/bin:$OPATH"
if [ -n "$SLOW_BUILD" ]; then
is_security_fix_only=$(python -c 'import sys; print(sys.version_info[:2] < (3, 10))')
if [ "$is_security_fix_only" == "True" ]; then
echo "Skipping build of $variant"
continue
fi
fi
echo "Building $variant $(python --version)"
python -mpip install -U pip
......@@ -127,14 +163,19 @@ if [ -d /gevent -a -d /opt/python ]; then
# The downside is that we must install dependencies manually.
# NOTE: We can't upgrade ``wheel`` because ``auditwheel`` depends on
# it, and auditwheel is installed in one of these environments.
python -m pip install -U 'cython>=3.0b3; python_version < "3.12"' "Cython @ https://github.com/cython/cython/archive/37f4dcdc04547875e2836fda076f5707ec50e579.zip; python_version >= '3.12'"
python -mpip install -U cffi 'greenlet >= 2.0.0; python_version < "3.12"' 'greenlet >= 3.0a1; python_version >= "3.12"' setuptools
# If there is no Cython binary wheel available, don't try to build one; it takes
# forever!
time python -m pip install --config-settings='--install-option="--no-cython-compile"' -U 'cython>=3.0b3; python_version < "3.12"' "Cython @ https://github.com/cython/cython/archive/37f4dcdc04547875e2836fda076f5707ec50e579.zip; python_version >= '3.12'"
time python -mpip install -U cffi 'greenlet >= 2.0.0; python_version < "3.12"' 'greenlet >= 3.0a1; python_version >= "3.12"' setuptools
echo "$variant: Building wheel"
time (python setup.py bdist_wheel)
PATH="$OPATH" auditwheel repair dist/gevent*.whl
cp wheelhouse/gevent*.whl /gevent/wheelhouse
python -mpip install -U --no-compile `ls dist/gevent*whl`[test]
echo 'Installation details'
# Install it and its deps to be sure that it can be done; no sense
# trying to publish a wheel that can't be installed.
time python -mpip install -U --no-compile $(ls dist/gevent*whl)
# Basic sanity checks
echo "$variant: Installation details"
python -c 'from __future__ import print_function; import gevent; print(gevent, gevent.__version__)'
python -c 'from __future__ import print_function; from gevent._compat import get_clock_info; print("clock info", get_clock_info("perf_counter"))'
python -c 'from __future__ import print_function; import greenlet; print(greenlet, greenlet.__version__)'
......@@ -147,6 +188,8 @@ if [ -d /gevent -a -d /opt/python ]; then
fi
if [ -z "$GEVENTTEST_SKIP_ALL" ]; then
# With the test extra
time python -mpip install -U --no-compile $(ls dist/gevent*whl)[test]
python -mgevent.tests --second-chance
else
# Allow skipping the bulk of the tests. If we're emulating Arm,
......@@ -165,15 +208,17 @@ fi
# Mount the current directory as /gevent
# Mount the pip cache directory as /cache
# `pip cache` requires pip 20.1
echo $SEP
echo Setting up caching
python --version
python -mpip --version
LCACHE="$(dirname `python -mpip cache dir`)"
echo Sharing pip cache at $LCACHE $(ls -ld $LCACHE)
echo Sharing ccache dir at $HOME/.ccache
if [ ! -d $HOME/.ccache ]; then
mkdir $HOME/.ccache
if [ ! -d "$HOME/.ccache" ]; then
mkdir "$HOME/.ccache"
fi
echo $SEP
# Travis CI and locally we want `-ti`, but github actions doesn't have a TTY, so one
# or the other of the arguments causes this to fail with 'input device is not a TTY'
# Pas through whether we're running on github or not to help with caching.
......
from __future__ import absolute_import, print_function, division
# Important: This file should have no dependencies that are part of the
# ``test`` extra, because it is sometimes run for quick checks without those
# installed.
import unittest
import sys
import gevent.testing as greentest
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment