Use setuptools to build libuv. This should be much simpler, especially on...

Use setuptools to build libuv. This should be much simpler, especially on windows, at the cost of manually keeping up with link arguments and file deps.

CHANGES.rst

@@ -101,8 +101,7 @@
 - Add initial *experimental* support for using libuv as a backend
   instead of libev, controlled by setting the environment variable
   ``GEVENT_CORE_CFFI_ONLY=libuv`` before importing gevent. This only
-  works on POSIX systems and it still suffers a number of limitations
-  compared to libev, notably:
+  suffers a number of limitations compared to libev, notably:
 
   - Timers (such as ``gevent.sleep`` and ``gevent.Timeout``) only
     support a resolution of 1ms. Attempting to use something smaller
@@ -133,8 +132,9 @@
     than either of the libev implementations (cython or CFFI), so
     pressure on the garbage collector will be higher.
 
-  - The build system may not be correctly producing embedded static
-    libraries, at least on OS X. libuv 1.18 or higher is required.
+  - The build system does not support using a system libuv; the
+    embedded copy must be used. Using setuptools to compile libuv was
+    the most portable method found.
 
   - If anything unexpected happens, libuv likes to ``abort()`` the
     entire process instead of reporting an error. For example, closing
@@ -142,7 +142,7 @@
     process to be exited.
 
   Feedback and pull requests are welcome, especially to address the
-  issues mentioned above and for Windows support.
+  issues mentioned above.
 
   Again, this is extremely experimental and all of it is subject to
   change.

_setuplibuv.py

-# -*- coding: utf-8 -*-
-"""
-libuv build utilities.
-"""
-
-from __future__ import print_function, absolute_import, division
-
-import ast
-import os
-import platform
-import subprocess
-import sys
-
-from _setuputils import WIN
-from _setuputils import DEFINE_MACROS
-from _setuputils import Extension
-from _setuputils import dep_abspath
-from _setuputils import system
-from _setuputils import glob_many
-from _setuputils import should_embed
-from _setuputils import _parse_environ
-
-from distutils import log # pylint:disable=no-name-in-module
-from distutils.errors import DistutilsError # pylint: disable=no-name-in-module,import-error
-
-# Inspired by code from https://github.com/saghul/pyuv
-# Note that they as of June 2017, they now use setuptools to build libuv
-# See https://github.com/saghul/pyuv/commit/2c398a9fe47deaebbf47f7bd37d9bb0c350656d1#diff-b30741570389940d7935f51cd0c084cb
-
-LIBUV_EMBED = should_embed('libuv')
-
-if WIN and not LIBUV_EMBED:
-    raise DistutilsError('using a system provided libuv is unsupported on Windows')
-
-LIBUV_INCLUDE_DIR = dep_abspath('libuv', 'include')
-# Set this to force dynamic linking of libuv, even when building the
-# embedded copy. This is most useful when upgrading/changing libuv
-# in place.
-LIBUV_DYNAMIC_EMBED = _parse_environ("GEVENT_LIBUV_DYNAMIC_EMBED")
-if LIBUV_DYNAMIC_EMBED is None:
-    # Not set in the environment. Are we developing?
-    # This is convenient for my workflow
-    # XXX Is there a better way to get this?
-    # This probably doesn't work for 'pip install -e .'
-    # or dev-requirments.txt
-    if 'develop' in sys.argv:
-        LIBUV_DYNAMIC_EMBED = LIBUV_EMBED
-
-LIBUV_LIBRARIES = []
-if sys.platform.startswith('linux'):
-    LIBUV_LIBRARIES.append('rt')
-elif WIN:
-    LIBUV_LIBRARIES.append('advapi32')
-    LIBUV_LIBRARIES.append('iphlpapi')
-    LIBUV_LIBRARIES.append('psapi')
-    LIBUV_LIBRARIES.append('shell32')
-    LIBUV_LIBRARIES.append('userenv')
-    LIBUV_LIBRARIES.append('ws2_32')
-elif sys.platform.startswith('freebsd'):
-    LIBUV_LIBRARIES.append('kvm')
-
-if not LIBUV_EMBED or LIBUV_DYNAMIC_EMBED:
-    LIBUV_LIBRARIES.append('uv')
-
-def prepare_windows_env(env):
-    env.pop('VS140COMNTOOLS', None)
-    env.pop('VS120COMNTOOLS', None)
-    env.pop('VS110COMNTOOLS', None)
-    if sys.version_info < (3, 3):
-        env.pop('VS100COMNTOOLS', None)
-        env['GYP_MSVS_VERSION'] = '2008'
-    else:
-        env['GYP_MSVS_VERSION'] = '2010'
-
-    if not env.get('PYTHON', '').endswith('.exe'):
-        env.pop('PYTHON', None)
-
-    if env.get('PYTHON'):
-        log.info("Using python from env %s", env['PYTHON'])
-        return  # Already manually set by user.
-
-    if sys.version_info[:2] == (2, 7):
-        env['PYTHON'] = sys.executable
-        return  # The current executable is fine.
-
-    # Try if `python` on PATH is the right one. If we would execute
-    # `python` directly the current executable might be used so we
-    # delegate this to cmd.
-    cmd = ['cmd.exe', '/C', 'python', '-c', 'import sys; '
-           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
-           'sys.stdout.flush()']
-    try:
-        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-        stdout, _ = sub.communicate()
-        version = ast.literal_eval(stdout.decode(sys.stdout.encoding).strip()) # pylint:disable=no-member
-        if version == (2, 7):
-            return  # Python on PATH is fine
-    except OSError:
-        pass
-
-    # Check default install locations
-    path = os.path.join('%SYSTEMDRIVE%', 'Python27', 'python.exe')
-    path = os.path.expandvars(path)
-    if os.path.isfile(path):
-        log.info('Using "%s" to build libuv...' % path)
-        env['PYTHON'] = path
-        # Things needed for run_with_env.cmd
-        # What we're building for, not what we're building with
-        # (because it's just the C library)
-        env['PYTHON_VERSION'] = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) + ".x"
-        # XXX: Just guessing here. Is PYTHON_ARCH correct?
-        if 'PYTHON_ARCH' not in env:
-            env['PYTHON_ARCH'] = '64' if platform.architecture()[0] == '64bit' else '32'
-        from distutils.msvc9compiler import query_vcvarsall # pylint:disable=import-error,no-name-in-module
-        if sys.version_info[:2] >= (3, 5):
-            version = 14
-        else:
-            version = 9 # for 2.7, but probably not right for 3.4?
-
-        env.update(query_vcvarsall(version))
-    else:
-        raise DistutilsError('No appropriate Python version found. An '
-                             'installation of 2.7 is required to '
-                             'build libuv. You can set the environment '
-                             'variable "PYTHON" to point to a custom '
-                             'installation location.')
-
-# This is a dummy extension that serves to let us hook into
-# when we need to compile libuv
-LIBUV = Extension(name='gevent.libuv._libuv',
-                  sources=['src/gevent/libuv/_libuv.c'],
-                  include_dirs=[LIBUV_INCLUDE_DIR],
-                  libraries=LIBUV_LIBRARIES,
-                  define_macros=list(DEFINE_MACROS),
-                  depends=glob_many('deps/libuv/src/*.[ch]'))
-
-if LIBUV_EMBED:
-    libuv_dir = dep_abspath('libuv')
-    if WIN:
-        libuv_library_dir = os.path.join(libuv_dir, 'Release', 'lib')
-        libuv_lib = os.path.join(libuv_library_dir, 'libuv.lib')
-        LIBUV.extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
-        LIBUV.extra_objects.append(libuv_lib)
-    else:
-        libuv_library_dir = os.path.join(libuv_dir, '.libs')
-        libuv_lib = os.path.join(libuv_library_dir, 'libuv.a')
-        if not LIBUV_DYNAMIC_EMBED:
-            LIBUV.extra_objects.append(libuv_lib)
-        else:
-            LIBUV.library_dirs.append(libuv_library_dir)
-            LIBUV.extra_link_args.extend(["-Wl,-rpath", libuv_library_dir])
-
-def configure_libuv(_bext, _ext):
-    def build_libuv():
-        cflags = '-fPIC'
-        env = os.environ.copy()
-        env['CFLAGS'] = ' '.join(x for x in (cflags,
-                                             env.get('CFLAGS', None),
-                                             env.get('ARCHFLAGS', None))
-                                 if x)
-        # Since we're building a static library, if link-time-optimization is requested, it
-        # results in failure to properly create the library archive. This goes unnoticed on
-        # OS X until import time because of '-undefined dynamic_lookup'. On the raspberry
-        # pi, it causes the linker to crash
-        if '-flto' in env['CFLAGS']:
-            log.info("Removing LTO")
-            env['CFLAGS'] = env['CFLAGS'].replace('-flto', '')
-        log.info('Building libuv with cflags %s', env['CFLAGS'])
-        if WIN:
-            prepare_windows_env(env)
-            libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
-            system(["cmd", "/E:ON", "/V:ON", "/C",
-                    #"..\\..\\appveyor\\run_with_env.cmd",
-                    'vcbuild.bat', libuv_arch, 'release'],
-                   cwd=libuv_dir,
-                   env=env,
-                   shell=False)
-        else:
-            # autogen: requires automake and libtool installed
-            system(['./autogen.sh'],
-                   cwd=libuv_dir,
-                   env=env)
-            # On OS X, the linker will link to the full path
-            # of the library libuv *as encoded in the dylib it finds*.
-            # So the libdir is important and must match the actual location
-            # of the dynamic library if we want to dynamically link to it.
-            # Otherwise, we wind up linking to /usr/local/lib/libuv.dylib by
-            # default, which can't be found. `otool -D libuv.dylib` will show
-            # this name, and `otool -L src/gevent/libuv/_corecffi.so` will show
-            # what got linked. Note that this approach results in libuv.dylib
-            # apparently linking to *itself*, which is weird, but not harmful
-            system(['./configure', '--libdir=' + libuv_library_dir],
-                   cwd=libuv_dir,
-                   env=env)
-            system(['make'],
-                   cwd=libuv_dir,
-                   env=env)
-
-    if not os.path.exists(libuv_lib):
-        log.info('libuv needs to be compiled.')
-        build_libuv()
-    else:
-        log.info('No need to build libuv.')
-
-if LIBUV_EMBED:
-    LIBUV.configure = configure_libuv

appveyor.yml

@@ -126,7 +126,6 @@ install:
       if ("${env:PYTHON_ID}" -ne "pypy") {
          pip install psutil | Out-Null;
       }
-  - ps: "ls \"C:/Program Files (x86)/MSBuild/\""
   - ps: "if(Test-Path(\"${env:PYTHON}\\bin\")) {ls ${env:PYTHON}\\bin;}"
   - ps: "if(Test-Path(\"${env:PYTHON}\\Scripts\")) {ls ${env:PYTHON}\\Scripts;}"
 

setup.py

@@ -43,8 +43,6 @@ from _setuplibev import CORE
 
 from _setupares import ARES
 
-from _setuplibuv import LIBUV
-from _setuplibuv import configure_libuv
 
 SEMAPHORE = Extension(name="gevent._semaphore",
                       sources=["src/gevent/gevent._semaphore.c"])
@@ -75,11 +73,8 @@ if not WIN:
         LIBEV_CFFI_MODULE
     )
 
-if not WIN:
-    EXT_MODULES.append(LIBUV)
 
-if not WIN or PYPY:
-    cffi_modules.append(LIBUV_CFFI_MODULE)
+cffi_modules.append(LIBUV_CFFI_MODULE)
 
 if PYPY:
     install_requires = []
@@ -142,8 +137,6 @@ def run_setup(ext_modules, run_make):
             # TODO: Generalize this.
             if LIBEV_CFFI_MODULE in cffi_modules and not WIN:
                 system(libev_configure_command)
-            if LIBUV_CFFI_MODULE in cffi_modules and PYPY and WIN:
-                configure_libuv(None, None) # This actually also builds it too
 
         MakeSdist.make()
 

src/gevent/libuv/_corecffi_build.py

@@ -47,39 +47,182 @@ _cdef = _cdef.replace('GEVENT_ST_NLINK_T', st_nlink_type())
 _cdef = _cdef.replace("GEVENT_STRUCT_DONE _;", '...;')
 _cdef = _cdef.replace("GEVENT_UV_OS_SOCK_T", 'int' if not WIN else 'SOCKET')
 
-# if sys.platform.startswith('win'):
-#     # We must have the vfd_open, etc, functions on
-#     # Windows. But on other platforms, going through
-#     # CFFI to just return the file-descriptor is slower
-#     # than just doing it in Python, so we check for and
-#     # workaround their absence in corecffi.py
-#     _cdef += """
-# typedef int... vfd_socket_t;
-# int vfd_open(vfd_socket_t);
-# vfd_socket_t vfd_get(int);
-# void vfd_free(int);
-# """
 
 setup_py_dir = os.path.abspath(os.path.join(thisdir, '..', '..', '..'))
 libuv_dir = os.path.abspath(os.path.join(setup_py_dir, 'deps', 'libuv'))
-sys.path.append(setup_py_dir)
 
-include_dirs = [
+
+LIBUV_INCLUDE_DIRS = [
     thisdir, # libev_vfd.h
     os.path.join(libuv_dir, 'include'),
+    os.path.join(libuv_dir, 'src'),
 ]
 
+# Initially based on https://github.com/saghul/pyuv/blob/v1.x/setup_libuv.py
+
+def _libuv_source(rel_path):
+    path = os.path.join(libuv_dir, 'src', rel_path)
+    assert os.path.isfile(path), path
+    return path
 
+LIBUV_SOURCES = [
+    _libuv_source('fs-poll.c'),
+    _libuv_source('inet.c'),
+    _libuv_source('threadpool.c'),
+    _libuv_source('uv-common.c'),
+    _libuv_source('version.c'),
+]
 
-from _setuplibuv import LIBUV_LIBRARIES # pylint:disable=import-error
-from _setuplibuv import LIBUV # pylint:disable=import-error
+if WIN:
+    LIBUV_SOURCES += [
+        _libuv_source('win/async.c'),
+        _libuv_source('win/core.c'),
+        _libuv_source('win/detect-wakeup.c'),
+        _libuv_source('win/dl.c'),
+        _libuv_source('win/error.c'),
+        _libuv_source('win/fs-event.c'),
+        _libuv_source('win/fs.c'),
+        _libuv_source('win/getaddrinfo.c'),
+        _libuv_source('win/getnameinfo.c'),
+        _libuv_source('win/handle.c'),
+        _libuv_source('win/loop-watcher.c'),
+        _libuv_source('win/pipe.c'),
+        _libuv_source('win/poll.c'),
+        _libuv_source('win/process-stdio.c'),
+        _libuv_source('win/process.c'),
+        _libuv_source('win/req.c'),
+        _libuv_source('win/signal.c'),
+        _libuv_source('win/snprintf.c'),
+        _libuv_source('win/stream.c'),
+        _libuv_source('win/tcp.c'),
+        _libuv_source('win/thread.c'),
+        _libuv_source('win/timer.c'),
+        _libuv_source('win/tty.c'),
+        _libuv_source('win/udp.c'),
+        _libuv_source('win/util.c'),
+        _libuv_source('win/winapi.c'),
+        _libuv_source('win/winsock.c'),
+    ]
+else:
+    LIBUV_SOURCES += [
+        _libuv_source('unix/async.c'),
+        _libuv_source('unix/core.c'),
+        _libuv_source('unix/dl.c'),
+        _libuv_source('unix/fs.c'),
+        _libuv_source('unix/getaddrinfo.c'),
+        _libuv_source('unix/getnameinfo.c'),
+        _libuv_source('unix/loop-watcher.c'),
+        _libuv_source('unix/loop.c'),
+        _libuv_source('unix/pipe.c'),
+        _libuv_source('unix/poll.c'),
+        _libuv_source('unix/process.c'),
+        _libuv_source('unix/signal.c'),
+        _libuv_source('unix/stream.c'),
+        _libuv_source('unix/tcp.c'),
+        _libuv_source('unix/thread.c'),
+        _libuv_source('unix/timer.c'),
+        _libuv_source('unix/tty.c'),
+        _libuv_source('unix/udp.c'),
+    ]
+
+
+if sys.platform.startswith('linux'):
+    LIBUV_SOURCES += [
+        _libuv_source('unix/linux-core.c'),
+        _libuv_source('unix/linux-inotify.c'),
+        _libuv_source('unix/linux-syscalls.c'),
+        _libuv_source('unix/procfs-exepath.c'),
+        _libuv_source('unix/proctitle.c'),
+        _libuv_source('unix/sysinfo-loadavg.c'),
+        _libuv_source('unix/sysinfo-memory.c'),
+    ]
+elif sys.platform == 'darwin':
+    LIBUV_SOURCES += [
+        _libuv_source('unix/bsd-ifaddrs.c'),
+        _libuv_source('unix/darwin.c'),
+        _libuv_source('unix/darwin-proctitle.c'),
+        _libuv_source('unix/fsevents.c'),
+        _libuv_source('unix/kqueue.c'),
+        _libuv_source('unix/proctitle.c'),
+    ]
+elif sys.platform.startswith(('freebsd', 'dragonfly')):
+    LIBUV_SOURCES += [
+        _libuv_source('unix/bsd-ifaddrs.c'),
+        _libuv_source('unix/freebsd.c'),
+        _libuv_source('unix/kqueue.c'),
+        _libuv_source('unix/posix-hrtime.c'),
+    ]
+elif sys.platform.startswith('openbsd'):
+    LIBUV_SOURCES += [
+        _libuv_source('unix/bsd-ifaddrs.c'),
+        _libuv_source('unix/kqueue.c'),
+        _libuv_source('unix/openbsd.c'),
+        _libuv_source('unix/posix-hrtime.c'),
+    ]
+elif sys.platform.startswith('netbsd'):
+    LIBUV_SOURCES += [
+        _libuv_source('unix/bsd-ifaddrs.c'),
+        _libuv_source('unix/kqueue.c'),
+        _libuv_source('unix/netbsd.c'),
+        _libuv_source('unix/posix-hrtime.c'),
+    ]
+
+elif sys.platform.startswith('sunos'):
+    LIBUV_SOURCES += [
+        _libuv_source('unix/no-proctitle.c'),
+        _libuv_source('unix/sunos.c'),
+    ]
+
+
+LIBUV_MACROS = []
+
+def _define_macro(name, value):
+    LIBUV_MACROS.append((name, value))
+
+LIBUV_LIBRARIES = []
+
+def _add_library(name):
+    LIBUV_LIBRARIES.append(name)
+
+if sys.platform != 'win32':
+    _define_macro('_LARGEFILE_SOURCE', 1)
+    _define_macro('_FILE_OFFSET_BITS', 64)
+
+if sys.platform.startswith('linux'):
+    _add_library('dl')
+    _add_library('rt')
+elif sys.platform == 'darwin':
+    _define_macro('_DARWIN_USE_64_BIT_INODE', 1)
+    _define_macro('_DARWIN_UNLIMITED_SELECT', 1)
+elif sys.platform.startswith('netbsd'):
+    _add_library('kvm')
+elif sys.platform.startswith('sunos'):
+    _define_macro('__EXTENSIONS__', 1)
+    _define_macro('_XOPEN_SOURCE', 500)
+    _add_library('kstat')
+    _add_library('nsl')
+    _add_library('sendfile')
+    _add_library('socket')
+elif WIN:
+    _define_macro('_GNU_SOURCE', 1)
+    _define_macro('WIN32', 1)
+    _define_macro('_CRT_SECURE_NO_DEPRECATE', 1)
+    _define_macro('_CRT_NONSTDC_NO_DEPRECATE', 1)
+    _define_macro('_WIN32_WINNT', '0x0600')
+    _add_library('advapi32')
+    _add_library('iphlpapi')
+    _add_library('psapi')
+    _add_library('shell32')
+    _add_library('user32')
+    _add_library('userenv')
+    _add_library('ws2_32')
 
 ffi.cdef(_cdef)
-ffi.set_source('gevent.libuv._corecffi', _source,
-               include_dirs=include_dirs,
-               library_dirs=LIBUV.library_dirs,
-               extra_objects=list(LIBUV.extra_objects),
-               extra_link_args=list(LIBUV.extra_link_args),
+ffi.set_source('gevent.libuv._corecffi',
+               _source,
+               sources=LIBUV_SOURCES,
+               depends=LIBUV_SOURCES,
+               include_dirs=LIBUV_INCLUDE_DIRS,
                libraries=list(LIBUV_LIBRARIES))
 
 if __name__ == '__main__':

src/gevent/libuv/_corecffi_cdef.c

@@ -163,6 +163,11 @@ typedef struct uv_fs_poll_s uv_fs_poll_t;
 
 
 // callbacks with the same signature
+// XXX: Note that these, and all callbacks, are defined to take
+// a void* or handle* instead of the more specific, correct,
+// value. This allows us to use the same gevent_generic_callback
+// without having to do a bunch of casts everywhere. This does produce
+// minor warnings when compiling the CFFI extension, though.
 typedef void (*uv_close_cb)(uv_handle_t *handle);
 typedef void (*uv_idle_cb)(void *handle);
 typedef void (*uv_timer_cb)(void *handle);

src/gevent/libuv/_libuv.py

-# Empty module, exists to build libuv library