setup.py 15.9 KB
Newer Older
1
# Wendelin.core | pythonic package setup
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
2
# Copyright (C) 2014-2020  Nexedi SA and Contributors.
3 4 5 6 7 8 9
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
10 11 12 13
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
14 15 16 17 18
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
19
# See https://www.nexedi.com/licensing for rationale and options.
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
20
from golang.pyx.build import setup, DSO as _DSO, Extension as _PyGoExt, build_ext as _build_ext
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
21
from setuptools_dso import Extension
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
22
from setuptools import Command, find_packages
23
from setuptools.command.build_py import build_py as _build_py
24
from pkg_resources import working_set, EntryPoint
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
25
from distutils.file_util import copy_file
26
from distutils.errors import DistutilsExecError
27
from subprocess import Popen, PIPE
28 29

import os
30
import sys
31

32 33 34 35 36 37 38 39 40 41 42 43 44

# tell cython to resolve `cimport wendelin.*` modules hierarcy starting at top-level.
# see wendelin.py for details.
from Cython.Compiler.Main import Context as CyContext
cy_search_inc_dirs = CyContext.search_include_directories
def wendelin_cy_searh_in_dirs(self, qualified_name, suffix, pos, include=False, sys_path=False):
    _ = qualified_name.split('.')
    if len(_) >= 1 and _[0] == "wendelin":
        qualified_name = '.'.join(_[1:])
    return cy_search_inc_dirs(self, qualified_name, suffix, pos, include, sys_path)
CyContext.search_include_directories = wendelin_cy_searh_in_dirs


Kirill Smelkov's avatar
.  
Kirill Smelkov committed
45 46 47
# _with_defaults calls what(*argv, **kw) with kw amended with default build flags.
# e.g. _with_defaults(_DSO, *argv, **kw)
def _with_defaults(what, *argv, **kw):
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
48 49 50 51 52 53 54
    kw = kw.copy()
    kw['include_dirs'] = [
            '.',
            './include',
            './3rdparty/ccan',
            './3rdparty/include',
        ]
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98

    ccdefault = []
    if kw.get('language') == 'c':
        ccdefault += [
            '-std=gnu99',           # declarations inside for-loop
            '-fplan9-extensions',   # anonymous-structs + simple inheritance

            # in C99 declaration after statement is ok, and we explicitly compile with -std=gnu99.
            # Python >= 3.4 however adds -Werror=declaration-after-statement even for extension
            # modules irregardless of their compilation flags:
            #
            #   https://bugs.python.org/issue21121
            #
            # ensure there is no warnings / errors for decl-after-statements.
            '-Wno-declaration-after-statement',
            '-Wno-error=declaration-after-statement',
        ]
    else:
        ccdefault.append('-std=gnu++11')    # not c++11 since we use typeof

    # DSOs are not yet annoated for visibility
    # XXX pyext besides _bigfile.so also cannot do this because PyMODINIT_FUNC
    # does not include export in it. TODO reenable for _bigfile.so
    """
    if what != _DSO:
        ccdefault.append('-fvisibility=hidden')  # by default symbols not visible outside DSO
    """

    _ = kw.get('extra_compile_args', [])[:]
    _[0:0] = ccdefault
    kw['extra_compile_args'] = _

    lddefault = []
    # python extensions cannot be built with -Wl,--no-undefined: at runtime
    # they links with either python (without libpython) or libpython. linking
    # with both libpython and python would be wrong
    if what == _DSO:
        lddefault.append('-Wl,--no-undefined')  # check DSO for undefined symbols at link time

    _ = kw.get('extra_link_args', [])[:]
    _[0:0] = lddefault
    kw['extra_link_args'] = _


Kirill Smelkov's avatar
.  
Kirill Smelkov committed
99 100
    return what(*argv, **kw)

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
101 102
def PyGoExt(*argv, **kw):   return _with_defaults(_PyGoExt, *argv, **kw)
def DSO(*argv, **kw):       return _with_defaults(_DSO, *argv, **kw)
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
103

104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122


# build_py that
# - prevents in-tree wendelin.py & setup.py to be installed
# - synthesizes wendelin/__init__.py on install
class build_py(_build_py):

    def find_package_modules(self, package, package_dir):
        modules = _build_py.find_package_modules(self, package, package_dir)
        try:
            modules.remove(('wendelin', 'wendelin', 'wendelin.py'))
            modules.remove(('wendelin', 'setup',    'setup.py'))
        except ValueError:
            pass    # was not there

        return modules

    def build_packages(self):
        _build_py.build_packages(self)
Kirill Smelkov's avatar
Kirill Smelkov committed
123
        # emit std namespacing mantra to wendelin/__init__.py
124 125 126 127 128 129 130 131 132
        self.initfile = self.get_module_outfile(self.build_lib, ('wendelin',), '__init__')
        with open(self.initfile, 'w') as f:
            f.write("# this is a namespace package (autogenerated)\n")
            f.write("__import__('pkg_resources').declare_namespace(__name__)\n")


    def get_outputs(self, include_bytecode=1):
        outputs = _build_py.get_outputs(self, include_bytecode)

Kirill Smelkov's avatar
Kirill Smelkov committed
133
        # add synthesized __init__.py to outputs, so that `pip uninstall`
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
        # works without leaving it
        outputs.append(self.initfile)
        if include_bytecode:
            if self.compile:
                outputs.append(self.initfile + 'c')
            if self.optimize:
                outputs.append(self.initfile + 'o')

        return outputs






# run `make <target>`
def runmake(target):
151 152 153 154 155 156 157 158 159
    # NOTE we care to propagate setuptools path to subpython because it could
    # be "inserted" to us by buildout. Propagating whole sys.path is more
    # risky, as e.g. it can break gdb which is using python3, while
    # building/testing under python2.
    pypathv = [working_set.by_key['setuptools'].location]
    pypath  = os.environ.get('PYTHONPATH')
    if pypath is not None:
        pypathv.append(pypath)

160
    err = os.system('make %s PYTHON="%s" PYTHONPATH="%s"' % \
161
            (target, sys.executable, ':'.join(pypathv)))
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
    if err:
        raise DistutilsExecError('Failed to execute `make %s`' % target)


# create distutils command to "run `make <target>`"
def viamake(target, help_text):
    class run_viamake(Command):
        user_options = []
        def initialize_options(self):   pass
        def finalize_options(self):     pass

        description  = help_text
        def run(self):
            runmake(target)

    return run_viamake


# build_ext that
# - builds via Makefile  (and thus pre-builds ccan)  XXX hacky
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
182
# - integrates built wcfs/wcfs.go exe into wcfs/ python package
183 184 185 186
class build_ext(_build_ext):

    def run(self):
        runmake('all')
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
187 188 189
        _build_ext.run(self)

        # copy wcfs/wcfs built from wcfs.go by make into build/lib.linux-x86_64-2.7/...
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
190
        # so that py packaging tools see built wcfs.go as part of wendelin/wcfs package.
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
191 192 193
        copy_file('wcfs/wcfs', os.path.join(self.build_lib, 'wendelin/wcfs/wcfs'),
            verbose=self.verbose, dry_run=self.dry_run)

194 195 196 197 198 199



# register `func` as entry point `entryname` in `groupname` in distribution `distname` on the fly
def register_as_entrypoint(func, entryname, groupname, distname):
    dist = working_set.by_key[distname]
200 201 202 203 204 205 206

    # register group if it is not yet registered
    # else dist.get_entry_map(groupname) returns just {} not connected to entry map
    entry_map = dist.get_entry_map()
    if groupname not in entry_map:
        entry_map[groupname] = {}

207 208 209 210 211 212
    entrypoint = EntryPoint(entryname, func.__module__, attrs=(func.__name__,),
                                    extras=(), dist=dist)
    group = dist.get_entry_map(groupname)
    assert entryname not in group
    group[entryname] = entrypoint

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
213 214 215 216 217 218 219 220 221
    # XXX hack to workaround ImportError in PEP517 mode: pip -> pep517 -> _in_process
    # sources, not imports, setup.py and so there, even though git_lsfiles.__module__=='__main__',
    # the module that is actually __main__ is pip. This leads to ImportError
    # when trying to resolve the entrypoint.
    mod = sys.modules[func.__module__]
    _ = getattr(mod, func.__name__, func)
    assert _ is func
    setattr(mod, func.__name__, func)

222 223 224 225 226 227 228

# like subprocess.check_output(), but properly report errors, if e.g. commands is not found
# check_output(['missing-command']) -> error: [Errno 2] No such file or directory
# runcmd      (['missing-command']) -> error: ['missing-command']: [Errno 2] No such file or directory
def runcmd(argv):
    try:
        process = Popen(argv, stdout=PIPE)
Kirill Smelkov's avatar
Kirill Smelkov committed
229
    except Exception as e:
230 231 232 233 234 235 236 237 238 239
        raise RuntimeError("%s: %s" % (argv, e))

    output, _err = process.communicate()
    retcode = process.poll()
    if retcode:
        raise RuntimeError("%s -> failed (status %s)" % (argv, retcode))

    return output


240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
def git_lsfiles(dirname):
    # FIXME dirname is currently ignored
    # XXX non-ascii names, etc
    for _ in runcmd(['git', 'ls-files']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded
    # XXX recursive submodules
    for _ in runcmd(['git', 'submodule', 'foreach', '--quiet',  \
                r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded

# if we are in git checkout - inject git_lsfiles to setuptools.file_finders
# entry-point on the fly.
#
# otherwise, for released sdist tarball we do not - that tarball already
# contains generated SOURCES.txt and sdist, if run from unpacked tarball, would
# reuse that information.
#
Kirill Smelkov's avatar
Kirill Smelkov committed
257
# (to setuptools - because we have to use some distribution and we already
258 259 260 261 262
#  depend on setuptools)
if os.path.exists('.git'):  # FIXME won't work if we are checked out as e.g. submodule
    register_as_entrypoint(git_lsfiles, 'git', 'setuptools.file_finders', 'setuptools')


263 264 265 266
# read file content
def readfile(path):
    with open(path, 'r') as f:
        return f.read()
267

268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
libvirtmem_h = [
    '3rdparty/include/linux/list.h',
    'include/wendelin/bigfile/file.h',
    'include/wendelin/bigfile/pagemap.h',
    'include/wendelin/bigfile/ram.h',
    'include/wendelin/bigfile/types.h',
    'include/wendelin/bigfile/virtmem.h',
    'include/wendelin/bug.h',
    'include/wendelin/list.h',
    'include/wendelin/utils.h',
]

libwcfs_h = [
    'wcfs/client/wcfs.h',
    'wcfs/client/wcfs_misc.h',
    'wcfs/client/wcfs_watchlink.h',
]

286 287
setup(
    name        = 'wendelin.core',
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
288
    version     = '0.14',   # XXX aka 2.0.0.dev1
289
    description = 'Out-of-core NumPy arrays',
290
    long_description = '%s\n----\n\n%s' % (
291
                            readfile('README.rst'), readfile('CHANGELOG.rst')),
292
    url         = 'https://lab.nexedi.com/nexedi/wendelin.core',
293 294 295 296 297 298
    license     = 'GPLv3+ with wide exception for Open-Source',
    author      = 'Kirill Smelkov',
    author_email= 'kirr@nexedi.com',

    keywords    = 'bigdata out-of-core numpy virtual-memory',

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
299 300 301 302 303 304 305 306 307
    x_dsos      = [DSO('wendelin.bigfile.libvirtmem',
                    ['bigfile/pagefault.c',
                     'bigfile/pagemap.c',
                     'bigfile/ram.c',
                     'bigfile/ram_shmfs.c',
                     'bigfile/ram_hugetlbfs.c',
                     'bigfile/virtmem.c',
                     'lib/bug.c',
                     'lib/utils.c'],
308
                    depends = libvirtmem_h,
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
309 310 311
                    define_macros       = [('_GNU_SOURCE',None), ('BUILDING_LIBVIRTMEM',None)],
                    language = 'c'),

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
312 313 314 315
                   DSO('wendelin.wcfs.client.libwcfs',
                    ['wcfs/client/wcfs.cpp',
                     'wcfs/client/wcfs_watchlink.cpp',
                     'wcfs/client/wcfs_misc.cpp'],
316
                    depends = libvirtmem_h + libwcfs_h,
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
317
                    dsos = ['wendelin.bigfile.libvirtmem'])],
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
318

319
    ext_modules = [
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
320 321
                    PyGoExt('wendelin.bigfile._bigfile',
                        ['bigfile/_bigfile.c'],
322 323 324 325
                        depends = [
                         'bigfile/_bigfile.h',
                         'include/wendelin/compat_py2.h',
                        ] + libvirtmem_h,
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
326 327
                        define_macros   = [('_GNU_SOURCE',None)],
                        language        = 'c',
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
328
                        dsos = ['wendelin.bigfile.libvirtmem']),
329

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
330
                    PyGoExt('wendelin.bigfile._file_zodb',
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
331 332
                        ['bigfile/_file_zodb.pyx',
                         'bigfile/file_zodb.cpp'],
333 334
                        depends = [
                         'wcfs/client/_wcfs.pxd',
335
                         'wcfs/client/_wczsync.pxd',
336 337
                         'bigfile/_bigfile.h',
                        ] + libwcfs_h + libvirtmem_h,
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
338
                        dsos = ['wendelin.wcfs.client.libwcfs']),
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
339

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
340 341
                    PyGoExt('wendelin.wcfs.client._wcfs',
                        ['wcfs/client/_wcfs.pyx'],
342
                        depends = libwcfs_h + libvirtmem_h,
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
343
                        dsos = ['wendelin.wcfs.client.libwcfs']),
344

345 346 347 348 349 350 351
                    PyGoExt('wendelin.wcfs.client._wczsync',
                        ['wcfs/client/_wczsync.pyx'],
                        depends = [
                         'wcfs/client/_wcfs.pxd',
                        ] + libwcfs_h + libvirtmem_h,
                        dsos = ['wendelin.wcfs.client.libwcfs']),

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
352
                    PyGoExt('wendelin.wcfs.internal.wcfs_test',
353 354
                        ['wcfs/internal/wcfs_test.pyx']),

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
355
                    Extension('wendelin.wcfs.internal.io',
356 357
                        ['wcfs/internal/io.pyx']),

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
358
                    Extension('wendelin.wcfs.internal.mm',
359 360
                        ['wcfs/internal/mm.pyx']),
                  ],
361 362 363 364

    package_dir = {'wendelin': ''},
    packages    = ['wendelin'] + ['wendelin.%s' % _ for _ in
                        find_packages(exclude='3rdparty')],
365
    install_requires = [
366
                   'numpy',     # BigArray + its children
367

368
                   # for ZBigFile / ZBigArray
369 370 371
                   # ( NOTE: ZODB3 3.11 just pulls in latest ZODB _4_, so this way
                   #   specifying ZODB _3_ we allow external requirements to
                   #   specify either to use e.g. ZODB3.10 or ZODB4 )
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
372 373 374 375 376 377
                   #
                   # TODO if ZODB5: require ZODB >= 5.5.2 ?
                   #    https://github.com/zopefoundation/ZODB/pull/298
                   #    https://github.com/zopefoundation/ZODB/pull/291
                   #
                   # ----//---- for ZODB4 >= 4.4.6
378
                   'ZODB3 >= 3.10',
379

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
380
                   'pygolang >= 0.0.7', # defer, sync.WaitGroup, pyx/nogil channels ...
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
381

382
                   'six',       # compat py2/py3
383 384

                   'psutil',    # demo_zbigarray
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
385

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
386
                   'zodbtools', # XXX clarify whether we can always require it (check lib/zodb.py)
387 388
                  ],

389
    extras_require = {
390
                   'test': ['pytest', 'scipy'],
391 392
    },

393 394 395
    cmdclass    = {'build_ext':     build_ext,
                   'll_build_ext':  _build_ext, # original build_ext for Makefile
                   'build_py':      build_py,
396
                   'test':          viamake('test',     'run tests'),
397
                   'bench':         viamake('bench',    'run benchmarks'),
398
                  },
399 400

    entry_points= {'console_scripts': [
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
401
                        # start wcfs for zurl
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
402 403
                        'wcfs           = wendelin.wcfs:main',

404 405 406 407
                        # demo to test
                        'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
                      ]
                  },
Kirill Smelkov's avatar
Kirill Smelkov committed
408 409

    classifiers = [_.strip() for _ in """\
410
        Development Status :: 5 - Production/Stable
Kirill Smelkov's avatar
Kirill Smelkov committed
411 412 413 414 415 416
        Intended Audience :: Developers
        Intended Audience :: Science/Research
        Operating System :: POSIX :: Linux
        Programming Language :: Python :: 2
        Programming Language :: Python :: 2.7
        Programming Language :: Python :: 3
Kirill Smelkov's avatar
Kirill Smelkov committed
417
        Programming Language :: Python :: 3.6
418
        Programming Language :: Python :: 3.7
Kirill Smelkov's avatar
Kirill Smelkov committed
419 420 421 422
        Programming Language :: Python :: Implementation :: CPython
        Topic :: Software Development :: Libraries :: Python Modules
        Framework :: ZODB\
    """.splitlines()]
423
)