setup.py 10.8 KB
Newer Older
1
# Wendelin.core | pythonic package setup
2
# Copyright (C) 2014-2020  Nexedi SA and Contributors.
3 4 5 6 7 8 9
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
10 11 12 13
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
14 15 16 17 18
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
19
# See https://www.nexedi.com/licensing for rationale and options.
20 21 22
from setuptools import setup, Extension, Command, find_packages
from setuptools.command.build_py import build_py as _build_py
from setuptools.command.build_ext import build_ext as _build_ext
23
from pkg_resources import working_set, EntryPoint
24
from distutils.errors import DistutilsExecError
25
from subprocess import Popen, PIPE
26 27

import os
28
import sys
29 30 31 32 33


_bigfile = Extension('wendelin.bigfile._bigfile',
            sources = [
                'bigfile/_bigfile.c',
34
                'bigfile/pagefault.c',
35
                'bigfile/pagemap.c',
Kirill Smelkov's avatar
Kirill Smelkov committed
36 37 38
                'bigfile/ram.c',
                'bigfile/ram_shmfs.c',
                'bigfile/ram_hugetlbfs.c',
39
                'bigfile/virtmem.c',
40
                'lib/bug.c',
41
                'lib/utils.c',
42 43
            ],
            include_dirs = [
44
                '.',
45 46 47 48 49 50 51 52 53
                './include',
                './3rdparty/ccan',
                './3rdparty/include'
            ],
            define_macros       = [('_GNU_SOURCE',None)],
            extra_compile_args  = [
                '-std=gnu99',           # declarations inside for-loop
                '-fplan9-extensions',   # anonymous-structs + simple inheritance
                '-fvisibility=hidden',  # by default symbols not visible outside DSO
54 55 56 57 58 59 60 61 62 63

                # in C99 declaration after statement is ok, and we explicitly compile with -std=gnu99.
                # Python >= 3.4 however adds -Werror=declaration-after-statement even for extension
                # modules irregardless of their compilation flags:
                #
                #   https://bugs.python.org/issue21121
                #
                # ensure there is no warnings / errors for decl-after-statements.
                '-Wno-declaration-after-statement',
                '-Wno-error=declaration-after-statement',
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
            ],

            # can't - at runtime links with either python (without libpython) or libpython
            # linking with both libpython and python would be wrong
            #extra_link_args     = [
            #    '-Wl,--no-undefined',   # check DSO for undefined symbols at link time
            #]
            )


# build_py that
# - prevents in-tree wendelin.py & setup.py to be installed
# - synthesizes wendelin/__init__.py on install
class build_py(_build_py):

    def find_package_modules(self, package, package_dir):
        modules = _build_py.find_package_modules(self, package, package_dir)
        try:
            modules.remove(('wendelin', 'wendelin', 'wendelin.py'))
            modules.remove(('wendelin', 'setup',    'setup.py'))
        except ValueError:
            pass    # was not there

        return modules

    def build_packages(self):
        _build_py.build_packages(self)
Kirill Smelkov's avatar
Kirill Smelkov committed
91
        # emit std namespacing mantra to wendelin/__init__.py
92 93 94 95 96 97 98 99 100
        self.initfile = self.get_module_outfile(self.build_lib, ('wendelin',), '__init__')
        with open(self.initfile, 'w') as f:
            f.write("# this is a namespace package (autogenerated)\n")
            f.write("__import__('pkg_resources').declare_namespace(__name__)\n")


    def get_outputs(self, include_bytecode=1):
        outputs = _build_py.get_outputs(self, include_bytecode)

Kirill Smelkov's avatar
Kirill Smelkov committed
101
        # add synthesized __init__.py to outputs, so that `pip uninstall`
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
        # works without leaving it
        outputs.append(self.initfile)
        if include_bytecode:
            if self.compile:
                outputs.append(self.initfile + 'c')
            if self.optimize:
                outputs.append(self.initfile + 'o')

        return outputs






# run `make <target>`
def runmake(target):
119 120 121 122 123 124 125 126 127
    # NOTE we care to propagate setuptools path to subpython because it could
    # be "inserted" to us by buildout. Propagating whole sys.path is more
    # risky, as e.g. it can break gdb which is using python3, while
    # building/testing under python2.
    pypathv = [working_set.by_key['setuptools'].location]
    pypath  = os.environ.get('PYTHONPATH')
    if pypath is not None:
        pypathv.append(pypath)

128
    err = os.system('make %s PYTHON="%s" PYTHONPATH="%s"' % \
129
            (target, sys.executable, ':'.join(pypathv)))
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
    if err:
        raise DistutilsExecError('Failed to execute `make %s`' % target)


# create distutils command to "run `make <target>`"
def viamake(target, help_text):
    class run_viamake(Command):
        user_options = []
        def initialize_options(self):   pass
        def finalize_options(self):     pass

        description  = help_text
        def run(self):
            runmake(target)

    return run_viamake


# build_ext that
# - builds via Makefile  (and thus pre-builds ccan)  XXX hacky
class build_ext(_build_ext):

    def run(self):
        runmake('all')
        return _build_ext.run(self)



# register `func` as entry point `entryname` in `groupname` in distribution `distname` on the fly
def register_as_entrypoint(func, entryname, groupname, distname):
    dist = working_set.by_key[distname]
161 162 163 164 165 166 167

    # register group if it is not yet registered
    # else dist.get_entry_map(groupname) returns just {} not connected to entry map
    entry_map = dist.get_entry_map()
    if groupname not in entry_map:
        entry_map[groupname] = {}

168 169 170 171 172 173
    entrypoint = EntryPoint(entryname, func.__module__, attrs=(func.__name__,),
                                    extras=(), dist=dist)
    group = dist.get_entry_map(groupname)
    assert entryname not in group
    group[entryname] = entrypoint

174 175 176 177 178 179 180 181 182
    # XXX hack to workaround ImportError in PEP517 mode: pip -> pep517 -> _in_process
    # sources, not imports, setup.py and so there, even though git_lsfiles.__module__=='__main__',
    # the module that is actually __main__ is pip. This leads to ImportError
    # when trying to resolve the entrypoint.
    mod = sys.modules[func.__module__]
    _ = getattr(mod, func.__name__, func)
    assert _ is func
    setattr(mod, func.__name__, func)

183 184 185 186 187 188 189

# like subprocess.check_output(), but properly report errors, if e.g. commands is not found
# check_output(['missing-command']) -> error: [Errno 2] No such file or directory
# runcmd      (['missing-command']) -> error: ['missing-command']: [Errno 2] No such file or directory
def runcmd(argv):
    try:
        process = Popen(argv, stdout=PIPE)
Kirill Smelkov's avatar
Kirill Smelkov committed
190
    except Exception as e:
191 192 193 194 195 196 197 198 199 200
        raise RuntimeError("%s: %s" % (argv, e))

    output, _err = process.communicate()
    retcode = process.poll()
    if retcode:
        raise RuntimeError("%s -> failed (status %s)" % (argv, retcode))

    return output


201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
def git_lsfiles(dirname):
    # FIXME dirname is currently ignored
    # XXX non-ascii names, etc
    for _ in runcmd(['git', 'ls-files']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded
    # XXX recursive submodules
    for _ in runcmd(['git', 'submodule', 'foreach', '--quiet',  \
                r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded

# if we are in git checkout - inject git_lsfiles to setuptools.file_finders
# entry-point on the fly.
#
# otherwise, for released sdist tarball we do not - that tarball already
# contains generated SOURCES.txt and sdist, if run from unpacked tarball, would
# reuse that information.
#
Kirill Smelkov's avatar
Kirill Smelkov committed
218
# (to setuptools - because we have to use some distribution and we already
219 220 221 222 223
#  depend on setuptools)
if os.path.exists('.git'):  # FIXME won't work if we are checked out as e.g. submodule
    register_as_entrypoint(git_lsfiles, 'git', 'setuptools.file_finders', 'setuptools')


224 225 226 227
# read file content
def readfile(path):
    with open(path, 'r') as f:
        return f.read()
228 229 230

setup(
    name        = 'wendelin.core',
Kirill Smelkov's avatar
Kirill Smelkov committed
231
    version     = '0.13',
232
    description = 'Out-of-core NumPy arrays',
233
    long_description = '%s\n----\n\n%s' % (
234
                            readfile('README.rst'), readfile('CHANGELOG.rst')),
235
    url         = 'https://lab.nexedi.com/nexedi/wendelin.core',
236 237 238 239 240 241 242 243 244 245 246
    license     = 'GPLv3+ with wide exception for Open-Source',
    author      = 'Kirill Smelkov',
    author_email= 'kirr@nexedi.com',

    keywords    = 'bigdata out-of-core numpy virtual-memory',

    ext_modules = [_bigfile],

    package_dir = {'wendelin': ''},
    packages    = ['wendelin'] + ['wendelin.%s' % _ for _ in
                        find_packages(exclude='3rdparty')],
247
    install_requires = [
248
                   'numpy',     # BigArray + its children
249

250
                   # for ZBigFile / ZBigArray
251 252 253
                   # ( NOTE: ZODB3 3.11 just pulls in latest ZODB _4_, so this way
                   #   specifying ZODB _3_ we allow external requirements to
                   #   specify either to use e.g. ZODB3.10 or ZODB4 )
254
                   'ZODB3 >= 3.10',
255

256 257
                   'pygolang >= 0.0.2', # defer, sync.WaitGroup, ...

258
                   'six',       # compat py2/py3
259 260

                   'psutil',    # demo_zbigarray
261 262
                  ],

263
    extras_require = {
264
                   'test': ['pytest'],
265 266
    },

267 268 269
    cmdclass    = {'build_ext':     build_ext,
                   'll_build_ext':  _build_ext, # original build_ext for Makefile
                   'build_py':      build_py,
270
                   'test':          viamake('test',     'run tests'),
271
                   'bench':         viamake('bench',    'run benchmarks'),
272
                  },
273 274 275 276 277 278

    entry_points= {'console_scripts': [
                        # demo to test
                        'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
                      ]
                  },
Kirill Smelkov's avatar
Kirill Smelkov committed
279 280

    classifiers = [_.strip() for _ in """\
281
        Development Status :: 5 - Production/Stable
Kirill Smelkov's avatar
Kirill Smelkov committed
282 283 284 285 286 287
        Intended Audience :: Developers
        Intended Audience :: Science/Research
        Operating System :: POSIX :: Linux
        Programming Language :: Python :: 2
        Programming Language :: Python :: 2.7
        Programming Language :: Python :: 3
Kirill Smelkov's avatar
Kirill Smelkov committed
288
        Programming Language :: Python :: 3.6
289
        Programming Language :: Python :: 3.7
Kirill Smelkov's avatar
Kirill Smelkov committed
290 291 292 293
        Programming Language :: Python :: Implementation :: CPython
        Topic :: Software Development :: Libraries :: Python Modules
        Framework :: ZODB\
    """.splitlines()]
294
)