setup.py 10.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# Wendelin.core | pythonic package setup
# Copyright (C) 2014-2015  Nexedi SA and Contributors.
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
from setuptools import setup, Extension, Command, find_packages
from setuptools.command.build_py import build_py as _build_py
from setuptools.command.build_ext import build_ext as _build_ext
21
from pkg_resources import working_set, EntryPoint
22
from distutils.errors import DistutilsExecError
23
from subprocess import Popen, PIPE
24 25

import os
26
import sys
27 28 29 30 31


_bigfile = Extension('wendelin.bigfile._bigfile',
            sources = [
                'bigfile/_bigfile.c',
32
                'bigfile/pagefault.c',
33
                'bigfile/pagemap.c',
Kirill Smelkov's avatar
Kirill Smelkov committed
34 35 36
                'bigfile/ram.c',
                'bigfile/ram_shmfs.c',
                'bigfile/ram_hugetlbfs.c',
37
                'bigfile/virtmem.c',
38
                'lib/bug.c',
39
                'lib/utils.c',
40 41 42 43 44 45 46 47 48 49 50
            ],
            include_dirs = [
                './include',
                './3rdparty/ccan',
                './3rdparty/include'
            ],
            define_macros       = [('_GNU_SOURCE',None)],
            extra_compile_args  = [
                '-std=gnu99',           # declarations inside for-loop
                '-fplan9-extensions',   # anonymous-structs + simple inheritance
                '-fvisibility=hidden',  # by default symbols not visible outside DSO
51 52 53 54 55 56 57 58 59 60

                # in C99 declaration after statement is ok, and we explicitly compile with -std=gnu99.
                # Python >= 3.4 however adds -Werror=declaration-after-statement even for extension
                # modules irregardless of their compilation flags:
                #
                #   https://bugs.python.org/issue21121
                #
                # ensure there is no warnings / errors for decl-after-statements.
                '-Wno-declaration-after-statement',
                '-Wno-error=declaration-after-statement',
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
            ],

            # can't - at runtime links with either python (without libpython) or libpython
            # linking with both libpython and python would be wrong
            #extra_link_args     = [
            #    '-Wl,--no-undefined',   # check DSO for undefined symbols at link time
            #]
            )


# build_py that
# - prevents in-tree wendelin.py & setup.py to be installed
# - synthesizes wendelin/__init__.py on install
class build_py(_build_py):

    def find_package_modules(self, package, package_dir):
        modules = _build_py.find_package_modules(self, package, package_dir)
        try:
            modules.remove(('wendelin', 'wendelin', 'wendelin.py'))
            modules.remove(('wendelin', 'setup',    'setup.py'))
        except ValueError:
            pass    # was not there

        return modules

    def build_packages(self):
        _build_py.build_packages(self)
Kirill Smelkov's avatar
Kirill Smelkov committed
88
        # emit std namespacing mantra to wendelin/__init__.py
89 90 91 92 93 94 95 96 97
        self.initfile = self.get_module_outfile(self.build_lib, ('wendelin',), '__init__')
        with open(self.initfile, 'w') as f:
            f.write("# this is a namespace package (autogenerated)\n")
            f.write("__import__('pkg_resources').declare_namespace(__name__)\n")


    def get_outputs(self, include_bytecode=1):
        outputs = _build_py.get_outputs(self, include_bytecode)

Kirill Smelkov's avatar
Kirill Smelkov committed
98
        # add synthesized __init__.py to outputs, so that `pip uninstall`
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
        # works without leaving it
        outputs.append(self.initfile)
        if include_bytecode:
            if self.compile:
                outputs.append(self.initfile + 'c')
            if self.optimize:
                outputs.append(self.initfile + 'o')

        return outputs






# run `make <target>`
def runmake(target):
116 117 118 119 120 121 122 123 124
    # NOTE we care to propagate setuptools path to subpython because it could
    # be "inserted" to us by buildout. Propagating whole sys.path is more
    # risky, as e.g. it can break gdb which is using python3, while
    # building/testing under python2.
    pypathv = [working_set.by_key['setuptools'].location]
    pypath  = os.environ.get('PYTHONPATH')
    if pypath is not None:
        pypathv.append(pypath)

125
    err = os.system('make %s PYTHON="%s" PYTHONPATH="%s"' % \
126
            (target, sys.executable, ':'.join(pypathv)))
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
    if err:
        raise DistutilsExecError('Failed to execute `make %s`' % target)


# create distutils command to "run `make <target>`"
def viamake(target, help_text):
    class run_viamake(Command):
        user_options = []
        def initialize_options(self):   pass
        def finalize_options(self):     pass

        description  = help_text
        def run(self):
            runmake(target)

    return run_viamake


# build_ext that
# - builds via Makefile  (and thus pre-builds ccan)  XXX hacky
class build_ext(_build_ext):

    def run(self):
        runmake('all')
        return _build_ext.run(self)



# register `func` as entry point `entryname` in `groupname` in distribution `distname` on the fly
def register_as_entrypoint(func, entryname, groupname, distname):
    dist = working_set.by_key[distname]
158 159 160 161 162 163 164

    # register group if it is not yet registered
    # else dist.get_entry_map(groupname) returns just {} not connected to entry map
    entry_map = dist.get_entry_map()
    if groupname not in entry_map:
        entry_map[groupname] = {}

165 166 167 168 169 170
    entrypoint = EntryPoint(entryname, func.__module__, attrs=(func.__name__,),
                                    extras=(), dist=dist)
    group = dist.get_entry_map(groupname)
    assert entryname not in group
    group[entryname] = entrypoint

171 172 173 174 175 176 177

# like subprocess.check_output(), but properly report errors, if e.g. commands is not found
# check_output(['missing-command']) -> error: [Errno 2] No such file or directory
# runcmd      (['missing-command']) -> error: ['missing-command']: [Errno 2] No such file or directory
def runcmd(argv):
    try:
        process = Popen(argv, stdout=PIPE)
Kirill Smelkov's avatar
Kirill Smelkov committed
178
    except Exception as e:
179 180 181 182 183 184 185 186 187 188
        raise RuntimeError("%s: %s" % (argv, e))

    output, _err = process.communicate()
    retcode = process.poll()
    if retcode:
        raise RuntimeError("%s -> failed (status %s)" % (argv, retcode))

    return output


189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
def git_lsfiles(dirname):
    # FIXME dirname is currently ignored
    # XXX non-ascii names, etc
    for _ in runcmd(['git', 'ls-files']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded
    # XXX recursive submodules
    for _ in runcmd(['git', 'submodule', 'foreach', '--quiet',  \
                r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded

# if we are in git checkout - inject git_lsfiles to setuptools.file_finders
# entry-point on the fly.
#
# otherwise, for released sdist tarball we do not - that tarball already
# contains generated SOURCES.txt and sdist, if run from unpacked tarball, would
# reuse that information.
#
Kirill Smelkov's avatar
Kirill Smelkov committed
206
# (to setuptools - because we have to use some distribution and we already
207 208 209 210 211
#  depend on setuptools)
if os.path.exists('.git'):  # FIXME won't work if we are checked out as e.g. submodule
    register_as_entrypoint(git_lsfiles, 'git', 'setuptools.file_finders', 'setuptools')


212 213 214 215
# read file content
def readfile(path):
    with open(path, 'r') as f:
        return f.read()
216 217 218

setup(
    name        = 'wendelin.core',
Kirill Smelkov's avatar
Kirill Smelkov committed
219
    version     = '0.9',
220
    description = 'Out-of-core NumPy arrays',
221
    long_description = '%s\n----\n\n%s' % (
222
                            readfile('README.rst'), readfile('CHANGELOG.rst')),
223
    url         = 'https://lab.nexedi.com/nexedi/wendelin.core',
224 225 226 227 228 229 230 231 232 233 234
    license     = 'GPLv3+ with wide exception for Open-Source',
    author      = 'Kirill Smelkov',
    author_email= 'kirr@nexedi.com',

    keywords    = 'bigdata out-of-core numpy virtual-memory',

    ext_modules = [_bigfile],

    package_dir = {'wendelin': ''},
    packages    = ['wendelin'] + ['wendelin.%s' % _ for _ in
                        find_packages(exclude='3rdparty')],
235
    install_requires = [
236
                   'numpy',     # BigArray + its children
237

238
                   # for ZBigFile / ZBigArray
239 240 241
                   # ( NOTE: ZODB3 3.11 just pulls in latest ZODB _4_, so this way
                   #   specifying ZODB _3_ we allow external requirements to
                   #   specify either to use e.g. ZODB3.10 or ZODB4 )
242
                   'ZODB3 >= 3.10',
243

244
                   'six',       # compat py2/py3
245 246

                   'psutil',    # demo_zbigarray
247 248
                  ],

249 250 251 252
    extras_require = {
                   'test': ['pytest'],
    },

253 254 255
    cmdclass    = {'build_ext':     build_ext,
                   'll_build_ext':  _build_ext, # original build_ext for Makefile
                   'build_py':      build_py,
256
                   'test':          viamake('test',     'run tests'),
257
                   'bench':         viamake('bench',    'run benchmarks'),
258
                  },
259 260 261 262 263 264

    entry_points= {'console_scripts': [
                        # demo to test
                        'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
                      ]
                  },
Kirill Smelkov's avatar
Kirill Smelkov committed
265 266 267 268 269 270 271 272 273 274

    classifiers = [_.strip() for _ in """\
        Development Status :: 3 - Alpha
        Intended Audience :: Developers
        Intended Audience :: Science/Research
        Operating System :: POSIX :: Linux
        Programming Language :: Python :: 2
        Programming Language :: Python :: 2.7
        Programming Language :: Python :: 3
        Programming Language :: Python :: 3.4
275
        Programming Language :: Python :: 3.5
Kirill Smelkov's avatar
Kirill Smelkov committed
276 277 278 279
        Programming Language :: Python :: Implementation :: CPython
        Topic :: Software Development :: Libraries :: Python Modules
        Framework :: ZODB\
    """.splitlines()]
280
)