setup.py 9.47 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
# Wendelin.core | pythonic package setup
# Copyright (C) 2014-2015  Nexedi SA and Contributors.
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
from setuptools import setup, Extension, Command, find_packages
from setuptools.command.build_py import build_py as _build_py
from setuptools.command.build_ext import build_ext as _build_ext
from distutils.errors import DistutilsExecError
22
from subprocess import Popen, PIPE
23 24

import os
25
import sys
26 27 28 29 30


_bigfile = Extension('wendelin.bigfile._bigfile',
            sources = [
                'bigfile/_bigfile.c',
31
                'bigfile/pagefault.c',
32
                'bigfile/pagemap.c',
Kirill Smelkov's avatar
Kirill Smelkov committed
33 34 35
                'bigfile/ram.c',
                'bigfile/ram_shmfs.c',
                'bigfile/ram_hugetlbfs.c',
36
                'bigfile/virtmem.c',
37
                'lib/bug.c',
38
                'lib/utils.c',
39 40 41 42 43 44 45 46 47 48 49
            ],
            include_dirs = [
                './include',
                './3rdparty/ccan',
                './3rdparty/include'
            ],
            define_macros       = [('_GNU_SOURCE',None)],
            extra_compile_args  = [
                '-std=gnu99',           # declarations inside for-loop
                '-fplan9-extensions',   # anonymous-structs + simple inheritance
                '-fvisibility=hidden',  # by default symbols not visible outside DSO
50 51 52 53 54 55 56 57 58 59

                # in C99 declaration after statement is ok, and we explicitly compile with -std=gnu99.
                # Python >= 3.4 however adds -Werror=declaration-after-statement even for extension
                # modules irregardless of their compilation flags:
                #
                #   https://bugs.python.org/issue21121
                #
                # ensure there is no warnings / errors for decl-after-statements.
                '-Wno-declaration-after-statement',
                '-Wno-error=declaration-after-statement',
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
            ],

            # can't - at runtime links with either python (without libpython) or libpython
            # linking with both libpython and python would be wrong
            #extra_link_args     = [
            #    '-Wl,--no-undefined',   # check DSO for undefined symbols at link time
            #]
            )


# build_py that
# - prevents in-tree wendelin.py & setup.py to be installed
# - synthesizes wendelin/__init__.py on install
class build_py(_build_py):

    def find_package_modules(self, package, package_dir):
        modules = _build_py.find_package_modules(self, package, package_dir)
        try:
            modules.remove(('wendelin', 'wendelin', 'wendelin.py'))
            modules.remove(('wendelin', 'setup',    'setup.py'))
        except ValueError:
            pass    # was not there

        return modules

    def build_packages(self):
        _build_py.build_packages(self)
Kirill Smelkov's avatar
Kirill Smelkov committed
87
        # emit std namespacing mantra to wendelin/__init__.py
88 89 90 91 92 93 94 95 96
        self.initfile = self.get_module_outfile(self.build_lib, ('wendelin',), '__init__')
        with open(self.initfile, 'w') as f:
            f.write("# this is a namespace package (autogenerated)\n")
            f.write("__import__('pkg_resources').declare_namespace(__name__)\n")


    def get_outputs(self, include_bytecode=1):
        outputs = _build_py.get_outputs(self, include_bytecode)

Kirill Smelkov's avatar
Kirill Smelkov committed
97
        # add synthesized __init__.py to outputs, so that `pip uninstall`
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
        # works without leaving it
        outputs.append(self.initfile)
        if include_bytecode:
            if self.compile:
                outputs.append(self.initfile + 'c')
            if self.optimize:
                outputs.append(self.initfile + 'o')

        return outputs






# run `make <target>`
def runmake(target):
115
    err = os.system('make %s PYTHON="%s"' % (target, sys.executable))
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
    if err:
        raise DistutilsExecError('Failed to execute `make %s`' % target)


# create distutils command to "run `make <target>`"
def viamake(target, help_text):
    class run_viamake(Command):
        user_options = []
        def initialize_options(self):   pass
        def finalize_options(self):     pass

        description  = help_text
        def run(self):
            runmake(target)

    return run_viamake


# build_ext that
# - builds via Makefile  (and thus pre-builds ccan)  XXX hacky
class build_ext(_build_ext):

    def run(self):
        runmake('all')
        return _build_ext.run(self)



# register `func` as entry point `entryname` in `groupname` in distribution `distname` on the fly
def register_as_entrypoint(func, entryname, groupname, distname):
    from pkg_resources import working_set, EntryPoint
    dist = working_set.by_key[distname]
148 149 150 151 152 153 154

    # register group if it is not yet registered
    # else dist.get_entry_map(groupname) returns just {} not connected to entry map
    entry_map = dist.get_entry_map()
    if groupname not in entry_map:
        entry_map[groupname] = {}

155 156 157 158 159 160
    entrypoint = EntryPoint(entryname, func.__module__, attrs=(func.__name__,),
                                    extras=(), dist=dist)
    group = dist.get_entry_map(groupname)
    assert entryname not in group
    group[entryname] = entrypoint

161 162 163 164 165 166 167

# like subprocess.check_output(), but properly report errors, if e.g. commands is not found
# check_output(['missing-command']) -> error: [Errno 2] No such file or directory
# runcmd      (['missing-command']) -> error: ['missing-command']: [Errno 2] No such file or directory
def runcmd(argv):
    try:
        process = Popen(argv, stdout=PIPE)
Kirill Smelkov's avatar
Kirill Smelkov committed
168
    except Exception as e:
169 170 171 172 173 174 175 176 177 178
        raise RuntimeError("%s: %s" % (argv, e))

    output, _err = process.communicate()
    retcode = process.poll()
    if retcode:
        raise RuntimeError("%s -> failed (status %s)" % (argv, retcode))

    return output


179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
def git_lsfiles(dirname):
    # FIXME dirname is currently ignored
    # XXX non-ascii names, etc
    for _ in runcmd(['git', 'ls-files']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded
    # XXX recursive submodules
    for _ in runcmd(['git', 'submodule', 'foreach', '--quiet',  \
                r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
        yield _.decode('utf8')  # XXX utf8 hardcoded

# if we are in git checkout - inject git_lsfiles to setuptools.file_finders
# entry-point on the fly.
#
# otherwise, for released sdist tarball we do not - that tarball already
# contains generated SOURCES.txt and sdist, if run from unpacked tarball, would
# reuse that information.
#
Kirill Smelkov's avatar
Kirill Smelkov committed
196
# (to setuptools - because we have to use some distribution and we already
197 198 199 200 201 202 203 204 205 206
#  depend on setuptools)
if os.path.exists('.git'):  # FIXME won't work if we are checked out as e.g. submodule
    register_as_entrypoint(git_lsfiles, 'git', 'setuptools.file_finders', 'setuptools')





setup(
    name        = 'wendelin.core',
Kirill Smelkov's avatar
Kirill Smelkov committed
207
    version     = '0.3',
208 209 210 211 212 213 214 215 216 217 218 219 220
    description = 'Out-of-core NumPy arrays',
    url         = 'http://www.wendelin.io/',
    license     = 'GPLv3+ with wide exception for Open-Source',
    author      = 'Kirill Smelkov',
    author_email= 'kirr@nexedi.com',

    keywords    = 'bigdata out-of-core numpy virtual-memory',

    ext_modules = [_bigfile],

    package_dir = {'wendelin': ''},
    packages    = ['wendelin'] + ['wendelin.%s' % _ for _ in
                        find_packages(exclude='3rdparty')],
221
    install_requires = [
222
                   'numpy',     # BigArray + its children
223

224
                   # for ZBigFile / ZBigArray
225 226 227 228 229
                   # ( NOTE: ZODB3 3.11 just pulls in latest ZODB _4_, so this way
                   #   specifying ZODB _3_ we allow external requirements to
                   #   specify either to use e.g. ZODB3.10 or ZODB4 )
                   'ZODB3',

230
                   'six',       # compat py2/py3
231 232

                   'psutil',    # demo_zbigarray
233 234
                  ],

235 236 237 238
    extras_require = {
                   'test': ['pytest'],
    },

239 240 241
    cmdclass    = {'build_ext':     build_ext,
                   'll_build_ext':  _build_ext, # original build_ext for Makefile
                   'build_py':      build_py,
242
                   'test':          viamake('test',     'run tests'),
243
                   'bench':         viamake('bench',    'run benchmarks'),
244
                  },
245 246 247 248 249 250

    entry_points= {'console_scripts': [
                        # demo to test
                        'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
                      ]
                  },
Kirill Smelkov's avatar
Kirill Smelkov committed
251 252 253 254 255 256 257 258 259 260 261 262 263 264

    classifiers = [_.strip() for _ in """\
        Development Status :: 3 - Alpha
        Intended Audience :: Developers
        Intended Audience :: Science/Research
        Operating System :: POSIX :: Linux
        Programming Language :: Python :: 2
        Programming Language :: Python :: 2.7
        Programming Language :: Python :: 3
        Programming Language :: Python :: 3.4
        Programming Language :: Python :: Implementation :: CPython
        Topic :: Software Development :: Libraries :: Python Modules
        Framework :: ZODB\
    """.splitlines()]
265
)