cystdlib.py 5.38 KB
Newer Older
1 2 3 4 5 6 7
"""
Highly experimental script that compiles the CPython standard library using Cython.

Execute the script either in the CPython 'Lib' directory or pass the
option '--current-python' to compile the standard library of the running
Python interpreter.

8
Pass '-j N' to get a parallel build with N processes.
9 10 11 12 13 14

Usage example::

    $ python cystdlib.py --current-python build_ext -i
"""

15
import os
16
import sys
17 18 19 20 21 22
from distutils.core import setup
from Cython.Build import cythonize
from Cython.Compiler import Options

# improve Python compatibility by allowing some broken code
Options.error_on_unknown_names = False
23
Options.error_on_uninitialized = False
24

25
exclude_patterns = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
26 27 28 29 30
broken = [
    'idlelib/MultiCall.py',
    'email/utils.py',
    'multiprocessing/reduction.py',
    'multiprocessing/util.py',
31
    'threading.py',      # interrupt handling
32 33
    'lib2to3/fixes/fix_sys_exc.py',
    'traceback.py',
34 35
    'types.py',
    'enum.py',
36 37
    'keyword.py',
    '_collections_abc.py',
38
    'importlib/_bootstrap',
39
]
40

41
default_directives = dict(
42
    auto_cpdef=False,   # enable when it's safe, see long list of failures below
43
    binding=True,
44
    set_initial_path='SOURCEFILE')
45
default_directives['optimize.inline_defnode_calls'] = True
46

47
special_directives = [
48
    (['pkgutil.py',
49
      'decimal.py',
50 51 52 53 54 55
      'datetime.py',
      'optparse.py',
      'sndhdr.py',
      'opcode.py',
      'ntpath.py',
      'urllib/request.py',
56
      'plat-*/TYPES.py',
57
      'plat-*/IN.py',
58
      'tkinter/_fix.py',
59
      'lib2to3/refactor.py',
60
      'webbrowser.py',
61
      'shutil.py',
62
      'multiprocessing/forking.py',
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
      'xml/sax/expatreader.py',
      'xmlrpc/client.py',
      'pydoc.py',
      'xml/etree/ElementTree.py',
      'posixpath.py',
      'inspect.py',
      'ctypes/util.py',
      'urllib/parse.py',
      'warnings.py',
      'tempfile.py',
      'trace.py',
      'heapq.py',
      'pickletools.py',
      'multiprocessing/connection.py',
      'hashlib.py',
      'getopt.py',
79 80
      'os.py',
      'types.py',
81
     ], dict(auto_cpdef=False)),
82
]
83
del special_directives[:]  # currently unused
84

85
def build_extensions(includes='**/*.py',
86
                     excludes=None,
87
                     special_directives=special_directives,
88
                     language_level=sys.version_info[0],
89 90 91
                     parallel=None):
    if isinstance(includes, str):
        includes = [includes]
92
    excludes = list(excludes or exclude_patterns) + broken
93 94 95 96 97 98 99 100 101 102 103 104 105

    all_groups = (special_directives or []) + [(includes, {})]
    extensions = []
    for modules, directives in all_groups:
        exclude_now = excludes[:]
        for other_modules, _ in special_directives:
            if other_modules != modules:
                exclude_now.extend(other_modules)

        d = dict(default_directives)
        d.update(directives)

        extensions.extend(
106 107
            cythonize(
                modules,
108 109
                exclude=exclude_now,
                exclude_failures=True,
110
                language_level=language_level,
111 112
                compiler_directives=d,
                nthreads=parallel,
113
            ))
114 115
    return extensions

116

117 118
def build(extensions):
    try:
119 120
        setup(ext_modules=extensions)
        result = True
121 122
    except:
        import traceback
123 124
        print('error building extensions %s' % (
            [ext.name for ext in extensions],))
125
        traceback.print_exc()
126 127 128
        result = False
    return extensions, result

129 130 131 132 133 134 135

def _build(args):
    sys_args, ext = args
    sys.argv[1:] = sys_args
    return build([ext])


136 137 138 139 140 141 142 143 144 145
def parse_args():
    from optparse import OptionParser
    parser = OptionParser('%prog [options] [LIB_DIR (default: ./Lib)]')
    parser.add_option(
        '--current-python', dest='current_python', action='store_true',
        help='compile the stdlib of the running Python')
    parser.add_option(
        '-j', '--jobs', dest='parallel_jobs', metavar='N',
        type=int, default=1,
        help='run builds in N parallel jobs (default: 1)')
146 147 148
    parser.add_option(
        '-x', '--exclude', dest='excludes', metavar='PATTERN',
        action="append", help='exclude modules/packages matching PATTERN')
149 150 151 152 153 154 155 156
    options, args = parser.parse_args()
    if not args:
        args = ['./Lib']
    elif len(args) > 1:
        parser.error('only one argument expected, got %d' % len(args))
    return options, args


157
if __name__ == '__main__':
158 159
    options, args = parse_args()
    if options.current_python:
160 161
        # assume that the stdlib is where the "os" module lives
        os.chdir(os.path.dirname(os.__file__))
162 163 164 165 166 167 168 169 170 171 172 173 174 175
    else:
        os.chdir(args[0])

    pool = None
    parallel_jobs = options.parallel_jobs
    if options.parallel_jobs:
        try:
            import multiprocessing
            pool = multiprocessing.Pool(parallel_jobs)
            print("Building in %d parallel processes" % parallel_jobs)
        except (ImportError, OSError):
            print("Not building in parallel")
            parallel_jobs = 0

176 177 178
    extensions = build_extensions(
        parallel=parallel_jobs,
        excludes=options.excludes)
Stefan Behnel's avatar
Stefan Behnel committed
179
    sys_args = ['build_ext', '-i']
180 181
    if pool is not None:
        results = pool.map(_build, [(sys_args, ext) for ext in extensions])
182 183 184 185
        pool.close()
        pool.join()
        for ext, result in results:
            if not result:
186
                print("building extension %s failed" % (ext[0].name,))
187
    else:
Stefan Behnel's avatar
Stefan Behnel committed
188
        sys.argv[1:] = sys_args
189
        build(extensions)