somewhat ugly hack for Py3.1+ to get Cython modules compiled on installation

Cython/Compiler/Parsing.py

@@ -12,10 +12,12 @@ import re
 import sys
 
 try:
-    set
-except NameError:
-    # Python 2.3
-    from sets import Set as set
+    from __builtin__ import set
+except ImportError:
+    try:
+        from builtins import set
+    except ImportError:
+        from sets import Set as set
 
 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor
 import Nodes

Cython/Compiler/Scanning.py

@@ -97,7 +97,10 @@ def initial_compile_time_env():
         'UNAME_VERSION', 'UNAME_MACHINE')
     for name, value in zip(names, platform.uname()):
         benv.declare(name, value)
-    import __builtin__ as builtins
+    try:
+        import __builtin__ as builtins
+    except ImportError:
+        import builtins
     names = ('False', 'True',
         'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate',
         'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min',

Cython/Distutils/__init__.py

@@ -7,5 +7,6 @@
 # and keep the old one under the module name _build_ext,
 # so that *our* build_ext can make use of it.
 
-from build_ext import build_ext
+from Cython.Distutils.build_ext import build_ext
+    
 # from extension import Extension

Cython/Distutils/build_ext.py

@@ -15,16 +15,6 @@ from distutils.sysconfig import customize_compiler, get_python_version
 from distutils.dep_util import newer, newer_group
 from distutils import log
 from distutils.dir_util import mkpath
-try:
-    from Cython.Compiler.Main \
-        import CompilationOptions, \
-               default_options as pyrex_default_options, \
-               compile as cython_compile
-    from Cython.Compiler.Errors import PyrexError
-except ImportError, e:
-    print "failed to import Cython: %s" % e
-    PyrexError = None
-
 from distutils.command import build_ext as _build_ext
 
 extension_name_re = _build_ext.extension_name_re
@@ -83,18 +73,22 @@ class build_ext(_build_ext.build_ext):
             self.build_extension(ext)
 
     def cython_sources(self, sources, extension):
-
         """
         Walk the list of source files in 'sources', looking for Cython
         source files (.pyx and .py).  Run Cython on all that are
         found, and return a modified 'sources' list with Cython source
         files replaced by the generated C (or C++) files.
         """
-
-        if PyrexError == None:
-            raise DistutilsPlatformError, \
-                  ("Cython does not appear to be installed "
-                   "on platform '%s'") % os.name
+        try:
+            from Cython.Compiler.Main \
+                import CompilationOptions, \
+                       default_options as pyrex_default_options, \
+                       compile as cython_compile
+            from Cython.Compiler.Errors import PyrexError
+        except ImportError:
+            e = sys.exc_info()[1]
+            print("failed to import Cython: %s" % e)
+            raise DistutilsPlatformError("Cython does not appear to be installed")
 
         new_sources = []
         pyrex_sources = []

setup.py

@@ -25,6 +25,11 @@ if sys.platform == "darwin":
 
 setup_args = {}
 
+def add_command_class(name, cls):
+    cmdclasses = setup_args.get('cmdclass', {})
+    cmdclasses[name] = cls
+    setup_args['cmdclass'] = cmdclasses
+
 if sys.version_info[0] >= 3:
     import lib2to3.refactor
     from distutils.command.build_py \
@@ -34,7 +39,7 @@ if sys.version_info[0] >= 3:
                if fix.split('fix_')[-1] not in ('next',)
                ]
     build_py.fixer_names = fixers
-    setup_args['cmdclass'] = {"build_py" : build_py}
+    add_command_class("build_py", build_py)
 
 
 if sys.version_info < (2,4):
@@ -72,54 +77,84 @@ else:
     else:
         scripts = ["cython.py"]
 
+def compile_cython_modules():
+    source_root = os.path.abspath(os.path.dirname(__file__))
+    compiled_modules = ["Cython.Plex.Scanners",
+                        "Cython.Compiler.Scanning",
+                        "Cython.Compiler.Parsing",
+                        "Cython.Compiler.Visitor",
+                        "Cython.Runtime.refnanny"]
+    extensions = []
 
-try:
     if sys.version_info[0] >= 3:
-        raise ValueError
-    sys.argv.remove("--no-cython-compile")
-except ValueError:
-    try:
-        from distutils.command.build_ext import build_ext as build_ext_orig
-        class build_ext(build_ext_orig):
-            def build_extension(self, ext, *args, **kargs):
-                try:
-                    build_ext_orig.build_extension(self, ext, *args, **kargs)
-                except StandardError:
-                    print("Compilation of '%s' failed" % ext.sources[0])
-        from Cython.Compiler.Main import compile
-        from Cython import Utils
-        source_root = os.path.dirname(__file__)
-        compiled_modules = ["Cython.Plex.Scanners",
-                            "Cython.Compiler.Scanning",
-                            "Cython.Compiler.Parsing",
-                            "Cython.Compiler.Visitor",
-                            "Cython.Runtime.refnanny"]
-        extensions = []
+        from Cython.Distutils import build_ext as build_ext_orig
         for module in compiled_modules:
             source_file = os.path.join(source_root, *module.split('.'))
             if os.path.exists(source_file + ".py"):
                 pyx_source_file = source_file + ".py"
             else:
                 pyx_source_file = source_file + ".pyx"
-            c_source_file = source_file + ".c"
-            if not os.path.exists(c_source_file) or \
-               Utils.file_newer_than(pyx_source_file,
-                                     Utils.modification_time(c_source_file)):
-                print("Compiling module %s ..." % module)
-                result = compile(pyx_source_file)
-                c_source_file = result.c_file
-            if c_source_file:
-                extensions.append(
-                    Extension(module, sources = [c_source_file])
-                    )
-            else:
-                print("Compilation failed")
-        if extensions:
-            setup_args['ext_modules'] = extensions
-            setup_args['cmdclass'] = {"build_ext" : build_ext}
-    except Exception:
-        print("ERROR: %s" % sys.exc_info()[1])
-        print("Extension module compilation failed, using plain Python implementation")
+            extensions.append(
+                Extension(module, sources = [pyx_source_file])
+                )
+
+        class build_ext(build_ext_orig):
+            def build_extensions(self):
+                # add path where 2to3 installed the transformed sources
+                # and make sure Python (re-)imports them from there
+                already_imported = [ module for module in sys.modules
+                                     if module == 'Cython' or module.startswith('Cython.') ]
+                for module in already_imported:
+                    del sys.modules[module]
+                sys.path.insert(0, os.path.join(source_root, self.build_lib))
+
+                build_ext_orig.build_extensions(self)
+
+        setup_args['ext_modules'] = extensions
+        add_command_class("build_ext", build_ext)
+
+    else: # Python 2.x
+        from distutils.command.build_ext import build_ext as build_ext_orig
+        try:
+            class build_ext(build_ext_orig):
+                def build_extension(self, ext, *args, **kargs):
+                    try:
+                        build_ext_orig.build_extension(self, ext, *args, **kargs)
+                    except StandardError:
+                        print("Compilation of '%s' failed" % ext.sources[0])
+            from Cython.Compiler.Main import compile
+            from Cython import Utils
+            source_root = os.path.dirname(__file__)
+            for module in compiled_modules:
+                source_file = os.path.join(source_root, *module.split('.'))
+                if os.path.exists(source_file + ".py"):
+                    pyx_source_file = source_file + ".py"
+                else:
+                    pyx_source_file = source_file + ".pyx"
+                c_source_file = source_file + ".c"
+                if not os.path.exists(c_source_file) or \
+                   Utils.file_newer_than(pyx_source_file,
+                                         Utils.modification_time(c_source_file)):
+                    print("Compiling module %s ..." % module)
+                    result = compile(pyx_source_file)
+                    c_source_file = result.c_file
+                if c_source_file:
+                    extensions.append(
+                        Extension(module, sources = [c_source_file])
+                        )
+                else:
+                    print("Compilation failed")
+            if extensions:
+                setup_args['ext_modules'] = extensions
+                add_command_class("build_ext", build_ext)
+        except Exception:
+            print("ERROR: %s" % sys.exc_info()[1])
+            print("Extension module compilation failed, using plain Python implementation")
+
+try:
+    sys.argv.remove("--no-cython-compile")
+except ValueError:
+    compile_cython_modules()
 
 setup_args.update(setuptools_extra_args)