always set __path__ for compiled packages and in Py3.3 actually implement...

always set __path__ for compiled packages and in Py3.3 actually implement finding the file path at module init time by rerunning the import lookup

CHANGES.rst

@@ -8,6 +8,9 @@ Cython Changelog
 Features added
 --------------
 
+* Package compilation (i.e. ``__init__.py`` files) now works, starting
+  with Python 3.3.
+
 * The cython-mode.el script for Emacs was updated.  Patch by Ivan Andrus.
 
 Bugs fixed

Cython/Compiler/ModuleNode.py

@@ -1973,7 +1973,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 code.error_goto(self.pos)))
         code.putln("}")
 
-        self.generate_module_path_setup(env, code)
+        # set up __file__ and __path__, then add the module to sys.modules
+        self.generate_module_import_setup(env, code)
 
         if Options.cache_builtins:
             code.putln("/*--- Builtin init code ---*/")
@@ -2043,34 +2044,65 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.exit_cfunc_scope()
 
-    def generate_module_path_setup(self, env, code):
-        if not env.directives['set_initial_path']:
-            return
+    def generate_module_import_setup(self, env, code):
         module_path = env.directives['set_initial_path']
         if module_path == 'SOURCEFILE':
             module_path = self.pos[0].filename
-            if not module_path:
-                return
-        code.putln('if (__Pyx_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
-            env.module_cname,
-            code.globalstate.get_py_string_const(
-                EncodedString(decode_filename(module_path))).cname,
-            code.error_goto(self.pos)))
-        if env.is_package:
-            # compiling a package => set __path__ as well
-            temp = code.funcstate.allocate_temp(py_object_type, True)
-            code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
-                temp,
-                code.globalstate.get_py_string_const(
-                    EncodedString(decode_filename(os.path.dirname(module_path)))).cname,
-                code.error_goto_if_null(temp, self.pos)))
-            code.put_gotref(temp)
-            code.putln('if (__Pyx_SetAttrString(%s, "__path__", %s) < 0) %s;' % (
+
+        if module_path:
+            code.putln('if (__Pyx_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
                 env.module_cname,
-                temp,
+                code.globalstate.get_py_string_const(
+                    EncodedString(decode_filename(module_path))).cname,
                 code.error_goto(self.pos)))
-            code.put_decref_clear(temp, py_object_type)
-            code.funcstate.release_temp(temp)
+
+            if env.is_package:
+                # set __path__ to mark the module as package
+                temp = code.funcstate.allocate_temp(py_object_type, True)
+                code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
+                    temp,
+                    code.globalstate.get_py_string_const(
+                        EncodedString(decode_filename(
+                            os.path.dirname(module_path)))).cname,
+                    code.error_goto_if_null(temp, self.pos)))
+                code.put_gotref(temp)
+                code.putln(
+                    'if (__Pyx_SetAttrString(%s, "__path__", %s) < 0) %s;' % (
+                        env.module_cname, temp, code.error_goto(self.pos)))
+                code.put_decref_clear(temp, py_object_type)
+                code.funcstate.release_temp(temp)
+
+        elif env.is_package:
+            # packages require __path__, so all we can do is try to figure
+            # out the module path at runtime by rerunning the import lookup
+            package_name, _ = self.full_module_name.rsplit('.', 1)
+            if '.' in package_name:
+                parent_name = '"%s"' % (package_name.rsplit('.', 1)[0],)
+            else:
+                parent_name = 'NULL'
+            code.globalstate.use_utility_code(UtilityCode.load(
+                "SetPackagePathFromImportLib", "ImportExport.c"))
+            code.putln(code.error_goto_if_neg(
+                '__Pyx_SetPackagePathFromImportLib(%s, %s)' % (
+                    parent_name,
+                    code.globalstate.get_py_string_const(
+                        EncodedString(env.module_name)).cname),
+                self.pos))
+
+        # CPython may not have put us into sys.modules yet, but relative imports and reimports require it
+        fq_module_name = self.full_module_name
+        if fq_module_name.endswith('.__init__'):
+            fq_module_name = fq_module_name[:-len('.__init__')]
+        code.putln("#if PY_MAJOR_VERSION >= 3")
+        code.putln("{")
+        code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" %
+                   code.error_goto_if_null("modules", self.pos))
+        code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name)
+        code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % (
+            fq_module_name, env.module_cname), self.pos))
+        code.putln("}")
+        code.putln("}")
+        code.putln("#endif")
 
     def generate_module_cleanup_func(self, env, code):
         if not Options.generate_cleanup_code:
@@ -2210,21 +2242,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 code.error_goto_if_null(env.module_dict_cname, self.pos)))
         code.put_incref(env.module_dict_cname, py_object_type, nanny=False)
 
-        # CPython may not have put us into sys.modules yet, but relative imports and reimports require it
-        fq_module_name = env.qualified_name
-        if fq_module_name.endswith('.__init__'):
-            fq_module_name = fq_module_name[:-len('.__init__')]
-        code.putln("#if PY_MAJOR_VERSION >= 3")
-        code.putln("{")
-        code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" %
-                   code.error_goto_if_null("modules", self.pos))
-        code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name)
-        code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % (
-            fq_module_name, env.module_cname), self.pos))
-        code.putln("}")
-        code.putln("}")
-        code.putln("#endif")
-
         code.putln(
             '%s = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); %s' % (
                 Naming.builtins_cname,

Cython/Utility/ImportExport.c

@@ -147,6 +147,89 @@ bad:
 }
 #endif
 
+
+/////////////// SetPackagePathFromImportLib.proto ///////////////
+
+#if PY_MAJOR_VERSION >= 3
+static int __Pyx_SetPackagePathFromImportLib(const char* parent_package_name, PyObject *module_name);
+#else
+#define __Pyx_SetPackagePathFromImportLib(a, b) 0
+#endif
+
+/////////////// SetPackagePathFromImportLib ///////////////
+//@requires: ObjectHandling.c::PyObjectGetAttrStr
+//@substitute: naming
+
+#if PY_MAJOR_VERSION >= 3
+static int __Pyx_SetPackagePathFromImportLib(const char* parent_package_name, PyObject *module_name) {
+    PyObject *importlib, *loader, *osmod, *ossep, *parts, *package_path;
+    PyObject *path = NULL, *file_path = NULL;
+    int result;
+    if (parent_package_name) {
+        PyObject *package = PyImport_ImportModule(parent_package_name);
+        if (unlikely(!package))
+            goto bad;
+        path = PyObject_GetAttrString(package, "__path__");
+        Py_DECREF(package);
+        if (unlikely(!path) || unlikely(path == Py_None))
+            goto bad;
+    } else {
+        path = Py_None; Py_INCREF(Py_None);
+    }
+    // package_path = [importlib.find_loader(module_name, path).path.rsplit(os.sep, 1)[0]]
+    importlib = PyImport_ImportModule("importlib");
+    if (unlikely(!importlib))
+        goto bad;
+    loader = PyObject_CallMethod(importlib, "find_loader", "(OO)", module_name, path);
+    Py_DECREF(importlib);
+    Py_DECREF(path); path = NULL;
+    if (unlikely(!loader))
+        goto bad;
+    file_path = PyObject_GetAttrString(loader, "path");
+    Py_DECREF(loader);
+    if (unlikely(!file_path))
+        goto bad;
+
+    if (unlikely(__Pyx_SetAttrString($module_cname, "__file__", file_path) < 0))
+        goto bad;
+
+    osmod = PyImport_ImportModule("os");
+    if (unlikely(!osmod))
+        goto bad;
+    ossep = PyObject_GetAttrString(osmod, "sep");
+    Py_DECREF(osmod);
+    if (unlikely(!ossep))
+        goto bad;
+    parts = PyObject_CallMethod(file_path, "rsplit", "(Oi)", ossep, 1);
+    Py_DECREF(file_path); file_path = NULL;
+    Py_DECREF(ossep);
+    if (unlikely(!parts))
+        goto bad;
+    package_path = Py_BuildValue("[O]", PyList_GET_ITEM(parts, 0));
+    Py_DECREF(parts);
+    if (unlikely(!package_path))
+        goto bad;
+    goto set_path;
+
+bad:
+    PyErr_WriteUnraisable(module_name);
+    Py_XDECREF(path);
+    Py_XDECREF(file_path);
+
+    // set an empty path list on failure
+    PyErr_Clear();
+    package_path = PyList_New(0);
+    if (unlikely(!package_path))
+        return -1;
+
+set_path:
+    result = __Pyx_SetAttrString($module_cname, "__path__", package_path);
+    Py_DECREF(package_path);
+    return result;
+}
+#endif
+
+
 /////////////// TypeImport.proto ///////////////
 
 static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict);  /*proto*/

runtests.py

@@ -248,6 +248,8 @@ VER_DEP_MODULES = {
                                         'compile.extsetslice',
                                         'compile.extdelslice',
                                         'run.special_methods_T561_py2']),
+    (3,3) : (operator.lt, lambda x: x in ['build.package_compilation',
+                                          ]),
 }
 
 # files that should not be converted to Python 3 code with 2to3

tests/build/package_compilation.srctree

+PYTHON setup.py build_ext --inplace
+PYTHON -c "import toppkg; assert '.py' not in toppkg.__file__; assert toppkg.PACKAGE == 1"
+PYTHON -c "import toppkg.subpkg; assert '.py' not in toppkg.__file__; assert '.py' not in toppkg.subpkg.__file__; assert toppkg.subpkg.PACKAGE == 2"
+PYTHON -c "import toppkg.a; assert toppkg.a.MODULE == 'a'"
+PYTHON -c "from toppkg.subpkg import a; assert a.MODULE == 'subpkg.a'"
+
+######## setup.py ########
+
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize("toppkg/**/*.py"),
+)
+
+######## toppkg/__init__.py ########
+
+import sys
+assert 'toppkg' in sys.modules
+
+assert __path__ is not None, "__path__ is None"
+
+assert __path__, "__path__ is empty"
+assert 'toppkg' in __path__[0], "toppkg not in __path__[0]"
+
+assert 'toppkg' in __file__
+
+from . import a
+assert a.MODULE == 'a'
+
+from . import b
+assert b.MODULE == 'b'
+
+PACKAGE = 1
+
+######## toppkg/a.py ########
+
+MODULE = 'a'
+
+######## toppkg/b.py ########
+
+MODULE = 'b'
+
+######## toppkg/subpkg/__init__.py ########
+
+PACKAGE = 2
+
+######## toppkg/subpkg/a.py ########
+
+MODULE = 'subpkg.a'