Merge pull request #1794 from scoder/pep489_multi_phase_init

implement PEP 489 multi-phase module initialisation in Py3.5+

Cython/Compiler/ModuleNode.py

@@ -201,10 +201,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             h_code.putln("")
             h_code.putln("#endif /* !%s */" % api_guard)
             h_code.putln("")
+            h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */")
+            h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */")
+            h_code.putln("")
             h_code.putln("#if PY_MAJOR_VERSION < 3")
             h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name)
             h_code.putln("#else")
-            h_code.putln("PyMODINIT_FUNC PyInit_%s(void);" % env.module_name)
+            h_code.putln("PyMODINIT_FUNC %s(void);" % self.mod_init_func_cname('PyInit', env))
             h_code.putln("#endif")
             h_code.putln("")
             h_code.putln("#endif /* !%s */" % h_guard)
@@ -2110,15 +2113,35 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.enter_cfunc_scope(self.scope)
         code.putln("")
         header2 = "PyMODINIT_FUNC init%s(void)" % env.module_name
-        header3 = "PyMODINIT_FUNC PyInit_%s(void)" % env.module_name
+        header3 = "PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env)
         code.putln("#if PY_MAJOR_VERSION < 3")
         code.putln("%s; /*proto*/" % header2)
         code.putln(header2)
         code.putln("#else")
         code.putln("%s; /*proto*/" % header3)
         code.putln(header3)
-        code.putln("#endif")
+
+        # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.)
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
         code.putln("{")
+        code.putln("return PyModuleDef_Init(&%s);" % Naming.pymoduledef_cname)
+        code.putln("}")
+
+        mod_create_func = UtilityCode.load_as_string("ModuleCreationPEP489", "ModuleSetupCode.c")[1]
+        code.put(mod_create_func)
+
+        code.putln("")
+        # main module init code lives in Py_mod_exec function, not in PyInit function
+        code.putln("static int %s(PyObject *%s)" % (
+            self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env),
+            Naming.pymodinit_module_arg))
+        code.putln("#endif")  # PEP489
+
+        code.putln("#endif")  # Py3
+
+        # start of module init/exec function (pre/post PEP 489)
+        code.putln("{")
+
         tempdecl_code = code.insertion_point()
 
         profile = code.globalstate.directives['profile']
@@ -2271,10 +2294,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.put_finish_refcount_context()
 
-        code.putln("#if PY_MAJOR_VERSION < 3")
-        code.putln("return;")
-        code.putln("#else")
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("return (%s != NULL) ? 0 : -1;" % env.module_cname)
+        code.putln("#elif PY_MAJOR_VERSION >= 3")
         code.putln("return %s;" % env.module_cname)
+        code.putln("#else")
+        code.putln("return;")
         code.putln("#endif")
         code.putln('}')
 
@@ -2288,14 +2313,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             module_path = self.pos[0].filename
 
         if module_path:
+            code.putln('if (!CYTHON_PEP489_MULTI_PHASE_INIT) {')
             code.putln('if (PyObject_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
                 env.module_cname,
                 code.globalstate.get_py_string_const(
                     EncodedString(decode_filename(module_path))).cname,
                 code.error_goto(self.pos)))
+            code.putln("}")
 
             if env.is_package:
                 # set __path__ to mark the module as package
+                code.putln('if (!CYTHON_PEP489_MULTI_PHASE_INIT) {')
                 temp = code.funcstate.allocate_temp(py_object_type, True)
                 code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
                     temp,
@@ -2309,10 +2337,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                         env.module_cname, temp, code.error_goto(self.pos)))
                 code.put_decref_clear(temp, py_object_type)
                 code.funcstate.release_temp(temp)
+                code.putln("}")
 
         elif env.is_package:
             # packages require __path__, so all we can do is try to figure
             # out the module path at runtime by rerunning the import lookup
+            code.putln("if (!CYTHON_PEP489_MULTI_PHASE_INIT) {")
             package_name, _ = self.full_module_name.rsplit('.', 1)
             if '.' in package_name:
                 parent_name = '"%s"' % (package_name.rsplit('.', 1)[0],)
@@ -2326,6 +2356,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     code.globalstate.get_py_string_const(
                         EncodedString(env.module_name)).cname),
                 self.pos))
+            code.putln("}")
 
         # CPython may not have put us into sys.modules yet, but relative imports and reimports require it
         fq_module_name = self.full_module_name
@@ -2425,6 +2456,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 main_method=Options.embed,
                 wmain_method=wmain))
 
+    def mod_init_func_cname(self, prefix, env):
+        return '%s_%s' % (prefix, env.module_name)
+
     def generate_pymoduledef_struct(self, env, code):
         if env.doc:
             doc = "%s" % code.get_string_const(env.doc)
@@ -2437,18 +2471,35 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.putln("")
         code.putln("#if PY_MAJOR_VERSION >= 3")
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env)
+        code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" %
+                   Naming.pymodule_create_func_cname)
+        code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname)
+
+        code.putln("static PyModuleDef_Slot %s[] = {" % Naming.pymoduledef_slots_cname)
+        code.putln("{Py_mod_create, (void*)%s}," % Naming.pymodule_create_func_cname)
+        code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname)
+        code.putln("{0, NULL}")
+        code.putln("};")
+        code.putln("#endif")
+
+        code.putln("")
         code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname)
-        code.putln("#if PY_VERSION_HEX < 0x03020000")
-        # fix C compiler warnings due to missing initialisers
-        code.putln("  { PyObject_HEAD_INIT(NULL) NULL, 0, NULL },")
-        code.putln("#else")
         code.putln("  PyModuleDef_HEAD_INIT,")
-        code.putln("#endif")
         code.putln('  "%s",' % env.module_name)
         code.putln("  %s, /* m_doc */" % doc)
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("  0, /* m_size */")
+        code.putln("#else")
         code.putln("  -1, /* m_size */")
+        code.putln("#endif")
         code.putln("  %s /* m_methods */," % env.method_table_cname)
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("  %s, /* m_slots */" % Naming.pymoduledef_slots_cname)
+        code.putln("#else")
         code.putln("  NULL, /* m_reload */")
+        code.putln("#endif")
         code.putln("  NULL, /* m_traverse */")
         code.putln("  NULL, /* m_clear */")
         code.putln("  %s /* m_free */" % cleanup_func)
@@ -2462,6 +2513,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             doc = "%s" % code.get_string_const(env.doc)
         else:
             doc = "0"
+
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("%s = %s;" % (
+            env.module_cname,
+            Naming.pymodinit_module_arg))
+        code.put_incref(env.module_cname, py_object_type, nanny=False)
+        code.putln("#else")
         code.putln("#if PY_MAJOR_VERSION < 3")
         code.putln(
             '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % (
@@ -2477,6 +2535,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 Naming.pymoduledef_cname))
         code.putln("#endif")
         code.putln(code.error_goto_if_null(env.module_cname, self.pos))
+        code.putln("#endif")  # CYTHON_PEP489_MULTI_PHASE_INIT
+
         code.putln(
             "%s = PyModule_GetDict(%s); %s" % (
                 env.module_dict_cname, env.module_cname,

Cython/Compiler/Naming.py

@@ -101,6 +101,10 @@ print_function   = pyrex_prefix + "print"
 print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
 pymoduledef_cname = pyrex_prefix + "moduledef"
+pymoduledef_slots_cname = pyrex_prefix + "moduledef_slots"
+pymodinit_module_arg = pyrex_prefix + "pyinit_module"
+pymodule_create_func_cname = pyrex_prefix + "pymod_create"
+pymodule_exec_func_cname = pyrex_prefix + "pymod_exec"
 optional_args_cname = pyrex_prefix + "optional_args"
 import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"

Cython/Compiler/Symtab.py

@@ -1103,7 +1103,8 @@ class ModuleScope(Scope):
         self.undeclared_cached_builtins = []
         self.namespace_cname = self.module_cname
         self._cached_tuple_types = {}
-        for var_name in ['__builtins__', '__name__', '__file__', '__doc__', '__path__']:
+        for var_name in ['__builtins__', '__name__', '__file__', '__doc__', '__path__',
+                         '__spec__', '__loader__', '__package__', '__cached__']:
             self.declare_var(EncodedString(var_name), py_object_type, None)
 
     def qualifying_scope(self):

Cython/Utility/Embed.c

@@ -32,6 +32,20 @@ static int __Pyx_main(int argc, wchar_t **argv) {
       %(module_is_main)s = 1;
       #if PY_MAJOR_VERSION < 3
           init%(module_name)s();
+      #elif CYTHON_PEP489_MULTI_PHASE_INIT
+          m = PyInit_%(module_name)s();
+          if (!PyModule_Check(m)) {
+              PyModuleDef *mdef = (PyModuleDef *) m;
+              PyObject *modname = PyUnicode_FromString("__main__");
+              m = NULL;
+              if (modname) {
+                  // FIXME: not currently calling PyModule_FromDefAndSpec() here because we do not have a module spec!
+                  // FIXME: not currently setting __file__, __path__, __spec__, ...
+                  m = PyModule_NewObject(modname);
+                  Py_DECREF(modname);
+                  if (m) PyModule_ExecDef(m, mdef);
+              }
+          }
       #else
           m = PyInit_%(module_name)s();
       #endif

Cython/Utility/ModuleSetupCode.c

@@ -71,6 +71,8 @@
   #define CYTHON_FAST_THREAD_STATE 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #undef CYTHON_PEP489_MULTI_PHASE_INIT
+  #define CYTHON_PEP489_MULTI_PHASE_INIT 0
 
 #elif defined(PYSTON_VERSION)
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -106,6 +108,8 @@
   #define CYTHON_FAST_THREAD_STATE 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #undef CYTHON_PEP489_MULTI_PHASE_INIT
+  #define CYTHON_PEP489_MULTI_PHASE_INIT 0
 
 #else
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -161,6 +165,9 @@
   #ifndef CYTHON_FAST_PYCALL
     #define CYTHON_FAST_PYCALL 1
   #endif
+  #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
+    #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000)
+  #endif
 #endif
 
 #if !defined(CYTHON_FAST_PYCCALL)
@@ -684,6 +691,52 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc
 PyEval_InitThreads();
 #endif
 
+
+/////////////// ModuleCreationPEP489 ///////////////
+//@substitute: naming
+
+//#if CYTHON_PEP489_MULTI_PHASE_INIT
+static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name) {
+    PyObject *value = PyObject_GetAttrString(spec, from_name);
+    int result = 0;
+    if (likely(value)) {
+        result = PyDict_SetItemString(moddict, to_name, value);
+        Py_DECREF(value);
+    } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+        PyErr_Clear();
+    } else {
+        result = -1;
+    }
+    return result;
+}
+
+static PyObject* ${pymodule_create_func_cname}(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) {
+    PyObject *module = NULL, *moddict, *modname;
+
+    modname = PyObject_GetAttrString(spec, "name");
+    if (unlikely(!modname)) goto bad;
+
+    module = PyModule_NewObject(modname);
+    Py_DECREF(modname);
+    if (unlikely(!module)) goto bad;
+
+    moddict = PyModule_GetDict(module);
+    if (unlikely(!moddict)) goto bad;
+    // moddict is a borrowed reference
+
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__") < 0)) goto bad;
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__") < 0)) goto bad;
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__") < 0)) goto bad;
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__") < 0)) goto bad;
+
+    return module;
+bad:
+    Py_XDECREF(module);
+    return NULL;
+}
+//#endif
+
+
 /////////////// CodeObjectCache.proto ///////////////
 
 typedef struct {

runtests.py

@@ -348,7 +348,10 @@ VER_DEP_MODULES = {
                                           ]),
     (3,4): (operator.lt, lambda x: x in ['run.py34_signature',
                                          ]),
+    (3,4,999): (operator.gt, lambda x: x in ['run.initial_file_path',
+                                             ]),
     (3,5): (operator.lt, lambda x: x in ['run.py35_pep492_interop',
+                                         'run.mod__spec__',
                                          ]),
 }
 

tests/build/module_api.srctree

@@ -108,6 +108,18 @@ void inita(void)
    if (!sys_modules) return;
    mod = PyInit_a();
    if (!mod) return;
+#if PY_VERSION_HEX >= 0x03050000
+   /* FIXME: this is incomplete and users shouldn't have to do this in the first place... */
+   if (!PyModule_Check(mod)) {
+      PyModuleDef *mdef = (PyModuleDef*)mod;
+      PyObject *modname = PyUnicode_FromString("a");
+      if (!modname) return;
+      mod = PyModule_NewObject(modname);
+      Py_DECREF(modname);
+      if (!mod) return;
+      PyModule_ExecDef(mod, mdef);
+   }
+#endif
    PyDict_SetItemString(sys_modules, (char*)"a", mod);
 }
 #endif

tests/run/mod__spec__.pyx

+# mode: run
+# tag: pep489
+
+import os.path
+
+module_spec = __spec__
+module_file = __file__
+
+
+def check_spec(spec=__spec__):
+    """
+    >>> check_spec()
+    """
+    assert __spec__ is not None
+    assert __spec__ is spec
+
+    assert __name__
+    assert __name__ == spec.name
+
+    assert spec.loader is not None
+    assert spec.loader is __loader__
+
+    assert not spec.parent
+    assert not __package__
+
+    assert spec.origin
+    assert spec.origin == module_file
+    assert spec.origin == __file__
+    assert os.path.basename(spec.origin).startswith(__name__)
+
+
+# validate that ModuleSpec is already complete at module initialisation time
+check_spec()
+check_spec(__spec__)
+check_spec(module_spec)
+
+
+def file_in_module():
+    """
+    >>> print(file_in_module())
+    mod__spec__
+    """
+    return os.path.basename(module_file).split('.', 1)[0]
+
+
+def file_in_function():
+    """
+    >>> print(file_in_function())
+    mod__spec__
+    """
+    return os.path.basename(__file__).split('.', 1)[0]

tests/run/relativeimport_star_T542.pyx

 from distutils import core, version
+
 __name__='distutils.core.cytest_relativeimport_T542' # fool Python we are in distutils
+__package__='distutils.core' # fool Python we are in distutils
+
 from . import *
 
 __doc__ = """