implement PEP 489 multi-phase module initialisation in Py3.5+

Cython/Compiler/ModuleNode.py

@@ -2116,8 +2116,28 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#else")
         code.putln("%s; /*proto*/" % header3)
         code.putln(header3)
-        code.putln("#endif")
+
+        # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.)
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
         code.putln("{")
+        code.putln("return PyModuleDef_Init(&%s);" % Naming.pymoduledef_cname)
+        code.putln("}")
+
+        mod_create_func = UtilityCode.load_cached("ModuleCreationPEP489", "ModuleSetupCode.c")
+        code.put(mod_create_func.impl.strip())
+
+        code.putln("")
+        # main module init code lives in Py_mod_exec function, not in PyInit function
+        code.putln("static int %s(PyObject *%s)" % (
+            Naming.pymodule_exec_func_cname,
+            Naming.pymodinit_module_arg))
+        code.putln("#endif")  # PEP489
+
+        code.putln("#endif")  # Py3
+
+        # start of module init/exec function (pre/post PEP 489)
+        code.putln("{")
+
         tempdecl_code = code.insertion_point()
 
         profile = code.globalstate.directives['profile']
@@ -2270,10 +2290,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.put_finish_refcount_context()
 
-        code.putln("#if PY_MAJOR_VERSION < 3")
-        code.putln("return;")
-        code.putln("#else")
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("return (%s != NULL) ? 0 : -1;" % env.module_cname)
+        code.putln("#elif PY_MAJOR_VERSION >= 3")
         code.putln("return %s;" % env.module_cname)
+        code.putln("#else")
+        code.putln("return;")
         code.putln("#endif")
         code.putln('}')
 
@@ -2436,18 +2458,35 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.putln("")
         code.putln("#if PY_MAJOR_VERSION >= 3")
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" %
+                   Naming.pymodule_create_func_cname)
+        code.putln("static int %s(PyObject* module); /*proto*/" %
+                   Naming.pymodule_exec_func_cname)
+
+        code.putln("static PyModuleDef_Slot %s[] = {" % Naming.pymoduledef_slots_cname)
+        code.putln("{Py_mod_create, %s}," % Naming.pymodule_create_func_cname)
+        code.putln("{Py_mod_exec, %s}," % Naming.pymodule_exec_func_cname)
+        code.putln("{0, NULL}")
+        code.putln("};")
+        code.putln("#endif")
+
+        code.putln("")
         code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname)
-        code.putln("#if PY_VERSION_HEX < 0x03020000")
-        # fix C compiler warnings due to missing initialisers
-        code.putln("  { PyObject_HEAD_INIT(NULL) NULL, 0, NULL },")
-        code.putln("#else")
         code.putln("  PyModuleDef_HEAD_INIT,")
-        code.putln("#endif")
         code.putln('  "%s",' % env.module_name)
         code.putln("  %s, /* m_doc */" % doc)
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("  0, /* m_size */")
+        code.putln("#else")
         code.putln("  -1, /* m_size */")
+        code.putln("#endif")
         code.putln("  %s /* m_methods */," % env.method_table_cname)
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("  %s, /* m_slots */" % Naming.pymoduledef_slots_cname)
+        code.putln("#else")
         code.putln("  NULL, /* m_reload */")
+        code.putln("#endif")
         code.putln("  NULL, /* m_traverse */")
         code.putln("  NULL, /* m_clear */")
         code.putln("  %s /* m_free */" % cleanup_func)
@@ -2461,6 +2500,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             doc = "%s" % code.get_string_const(env.doc)
         else:
             doc = "0"
+
+        code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT")
+        code.putln("%s = %s;" % (
+            env.module_cname,
+            Naming.pymodinit_module_arg))
+        code.put_incref(env.module_cname, py_object_type, nanny=False)
+        code.putln("#else")
         code.putln("#if PY_MAJOR_VERSION < 3")
         code.putln(
             '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % (
@@ -2476,6 +2522,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 Naming.pymoduledef_cname))
         code.putln("#endif")
         code.putln(code.error_goto_if_null(env.module_cname, self.pos))
+        code.putln("#endif")  # CYTHON_PEP489_MULTI_PHASE_INIT
+
         code.putln(
             "%s = PyModule_GetDict(%s); %s" % (
                 env.module_dict_cname, env.module_cname,

Cython/Compiler/Naming.py

@@ -101,6 +101,10 @@ print_function   = pyrex_prefix + "print"
 print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
 pymoduledef_cname = pyrex_prefix + "moduledef"
+pymoduledef_slots_cname = pyrex_prefix + "moduledef_slots"
+pymodinit_module_arg = pyrex_prefix + "pyinit_module"
+pymodule_create_func_cname = pyrex_prefix + "pyinit_module_create"
+pymodule_exec_func_cname = pyrex_prefix + "pyinit_module_exec"
 optional_args_cname = pyrex_prefix + "optional_args"
 import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"

Cython/Compiler/Symtab.py

@@ -1092,7 +1092,8 @@ class ModuleScope(Scope):
         self.undeclared_cached_builtins = []
         self.namespace_cname = self.module_cname
         self._cached_tuple_types = {}
-        for var_name in ['__builtins__', '__name__', '__file__', '__doc__', '__path__']:
+        for var_name in ['__builtins__', '__name__', '__file__', '__doc__', '__path__',
+                         '__spec__', '__loader__', '__package__', '__cached__']:
             self.declare_var(EncodedString(var_name), py_object_type, None)
 
     def qualifying_scope(self):

Cython/Utility/ModuleSetupCode.c

@@ -69,6 +69,8 @@
   #define CYTHON_FAST_THREAD_STATE 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #undef CYTHON_PEP489_MULTI_PHASE_INIT
+  #define CYTHON_PEP489_MULTI_PHASE_INIT 0
 
 #elif defined(PYSTON_VERSION)
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -102,6 +104,8 @@
   #define CYTHON_FAST_THREAD_STATE 0
   #undef CYTHON_FAST_PYCALL
   #define CYTHON_FAST_PYCALL 0
+  #undef CYTHON_PEP489_MULTI_PHASE_INIT
+  #define CYTHON_PEP489_MULTI_PHASE_INIT 0
 
 #else
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -150,6 +154,9 @@
   #ifndef CYTHON_FAST_PYCALL
     #define CYTHON_FAST_PYCALL 1
   #endif
+  #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
+    #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000)
+  #endif
 #endif
 
 #if !defined(CYTHON_FAST_PYCCALL)
@@ -568,6 +575,52 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc
 PyEval_InitThreads();
 #endif
 
+
+/////////////// ModuleCreationPEP489 ///////////////
+//@substitute: naming
+
+#if CYTHON_PEP489_MULTI_PHASE_INIT
+static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name) {
+    PyObject *value = PyObject_GetAttrString(spec, from_name);
+    int result = 0;
+    if (likely(value)) {
+        result = PyDict_SetItemString(moddict, to_name, value);
+        Py_DECREF(value);
+    } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+        PyErr_Clear();
+    } else {
+        result = -1;
+    }
+    return result;
+}
+
+static PyObject* ${pymodule_create_func_cname}(PyObject *spec, PyModuleDef *def) {
+    PyObject *module = NULL, *moddict, *modname;
+
+    modname = PyObject_GetAttrString(spec, "name");
+    if (unlikely(!modname)) goto bad;
+
+    module = PyModule_NewObject(modname);
+    Py_DECREF(modname);
+    if (unlikely(!module)) goto bad;
+
+    moddict = PyModule_GetDict(module);
+    if (unlikely(!moddict)) goto bad;
+    // moddict is a borrowed reference
+
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__") < 0)) goto bad;
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__") < 0)) goto bad;
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__") < 0)) goto bad;
+    if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__") < 0)) goto bad;
+
+    return module;
+bad:
+    Py_XDECREF(module);
+    return NULL;
+}
+#endif
+
+
 /////////////// CodeObjectCache.proto ///////////////
 
 typedef struct {

runtests.py

@@ -349,6 +349,7 @@ VER_DEP_MODULES = {
     (3,4): (operator.lt, lambda x: x in ['run.py34_signature',
                                          ]),
     (3,5): (operator.lt, lambda x: x in ['run.py35_pep492_interop',
+                                         'run.mod__spec__',
                                          ]),
 }
 

tests/run/mod__spec__.pyx

+# mode: run
+# tag: pep489
+
+import os.path
+
+module_spec = __spec__
+module_file = __file__
+
+
+def check_spec(spec=__spec__):
+    """
+    >>> check_spec()
+    """
+    assert __spec__ is not None
+    assert __spec__ is spec
+
+    assert __name__
+    assert __name__ == spec.name
+
+    assert spec.loader is not None
+    assert spec.loader is __loader__
+
+    assert not spec.parent
+    assert not __package__
+
+    assert spec.origin
+    assert spec.origin == module_file
+    assert spec.origin == __file__
+    assert os.path.basename(spec.origin).startswith(__name__)
+
+
+# validate that ModuleSpec is already complete at module initialisation time
+check_spec()
+check_spec(__spec__)
+check_spec(module_spec)
+
+
+def file_in_module():
+    """
+    >>> print(file_in_module())
+    mod__spec__
+    """
+    return os.path.basename(module_file).split('.', 1)[0]
+
+
+def file_in_function():
+    """
+    >>> print(file_in_function())
+    mod__spec__
+    """
+    return os.path.basename(__file__).split('.', 1)[0]