Merge remote-tracking branch 'main/master'

Cython/Compiler/Main.py

@@ -332,8 +332,6 @@ class Context(object):
         module_name, _ = os.path.splitext(filename)
         if "." in module_name:
             return module_name
-        if module_name == "__init__":
-            dir, module_name = os.path.split(dir)
         names = [module_name]
         while self.is_package_dir(dir):
             parent, package_name = os.path.split(dir)
@@ -384,10 +382,11 @@ def create_default_resultobj(compilation_source, options):
         result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
     return result
 
-def run_pipeline(source, options, full_module_name = None):
+def run_pipeline(source, options, full_module_name=None, context=None):
     import Pipeline
 
-    context = options.create_context()
+    if context is None:
+        context = options.create_context()
 
     # Set up source object
     cwd = os.getcwd()
@@ -551,16 +550,20 @@ def compile_multiple(sources, options):
     if timestamps is None:
         timestamps = recursive
     verbose = options.verbose or ((recursive or timestamps) and not options.quiet)
+    context = None
     for source in sources:
         if source not in processed:
-            # Compiling multiple sources in one context doesn't quite
-            # work properly yet.
+            if context is None:
+                context = options.create_context()
             if not timestamps or context.c_file_out_of_date(source):
                 if verbose:
                     sys.stderr.write("Compiling %s\n" % source)
 
-                result = run_pipeline(source, options)
+                result = run_pipeline(source, options, context=context)
                 results.add(source, result)
+                # Compiling multiple sources in one context doesn't quite
+                # work properly yet.
+                context = None
             processed.add(source)
             if recursive:
                 for module_name in context.find_cimported_module_names(source):

Cython/Compiler/ModuleNode.py

@@ -1840,29 +1840,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 code.error_goto(self.pos)))
         code.putln("}")
 
-        if env.directives['set_initial_path_from_source'] and self.pos[0]:
-            source_path = self.pos[0].filename
-            if source_path:
-                code.putln('if (__Pyx_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
-                    env.module_cname,
-                    code.globalstate.get_py_string_const(
-                        EncodedString(decode_filename(source_path))).cname,
-                code.error_goto(self.pos)))
-                if os.path.splitext(os.path.basename(source_path))[0] == '__init__':
-                    # compiling a package => set __path__ as well
-                    temp = code.funcstate.allocate_temp(py_object_type, True)
-                    code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
-                        temp,
-                        code.globalstate.get_py_string_const(
-                            EncodedString(decode_filename(os.path.dirname(source_path)))).cname,
-                        code.error_goto_if_null(temp, self.pos)))
-                    code.put_gotref(temp)
-                    code.putln('if (__Pyx_SetAttrString(%s, "__path__", %s) < 0) %s;' % (
-                        env.module_cname,
-                        temp,
-                        code.error_goto(self.pos)))
-                    code.put_decref_clear(temp, py_object_type)
-                    code.funcstate.release_temp(temp)
+        self.generate_module_path_setup(env, code)
 
         if Options.cache_builtins:
             code.putln("/*--- Builtin init code ---*/")
@@ -1932,6 +1910,35 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.exit_cfunc_scope()
 
+    def generate_module_path_setup(self, env, code):
+        if not env.directives['set_initial_path']:
+            return
+        module_path = env.directives['set_initial_path']
+        if module_path == 'SOURCEFILE':
+            module_path = self.pos[0].filename
+            if not module_path:
+                return
+        code.putln('if (__Pyx_SetAttrString(%s, "__file__", %s) < 0) %s;' % (
+            env.module_cname,
+            code.globalstate.get_py_string_const(
+                EncodedString(decode_filename(module_path))).cname,
+            code.error_goto(self.pos)))
+        if os.path.splitext(os.path.basename(module_path))[0] == '__init__':
+            # compiling a package => set __path__ as well
+            temp = code.funcstate.allocate_temp(py_object_type, True)
+            code.putln('%s = Py_BuildValue("[O]", %s); %s' % (
+                temp,
+                code.globalstate.get_py_string_const(
+                    EncodedString(decode_filename(os.path.dirname(module_path)))).cname,
+                code.error_goto_if_null(temp, self.pos)))
+            code.put_gotref(temp)
+            code.putln('if (__Pyx_SetAttrString(%s, "__path__", %s) < 0) %s;' % (
+                env.module_cname,
+                temp,
+                code.error_goto(self.pos)))
+            code.put_decref_clear(temp, py_object_type)
+            code.funcstate.release_temp(temp)
+
     def generate_module_cleanup_func(self, env, code):
         if not Options.generate_cleanup_code:
             return
@@ -2038,6 +2045,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             "if (!%s) %s;" % (
                 env.module_cname,
                 code.error_goto(self.pos)))
+        if env.is_package:
+            # some CPython versions have not registered us in sys.modules yet
+            code.putln("{")
+            code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" %
+                       code.error_goto_if_null("modules", self.pos))
+            code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % env.module_name)
+            code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % (
+                env.module_name, env.module_cname), self.pos))
+            code.putln("}")
+            code.putln("}")
         code.putln("#if PY_MAJOR_VERSION < 3")
         code.putln(
             "Py_INCREF(%s);" %

Cython/Compiler/Options.py

@@ -98,8 +98,8 @@ directive_defaults = {
     'fast_getattr': False, # Undocumented until we come up with a better way to handle this everywhere.
     'py2_import': False, # For backward compatibility of Cython's source code in Py3 source mode
 
-    # set __file__ and/or __path__ to source file path at import time (instead of not having them available)
-    'set_initial_path_from_source' : False,
+    # set __file__ and/or __path__ to known source/target path at import time (instead of not having them available)
+    'set_initial_path' : None,  # SOURCEFILE or "/full/path/to/module"
 
     'warn': None,
     'warn.undeclared': False,
@@ -143,6 +143,7 @@ directive_types = {
     'ccall' : None,
     'cclass' : None,
     'returns' : type,
+    'set_initial_path': str,
     }
 
 for key, val in directive_defaults.items():
@@ -156,7 +157,7 @@ directive_scopes = { # defaults to available everywhere
     'autotestdict' : ('module',),
     'autotestdict.all' : ('module',),
     'autotestdict.cdef' : ('module',),
-    'set_initial_path_from_source' : ('module',),
+    'set_initial_path' : ('module',),
     'test_assert_path_exists' : ('function', 'class', 'cclass'),
     'test_fail_if_path_exists' : ('function', 'class', 'cclass'),
 }

Cython/Compiler/Symtab.py

@@ -957,6 +957,7 @@ class ModuleScope(Scope):
     # has_import_star      boolean            Module contains import *
     # cpp                  boolean            Compiling a C++ file
     # is_cython_builtin    boolean            Is this the Cython builtin scope (or a child scope)
+    # is_package           boolean            Is this a package module? (__init__)
 
     is_module_scope = 1
     has_import_star = 0
@@ -969,10 +970,12 @@ class ModuleScope(Scope):
         Scope.__init__(self, name, outer_scope, parent_module)
         if name != "__init__":
             self.module_name = name
+            self.is_package = False
         else:
             # Treat Spam/__init__.pyx specially, so that when Python loads
             # Spam/__init__.so, initSpam() is defined.
             self.module_name = parent_module.module_name
+            self.is_package = True
         self.module_name = EncodedString(self.module_name)
         self.context = context
         self.module_cname = Naming.module_cname

runtests.py

@@ -210,8 +210,6 @@ VER_DEP_MODULES = {
                                         'compile.extsetslice',
                                         'compile.extdelslice',
                                         'run.special_methods_T561_py2']),
-    (3,3): (operator.ge, lambda x: x in ['run.initial_file_path', # new importlib currently fails to load __init__.so
-                                         ]),
 }
 
 # files that should not be converted to Python 3 code with 2to3

tests/run/initial_file_path.srctree

 PYTHON setup.py build_ext --inplace
-PYTHON -c "import my_test_package, my_test_package.a; my_test_package.test(); my_test_package.a.test()"
+PYTHON -c "import my_test_package; assert not my_test_package.__file__.rstrip('co').endswith('.py'), my_test_package.__file__; my_test_package.test()"
+PYTHON -c "import my_test_package.a; my_test_package.a.test()"
 
 ######## setup.py ########
 
@@ -12,7 +13,7 @@ setup(
 
 ######## my_test_package/__init__.py ########
 
-# cython: set_initial_path_from_source=True
+# cython: set_initial_path=SOURCEFILE
 
 initial_path = __path__
 initial_file = __file__
@@ -34,7 +35,7 @@ def test():
 
 ######## my_test_package/a.py ########
 
-# cython: set_initial_path_from_source=True
+# cython: set_initial_path=SOURCEFILE
 
 initial_file = __file__