Commit 4993ba6a authored by Stefan Behnel's avatar Stefan Behnel

GH-2854: Reimplement absolute module imports to speed them up, especially...

GH-2854: Reimplement absolute module imports to speed them up, especially under Py3.7. Cython used to be 3x slower than CPython here.
Also enable absolute imports automatically in top-level (non-package) modules.
parent a8d7ccd2
...@@ -2527,44 +2527,61 @@ class ImportNode(ExprNode): ...@@ -2527,44 +2527,61 @@ class ImportNode(ExprNode):
# relative to the current module. # relative to the current module.
# None: decide the level according to language level and # None: decide the level according to language level and
# directives # directives
# get_top_level_module int true: return top-level module, false: return imported module
# module_names TupleNode the separate names of the module and submodules, or None
type = py_object_type type = py_object_type
module_names = None
get_top_level_module = False
is_temp = True
subexprs = ['module_name', 'name_list'] subexprs = ['module_name', 'name_list', 'module_names']
def analyse_types(self, env): def analyse_types(self, env):
if self.level is None: if self.level is None:
if (env.directives['py2_import'] or # For modules in packages, and without 'absolute_import' enabled, try relative (Py2) import first.
Future.absolute_import not in env.global_scope().context.future_directives): if env.global_scope().parent_module and (
env.directives['py2_import'] or
Future.absolute_import not in env.global_scope().context.future_directives):
self.level = -1 self.level = -1
else: else:
self.level = 0 self.level = 0
module_name = self.module_name.analyse_types(env) module_name = self.module_name.analyse_types(env)
self.module_name = module_name.coerce_to_pyobject(env) self.module_name = module_name.coerce_to_pyobject(env)
assert self.module_name.is_string_literal
if self.name_list: if self.name_list:
name_list = self.name_list.analyse_types(env) name_list = self.name_list.analyse_types(env)
self.name_list = name_list.coerce_to_pyobject(env) self.name_list = name_list.coerce_to_pyobject(env)
self.is_temp = 1 elif '.' in self.module_name.value:
self.module_names = TupleNode(self.module_name.pos, args=[
IdentifierStringNode(self.module_name.pos, value=part, constant_result=part)
for part in map(StringEncoding.EncodedString, self.module_name.value.split('.'))
]).analyse_types(env)
return self return self
gil_message = "Python import" gil_message = "Python import"
def generate_result_code(self, code): def generate_result_code(self, code):
if self.name_list: assert self.module_name.is_string_literal
name_list_code = self.name_list.py_result() module_name = self.module_name.value
if self.level == 0 and not self.name_list and not self.get_top_level_module:
if self.module_names:
assert self.module_names.is_literal # make sure we create the tuple only once
code.globalstate.use_utility_code(UtilityCode.load_cached("ImportDottedModule", "ImportExport.c"))
import_code = "__Pyx_ImportDottedModule(%s, %s)" % (
self.module_name.py_result(),
self.module_names.py_result() if self.module_names else 'NULL',
)
else: else:
name_list_code = "0" code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c"))
import_code = "__Pyx_Import(%s, %s, %d)" % (
code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) self.module_name.py_result(),
import_code = "__Pyx_Import(%s, %s, %d)" % ( self.name_list.py_result() if self.name_list else '0',
self.module_name.py_result(), self.level)
name_list_code,
self.level)
if (self.level <= 0 and if self.level <= 0 and module_name in utility_code_for_imports:
self.module_name.is_string_literal and helper_func, code_name, code_file = utility_code_for_imports[module_name]
self.module_name.value in utility_code_for_imports):
helper_func, code_name, code_file = utility_code_for_imports[self.module_name.value]
code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file)) code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file))
import_code = '%s(%s)' % (helper_func, import_code) import_code = '%s(%s)' % (helper_func, import_code)
......
...@@ -1675,11 +1675,6 @@ def p_import_statement(s): ...@@ -1675,11 +1675,6 @@ def p_import_statement(s):
as_name=as_name, as_name=as_name,
is_absolute=is_absolute) is_absolute=is_absolute)
else: else:
if as_name and "." in dotted_name:
name_list = ExprNodes.ListNode(pos, args=[
ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))])
else:
name_list = None
stat = Nodes.SingleAssignmentNode( stat = Nodes.SingleAssignmentNode(
pos, pos,
lhs=ExprNodes.NameNode(pos, name=as_name or target_name), lhs=ExprNodes.NameNode(pos, name=as_name or target_name),
...@@ -1687,7 +1682,8 @@ def p_import_statement(s): ...@@ -1687,7 +1682,8 @@ def p_import_statement(s):
pos, pos,
module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name), module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name),
level=0 if is_absolute else None, level=0 if is_absolute else None,
name_list=name_list)) get_top_level_module='.' in dotted_name and as_name is None,
name_list=None))
stats.append(stat) stats.append(stat)
return Nodes.StatListNode(pos, stats=stats) return Nodes.StatListNode(pos, stats=stats)
......
...@@ -9,6 +9,135 @@ ...@@ -9,6 +9,135 @@
#endif #endif
/////////////// ImportDottedModule.proto ///////////////
static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); /*proto*/
/////////////// ImportDottedModule ///////////////
//@requires: Import
#if PY_MAJOR_VERSION >= 3
static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) {
PyObject *partial_name = NULL, *slice = NULL, *sep = NULL;
if (unlikely(PyErr_Occurred())) {
PyErr_Clear();
}
if (likely(PyTuple_GET_SIZE(parts_tuple) == count)) {
partial_name = name;
} else {
PyObject *sep;
PyObject *slice = PySequence_GetSlice(parts_tuple, 0, count);
if (unlikely(!slice))
goto bad;
sep = PyUnicode_FromStringAndSize(".", 1);
if (unlikely(!sep))
goto bad;
partial_name = PyUnicode_Join(sep, slice);
}
PyErr_Format(
#if PY_MAJOR_VERSION < 3
PyExc_ImportError,
"No module named '%s'", PyString_AS_STRING(partial_name));
#else
#if PY_VERSION_HEX >= 0x030600B1
PyExc_ModuleNotFoundError,
#else
PyExc_ImportError,
#endif
"No module named '%U'", partial_name);
#endif
bad:
Py_XDECREF(sep);
Py_XDECREF(slice);
Py_XDECREF(partial_name);
return NULL;
}
#endif
static PyObject *__Pyx__ImportDottedModule(PyObject *name, CYTHON_UNUSED PyObject *parts_tuple) {
#if PY_MAJOR_VERSION < 3
PyObject *module, *from_list = NULL, *star = PYIDENT("*");
from_list = PyList_New(1);
if (unlikely(!from_list))
return NULL;
Py_INCREF(star);
PyList_SET_ITEM(from_list, 0, star);
module = __Pyx_Import(name, from_list, 0);
Py_DECREF(from_list);
return module;
#else
Py_ssize_t i, nparts;
PyObject *module = __Pyx_Import(name, NULL, 0);
if (!parts_tuple || unlikely(!module))
return module;
nparts = PyTuple_GET_SIZE(parts_tuple);
for (i=1; i < nparts && module; i++) {
PyObject *part, *submodule;
#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
part = PyTuple_GET_ITEM(parts_tuple, i);
#else
part = PySequence_ITEM(parts_tuple, i);
#endif
submodule = __Pyx_PyObject_GetAttrStrNoError(module, part);
#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
Py_DECREF(part);
#endif
Py_DECREF(module);
module = submodule;
}
if (likely(module))
return module;
return __Pyx__ImportDottedModule_Error(name, parts_tuple, i);
#endif
}
static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) {
#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030400B1
PyObject *module;
#if PY_VERSION_HEX < 0x030700A1
PyObject *modules = PyImport_GetModuleDict();
if (unlikely(!modules))
return NULL;
module = __Pyx_PyDict_GetItemStr(modules, name);
Py_XINCREF(module);
#else
module = PyImport_GetModule(name);
#endif
if (likely(module)) {
// TODO: CPython guards against thread-concurrent imports in importlib,
// but importing and calling into importlib just for that rare case would
// be quite costly, so we skip it for now.
return module;
// -- Code copied from Py3.8-pre:
// PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, PYIDENT("__spec__"));
// if (likely(spec)) {
// PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, PYIDENT("_initializing"));
// if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) {
// Py_DECREF(spec);
// spec = NULL;
// }
// Py_XDECREF(unsafe);
// }
// if (likely(!spec)) {
// PyErr_Clear();
// return module;
// }
// Py_DECREF(spec);
// // Need to let PyImport_ImportModuleLevelObject() handle the locking.
// Py_DECREF(module);
} else if (PyErr_Occurred()) {
PyErr_Clear();
}
#endif
return __Pyx__ImportDottedModule(name, parts_tuple);
}
/////////////// Import.proto /////////////// /////////////// Import.proto ///////////////
static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /*proto*/ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /*proto*/
...@@ -18,30 +147,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); / ...@@ -18,30 +147,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /
//@substitute: naming //@substitute: naming
static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) {
PyObject *empty_list = 0;
PyObject *module = 0; PyObject *module = 0;
PyObject *global_dict = 0;
PyObject *empty_dict = 0; PyObject *empty_dict = 0;
PyObject *list; PyObject *empty_list = 0;
#if PY_MAJOR_VERSION < 3 #if PY_MAJOR_VERSION < 3
PyObject *py_import; PyObject *py_import;
py_import = __Pyx_PyObject_GetAttrStr($builtins_cname, PYIDENT("__import__")); py_import = __Pyx_PyObject_GetAttrStr($builtins_cname, PYIDENT("__import__"));
if (!py_import) if (unlikely(!py_import))
goto bad; goto bad;
#endif if (!from_list) {
if (from_list)
list = from_list;
else {
empty_list = PyList_New(0); empty_list = PyList_New(0);
if (!empty_list) if (unlikely(!empty_list))
goto bad; goto bad;
list = empty_list; from_list = empty_list;
} }
global_dict = PyModule_GetDict($module_cname); #endif
if (!global_dict)
goto bad;
empty_dict = PyDict_New(); empty_dict = PyDict_New();
if (!empty_dict) if (unlikely(!empty_dict))
goto bad; goto bad;
{ {
#if PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3
...@@ -49,9 +171,9 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { ...@@ -49,9 +171,9 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) {
if (strchr(__Pyx_MODULE_NAME, '.')) { if (strchr(__Pyx_MODULE_NAME, '.')) {
/* try package relative import first */ /* try package relative import first */
module = PyImport_ImportModuleLevelObject( module = PyImport_ImportModuleLevelObject(
name, global_dict, empty_dict, list, 1); name, $moddict_cname, empty_dict, from_list, 1);
if (!module) { if (unlikely(!module)) {
if (!PyErr_ExceptionMatches(PyExc_ImportError)) if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError)))
goto bad; goto bad;
PyErr_Clear(); PyErr_Clear();
} }
...@@ -62,23 +184,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { ...@@ -62,23 +184,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) {
if (!module) { if (!module) {
#if PY_MAJOR_VERSION < 3 #if PY_MAJOR_VERSION < 3
PyObject *py_level = PyInt_FromLong(level); PyObject *py_level = PyInt_FromLong(level);
if (!py_level) if (unlikely(!py_level))
goto bad; goto bad;
module = PyObject_CallFunctionObjArgs(py_import, module = PyObject_CallFunctionObjArgs(py_import,
name, global_dict, empty_dict, list, py_level, (PyObject *)NULL); name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL);
Py_DECREF(py_level); Py_DECREF(py_level);
#else #else
module = PyImport_ImportModuleLevelObject( module = PyImport_ImportModuleLevelObject(
name, global_dict, empty_dict, list, level); name, $moddict_cname, empty_dict, from_list, level);
#endif #endif
} }
} }
bad: bad:
Py_XDECREF(empty_dict);
Py_XDECREF(empty_list);
#if PY_MAJOR_VERSION < 3 #if PY_MAJOR_VERSION < 3
Py_XDECREF(py_import); Py_XDECREF(py_import);
#endif #endif
Py_XDECREF(empty_list);
Py_XDECREF(empty_dict);
return module; return module;
} }
......
# mode: run
# tag: all_language_levels
__doc__ = u""" __doc__ = u"""
>>> try: sys
... except NameError: pass
... else: print("sys was defined!")
>>> try: distutils
... except NameError: pass
... else: print("distutils was defined!")
>>> import sys as sous >>> import sys as sous
>>> import distutils.core as corey >>> import distutils.core as corey
>>> from copy import deepcopy as copey >>> from copy import deepcopy as copey
......
...@@ -17,8 +17,8 @@ setup( ...@@ -17,8 +17,8 @@ setup(
import sys import sys
import a import a
assert a in sys.modules.values(), list(sys.modules) assert a in sys.modules.values(), sorted(sys.modules)
assert sys.modules['a'] is a, list(sys.modules) assert sys.modules['a'] is a, sorted(sys.modules)
from atest.package import module from atest.package import module
...@@ -33,8 +33,8 @@ assert 'atest.package.module' in sys.modules ...@@ -33,8 +33,8 @@ assert 'atest.package.module' in sys.modules
import a import a
import atest.package.module as module import atest.package.module as module
assert module in sys.modules.values(), list(sys.modules) assert module in sys.modules.values(), sorted(sys.modules)
assert sys.modules['atest.package.module'] is module, list(sys.modules) assert sys.modules['atest.package.module'] is module, sorted(sys.modules)
if sys.version_info >= (3, 5): if sys.version_info >= (3, 5):
from . import pymodule from . import pymodule
......
...@@ -26,7 +26,8 @@ try: ...@@ -26,7 +26,8 @@ try:
except ImportError: except ImportError:
pass pass
else: else:
assert False, "absolute import succeeded" import sys
assert False, "absolute import succeeded: %s" % sorted(sys.modules)
import relimport.a import relimport.a
import relimport.bmod import relimport.bmod
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment