Issue #13959: Rename imp to _imp and add Lib/imp.py and begin

rewriting functionality in pure Python.

To start, imp.new_module() has been rewritten in pure Python, put into
importlib (privately) and then publicly exposed in imp.

Lib/imp.py

+"""This module provides the components needed to build your own __import__
+function.  Undocumented functions are obsolete.
+
+In most cases it is preferred you consider using the importlib module's
+functionality over this module.
+
+"""
+# (Probably) need to stay in _imp
+from _imp import (lock_held, acquire_lock, release_lock, reload,
+                  get_frozen_object, is_frozen_package, init_builtin,
+                  init_frozen, is_builtin, is_frozen, _fix_co_filename)
+# Can (probably) move to importlib
+from _imp import (get_magic, get_tag, get_suffixes, cache_from_source,
+                  source_from_cache)
+# Should be re-implemented here (and mostly deprecated)
+from _imp import (find_module, load_module, load_compiled, load_dynamic,
+                  load_package, load_source, NullImporter,
+                  SEARCH_ERROR, PY_SOURCE, PY_COMPILED, C_EXTENSION,
+                  PY_RESOURCE, PKG_DIRECTORY, C_BUILTIN, PY_FROZEN,
+                  PY_CODERESOURCE, IMP_HOOK)
+
+from importlib._bootstrap import _new_module as new_module

Lib/importlib/_bootstrap.py

@@ -7,7 +7,7 @@ work. One should use importlib as the public-facing version of this module.
 
 """
 
-# Injected modules are '_warnings', 'imp', 'sys', 'marshal', '_io',
+# Injected modules are '_warnings', '_imp', 'sys', 'marshal', '_io',
 # and '_os' (a.k.a. 'posix', 'nt' or 'os2').
 # Injected attribute is path_sep.
 # Most injection is handled by _setup().
@@ -158,6 +158,16 @@ def _wrap(new, old):
 
 code_type = type(_wrap.__code__)
 
+
+def _new_module(name):
+    """Create a new module.
+
+    The module is not entered into sys.modules.
+
+    """
+    return type(sys)(name)
+
+
 # Finder/loader utility code ##################################################
 
 def verbose_message(message, *args):
@@ -212,7 +222,7 @@ def module_for_loader(fxn):
             # This must be done before open() is called as the 'io' module
             # implicitly imports 'locale' and would otherwise trigger an
             # infinite loop.
-            module = imp.new_module(fullname)
+            module = _new_module(fullname)
             sys.modules[fullname] = module
         try:
             return fxn(self, module, *args, **kwargs)
@@ -254,7 +264,7 @@ def _requires_builtin(fxn):
 def _requires_frozen(fxn):
     """Decorator to verify the named module is frozen."""
     def _requires_frozen_wrapper(self, fullname):
-        if not imp.is_frozen(fullname):
+        if not _imp.is_frozen(fullname):
             raise ImportError("{0} is not a frozen module".format(fullname),
                               name=fullname)
         return fxn(self, fullname)
@@ -264,7 +274,7 @@ def _requires_frozen(fxn):
 
 def _suffix_list(suffix_type):
     """Return a list of file suffixes based on the imp file type."""
-    return [suffix[0] for suffix in imp.get_suffixes()
+    return [suffix[0] for suffix in _imp.get_suffixes()
             if suffix[2] == suffix_type]
 
 
@@ -288,7 +298,7 @@ class BuiltinImporter:
         """
         if path is not None:
             return None
-        return cls if imp.is_builtin(fullname) else None
+        return cls if _imp.is_builtin(fullname) else None
 
     @classmethod
     @set_package
@@ -298,7 +308,7 @@ class BuiltinImporter:
         """Load a built-in module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.init_builtin(fullname)
+            return _imp.init_builtin(fullname)
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -335,7 +345,7 @@ class FrozenImporter:
     @classmethod
     def find_module(cls, fullname, path=None):
         """Find a frozen module."""
-        return cls if imp.is_frozen(fullname) else None
+        return cls if _imp.is_frozen(fullname) else None
 
     @classmethod
     @set_package
@@ -345,7 +355,7 @@ class FrozenImporter:
         """Load a frozen module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.init_frozen(fullname)
+            return _imp.init_frozen(fullname)
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -355,7 +365,7 @@ class FrozenImporter:
     @_requires_frozen
     def get_code(cls, fullname):
         """Return the code object for the frozen module."""
-        return imp.get_frozen_object(fullname)
+        return _imp.get_frozen_object(fullname)
 
     @classmethod
     @_requires_frozen
@@ -367,7 +377,7 @@ class FrozenImporter:
     @_requires_frozen
     def is_package(cls, fullname):
         """Return if the frozen module is a package."""
-        return imp.is_frozen_package(fullname)
+        return _imp.is_frozen_package(fullname)
 
 
 class _LoaderBasics:
@@ -391,7 +401,7 @@ class _LoaderBasics:
         magic = data[:4]
         raw_timestamp = data[4:8]
         raw_size = data[8:12]
-        if len(magic) != 4 or magic != imp.get_magic():
+        if len(magic) != 4 or magic != _imp.get_magic():
             raise ImportError("bad magic number in {}".format(fullname),
                               name=fullname, path=bytecode_path)
         elif len(raw_timestamp) != 4:
@@ -434,7 +444,7 @@ class _LoaderBasics:
         code_object = self.get_code(name)
         module.__file__ = self.get_filename(name)
         if not sourceless:
-            module.__cached__ = imp.cache_from_source(module.__file__)
+            module.__cached__ = _imp.cache_from_source(module.__file__)
         else:
             module.__cached__ = module.__file__
         module.__package__ = name
@@ -497,7 +507,7 @@ class SourceLoader(_LoaderBasics):
 
         """
         source_path = self.get_filename(fullname)
-        bytecode_path = imp.cache_from_source(source_path)
+        bytecode_path = _imp.cache_from_source(source_path)
         source_mtime = None
         if bytecode_path is not None:
             try:
@@ -522,7 +532,7 @@ class SourceLoader(_LoaderBasics):
                                         source_path)
                         found = marshal.loads(bytes_data)
                         if isinstance(found, code_type):
-                            imp._fix_co_filename(found, source_path)
+                            _imp._fix_co_filename(found, source_path)
                             verbose_message('code object from {}',
                                             bytecode_path)
                             return found
@@ -539,7 +549,7 @@ class SourceLoader(_LoaderBasics):
             # If e.g. Jython ever implements imp.cache_from_source to have
             # their own cached file format, this block of code will most likely
             # throw an exception.
-            data = bytearray(imp.get_magic())
+            data = bytearray(_imp.get_magic())
             data.extend(_w_long(source_mtime))
             data.extend(_w_long(len(source_bytes)))
             data.extend(marshal.dumps(code_object))
@@ -664,7 +674,7 @@ class _ExtensionFileLoader:
         """Load an extension module."""
         is_reload = fullname in sys.modules
         try:
-            module = imp.load_dynamic(fullname, self._path)
+            module = _imp.load_dynamic(fullname, self._path)
             verbose_message('extension module loaded from {!r}', self._path)
             return module
         except:
@@ -841,7 +851,7 @@ class _SourceFinderDetails:
     supports_packages = True
 
     def __init__(self):
-        self.suffixes = _suffix_list(imp.PY_SOURCE)
+        self.suffixes = _suffix_list(_imp.PY_SOURCE)
 
 class _SourcelessFinderDetails:
 
@@ -849,7 +859,7 @@ class _SourcelessFinderDetails:
     supports_packages = True
 
     def __init__(self):
-        self.suffixes = _suffix_list(imp.PY_COMPILED)
+        self.suffixes = _suffix_list(_imp.PY_COMPILED)
 
 
 class _ExtensionFinderDetails:
@@ -858,7 +868,7 @@ class _ExtensionFinderDetails:
     supports_packages = False
 
     def __init__(self):
-        self.suffixes = _suffix_list(imp.C_EXTENSION)
+        self.suffixes = _suffix_list(_imp.C_EXTENSION)
 
 
 # Import itself ###############################################################
@@ -886,7 +896,7 @@ class _DefaultPathFinder(PathFinder):
         try:
             return super()._path_hooks(path)
         except ImportError:
-            implicit_hooks = [_DEFAULT_PATH_HOOK, imp.NullImporter]
+            implicit_hooks = [_DEFAULT_PATH_HOOK, _imp.NullImporter]
             return super()._path_hooks(path, implicit_hooks)
 
     @classmethod
@@ -902,11 +912,11 @@ class _ImportLockContext:
 
     def __enter__(self):
         """Acquire the import lock."""
-        imp.acquire_lock()
+        _imp.acquire_lock()
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
         """Release the import lock regardless of any raised exceptions."""
-        imp.release_lock()
+        _imp.release_lock()
 
 
 def _resolve_name(name, package, level):
@@ -1092,19 +1102,19 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0):
         return _handle_fromlist(module, fromlist, _gcd_import)
 
 
-def _setup(sys_module, imp_module):
+def _setup(sys_module, _imp_module):
     """Setup importlib by importing needed built-in modules and injecting them
     into the global namespace.
 
-    As sys is needed for sys.modules access and imp is needed to load built-in
+    As sys is needed for sys.modules access and _imp is needed to load built-in
     modules, those two modules must be explicitly passed in.
 
     """
-    global imp, sys
-    imp = imp_module
+    global _imp, sys
+    _imp = _imp_module
     sys = sys_module
 
-    for module in (imp, sys):
+    for module in (_imp, sys):
         if not hasattr(module, '__loader__'):
             module.__loader__ = BuiltinImporter
 
@@ -1137,14 +1147,14 @@ def _setup(sys_module, imp_module):
     setattr(self_module, '_relax_case', _make_relax_case())
 
 
-def _install(sys_module, imp_module):
+def _install(sys_module, _imp_module):
     """Install importlib as the implementation of import.
 
-    It is assumed that imp and sys have been imported and injected into the
+    It is assumed that _imp and sys have been imported and injected into the
     global namespace for the module prior to calling this function.
 
     """
-    _setup(sys_module, imp_module)
+    _setup(sys_module, _imp_module)
     orig_import = builtins.__import__
     builtins.__import__ = __import__
     builtins.__original_import__ = orig_import

Misc/NEWS

@@ -32,6 +32,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #13959: Add imp.py and rename the built-in module to _imp, allowing for
+  re-implementing parts of the module in pure Python.
+
 - Issue #13496: Fix potential overflow in bisect.bisect algorithm when applied
   to a collection of size > sys.maxsize / 2.
 

Modules/config.c.in

@@ -39,7 +39,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"marshal", PyMarshal_Init},
 
     /* This lives in import.c */
-    {"imp", PyInit_imp},
+    {"_imp", PyInit_imp},
 
     /* This lives in Python/Python-ast.c */
     {"_ast", PyInit__ast},

PC/config.c

@@ -143,7 +143,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"marshal", PyMarshal_Init},
 
     /* This lives it with import.c */
-    {"imp", PyInit_imp},
+    {"_imp", PyInit_imp},
 
     /* These entries are here for sys.builtin_module_names */
     {"__main__", NULL},

PC/os2emx/config.c

@@ -150,7 +150,7 @@ struct _inittab _PyImport_Inittab[] = {
     {"marshal", PyMarshal_Init},
 
     /* This lives it with import.c */
-    {"imp", initimp},
+    {"_imp", initimp},
 
     /* These entries are here for sys.builtin_module_names */
     {"__main__", NULL},

PC/os2vacpp/config.c

@@ -88,7 +88,7 @@ struct _inittab _PyImport_Inittab[] = {
         {"marshal", PyMarshal_Init},
 
         /* This lives it with import.c */
-        {"imp", initimp},
+        {"_imp", initimp},
 
         /* These entries are here for sys.builtin_module_names */
         {"__main__", NULL},

Python/import.c

@@ -3684,15 +3684,6 @@ imp_load_package(PyObject *self, PyObject *args)
     return ret;
 }
 
-static PyObject *
-imp_new_module(PyObject *self, PyObject *args)
-{
-    PyObject *name;
-    if (!PyArg_ParseTuple(args, "U:new_module", &name))
-        return NULL;
-    return PyModule_NewObject(name);
-}
-
 static PyObject *
 imp_reload(PyObject *self, PyObject *v)
 {
@@ -3781,8 +3772,7 @@ does not conform to PEP 3147 format, ValueError will be raised.");
 /* Doc strings */
 
 PyDoc_STRVAR(doc_imp,
-"This module provides the components needed to build your own\n\
-__import__ function.  Undocumented functions are obsolete.");
+"(Extremely) low-level import machinery bits as used by importlib and imp.");
 
 PyDoc_STRVAR(doc_find_module,
 "find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\
@@ -3809,11 +3799,6 @@ PyDoc_STRVAR(doc_get_suffixes,
 Return a list of (suffix, mode, type) tuples describing the files\n\
 that find_module() looks for.");
 
-PyDoc_STRVAR(doc_new_module,
-"new_module(name) -> module\n\
-Create a new module.  Do not enter it in sys.modules.\n\
-The module name must include the full package name, if any.");
-
 PyDoc_STRVAR(doc_lock_held,
 "lock_held() -> boolean\n\
 Return True if the import lock is currently held, else False.\n\
@@ -3837,7 +3822,6 @@ static PyMethodDef imp_methods[] = {
     {"get_tag",          imp_get_tag,      METH_NOARGS,  doc_get_tag},
     {"get_suffixes", imp_get_suffixes, METH_NOARGS,  doc_get_suffixes},
     {"load_module",      imp_load_module,  METH_VARARGS, doc_load_module},
-    {"new_module",       imp_new_module,   METH_VARARGS, doc_new_module},
     {"lock_held",        imp_lock_held,    METH_NOARGS,  doc_lock_held},
     {"acquire_lock", imp_acquire_lock, METH_NOARGS,  doc_acquire_lock},
     {"release_lock", imp_release_lock, METH_NOARGS,  doc_release_lock},
@@ -4005,7 +3989,7 @@ PyTypeObject PyNullImporter_Type = {
 
 static struct PyModuleDef impmodule = {
     PyModuleDef_HEAD_INIT,
-    "imp",
+    "_imp",
     doc_imp,
     0,
     imp_methods,

Python/pythonrun.c

@@ -225,8 +225,8 @@ import_init(PyInterpreterState *interp, PyObject *sysmod)
     if (Py_VerboseFlag) {
         PySys_FormatStderr("import sys # builtin\n");
     }
-    if (PyDict_SetItemString(sys_modules, "imp", impmod) < 0) {
-        Py_FatalError("Py_Initialize: can't save imp to sys.modules");
+    if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) {
+        Py_FatalError("Py_Initialize: can't save _imp to sys.modules");
     }
 
     value = PyObject_CallMethod(importlib, "_setup", "OO", sysmod, impmod);