Issue #2377: Make importlib the implementation of __import__().

importlib._bootstrap is now frozen into Python/importlib.h and stored
as _frozen_importlib in sys.modules. Py_Initialize() loads the frozen
code along with sys and imp and then uses _frozen_importlib._install()
to set builtins.__import__() w/ _frozen_importlib.__import__().

Include/abstract.h

@@ -339,6 +339,10 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 
      PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(PyObject *o,
                                                        PyObject *method, ...);
+     PyAPI_FUNC(PyObject *) _PyObject_CallMethodObjIdArgs(PyObject *o,
+                                               struct _Py_Identifier *method,
+                                               ...);
+
 
        /*
      Call the method named m of object o with a variable number of

Include/dictobject.h

@@ -109,6 +109,8 @@ PyAPI_DATA(PyTypeObject) PyDictValues_Type;
 PyAPI_FUNC(PyObject *) PyDict_New(void);
 PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
+PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
+                                                  struct _Py_Identifier *key);
 PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);

Include/import.h

@@ -7,6 +7,9 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(void) _PyImportZip_Init(void);
+
+PyMODINIT_FUNC PyInit_imp(void);
 PyAPI_FUNC(long) PyImport_GetMagicNumber(void);
 PyAPI_FUNC(const char *) PyImport_GetMagicTag(void);
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(

Include/pystate.h

@@ -25,6 +25,7 @@ typedef struct _is {
     PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;
+    PyObject *importlib;
     PyObject *modules_reloading;
 
     PyObject *codec_search_path;
@@ -33,6 +34,7 @@ typedef struct _is {
     int codecs_initialized;
     int fscodec_initialized;
 
+
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif

Include/pythonrun.h

@@ -188,7 +188,7 @@ PyAPI_FUNC(const char *) _Py_hgversion(void);
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
 PyAPI_FUNC(PyObject *) _PySys_Init(void);
 PyAPI_FUNC(void) _PyImport_Init(void);
-PyAPI_FUNC(void) _PyExc_Init(void);
+PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
 PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(void) _PyFloat_Init(void);

Lib/importlib/_bootstrap.py

@@ -160,6 +160,13 @@ code_type = type(_wrap.__code__)
 
 # Finder/loader utility code ##################################################
 
+def verbose_message(message, *args):
+    """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
+    if sys.flags.verbose:
+        if not message.startswith('#') and not message.startswith('import '):
+            message = '# ' + message
+        print(message.format(*args), file=sys.stderr)
+
 
 def set_package(fxn):
     """Set __package__ on the returned module."""
@@ -388,9 +395,13 @@ class _LoaderBasics:
             raise ImportError("bad magic number in {}".format(fullname),
                               name=fullname, path=bytecode_path)
         elif len(raw_timestamp) != 4:
-            raise EOFError("bad timestamp in {}".format(fullname))
+            message = 'bad timestamp in {}'.format(fullname)
+            verbose_message(message)
+            raise EOFError(message)
         elif len(raw_size) != 4:
-            raise EOFError("bad size in {}".format(fullname))
+            message = 'bad size in {}'.format(fullname)
+            verbose_message(message)
+            raise EOFError(message)
         if source_stats is not None:
             try:
                 source_mtime = int(source_stats['mtime'])
@@ -398,9 +409,10 @@ class _LoaderBasics:
                 pass
             else:
                 if _r_long(raw_timestamp) != source_mtime:
-                    raise ImportError(
-                        "bytecode is stale for {}".format(fullname),
-                        name=fullname, path=bytecode_path)
+                    message = 'bytecode is stale for {}'.format(fullname)
+                    verbose_message(message)
+                    raise ImportError(message, name=fullname,
+                                      path=bytecode_path)
             try:
                 source_size = source_stats['size'] & 0xFFFFFFFF
             except KeyError:
@@ -506,9 +518,13 @@ class SourceLoader(_LoaderBasics):
                     except (ImportError, EOFError):
                         pass
                     else:
+                        verbose_message('{} matches {}', bytecode_path,
+                                        source_path)
                         found = marshal.loads(bytes_data)
                         if isinstance(found, code_type):
                             imp._fix_co_filename(found, source_path)
+                            verbose_message('code object from {}',
+                                            bytecode_path)
                             return found
                         else:
                             msg = "Non-code object in {}"
@@ -517,6 +533,7 @@ class SourceLoader(_LoaderBasics):
         source_bytes = self.get_data(source_path)
         code_object = compile(source_bytes, source_path, 'exec',
                                 dont_inherit=True)
+        verbose_message('code object from {}', source_path)
         if (not sys.dont_write_bytecode and bytecode_path is not None and
             source_mtime is not None):
             # If e.g. Jython ever implements imp.cache_from_source to have
@@ -528,6 +545,7 @@ class SourceLoader(_LoaderBasics):
             data.extend(marshal.dumps(code_object))
             try:
                 self.set_data(bytecode_path, data)
+                verbose_message('wrote {!r}', bytecode_path)
             except NotImplementedError:
                 pass
         return code_object
@@ -596,6 +614,7 @@ class _SourceFileLoader(_FileLoader, SourceLoader):
                 return
         try:
             _write_atomic(path, data)
+            verbose_message('created {!r}', path)
         except (PermissionError, FileExistsError):
             # Don't worry if you can't write bytecode or someone is writing
             # it at the same time.
@@ -615,6 +634,7 @@ class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
         bytes_data = self._bytes_from_bytecode(fullname, data, path, None)
         found = marshal.loads(bytes_data)
         if isinstance(found, code_type):
+            verbose_message('code object from {!r}', path)
             return found
         else:
             raise ImportError("Non-code object in {}".format(path),
@@ -644,7 +664,9 @@ class _ExtensionFileLoader:
         """Load an extension module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.load_dynamic(fullname, self._path)
+            module = imp.load_dynamic(fullname, self._path)
+            verbose_message('extension module loaded from {!r}', self._path)
+            return module
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -953,6 +975,7 @@ def _find_and_load(name, import_):
     elif name not in sys.modules:
         # The parent import may have already imported this module.
         loader.load_module(name)
+        verbose_message('import {!r} # {!r}', name, loader)
     # Backwards-compatibility; be nicer to skip the dict lookup.
     module = sys.modules[name]
     if parent:

Lib/importlib/test/import_/test_path.py

@@ -87,7 +87,7 @@ class FinderTests(unittest.TestCase):
 
 class DefaultPathFinderTests(unittest.TestCase):
 
-    """Test importlib._bootstrap._DefaultPathFinder."""
+    """Test _bootstrap._DefaultPathFinder."""
 
     def test_implicit_hooks(self):
         # Test that the implicit path hooks are used.

Lib/importlib/test/import_/util.py

 import functools
 import importlib
-import importlib._bootstrap
 import unittest
 
 

Lib/importlib/test/regrtest.py

@@ -13,16 +13,4 @@ from test import regrtest
 if __name__ == '__main__':
     __builtins__.__import__ = importlib.__import__
 
-    exclude = ['--exclude',
-                'test_frozen', # Does not expect __loader__ attribute
-                'test_pkg',  # Does not expect __loader__ attribute
-                'test_pydoc', # Does not expect __loader__ attribute
-              ]
-
-    # Switching on --exclude implies running all test but the ones listed, so
-    # only use it when one is not running an explicit test
-    if len(sys.argv) == 1:
-        # No programmatic way to specify tests to exclude
-        sys.argv.extend(exclude)
-
     regrtest.main(quiet=True, verbose2=True)

Lib/importlib/test/source/test_file_loader.py

+from ... import _bootstrap
 import importlib
-from importlib import _bootstrap
 from .. import abc
 from .. import util
 from . import util as source_util

Lib/importlib/test/source/test_finder.py

-from importlib import _bootstrap
 from .. import abc
 from . import util as source_util
-from test.support import make_legacy_pyc
-import os
+
+from importlib import _bootstrap
 import errno
+import os
 import py_compile
+from test.support import make_legacy_pyc
 import unittest
 import warnings
 

Lib/importlib/test/source/test_path_hook.py

-from importlib import _bootstrap
 from . import util as source_util
+
+from importlib import _bootstrap
 import unittest
 
 

Lib/importlib/test/source/test_source_encoding.py

-from importlib import _bootstrap
 from . import util as source_util
 
+from importlib import _bootstrap
 import codecs
 import re
 import sys
@@ -36,7 +36,7 @@ class EncodingTest(unittest.TestCase):
             with open(mapping[self.module_name], 'wb') as file:
                 file.write(source)
             loader = _bootstrap._SourceFileLoader(self.module_name,
-                                       mapping[self.module_name])
+                                                  mapping[self.module_name])
             return loader.load_module(self.module_name)
 
     def create_source(self, encoding):

Lib/importlib/test/util.py

@@ -35,7 +35,7 @@ def uncache(*names):
     for name in names:
         if name in ('sys', 'marshal', 'imp'):
             raise ValueError(
-                "cannot uncache {0} as it will break _importlib".format(name))
+                "cannot uncache {0}".format(name))
         try:
             del sys.modules[name]
         except KeyError:

Lib/os.py

@@ -42,6 +42,8 @@ def _get_exports_list(module):
     except AttributeError:
         return [n for n in dir(module) if n[0] != '_']
 
+# Any new dependencies of the os module and/or changes in path separator
+# requires updating importlib as well.
 if 'posix' in _names:
     name = 'posix'
     linesep = '\n'

Lib/pydoc.py

@@ -299,9 +299,8 @@ def safeimport(path, forceload=0, cache={}):
         elif exc is SyntaxError:
             # A SyntaxError occurred before we could execute the module.
             raise ErrorDuringImport(value.filename, info)
-        elif exc is ImportError and extract_tb(tb)[-1][2]=='safeimport':
-            # The import error occurred directly in this function,
-            # which means there is no such module in the path.
+        elif exc is ImportError and value.name == path:
+            # No such module in the path.
             return None
         else:
             # Some other error occurred during the importing process.

Lib/site.py

@@ -81,7 +81,8 @@ def makepath(*paths):
 def abs_paths():
     """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in set(sys.modules.values()):
-        if hasattr(m, '__loader__'):
+        if (getattr(getattr(m, '__loader__', None), '__module__', None) !=
+                '_frozen_importlib'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
             m.__file__ = os.path.abspath(m.__file__)

Lib/test/test_frozen.py

@@ -5,6 +5,12 @@ import unittest
 import sys
 
 class FrozenTests(unittest.TestCase):
+
+    module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
+                              '__file__', '__loader__', '__name__',
+                              '__package__'])
+    package_attrs = frozenset(list(module_attrs) + ['__path__'])
+
     def test_frozen(self):
         with captured_stdout() as stdout:
             try:
@@ -12,7 +18,9 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __hello__ failed:" + str(x))
             self.assertEqual(__hello__.initialized, True)
-            self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
+            expect = set(self.module_attrs)
+            expect.add('initialized')
+            self.assertEqual(set(dir(__hello__)), expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         with captured_stdout() as stdout:
@@ -21,10 +29,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__ failed:" + str(x))
             self.assertEqual(__phello__.initialized, True)
+            expect = set(self.package_attrs)
+            expect.add('initialized')
             if not "__phello__.spam" in sys.modules:
-                self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+                self.assertEqual(set(dir(__phello__)), expect)
             else:
-                self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
+                expect.add('spam')
+                self.assertEqual(set(dir(__phello__)), expect)
             self.assertEqual(__phello__.__path__, [__phello__.__name__])
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
@@ -34,8 +45,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__.spam failed:" + str(x))
             self.assertEqual(__phello__.spam.initialized, True)
-            self.assertEqual(len(dir(__phello__.spam)), 7)
-            self.assertEqual(len(dir(__phello__)), 9)
+            spam_expect = set(self.module_attrs)
+            spam_expect.add('initialized')
+            self.assertEqual(set(dir(__phello__.spam)), spam_expect)
+            phello_expect = set(self.package_attrs)
+            phello_expect.add('initialized')
+            phello_expect.add('spam')
+            self.assertEqual(set(dir(__phello__)), phello_expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         try:

Lib/test/test_import.py

@@ -73,6 +73,7 @@ class ImportTests(unittest.TestCase):
 
             if TESTFN in sys.modules:
                 del sys.modules[TESTFN]
+            importlib.invalidate_caches()
             try:
                 try:
                     mod = __import__(TESTFN)
@@ -402,6 +403,7 @@ func_filename = func.__code__.co_filename
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
         pyc_file = make_legacy_pyc(self.file_name)
+        importlib.invalidate_caches()
         mod = self.import_module()
         self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
@@ -509,7 +511,7 @@ class RelativeImportTests(unittest.TestCase):
 
         # Check relative import fails with package set to a non-string
         ns = dict(__package__=object())
-        self.assertRaises(ValueError, check_relative)
+        self.assertRaises(TypeError, check_relative)
 
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
@@ -644,6 +646,7 @@ class PycacheTests(unittest.TestCase):
             pass
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
@@ -666,9 +669,11 @@ class PycacheTests(unittest.TestCase):
             pass
         with open(os.path.join('pep3147', 'foo.py'), 'w'):
             pass
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))

Lib/test/test_pkg.py

@@ -196,14 +196,15 @@ class TestPkg(unittest.TestCase):
 
         import t5
         self.assertEqual(fixdir(dir(t5)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__', 'foo', 'string', 't5'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__', 'foo',
+                          'string', 't5'])
         self.assertEqual(fixdir(dir(t5.foo)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', 'string'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', 'string'])
         self.assertEqual(fixdir(dir(t5.string)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', 'spam'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', 'spam'])
 
     def test_6(self):
         hier = [
@@ -219,14 +220,14 @@ class TestPkg(unittest.TestCase):
         import t6
         self.assertEqual(fixdir(dir(t6)),
                          ['__all__', '__cached__', '__doc__', '__file__',
-                          '__name__', '__package__', '__path__'])
+                          '__loader__', '__name__', '__package__', '__path__'])
         s = """
             import t6
             from t6 import *
             self.assertEqual(fixdir(dir(t6)),
                              ['__all__', '__cached__', '__doc__', '__file__',
-                              '__name__', '__package__', '__path__',
-                              'eggs', 'ham', 'spam'])
+                              '__loader__', '__name__', '__package__',
+                              '__path__', 'eggs', 'ham', 'spam'])
             self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
             """
         self.run_code(s)
@@ -252,19 +253,19 @@ class TestPkg(unittest.TestCase):
         t7, sub, subsub = None, None, None
         import t7 as tas
         self.assertEqual(fixdir(dir(tas)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         from t7 import sub as subpar
         self.assertEqual(fixdir(dir(subpar)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         from t7.sub import subsub as subsubsub
         self.assertEqual(fixdir(dir(subsubsub)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                         '__package__', '__path__', 'spam'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__', 'spam'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         self.assertFalse(subsub)

Lib/test/test_pydoc.py

@@ -383,11 +383,10 @@ class PydocImportTest(unittest.TestCase):
         modname = 'testmod_xyzzy'
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
-            ('test.i_am_not_here_either', 'i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname),
-             'i_am_not_here.{}'.format(modname)),
-            ('test.{}'.format(modname), modname),
+            ('test.i_am_not_here_either', 'test.i_am_not_here_either'),
+            ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
+            ('test.{}'.format(modname), 'test.{}'.format(modname)),
             )
 
         sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py"

Lib/test/test_runpy.py

@@ -5,6 +5,7 @@ import os.path
 import sys
 import re
 import tempfile
+import importlib
 import py_compile
 from test.support import (
     forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -172,11 +173,13 @@ class RunModuleTest(unittest.TestCase):
             self.assertIn("x", d1)
             self.assertEqual(d1["x"], 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name) # Read from bytecode
             self.assertIn("x", d2)
             self.assertEqual(d2["x"], 1)
@@ -196,11 +199,13 @@ class RunModuleTest(unittest.TestCase):
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", pkg_name)
+            importlib.invalidate_caches()
             d2 = run_module(pkg_name) # Read from bytecode
             self.assertIn("x", d2)
             self.assertTrue(d2["x"] == 1)
@@ -250,11 +255,13 @@ from ..uncle.cousin import nephew
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case the loader caches paths
             if verbose: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
             self.assertIn("__package__", d2)
             self.assertTrue(d2["__package__"] == pkg_name)

Lib/test/test_support.py

 #!/usr/bin/env python
 
+import importlib
 import sys
 import os
 import unittest
@@ -59,6 +60,7 @@ class TestSupport(unittest.TestCase):
         with open(mod_filename, 'w') as f:
             print('foo = 1', file=f)
         sys.path.insert(0, os.curdir)
+        importlib.invalidate_caches()
         try:
             mod = __import__(TESTFN)
             self.assertIn(TESTFN, sys.modules)

Lib/test/test_trace.py

@@ -322,7 +322,7 @@ class TestCoverage(unittest.TestCase):
             self._coverage(tracer)
         if os.path.exists(TESTFN):
             files = os.listdir(TESTFN)
-            self.assertEqual(files, [])
+            self.assertEqual(files, ['_importlib.cover'])  # Ignore __import__
 
     def test_issue9936(self):
         tracer = trace.Trace(trace=0, count=1)

Makefile.pre.in

@@ -573,7 +573,12 @@ Modules/Setup: $(srcdir)/Modules/Setup.dist
 
 Modules/_testembed: Modules/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
 	$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Modules/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
+############################################################################
+# Importlib
 
+Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/freeze_importlib.py
+	./$(BUILDPYTHON) $(srcdir)/Python/freeze_importlib.py \
+	    $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
 ############################################################################
 # Special rules for object files
 
@@ -787,6 +792,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/unicodeobject.h \
 		$(srcdir)/Include/warnings.h \
 		$(srcdir)/Include/weakrefobject.h \
+		$(srcdir)/Python/importlib.h \
 		pyconfig.h \
 		$(PARSER_HEADERS)
 

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.3.0 Alpha 3?
 Core and Builtins
 -----------------
 
+- Issue #2377: Make importlib the implementation of __import__().
+
 - Issue #1559549: ImportError now has 'name' and 'path' attributes that are set
   using keyword arguments to its constructor. They are currently not set by
   import as they are meant for use by importlib.

Objects/abstract.c

@@ -2376,6 +2376,35 @@ PyObject_CallMethodObjArgs(PyObject *callable, PyObject *name, ...)
     return tmp;
 }
 
+PyObject *
+_PyObject_CallMethodObjIdArgs(PyObject *callable,
+        struct _Py_Identifier *name, ...)
+{
+    PyObject *args, *tmp;
+    va_list vargs;
+
+    if (callable == NULL || name == NULL)
+        return null_error();
+
+    callable = _PyObject_GetAttrId(callable, name);
+    if (callable == NULL)
+        return NULL;
+
+    /* count the args */
+    va_start(vargs, name);
+    args = objargs_mktuple(vargs);
+    va_end(vargs);
+    if (args == NULL) {
+        Py_DECREF(callable);
+        return NULL;
+    }
+    tmp = PyObject_Call(callable, args, NULL);
+    Py_DECREF(args);
+    Py_DECREF(callable);
+
+    return tmp;
+}
+
 PyObject *
 PyObject_CallFunctionObjArgs(PyObject *callable, ...)
 {

Objects/dictobject.c

@@ -788,6 +788,16 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key)
     return ep->me_value;
 }
 
+PyObject *
+_PyDict_GetItemIdWithError(PyObject *dp, struct _Py_Identifier *key)
+{
+    PyObject *kv;
+    kv = _PyUnicode_FromId(key); /* borrowed */
+    if (kv == NULL)
+        return NULL;
+    return PyDict_GetItemWithError(dp, kv);
+}
+
 static int
 dict_set_item_by_hash_or_entry(register PyObject *op, PyObject *key,
                                Py_hash_t hash, PyDictEntry *ep, PyObject *value)

Objects/exceptions.c

@@ -2344,9 +2344,9 @@ PyObject *PyExc_RecursionErrorInst = NULL;
 #endif /* MS_WINDOWS */
 
 void
-_PyExc_Init(void)
+_PyExc_Init(PyObject *bltinmod)
 {
-    PyObject *bltinmod, *bdict;
+    PyObject *bdict;
 
     PRE_INIT(BaseException)
     PRE_INIT(Exception)
@@ -2414,9 +2414,6 @@ _PyExc_Init(void)
     PRE_INIT(ProcessLookupError);
     PRE_INIT(TimeoutError);
 
-    bltinmod = PyImport_ImportModule("builtins");
-    if (bltinmod == NULL)
-        Py_FatalError("exceptions bootstrapping error.");
     bdict = PyModule_GetDict(bltinmod);
     if (bdict == NULL)
         Py_FatalError("exceptions bootstrapping error.");
@@ -2546,7 +2543,6 @@ _PyExc_Init(void)
             Py_DECREF(args_tuple);
         }
     }
-    Py_DECREF(bltinmod);
 }
 
 void

Python/bltinmodule.c

@@ -187,7 +187,7 @@ builtin___import__(PyObject *self, PyObject *args, PyObject *kwds)
     static char *kwlist[] = {"name", "globals", "locals", "fromlist",
                              "level", 0};
     PyObject *name, *globals = NULL, *locals = NULL, *fromlist = NULL;
-    int level = -1;
+    int level = 0;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|OOOi:__import__",
                     kwlist, &name, &globals, &locals, &fromlist, &level))

Python/dynload_shlib.c

@@ -115,10 +115,6 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname,
     dlopenflags = PyThreadState_GET()->interp->dlopenflags;
 #endif
 
-    if (Py_VerboseFlag)
-        PySys_WriteStderr("dlopen(\"%s\", %x);\n", pathname,
-                          dlopenflags);
-
 #ifdef __VMS
     /* VMS currently don't allow a pathname, use a logical name instead */
     /* Concatenate 'python_module_' and shortname */

Python/freeze_importlib.py

+#! /usr/bin/env python
+"""Freeze importlib for use as the implementation of import."""
+import marshal
+
+
+header = """/* Auto-generated by Python/freeze_importlib.py */"""
+
+
+def main(input_path, output_path):
+    with open(input_path, 'r', encoding='utf-8') as input_file:
+        source = input_file.read()
+
+    code = compile(source, '<frozen importlib._bootstrap>', 'exec')
+
+    lines = [header]
+    lines.append('unsigned char _Py_M__importlib[] = {')
+    data = marshal.dumps(code)
+    # Code from Tools/freeze/makefreeze.py:writecode()
+    for i in range(0, len(data), 16):
+        line = ['    ']
+        for c in data[i:i+16]:
+            line.append('%d,' % c)
+        lines.append(''.join(line))
+    lines.append('};\n')
+    with open(output_path, 'w') as output_file:
+        output_file.write('\n'.join(lines))
+
+
+if __name__ == '__main__':
+    import sys
+
+    args = sys.argv[1:]
+    if len(args) != 2:
+        print('Need to specify input and output file paths', file=sys.stderr)
+        sys.exit(1)
+
+    main(*args)

Python/frozen.c

@@ -2,6 +2,7 @@
 /* Dummy frozen modules initializer */
 
 #include "Python.h"
+#include "importlib.h"
 
 /* In order to test the support for frozen modules, by default we
    define a single frozen module, __hello__.  Loading it will print
@@ -28,6 +29,8 @@ static unsigned char M___hello__[] = {
 #define SIZE (int)sizeof(M___hello__)
 
 static struct _frozen _PyImport_FrozenModules[] = {
+    /* importlib */
+    {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)},
     /* Test module */
     {"__hello__", M___hello__, SIZE},
     /* Test package (negative size indicates package-ness) */

Python/importdl.c

@@ -106,10 +106,6 @@ _PyImport_LoadDynamicModule(PyObject *name, PyObject *path, FILE *fp)
 
     if (_PyImport_FixupExtensionObject(m, name, path) < 0)
         goto error;
-    if (Py_VerboseFlag)
-        PySys_FormatStderr(
-            "import %U # dynamically loaded from %R\n",
-            name, path);
     Py_DECREF(nameascii);
     return m;
 

Python/pystate.c

@@ -79,6 +79,7 @@ PyInterpreterState_New(void)
         interp->codec_error_registry = NULL;
         interp->codecs_initialized = 0;
         interp->fscodec_initialized = 0;
+        interp->importlib = NULL;
 #ifdef HAVE_DLOPEN
 #ifdef RTLD_NOW
         interp->dlopenflags = RTLD_NOW;
@@ -116,6 +117,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
     Py_CLEAR(interp->modules_reloading);
     Py_CLEAR(interp->sysdict);
     Py_CLEAR(interp->builtins);
+    Py_CLEAR(interp->importlib);
 }
 
 

Python/pythonrun.c

@@ -190,6 +190,58 @@ get_locale_encoding(void)
 #endif
 }
 
+static void
+import_init(PyInterpreterState *interp, PyObject *sysmod)
+{
+    PyObject *importlib;
+    PyObject *impmod;
+    PyObject *sys_modules;
+    PyObject *value;
+
+    /* Import _importlib through its frozen version, _frozen_importlib. */
+    /* XXX(bcannon): The file path for _frozen_importlib is completely off
+     */
+    if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) {
+        Py_FatalError("Py_Initialize: can't import _frozen_importlib");
+    }
+    else if (Py_VerboseFlag) {
+        PySys_FormatStderr("import _frozen_importlib # frozen\n");
+    }
+    importlib = PyImport_AddModule("_frozen_importlib");
+    if (importlib == NULL) {
+        Py_FatalError("Py_Initialize: couldn't get _frozen_importlib from "
+                      "sys.modules");
+    }
+    interp->importlib = importlib;
+    Py_INCREF(interp->importlib);
+
+    /* Install _importlib as __import__ */
+    impmod = PyInit_imp();
+    if (impmod == NULL) {
+        Py_FatalError("Py_Initialize: can't import imp");
+    }
+    else if (Py_VerboseFlag) {
+        PySys_FormatStderr("import imp # builtin\n");
+    }
+    sys_modules = PyImport_GetModuleDict();
+    if (Py_VerboseFlag) {
+        PySys_FormatStderr("import sys # builtin\n");
+    }
+    if (PyDict_SetItemString(sys_modules, "imp", impmod) < 0) {
+        Py_FatalError("Py_Initialize: can't save imp to sys.modules");
+    }
+
+    value = PyObject_CallMethod(importlib, "_setup", "OO", sysmod, impmod);
+    if (value == NULL) {
+        PyErr_Print();
+        Py_FatalError("Py_Initialize: importlib install failed");
+    }
+    Py_DECREF(value);
+
+    _PyImportZip_Init();
+}
+
+
 void
 Py_InitializeEx(int install_sigs)
 {
@@ -281,7 +333,7 @@ Py_InitializeEx(int install_sigs)
     Py_INCREF(interp->builtins);
 
     /* initialize builtin exceptions */
-    _PyExc_Init();
+    _PyExc_Init(bimod);
 
     sysmod = _PySys_Init();
     if (sysmod == NULL)
@@ -315,6 +367,8 @@ Py_InitializeEx(int install_sigs)
     /* Initialize _warnings. */
     _PyWarnings_Init();
 
+    import_init(interp, sysmod);
+
     _PyTime_Init();
 
     if (initfsencoding(interp) < 0)
@@ -638,11 +692,12 @@ Py_NewInterpreter(void)
     }
 
     /* initialize builtin exceptions */
-    _PyExc_Init();
+    _PyExc_Init(bimod);
 
     sysmod = _PyImport_FindBuiltin("sys");
     if (bimod != NULL && sysmod != NULL) {
         PyObject *pstderr;
+
         interp->sysdict = PyModule_GetDict(sysmod);
         if (interp->sysdict == NULL)
             goto handle_error;
@@ -661,6 +716,8 @@ Py_NewInterpreter(void)
 
         _PyImportHooks_Init();
 
+        import_init(interp, sysmod);
+
         if (initfsencoding(interp) < 0)
             goto handle_error;