Make importlib.abc.SourceLoader the primary mechanism for importlib.

This required moving the class from importlib/abc.py into
importlib/_bootstrap.py and jiggering some code to work better with the class.
This included changing how the file finder worked to better meet import
semantics. This also led to fixing importlib to handle the empty string from
sys.path as import currently does (and making me wish we didn't support that
instead just required people to insert '.' instead to represent cwd).

It also required making the new set_data abstractmethod create
any needed subdirectories implicitly thanks to __pycache__ (it was either this
or grow the SourceLoader ABC to gain an 'exists' method and either a mkdir
method or have set_data with no data arg mean to create a directory).

Lastly, as an optimization the file loaders cache the file path where the
finder found something to use for loading (this is thanks to having a
sourceless loader separate from the source loader to simplify the code and
cut out stat calls).
Unfortunately test_runpy assumed a loader would always work for a module, even
if you changed from underneath it what it was expected to work with. By simply
dropping the previous loader in test_runpy so the proper loader can be returned
by the finder fixed the failure.

At this point importlib deviates from import on two points:

1. The exception raised when trying to import a file is different (import does
an explicit file check to print a special message, importlib just says the path
cannot be imported as if it was just some module name).

2. the co_filename on a code object is not being set to where bytecode was
actually loaded from instead of where the marshalled code object originally
came from (a solution for this has already been agreed upon on python-dev but has
not been implemented yet; issue8611).

Doc/library/importlib.rst

@@ -247,8 +247,11 @@ are also provided to help in implementing the core ABCs.
     .. method:: set_data(self, path, data)
 
         Optional abstract method which writes the specified bytes to a file
-        path. When writing to the path fails because the path is read-only, do
-        not propagate the exception.
+        path. Any intermediate directories which do not exist are to be created
+        automatically.
+
+        When writing to the path fails because the path is read-only
+        (:attr:`errno.EACCES`), do not propagate the exception.
 
     .. method:: get_code(self, fullname)
 

Lib/importlib/__init__.py

@@ -36,7 +36,7 @@ def _case_ok(directory, check):
     """
     if 'PYTHONCASEOK' in os.environ:
         return True
-    elif check in os.listdir(directory):
+    elif check in os.listdir(directory if directory else os.getcwd()):
         return True
     return False
 

Lib/importlib/abc.py

@@ -182,8 +182,6 @@ class PyLoader(SourceLoader):
         else:
             return path
 
-PyLoader.register(_bootstrap.PyLoader)
-
 
 class PyPycLoader(PyLoader):
 
@@ -266,7 +264,6 @@ class PyPycLoader(PyLoader):
             self.write_bytecode(fullname, data)
         return code_object
 
-
     @abc.abstractmethod
     def source_mtime(self, fullname:str) -> int:
         """Abstract method which when implemented should return the
@@ -285,5 +282,3 @@ class PyPycLoader(PyLoader):
         bytecode for the module, returning a boolean representing whether the
         bytecode was written or not."""
         raise NotImplementedError
-
-PyPycLoader.register(_bootstrap.PyPycLoader)

Lib/importlib/test/extension/test_case_sensitivity.py

@@ -13,7 +13,8 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
         good_name = ext_util.NAME
         bad_name = good_name.upper()
         assert good_name != bad_name
-        finder = _bootstrap._ExtensionFileFinder(ext_util.PATH)
+        finder = _bootstrap._FileFinder(ext_util.PATH,
+                                        _bootstrap._ExtensionFinderDetails())
         return finder.find_module(bad_name)
 
     def test_case_sensitive(self):

Lib/importlib/test/extension/test_finder.py

@@ -9,7 +9,8 @@ class FinderTests(abc.FinderTests):
     """Test the finder for extension modules."""
 
     def find_module(self, fullname):
-        importer = _bootstrap._ExtensionFileFinder(util.PATH)
+        importer = _bootstrap._FileFinder(util.PATH,
+                                          _bootstrap._ExtensionFinderDetails())
         return importer.find_module(fullname)
 
     def test_module(self):

Lib/importlib/test/extension/test_loader.py

@@ -13,7 +13,7 @@ class LoaderTests(abc.LoaderTests):
 
     def load_module(self, fullname):
         loader = _bootstrap._ExtensionFileLoader(ext_util.NAME,
-                                                ext_util.FILEPATH, False)
+                                                ext_util.FILEPATH)
         return loader.load_module(fullname)
 
     def test_module(self):

Lib/importlib/test/extension/test_path_hook.py

@@ -14,7 +14,7 @@ class PathHookTests(unittest.TestCase):
     # XXX Should it only work for directories containing an extension module?
 
     def hook(self, entry):
-        return _bootstrap._ExtensionFileFinder(entry)
+        return _bootstrap._file_path_hook(entry)
 
     def test_success(self):
         # Path hook should handle a directory where a known extension module

Lib/importlib/test/import_/test_path.py

@@ -5,6 +5,7 @@ from . import util as import_util
 import imp
 import os
 import sys
+import tempfile
 from test import support
 from types import MethodType
 import unittest
@@ -80,23 +81,28 @@ class DefaultPathFinderTests(unittest.TestCase):
 
     def test_implicit_hooks(self):
         # Test that the implicit path hooks are used.
-        existing_path = os.path.dirname(support.TESTFN)
         bad_path = '<path>'
         module = '<module>'
         assert not os.path.exists(bad_path)
-        with util.import_state():
-            nothing = _bootstrap._DefaultPathFinder.find_module(module,
-                                                           path=[existing_path])
-            self.assertTrue(nothing is None)
-            self.assertTrue(existing_path in sys.path_importer_cache)
-            self.assertTrue(not isinstance(sys.path_importer_cache[existing_path],
-                                        imp.NullImporter))
-            nothing = _bootstrap._DefaultPathFinder.find_module(module,
-                                                                path=[bad_path])
-            self.assertTrue(nothing is None)
-            self.assertTrue(bad_path in sys.path_importer_cache)
-            self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
-                                    imp.NullImporter))
+        existing_path = tempfile.mkdtemp()
+        try:
+            with util.import_state():
+                nothing = _bootstrap._DefaultPathFinder.find_module(module,
+                                                        path=[existing_path])
+                self.assertTrue(nothing is None)
+                self.assertTrue(existing_path in sys.path_importer_cache)
+                result = isinstance(sys.path_importer_cache[existing_path],
+                                    imp.NullImporter)
+                self.assertFalse(result)
+                nothing = _bootstrap._DefaultPathFinder.find_module(module,
+                                                            path=[bad_path])
+                self.assertTrue(nothing is None)
+                self.assertTrue(bad_path in sys.path_importer_cache)
+                self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
+                                           imp.NullImporter))
+        finally:
+            os.rmdir(existing_path)
+
 
     def test_path_importer_cache_has_None(self):
         # Test that the default hook is used when sys.path_importer_cache

Lib/importlib/test/regrtest.py

@@ -6,9 +6,11 @@ Otherwise all command-line options valid for test.regrtest are also valid for
 this script.
 
 XXX FAILING
-    test_import
-        execution bit
+  * test_import
+    - test_incorrect_code_name
         file name differing between __file__ and co_filename (r68360 on trunk)
+    - test_import_by_filename
+        exception for trying to import by file name does not match
 
 """
 import importlib

Lib/importlib/test/source/test_abc_loader.py

@@ -815,6 +815,7 @@ class AbstractMethodImplTests(unittest.TestCase):
     def test_Loader(self):
         self.raises_NotImplementedError(self.Loader(), 'load_module')
 
+    # XXX misplaced; should be somewhere else
     def test_Finder(self):
         self.raises_NotImplementedError(self.Finder(), 'find_module')
 

Lib/importlib/test/source/test_case_sensitivity.py

@@ -19,7 +19,9 @@ class CaseSensitivityTest(unittest.TestCase):
     assert name != name.lower()
 
     def find(self, path):
-        finder = _bootstrap._PyPycFileFinder(path)
+        finder = _bootstrap._FileFinder(path,
+                                        _bootstrap._SourceFinderDetails(),
+                                        _bootstrap._SourcelessFinderDetails())
         return finder.find_module(self.name)
 
     def sensitivity_test(self):
@@ -27,7 +29,7 @@ class CaseSensitivityTest(unittest.TestCase):
         sensitive_pkg = 'sensitive.{0}'.format(self.name)
         insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
         context = source_util.create_modules(insensitive_pkg, sensitive_pkg)
-        with  context as mapping:
+        with context as mapping:
             sensitive_path = os.path.join(mapping['.root'], 'sensitive')
             insensitive_path = os.path.join(mapping['.root'], 'insensitive')
             return self.find(sensitive_path), self.find(insensitive_path)
@@ -37,7 +39,7 @@ class CaseSensitivityTest(unittest.TestCase):
             env.unset('PYTHONCASEOK')
             sensitive, insensitive = self.sensitivity_test()
             self.assertTrue(hasattr(sensitive, 'load_module'))
-            self.assertIn(self.name, sensitive._base_path)
+            self.assertIn(self.name, sensitive.get_filename(self.name))
             self.assertIsNone(insensitive)
 
     def test_insensitive(self):
@@ -45,9 +47,9 @@ class CaseSensitivityTest(unittest.TestCase):
             env.set('PYTHONCASEOK', '1')
             sensitive, insensitive = self.sensitivity_test()
             self.assertTrue(hasattr(sensitive, 'load_module'))
-            self.assertIn(self.name, sensitive._base_path)
+            self.assertIn(self.name, sensitive.get_filename(self.name))
             self.assertTrue(hasattr(insensitive, 'load_module'))
-            self.assertIn(self.name, insensitive._base_path)
+            self.assertIn(self.name, insensitive.get_filename(self.name))
 
 
 def test_main():

Lib/importlib/test/source/test_finder.py

@@ -34,7 +34,9 @@ class FinderTests(abc.FinderTests):
     """
 
     def import_(self, root, module):
-        finder = _bootstrap._PyPycFileFinder(root)
+        finder = _bootstrap._FileFinder(root,
+                                        _bootstrap._SourceFinderDetails(),
+                                        _bootstrap._SourcelessFinderDetails())
         return finder.find_module(module)
 
     def run_test(self, test, create=None, *, compile_=None, unlink=None):
@@ -116,7 +118,7 @@ class FinderTests(abc.FinderTests):
         # XXX This is not a blackbox test!
         name = '_temp'
         loader = self.run_test(name, {'{0}.__init__'.format(name), name})
-        self.assertTrue('__init__' in loader._base_path)
+        self.assertTrue('__init__' in loader.get_filename(name))
 
 
     def test_failure(self):

Lib/importlib/test/source/test_path_hook.py

@@ -8,9 +8,8 @@ class PathHookTest(unittest.TestCase):
     """Test the path hook for source."""
 
     def test_success(self):
-        # XXX Only work on existing directories?
         with source_util.create_modules('dummy') as mapping:
-            self.assertTrue(hasattr(_bootstrap._FileFinder(mapping['.root']),
+            self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']),
                                  'find_module'))
 
 

Lib/importlib/test/source/test_source_encoding.py

@@ -35,8 +35,8 @@ class EncodingTest(unittest.TestCase):
         with source_util.create_modules(self.module_name) as mapping:
             with open(mapping[self.module_name], 'wb') as file:
                 file.write(source)
-            loader = _bootstrap._PyPycFileLoader(self.module_name,
-                                       mapping[self.module_name], False)
+            loader = _bootstrap._SourceFileLoader(self.module_name,
+                                       mapping[self.module_name])
             return loader.load_module(self.module_name)
 
     def create_source(self, encoding):
@@ -97,8 +97,8 @@ class LineEndingTest(unittest.TestCase):
         with source_util.create_modules(module_name) as mapping:
             with open(mapping[module_name], 'wb') as file:
                 file.write(source)
-            loader = _bootstrap._PyPycFileLoader(module_name,
-                                                 mapping[module_name], False)
+            loader = _bootstrap._SourceFileLoader(module_name,
+                                                 mapping[module_name])
             return loader.load_module(module_name)
 
     # [cr]

Lib/test/test_runpy.py

@@ -6,7 +6,7 @@ import sys
 import re
 import tempfile
 import py_compile
-from test.support import forget, make_legacy_pyc, run_unittest, verbose
+from test.support import forget, make_legacy_pyc, run_unittest, unload, verbose
 from test.script_helper import (
     make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
 
@@ -174,6 +174,7 @@ class RunModuleTest(unittest.TestCase):
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
+            unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", mod_name)
             d2 = run_module(mod_name) # Read from bytecode
             self.assertIn("x", d2)
@@ -197,6 +198,7 @@ class RunModuleTest(unittest.TestCase):
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
+            unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", pkg_name)
             d2 = run_module(pkg_name) # Read from bytecode
             self.assertIn("x", d2)
@@ -252,6 +254,7 @@ from ..uncle.cousin import nephew
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
+            unload(mod_name)  # In case the loader caches paths
             if verbose: print("Running from compiled:", mod_name)
             d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
             self.assertIn("__package__", d2)
@@ -405,7 +408,11 @@ argv0 = sys.argv[0]
 
 
 def test_main():
-    run_unittest(RunModuleCodeTest, RunModuleTest, RunPathTest)
+    run_unittest(
+                 RunModuleCodeTest,
+                 RunModuleTest,
+                 RunPathTest
+                 )
 
 if __name__ == "__main__":
     test_main()