Skip to content

C
cpython
Commit 61b14251 authored by Brett Cannon's avatar Brett Cannon
Browse files
Options

Make importlib.abc.SourceLoader the primary mechanism for importlib. 

This required moving the class from importlib/abc.py into
importlib/_bootstrap.py and jiggering some code to work better with the class.
This included changing how the file finder worked to better meet import
semantics. This also led to fixing importlib to handle the empty string from
sys.path as import currently does (and making me wish we didn't support that
instead just required people to insert '.' instead to represent cwd).

It also required making the new set_data abstractmethod create
any needed subdirectories implicitly thanks to __pycache__ (it was either this
or grow the SourceLoader ABC to gain an 'exists' method and either a mkdir
method or have set_data with no data arg mean to create a directory).

Lastly, as an optimization the file loaders cache the file path where the
finder found something to use for loading (this is thanks to having a
sourceless loader separate from the source loader to simplify the code and
cut out stat calls).
Unfortunately test_runpy assumed a loader would always work for a module, even
if you changed from underneath it what it was expected to work with. By simply
dropping the previous loader in test_runpy so the proper loader can be returned
by the finder fixed the failure.

At this point importlib deviates from import on two points:

1. The exception raised when trying to import a file is different (import does
an explicit file check to print a special message, importlib just says the path
cannot be imported as if it was just some module name).

2. the co_filename on a code object is not being set to where bytecode was
actually loaded from instead of where the marshalled code object originally
came from (a solution for this has already been agreed upon on python-dev but has
not been implemented yet; issue8611).
parent bb3565d4
Hide whitespace changes
Inline Side-by-side
Showing with 420 additions and 544 deletions
Doc/library/importlib.rst
View file @ 61b14251
......@@ -247,8 +247,11 @@ are also provided to help in implementing the core ABCs.
.. method:: set_data(self, path, data)
Optional abstract method which writes the specified bytes to a file
path. When writing to the path fails because the path is read-only, do
not propagate the exception.
path. Any intermediate directories which do not exist are to be created
automatically.
When writing to the path fails because the path is read-only
(:attr:`errno.EACCES`), do not propagate the exception.
.. method:: get_code(self, fullname)
......
Lib/importlib/__init__.py
View file @ 61b14251
......@@ -36,7 +36,7 @@ def _case_ok(directory, check):
"""
if 'PYTHONCASEOK' in os.environ:
return True
elif check in os.listdir(directory):
elif check in os.listdir(directory if directory else os.getcwd()):
return True
return False
......
Lib/importlib/_bootstrap.py
View file @ 61b14251
......@@ -22,7 +22,7 @@ work. One should use importlib as the public-facing version of this module.
def _path_join(*args):
"""Replacement for os.path.join."""
return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x
for x in args)
for x in args if x)
def _path_exists(path):
......@@ -53,6 +53,8 @@ def _path_isfile(path):
# XXX Could also expose Modules/getpath.c:isdir()
def _path_isdir(path):
"""Replacement for os.path.isdir."""
if not path:
path = _os.getcwd()
return _path_is_mode_type(path, 0o040000)
......@@ -99,6 +101,8 @@ def _wrap(new, old):
new.__dict__.update(old.__dict__)
code_type = type(_wrap.__code__)
# Finder/loader utility code ##################################################
def set_package(fxn):
......@@ -138,7 +142,7 @@ def module_for_loader(fxn):
the second argument.
"""
def decorated(self, fullname):
def decorated(self, fullname, *args, **kwargs):
module = sys.modules.get(fullname)
is_reload = bool(module)
if not is_reload:
......@@ -148,7 +152,7 @@ def module_for_loader(fxn):
module = imp.new_module(fullname)
sys.modules[fullname] = module
try:
return fxn(self, module)
return fxn(self, module, *args, **kwargs)
except:
if not is_reload:
del sys.modules[fullname]
......@@ -301,7 +305,59 @@ class FrozenImporter:
return imp.is_frozen_package(fullname)
class SourceLoader:
class _LoaderBasics:
"""Base class of common code needed by both SourceLoader and
_SourcelessFileLoader."""
def is_package(self, fullname):
"""Concrete implementation of InspectLoader.is_package by checking if
the path returned by get_filename has a filename of '__init__.py'."""
filename = self.get_filename(fullname).rpartition(path_sep)[2]
return filename.rsplit('.', 1)[0] == '__init__'
def _bytes_from_bytecode(self, fullname, data, source_mtime):
"""Return the marshalled bytes from bytecode, verifying the magic
number and timestamp alon the way.
If source_mtime is None then skip the timestamp check.
"""
magic = data[:4]
raw_timestamp = data[4:8]
if len(magic) != 4 or magic != imp.get_magic():
raise ImportError("bad magic number in {}".format(fullname))
elif len(raw_timestamp) != 4:
raise EOFError("bad timestamp in {}".format(fullname))
elif source_mtime is not None:
if marshal._r_long(raw_timestamp) != source_mtime:
raise ImportError("bytecode is stale for {}".format(fullname))
# Can't return the code object as errors from marshal loading need to
# propagate even when source is available.
return data[8:]
@module_for_loader
def _load_module(self, module, *, sourceless=False):
"""Helper for load_module able to handle either source or sourceless
loading."""
name = module.__name__
code_object = self.get_code(name)
module.__file__ = self.get_filename(name)
if not sourceless:
module.__cached__ = imp.cache_from_source(module.__file__)
else:
module.__cached__ = module.__file__
module.__package__ = name
if self.is_package(name):
module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
else:
module.__package__ = module.__package__.rpartition('.')[0]
module.__loader__ = self
exec(code_object, module.__dict__)
return module
class SourceLoader(_LoaderBasics):
def path_mtime(self, path:str) -> int:
"""Optional method that returns the modification time for the specified
......@@ -320,11 +376,6 @@ class SourceLoader:
"""
raise NotImplementedError
def is_package(self, fullname):
"""Concrete implementation of InspectLoader.is_package by checking if
the path returned by get_filename has a filename of '__init__.py'."""
filename = self.get_filename(fullname).rsplit(path_sep, 1)[1]
return filename.rsplit('.', 1)[0] == '__init__'
def get_source(self, fullname):
"""Concrete implementation of InspectLoader.get_source."""
......@@ -359,12 +410,18 @@ class SourceLoader:
except IOError:
pass
else:
magic = data[:4]
raw_timestamp = data[4:8]
if (len(magic) == 4 and len(raw_timestamp) == 4 and
magic == imp.get_magic() and
marshal._r_long(raw_timestamp) == source_mtime):
return marshal.loads(data[8:])
try:
bytes_data = self._bytes_from_bytecode(fullname, data,
source_mtime)
except (ImportError, EOFError):
pass
else:
found = marshal.loads(bytes_data)
if isinstance(found, code_type):
return found
else:
msg = "Non-code object in {}"
raise ImportError(msg.format(bytecode_path))
source_bytes = self.get_data(source_path)
code_object = compile(source_bytes, source_path, 'exec',
dont_inherit=True)
......@@ -382,8 +439,7 @@ class SourceLoader:
pass
return code_object
@module_for_loader
def load_module(self, module):
def load_module(self, fullname):
"""Concrete implementation of Loader.load_module.
Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be
......@@ -391,31 +447,17 @@ class SourceLoader:
get_code uses/writes bytecode.
"""
name = module.__name__
code_object = self.get_code(name)
module.__file__ = self.get_filename(name)
module.__cached__ = imp.cache_from_source(module.__file__)
module.__package__ = name
is_package = self.is_package(name)
if is_package:
module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
else:
module.__package__ = module.__package__.rpartition('.')[0]
module.__loader__ = self
exec(code_object, module.__dict__)
return module
return self._load_module(fullname)
class _SourceFileLoader(SourceLoader):
"""Concrete implementation of SourceLoader.
class _FileLoader:
NOT A PUBLIC CLASS! Do not expect any API stability from this class, so DO
NOT SUBCLASS IN YOUR OWN CODE!
"""
"""Base file loader class which implements the loader protocol methods that
require file system usage."""
def __init__(self, fullname, path):
"""Cache the module name and the path to the file found by the
finder."""
self._name = fullname
self._path = path
......@@ -424,272 +466,66 @@ class _SourceFileLoader(SourceLoader):
"""Return the path to the source file as found by the finder."""
return self._path
def get_data(self, path):
"""Return the data from path as raw bytes."""
with _closing(_io.FileIO(path, 'r')) as file:
return file.read()
class _SourceFileLoader(_FileLoader, SourceLoader):
"""Concrete implementation of SourceLoader using the file system."""
def path_mtime(self, path):
"""Return the modification time for the path."""
return int(_os.stat(path).st_mtime)
def set_data(self, data, path):
def set_data(self, path, data):
"""Write bytes data to a file."""
try:
with _closing(_io.FileIO(bytecode_path, 'w')) as file:
with _closing(_io.FileIO(path, 'wb')) as file:
file.write(data)
except IOError as exc:
if exc.errno != errno.EACCES:
if exc.errno == errno.ENOENT:
directory, _, filename = path.rpartition(path_sep)
sub_directories = []
while not _path_isdir(directory):
directory, _, sub_dir = directory.rpartition(path_sep)
sub_directories.append(sub_dir)
for part in reversed(sub_directories):
directory = _path_join(directory, part)
try:
_os.mkdir(directory)
except IOError as exc:
if exc.errno != errno.EACCES:
raise
else:
return
return self.set_data(path, data)
elif exc.errno != errno.EACCES:
raise
class PyLoader:
"""Loader base class for Python source code.
Subclasses need to implement the methods:
- source_path
- get_data
- is_package
"""
@module_for_loader
def load_module(self, module):
"""Initialize the module."""
name = module.__name__
code_object = self.get_code(module.__name__)
module.__file__ = self.get_filename(name)
if self.is_package(name):
module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
module.__package__ = module.__name__
if not hasattr(module, '__path__'):
module.__package__ = module.__package__.rpartition('.')[0]
module.__loader__ = self
exec(code_object, module.__dict__)
return module
def get_filename(self, fullname):
"""Return the path to the source file, else raise ImportError."""
path = self.source_path(fullname)
if path is not None:
return path
else:
raise ImportError("no source path available for "
"{0!r}".format(fullname))
def get_code(self, fullname):
"""Get a code object from source."""
source_path = self.source_path(fullname)
if source_path is None:
message = "a source path must exist to load {0}".format(fullname)
raise ImportError(message)
source = self.get_data(source_path)
return compile(source, source_path, 'exec', dont_inherit=True)
# Never use in implementing import! Imports code within the method.
def get_source(self, fullname):
"""Return the source code for a module.
self.source_path() and self.get_data() are used to implement this
method.
"""
path = self.source_path(fullname)
if path is None:
return None
try:
source_bytes = self.get_data(path)
except IOError:
return ImportError("source not available through get_data()")
import io
import tokenize
encoding = tokenize.detect_encoding(io.BytesIO(source_bytes).readline)
return source_bytes.decode(encoding[0])
class PyPycLoader(PyLoader):
"""Loader base class for Python source and bytecode.
class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
Requires implementing the methods needed for PyLoader as well as
source_mtime, bytecode_path, and write_bytecode.
"""Loader which handles sourceless file imports."""
"""
def get_filename(self, fullname):
"""Return the source or bytecode file path."""
path = self.source_path(fullname)
if path is not None:
return path
path = self.bytecode_path(fullname)
if path is not None:
return path
raise ImportError("no source or bytecode path available for "
"{0!r}".format(fullname))
def load_module(self, fullname):
return self._load_module(fullname, sourceless=True)
def get_code(self, fullname):
"""Get a code object from source or bytecode."""
# XXX Care enough to make sure this call does not happen if the magic
# number is bad?
source_timestamp = self.source_mtime(fullname)
# Try to use bytecode if it is available.
bytecode_path = self.bytecode_path(fullname)
if bytecode_path:
data = self.get_data(bytecode_path)
try:
magic = data[:4]
if len(magic) < 4:
raise ImportError("bad magic number in {}".format(fullname))
raw_timestamp = data[4:8]
if len(raw_timestamp) < 4:
raise EOFError("bad timestamp in {}".format(fullname))
pyc_timestamp = marshal._r_long(raw_timestamp)
bytecode = data[8:]
# Verify that the magic number is valid.
if imp.get_magic() != magic:
raise ImportError("bad magic number in {}".format(fullname))
# Verify that the bytecode is not stale (only matters when
# there is source to fall back on.
if source_timestamp:
if pyc_timestamp < source_timestamp:
raise ImportError("bytecode is stale")
except (ImportError, EOFError):
# If source is available give it a shot.
if source_timestamp is not None:
pass
else:
raise
else:
# Bytecode seems fine, so try to use it.
# XXX If the bytecode is ill-formed, would it be beneficial to
# try for using source if available and issue a warning?
return marshal.loads(bytecode)
elif source_timestamp is None:
raise ImportError("no source or bytecode available to create code "
"object for {0!r}".format(fullname))
# Use the source.
code_object = super().get_code(fullname)
# Generate bytecode and write it out.
if not sys.dont_write_bytecode:
data = bytearray(imp.get_magic())
data.extend(marshal._w_long(source_timestamp))
data.extend(marshal.dumps(code_object))
self.write_bytecode(fullname, data)
return code_object
class _PyFileLoader(PyLoader):
"""Load a Python source file."""
def __init__(self, name, path, is_pkg):
self._name = name
self._is_pkg = is_pkg
# Figure out the base path based on whether it was source or bytecode
# that was found.
try:
self._base_path = _path_without_ext(path, imp.PY_SOURCE)
except ValueError:
self._base_path = _path_without_ext(path, imp.PY_COMPILED)
def _find_path(self, ext_type):
"""Find a path from the base path and the specified extension type that
exists, returning None if one is not found."""
for suffix in _suffix_list(ext_type):
path = self._base_path + suffix
if _path_exists(path):
return path
path = self.get_filename(fullname)
data = self.get_data(path)
bytes_data = self._bytes_from_bytecode(fullname, data, None)
found = marshal.loads(bytes_data)
if isinstance(found, code_type):
return found
else:
return None
@_check_name
def source_path(self, fullname):
"""Return the path to an existing source file for the module, or None
if one cannot be found."""
# Not a property so that it is easy to override.
return self._find_path(imp.PY_SOURCE)
raise ImportError("Non-code object in {}".format(path))
def get_data(self, path):
"""Return the data from path as raw bytes."""
return _io.FileIO(path, 'r').read() # Assuming bytes.
@_check_name
def is_package(self, fullname):
"""Return a boolean based on whether the module is a package.
Raises ImportError (like get_source) if the loader cannot handle the
package.
"""
return self._is_pkg
class _PyPycFileLoader(PyPycLoader, _PyFileLoader):
"""Load a module from a source or bytecode file."""
def _find_path(self, ext_type):
"""Return PEP 3147 path if ext_type is PY_COMPILED, otherwise
super()._find_path() is called."""
if ext_type == imp.PY_COMPILED:
# We don't really care what the extension on self._base_path is,
# as long as it has exactly one dot.
source_path = self._base_path + '.py'
pycache_path = imp.cache_from_source(source_path)
legacy_path = self._base_path + '.pyc'
# The rule is: if the source file exists, then Python always uses
# the __pycache__/foo.<tag>.pyc file. If the source file does not
# exist, then Python uses the legacy path.
pyc_path = (pycache_path
if _path_exists(source_path)
else legacy_path)
return (pyc_path if _path_exists(pyc_path) else None)
return super()._find_path(ext_type)
@_check_name
def source_mtime(self, name):
"""Return the modification time of the source for the specified
module."""
source_path = self.source_path(name)
if not source_path:
return None
return int(_os.stat(source_path).st_mtime)
@_check_name
def bytecode_path(self, fullname):
"""Return the path to a bytecode file, or None if one does not
exist."""
# Not a property for easy overriding.
return self._find_path(imp.PY_COMPILED)
@_check_name
def write_bytecode(self, name, data):
"""Write out 'data' for the specified module, returning a boolean
signifying if the write-out actually occurred.
Raises ImportError (just like get_source) if the specified module
cannot be handled by the loader.
"""
bytecode_path = self.bytecode_path(name)
if not bytecode_path:
source_path = self.source_path(name)
bytecode_path = imp.cache_from_source(source_path)
# Ensure that the __pycache__ directory exists. We can't use
# os.path.dirname() here.
dirname, sep, basename = bytecode_path.rpartition(path_sep)
try:
_os.mkdir(dirname)
except OSError as error:
if error.errno != errno.EEXIST:
raise
try:
# Assuming bytes.
with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file:
bytecode_file.write(data)
return True
except IOError as exc:
if exc.errno == errno.EACCES:
return False
else:
raise
def get_source(self, fullname):
"""Return None as there is no source code."""
return None
class _ExtensionFileLoader:
......@@ -700,7 +536,7 @@ class _ExtensionFileLoader:
"""
def __init__(self, name, path, is_pkg):
def __init__(self, name, path):
"""Initialize the loader.
If is_pkg is True then an exception is raised as extension modules
......@@ -709,8 +545,6 @@ class _ExtensionFileLoader:
"""
self._name = name
self._path = path
if is_pkg:
raise ValueError("extension modules cannot be packages")
@_check_name
@set_package
......@@ -808,147 +642,88 @@ class PathFinder:
return None
class _ChainedFinder:
"""Finder that sequentially calls other finders."""
def __init__(self, *finders):
self._finders = finders
def find_module(self, fullname, path=None):
for finder in self._finders:
result = finder.find_module(fullname, path)
if result:
return result
else:
return None
class _FileFinder:
"""Base class for file finders.
Subclasses are expected to define the following attributes:
* _suffixes
Sequence of file suffixes whose order will be followed.
"""File-based finder.
* _possible_package
True if importer should check for packages.
* _loader
A callable that takes the module name, a file path, and whether
the path points to a package and returns a loader for the module
found at that path.
Constructor takes a list of objects detailing what file extensions their
loader supports along with whether it can be used for a package.
"""
def __init__(self, path_entry):
"""Initialize an importer for the passed-in sys.path entry (which is
assumed to have already been verified as an existing directory).
Can be used as an entry on sys.path_hook.
"""
absolute_path = _path_absolute(path_entry)
if not _path_isdir(absolute_path):
raise ImportError("only directories are supported")
self._path_entry = absolute_path
def find_module(self, fullname, path=None):
def __init__(self, path, *details):
"""Initialize with finder details."""
packages = []
modules = []
for detail in details:
modules.extend((suffix, detail.loader) for suffix in detail.suffixes)
if detail.supports_packages:
packages.extend((suffix, detail.loader)
for suffix in detail.suffixes)
self.packages = packages
self.modules = modules
self.path = path
def find_module(self, fullname):
"""Try to find a loader for the specified module."""
tail_module = fullname.rpartition('.')[2]
package_directory = None
if self._possible_package:
for ext in self._suffixes:
package_directory = _path_join(self._path_entry, tail_module)
init_filename = '__init__' + ext
package_init = _path_join(package_directory, init_filename)
if (_path_isfile(package_init) and
_case_ok(self._path_entry, tail_module) and
_case_ok(package_directory, init_filename)):
return self._loader(fullname, package_init, True)
for ext in self._suffixes:
file_name = tail_module + ext
file_path = _path_join(self._path_entry, file_name)
if (_path_isfile(file_path) and
_case_ok(self._path_entry, file_name)):
return self._loader(fullname, file_path, False)
else:
# Raise a warning if it matches a directory w/o an __init__ file.
if (package_directory is not None and
_path_isdir(package_directory) and
_case_ok(self._path_entry, tail_module)):
_warnings.warn("Not importing directory %s: missing __init__"
% package_directory, ImportWarning)
return None
class _PyFileFinder(_FileFinder):
"""Importer for source/bytecode files."""
_possible_package = True
_loader = _PyFileLoader
def __init__(self, path_entry):
# Lack of imp during class creation means _suffixes is set here.
# Make sure that Python source files are listed first! Needed for an
# optimization by the loader.
self._suffixes = _suffix_list(imp.PY_SOURCE)
super().__init__(path_entry)
class _PyPycFileFinder(_PyFileFinder):
base_path = _path_join(self.path, tail_module)
if _path_isdir(base_path) and _case_ok(self.path, tail_module):
for suffix, loader in self.packages:
init_filename = '__init__' + suffix
full_path = _path_join(base_path, init_filename)
if (_path_isfile(full_path) and
_case_ok(base_path, init_filename)):
return loader(fullname, full_path)
else:
msg = "Not importing directory {}: missing __init__"
_warnings.warn(msg.format(base_path), ImportWarning)
for suffix, loader in self.modules:
mod_filename = tail_module + suffix
full_path = _path_join(self.path, mod_filename)
if _path_isfile(full_path) and _case_ok(self.path, mod_filename):
return loader(fullname, full_path)
return None
"""Finder for source and bytecode files."""
class _SourceFinderDetails:
_loader = _PyPycFileLoader
loader = _SourceFileLoader
supports_packages = True
def __init__(self, path_entry):
super().__init__(path_entry)
self._suffixes += _suffix_list(imp.PY_COMPILED)
def __init__(self):
self.suffixes = _suffix_list(imp.PY_SOURCE)
class _SourcelessFinderDetails:
loader = _SourcelessFileLoader
supports_packages = True
def __init__(self):
self.suffixes = _suffix_list(imp.PY_COMPILED)
class _ExtensionFileFinder(_FileFinder):
"""Importer for extension files."""
class _ExtensionFinderDetails:
_possible_package = False
_loader = _ExtensionFileLoader
loader = _ExtensionFileLoader
supports_packages = False
def __init__(self, path_entry):
# Assigning to _suffixes here instead of at the class level because
# imp is not imported at the time of class creation.
self._suffixes = _suffix_list(imp.C_EXTENSION)
super().__init__(path_entry)
def __init__(self):
self.suffixes = _suffix_list(imp.C_EXTENSION)
# Import itself ###############################################################
def _chained_path_hook(*path_hooks):
"""Create a closure which sequentially checks path hooks to see which ones
(if any) can work with a path."""
def path_hook(entry):
"""Check to see if 'entry' matches any of the enclosed path hooks."""
finders = []
for hook in path_hooks:
try:
finder = hook(entry)
except ImportError:
continue
else:
finders.append(finder)
if not finders:
raise ImportError("no finder found")
else:
return _ChainedFinder(*finders)
return path_hook
def _file_path_hook(path):
"""If the path is a directory, return a file-based finder."""
if _path_isdir(path):
return _FileFinder(path, _ExtensionFinderDetails(),
_SourceFinderDetails(),
_SourcelessFinderDetails())
else:
raise ImportError("only directories are supported")
_DEFAULT_PATH_HOOK = _chained_path_hook(_ExtensionFileFinder, _PyPycFileFinder)
_DEFAULT_PATH_HOOK = _file_path_hook
class _DefaultPathFinder(PathFinder):
......
Lib/importlib/abc.py
View file @ 61b14251
......@@ -182,8 +182,6 @@ class PyLoader(SourceLoader):
else:
return path
PyLoader.register(_bootstrap.PyLoader)
class PyPycLoader(PyLoader):
......@@ -266,7 +264,6 @@ class PyPycLoader(PyLoader):
self.write_bytecode(fullname, data)
return code_object
@abc.abstractmethod
def source_mtime(self, fullname:str) -> int:
"""Abstract method which when implemented should return the
......@@ -285,5 +282,3 @@ class PyPycLoader(PyLoader):
bytecode for the module, returning a boolean representing whether the
bytecode was written or not."""
raise NotImplementedError
PyPycLoader.register(_bootstrap.PyPycLoader)
Lib/importlib/test/extension/test_case_sensitivity.py
View file @ 61b14251
......@@ -13,7 +13,8 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
good_name = ext_util.NAME
bad_name = good_name.upper()
assert good_name != bad_name
finder = _bootstrap._ExtensionFileFinder(ext_util.PATH)
finder = _bootstrap._FileFinder(ext_util.PATH,
_bootstrap._ExtensionFinderDetails())
return finder.find_module(bad_name)
def test_case_sensitive(self):
......
Lib/importlib/test/extension/test_finder.py
View file @ 61b14251
......@@ -9,7 +9,8 @@ class FinderTests(abc.FinderTests):
"""Test the finder for extension modules."""
def find_module(self, fullname):
importer = _bootstrap._ExtensionFileFinder(util.PATH)
importer = _bootstrap._FileFinder(util.PATH,
_bootstrap._ExtensionFinderDetails())
return importer.find_module(fullname)
def test_module(self):
......
Lib/importlib/test/extension/test_loader.py
View file @ 61b14251
......@@ -13,7 +13,7 @@ class LoaderTests(abc.LoaderTests):
def load_module(self, fullname):
loader = _bootstrap._ExtensionFileLoader(ext_util.NAME,
ext_util.FILEPATH, False)
ext_util.FILEPATH)
return loader.load_module(fullname)
def test_module(self):
......
Lib/importlib/test/extension/test_path_hook.py
View file @ 61b14251
......@@ -14,7 +14,7 @@ class PathHookTests(unittest.TestCase):
# XXX Should it only work for directories containing an extension module?
def hook(self, entry):
return _bootstrap._ExtensionFileFinder(entry)
return _bootstrap._file_path_hook(entry)
def test_success(self):
# Path hook should handle a directory where a known extension module
......
Lib/importlib/test/import_/test_path.py
View file @ 61b14251
......@@ -5,6 +5,7 @@ from . import util as import_util
import imp
import os
import sys
import tempfile
from test import support
from types import MethodType
import unittest
......@@ -80,23 +81,28 @@ class DefaultPathFinderTests(unittest.TestCase):
def test_implicit_hooks(self):
# Test that the implicit path hooks are used.
existing_path = os.path.dirname(support.TESTFN)
bad_path = '<path>'
module = '<module>'
assert not os.path.exists(bad_path)
with util.import_state():
nothing = _bootstrap._DefaultPathFinder.find_module(module,
path=[existing_path])
self.assertTrue(nothing is None)
self.assertTrue(existing_path in sys.path_importer_cache)
self.assertTrue(not isinstance(sys.path_importer_cache[existing_path],
imp.NullImporter))
nothing = _bootstrap._DefaultPathFinder.find_module(module,
path=[bad_path])
self.assertTrue(nothing is None)
self.assertTrue(bad_path in sys.path_importer_cache)
self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
imp.NullImporter))
existing_path = tempfile.mkdtemp()
try:
with util.import_state():
nothing = _bootstrap._DefaultPathFinder.find_module(module,
path=[existing_path])
self.assertTrue(nothing is None)
self.assertTrue(existing_path in sys.path_importer_cache)
result = isinstance(sys.path_importer_cache[existing_path],
imp.NullImporter)
self.assertFalse(result)
nothing = _bootstrap._DefaultPathFinder.find_module(module,
path=[bad_path])
self.assertTrue(nothing is None)
self.assertTrue(bad_path in sys.path_importer_cache)
self.assertTrue(isinstance(sys.path_importer_cache[bad_path],
imp.NullImporter))
finally:
os.rmdir(existing_path)
def test_path_importer_cache_has_None(self):
# Test that the default hook is used when sys.path_importer_cache
......
Lib/importlib/test/regrtest.py
View file @ 61b14251
......@@ -6,9 +6,11 @@ Otherwise all command-line options valid for test.regrtest are also valid for
this script.
XXX FAILING
test_import
execution bit
* test_import
- test_incorrect_code_name
file name differing between __file__ and co_filename (r68360 on trunk)
- test_import_by_filename
exception for trying to import by file name does not match
"""
import importlib
......
Lib/importlib/test/source/test_abc_loader.py
View file @ 61b14251
......@@ -815,6 +815,7 @@ class AbstractMethodImplTests(unittest.TestCase):
def test_Loader(self):
self.raises_NotImplementedError(self.Loader(), 'load_module')
# XXX misplaced; should be somewhere else
def test_Finder(self):
self.raises_NotImplementedError(self.Finder(), 'find_module')
......
Lib/importlib/test/source/test_case_sensitivity.py
View file @ 61b14251
......@@ -19,7 +19,9 @@ class CaseSensitivityTest(unittest.TestCase):
assert name != name.lower()
def find(self, path):
finder = _bootstrap._PyPycFileFinder(path)
finder = _bootstrap._FileFinder(path,
_bootstrap._SourceFinderDetails(),
_bootstrap._SourcelessFinderDetails())
return finder.find_module(self.name)
def sensitivity_test(self):
......@@ -27,7 +29,7 @@ class CaseSensitivityTest(unittest.TestCase):
sensitive_pkg = 'sensitive.{0}'.format(self.name)
insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
context = source_util.create_modules(insensitive_pkg, sensitive_pkg)
with context as mapping:
with context as mapping:
sensitive_path = os.path.join(mapping['.root'], 'sensitive')
insensitive_path = os.path.join(mapping['.root'], 'insensitive')
return self.find(sensitive_path), self.find(insensitive_path)
......@@ -37,7 +39,7 @@ class CaseSensitivityTest(unittest.TestCase):
env.unset('PYTHONCASEOK')
sensitive, insensitive = self.sensitivity_test()
self.assertTrue(hasattr(sensitive, 'load_module'))
self.assertIn(self.name, sensitive._base_path)
self.assertIn(self.name, sensitive.get_filename(self.name))
self.assertIsNone(insensitive)
def test_insensitive(self):
......@@ -45,9 +47,9 @@ class CaseSensitivityTest(unittest.TestCase):
env.set('PYTHONCASEOK', '1')
sensitive, insensitive = self.sensitivity_test()
self.assertTrue(hasattr(sensitive, 'load_module'))
self.assertIn(self.name, sensitive._base_path)
self.assertIn(self.name, sensitive.get_filename(self.name))
self.assertTrue(hasattr(insensitive, 'load_module'))
self.assertIn(self.name, insensitive._base_path)
self.assertIn(self.name, insensitive.get_filename(self.name))
def test_main():
......
Lib/importlib/test/source/test_file_loader.py
View file @ 61b14251
......@@ -4,6 +4,7 @@ from .. import abc
from . import util as source_util
import imp
import marshal
import os
import py_compile
import stat
......@@ -23,8 +24,7 @@ class SimpleTest(unittest.TestCase):
# [basic]
def test_module(self):
with source_util.create_modules('_temp') as mapping:
loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
False)
loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
module = loader.load_module('_temp')
self.assertTrue('_temp' in sys.modules)
check = {'__name__': '_temp', '__file__': mapping['_temp'],
......@@ -34,9 +34,8 @@ class SimpleTest(unittest.TestCase):
def test_package(self):
with source_util.create_modules('_pkg.__init__') as mapping:
loader = _bootstrap._PyPycFileLoader('_pkg',
mapping['_pkg.__init__'],
True)
loader = _bootstrap._SourceFileLoader('_pkg',
mapping['_pkg.__init__'])
module = loader.load_module('_pkg')
self.assertTrue('_pkg' in sys.modules)
check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
......@@ -48,8 +47,8 @@ class SimpleTest(unittest.TestCase):
def test_lacking_parent(self):
with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
loader = _bootstrap._PyPycFileLoader('_pkg.mod',
mapping['_pkg.mod'], False)
loader = _bootstrap._SourceFileLoader('_pkg.mod',
mapping['_pkg.mod'])
module = loader.load_module('_pkg.mod')
self.assertTrue('_pkg.mod' in sys.modules)
check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
......@@ -63,8 +62,7 @@ class SimpleTest(unittest.TestCase):
def test_module_reuse(self):
with source_util.create_modules('_temp') as mapping:
loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
False)
loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
module = loader.load_module('_temp')
module_id = id(module)
module_dict_id = id(module.__dict__)
......@@ -74,7 +72,7 @@ class SimpleTest(unittest.TestCase):
# everything that has happened above can be too fast;
# force an mtime on the source that is guaranteed to be different
# than the original mtime.
loader.source_mtime = self.fake_mtime(loader.source_mtime)
loader.path_mtime = self.fake_mtime(loader.path_mtime)
module = loader.load_module('_temp')
self.assertTrue('testing_var' in module.__dict__,
"'testing_var' not in "
......@@ -94,8 +92,7 @@ class SimpleTest(unittest.TestCase):
setattr(orig_module, attr, value)
with open(mapping[name], 'w') as file:
file.write('+++ bad syntax +++')
loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
False)
loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
loader.load_module(name)
for attr in attributes:
......@@ -106,8 +103,7 @@ class SimpleTest(unittest.TestCase):
with source_util.create_modules('_temp') as mapping:
with open(mapping['_temp'], 'w') as file:
file.write('=')
loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'],
False)
loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
loader.load_module('_temp')
self.assertTrue('_temp' not in sys.modules)
......@@ -116,7 +112,7 @@ class SimpleTest(unittest.TestCase):
class BadBytecodeTest(unittest.TestCase):
def import_(self, file, module_name):
loader = _bootstrap._PyPycFileLoader(module_name, file, False)
loader = self.loader(module_name, file)
module = loader.load_module(module_name)
self.assertTrue(module_name in sys.modules)
......@@ -129,101 +125,156 @@ class BadBytecodeTest(unittest.TestCase):
except KeyError:
pass
py_compile.compile(mapping[name])
bytecode_path = imp.cache_from_source(mapping[name])
with open(bytecode_path, 'rb') as file:
bc = file.read()
new_bc = manipulator(bc)
with open(bytecode_path, 'wb') as file:
if new_bc:
file.write(new_bc)
if del_source:
if not del_source:
bytecode_path = imp.cache_from_source(mapping[name])
else:
os.unlink(mapping[name])
make_legacy_pyc(mapping[name])
bytecode_path = make_legacy_pyc(mapping[name])
if manipulator:
with open(bytecode_path, 'rb') as file:
bc = file.read()
new_bc = manipulator(bc)
with open(bytecode_path, 'wb') as file:
if new_bc is not None:
file.write(new_bc)
return bytecode_path
@source_util.writes_bytecode_files
def test_empty_file(self):
# When a .pyc is empty, regenerate it if possible, else raise
# ImportError.
def _test_empty_file(self, test, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: None)
self.import_(mapping['_temp'], '_temp')
with open(bc_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
self.manipulate_bytecode('_temp', mapping, lambda bc: None,
del_source=True)
with self.assertRaises(ImportError):
self.import_(mapping['_temp'], '_temp')
lambda bc: b'',
del_source=del_source)
test('_temp', mapping, bc_path)
@source_util.writes_bytecode_files
def test_partial_magic(self):
def _test_partial_magic(self, test, *, del_source=False):
# When their are less than 4 bytes to a .pyc, regenerate it if
# possible, else raise ImportError.
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:3])
self.import_(mapping['_temp'], '_temp')
with open(bc_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:3],
del_source=True)
lambda bc: bc[:3],
del_source=del_source)
test('_temp', mapping, bc_path)
def _test_magic_only(self, test, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:4],
del_source=del_source)
test('_temp', mapping, bc_path)
def _test_partial_timestamp(self, test, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:7],
del_source=del_source)
test('_temp', mapping, bc_path)
def _test_no_marshal(self, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:8],
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bc_path
with self.assertRaises(EOFError):
self.import_(file_path, '_temp')
def _test_non_code_marshal(self, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bytecode_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:8] + marshal.dumps(b'abcd'),
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bytecode_path
with self.assertRaises(ImportError):
self.import_(mapping['_temp'], '_temp')
self.import_(file_path, '_temp')
def _test_bad_marshal(self, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bytecode_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:8] + b'<test>',
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bytecode_path
with self.assertRaises(ValueError):
self.import_(file_path, '_temp')
def _test_bad_magic(self, test, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: b'\x00\x00\x00\x00' + bc[4:])
test('_temp', mapping, bc_path)
class SourceLoaderBadBytecodeTest(BadBytecodeTest):
loader = _bootstrap._SourceFileLoader
@source_util.writes_bytecode_files
def test_empty_file(self):
# When a .pyc is empty, regenerate it if possible, else raise
# ImportError.
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
self._test_empty_file(test)
def test_partial_magic(self):
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
self._test_partial_magic(test)
@source_util.writes_bytecode_files
def test_magic_only(self):
# When there is only the magic number, regenerate the .pyc if possible,
# else raise EOFError.
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:4])
self.import_(mapping['_temp'], '_temp')
with open(bc_path, 'rb') as file:
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:4],
del_source=True)
with self.assertRaises(EOFError):
self.import_(mapping['_temp'], '_temp')
@source_util.writes_bytecode_files
def test_bad_magic(self):
# When the magic number is different, the bytecode should be
# regenerated.
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as bytecode_file:
self.assertEqual(bytecode_file.read(4), imp.get_magic())
self._test_bad_magic(test)
@source_util.writes_bytecode_files
def test_partial_timestamp(self):
# When the timestamp is partial, regenerate the .pyc, else
# raise EOFError.
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:7])
self.import_(mapping['_temp'], '_temp')
def test(name, mapping, bc_path):
self.import_(mapping[name], name)
with open(bc_path, 'rb') as file:
self.assertGreater(len(file.read()), 8)
self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:7],
del_source=True)
with self.assertRaises(EOFError):
self.import_(mapping['_temp'], '_temp')
@source_util.writes_bytecode_files
def test_no_marshal(self):
# When there is only the magic number and timestamp, raise EOFError.
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: bc[:8])
with self.assertRaises(EOFError):
self.import_(mapping['_temp'], '_temp')
self._test_no_marshal()
@source_util.writes_bytecode_files
def test_bad_magic(self):
# When the magic number is different, the bytecode should be
# regenerated.
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
lambda bc: b'\x00\x00\x00\x00' + bc[4:])
self.import_(mapping['_temp'], '_temp')
with open(bc_path, 'rb') as bytecode_file:
self.assertEqual(bytecode_file.read(4), imp.get_magic())
def test_non_code_marshal(self):
self._test_non_code_marshal()
# XXX ImportError when sourceless
# [bad marshal]
@source_util.writes_bytecode_files
def test_bad_marshal(self):
# Bad marshal data should raise a ValueError.
self._test_bad_marshal()
# [bad timestamp]
@source_util.writes_bytecode_files
def test_bad_bytecode(self):
def test_old_timestamp(self):
# When the timestamp is older than the source, bytecode should be
# regenerated.
zeros = b'\x00\x00\x00\x00'
......@@ -240,23 +291,6 @@ class BadBytecodeTest(unittest.TestCase):
bytecode_file.seek(4)
self.assertEqual(bytecode_file.read(4), source_timestamp)
# [bad marshal]
@source_util.writes_bytecode_files
def test_bad_marshal(self):
# Bad marshal data should raise a ValueError.
with source_util.create_modules('_temp') as mapping:
bytecode_path = imp.cache_from_source(mapping['_temp'])
source_mtime = os.path.getmtime(mapping['_temp'])
source_timestamp = importlib._w_long(source_mtime)
source_util.ensure_bytecode_path(bytecode_path)
with open(bytecode_path, 'wb') as bytecode_file:
bytecode_file.write(imp.get_magic())
bytecode_file.write(source_timestamp)
bytecode_file.write(b'AAAA')
with self.assertRaises(ValueError):
self.import_(mapping['_temp'], '_temp')
self.assertTrue('_temp' not in sys.modules)
# [bytecode read-only]
@source_util.writes_bytecode_files
def test_read_only_bytecode(self):
......@@ -279,9 +313,57 @@ class BadBytecodeTest(unittest.TestCase):
os.chmod(bytecode_path, stat.S_IWUSR)
class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
loader = _bootstrap._SourcelessFileLoader
def test_empty_file(self):
def test(name, mapping, bytecode_path):
with self.assertRaises(ImportError):
self.import_(bytecode_path, name)
self._test_empty_file(test, del_source=True)
def test_partial_magic(self):
def test(name, mapping, bytecode_path):
with self.assertRaises(ImportError):
self.import_(bytecode_path, name)
self._test_partial_magic(test, del_source=True)
def test_magic_only(self):
def test(name, mapping, bytecode_path):
with self.assertRaises(EOFError):
self.import_(bytecode_path, name)
self._test_magic_only(test, del_source=True)
def test_bad_magic(self):
def test(name, mapping, bytecode_path):
with self.assertRaises(ImportError):
self.import_(bytecode_path, name)
self._test_bad_magic(test, del_source=True)
def test_partial_timestamp(self):
def test(name, mapping, bytecode_path):
with self.assertRaises(EOFError):
self.import_(bytecode_path, name)
self._test_partial_timestamp(test, del_source=True)
def test_no_marshal(self):
self._test_no_marshal(del_source=True)
def test_non_code_marshal(self):
self._test_non_code_marshal(del_source=True)
def test_main():
from test.support import run_unittest
run_unittest(SimpleTest, BadBytecodeTest)
run_unittest(SimpleTest,
SourceLoaderBadBytecodeTest,
SourcelessLoaderBadBytecodeTest
)
if __name__ == '__main__':
......
Lib/importlib/test/source/test_finder.py
View file @ 61b14251
......@@ -34,7 +34,9 @@ class FinderTests(abc.FinderTests):
"""
def import_(self, root, module):
finder = _bootstrap._PyPycFileFinder(root)
finder = _bootstrap._FileFinder(root,
_bootstrap._SourceFinderDetails(),
_bootstrap._SourcelessFinderDetails())
return finder.find_module(module)
def run_test(self, test, create=None, *, compile_=None, unlink=None):
......@@ -116,7 +118,7 @@ class FinderTests(abc.FinderTests):
# XXX This is not a blackbox test!
name = '_temp'
loader = self.run_test(name, {'{0}.__init__'.format(name), name})
self.assertTrue('__init__' in loader._base_path)
self.assertTrue('__init__' in loader.get_filename(name))
def test_failure(self):
......
Lib/importlib/test/source/test_path_hook.py
View file @ 61b14251
......@@ -8,9 +8,8 @@ class PathHookTest(unittest.TestCase):
"""Test the path hook for source."""
def test_success(self):
# XXX Only work on existing directories?
with source_util.create_modules('dummy') as mapping:
self.assertTrue(hasattr(_bootstrap._FileFinder(mapping['.root']),
self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']),
'find_module'))
......
Lib/importlib/test/source/test_source_encoding.py
View file @ 61b14251
......@@ -35,8 +35,8 @@ class EncodingTest(unittest.TestCase):
with source_util.create_modules(self.module_name) as mapping:
with open(mapping[self.module_name], 'wb') as file:
file.write(source)
loader = _bootstrap._PyPycFileLoader(self.module_name,
mapping[self.module_name], False)
loader = _bootstrap._SourceFileLoader(self.module_name,
mapping[self.module_name])
return loader.load_module(self.module_name)
def create_source(self, encoding):
......@@ -97,8 +97,8 @@ class LineEndingTest(unittest.TestCase):
with source_util.create_modules(module_name) as mapping:
with open(mapping[module_name], 'wb') as file:
file.write(source)
loader = _bootstrap._PyPycFileLoader(module_name,
mapping[module_name], False)
loader = _bootstrap._SourceFileLoader(module_name,
mapping[module_name])
return loader.load_module(module_name)
# [cr]
......
Lib/test/test_runpy.py
View file @ 61b14251
......@@ -6,7 +6,7 @@ import sys
import re
import tempfile
import py_compile
from test.support import forget, make_legacy_pyc, run_unittest, verbose
from test.support import forget, make_legacy_pyc, run_unittest, unload, verbose
from test.script_helper import (
make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
......@@ -174,6 +174,7 @@ class RunModuleTest(unittest.TestCase):
__import__(mod_name)
os.remove(mod_fname)
make_legacy_pyc(mod_fname)
unload(mod_name) # In case loader caches paths
if verbose: print("Running from compiled:", mod_name)
d2 = run_module(mod_name) # Read from bytecode
self.assertIn("x", d2)
......@@ -197,6 +198,7 @@ class RunModuleTest(unittest.TestCase):
__import__(mod_name)
os.remove(mod_fname)
make_legacy_pyc(mod_fname)
unload(mod_name) # In case loader caches paths
if verbose: print("Running from compiled:", pkg_name)
d2 = run_module(pkg_name) # Read from bytecode
self.assertIn("x", d2)
......@@ -252,6 +254,7 @@ from ..uncle.cousin import nephew
__import__(mod_name)
os.remove(mod_fname)
make_legacy_pyc(mod_fname)
unload(mod_name) # In case the loader caches paths
if verbose: print("Running from compiled:", mod_name)
d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
self.assertIn("__package__", d2)
......@@ -405,7 +408,11 @@ argv0 = sys.argv[0]
def test_main():
run_unittest(RunModuleCodeTest, RunModuleTest, RunPathTest)
run_unittest(
RunModuleCodeTest,
RunModuleTest,
RunPathTest
)
if __name__ == "__main__":
test_main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7