Skip to content

C
cpython
Commit 3c536491 authored by Phillip J. Eby's avatar Phillip J. Eby
Browse files
Options

First phase of refactoring for runpy, pkgutil, pydoc, and setuptools 

to share common PEP 302 support code, as described here:

http://mail.python.org/pipermail/python-dev/2006-April/063724.html

This revision strips all the PEP 302 emulation code from runpy,
replacing it with published API classes and functions in pkgutil,
mostly using setuptools' implementation of common functionality,
but adding features from runpy, and doing some refactoring to make
the layer pydoc needs easier to implement on top of this.

One step down, four to go, although step #4 (adding C versions of
the new APIs to 'imp') may not be able to make it in time for
alpha 2.  We'll see how that goes.
parent 9af5e955
Hide whitespace changes
Inline Side-by-side
Showing with 280 additions and 339 deletions
Lib/pkgutil.py
View file @ 3c536491
"""Utilities to support packages."""
# NOTE: This module must remain compatible with Python 2.3, as it is shared
# by setuptools for distribution with Python 2.3 and up.
import os
import sys
import imp
import os.path
from types import ModuleType
__all__ = [
'get_importer', 'iter_importers', 'get_loader', 'find_loader',
'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
]
def read_code(stream):
# This helper is needed in order for the PEP 302 emulation to
# correctly handle compiled files
import marshal
magic = stream.read(4)
if magic != imp.get_magic():
return None
stream.read(4) # Skip timestamp
return marshal.load(stream)
class ImpImporter:
"""PEP 302 Importer that wraps Python's "classic" import algorithm
ImpImporter(dirname) produces a PEP 302 importer that searches that
directory. ImpImporter(None) produces a PEP 302 importer that searches
the current sys.path, plus any modules that are frozen or built-in.
Note that ImpImporter does not currently support being used by placement
on sys.meta_path.
"""
def __init__(self, path=None):
self.path = path
def find_module(self, fullname, path=None):
# Note: we ignore 'path' argument since it is only used via meta_path
subname = fullname.split(".")[-1]
if subname != fullname and self.path is None:
return None
if self.path is None:
path = None
else:
path = [self.path]
try:
file, filename, etc = imp.find_module(subname, path)
except ImportError:
return None
return ImpLoader(fullname, file, filename, etc)
class ImpLoader:
"""PEP 302 Loader that wraps Python's "classic" import algorithm
"""
code = source = None
def __init__(self, fullname, file, filename, etc):
self.file = file
self.filename = filename
self.fullname = fullname
self.etc = etc
def load_module(self, fullname):
self._reopen()
try:
mod = imp.load_module(fullname, self.file, self.filename, self.etc)
finally:
if self.file:
self.file.close()
# Note: we don't set __loader__ because we want the module to look
# normal; i.e. this is just a wrapper for standard import machinery
return mod
def get_data(self, pathname):
return open(pathname, "rb").read()
def _reopen(self):
if self.file and self.file.closed:
if mod_type==imp.PY_SOURCE:
self.file = open(self.filename, 'rU')
elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
self.file = open(self.filename, 'rb')
def _fix_name(self, fullname):
if fullname is None:
fullname = self.fullname
elif fullname != self.fullname:
raise ImportError("Loader for module %s cannot handle "
"module %s" % (self.fullname, fullname))
return fullname
def is_package(self):
return self.etc[2]==imp.PKG_DIRECTORY
def get_code(self, fullname=None):
fullname = self._fix_name(fullname)
if self.code is None:
mod_type = self.etc[2]
if mod_type==imp.PY_SOURCE:
source = self.get_source(fullname)
self.code = compile(source, self.filename, 'exec')
elif mod_type==imp.PY_COMPILED:
self._reopen()
try:
self.code = read_code(self.file)
finally:
self.file.close()
elif mod_type==imp.PKG_DIRECTORY:
self.code = self._get_delegate().get_code()
return self.code
def get_source(self, fullname=None):
fullname = self._fix_name(fullname)
if self.source is None:
mod_type = self.etc[2]
if mod_type==imp.PY_SOURCE:
self._reopen()
try:
self.source = self.file.read()
finally:
self.file.close()
elif mod_type==imp.PY_COMPILED:
if os.path.exists(self.filename[:-1]):
f = open(self.filename[:-1], 'rU')
self.source = f.read()
f.close()
elif mod_type==imp.PKG_DIRECTORY:
self.source = self._get_delegate().get_source()
return self.source
def _get_delegate(self):
return ImpImporter(self.filename).find_module('__init__')
def get_filename(self, fullname=None):
fullname = self._fix_name(fullname)
mod_type = self.etc[2]
if self.etc[2]==imp.PKG_DIRECTORY:
return self._get_delegate().get_filename()
elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
return self.filename
return None
def get_importer(path_item):
"""Retrieve a PEP 302 importer for the given path item
The returned importer is cached in sys.path_importer_cache
if it was newly created by a path hook.
If there is no importer, a wrapper around the basic import
machinery is returned. This wrapper is never inserted into
the importer cache (None is inserted instead).
The cache (or part of it) can be cleared manually if a
rescan of sys.path_hooks is necessary.
"""
try:
importer = sys.path_importer_cache[path_item]
except KeyError:
for path_hook in sys.path_hooks:
try:
importer = path_hook(path_item)
break
except ImportError:
pass
else:
importer = None
sys.path_importer_cache.setdefault(path_item,importer)
if importer is None:
try:
importer = ImpImporter(path_item)
except ImportError:
pass
return importer
def iter_importers(fullname):
"""Yield PEP 302 importers for the given module name
If fullname contains a '.', the importers will be for the package
containing fullname, otherwise they will be importers for sys.meta_path,
sys.path, and Python's "classic" import machinery, in that order. If
the named module is in a package, that package is imported as a side
effect of invoking this function.
Non PEP 302 mechanisms (e.g. the Windows registry) used by the
standard import machinery to find files in alternative locations
are partially supported, but are searched AFTER sys.path. Normally,
these locations are searched BEFORE sys.path, preventing sys.path
entries from shadowing them.
For this to cause a visible difference in behaviour, there must
be a module or package name that is accessible via both sys.path
and one of the non PEP 302 file system mechanisms. In this case,
the emulation will find the former version, while the builtin
import mechanism will find the latter.
Items of the following types can be affected by this discrepancy:
imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
"""
if fullname.startswith('.'):
raise ImportError("Relative module names not supported")
if '.' in fullname:
# Get the containing package's __path__
pkg = '.'.join(fullname.split('.')[:-1])
if pkg not in sys.modules:
__import__(pkg)
path = getattr(sys.modules[pkg],'__path__',None) or []
else:
for importer in sys.meta_path:
yield importer
path = sys.path
for item in path:
yield get_importer(item)
if '.' not in fullname:
yield ImpImporter()
def get_loader(module_or_name):
"""Get a PEP 302 "loader" object for module_or_name
If the module or package is accessible via the normal import
mechanism, a wrapper around the relevant part of that machinery
is returned. Returns None if the module cannot be found or imported.
If the named module is not already imported, its containing package
(if any) is imported, in order to establish the package __path__.
This function uses iter_importers(), and is thus subject to the same
limitations regarding platform-specific special import locations such
as the Windows registry.
"""
if module_or_name in sys.modules:
module_or_name = sys.modules[module_or_name]
if isinstance(module_or_name, ModuleType):
module = module_or_name
loader = getattr(module,'__loader__',None)
if loader is not None:
return loader
fullname = module.__name__
else:
fullname = module_or_name
return find_loader(fullname)
def find_loader(fullname):
"""Find a PEP 302 "loader" object for fullname
If fullname contains dots, path must be the containing package's __path__.
Returns None if the module cannot be found or imported. This function uses
iter_importers(), and is thus subject to the same limitations regarding
platform-specific special import locations such as the Windows registry.
"""
for importer in iter_importers(fullname):
loader = importer.find_module(fullname)
if loader is not None:
return loader
return None
def extend_path(path, name):
"""Extend a package's path.
......
Lib/runpy.py
View file @ 3c536491
......@@ -11,349 +11,15 @@ importers when locating support scripts as well as when importing modules.
import sys
import imp
try:
from imp import get_loader
except ImportError:
from pkgutil import get_loader
__all__ = [
"run_module",
]
try:
_get_loader = imp.get_loader
except AttributeError:
# get_loader() is not provided by the imp module, so emulate it
# as best we can using the PEP 302 import machinery exposed since
# Python 2.3. The emulation isn't perfect, but the differences
# in the way names are shadowed shouldn't matter in practice.
import os.path
import marshal # Handle compiled Python files
# This helper is needed in order for the PEP 302 emulation to
# correctly handle compiled files
def _read_compiled_file(compiled_file):
magic = compiled_file.read(4)
if magic != imp.get_magic():
return None
try:
compiled_file.read(4) # Skip timestamp
return marshal.load(compiled_file)
except Exception:
return None
class _AbsoluteImporter(object):
"""PEP 302 importer wrapper for top level import machinery"""
def find_module(self, mod_name, path=None):
if path is not None:
return None
try:
file, filename, mod_info = imp.find_module(mod_name)
except ImportError:
return None
suffix, mode, mod_type = mod_info
if mod_type == imp.PY_SOURCE:
loader = _SourceFileLoader(mod_name, file,
filename, mod_info)
elif mod_type == imp.PY_COMPILED:
loader = _CompiledFileLoader(mod_name, file,
filename, mod_info)
elif mod_type == imp.PKG_DIRECTORY:
loader = _PackageDirLoader(mod_name, file,
filename, mod_info)
elif mod_type == imp.C_EXTENSION:
loader = _FileSystemLoader(mod_name, file,
filename, mod_info)
else:
loader = _BasicLoader(mod_name, file,
filename, mod_info)
return loader
class _FileSystemImporter(object):
"""PEP 302 importer wrapper for filesystem based imports"""
def __init__(self, path_item=None):
if path_item is not None:
if path_item != '' and not os.path.isdir(path_item):
raise ImportError("%s is not a directory" % path_item)
self.path_dir = path_item
else:
raise ImportError("Filesystem importer requires "
"a directory name")
def find_module(self, mod_name, path=None):
if path is not None:
return None
path_dir = self.path_dir
if path_dir == '':
path_dir = os.getcwd()
sub_name = mod_name.rsplit(".", 1)[-1]
try:
file, filename, mod_info = imp.find_module(sub_name,
[path_dir])
except ImportError:
return None
if not filename.startswith(path_dir):
return None
suffix, mode, mod_type = mod_info
if mod_type == imp.PY_SOURCE:
loader = _SourceFileLoader(mod_name, file,
filename, mod_info)
elif mod_type == imp.PY_COMPILED:
loader = _CompiledFileLoader(mod_name, file,
filename, mod_info)
elif mod_type == imp.PKG_DIRECTORY:
loader = _PackageDirLoader(mod_name, file,
filename, mod_info)
elif mod_type == imp.C_EXTENSION:
loader = _FileSystemLoader(mod_name, file,
filename, mod_info)
else:
loader = _BasicLoader(mod_name, file,
filename, mod_info)
return loader
class _BasicLoader(object):
"""PEP 302 loader wrapper for top level import machinery"""
def __init__(self, mod_name, file, filename, mod_info):
self.mod_name = mod_name
self.file = file
self.filename = filename
self.mod_info = mod_info
def _fix_name(self, mod_name):
if mod_name is None:
mod_name = self.mod_name
elif mod_name != self.mod_name:
raise ImportError("Loader for module %s cannot handle "
"module %s" % (self.mod_name, mod_name))
return mod_name
def load_module(self, mod_name=None):
mod_name = self._fix_name(mod_name)
mod = imp.load_module(mod_name, self.file,
self.filename, self.mod_info)
mod.__loader__ = self # for introspection
return mod
def get_code(self, mod_name=None):
return None
def get_source(self, mod_name=None):
return None
def is_package(self, mod_name=None):
return False
def close(self):
if self.file:
self.file.close()
def __del__(self):
self.close()
class _FileSystemLoader(_BasicLoader):
"""PEP 302 loader wrapper for filesystem based imports"""
def get_code(self, mod_name=None):
mod_name = self._fix_name(mod_name)
return self._get_code(mod_name)
def get_data(self, pathname):
return open(pathname, "rb").read()
def get_filename(self, mod_name=None):
mod_name = self._fix_name(mod_name)
return self._get_filename(mod_name)
def get_source(self, mod_name=None):
mod_name = self._fix_name(mod_name)
return self._get_source(mod_name)
def is_package(self, mod_name=None):
mod_name = self._fix_name(mod_name)
return self._is_package(mod_name)
def _get_code(self, mod_name):
return None
def _get_filename(self, mod_name):
return self.filename
def _get_source(self, mod_name):
return None
def _is_package(self, mod_name):
return False
class _PackageDirLoader(_FileSystemLoader):
"""PEP 302 loader wrapper for PKG_DIRECTORY directories"""
def _is_package(self, mod_name):
return True
class _SourceFileLoader(_FileSystemLoader):
"""PEP 302 loader wrapper for PY_SOURCE modules"""
def _get_code(self, mod_name):
return compile(self._get_source(mod_name),
self.filename, 'exec')
def _get_source(self, mod_name):
f = self.file
f.seek(0)
return f.read()
class _CompiledFileLoader(_FileSystemLoader):
"""PEP 302 loader wrapper for PY_COMPILED modules"""
def _get_code(self, mod_name):
f = self.file
f.seek(0)
return _read_compiled_file(f)
def _get_importer(path_item):
"""Retrieve a PEP 302 importer for the given path item
The returned importer is cached in sys.path_importer_cache
if it was newly created by a path hook.
If there is no importer, a wrapper around the basic import
machinery is returned. This wrapper is never inserted into
the importer cache (None is inserted instead).
The cache (or part of it) can be cleared manually if a
rescan of sys.path_hooks is necessary.
"""
try:
importer = sys.path_importer_cache[path_item]
except KeyError:
for path_hook in sys.path_hooks:
try:
importer = path_hook(path_item)
break
except ImportError:
pass
else:
importer = None
sys.path_importer_cache[path_item] = importer
if importer is None:
try:
importer = _FileSystemImporter(path_item)
except ImportError:
pass
return importer
def _get_path_loader(mod_name, path=None):
"""Retrieve a PEP 302 loader using a path importer"""
if path is None:
path = sys.path
absolute_loader = _AbsoluteImporter().find_module(mod_name)
if isinstance(absolute_loader, _FileSystemLoader):
# Found in filesystem, so scan path hooks
# before accepting this one as the right one
loader = None
else:
# Not found in filesystem, so use top-level loader
loader = absolute_loader
else:
loader = absolute_loader = None
if loader is None:
for path_item in path:
importer = _get_importer(path_item)
if importer is not None:
loader = importer.find_module(mod_name)
if loader is not None:
# Found a loader for our module
break
else:
# No path hook found, so accept the top level loader
loader = absolute_loader
return loader
def _get_package(pkg_name):
"""Retrieve a named package"""
pkg = __import__(pkg_name)
sub_pkg_names = pkg_name.split(".")
for sub_pkg in sub_pkg_names[1:]:
pkg = getattr(pkg, sub_pkg)
return pkg
def _get_loader(mod_name, path=None):
"""Retrieve a PEP 302 loader for the given module or package
If the module or package is accessible via the normal import
mechanism, a wrapper around the relevant part of that machinery
is returned.
Non PEP 302 mechanisms (e.g. the Windows registry) used by the
standard import machinery to find files in alternative locations
are partially supported, but are searched AFTER sys.path. Normally,
these locations are searched BEFORE sys.path, preventing sys.path
entries from shadowing them.
For this to cause a visible difference in behaviour, there must
be a module or package name that is accessible via both sys.path
and one of the non PEP 302 file system mechanisms. In this case,
the emulation will find the former version, while the builtin
import mechanism will find the latter.
Items of the following types can be affected by this discrepancy:
imp.C_EXTENSION
imp.PY_SOURCE
imp.PY_COMPILED
imp.PKG_DIRECTORY
"""
try:
loader = sys.modules[mod_name].__loader__
except (KeyError, AttributeError):
loader = None
if loader is None:
imp.acquire_lock()
try:
# Module not in sys.modules, or uses an unhooked loader
parts = mod_name.rsplit(".", 1)
if len(parts) == 2:
# Sub package, so use parent package's path
pkg_name, sub_name = parts
if pkg_name and pkg_name[0] != '.':
if path is not None:
raise ImportError("Path argument must be None "
"for a dotted module name")
pkg = _get_package(pkg_name)
try:
path = pkg.__path__
except AttributeError:
raise ImportError(pkg_name +
" is not a package")
else:
raise ImportError("Relative import syntax is not "
"supported by _get_loader()")
else:
# Top level module, so stick with default path
sub_name = mod_name
for importer in sys.meta_path:
loader = importer.find_module(mod_name, path)
if loader is not None:
# Found a metahook to handle the module
break
else:
# Handling via the standard path mechanism
loader = _get_path_loader(mod_name, path)
finally:
imp.release_lock()
return loader
# This helper is needed due to a missing component in the PEP 302
# loader protocol (specifically, "get_filename" is non-standard)
def _get_filename(loader, mod_name):
try:
get_filename = loader.get_filename
except AttributeError:
return None
else:
return get_filename(mod_name)
# ------------------------------------------------------------
# Done with the import machinery emulation, on with the code!
def _run_code(code, run_globals, init_globals,
mod_name, mod_fname, mod_loader):
......@@ -399,13 +65,24 @@ def _run_module_code(code, init_globals=None,
mod_name, mod_fname, mod_loader)
# This helper is needed due to a missing component in the PEP 302
# loader protocol (specifically, "get_filename" is non-standard)
def _get_filename(loader, mod_name):
try:
get_filename = loader.get_filename
except AttributeError:
return None
else:
return get_filename(mod_name)
def run_module(mod_name, init_globals=None,
run_name=None, alter_sys=False):
"""Execute a module's code without importing it
Returns the resulting top level namespace dictionary
"""
loader = _get_loader(mod_name)
loader = get_loader(mod_name)
if loader is None:
raise ImportError("No module named " + mod_name)
code = loader.get_code(mod_name)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7