Skip to content

C
cpython
Commit 16475adc authored by Brett Cannon's avatar Brett Cannon
Browse files
Options

Issue #13959: Re-implement imp.load_source() in imp.py.

parent 4132368d
Show whitespace changes
Inline Side-by-side
Showing with 28 additions and 391 deletions
Lib/imp.py
View file @ 16475adc
......@@ -14,7 +14,7 @@ from _imp import (lock_held, acquire_lock, release_lock, reload,
from _imp import (get_magic, get_tag, get_suffixes, cache_from_source,
source_from_cache)
# Should be re-implemented here (and mostly deprecated)
from _imp import (find_module, load_compiled, load_source, NullImporter,
from _imp import (find_module, load_compiled, NullImporter,
SEARCH_ERROR, PY_SOURCE, PY_COMPILED, C_EXTENSION,
PY_RESOURCE, PKG_DIRECTORY, C_BUILTIN, PY_FROZEN,
PY_CODERESOURCE, IMP_HOOK)
......@@ -25,6 +25,33 @@ from importlib import _bootstrap
import os
class _LoadSourceCompatibility(_bootstrap._SourceFileLoader):
"""Compatibility support for implementing load_source()."""
def __init__(self, fullname, path, file=None):
super().__init__(fullname, path)
self.file = file
def get_data(self, path):
"""Gross hack to contort SourceFileLoader to deal w/ load_source()'s bad
API."""
if path == self._path:
with self.file:
# Technically should be returning bytes, but
# SourceLoader.get_code() just passed what is returned to
# compile() which can handle str. And converting to bytes would
# require figuring out the encoding to decode to and
# tokenize.detect_encoding() only accepts bytes.
return self.file.read()
else:
return super().get_data(path)
def load_source(name, pathname, file=None):
return _LoadSourceCompatibility(name, pathname, file).load_module(name)
def load_package(name, path):
if os.path.isdir(path):
extensions = _bootstrap._suffix_list(PY_SOURCE)
......
Python/import.c
View file @ 16475adc
......@@ -904,26 +904,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
}
/* Like strrchr(string, '/') but searches for the rightmost of either SEP
or ALTSEP, if the latter is defined.
*/
static Py_UCS4*
rightmost_sep(Py_UCS4 *s)
{
Py_UCS4 *found, c;
for (found = NULL; (c = *s); s++) {
if (c == SEP
#ifdef ALTSEP
|| c == ALTSEP
#endif
)
{
found = s;
}
}
return found;
}
/* Like rightmost_sep, but operate on unicode objects. */
static Py_ssize_t
rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end)
......@@ -1081,50 +1061,6 @@ make_source_pathname(PyObject *path)
return result;
}
/* Given a pathname for a Python source file, its time of last
modification, and a pathname for a compiled file, check whether the
compiled file represents the same version of the source. If so,
return a FILE pointer for the compiled file, positioned just after
the header; if not, return NULL.
Doesn't set an exception. */
static FILE *
check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname)
{
FILE *fp;
long magic;
long pyc_mtime;
long pyc_size;
fp = _Py_fopen(cpathname, "rb");
if (fp == NULL)
return NULL;
magic = PyMarshal_ReadLongFromFile(fp);
if (magic != pyc_magic) {
if (Py_VerboseFlag)
PySys_FormatStderr("# %R has bad magic\n", cpathname);
fclose(fp);
return NULL;
}
pyc_mtime = PyMarshal_ReadLongFromFile(fp);
if (pyc_mtime != srcstat->st_mtime) {
if (Py_VerboseFlag)
PySys_FormatStderr("# %R has bad mtime\n", cpathname);
fclose(fp);
return NULL;
}
pyc_size = PyMarshal_ReadLongFromFile(fp);
if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) {
if (Py_VerboseFlag)
PySys_FormatStderr("# %R has bad size\n", cpathname);
fclose(fp);
return NULL;
}
if (Py_VerboseFlag)
PySys_FormatStderr("# %R matches %R\n", cpathname, pathname);
return fp;
}
/* Read a code object from a file and check it for validity */
......@@ -1178,238 +1114,6 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp)
return m;
}
/* Parse a source file and return the corresponding code object */
static PyCodeObject *
parse_source_module(PyObject *pathname, FILE *fp)
{
PyCodeObject *co;
PyObject *pathbytes;
mod_ty mod;
PyCompilerFlags flags;
PyArena *arena;
pathbytes = PyUnicode_EncodeFSDefault(pathname);
if (pathbytes == NULL)
return NULL;
arena = PyArena_New();
if (arena == NULL) {
Py_DECREF(pathbytes);
return NULL;
}
flags.cf_flags = 0;
mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL,
Py_file_input, 0, 0, &flags,
NULL, arena);
if (mod != NULL)
co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena);
else
co = NULL;
Py_DECREF(pathbytes);
PyArena_Free(arena);
return co;
}
/* Write a compiled module to a file, placing the time of last
modification of its source into the header.
Errors are ignored, if a write error occurs an attempt is made to
remove the file. */
static void
write_compiled_module(PyCodeObject *co, PyObject *cpathname,
struct stat *srcstat)
{
Py_UCS4 *cpathname_ucs4;
FILE *fp;
time_t mtime = srcstat->st_mtime;
long size = srcstat->st_size & 0xFFFFFFFF;
PyObject *cpathname_tmp;
#ifdef MS_WINDOWS /* since Windows uses different permissions */
mode_t mode = srcstat->st_mode & ~S_IEXEC;
wchar_t *wdirname, *wpathname, *wpathname_tmp;
#else
mode_t dirmode = (srcstat->st_mode |
S_IXUSR | S_IXGRP | S_IXOTH |
S_IWUSR | S_IWGRP | S_IWOTH);
PyObject *dirbytes;
PyObject *cpathbytes, *cpathbytes_tmp;
#endif
int fd;
PyObject *dirname;
Py_UCS4 *dirsep;
int res, ok;
/* Ensure that the __pycache__ directory exists. */
cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname);
if (!cpathname_ucs4)
return;
dirsep = rightmost_sep(cpathname_ucs4);
if (dirsep == NULL) {
if (Py_VerboseFlag)
PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname);
return;
}
dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
cpathname_ucs4,
dirsep - cpathname_ucs4);
PyMem_Free(cpathname_ucs4);
if (dirname == NULL) {
PyErr_Clear();
return;
}
#ifdef MS_WINDOWS
wdirname = PyUnicode_AsUnicode(dirname);
if (wdirname == NULL) {
PyErr_Clear();
return;
}
res = CreateDirectoryW(wdirname, NULL);
ok = (res != 0);
if (!ok && GetLastError() == ERROR_ALREADY_EXISTS)
ok = 1;
#else
dirbytes = PyUnicode_EncodeFSDefault(dirname);
if (dirbytes == NULL) {
PyErr_Clear();
return;
}
res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode);
Py_DECREF(dirbytes);
if (0 <= res)
ok = 1;
else
ok = (errno == EEXIST);
#endif
if (!ok) {
if (Py_VerboseFlag)
PySys_FormatStderr("# cannot create cache dir %R\n", dirname);
Py_DECREF(dirname);
return;
}
Py_DECREF(dirname);
/* We first write to a tmp file and then take advantage
of atomic renaming (which *should* be true even under Windows).
As in importlib, we use id(something) to generate a pseudo-random
filename. mkstemp() can't be used since it doesn't allow specifying
the file access permissions.
*/
cpathname_tmp = PyUnicode_FromFormat("%U.%zd",
cpathname, (Py_ssize_t) co);
if (cpathname_tmp == NULL) {
PyErr_Clear();
return;
}
#ifdef MS_WINDOWS
wpathname = PyUnicode_AsUnicode(cpathname);
if (wpathname == NULL) {
PyErr_Clear();
return;
}
wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp);
if (wpathname_tmp == NULL) {
PyErr_Clear();
return;
}
(void)DeleteFileW(wpathname_tmp);
fd = _wopen(wpathname_tmp,
O_EXCL | O_CREAT | O_WRONLY | O_BINARY,
mode);
if (0 <= fd)
fp = fdopen(fd, "wb");
else
fp = NULL;
#else
cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp);
Py_DECREF(cpathname_tmp);
if (cpathbytes_tmp == NULL) {
PyErr_Clear();
return;
}
cpathbytes = PyUnicode_EncodeFSDefault(cpathname);
if (cpathbytes == NULL) {
PyErr_Clear();
return;
}
fd = open(PyBytes_AS_STRING(cpathbytes_tmp),
O_CREAT | O_EXCL | O_WRONLY, 0666);
if (0 <= fd)
fp = fdopen(fd, "wb");
else
fp = NULL;
#endif
if (fp == NULL) {
if (Py_VerboseFlag)
PySys_FormatStderr(
"# can't create %R\n", cpathname);
#ifdef MS_WINDOWS
Py_DECREF(cpathname_tmp);
#else
Py_DECREF(cpathbytes);
Py_DECREF(cpathbytes_tmp);
#endif
return;
}
PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
/* First write a 0 for mtime and size */
PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
fflush(fp);
/* Now write the true mtime and size (as 32-bit fields) */
fseek(fp, 4L, 0);
assert(mtime <= 0xFFFFFFFF);
PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION);
if (fflush(fp) != 0 || ferror(fp)) {
if (Py_VerboseFlag)
PySys_FormatStderr("# can't write %R\n", cpathname);
/* Don't keep partial file */
fclose(fp);
#ifdef MS_WINDOWS
(void)DeleteFileW(wpathname_tmp);
Py_DECREF(cpathname_tmp);
#else
(void) unlink(PyBytes_AS_STRING(cpathbytes_tmp));
Py_DECREF(cpathbytes);
Py_DECREF(cpathbytes_tmp);
#endif
return;
}
fclose(fp);
/* Do a (hopefully) atomic rename */
#ifdef MS_WINDOWS
if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) {
if (Py_VerboseFlag)
PySys_FormatStderr("# can't write %R\n", cpathname);
/* Don't keep tmp file */
(void) DeleteFileW(wpathname_tmp);
Py_DECREF(cpathname_tmp);
return;
}
Py_DECREF(cpathname_tmp);
#else
if (rename(PyBytes_AS_STRING(cpathbytes_tmp),
PyBytes_AS_STRING(cpathbytes))) {
if (Py_VerboseFlag)
PySys_FormatStderr("# can't write %R\n", cpathname);
/* Don't keep tmp file */
unlink(PyBytes_AS_STRING(cpathbytes_tmp));
Py_DECREF(cpathbytes);
Py_DECREF(cpathbytes_tmp);
return;
}
Py_DECREF(cpathbytes);
Py_DECREF(cpathbytes_tmp);
#endif
if (Py_VerboseFlag)
PySys_FormatStderr("# wrote %R\n", cpathname);
}
static void
update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname)
{
......@@ -1474,76 +1178,6 @@ imp_fix_co_filename(PyObject *self, PyObject *args)
Py_RETURN_NONE;
}
/* Load a source module from a given file and return its module
object WITH INCREMENTED REFERENCE COUNT. If there's a matching
byte-compiled file, use that instead. */
static PyObject *
load_source_module(PyObject *name, PyObject *pathname, FILE *fp)
{
struct stat st;
FILE *fpc;
PyObject *cpathname = NULL, *cpathbytes = NULL;
PyCodeObject *co;
PyObject *m = NULL;
if (fstat(fileno(fp), &st) != 0) {
PyErr_Format(PyExc_RuntimeError,
"unable to get file status from %R",
pathname);
goto error;
}
if (sizeof st.st_mtime > 4) {
/* Python's .pyc timestamp handling presumes that the timestamp fits
in 4 bytes. Since the code only does an equality comparison,
ordering is not important and we can safely ignore the higher bits
(collisions are extremely unlikely).
*/
st.st_mtime &= 0xFFFFFFFF;
}
if (PyUnicode_READY(pathname) < 0)
return NULL;
cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag);
if (cpathname != NULL)
fpc = check_compiled_module(pathname, &st, cpathname);
else
fpc = NULL;
if (fpc) {
co = read_compiled_module(cpathname, fpc);
fclose(fpc);
if (co == NULL)
goto error;
update_compiled_module(co, pathname);
if (Py_VerboseFlag)
PySys_FormatStderr("import %U # precompiled from %R\n",
name, cpathname);
m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
cpathname, cpathname);
}
else {
co = parse_source_module(pathname, fp);
if (co == NULL)
goto error;
if (Py_VerboseFlag)
PySys_FormatStderr("import %U # from %R\n",
name, pathname);
if (cpathname != NULL) {
PyObject *ro = PySys_GetObject("dont_write_bytecode");
if (ro == NULL || !PyObject_IsTrue(ro))
write_compiled_module(co, cpathname, &st);
}
m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
pathname, cpathname);
}
Py_DECREF(co);
error:
Py_XDECREF(cpathbytes);
Py_XDECREF(cpathname);
return m;
}
/* Get source file -> unicode or None
* Returns the path to the py file if available, else the given path
......@@ -3426,29 +3060,6 @@ imp_load_dynamic(PyObject *self, PyObject *args)
#endif /* HAVE_DYNAMIC_LOADING */
static PyObject *
imp_load_source(PyObject *self, PyObject *args)
{
PyObject *name, *pathname;
PyObject *fob = NULL;
PyObject *m;
FILE *fp;
if (!PyArg_ParseTuple(args, "UO&|O:load_source",
&name,
PyUnicode_FSDecoder, &pathname,
&fob))
return NULL;
fp = get_file(pathname, fob, "r");
if (fp == NULL) {
Py_DECREF(pathname);
return NULL;
}
m = load_source_module(name, pathname, fp);
Py_DECREF(pathname);
fclose(fp);
return m;
}
static PyObject *
imp_reload(PyObject *self, PyObject *v)
{
......@@ -3600,7 +3211,6 @@ static PyMethodDef imp_methods[] = {
#ifdef HAVE_DYNAMIC_LOADING
{"load_dynamic", imp_load_dynamic, METH_VARARGS},
#endif
{"load_source", imp_load_source, METH_VARARGS},
{"_fix_co_filename", imp_fix_co_filename, METH_VARARGS},
{NULL, NULL} /* sentinel */
};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7