Commit 0fc8c3f3 authored by Marius Wachtler's avatar Marius Wachtler

Add the zipimport module and add sys.path_hook support

parent 1b0af1c7
......@@ -292,9 +292,9 @@ STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
STDLIB_RELEASE_OBJS := stdlib.release.bc.o
ASM_SRCS := $(wildcard src/runtime/*.S)
STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c $(EXTRA_STDMODULE_SRCS)
STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c zipimport.c $(EXTRA_STDMODULE_SRCS)
STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c iterobject.c $(EXTRA_STDOBJECT_SRCS)
STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c $(EXTRA_STDPYTHON_SRCS)
STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c marshal.c $(EXTRA_STDPYTHON_SRCS)
FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))
# The stdlib objects have slightly longer dependency chains,
......
......@@ -15,13 +15,13 @@ endforeach(STDLIB_FILE)
add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
# compile specified files in from_cpython/Modules
file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c bufferedio.c bytesio.c fileio.c iobase.c _iomodule.c stringio.c textio.c)
file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c bufferedio.c bytesio.c fileio.c iobase.c _iomodule.c stringio.c textio.c zipimport.c)
# compile specified files in from_cpython/Objects
file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c iterobject.c)
# compile specified files in from_cpython/Python
file(GLOB_RECURSE STDPYTHON_SRCS Python getargs.c pyctype.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c)
file(GLOB_RECURSE STDPYTHON_SRCS Python getargs.c pyctype.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c marshal.c)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing")
add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS})
// This file is originally from CPython 2.7, with modifications for Pyston
/* Interface for marshal.c */
......@@ -9,15 +10,15 @@ extern "C" {
#define Py_MARSHAL_VERSION 2
PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);
PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int);
PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int) PYSTON_NOEXCEPT;
PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int) PYSTON_NOEXCEPT;
PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *);
PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *);
PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *);
PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *);
PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(char *, Py_ssize_t);
PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *) PYSTON_NOEXCEPT;
PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(char *, Py_ssize_t) PYSTON_NOEXCEPT;
#ifdef __cplusplus
}
......
......@@ -4,6 +4,8 @@
#include "marshal.h"
#include <time.h>
// Pyston change:
#include <sys/stat.h>
#define IS_SOURCE 0x0
#define IS_BYTECODE 0x1
......@@ -46,10 +48,10 @@ static PyObject *zip_directory_cache = NULL;
/* forward decls */
static PyObject *read_directory(char *archive);
static PyObject *get_data(char *archive, PyObject *toc_entry);
static PyObject *get_module_code(ZipImporter *self, char *fullname,
int *p_ispackage, char **p_modpath);
#define ZipImporter_Check(op) PyObject_TypeCheck(op, &ZipImporter_Type)
......@@ -956,6 +958,8 @@ eq_mtime(time_t t1, time_t t2)
return d <= 1;
}
// Pyston change: We don't support bytecode archives yet
#if 0
/* Given the contents of a .py[co] file in a buffer, unmarshal the data
and return the code object. Return None if it the magic word doesn't
match (we do this instead of raising an exception as we fall back
......@@ -1103,6 +1107,7 @@ get_mtime_of_source(ZipImporter *self, char *path)
path[lastchar] = savechar;
return mtime;
}
#endif // Pyston change
/* Return the code object for the module named by 'fullname' from the
Zip archive as a new reference. */
......@@ -1123,6 +1128,11 @@ get_code_from_data(ZipImporter *self, int ispackage, int isbytecode,
modpath = PyString_AsString(PyTuple_GetItem(toc_entry, 0));
// Pyston change: Just return the source code for now
#if 1
assert(!isbytecode);
return data;
#else // Pyston change
if (isbytecode) {
code = unmarshal_code(modpath, data, mtime);
}
......@@ -1131,6 +1141,7 @@ get_code_from_data(ZipImporter *self, int ispackage, int isbytecode,
}
Py_DECREF(data);
return code;
#endif // Pyston change
}
/* Get the code object associated with the module specified by
......@@ -1164,8 +1175,14 @@ get_module_code(ZipImporter *self, char *fullname,
int ispackage = zso->type & IS_PACKAGE;
int isbytecode = zso->type & IS_BYTECODE;
if (isbytecode)
mtime = get_mtime_of_source(self, path);
if (isbytecode) {
// Pyston change: We don't support bytecode archives currently
// mtime = get_mtime_of_source(self, path);
PyErr_Format(ZipImportError, "Pyston can't load bytecode from archives yet. ' '%.200s'", fullname);
return NULL;
}
if (p_ispackage != NULL)
*p_ispackage = ispackage;
code = get_code_from_data(self, ispackage,
......@@ -1254,4 +1271,8 @@ initzipimport(void)
if (PyModule_AddObject(mod, "_zip_directory_cache",
zip_directory_cache) < 0)
return;
// Pyston change: register zip module import hook
PyObject* zipimporter = PyObject_GetAttrString(mod, "zipimporter");
PyList_Append(PySys_GetObject("path_hooks"), zipimporter);
}
// This file is originally from CPython 2.7, with modifications for Pyston
/* Write Python objects to files and read them back.
This is intended for writing and reading compiled Python code only;
......@@ -7,6 +8,9 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
// Pyston change: not yet ported
#if 0
#include "longintrepr.h"
#include "code.h"
#include "marshal.h"
......@@ -48,6 +52,7 @@
#define WFERR_UNMARSHALLABLE 1
#define WFERR_NESTEDTOODEEP 2
#define WFERR_NOMEMORY 3
#endif // Pyston change
typedef struct {
FILE *fp;
......@@ -61,6 +66,8 @@ typedef struct {
int version;
} WFILE;
// Pyston change: not yet ported
#if 0
#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
else w_more(c, p)
......@@ -479,6 +486,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
w_object(x, &wf);
Py_XDECREF(wf.strings);
}
#endif // Pyston change
typedef WFILE RFILE; /* Same struct with different invariants */
......@@ -565,6 +573,8 @@ r_long64(RFILE *p)
#endif
}
// Pyston change: not yet ported
#if 0
static PyObject *
r_PyLong(RFILE *p)
{
......@@ -624,7 +634,6 @@ r_PyLong(RFILE *p)
return NULL;
}
static PyObject *
r_object(RFILE *p)
{
......@@ -1047,7 +1056,6 @@ r_object(RFILE *p)
code, consts, names, varnames,
freevars, cellvars, filename, name,
firstlineno, lnotab);
code_error:
Py_XDECREF(code);
Py_XDECREF(consts);
......@@ -1088,6 +1096,7 @@ read_object(RFILE *p)
PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
return v;
}
#endif // Pyston change
int
PyMarshal_ReadShortFromFile(FILE *fp)
......@@ -1123,6 +1132,8 @@ getfilesize(FILE *fp)
}
#endif
// Pyston change: not yet ported
#if 0
/* If we can get the size of the file up-front, and it's reasonably small,
* read it in one gulp and delegate to ...FromString() instead. Much quicker
* than reading a byte at a time from file; speeds .pyc imports.
......@@ -1410,3 +1421,4 @@ PyMarshal_Init(void)
return;
PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
}
#endif // Pyston change
......@@ -75,5 +75,7 @@ bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;
extern "C" {
int Py_IgnoreEnvironmentFlag = 1;
int Py_OptimizeFlag = 0;
int Py_VerboseFlag = 0;
}
}
......@@ -275,6 +275,8 @@ void setupSys() {
"getfilesystemencoding"));
sys_module->giveAttr("meta_path", new BoxedList());
sys_module->giveAttr("path_hooks", new BoxedList());
sys_module->giveAttr("path_importer_cache", new BoxedDict());
// TODO: should configure this in a better way
sys_module->giveAttr("prefix", boxStrConstant("/usr"));
......
......@@ -30,6 +30,7 @@ static const std::string all_str("__all__");
static const std::string name_str("__name__");
static const std::string path_str("__path__");
static const std::string package_str("__package__");
static BoxedClass* null_importer_cls;
BoxedModule* createAndRunModule(const std::string& name, const std::string& fn) {
BoxedModule* module = createModule(name, fn);
......@@ -71,6 +72,65 @@ static bool pathExists(const std::string& path) {
return exists;
}
/* Return an importer object for a sys.path/pkg.__path__ item 'p',
possibly by fetching it from the path_importer_cache dict. If it
wasn't yet cached, traverse path_hooks until a hook is found
that can handle the path item. Return None if no hook could;
this tells our caller it should fall back to the builtin
import mechanism. Cache the result in path_importer_cache.
Returns a borrowed reference. */
extern "C" PyObject* get_path_importer(PyObject* path_importer_cache, PyObject* path_hooks, PyObject* p) noexcept {
PyObject* importer;
Py_ssize_t j, nhooks;
/* These conditions are the caller's responsibility: */
assert(PyList_Check(path_hooks));
assert(PyDict_Check(path_importer_cache));
nhooks = PyList_Size(path_hooks);
if (nhooks < 0)
return NULL; /* Shouldn't happen */
importer = PyDict_GetItem(path_importer_cache, p);
if (importer != NULL)
return importer;
/* set path_importer_cache[p] to None to avoid recursion */
if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0)
return NULL;
for (j = 0; j < nhooks; j++) {
PyObject* hook = PyList_GetItem(path_hooks, j);
if (hook == NULL)
return NULL;
importer = PyObject_CallFunctionObjArgs(hook, p, NULL);
if (importer != NULL)
break;
if (!PyErr_ExceptionMatches(PyExc_ImportError)) {
return NULL;
}
PyErr_Clear();
}
if (importer == NULL) {
importer = PyObject_CallFunctionObjArgs(null_importer_cls, p, NULL);
if (importer == NULL) {
if (PyErr_ExceptionMatches(PyExc_ImportError)) {
PyErr_Clear();
return Py_None;
}
}
}
if (importer != NULL) {
int err = PyDict_SetItem(path_importer_cache, p, importer);
Py_DECREF(importer);
if (err != 0)
return NULL;
}
return importer;
}
struct SearchResult {
// Each of these fields are only valid/used for certain filetypes:
std::string path;
......@@ -119,6 +179,14 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
raiseExcHelper(RuntimeError, "sys.path must be a list of directory names");
}
BoxedList* path_hooks = static_cast<BoxedList*>(sys_module->getattr("path_hooks"));
if (!path_hooks || path_hooks->cls != list_cls)
raiseExcHelper(RuntimeError, "sys.path_hooks must be a list of import hooks");
BoxedDict* path_importer_cache = static_cast<BoxedDict*>(sys_module->getattr("path_importer_cache"));
if (!path_importer_cache || path_importer_cache->cls != dict_cls)
raiseExcHelper(RuntimeError, "sys.path_importer_cache must be a dict");
llvm::SmallString<128> joined_path;
for (int i = 0; i < path_list->size; i++) {
Box* _p = path_list->elts->elts[i];
......@@ -133,6 +201,19 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
llvm::sys::path::append(joined_path, "__init__.py");
std::string fn(joined_path.str());
PyObject* importer = get_path_importer(path_importer_cache, path_hooks, _p);
if (importer == NULL)
return SearchResult("", SearchResult::SEARCH_ERROR);
if (importer != None) {
auto path_pass = path_list ? path_list : None;
Box* loader = callattr(importer, &find_module_str,
CallattrFlags({.cls_only = false, .null_on_nonexistent = false }), ArgPassSpec(2),
new BoxedString(full_name), path_pass, NULL, NULL, NULL);
if (loader != None)
return SearchResult(loader);
}
if (pathExists(fn))
return SearchResult(std::move(dn), SearchResult::PKG_DIRECTORY);
......@@ -416,7 +497,7 @@ extern "C" void _PyImport_ReInitLock() noexcept {
}
extern "C" PyObject* PyImport_ImportModuleNoBlock(const char* name) noexcept {
Py_FatalError("unimplemented");
return PyImport_ImportModule(name);
}
// This function has the same behaviour as __import__()
......@@ -476,6 +557,70 @@ extern "C" PyObject* PyImport_ImportModule(const char* name) noexcept {
}
}
/* Get the module object corresponding to a module name.
First check the modules dictionary if there's one there,
if not, create a new one and insert it in the modules dictionary.
Because the former action is most common, THIS DOES NOT RETURN A
'NEW' REFERENCE! */
extern "C" PyObject* PyImport_AddModule(const char* name) noexcept {
try {
PyObject* modules = getSysModulesDict();
PyObject* m = PyDict_GetItemString(modules, name);
if (m != NULL && m->cls == module_cls)
return m;
return createModule(name, name);
} catch (ExcInfo e) {
setCAPIException(e);
return NULL;
}
}
extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* pathname) noexcept {
try {
RELEASE_ASSERT(co->cls == str_cls, "");
BoxedString* code = (BoxedString*)co;
BoxedModule* module = (BoxedModule*)PyImport_AddModule(name);
if (module == NULL)
return NULL;
AST_Module* ast = parse_string(code->s.c_str());
compileAndRunModule(ast, module);
module->setattr("__file__", boxString(pathname), NULL);
return module;
} catch (ExcInfo e) {
setCAPIException(e);
return NULL;
}
}
static int isdir(const char* path) {
struct stat statbuf;
return stat(path, &statbuf) == 0 && S_ISDIR(statbuf.st_mode);
}
Box* nullImporterInit(Box* self, Box* _path) {
RELEASE_ASSERT(self->cls == null_importer_cls, "");
if (_path->cls != str_cls)
raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_path));
BoxedString* path = (BoxedString*)_path;
if (path->s.empty())
raiseExcHelper(ImportError, "empty pathname");
if (isdir(path->s.c_str()))
raiseExcHelper(ImportError, "existing directory");
return None;
}
Box* nullImporterFindModule(Box* self, Box* fullname, Box* path) {
return None;
}
extern "C" Box* import(int level, Box* from_imports, const std::string* module_name) {
std::string _module_name(*module_name);
return importModuleLevel(&_module_name, getCurrentModule(), from_imports, level);
......@@ -548,6 +693,16 @@ void setupImport() {
imp_module->giveAttr("C_BUILTIN", boxInt(SearchResult::C_BUILTIN));
imp_module->giveAttr("PY_FROZEN", boxInt(SearchResult::PY_FROZEN));
null_importer_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "NullImporter");
null_importer_cls->giveAttr(
"__init__", new BoxedFunction(boxRTFunction((void*)nullImporterInit, NONE, 2, 1, false, false), { None }));
null_importer_cls->giveAttr(
"find_module",
new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)nullImporterFindModule, NONE, 2, 1, false, false),
"find_module", { None }));
null_importer_cls->freeze();
imp_module->giveAttr("NullImporter", null_importer_cls);
CLFunction* find_module_func
= boxRTFunction((void*)impFindModule, UNKNOWN, 2, 1, false, false, ParamNames({ "name", "path" }, "", ""));
imp_module->giveAttr("find_module", new BoxedBuiltinFunctionOrMethod(find_module_func, "find_module", { None }));
......
......@@ -317,6 +317,10 @@ extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
return new BoxedString(s);
}
extern "C" void PyString_InternInPlace(PyObject**) noexcept {
Py_FatalError("unimplemented");
}
/* Format codes
* F_LJUST '-'
* F_SIGN '+'
......
......@@ -70,6 +70,7 @@ extern "C" void initunicodedata();
extern "C" void init_weakref();
extern "C" void initcStringIO();
extern "C" void init_io();
extern "C" void initzipimport();
namespace pyston {
......@@ -1549,6 +1550,7 @@ void setupRuntime() {
init_weakref();
initcStringIO();
init_io();
initzipimport();
// some additional setup to ensure weakrefs participate in our GC
BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;
......
......@@ -4,3 +4,12 @@ e = imp.find_module("encodings")
print e[0]
m = imp.load_module("myenc", e[0], e[1], e[2])
print m.__name__
# Null Importer tests
for a in (1, "", "/proc", "nonexisting_dir"):
try:
i = imp.NullImporter(a)
print i.find_module("foo")
except Exception as e:
print e
import sys, os
# this zip contains two pkgs: test1 and test2
zipdir = os.path.dirname(os.path.realpath(__file__))
filename = zipdir + "/zipimport_test_file.zip"
sys.path.append(filename)
import test1
import test2
import os
import zipimport
# this zip contains two pkgs: test1 and test2
zipdir = os.path.dirname(os.path.realpath(__file__))
filename = zipdir + "/zipimport_test_file.zip"
z = zipimport.zipimporter(filename)
for name in ("test1", "test2", "test3"):
try:
z.is_package(name)
print z.archive
print z.prefix
m = z.load_module(name)
print m.__file__
print m.__name__
except Exception as e:
print e
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment