Skip to content

P
Pyston
Commit dca67a9f authored by Kevin Modzelewski's avatar Kevin Modzelewski
Browse files
Options

Misc changes from trying to run metaserver 

- generalize dynamic loading support
- copy include files to cmake build directory: we don't need it for our build,
  but we need them to be there for building extension modules
- some beginning hacks for supporting Cython
parent 1a926173
Hide whitespace changes
Inline Side-by-side
Showing with 37 additions and 38 deletions
Makefile
View file @ dca67a9f
......@@ -960,35 +960,35 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
.PHONY: run$1 dbg$1
run$1: pyston$1 $$(RUN_DEPS)
./pyston$1 $$(ARGS)
PYTHONPATH=test/test_extension ./pyston$1 $$(ARGS)
dbg$1: pyston$1 $$(RUN_DEPS)
zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
nosearch_run$1_%: %.py pyston$1 $$(RUN_DEPS)
$(VERB) zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
$$(call make_search,run$1_%)
nosearch_dbg$1_%: %.py pyston$1 $$(RUN_DEPS)
$(VERB) zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
$$(call make_search,dbg$1_%)
ifneq ($$(ENABLE_VALGRIND),0)
nosearch_memcheck$1_%: %.py pyston$1 $$(RUN_DEPS)
$$(VALGRIND) --tool=memcheck --leak-check=no --db-attach=yes ./pyston$1 $$(ARGS) $$<
PYTHONPATH=test/test_extension $$(VALGRIND) --tool=memcheck --leak-check=no --db-attach=yes ./pyston$1 $$(ARGS) $$<
$$(call make_search,memcheck$1_%)
nosearch_memcheck_gdb$1_%: %.py pyston$1 $$(RUN_DEPS)
set +e; $$(VALGRIND) -v -v -v -v -v --tool=memcheck --leak-check=no --track-origins=yes --vgdb=yes --vgdb-error=0 ./pyston$1 $$(ARGS) $$< & export PID=$$$$! ; \
set +e; PYTHONPATH=test/test_extension $$(VALGRIND) -v -v -v -v -v --tool=memcheck --leak-check=no --track-origins=yes --vgdb=yes --vgdb-error=0 ./pyston$1 $$(ARGS) $$< & export PID=$$$$! ; \
$$(GDB) --ex "set confirm off" --ex "target remote | $$(DEPS_DIR)/valgrind-3.10.0-install/bin/vgdb" --ex "continue" --ex "bt" ./pyston$1; kill -9 $$$$PID
$$(call make_search,memcheck_gdb$1_%)
nosearch_memleaks$1_%: %.py pyston$1 $$(RUN_DEPS)
$$(VALGRIND) --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston$1 $$(ARGS) $$<
PYTHONPATH=test/test_extension $$(VALGRIND) --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston$1 $$(ARGS) $$<
$$(call make_search,memleaks$1_%)
nosearch_cachegrind$1_%: %.py pyston$1 $$(RUN_DEPS)
$$(VALGRIND) --tool=cachegrind ./pyston$1 $$(ARGS) $$<
PYTHONPATH=test/test_extension $$(VALGRIND) --tool=cachegrind ./pyston$1 $$(ARGS) $$<
$$(call make_search,cachegrind$1_%)
endif
.PHONY: perf$1_%
nosearch_perf$1_%: %.py pyston$1
perf record -g -- ./pyston$1 -q -p $$(ARGS) $$<
PYTHONPATH=test/test_extension perf record -g -- ./pyston$1 -q -p $$(ARGS) $$<
@$(MAKE) perf_report
$$(call make_search,perf$1_%)
......
from_cpython/CMakeLists.txt
View file @ dca67a9f
# Copy any changed stdlib files to the destination:
file(GLOB_RECURSE STDLIB_SRCS Lib/ "*.py")
file(GLOB_RECURSE STD_INCLUDES Include/ "*.h")
set(STDLIB_TARGETS "")
foreach(STDLIB_FILE ${STDLIB_SRCS})
foreach(STDLIB_FILE ${STDLIB_SRCS} ${STD_INCLUDES})
file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})
set(TARGET ${CMAKE_BINARY_DIR}/${FN_REL})
......
from_cpython/Include/Python.h
View file @ dca67a9f
......@@ -15,6 +15,12 @@
#ifndef PYSTON_EXTINCLUDE_PYTHON_H
#define PYSTON_EXTINCLUDE_PYTHON_H
// Cython depends on having this define set:
#define Py_PYTHON_H
// Cython has some "not targeting CPython" support that is triggered by having PYPY_VERSION defined:
#define PYPY_VERSION "Pyston"
// These include orders come from CPython:
#include "patchlevel.h"
#include "pyconfig.h"
......
src/runtime/capi.cpp
View file @ dca67a9f
......@@ -1275,27 +1275,15 @@ extern "C" char* PyModule_GetName(PyObject* m) noexcept {
return &static_cast<BoxedModule*>(m)->fn[0];
}
BoxedModule* importTestExtension(const std::string& name) {
llvm::SmallString<128> pathname_str;
// TODO supposed to pass argv0, main_addr to this function:
pathname_str = llvm::sys::fs::getMainExecutable(NULL, NULL);
assert(pathname_str.size() && "could not find the path to the pyston src dir");
// Start by removing the binary name
llvm::sys::path::remove_filename(pathname_str);
llvm::sys::path::append(pathname_str, "test/test_extension");
llvm::sys::path::append(pathname_str, name + ".pyston.so");
const char* pathname = pathname_str.str().str().c_str();
void* handle = dlopen(pathname, RTLD_NOW);
BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
void* handle = dlopen(path.c_str(), RTLD_NOW);
if (!handle) {
fprintf(stderr, "%s\n", dlerror());
exit(1);
}
assert(handle);
std::string initname = "init" + name;
std::string initname = "init" + last_name;
void (*init)() = (void (*)())dlsym(handle, initname.c_str());
char* error;
......@@ -1308,14 +1296,14 @@ BoxedModule* importTestExtension(const std::string& name) {
(*init)();
BoxedDict* sys_modules = getSysModulesDict();
Box* s = boxStrConstant(name.c_str());
Box* s = boxStrConstant(full_name.c_str());
Box* _m = sys_modules->d[s];
RELEASE_ASSERT(_m, "module failed to initialize properly?");
assert(_m->cls == module_cls);
BoxedModule* m = static_cast<BoxedModule*>(_m);
m->setattr("__file__", boxStrConstant(pathname), NULL);
m->fn = pathname;
m->setattr("__file__", boxString(path), NULL);
m->fn = path;
return m;
}
......
src/runtime/capi.h
View file @ dca67a9f
......@@ -21,7 +21,7 @@ namespace pyston {
class Box;
class BoxedModule;
BoxedModule* importTestExtension(const std::string&);
BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path);
void checkAndThrowCAPIException();
void throwCAPIException() __attribute__((noreturn));
......
src/runtime/import.cpp
View file @ dca67a9f
......@@ -223,6 +223,13 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
if (pathExists(fn))
return SearchResult(std::move(fn), SearchResult::PY_SOURCE);
joined_path.clear();
llvm::sys::path::append(joined_path, p->s, name + ".pyston.so");
fn = joined_path.str();
if (pathExists(fn))
return SearchResult(std::move(fn), SearchResult::C_EXTENSION);
}
return SearchResult("", SearchResult::SEARCH_ERROR);
......@@ -353,6 +360,8 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
module = createAndRunModule(full_name, sr.path);
else if (sr.type == SearchResult::PKG_DIRECTORY)
module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path);
else if (sr.type == SearchResult::C_EXTENSION)
module = importCExtension(full_name, name, sr.path);
else if (sr.type == SearchResult::IMP_HOOK) {
const static std::string load_module_str("load_module");
module = callattr(sr.loader, &load_module_str,
......@@ -366,14 +375,6 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
return module;
}
// TODO: these are a crude hack to help our tests, find a better way
if (name == "basic_test")
return importTestExtension("basic_test");
if (name == "descr_test")
return importTestExtension("descr_test");
if (name == "slots_test")
return importTestExtension("slots_test");
return None;
}
......
tools/tester.py
View file @ dca67a9f
......@@ -57,6 +57,7 @@ def set_ulimits():
resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024))
EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/")
EXTMODULE_DIR_PYSTON = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/")
THIS_FILE = os.path.abspath(__file__)
_global_mtime = None
......@@ -141,7 +142,9 @@ def run_test(fn, check_stats, run_memcheck):
run_args = [os.path.abspath(IMAGE)] + opts.jit_args + [fn]
start = time.time()
p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits)
env = dict(os.environ)
env["PYTHONPATH"] = EXTMODULE_DIR_PYSTON
p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits, env=env)
out, stderr = p.communicate()
code = p.wait()
elapsed = time.time() - start
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7