Commit dca67a9f authored by Kevin Modzelewski's avatar Kevin Modzelewski

Misc changes from trying to run metaserver

- generalize dynamic loading support
- copy include files to cmake build directory: we don't need it for our build,
  but we need them to be there for building extension modules
- some beginning hacks for supporting Cython
parent 1a926173
...@@ -960,35 +960,35 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston ...@@ -960,35 +960,35 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
.PHONY: run$1 dbg$1 .PHONY: run$1 dbg$1
run$1: pyston$1 $$(RUN_DEPS) run$1: pyston$1 $$(RUN_DEPS)
./pyston$1 $$(ARGS) PYTHONPATH=test/test_extension ./pyston$1 $$(ARGS)
dbg$1: pyston$1 $$(RUN_DEPS) dbg$1: pyston$1 $$(RUN_DEPS)
zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)' PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
nosearch_run$1_%: %.py pyston$1 $$(RUN_DEPS) nosearch_run$1_%: %.py pyston$1 $$(RUN_DEPS)
$(VERB) zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<' $(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
$$(call make_search,run$1_%) $$(call make_search,run$1_%)
nosearch_dbg$1_%: %.py pyston$1 $$(RUN_DEPS) nosearch_dbg$1_%: %.py pyston$1 $$(RUN_DEPS)
$(VERB) zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<' $(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
$$(call make_search,dbg$1_%) $$(call make_search,dbg$1_%)
ifneq ($$(ENABLE_VALGRIND),0) ifneq ($$(ENABLE_VALGRIND),0)
nosearch_memcheck$1_%: %.py pyston$1 $$(RUN_DEPS) nosearch_memcheck$1_%: %.py pyston$1 $$(RUN_DEPS)
$$(VALGRIND) --tool=memcheck --leak-check=no --db-attach=yes ./pyston$1 $$(ARGS) $$< PYTHONPATH=test/test_extension $$(VALGRIND) --tool=memcheck --leak-check=no --db-attach=yes ./pyston$1 $$(ARGS) $$<
$$(call make_search,memcheck$1_%) $$(call make_search,memcheck$1_%)
nosearch_memcheck_gdb$1_%: %.py pyston$1 $$(RUN_DEPS) nosearch_memcheck_gdb$1_%: %.py pyston$1 $$(RUN_DEPS)
set +e; $$(VALGRIND) -v -v -v -v -v --tool=memcheck --leak-check=no --track-origins=yes --vgdb=yes --vgdb-error=0 ./pyston$1 $$(ARGS) $$< & export PID=$$$$! ; \ set +e; PYTHONPATH=test/test_extension $$(VALGRIND) -v -v -v -v -v --tool=memcheck --leak-check=no --track-origins=yes --vgdb=yes --vgdb-error=0 ./pyston$1 $$(ARGS) $$< & export PID=$$$$! ; \
$$(GDB) --ex "set confirm off" --ex "target remote | $$(DEPS_DIR)/valgrind-3.10.0-install/bin/vgdb" --ex "continue" --ex "bt" ./pyston$1; kill -9 $$$$PID $$(GDB) --ex "set confirm off" --ex "target remote | $$(DEPS_DIR)/valgrind-3.10.0-install/bin/vgdb" --ex "continue" --ex "bt" ./pyston$1; kill -9 $$$$PID
$$(call make_search,memcheck_gdb$1_%) $$(call make_search,memcheck_gdb$1_%)
nosearch_memleaks$1_%: %.py pyston$1 $$(RUN_DEPS) nosearch_memleaks$1_%: %.py pyston$1 $$(RUN_DEPS)
$$(VALGRIND) --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston$1 $$(ARGS) $$< PYTHONPATH=test/test_extension $$(VALGRIND) --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston$1 $$(ARGS) $$<
$$(call make_search,memleaks$1_%) $$(call make_search,memleaks$1_%)
nosearch_cachegrind$1_%: %.py pyston$1 $$(RUN_DEPS) nosearch_cachegrind$1_%: %.py pyston$1 $$(RUN_DEPS)
$$(VALGRIND) --tool=cachegrind ./pyston$1 $$(ARGS) $$< PYTHONPATH=test/test_extension $$(VALGRIND) --tool=cachegrind ./pyston$1 $$(ARGS) $$<
$$(call make_search,cachegrind$1_%) $$(call make_search,cachegrind$1_%)
endif endif
.PHONY: perf$1_% .PHONY: perf$1_%
nosearch_perf$1_%: %.py pyston$1 nosearch_perf$1_%: %.py pyston$1
perf record -g -- ./pyston$1 -q -p $$(ARGS) $$< PYTHONPATH=test/test_extension perf record -g -- ./pyston$1 -q -p $$(ARGS) $$<
@$(MAKE) perf_report @$(MAKE) perf_report
$$(call make_search,perf$1_%) $$(call make_search,perf$1_%)
......
# Copy any changed stdlib files to the destination: # Copy any changed stdlib files to the destination:
file(GLOB_RECURSE STDLIB_SRCS Lib/ "*.py") file(GLOB_RECURSE STDLIB_SRCS Lib/ "*.py")
file(GLOB_RECURSE STD_INCLUDES Include/ "*.h")
set(STDLIB_TARGETS "") set(STDLIB_TARGETS "")
foreach(STDLIB_FILE ${STDLIB_SRCS}) foreach(STDLIB_FILE ${STDLIB_SRCS} ${STD_INCLUDES})
file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE}) file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})
set(TARGET ${CMAKE_BINARY_DIR}/${FN_REL}) set(TARGET ${CMAKE_BINARY_DIR}/${FN_REL})
......
...@@ -15,6 +15,12 @@ ...@@ -15,6 +15,12 @@
#ifndef PYSTON_EXTINCLUDE_PYTHON_H #ifndef PYSTON_EXTINCLUDE_PYTHON_H
#define PYSTON_EXTINCLUDE_PYTHON_H #define PYSTON_EXTINCLUDE_PYTHON_H
// Cython depends on having this define set:
#define Py_PYTHON_H
// Cython has some "not targeting CPython" support that is triggered by having PYPY_VERSION defined:
#define PYPY_VERSION "Pyston"
// These include orders come from CPython: // These include orders come from CPython:
#include "patchlevel.h" #include "patchlevel.h"
#include "pyconfig.h" #include "pyconfig.h"
......
...@@ -1275,27 +1275,15 @@ extern "C" char* PyModule_GetName(PyObject* m) noexcept { ...@@ -1275,27 +1275,15 @@ extern "C" char* PyModule_GetName(PyObject* m) noexcept {
return &static_cast<BoxedModule*>(m)->fn[0]; return &static_cast<BoxedModule*>(m)->fn[0];
} }
BoxedModule* importTestExtension(const std::string& name) { BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
llvm::SmallString<128> pathname_str; void* handle = dlopen(path.c_str(), RTLD_NOW);
// TODO supposed to pass argv0, main_addr to this function:
pathname_str = llvm::sys::fs::getMainExecutable(NULL, NULL);
assert(pathname_str.size() && "could not find the path to the pyston src dir");
// Start by removing the binary name
llvm::sys::path::remove_filename(pathname_str);
llvm::sys::path::append(pathname_str, "test/test_extension");
llvm::sys::path::append(pathname_str, name + ".pyston.so");
const char* pathname = pathname_str.str().str().c_str();
void* handle = dlopen(pathname, RTLD_NOW);
if (!handle) { if (!handle) {
fprintf(stderr, "%s\n", dlerror()); fprintf(stderr, "%s\n", dlerror());
exit(1); exit(1);
} }
assert(handle); assert(handle);
std::string initname = "init" + name; std::string initname = "init" + last_name;
void (*init)() = (void (*)())dlsym(handle, initname.c_str()); void (*init)() = (void (*)())dlsym(handle, initname.c_str());
char* error; char* error;
...@@ -1308,14 +1296,14 @@ BoxedModule* importTestExtension(const std::string& name) { ...@@ -1308,14 +1296,14 @@ BoxedModule* importTestExtension(const std::string& name) {
(*init)(); (*init)();
BoxedDict* sys_modules = getSysModulesDict(); BoxedDict* sys_modules = getSysModulesDict();
Box* s = boxStrConstant(name.c_str()); Box* s = boxStrConstant(full_name.c_str());
Box* _m = sys_modules->d[s]; Box* _m = sys_modules->d[s];
RELEASE_ASSERT(_m, "module failed to initialize properly?"); RELEASE_ASSERT(_m, "module failed to initialize properly?");
assert(_m->cls == module_cls); assert(_m->cls == module_cls);
BoxedModule* m = static_cast<BoxedModule*>(_m); BoxedModule* m = static_cast<BoxedModule*>(_m);
m->setattr("__file__", boxStrConstant(pathname), NULL); m->setattr("__file__", boxString(path), NULL);
m->fn = pathname; m->fn = path;
return m; return m;
} }
......
...@@ -21,7 +21,7 @@ namespace pyston { ...@@ -21,7 +21,7 @@ namespace pyston {
class Box; class Box;
class BoxedModule; class BoxedModule;
BoxedModule* importTestExtension(const std::string&); BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path);
void checkAndThrowCAPIException(); void checkAndThrowCAPIException();
void throwCAPIException() __attribute__((noreturn)); void throwCAPIException() __attribute__((noreturn));
......
...@@ -223,6 +223,13 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B ...@@ -223,6 +223,13 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
if (pathExists(fn)) if (pathExists(fn))
return SearchResult(std::move(fn), SearchResult::PY_SOURCE); return SearchResult(std::move(fn), SearchResult::PY_SOURCE);
joined_path.clear();
llvm::sys::path::append(joined_path, p->s, name + ".pyston.so");
fn = joined_path.str();
if (pathExists(fn))
return SearchResult(std::move(fn), SearchResult::C_EXTENSION);
} }
return SearchResult("", SearchResult::SEARCH_ERROR); return SearchResult("", SearchResult::SEARCH_ERROR);
...@@ -353,6 +360,8 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box ...@@ -353,6 +360,8 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
module = createAndRunModule(full_name, sr.path); module = createAndRunModule(full_name, sr.path);
else if (sr.type == SearchResult::PKG_DIRECTORY) else if (sr.type == SearchResult::PKG_DIRECTORY)
module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path); module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path);
else if (sr.type == SearchResult::C_EXTENSION)
module = importCExtension(full_name, name, sr.path);
else if (sr.type == SearchResult::IMP_HOOK) { else if (sr.type == SearchResult::IMP_HOOK) {
const static std::string load_module_str("load_module"); const static std::string load_module_str("load_module");
module = callattr(sr.loader, &load_module_str, module = callattr(sr.loader, &load_module_str,
...@@ -366,14 +375,6 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box ...@@ -366,14 +375,6 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
return module; return module;
} }
// TODO: these are a crude hack to help our tests, find a better way
if (name == "basic_test")
return importTestExtension("basic_test");
if (name == "descr_test")
return importTestExtension("descr_test");
if (name == "slots_test")
return importTestExtension("slots_test");
return None; return None;
} }
......
...@@ -57,6 +57,7 @@ def set_ulimits(): ...@@ -57,6 +57,7 @@ def set_ulimits():
resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024)) resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024))
EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/") EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/")
EXTMODULE_DIR_PYSTON = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/")
THIS_FILE = os.path.abspath(__file__) THIS_FILE = os.path.abspath(__file__)
_global_mtime = None _global_mtime = None
...@@ -141,7 +142,9 @@ def run_test(fn, check_stats, run_memcheck): ...@@ -141,7 +142,9 @@ def run_test(fn, check_stats, run_memcheck):
run_args = [os.path.abspath(IMAGE)] + opts.jit_args + [fn] run_args = [os.path.abspath(IMAGE)] + opts.jit_args + [fn]
start = time.time() start = time.time()
p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits) env = dict(os.environ)
env["PYTHONPATH"] = EXTMODULE_DIR_PYSTON
p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits, env=env)
out, stderr = p.communicate() out, stderr = p.communicate()
code = p.wait() code = p.wait()
elapsed = time.time() - start elapsed = time.time() - start
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment