Commit 4c2252ad authored by Kevin Modzelewski's avatar Kevin Modzelewski

Merge pull request #462 from kmod/compile_exec

initial compile() support
parents 11ebf1cf d4858920
...@@ -1023,8 +1023,11 @@ $(call make_target,_gcc) ...@@ -1023,8 +1023,11 @@ $(call make_target,_gcc)
nosearch_runpy_% nosearch_pyrun_%: %.py ext_python nosearch_runpy_% nosearch_pyrun_%: %.py ext_python
$(VERB) PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7 zsh -c 'time python $<' $(VERB) PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7 zsh -c 'time python $<'
nosearch_pypyrun_%: %.py ext_python
$(VERB) PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7 zsh -c 'time python $<'
$(call make_search,runpy_%) $(call make_search,runpy_%)
$(call make_search,pyrun_%) $(call make_search,pyrun_%)
$(call make_search,pypyrun_%)
nosearch_check_%: %.py ext_python ext_pyston nosearch_check_%: %.py ext_python ext_pyston
$(MAKE) check_dbg ARGS="$(patsubst %.py,%,$(notdir $<)) -K" $(MAKE) check_dbg ARGS="$(patsubst %.py,%,$(notdir $<)) -K"
......
...@@ -359,7 +359,7 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const { ...@@ -359,7 +359,7 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const {
} }
} }
void DefinednessAnalysis::run(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel>&& initial_map, void DefinednessAnalysis::run(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map,
CFGBlock* initial_block, ScopeInfo* scope_info) { CFGBlock* initial_block, ScopeInfo* scope_info) {
Timer _t("DefinednessAnalysis()", 10); Timer _t("DefinednessAnalysis()", 10);
...@@ -397,7 +397,7 @@ const DefinednessAnalysis::RequiredSet& DefinednessAnalysis::getDefinedNamesAtEn ...@@ -397,7 +397,7 @@ const DefinednessAnalysis::RequiredSet& DefinednessAnalysis::getDefinedNamesAtEn
return defined_at_end_sets[block]; return defined_at_end_sets[block];
} }
PhiAnalysis::PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel>&& initial_map, PhiAnalysis::PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map,
CFGBlock* initial_block, bool initials_need_phis, LivenessAnalysis* liveness, CFGBlock* initial_block, bool initials_need_phis, LivenessAnalysis* liveness,
ScopeInfo* scope_info) ScopeInfo* scope_info)
: definedness(), liveness(liveness) { : definedness(), liveness(liveness) {
......
...@@ -70,7 +70,7 @@ private: ...@@ -70,7 +70,7 @@ private:
public: public:
DefinednessAnalysis() {} DefinednessAnalysis() {}
void run(llvm::DenseMap<InternedString, DefinitionLevel>&& initial_map, CFGBlock* initial_block, void run(llvm::DenseMap<InternedString, DefinitionLevel> initial_map, CFGBlock* initial_block,
ScopeInfo* scope_info); ScopeInfo* scope_info);
DefinitionLevel isDefinedAtEnd(InternedString name, CFGBlock* block); DefinitionLevel isDefinedAtEnd(InternedString name, CFGBlock* block);
...@@ -92,7 +92,7 @@ private: ...@@ -92,7 +92,7 @@ private:
public: public:
// Initials_need_phis specifies that initial_map should count as an additional entry point // Initials_need_phis specifies that initial_map should count as an additional entry point
// that may require phis. // that may require phis.
PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel>&& initial_map, PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map,
CFGBlock* initial_block, bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info); CFGBlock* initial_block, bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info);
bool isRequired(InternedString name, CFGBlock* block); bool isRequired(InternedString name, CFGBlock* block);
......
...@@ -224,13 +224,13 @@ void ASTInterpreter::gcVisit(GCVisitor* visitor) { ...@@ -224,13 +224,13 @@ void ASTInterpreter::gcVisit(GCVisitor* visitor) {
} }
ASTInterpreter::ASTInterpreter(CompiledFunction* compiled_function) ASTInterpreter::ASTInterpreter(CompiledFunction* compiled_function)
: compiled_func(compiled_function), source_info(compiled_function->clfunc->source), scope_info(0), phis(NULL), : compiled_func(compiled_function), source_info(compiled_function->clfunc->source.get()), scope_info(0), phis(NULL),
current_block(0), current_inst(0), last_exception(NULL, NULL, NULL), passed_closure(0), created_closure(0), current_block(0), current_inst(0), last_exception(NULL, NULL, NULL), passed_closure(0), created_closure(0),
generator(0), edgecount(0), frame_info(ExcInfo(NULL, NULL, NULL)) { generator(0), edgecount(0), frame_info(ExcInfo(NULL, NULL, NULL)) {
CLFunction* f = compiled_function->clfunc; CLFunction* f = compiled_function->clfunc;
if (!source_info->cfg) if (!source_info->cfg)
source_info->cfg = computeCFG(f->source, f->source->body); source_info->cfg = computeCFG(f->source.get(), f->source->body);
scope_info = source_info->getScopeInfo(); scope_info = source_info->getScopeInfo();
...@@ -767,8 +767,11 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v ...@@ -767,8 +767,11 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
} }
assert(closure); assert(closure);
} }
return boxCLFunction(cl, closure, is_generator, globals->cls == dict_cls ? static_cast<BoxedDict*>(globals) : NULL,
u.il); Box* passed_globals = NULL;
if (!getCF()->clfunc->source->scoping->areGlobalsFromModule())
passed_globals = globals;
return boxCLFunction(cl, closure, is_generator, passed_globals, u.il);
} }
Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) { Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) {
...@@ -804,9 +807,11 @@ Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) { ...@@ -804,9 +807,11 @@ Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
BoxedClosure* closure = scope_info->takesClosure() ? created_closure : 0; BoxedClosure* closure = scope_info->takesClosure() ? created_closure : 0;
CLFunction* cl = wrapFunction(node, nullptr, node->body, source_info); CLFunction* cl = wrapFunction(node, nullptr, node->body, source_info);
Box* attrDict = runtimeCall(
boxCLFunction(cl, closure, false, globals->cls == dict_cls ? static_cast<BoxedDict*>(globals) : NULL, {}), Box* passed_globals = NULL;
ArgPassSpec(0), 0, 0, 0, 0, 0); if (!getCF()->clfunc->source->scoping->areGlobalsFromModule())
passed_globals = globals;
Box* attrDict = runtimeCall(boxCLFunction(cl, closure, false, passed_globals, {}), ArgPassSpec(0), 0, 0, 0, 0, 0);
Box* classobj = createUserClass(&node->name.str(), basesTuple, attrDict); Box* classobj = createUserClass(&node->name.str(), basesTuple, attrDict);
...@@ -1202,8 +1207,9 @@ Value ASTInterpreter::visit_attribute(AST_Attribute* node) { ...@@ -1202,8 +1207,9 @@ Value ASTInterpreter::visit_attribute(AST_Attribute* node) {
const void* interpreter_instr_addr = (void*)&ASTInterpreter::execute; const void* interpreter_instr_addr = (void*)&ASTInterpreter::execute;
Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* generator, BoxedDict* globals, Box* arg1, Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
Box* arg2, Box* arg3, Box** args) { Box* arg2, Box* arg3, Box** args) {
assert((!globals) == cf->clfunc->source->scoping->areGlobalsFromModule());
bool can_reopt = ENABLE_REOPT && !FORCE_INTERPRETER && (globals == NULL); bool can_reopt = ENABLE_REOPT && !FORCE_INTERPRETER && (globals == NULL);
if (unlikely(can_reopt && cf->times_called > REOPT_THRESHOLD_INTERPRETER)) { if (unlikely(can_reopt && cf->times_called > REOPT_THRESHOLD_INTERPRETER)) {
assert(!globals); assert(!globals);
...@@ -1222,7 +1228,7 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge ...@@ -1222,7 +1228,7 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge
ASTInterpreter interpreter(cf); ASTInterpreter interpreter(cf);
ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo(); ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
SourceInfo* source_info = cf->clfunc->source; SourceInfo* source_info = cf->clfunc->source.get();
if (unlikely(scope_info->usesNameLookup())) { if (unlikely(scope_info->usesNameLookup())) {
interpreter.setBoxedLocals(new BoxedDict()); interpreter.setBoxedLocals(new BoxedDict());
} }
...@@ -1240,7 +1246,7 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge ...@@ -1240,7 +1246,7 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge
return v.o ? v.o : None; return v.o ? v.o : None;
} }
Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* boxedLocals) { Box* astInterpretFunctionEval(CompiledFunction* cf, Box* globals, Box* boxedLocals) {
++cf->times_called; ++cf->times_called;
ASTInterpreter interpreter(cf); ASTInterpreter interpreter(cf);
...@@ -1248,14 +1254,11 @@ Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* box ...@@ -1248,14 +1254,11 @@ Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* box
interpreter.setBoxedLocals(boxedLocals); interpreter.setBoxedLocals(boxedLocals);
ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo(); ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
SourceInfo* source_info = cf->clfunc->source; SourceInfo* source_info = cf->clfunc->source.get();
if (cf->clfunc->source->scoping->areGlobalsFromModule()) {
assert(!globals); assert(!cf->clfunc->source->scoping->areGlobalsFromModule());
interpreter.setGlobals(source_info->parent_module);
} else {
assert(globals); assert(globals);
interpreter.setGlobals(globals); interpreter.setGlobals(globals);
}
Value v = ASTInterpreter::execute(interpreter); Value v = ASTInterpreter::execute(interpreter);
...@@ -1273,7 +1276,7 @@ Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* encl ...@@ -1273,7 +1276,7 @@ Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* encl
ASTInterpreter interpreter(cf); ASTInterpreter interpreter(cf);
ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo(); ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
SourceInfo* source_info = cf->clfunc->source; SourceInfo* source_info = cf->clfunc->source.get();
assert(cf->clfunc->source->scoping->areGlobalsFromModule()); assert(cf->clfunc->source->scoping->areGlobalsFromModule());
interpreter.setGlobals(source_info->parent_module); interpreter.setGlobals(source_info->parent_module);
......
...@@ -33,9 +33,9 @@ struct LineInfo; ...@@ -33,9 +33,9 @@ struct LineInfo;
extern const void* interpreter_instr_addr; extern const void* interpreter_instr_addr;
Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, BoxedDict* globals, Box* arg1, Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
Box* arg2, Box* arg3, Box** args); Box* arg2, Box* arg3, Box** args);
Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* boxedLocals); Box* astInterpretFunctionEval(CompiledFunction* cf, Box* globals, Box* boxedLocals);
Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val, Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
FrameStackState frame_state); FrameStackState frame_state);
......
...@@ -34,8 +34,11 @@ namespace pyston { ...@@ -34,8 +34,11 @@ namespace pyston {
DS_DEFINE_RWLOCK(codegen_rwlock); DS_DEFINE_RWLOCK(codegen_rwlock);
SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body) SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body,
: parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), body(body) { std::string fn)
: parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), fn(std::move(fn)), body(body) {
assert(this->fn.size());
switch (ast->type) { switch (ast->type) {
case AST_TYPE::ClassDef: case AST_TYPE::ClassDef:
case AST_TYPE::Lambda: case AST_TYPE::Lambda:
......
...@@ -721,7 +721,7 @@ ConcreteCompilerVariable* UnknownType::hasnext(IREmitter& emitter, const OpInfo& ...@@ -721,7 +721,7 @@ ConcreteCompilerVariable* UnknownType::hasnext(IREmitter& emitter, const OpInfo&
} }
CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure, bool isGenerator, CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure, bool isGenerator,
BoxedDict* globals, const std::vector<ConcreteCompilerVariable*>& defaults) { Box* globals, const std::vector<ConcreteCompilerVariable*>& defaults) {
// Unlike the CLFunction*, which can be shared between recompilations, the Box* around it // Unlike the CLFunction*, which can be shared between recompilations, the Box* around it
// should be created anew every time the functiondef is encountered // should be created anew every time the functiondef is encountered
...@@ -752,7 +752,7 @@ CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariab ...@@ -752,7 +752,7 @@ CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariab
llvm::Value* isGenerator_v = llvm::ConstantInt::get(g.i1, isGenerator, false); llvm::Value* isGenerator_v = llvm::ConstantInt::get(g.i1, isGenerator, false);
assert(globals == NULL); assert(globals == NULL);
llvm::Value* globals_v = getNullPtr(g.llvm_dict_type_ptr); llvm::Value* globals_v = getNullPtr(g.llvm_value_type_ptr);
// We know this function call can't throw, so it's safe to use emitter.getBuilder()->CreateCall() rather than // We know this function call can't throw, so it's safe to use emitter.getBuilder()->CreateCall() rather than
// emitter.createCall(). // emitter.createCall().
......
...@@ -399,7 +399,7 @@ ConcreteCompilerVariable* makePureImaginary(IREmitter& emitter, double imag); ...@@ -399,7 +399,7 @@ ConcreteCompilerVariable* makePureImaginary(IREmitter& emitter, double imag);
CompilerVariable* makeStr(const std::string*); CompilerVariable* makeStr(const std::string*);
CompilerVariable* makeUnicode(IREmitter& emitter, const std::string*); CompilerVariable* makeUnicode(IREmitter& emitter, const std::string*);
CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure, bool isGenerator, CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure, bool isGenerator,
BoxedDict* globals, const std::vector<ConcreteCompilerVariable*>& defaults); Box* globals, const std::vector<ConcreteCompilerVariable*>& defaults);
ConcreteCompilerVariable* undefVariable(); ConcreteCompilerVariable* undefVariable();
CompilerVariable* makeTuple(const std::vector<CompilerVariable*>& elts); CompilerVariable* makeTuple(const std::vector<CompilerVariable*>& elts);
......
...@@ -129,6 +129,13 @@ public: ...@@ -129,6 +129,13 @@ public:
if (error_code) if (error_code)
return false; return false;
int uncompressed_size = data.size();
// Write the uncompressed size to the beginning of the file as a simple checksum.
// It looks like each lz4 block has its own data checksum, but we need to also
// make sure that we have all the blocks that we expected.
// In particular, without this, an empty file seems to be a valid lz4 stream.
file.write(reinterpret_cast<const char*>(&uncompressed_size), 4);
LZ4F_preferences_t preferences; LZ4F_preferences_t preferences;
memset(&preferences, 0, sizeof(preferences)); memset(&preferences, 0, sizeof(preferences));
preferences.frameInfo.contentChecksumFlag = contentChecksumEnabled; preferences.frameInfo.contentChecksumFlag = contentChecksumEnabled;
...@@ -158,8 +165,14 @@ public: ...@@ -158,8 +165,14 @@ public:
const char* start = (*compressed_content)->getBufferStart(); const char* start = (*compressed_content)->getBufferStart();
size_t pos = 0; size_t pos = 0;
size_t compressed_size = (*compressed_content)->getBufferSize(); size_t compressed_size = (*compressed_content)->getBufferSize();
if (compressed_size < 4)
return std::unique_ptr<llvm::MemoryBuffer>();
int orig_uncompressed_size = *reinterpret_cast<const int*>(start);
pos += 4;
size_t remaining = compressed_size - pos; size_t remaining = compressed_size - pos;
LZ4F_getFrameInfo(context, &frame_info, start, &remaining); LZ4F_getFrameInfo(context, &frame_info, start + pos, &remaining);
pos += remaining; pos += remaining;
std::vector<char> uncompressed; std::vector<char> uncompressed;
...@@ -182,6 +195,9 @@ public: ...@@ -182,6 +195,9 @@ public:
if (uncompressed.size() != frame_info.contentSize) if (uncompressed.size() != frame_info.contentSize)
return std::unique_ptr<llvm::MemoryBuffer>(); return std::unique_ptr<llvm::MemoryBuffer>();
if (uncompressed.size() != orig_uncompressed_size)
return std::unique_ptr<llvm::MemoryBuffer>();
return llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(uncompressed.data(), uncompressed.size())); return llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(uncompressed.data(), uncompressed.size()));
} }
}; };
......
...@@ -918,7 +918,7 @@ static llvm::MDNode* setupDebugInfo(SourceInfo* source, llvm::Function* f, std:: ...@@ -918,7 +918,7 @@ static llvm::MDNode* setupDebugInfo(SourceInfo* source, llvm::Function* f, std::
llvm::DIBuilder builder(*g.cur_module); llvm::DIBuilder builder(*g.cur_module);
std::string fn = source->parent_module->fn; const std::string& fn = source->fn;
std::string dir = ""; std::string dir = "";
std::string producer = "pyston; git rev " STRINGIFY(GITREV); std::string producer = "pyston; git rev " STRINGIFY(GITREV);
......
...@@ -186,7 +186,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E ...@@ -186,7 +186,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
assert((entry_descriptor != NULL) + (spec != NULL) == 1); assert((entry_descriptor != NULL) + (spec != NULL) == 1);
SourceInfo* source = f->source; SourceInfo* source = f->source.get();
assert(source); assert(source);
std::string name = source->getName(); std::string name = source->getName();
...@@ -198,7 +198,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E ...@@ -198,7 +198,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
llvm::raw_string_ostream ss(s); llvm::raw_string_ostream ss(s);
if (spec) { if (spec) {
ss << "\033[34;1mJIT'ing " << source->parent_module->fn << ":" << name << " with signature ("; ss << "\033[34;1mJIT'ing " << source->fn << ":" << name << " with signature (";
for (int i = 0; i < spec->arg_types.size(); i++) { for (int i = 0; i < spec->arg_types.size(); i++) {
if (i > 0) if (i > 0)
ss << ", "; ss << ", ";
...@@ -208,7 +208,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E ...@@ -208,7 +208,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
ss << ") -> "; ss << ") -> ";
ss << spec->rtn_type->debugName(); ss << spec->rtn_type->debugName();
} else { } else {
ss << "\033[34;1mDoing OSR-entry partial compile of " << source->parent_module->fn << ":" << name ss << "\033[34;1mDoing OSR-entry partial compile of " << source->fn << ":" << name
<< ", starting with backedge to block " << entry_descriptor->backedge->target->idx; << ", starting with backedge to block " << entry_descriptor->backedge->target->idx;
} }
ss << " at effort level " << (int)effort; ss << " at effort level " << (int)effort;
...@@ -259,7 +259,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E ...@@ -259,7 +259,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
static StatCounter us_compiling("us_compiling"); static StatCounter us_compiling("us_compiling");
us_compiling.log(us); us_compiling.log(us);
if (VERBOSITY() >= 1 && us > 100000) { if (VERBOSITY() >= 1 && us > 100000) {
printf("Took %ldms to compile %s::%s!\n", us / 1000, source->parent_module->fn.c_str(), name.c_str()); printf("Took %ldms to compile %s::%s!\n", us / 1000, source->fn.c_str(), name.c_str());
} }
static StatCounter num_compiles("num_compiles"); static StatCounter num_compiles("num_compiles");
...@@ -309,15 +309,17 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) { ...@@ -309,15 +309,17 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
Timer _t("for compileModule()"); Timer _t("for compileModule()");
bm->future_flags = getFutureFlags(m, bm->fn.c_str()); const char* fn = PyModule_GetFilename(bm);
RELEASE_ASSERT(fn, "");
bm->future_flags = getFutureFlags(m, fn);
ScopingAnalysis* scoping = new ScopingAnalysis(m); ScopingAnalysis* scoping = new ScopingAnalysis(m);
SourceInfo* si = new SourceInfo(bm, scoping, m, m->body); std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, m, m->body, fn));
CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
bm->setattr("__doc__", si->getDocString(), NULL); bm->setattr("__doc__", si->getDocString(), NULL);
CLFunction* cl_f = new CLFunction(0, 0, false, false, std::move(si));
EffortLevel effort = initialEffort(); EffortLevel effort = initialEffort();
assert(scoping->areGlobalsFromModule()); assert(scoping->areGlobalsFromModule());
...@@ -332,21 +334,10 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) { ...@@ -332,21 +334,10 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
((void (*)())cf->code)(); ((void (*)())cf->code)();
} }
template <typename AST_Type> Box* evalOrExec(CLFunction* cl, Box* globals, Box* boxedLocals) {
Box* evalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, BoxedModule* bm, BoxedDict* globals, Box* boxedLocals) { RELEASE_ASSERT(!cl->source->scoping->areGlobalsFromModule(), "");
CompiledFunction* cf;
assert(!globals || globals->cls == dict_cls);
{ // scope for limiting the locked region: assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
LOCK_REGION(codegen_rwlock.asWrite());
Timer _t("for evalOrExec()");
ScopingAnalysis* scoping = new ScopingAnalysis(source, globals == NULL);
SourceInfo* si = new SourceInfo(bm, scoping, source, body);
CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
// TODO Right now we only support going into an exec or eval through the // TODO Right now we only support going into an exec or eval through the
// intepretter, since the interpretter has a special function which lets // intepretter, since the interpretter has a special function which lets
...@@ -356,24 +347,95 @@ Box* evalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, BoxedModule* bm, ...@@ -356,24 +347,95 @@ Box* evalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, BoxedModule* bm,
// results (since sometimes eval or exec might be called on constant strings). // results (since sometimes eval or exec might be called on constant strings).
EffortLevel effort = EffortLevel::INTERPRETED; EffortLevel effort = EffortLevel::INTERPRETED;
cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL); CompiledFunction* cf = compileFunction(cl, new FunctionSpecialization(VOID), effort, NULL);
assert(cf->clfunc->versions.size()); assert(cf->clfunc->versions.size());
}
return astInterpretFunctionEval(cf, globals, boxedLocals); return astInterpretFunctionEval(cf, globals, boxedLocals);
} }
// Main entrypoints for eval and exec. template <typename AST_Type>
CLFunction* compileForEvalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, std::string fn) {
LOCK_REGION(codegen_rwlock.asWrite());
Timer _t("for evalOrExec()");
ScopingAnalysis* scoping = new ScopingAnalysis(source, false);
std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, source, body, std::move(fn)));
CLFunction* cl_f = new CLFunction(0, 0, false, false, std::move(si));
return cl_f;
}
static CLFunction* compileExec(llvm::StringRef source, llvm::StringRef fn) {
// TODO error message if parse fails or if it isn't an expr
// TODO should have a cleaner interface that can parse the Expression directly
// TODO this memory leaks
const char* code = source.data();
AST_Module* parsedModule = parse_string(code);
AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings));
parsedSuite->body = parsedModule->body;
return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn);
}
Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
Box* flags = _args[0];
Box* dont_inherit = _args[0];
RELEASE_ASSERT(flags == boxInt(0), "");
RELEASE_ASSERT(dont_inherit == boxInt(0), "");
// source is allowed to be an AST, unicode, or anything that supports the buffer protocol
if (source->cls == unicode_cls) {
source = PyUnicode_AsUTF8String(source);
if (!source)
throwCAPIException();
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8
}
if (isSubclass(fn->cls, unicode_cls)) {
fn = _PyUnicode_AsDefaultEncodedString(fn, NULL);
if (!fn)
throwCAPIException();
}
RELEASE_ASSERT(isSubclass(fn->cls, str_cls), "");
if (isSubclass(type->cls, unicode_cls)) {
type = _PyUnicode_AsDefaultEncodedString(type, NULL);
if (!type)
throwCAPIException();
}
RELEASE_ASSERT(isSubclass(type->cls, str_cls), "");
llvm::StringRef filename_str = static_cast<BoxedString*>(fn)->s;
llvm::StringRef type_str = static_cast<BoxedString*>(type)->s;
RELEASE_ASSERT(isSubclass(source->cls, str_cls), "");
llvm::StringRef source_str = static_cast<BoxedString*>(source)->s;
CLFunction* cl;
if (type_str == "exec") {
cl = compileExec(source_str, filename_str);
} else if (type_str == "eval") {
fatalOrError(NotImplemented, "unimplemented");
throwCAPIException();
} else if (type_str == "single") {
fatalOrError(NotImplemented, "unimplemented");
throwCAPIException();
} else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
}
return codeForCLFunction(cl);
}
Box* eval(Box* boxedCode) { Box* eval(Box* boxedCode) {
Box* boxedLocals = fastLocalsToBoxedLocals(); Box* boxedLocals = fastLocalsToBoxedLocals();
BoxedModule* module = getCurrentModule(); BoxedModule* module = getCurrentModule();
Box* globals = getGlobals(); Box* globals = getGlobals();
if (globals == module)
globals = NULL;
if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module) if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
globals = NULL; globals = module;
if (boxedCode->cls == unicode_cls) { if (boxedCode->cls == unicode_cls) {
boxedCode = PyUnicode_AsUTF8String(boxedCode); boxedCode = PyUnicode_AsUTF8String(boxedCode);
...@@ -408,9 +470,10 @@ Box* eval(Box* boxedCode) { ...@@ -408,9 +470,10 @@ Box* eval(Box* boxedCode) {
stmt->value = parsedExpr->body; stmt->value = parsedExpr->body;
std::vector<AST_stmt*> body = { stmt }; std::vector<AST_stmt*> body = { stmt };
assert(!globals || globals->cls == dict_cls); assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
return evalOrExec<AST_Expression>(parsedExpr, body, module, static_cast<BoxedDict*>(globals), boxedLocals); CLFunction* cl = compileForEvalOrExec(parsedExpr, body, "<string>");
return evalOrExec(cl, globals, boxedLocals);
} }
Box* exec(Box* boxedCode, Box* globals, Box* locals) { Box* exec(Box* boxedCode, Box* globals, Box* locals) {
...@@ -432,8 +495,6 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) { ...@@ -432,8 +495,6 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
if (locals == None) if (locals == None)
locals = NULL; locals = NULL;
// TODO boxedCode is allowed to be a tuple
// TODO need to handle passing in globals
if (locals == NULL) { if (locals == NULL) {
locals = globals; locals = globals;
} }
...@@ -446,19 +507,18 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) { ...@@ -446,19 +507,18 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
globals = getGlobals(); globals = getGlobals();
BoxedModule* module = getCurrentModule(); BoxedModule* module = getCurrentModule();
if (globals == module)
globals = NULL;
if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module) if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
globals = NULL; globals = module;
ASSERT(!globals || globals->cls == dict_cls, "%s", globals->cls->tp_name); assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
if (globals) { if (globals) {
// From CPython (they set it to be f->f_builtins): // From CPython (they set it to be f->f_builtins):
if (PyDict_GetItemString(globals, "__builtins__") == NULL) Box* globals_dict = globals;
PyDict_SetItemString(globals, "__builtins__", builtins_module); if (globals->cls == module_cls)
globals_dict = makeAttrWrapper(globals);
if (PyDict_GetItemString(globals_dict, "__builtins__") == NULL)
PyDict_SetItemString(globals_dict, "__builtins__", builtins_module);
} }
if (boxedCode->cls == unicode_cls) { if (boxedCode->cls == unicode_cls) {
...@@ -468,14 +528,17 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) { ...@@ -468,14 +528,17 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8 // cf.cf_flags |= PyCF_SOURCE_IS_UTF8
} }
// TODO same issues as in `eval` CLFunction* cl;
RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name); if (boxedCode->cls == str_cls) {
const char* code = static_cast<BoxedString*>(boxedCode)->s.data(); cl = compileExec(static_cast<BoxedString*>(boxedCode)->s, "<string>");
AST_Module* parsedModule = parse_string(code); } else if (boxedCode->cls == code_cls) {
AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings)); cl = clfunctionFromCode(boxedCode);
parsedSuite->body = parsedModule->body; } else {
abort();
}
assert(cl);
return evalOrExec<AST_Suite>(parsedSuite, parsedSuite->body, module, static_cast<BoxedDict*>(globals), locals); return evalOrExec(cl, globals, locals);
} }
// If a function version keeps failing its speculations, kill it (remove it // If a function version keeps failing its speculations, kill it (remove it
......
...@@ -39,6 +39,7 @@ CompiledFunction* cfForMachineFunctionName(const std::string&); ...@@ -39,6 +39,7 @@ CompiledFunction* cfForMachineFunctionName(const std::string&);
extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals); extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals);
extern "C" Box* eval(Box* boxedCode); extern "C" Box* eval(Box* boxedCode);
extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
} }
#endif #endif
...@@ -2591,12 +2591,12 @@ CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_s ...@@ -2591,12 +2591,12 @@ CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_s
CLFunction*& cl = made[node]; CLFunction*& cl = made[node];
if (cl == NULL) { if (cl == NULL) {
SourceInfo* si = new SourceInfo(source->parent_module, source->scoping, node, body); std::unique_ptr<SourceInfo> si(new SourceInfo(source->parent_module, source->scoping, node, body, source->fn));
if (args) if (args)
cl = new CLFunction(args->args.size(), args->defaults.size(), args->vararg.str().size(), cl = new CLFunction(args->args.size(), args->defaults.size(), args->vararg.str().size(),
args->kwarg.str().size(), si); args->kwarg.str().size(), std::move(si));
else else
cl = new CLFunction(0, 0, 0, 0, si); cl = new CLFunction(0, 0, 0, 0, std::move(si));
} }
return cl; return cl;
} }
......
...@@ -387,7 +387,7 @@ static unw_word_t getFunctionEnd(unw_word_t ip) { ...@@ -387,7 +387,7 @@ static unw_word_t getFunctionEnd(unw_word_t ip) {
// the stack frame that they were created in, so we need to use this approach (as opposed to // the stack frame that they were created in, so we need to use this approach (as opposed to
// C++11 range loops, for example). // C++11 range loops, for example).
// Return true from the handler to stop iteration at that frame. // Return true from the handler to stop iteration at that frame.
void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImpl>&&)> func) { void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImpl>)> func) {
static unw_word_t interpreter_instr_end = getFunctionEnd((unw_word_t)interpreter_instr_addr); static unw_word_t interpreter_instr_end = getFunctionEnd((unw_word_t)interpreter_instr_addr);
static unw_word_t generator_entry_end = getFunctionEnd((unw_word_t)generatorEntry); static unw_word_t generator_entry_end = getFunctionEnd((unw_word_t)generatorEntry);
...@@ -485,7 +485,7 @@ void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImp ...@@ -485,7 +485,7 @@ void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImp
static std::unique_ptr<PythonFrameIteratorImpl> getTopPythonFrame() { static std::unique_ptr<PythonFrameIteratorImpl> getTopPythonFrame() {
std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr); std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr);
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& iter) { unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> iter) {
rtn = std::move(iter); rtn = std::move(iter);
return true; return true;
}); });
...@@ -497,18 +497,9 @@ static const LineInfo* lineInfoForFrame(PythonFrameIteratorImpl& frame_it) { ...@@ -497,18 +497,9 @@ static const LineInfo* lineInfoForFrame(PythonFrameIteratorImpl& frame_it) {
auto* cf = frame_it.getCF(); auto* cf = frame_it.getCF();
assert(cf); assert(cf);
auto source = cf->clfunc->source; auto source = cf->clfunc->source.get();
// Hack: the "filename" for eval and exec statements is "<string>", not the filename return new LineInfo(current_stmt->lineno, current_stmt->col_offset, source->fn, source->getName());
// of the parent module. We can't currently represent this the same way that CPython does
// (but we probably should), so just check that here:
const std::string* fn = &source->parent_module->fn;
if (source->ast->type == AST_TYPE::Suite /* exec */ || source->ast->type == AST_TYPE::Expression /* eval */) {
static const std::string string_str("<string>");
fn = &string_str;
}
return new LineInfo(current_stmt->lineno, current_stmt->col_offset, *fn, source->getName());
} }
static StatCounter us_gettraceback("us_gettraceback"); static StatCounter us_gettraceback("us_gettraceback");
...@@ -525,7 +516,7 @@ BoxedTraceback* getTraceback() { ...@@ -525,7 +516,7 @@ BoxedTraceback* getTraceback() {
Timer _t("getTraceback", 1000); Timer _t("getTraceback", 1000);
std::vector<const LineInfo*> entries; std::vector<const LineInfo*> entries;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) { unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
const LineInfo* line_info = lineInfoForFrame(*frame_iter.get()); const LineInfo* line_info = lineInfoForFrame(*frame_iter.get());
if (line_info) if (line_info)
entries.push_back(line_info); entries.push_back(line_info);
...@@ -545,7 +536,7 @@ ExcInfo* getFrameExcInfo() { ...@@ -545,7 +536,7 @@ ExcInfo* getFrameExcInfo() {
ExcInfo* copy_from_exc = NULL; ExcInfo* copy_from_exc = NULL;
ExcInfo* cur_exc = NULL; ExcInfo* cur_exc = NULL;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) { unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
FrameInfo* frame_info = frame_iter->getFrameInfo(); FrameInfo* frame_info = frame_iter->getFrameInfo();
copy_from_exc = &frame_info->exc; copy_from_exc = &frame_info->exc;
...@@ -609,7 +600,7 @@ BoxedModule* getCurrentModule() { ...@@ -609,7 +600,7 @@ BoxedModule* getCurrentModule() {
PythonFrameIterator getPythonFrame(int depth) { PythonFrameIterator getPythonFrame(int depth) {
std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr); std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr);
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) { unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
if (depth == 0) { if (depth == 0) {
rtn = std::move(frame_iter); rtn = std::move(frame_iter);
return true; return true;
...@@ -631,7 +622,7 @@ void PythonFrameIterator::operator=(PythonFrameIterator&& rhs) { ...@@ -631,7 +622,7 @@ void PythonFrameIterator::operator=(PythonFrameIterator&& rhs) {
std::swap(this->impl, rhs.impl); std::swap(this->impl, rhs.impl);
} }
PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl>&& impl) { PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl> impl) {
std::swap(this->impl, impl); std::swap(this->impl, impl);
} }
...@@ -641,7 +632,7 @@ PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl ...@@ -641,7 +632,7 @@ PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl
FrameStackState getFrameStackState() { FrameStackState getFrameStackState() {
FrameStackState rtn(NULL, NULL); FrameStackState rtn(NULL, NULL);
bool found = false; bool found = false;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) { unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
BoxedDict* d; BoxedDict* d;
BoxedClosure* closure; BoxedClosure* closure;
CompiledFunction* cf; CompiledFunction* cf;
...@@ -897,7 +888,7 @@ FrameInfo* PythonFrameIterator::getFrameInfo() { ...@@ -897,7 +888,7 @@ FrameInfo* PythonFrameIterator::getFrameInfo() {
PythonFrameIterator PythonFrameIterator::getCurrentVersion() { PythonFrameIterator PythonFrameIterator::getCurrentVersion() {
std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr); std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr);
auto& impl = this->impl; auto& impl = this->impl;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) { unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
if (frame_iter->pointsToTheSameAs(*impl.get())) { if (frame_iter->pointsToTheSameAs(*impl.get())) {
rtn = std::move(frame_iter); rtn = std::move(frame_iter);
return true; return true;
......
...@@ -64,7 +64,7 @@ public: ...@@ -64,7 +64,7 @@ public:
PythonFrameIterator(PythonFrameIterator&& rhs); PythonFrameIterator(PythonFrameIterator&& rhs);
void operator=(PythonFrameIterator&& rhs); void operator=(PythonFrameIterator&& rhs);
PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl>&& impl); PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl> impl);
~PythonFrameIterator(); ~PythonFrameIterator();
}; };
......
...@@ -246,6 +246,7 @@ public: ...@@ -246,6 +246,7 @@ public:
LivenessAnalysis* liveness; LivenessAnalysis* liveness;
std::unordered_map<const OSREntryDescriptor*, PhiAnalysis*> phis; std::unordered_map<const OSREntryDescriptor*, PhiAnalysis*> phis;
bool is_generator; bool is_generator;
std::string fn; // equivalent of code.co_filename
InternedStringPool& getInternedStrings(); InternedStringPool& getInternedStrings();
...@@ -260,7 +261,7 @@ public: ...@@ -260,7 +261,7 @@ public:
Box* getDocString(); Box* getDocString();
SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body); SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body, std::string fn);
}; };
typedef std::vector<CompiledFunction*> FunctionList; typedef std::vector<CompiledFunction*> FunctionList;
...@@ -271,7 +272,7 @@ public: ...@@ -271,7 +272,7 @@ public:
int num_defaults; int num_defaults;
bool takes_varargs, takes_kwargs; bool takes_varargs, takes_kwargs;
SourceInfo* source; std::unique_ptr<SourceInfo> source;
ParamNames param_names; ParamNames param_names;
FunctionList FunctionList
...@@ -287,14 +288,15 @@ public: ...@@ -287,14 +288,15 @@ public:
const std::vector<const std::string*>*); const std::vector<const std::string*>*);
InternalCallable internal_callable = NULL; InternalCallable internal_callable = NULL;
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs, SourceInfo* source) CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs,
std::unique_ptr<SourceInfo> source)
: num_args(num_args), num_defaults(num_defaults), takes_varargs(takes_varargs), takes_kwargs(takes_kwargs), : num_args(num_args), num_defaults(num_defaults), takes_varargs(takes_varargs), takes_kwargs(takes_kwargs),
source(source), param_names(source->ast), always_use_version(NULL) { source(std::move(source)), param_names(this->source->ast), always_use_version(NULL) {
assert(num_args >= num_defaults); assert(num_args >= num_defaults);
} }
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs, const ParamNames& param_names) CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs, const ParamNames& param_names)
: num_args(num_args), num_defaults(num_defaults), takes_varargs(takes_varargs), takes_kwargs(takes_kwargs), : num_args(num_args), num_defaults(num_defaults), takes_varargs(takes_varargs), takes_kwargs(takes_kwargs),
source(NULL), param_names(param_names), always_use_version(NULL) { source(nullptr), param_names(param_names), always_use_version(NULL) {
assert(num_args >= num_defaults); assert(num_args >= num_defaults);
} }
......
...@@ -283,7 +283,7 @@ static int main(int argc, char** argv) { ...@@ -283,7 +283,7 @@ static int main(int argc, char** argv) {
if (!main_module) { if (!main_module) {
main_module = createModule("__main__", "<stdin>"); main_module = createModule("__main__", "<stdin>");
} else { } else {
main_module->fn = "<stdin>"; // main_module->fn = "<stdin>";
} }
for (;;) { for (;;) {
......
...@@ -1149,6 +1149,10 @@ void setupBuiltins() { ...@@ -1149,6 +1149,10 @@ void setupBuiltins() {
builtins_module->giveAttr("execfile", builtins_module->giveAttr("execfile",
new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)execfile, UNKNOWN, 1), "execfile")); new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)execfile, UNKNOWN, 1), "execfile"));
builtins_module->giveAttr(
"compile", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)compile, UNKNOWN, 5, 2, false, false),
"compile", { boxInt(0), boxInt(0) }));
builtins_module->giveAttr( builtins_module->giveAttr(
"map", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)map, LIST, 1, 0, true, false), "map")); "map", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)map, LIST, 1, 0, true, false), "map"));
builtins_module->giveAttr( builtins_module->giveAttr(
......
...@@ -1273,8 +1273,31 @@ extern "C" void PyEval_InitThreads(void) noexcept { ...@@ -1273,8 +1273,31 @@ extern "C" void PyEval_InitThreads(void) noexcept {
} }
extern "C" char* PyModule_GetName(PyObject* m) noexcept { extern "C" char* PyModule_GetName(PyObject* m) noexcept {
assert(m->cls == module_cls); PyObject* d;
return &static_cast<BoxedModule*>(m)->fn[0]; PyObject* nameobj;
if (!PyModule_Check(m)) {
PyErr_BadArgument();
return NULL;
}
if ((nameobj = m->getattr("__name__")) == NULL || !PyString_Check(nameobj)) {
PyErr_SetString(PyExc_SystemError, "nameless module");
return NULL;
}
return PyString_AsString(nameobj);
}
extern "C" char* PyModule_GetFilename(PyObject* m) noexcept {
PyObject* d;
PyObject* fileobj;
if (!PyModule_Check(m)) {
PyErr_BadArgument();
return NULL;
}
if ((fileobj = m->getattr("__file__")) == NULL || !PyString_Check(fileobj)) {
PyErr_SetString(PyExc_SystemError, "module filename missing");
return NULL;
}
return PyString_AsString(fileobj);
} }
BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) { BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
...@@ -1315,7 +1338,6 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l ...@@ -1315,7 +1338,6 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
BoxedModule* m = static_cast<BoxedModule*>(_m); BoxedModule* m = static_cast<BoxedModule*>(_m);
m->setattr("__file__", boxString(path), NULL); m->setattr("__file__", boxString(path), NULL);
m->fn = path;
return m; return m;
} }
......
...@@ -48,7 +48,7 @@ public: ...@@ -48,7 +48,7 @@ public:
static Box* filename(Box* b, void*) { static Box* filename(Box* b, void*) {
RELEASE_ASSERT(b->cls == code_cls, ""); RELEASE_ASSERT(b->cls == code_cls, "");
return boxString(static_cast<BoxedCode*>(b)->f->source->parent_module->fn); return boxString(static_cast<BoxedCode*>(b)->f->source->fn);
} }
static Box* argcount(Box* b, void*) { static Box* argcount(Box* b, void*) {
...@@ -96,6 +96,11 @@ Box* codeForCLFunction(CLFunction* f) { ...@@ -96,6 +96,11 @@ Box* codeForCLFunction(CLFunction* f) {
return new BoxedCode(f); return new BoxedCode(f);
} }
CLFunction* clfunctionFromCode(Box* code) {
assert(code->cls == code_cls);
return static_cast<BoxedCode*>(code)->f;
}
extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept { PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
RELEASE_ASSERT(0, "not implemented"); RELEASE_ASSERT(0, "not implemented");
......
...@@ -2975,7 +2975,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe ...@@ -2975,7 +2975,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
} }
Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure, Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
BoxedGenerator* generator, BoxedDict* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs) { BoxedGenerator* generator, Box* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs) {
CompiledFunction* chosen_cf = pickVersion(f, num_output_args, oarg1, oarg2, oarg3, oargs); CompiledFunction* chosen_cf = pickVersion(f, num_output_args, oarg1, oarg2, oarg3, oargs);
assert(chosen_cf->is_interpreted == (chosen_cf->code == NULL)); assert(chosen_cf->is_interpreted == (chosen_cf->code == NULL));
......
...@@ -150,7 +150,7 @@ bool isUserDefined(BoxedClass* cls); ...@@ -150,7 +150,7 @@ bool isUserDefined(BoxedClass* cls);
Box* processDescriptor(Box* obj, Box* inst, Box* owner); Box* processDescriptor(Box* obj, Box* inst, Box* owner);
Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure, Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
BoxedGenerator* generator, BoxedDict* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs); BoxedGenerator* generator, Box* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs);
static const char* objectNewParameterTypeErrorMsg() { static const char* objectNewParameterTypeErrorMsg() {
if (PYTHON_VERSION_HEX >= version_hex(2, 7, 4)) { if (PYTHON_VERSION_HEX >= version_hex(2, 7, 4)) {
......
...@@ -30,7 +30,7 @@ public: ...@@ -30,7 +30,7 @@ public:
std::vector<const LineInfo*> lines; std::vector<const LineInfo*> lines;
Box* py_lines; Box* py_lines;
BoxedTraceback(std::vector<const LineInfo*>&& lines) : lines(std::move(lines)), py_lines(NULL) {} BoxedTraceback(std::vector<const LineInfo*> lines) : lines(std::move(lines)), py_lines(NULL) {}
BoxedTraceback() : py_lines(NULL) {} BoxedTraceback() : py_lines(NULL) {}
DEFAULT_CLASS(traceback_cls); DEFAULT_CLASS(traceback_cls);
......
...@@ -311,9 +311,10 @@ BoxedFunction::BoxedFunction(CLFunction* f) : BoxedFunction(f, {}) { ...@@ -311,9 +311,10 @@ BoxedFunction::BoxedFunction(CLFunction* f) : BoxedFunction(f, {}) {
} }
BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure, BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure,
bool isGenerator, BoxedDict* globals) bool isGenerator, Box* globals)
: BoxedFunctionBase(f, defaults, closure, isGenerator) { : BoxedFunctionBase(f, defaults, closure, isGenerator) {
assert((!globals) == (!f->source || f->source->scoping->areGlobalsFromModule()));
this->globals = globals; this->globals = globals;
// TODO eventually we want this to assert(f->source), I think, but there are still // TODO eventually we want this to assert(f->source), I think, but there are still
...@@ -380,7 +381,8 @@ static void functionDtor(Box* b) { ...@@ -380,7 +381,8 @@ static void functionDtor(Box* b) {
self->dependent_ics.~ICInvalidator(); self->dependent_ics.~ICInvalidator();
} }
BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) : fn(fn) { // TODO(kmod): builtin modules are not supposed to have a __file__ attribute
BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) {
this->giveAttr("__name__", boxString(name)); this->giveAttr("__name__", boxString(name));
this->giveAttr("__file__", boxString(fn)); this->giveAttr("__file__", boxString(fn));
this->giveAttr("__doc__", doc ? boxStrConstant(doc) : None); this->giveAttr("__doc__", doc ? boxStrConstant(doc) : None);
...@@ -418,12 +420,10 @@ extern "C" void moduleGCHandler(GCVisitor* v, Box* b) { ...@@ -418,12 +420,10 @@ extern "C" void moduleGCHandler(GCVisitor* v, Box* b) {
// This mustn't throw; our IR generator generates calls to it without "invoke" even when there are exception handlers / // This mustn't throw; our IR generator generates calls to it without "invoke" even when there are exception handlers /
// finally-blocks in scope. // finally-blocks in scope.
// TODO: should we use C++11 `noexcept' here? // TODO: should we use C++11 `noexcept' here?
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, BoxedDict* globals, extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, Box* globals,
std::initializer_list<Box*> defaults) { std::initializer_list<Box*> defaults) {
if (closure) if (closure)
assert(closure->cls == closure_cls); assert(closure->cls == closure_cls);
if (globals)
assert(globals->cls == dict_cls);
return new BoxedFunction(f, defaults, closure, isGenerator, globals); return new BoxedFunction(f, defaults, closure, isGenerator, globals);
} }
...@@ -1095,10 +1095,13 @@ Box* moduleRepr(BoxedModule* m) { ...@@ -1095,10 +1095,13 @@ Box* moduleRepr(BoxedModule* m) {
os << "<module '" << m->name() << "' "; os << "<module '" << m->name() << "' ";
if (m->fn == "__builtin__") { const char* filename = PyModule_GetFilename((PyObject*)m);
// TODO(kmod): builtin modules are not supposed to have a __file__ attribute
if (!filename || !strcmp(filename, "__builtin__")) {
PyErr_Clear();
os << "(built-in)>"; os << "(built-in)>";
} else { } else {
os << "from '" << m->fn << "'>"; os << "from '" << filename << "'>";
} }
return boxString(os.str()); return boxString(os.str());
} }
......
...@@ -86,7 +86,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float ...@@ -86,7 +86,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float
*none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls,
*enumerate_cls, *xrange_cls, *member_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *enumerate_cls, *xrange_cls, *member_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
*property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls, *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
*builtin_function_or_method_cls, *set_cls, *frozenset_cls; *builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls;
} }
#define unicode_cls (&PyUnicode_Type) #define unicode_cls (&PyUnicode_Type)
#define memoryview_cls (&PyMemoryView_Type) #define memoryview_cls (&PyMemoryView_Type)
...@@ -128,7 +128,7 @@ char* getWriteableStringContents(BoxedString* s); ...@@ -128,7 +128,7 @@ char* getWriteableStringContents(BoxedString* s);
extern "C" void listAppendInternal(Box* self, Box* v); extern "C" void listAppendInternal(Box* self, Box* v);
extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts); extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, BoxedDict* globals, extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, Box* globals,
std::initializer_list<Box*> defaults); std::initializer_list<Box*> defaults);
extern "C" CLFunction* unboxCLFunction(Box* b); extern "C" CLFunction* unboxCLFunction(Box* b);
extern "C" Box* createUserClass(const std::string* name, Box* base, Box* attr_dict); extern "C" Box* createUserClass(const std::string* name, Box* base, Box* attr_dict);
...@@ -591,7 +591,7 @@ public: ...@@ -591,7 +591,7 @@ public:
// garbage values when the GC is run (BoxedFunctionBase's constructor might call the GC). // garbage values when the GC is run (BoxedFunctionBase's constructor might call the GC).
// So ick... needs to be fixed. // So ick... needs to be fixed.
BoxedClosure* closure; BoxedClosure* closure;
BoxedDict* globals; Box* globals;
bool isGenerator; bool isGenerator;
int ndefaults; int ndefaults;
...@@ -615,7 +615,7 @@ public: ...@@ -615,7 +615,7 @@ public:
BoxedFunction(CLFunction* f); BoxedFunction(CLFunction* f);
BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL, BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
bool isGenerator = false, BoxedDict* globals = NULL); bool isGenerator = false, Box* globals = NULL);
DEFAULT_CLASS(function_cls); DEFAULT_CLASS(function_cls);
}; };
...@@ -633,8 +633,6 @@ class BoxedModule : public Box { ...@@ -633,8 +633,6 @@ class BoxedModule : public Box {
public: public:
HCAttrs attrs; HCAttrs attrs;
// for traceback purposes; not the same as __file__. This corresponds to co_filename
std::string fn;
FutureFlags future_flags; FutureFlags future_flags;
BoxedModule(const std::string& name, const std::string& fn, const char* doc = NULL); BoxedModule(const std::string& name, const std::string& fn, const char* doc = NULL);
...@@ -823,6 +821,7 @@ extern Box* dict_descr; ...@@ -823,6 +821,7 @@ extern Box* dict_descr;
Box* codeForFunction(BoxedFunction*); Box* codeForFunction(BoxedFunction*);
Box* codeForCLFunction(CLFunction*); Box* codeForCLFunction(CLFunction*);
CLFunction* clfunctionFromCode(Box* code);
Box* getFrame(int depth); Box* getFrame(int depth);
} }
......
c = compile("a = 1; print a", "test.py", "exec")
print type(c), c.co_filename, c.co_name
print
a = 0
exec c
print a
print
a = 0
g = {}
exec c in g
print a, sorted(g.keys())
print
g = {}
exec """
c = compile("a = 1; print a", "test.py", "exec")
""" in g
a = 0
exec g['c']
print a, sorted(g.keys())
print
a = 0
g = {'_c':c}
exec "exec _c" in g
print a, sorted(g.keys())
from __future__ import division
# compile() inherits the future flags of the parent module
print 1 / 2
exec "print 1 / 2"
exec compile("print 1 / 2", "<string>", "exec")
# But you can explicitly request that they not be inherited:
# exec compile("print 1 / 2", "<string>", "exec", flags=0, dont_inherit=True)
# expected: fail
# - not currently supported
def f():
global b
b = 1
print b
print
b = 0
g = {}
exec f.func_code in g
print b, sorted(g.keys())
print
b = 0
g = {'f':f}
exec "f()" in g
print b, sorted(g.keys())
from __future__ import unicode_literals
print type("hello world")
...@@ -36,7 +36,7 @@ TEST_F(AnalysisTest, augassign) { ...@@ -36,7 +36,7 @@ TEST_F(AnalysisTest, augassign) {
ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL); ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL); ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
SourceInfo* si = new SourceInfo(createModule("augassign", fn), scoping, func, func->body); SourceInfo* si = new SourceInfo(createModule("augassign", fn), scoping, func, func->body, fn);
CFG* cfg = computeCFG(si, func->body); CFG* cfg = computeCFG(si, func->body);
LivenessAnalysis* liveness = computeLivenessInfo(cfg); LivenessAnalysis* liveness = computeLivenessInfo(cfg);
...@@ -64,7 +64,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) { ...@@ -64,7 +64,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
ScopeInfo* scope_info = scoping->getScopeInfoForNode(func); ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
SourceInfo* si = new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined), fn), SourceInfo* si = new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined), fn),
scoping, func, func->body); scoping, func, func->body, fn);
CFG* cfg = computeCFG(si, func->body); CFG* cfg = computeCFG(si, func->body);
LivenessAnalysis* liveness = computeLivenessInfo(cfg); LivenessAnalysis* liveness = computeLivenessInfo(cfg);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment