Merge pull request #311 from tjhance/eval

Eval

src/analysis/scoping_analysis.cpp

@@ -104,6 +104,9 @@ public:
     }
     bool refersToClosure(InternedString name) override { return false; }
     bool saveInClosure(InternedString name) override { return false; }
+    VarScopeType getScopeTypeOfName(InternedString name) override {
+        return refersToGlobal(name) ? VarScopeType::GLOBAL : VarScopeType::FAST;
+    }
 
     InternedString mangleName(InternedString id) override { return id; }
     InternedString internString(llvm::StringRef s) override { abort(); }
@@ -174,11 +177,13 @@ private:
     ScopeInfo* parent;
     ScopingAnalysis::ScopeNameUsage* usage;
     AST* ast;
+    bool usesNameLookup;
 
 public:
-    ScopeInfoBase(ScopeInfo* parent, ScopingAnalysis::ScopeNameUsage* usage, AST* ast)
-        : parent(parent), usage(usage), ast(ast) {
-        assert(parent);
+    ScopeInfoBase(ScopeInfo* parent, ScopingAnalysis::ScopeNameUsage* usage, AST* ast, bool usesNameLookup)
+        : parent(parent), usage(usage), ast(ast), usesNameLookup(usesNameLookup) {
+        // not true anymore: Expression
+        // assert(parent);
         assert(usage);
         assert(ast);
     }
@@ -202,8 +207,8 @@ public:
 
         if (usage->forced_globals.count(name))
             return true;
-        if (name.c_str() != name.c_str())
-            usage->dump();
+        if (usesNameLookup)
+            return false;
         return usage->written.count(name) == 0 && usage->got_from_closure.count(name) == 0;
     }
     bool refersToClosure(InternedString name) override {
@@ -214,11 +219,26 @@ public:
     }
     bool saveInClosure(InternedString name) override {
         // HAX
-        if (isCompilerCreatedName(name))
+        if (isCompilerCreatedName(name) || usesNameLookup)
             return false;
         return usage->referenced_from_nested.count(name) != 0;
     }
 
+    VarScopeType getScopeTypeOfName(InternedString name) override {
+        // HAX
+        if (isCompilerCreatedName(name))
+            return VarScopeType::FAST;
+        if (refersToClosure(name))
+            return VarScopeType::DEREF;
+        if (refersToGlobal(name))
+            return VarScopeType::GLOBAL;
+        if (saveInClosure(name))
+            return VarScopeType::CLOSURE;
+        if (usesNameLookup)
+            return VarScopeType::NAME;
+        return VarScopeType::FAST;
+    }
+
     InternedString mangleName(const InternedString id) override {
         return pyston::mangleName(id, usage->private_name, usage->scoping->getInternedStrings());
     }
@@ -294,6 +314,7 @@ public:
     bool visit_keyword(AST_keyword* node) override { return false; }
     bool visit_list(AST_List* node) override { return false; }
     bool visit_listcomp(AST_ListComp* node) override { return false; }
+    bool visit_expression(AST_Expression* node) override { return false; }
     // bool visit_module(AST_Module *node) override { return false; }
     // bool visit_name(AST_Name *node) override { return false; }
     bool visit_num(AST_Num* node) override { return false; }
@@ -530,15 +551,18 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
         ScopeInfo* parent_info = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node];
 
         switch (node->type) {
-            case AST_TYPE::ClassDef:
-            case AST_TYPE::FunctionDef:
-            case AST_TYPE::Lambda: {
-                ScopeInfoBase* scopeInfo = new ScopeInfoBase(parent_info, usage, usage->node);
+            case AST_TYPE::Expression:
+            case AST_TYPE::ClassDef: {
+                ScopeInfoBase* scopeInfo
+                    = new ScopeInfoBase(parent_info, usage, usage->node, true /* usesNameLookup */);
                 this->scopes[node] = scopeInfo;
                 break;
             }
+            case AST_TYPE::FunctionDef:
+            case AST_TYPE::Lambda:
             case AST_TYPE::GeneratorExp: {
-                ScopeInfoBase* scopeInfo = new ScopeInfoBase(parent_info, usage, usage->node);
+                ScopeInfoBase* scopeInfo
+                    = new ScopeInfoBase(parent_info, usage, usage->node, false /* usesNameLookup */);
                 this->scopes[node] = scopeInfo;
                 break;
             }
@@ -550,8 +574,7 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
 }
 
 InternedStringPool& ScopingAnalysis::getInternedStrings() {
-    assert(parent_module);
-    return *parent_module->interned_strings.get();
+    return interned_strings;
 }
 
 ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
@@ -602,4 +625,8 @@ ScopingAnalysis::ScopingAnalysis(AST_Module* m) : parent_module(m), interned_str
 ScopingAnalysis* runScopingAnalysis(AST_Module* m) {
     return new ScopingAnalysis(m);
 }
+
+ScopingAnalysis::ScopingAnalysis(AST_Expression* e) : interned_strings(*e->interned_strings.get()) {
+    scopes[e] = getScopeInfoForNode(e);
+}
 }

src/analysis/scoping_analysis.h

@@ -22,6 +22,7 @@ namespace pyston {
 
 class AST;
 class AST_Module;
+class AST_Expression;
 
 class ScopeInfo {
 public:
@@ -33,9 +34,39 @@ public:
     virtual bool takesClosure() = 0;
     virtual bool passesThroughClosure() = 0;
 
+    // Various ways a variable name can be resolved.
+    // These all correspond to STORE_* or LOAD_* bytecodes in CPython.
+    //
+    // By way of example:
+    //
+    //  def f():
+    //      print a # GLOBAL
+    //
+    //      b = 0
+    //      print b # FAST
+    //
+    //      c = 0 # CLOSURE
+    //      def g():
+    //          print c # DEREF
+    //
+    //  class C(object):
+    //      print d # NAME
+    //
+    //  def g():
+    //      exec "sdfasdfds()"
+    //      # existence of 'exec' statement forces this to NAME:
+    //      print e # NAME
+    //
+    //  # protip: you can figure this stuff out by doing something like this in CPython:
+    //  import dis
+    //  print dis.dis(g)
+
+    enum class VarScopeType { FAST, GLOBAL, CLOSURE, DEREF, NAME };
+
     virtual bool refersToGlobal(InternedString name) = 0;
     virtual bool refersToClosure(InternedString name) = 0;
     virtual bool saveInClosure(InternedString name) = 0;
+    virtual VarScopeType getScopeTypeOfName(InternedString name) = 0;
 
     virtual InternedString mangleName(InternedString id) = 0;
     virtual InternedString internString(llvm::StringRef) = 0;
@@ -67,6 +98,7 @@ public:
     void registerScopeReplacement(AST* original_node, AST* new_node);
 
     ScopingAnalysis(AST_Module* m);
+    ScopingAnalysis(AST_Expression* e);
     ScopeInfo* getScopeInfoForNode(AST* node);
 
     InternedStringPool& getInternedStrings();

src/analysis/type_analysis.cpp

@@ -394,7 +394,9 @@ private:
     }
 
     void* visit_name(AST_Name* node) override {
-        if (scope_info->refersToGlobal(node->id)) {
+        auto name_scope = scope_info->getScopeTypeOfName(node->id);
+
+        if (name_scope == ScopeInfo::VarScopeType::GLOBAL) {
             if (node->id.str() == "xrange") {
                 // printf("TODO guard here and return the classobj\n");
                 // return typeOfClassobj(xrange_cls);
@@ -402,19 +404,31 @@ private:
             return UNKNOWN;
         }
 
-        if (scope_info->refersToClosure(node->id)) {
+        if (name_scope == ScopeInfo::VarScopeType::CLOSURE) {
             return UNKNOWN;
         }
 
-        CompilerType*& t = sym_table[node->id];
-        if (t == NULL) {
-            // if (VERBOSITY() >= 2) {
-            // printf("%s is undefined!\n", node->id.c_str());
-            // raise(SIGTRAP);
-            //}
-            t = UNDEF;
+        if (name_scope == ScopeInfo::VarScopeType::NAME) {
+            return UNKNOWN;
+        }
+
+        if (name_scope == ScopeInfo::VarScopeType::DEREF) {
+            return UNKNOWN;
         }
-        return t;
+
+        if (name_scope == ScopeInfo::VarScopeType::FAST) {
+            CompilerType*& t = sym_table[node->id];
+            if (t == NULL) {
+                // if (VERBOSITY() >= 2) {
+                // printf("%s is undefined!\n", node->id.c_str());
+                // raise(SIGTRAP);
+                //}
+                t = UNDEF;
+            }
+            return t;
+        }
+
+        RELEASE_ASSERT(0, "Unknown scope type: %d", (int)name_scope);
     }
 
     void* visit_num(AST_Num* node) override {

src/codegen/ast_interpreter.cpp

@@ -137,7 +137,10 @@ public:
 
     CompiledFunction* getCF() { return compiled_func; }
     FrameInfo* getFrameInfo() { return &frame_info; }
+    BoxedClosure* getPassedClosure() { return passed_closure; }
     const SymMap& getSymbolTable() { return sym_table; }
+    const ScopeInfo* getScopeInfo() { return scope_info; }
+
     void addSymbol(InternedString name, Box* value, bool allow_duplicates);
     void gcVisit(GCVisitor* visitor);
 };
@@ -999,14 +1002,15 @@ Value ASTInterpreter::visit_str(AST_Str* node) {
 Value ASTInterpreter::visit_name(AST_Name* node) {
     switch (node->lookup_type) {
         case AST_Name::UNKNOWN: {
-            if (scope_info->refersToGlobal(node->id)) {
+            ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(node->id);
+            if (vst == ScopeInfo::VarScopeType::GLOBAL) {
                 node->lookup_type = AST_Name::GLOBAL;
                 return getGlobal(source_info->parent_module, &node->id.str());
-            } else if (scope_info->refersToClosure(node->id)) {
+            } else if (vst == ScopeInfo::VarScopeType::DEREF) {
                 node->lookup_type = AST_Name::CLOSURE;
                 return getattr(passed_closure, node->id.c_str());
             } else {
-                bool is_old_local = (source_info->ast->type == AST_TYPE::ClassDef);
+                bool is_old_local = (vst == ScopeInfo::VarScopeType::NAME);
                 node->lookup_type = is_old_local ? AST_Name::LOCAL : AST_Name::FAST_LOCAL;
 
                 SymMap::iterator it = sym_table.find(node->id);
@@ -1015,8 +1019,8 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
                     return value;
                 }
 
-                // classdefs have different scoping rules than functions:
-                if (source_info->ast->type == AST_TYPE::ClassDef)
+                // classdefs (and some other cases like eval) have different scoping rules than functions:
+                if (is_old_local)
                     return getGlobal(source_info->parent_module, &node->id.str());
 
                 assertNameDefined(0, node->id.c_str(), UnboundLocalError, true);
@@ -1101,6 +1105,22 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge
     return v.o ? v.o : None;
 }
 
+Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* locals) {
+    ++cf->times_called;
+
+    ASTInterpreter interpreter(cf);
+    for (const auto& p : locals->d) {
+        assert(p.first->cls == str_cls);
+        auto name = static_cast<BoxedString*>(p.first)->s;
+        InternedString interned = cf->clfunc->source->getInternedStrings().get(name);
+        interpreter.addSymbol(interned, p.second, false);
+    }
+
+    interpreter.initArguments(0, NULL, NULL, NULL, NULL, NULL, NULL);
+    Value v = ASTInterpreter::execute(interpreter);
+
+    return v.o ? v.o : None;
+}
 
 Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
                       BoxedDict* locals) {
@@ -1200,9 +1220,16 @@ BoxedDict* localsForInterpretedFrame(void* frame_ptr, bool only_user_visible) {
 
         rtn->d[new BoxedString(l.first.str())] = l.second;
     }
+
     return rtn;
 }
 
+BoxedClosure* passedClosureForInterpretedFrame(void* frame_ptr) {
+    ASTInterpreter* interpreter = s_interpreterMap[frame_ptr];
+    assert(interpreter);
+    return interpreter->getPassedClosure();
+}
+
 void gatherInterpreterRoots(GCVisitor* visitor) {
     for (const auto& p : s_interpreterMap) {
         p.second->gcVisit(visitor);

src/codegen/ast_interpreter.h

@@ -24,6 +24,7 @@ class GCVisitor;
 class AST_expr;
 class AST_stmt;
 class Box;
+class BoxedClosure;
 class BoxedDict;
 struct CompiledFunction;
 struct LineInfo;
@@ -32,6 +33,7 @@ extern const void* interpreter_instr_addr;
 
 Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, Box* arg1, Box* arg2, Box* arg3,
                           Box** args);
+Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* locals);
 Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
                       BoxedDict* locals);
 
@@ -39,6 +41,7 @@ AST_stmt* getCurrentStatementForInterpretedFrame(void* frame_ptr);
 CompiledFunction* getCFForInterpretedFrame(void* frame_ptr);
 struct FrameInfo;
 FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr);
+BoxedClosure* passedClosureForInterpretedFrame(void* frame_ptr);
 
 void gatherInterpreterRoots(gc::GCVisitor* visitor);
 BoxedDict* localsForInterpretedFrame(void* frame_ptr, bool only_user_visible);

src/codegen/codegen.cpp

@@ -39,6 +39,7 @@ SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const
         case AST_TYPE::ClassDef:
         case AST_TYPE::Lambda:
         case AST_TYPE::Module:
+        case AST_TYPE::Expression:
             is_generator = false;
             break;
         case AST_TYPE::FunctionDef:

src/codegen/compvars.cpp

@@ -1679,7 +1679,7 @@ public:
 
     Box* deserializeFromFrame(const FrameVals& vals) override {
         assert(vals.size() == 1);
-        abort();
+        return reinterpret_cast<Box*>(vals[0]);
     }
 } _CLOSURE;
 ConcreteCompilerType* CLOSURE = &_CLOSURE;

src/codegen/irgen/hooks.cpp

@@ -27,6 +27,7 @@
 #include "codegen/irgen/future.h"
 #include "codegen/irgen/util.h"
 #include "codegen/osrentry.h"
+#include "codegen/parser.h"
 #include "codegen/patchpoints.h"
 #include "codegen/stackmaps.h"
 #include "core/ast.h"
@@ -43,7 +44,7 @@ namespace pyston {
 
 // TODO terrible place for these!
 ParamNames::ParamNames(AST* ast) : takes_param_names(true) {
-    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef) {
+    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef || ast->type == AST_TYPE::Expression) {
         kwarg = "";
         vararg = "";
     } else if (ast->type == AST_TYPE::FunctionDef || ast->type == AST_TYPE::Lambda) {
@@ -93,6 +94,7 @@ const std::string SourceInfo::getName() {
         case AST_TYPE::Lambda:
             return "<lambda>";
         case AST_TYPE::Module:
+        case AST_TYPE::Expression:
             return "<module>";
         default:
             RELEASE_ASSERT(0, "%d", ast->type);
@@ -300,6 +302,42 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         ((void (*)())cf->code)();
 }
 
+static Box* compileAndRunExpression(AST_Expression* expr, BoxedModule* bm, BoxedDict* locals) {
+    CompiledFunction* cf;
+
+    { // scope for limiting the locked region:
+        LOCK_REGION(codegen_rwlock.asWrite());
+
+        Timer _t("for compileEval()");
+
+        ScopingAnalysis* scoping = new ScopingAnalysis(expr);
+
+        AST_Return* stmt = new AST_Return();
+        stmt->value = expr->body;
+        SourceInfo* si = new SourceInfo(bm, scoping, expr, { stmt });
+        CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
+
+        EffortLevel effort = EffortLevel::INTERPRETED;
+
+        cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL);
+        assert(cf->clfunc->versions.size());
+    }
+
+    return astInterpretFunctionEval(cf, locals);
+}
+
+Box* runEval(const char* code, BoxedDict* locals, BoxedModule* module) {
+    // TODO error message if parse fails or if it isn't an expr
+    // TODO should have a cleaner interface that can parse the Expression directly
+    // TODO this memory leaks
+    AST_Module* parsedModule = parse_string(code);
+    assert(parsedModule->body[0]->type == AST_TYPE::Expr);
+    AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
+    parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
+
+    return compileAndRunExpression(parsedExpr, module, locals);
+}
+
 // If a function version keeps failing its speculations, kill it (remove it
 // from the list of valid function versions).  The next time we go to call
 // the function, we will have to pick a different version, potentially recompiling.

src/codegen/irgen/hooks.h

@@ -22,6 +22,8 @@ namespace pyston {
 struct CompiledFunction;
 class CLFunction;
 class OSRExit;
+class Box;
+class BoxedDict;
 
 CompiledFunction* compilePartialFuncInternal(OSRExit* exit);
 void* compilePartialFunc(OSRExit*);
@@ -34,6 +36,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 
 // will we always want to generate unique function names? (ie will this function always be reasonable?)
 CompiledFunction* cfForMachineFunctionName(const std::string&);
+
+Box* runEval(const char* code, BoxedDict* locals, BoxedModule* module);
 }
 
 #endif

src/codegen/irgen/irgenerator.cpp

@@ -867,10 +867,11 @@ private:
         bool is_kill = irstate->getSourceInfo()->liveness->isKill(node, myblock);
         assert(!is_kill || node->id.str()[0] == '#');
 
-        if (scope_info->refersToGlobal(node->id)) {
+        ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(node->id);
+        if (vst == ScopeInfo::VarScopeType::GLOBAL) {
             assert(!is_kill);
             return _getGlobal(node, unw_info);
-        } else if (scope_info->refersToClosure(node->id)) {
+        } else if (vst == ScopeInfo::VarScopeType::DEREF) {
             assert(!is_kill);
             assert(scope_info->takesClosure());
 
@@ -881,7 +882,7 @@ private:
         } else {
             if (symbol_table.find(node->id) == symbol_table.end()) {
                 // classdefs have different scoping rules than functions:
-                if (irstate->getSourceInfo()->ast->type == AST_TYPE::ClassDef) {
+                if (vst == ScopeInfo::VarScopeType::NAME) {
                     return _getGlobal(node, unw_info);
                 }
 
@@ -900,8 +901,7 @@ private:
                 = static_cast<ConcreteCompilerVariable*>(_getFake(defined_name, true));
 
             if (is_defined_var) {
-                // classdefs have different scoping rules than functions:
-                if (irstate->getSourceInfo()->ast->type == AST_TYPE::ClassDef) {
+                if (vst == ScopeInfo::VarScopeType::NAME) {
                     llvm::Value* v = handlePotentiallyUndefined(
                         is_defined_var, g.llvm_value_type_ptr, curblock, emitter, false,
                         [=](IREmitter& emitter) {

src/codegen/parser.cpp

@@ -935,7 +935,27 @@ static std::string getParserCommandLine(const char* fn) {
     return std::string("python -S ") + parse_ast_fn.str().str() + " " + fn;
 }
 
-AST_Module* parse(const char* fn) {
+AST_Module* parse_string(const char* code) {
+    int size = strlen(code);
+    char buf[] = "pystontmp_XXXXXX";
+    char* tmpdir = mkdtemp(buf);
+    assert(tmpdir);
+    std::string tmp = std::string(tmpdir) + "/in.py";
+    if (VERBOSITY() >= 1) {
+        printf("writing %d bytes to %s\n", size, tmp.c_str());
+    }
+
+    FILE* f = fopen(tmp.c_str(), "w");
+    fwrite(code, 1, size, f);
+    fclose(f);
+
+    AST_Module* m = parse_file(tmp.c_str());
+    removeDirectoryIfExists(tmpdir);
+
+    return m;
+}
+
+AST_Module* parse_file(const char* fn) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {
@@ -1009,7 +1029,7 @@ static ParseResult _reparse(const char* fn, const std::string& cache_fn) {
 // Parsing the file is somewhat expensive since we have to shell out to cpython;
 // it's not a huge deal right now, but this caching version can significantly cut down
 // on the startup time (40ms -> 10ms).
-AST_Module* caching_parse(const char* fn) {
+AST_Module* caching_parse_file(const char* fn) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {
@@ -1069,7 +1089,7 @@ AST_Module* caching_parse(const char* fn) {
             if (result == ParseResult::PYC_UNWRITABLE) {
                 if (VERBOSITY())
                     printf("Unable to write to %s, falling back to non-caching parse\n", cache_fn.c_str());
-                return parse(fn);
+                return parse_file(fn);
             }
             code = stat(cache_fn.c_str(), &cache_stat);
             assert(code == 0);

src/codegen/parser.h

@@ -19,8 +19,10 @@ namespace pyston {
 
 class AST_Module;
 
-AST_Module* parse(const char* fn);
-AST_Module* caching_parse(const char* fn);
+AST_Module* parse_string(const char* code);
+
+AST_Module* parse_file(const char* fn);
+AST_Module* caching_parse_file(const char* fn);
 }
 
 #endif

src/codegen/unwinding.cpp

@@ -23,10 +23,12 @@
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/Object/ObjectFile.h"
 
+#include "analysis/scoping_analysis.h"
 #include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
 #include "codegen/compvars.h"
 #include "codegen/irgen/hooks.h"
+#include "codegen/irgen/irgenerator.h"
 #include "codegen/stackmaps.h"
 #include "core/util.h"
 #include "runtime/ctxswitching.h"
@@ -536,12 +538,15 @@ BoxedModule* getCurrentModule() {
     return compiledFunction->clfunc->source->parent_module;
 }
 
-BoxedDict* getLocals(bool only_user_visible) {
+BoxedDict* getLocals(bool only_user_visible, bool includeClosure) {
     for (PythonFrameIterator& frame_info : unwindPythonFrames()) {
+        BoxedDict* d;
+        BoxedClosure* closure;
+        CompiledFunction* cf;
         if (frame_info.getId().type == PythonFrameId::COMPILED) {
-            BoxedDict* d = new BoxedDict();
+            d = new BoxedDict();
 
-            CompiledFunction* cf = frame_info.getCF();
+            cf = frame_info.getCF();
             uint64_t ip = frame_info.getId().ip;
 
             assert(ip > cf->code_start);
@@ -601,11 +606,52 @@ BoxedDict* getLocals(bool only_user_visible) {
                 }
             }
 
-            return d;
+            closure = NULL;
+            if (includeClosure && cf->location_map->names.count(PASSED_CLOSURE_NAME) > 0) {
+                for (const LocationMap::LocationTable::LocationEntry& e :
+                     cf->location_map->names[PASSED_CLOSURE_NAME].locations) {
+                    if (e.offset < offset && offset <= e.offset + e.length) {
+                        const auto& locs = e.locations;
+
+                        llvm::SmallVector<uint64_t, 1> vals;
+
+                        for (auto& loc : locs) {
+                            vals.push_back(frame_info.readLocation(loc));
+                        }
+
+                        Box* v = e.type->deserializeFromFrame(vals);
+                        assert(gc::isValidGCObject(v));
+                        closure = static_cast<BoxedClosure*>(v);
+                    }
+                }
+            }
         } else if (frame_info.getId().type == PythonFrameId::INTERPRETED) {
-            return localsForInterpretedFrame((void*)frame_info.getId().bp, only_user_visible);
+            d = localsForInterpretedFrame((void*)frame_info.getId().bp, only_user_visible);
+            if (includeClosure) {
+                closure = passedClosureForInterpretedFrame((void*)frame_info.getId().bp);
+                cf = getCFForInterpretedFrame((void*)frame_info.getId().bp);
+            }
+        } else {
+            abort();
         }
-        abort();
+
+        if (includeClosure) {
+            // Add the locals from the closure
+            for (; closure != NULL; closure = closure->parent) {
+                assert(closure->cls == closure_cls);
+                for (auto& attr_offset : closure->attrs.hcls->attr_offsets) {
+                    const std::string& name = attr_offset.first;
+                    int offset = attr_offset.second;
+                    Box* val = closure->attrs.attr_list->attrs[offset];
+                    ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
+                    if (val != NULL && scope_info->refersToClosure(scope_info->internString(name))) {
+                        d->d[boxString(name)] = val;
+                    }
+                }
+            }
+        }
+
+        return d;
     }
     RELEASE_ASSERT(0, "Internal error: unable to find any python frames");
 }

src/codegen/unwinding.h

@@ -28,7 +28,7 @@ class BoxedTraceback;
 BoxedTraceback* getTraceback();
 
 class BoxedDict;
-BoxedDict* getLocals(bool only_user_visible);
+BoxedDict* getLocals(bool only_user_visible, bool includeClosure);
 
 // Fetches a writeable pointer to the frame-local excinfo object,
 // calculating it if necessary (from previous frames).

src/core/ast.cpp

@@ -692,6 +692,14 @@ void AST_Module::accept(ASTVisitor* v) {
     visitVector(body, v);
 }
 
+void AST_Expression::accept(ASTVisitor* v) {
+    bool skip = v->visit_expression(this);
+    if (skip)
+        return;
+
+    body->accept(v);
+}
+
 void AST_Name::accept(ASTVisitor* v) {
     bool skip = v->visit_name(this);
 }
@@ -1507,6 +1515,12 @@ bool PrintVisitor::visit_module(AST_Module* node) {
     return true;
 }
 
+bool PrintVisitor::visit_expression(AST_Expression* node) {
+    node->body->accept(this);
+    printf("\n");
+    return true;
+}
+
 bool PrintVisitor::visit_name(AST_Name* node) {
     printf("%s", node->id.c_str());
     // printf("%s(%d)", node->id.c_str(), node->ctx_type);

src/core/ast.h

@@ -118,6 +118,7 @@ enum AST_TYPE {
     DictComp = 15,
     Set = 43,
     Ellipsis = 87,
+    Expression = 88,
 
     // Pseudo-nodes that are specific to this compiler:
     Branch = 200,
@@ -657,6 +658,21 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Module;
 };
 
+// (Alternative to AST_Module, used for, e.g., eval)
+class AST_Expression : public AST {
+public:
+    std::unique_ptr<InternedStringPool> interned_strings;
+
+    AST_expr* body;
+
+    virtual void accept(ASTVisitor* v);
+
+    AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
+        : AST(AST_TYPE::Expression), interned_strings(std::move(interned_strings)) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expression;
+};
+
 class AST_Name : public AST_expr {
 public:
     AST_TYPE::AST_TYPE ctx_type;
@@ -1044,6 +1060,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_exec(AST_Exec* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_expr(AST_Expr* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_expression(AST_Expression* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_extslice(AST_ExtSlice* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_for(AST_For* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_functiondef(AST_FunctionDef* node) { RELEASE_ASSERT(0, ""); }
@@ -1112,6 +1129,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { return false; }
     virtual bool visit_exec(AST_Exec* node) { return false; }
     virtual bool visit_expr(AST_Expr* node) { return false; }
+    virtual bool visit_expr(AST_Expression* node) { return false; }
     virtual bool visit_extslice(AST_ExtSlice* node) { return false; }
     virtual bool visit_for(AST_For* node) { return false; }
     virtual bool visit_functiondef(AST_FunctionDef* node) { return false; }
@@ -1254,6 +1272,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node);
     virtual bool visit_exec(AST_Exec* node);
     virtual bool visit_expr(AST_Expr* node);
+    virtual bool visit_expression(AST_Expression* node);
     virtual bool visit_extslice(AST_ExtSlice* node);
     virtual bool visit_for(AST_For* node);
     virtual bool visit_functiondef(AST_FunctionDef* node);

src/core/cfg.cpp

@@ -1609,7 +1609,7 @@ public:
     }
 
     bool visit_return(AST_Return* node) override {
-        if (root_type != AST_TYPE::FunctionDef && root_type != AST_TYPE::Lambda) {
+        if (root_type != AST_TYPE::FunctionDef && root_type != AST_TYPE::Lambda && root_type != AST_TYPE::Expression) {
             raiseExcHelper(SyntaxError, "'return' outside function");
         }
 

src/jit.cpp

@@ -179,29 +179,14 @@ int main(int argc, char** argv) {
         while (repl) {
             char* line = readline(">> ");
 
+            AST_Module* m = parse_string(line);
+
             if (!line) {
                 repl = false;
             } else {
                 add_history(line);
-                int size = strlen(line);
 
                 Timer _t("repl");
-
-                char buf[] = "pystontmp_XXXXXX";
-                char* tmpdir = mkdtemp(buf);
-                assert(tmpdir);
-                std::string tmp = std::string(tmpdir) + "/in.py";
-                if (VERBOSITY() >= 1) {
-                    printf("writing %d bytes to %s\n", size, tmp.c_str());
-                }
-
-                FILE* f = fopen(tmp.c_str(), "w");
-                fwrite(line, 1, size, f);
-                fclose(f);
-
-                AST_Module* m = parse(tmp.c_str());
-                removeDirectoryIfExists(tmpdir);
-
                 if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
                     AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
                     AST_Call* c = new AST_Call();

src/runtime/builtin_modules/builtins.cpp

@@ -576,6 +576,16 @@ Box* zip2(Box* container1, Box* container2) {
     return rtn;
 }
 
+Box* eval(Box* code) {
+    // TODO implement full functionality (args and stuff)
+    RELEASE_ASSERT(code->cls == str_cls, "eval not implemented for non-strings");
+
+    BoxedDict* locals = getLocals(true /* only_user_visible */, true /* includeClosure */);
+    BoxedModule* module = getCurrentModule();
+
+    return runEval(static_cast<BoxedString*>(code)->s.c_str(), locals, module);
+}
+
 BoxedClass* notimplemented_cls;
 BoxedModule* builtins_module;
 
@@ -743,7 +753,7 @@ Box* globals() {
 }
 
 Box* locals() {
-    return getLocals(true /* filter */);
+    return getLocals(true /* filter */, true /* includeClosure */);
 }
 
 Box* divmod(Box* lhs, Box* rhs) {
@@ -776,7 +786,7 @@ Box* execfile(Box* _fn) {
         raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s.c_str());
 
     // Run directly inside the current module:
-    AST_Module* ast = caching_parse(fn->s.c_str());
+    AST_Module* ast = caching_parse_file(fn->s.c_str());
     compileAndRunModule(ast, getCurrentModule());
 
     return None;
@@ -1201,7 +1211,8 @@ void setupBuiltins() {
     builtins_module->giveAttr("property", property_cls);
     builtins_module->giveAttr("staticmethod", staticmethod_cls);
     builtins_module->giveAttr("classmethod", classmethod_cls);
-
+    builtins_module->giveAttr(
+        "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false)));
 
     PyExc_RecursionErrorInst = new (RuntimeError) BoxedException();
     gc::registerPermanentRoot(PyExc_RecursionErrorInst);

src/runtime/import.cpp

@@ -27,7 +27,7 @@ namespace pyston {
 BoxedModule* createAndRunModule(const std::string& name, const std::string& fn) {
     BoxedModule* module = createModule(name, fn);
 
-    AST_Module* ast = caching_parse(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str());
     compileAndRunModule(ast, module);
     return module;
 }
@@ -42,7 +42,7 @@ static BoxedModule* createAndRunModule(const std::string& name, const std::strin
 
     module->setattr("__path__", path_list, NULL);
 
-    AST_Module* ast = caching_parse(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str());
     compileAndRunModule(ast, module);
     return module;
 }

src/runtime/objmodel.cpp

@@ -154,7 +154,7 @@ extern "C" Box* deopt(AST_expr* expr, Box* value) {
     static StatCounter num_deopt("num_deopt");
     num_deopt.log();
 
-    auto locals = getLocals(false /* filter */);
+    auto locals = getLocals(false /* filter */, false /* includeClosure */);
     auto execution_point = getExecutionPoint();
 
     // Should we only do this selectively?

test/tests/eval_closures.py

-# expected: fail
-# - eval not implemented
-# - closures not implemented
-x = 2
-def wrap():
-    x = 1
-    y = 3
-
-    # The eval('x') in this function will resolve to the global scope:
-    def inner1():
-        y
-        print locals()
-        print eval('x')
-    inner1()
-
-    # The eval('x') in this function will resolve to the closure, since
-    # there is a textual reference to x which causes it to get captured:
-    def inner2():
-        x
-        print locals()
-        print eval('x')
-    inner2()
-
-wrap()

test/tests/eval_test.py

+# TODO lots of eval functionality not implemented
+
+print eval("3 + 4")
+
+a = 5
+print eval("a")
+
+#print eval("[b for b in range(5)]")
+#print b
+
+#c = 2
+#print eval("[c for c in range(5)]")
+#print c
+
+#try:
+#    print eval("int('abc')")
+#except ValueError:
+#    print 'got ValueError'
+
+d = 19
+e = 20
+i = 21
+def func():
+    loc = 231
+    print 'loc', eval("loc")
+
+    print eval("d")
+
+    e = 20
+    print eval("e")
+
+    eval("[f for f in range(5)]")
+
+    eval("[g for g in range(5)]")
+    try:
+        g
+    except NameError:
+        print 'g not found'
+
+    eval("[g2 for g2 in range(5)]")
+    try:
+        print g2
+    except NameError:
+        print 'g2 not found'
+    g2 = 5
+
+    h = 2
+    eval("[h for h in range(5)]")
+    print h
+
+    h2 = 2
+    print eval("h2 + sum([h2 for h2 in range(5)])")
+    print 'h2', h2
+
+    h3 = 2
+    print eval("sum([h3 for h3 in range(5)]) + h3")
+    print 'h3', h3
+
+    eval("[i for i in range(5)]")
+
+    j = 24
+    def inner():
+        return j
+    print 'j', eval("inner()")
+func()
+print i
+
+print eval("(lambda k : k+2)(3)")
+
+l = 100
+print eval("(lambda k : l+2)(3)")
+
+print eval("(lambda k : [m for m in range(5)])(3)")
+try:
+    print m
+except NameError:
+    print 'm not found'
+
+n = 200
+print eval("(lambda k : [n for n in range(5)])(3)")
+print n
+
+print eval("eval('3 + 2342')")
+o = 300
+print 'eval eval o', eval("eval('o')")
+
+#print eval("[(lambda p : p + o)(5) for o in range(5)]")
+
+shadow1 = 1000
+shadow2 = 1000
+shadow3 = 1000
+def func2():
+    shadow1 = 2000
+    print 'shadow1', eval("shadow1")
+
+    shadow2 = 2000
+    eval("[shadow2 for shadow2 in range(5)]")
+    print 'shadow2', shadow2
+
+    print 'shadow3', eval("shadow3 + sum([2 for shadow3 in range(5)]) + shadow3")
+func2()
+#print 'shadow1', shadow2
+#print 'shadow2', shadow2
+#print 'shadow3', shadow3
+
+
+def func3():
+    loc = 1234
+    try:
+        print eval("(lambda arg : arg + loc)(12)")
+    except NameError as e:
+        print 'NameError', e.message
+    try:
+        print eval("loc + (lambda arg : arg + loc)(12)")
+    except NameError as e:
+        print 'NameError', e.message
+func3()
+
+"""
+
+changing_global = -1
+def print_changing_global():
+    print 'changing_global is', changing_global
+    return 0
+eval("[print_changing_global() for changing_global in range(5)]")
+
+"""
+
+def do_changing_local():
+    # this won't get modified:
+    changing_local = -1
+    def print_changing_local():
+        print 'changing_local is', changing_local
+        return 0
+    eval("[print_changing_local() for changing_local in range(5)]")
+do_changing_local()
+
+x = 2
+def wrap():
+    x = 1
+    y = 3
+
+    # The eval('x') in this function will resolve to the global scope:
+    def inner1():
+        y
+        print locals()
+        print eval('x')
+    inner1()
+
+    # The eval('x') in this function will resolve to the closure, since
+    # there is a textual reference to x which causes it to get captured:
+    def inner2():
+        x
+        print locals()
+        print eval('x')
+    inner2()
+
+wrap()

test/unittests/analysis.cpp

@@ -23,7 +23,7 @@ protected:
 
 TEST_F(AnalysisTest, augassign) {
     const std::string fn("test/unittests/analysis_listcomp.py");
-    AST_Module* module = caching_parse(fn.c_str());
+    AST_Module* module = caching_parse_file(fn.c_str());
     assert(module);
 
     ScopingAnalysis *scoping = runScopingAnalysis(module);