Split parsing from compiling

This lets us support `exec compile(ast.parse(s))`.

Also, add 3-arg eval() support, and simple support for the _ast module.

from_cpython/Include/Python.h

@@ -132,6 +132,9 @@ PyObject* PyGC_AddRoot(PyObject*) PYSTON_NOEXCEPT;
 #define PyDoc_STRVAR(name, str) PyDoc_VAR(name) = PyDoc_STR(str)
 #define PyDoc_STR(str) str
 
+// This is in Python-ast.h in CPython, which we don't yet have:
+int PyAST_Check(PyObject* obj) PYSTON_NOEXCEPT;
+
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" void
 #else

src/CMakeLists.txt

@@ -73,6 +73,7 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
 		gc/gc_alloc.cpp
 		gc/heap.cpp
 		runtime/bool.cpp
+		runtime/builtin_modules/ast.cpp
 		runtime/builtin_modules/builtins.cpp
 		runtime/builtin_modules/gc.cpp
 		runtime/builtin_modules/pyston.cpp

src/analysis/scoping_analysis.cpp

@@ -845,7 +845,7 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
 }
 
 InternedStringPool& ScopingAnalysis::getInternedStrings() {
-    return interned_strings;
+    return *interned_strings;
 }
 
 ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
@@ -889,19 +889,25 @@ ScopeInfo* ScopingAnalysis::getScopeInfoForNode(AST* node) {
     return analyzeSubtree(node);
 }
 
-ScopingAnalysis::ScopingAnalysis(AST_Module* m)
-    : parent_module(m), interned_strings(*m->interned_strings.get()), globals_from_module(true) {
-    scopes[m] = new ModuleScopeInfo();
-}
-
-ScopingAnalysis::ScopingAnalysis(AST_Expression* e, bool globals_from_module)
-    : interned_strings(*e->interned_strings.get()), globals_from_module(globals_from_module) {
-    // It's an expression, so it can't have a `global` statement
-    scopes[e] = new EvalExprScopeInfo(globals_from_module);
-}
-
-ScopingAnalysis::ScopingAnalysis(AST_Suite* s, bool globals_from_module)
-    : interned_strings(*s->interned_strings.get()), globals_from_module(globals_from_module) {
-    scopes[s] = new EvalExprScopeInfo(s, globals_from_module);
+ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
+    : parent_module(NULL), globals_from_module(globals_from_module) {
+    switch (ast->type) {
+        case AST_TYPE::Module:
+            assert(globals_from_module);
+            scopes[ast] = new ModuleScopeInfo();
+            interned_strings = static_cast<AST_Module*>(ast)->interned_strings.get();
+            parent_module = static_cast<AST_Module*>(ast);
+            break;
+        case AST_TYPE::Expression:
+            scopes[ast] = new EvalExprScopeInfo(globals_from_module);
+            interned_strings = static_cast<AST_Expression*>(ast)->interned_strings.get();
+            break;
+        case AST_TYPE::Suite:
+            scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
+            interned_strings = static_cast<AST_Suite*>(ast)->interned_strings.get();
+            break;
+        default:
+            RELEASE_ASSERT(0, "%d", ast->type);
+    }
 }
 }

src/analysis/scoping_analysis.h

@@ -151,7 +151,7 @@ public:
 private:
     std::unordered_map<AST*, ScopeInfo*> scopes;
     AST_Module* parent_module;
-    InternedStringPool& interned_strings;
+    InternedStringPool* interned_strings;
 
     std::unordered_map<AST*, AST*> scope_replacements;
 
@@ -170,9 +170,7 @@ public:
     // a scope-node with a different node.
     void registerScopeReplacement(AST* original_node, AST* new_node);
 
-    ScopingAnalysis(AST_Module* m);
-    ScopingAnalysis(AST_Expression* e, bool globals_from_module);
-    ScopingAnalysis(AST_Suite* s, bool globals_from_module);
+    ScopingAnalysis(AST* ast, bool globals_from_module);
     ScopeInfo* getScopeInfoForNode(AST* node);
 
     InternedStringPool& getInternedStrings();

src/codegen/codegen.cpp

@@ -34,9 +34,9 @@ namespace pyston {
 
 DS_DEFINE_RWLOCK(codegen_rwlock);
 
-SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body,
-                       std::string fn)
-    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), fn(std::move(fn)), body(body) {
+SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, std::vector<AST_stmt*> body, std::string fn)
+    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), fn(std::move(fn)),
+      body(std::move(body)) {
     assert(this->fn.size());
 
     switch (ast->type) {

src/codegen/irgen/hooks.cpp

@@ -313,7 +313,7 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         RELEASE_ASSERT(fn, "");
         bm->future_flags = getFutureFlags(m, fn);
 
-        ScopingAnalysis* scoping = new ScopingAnalysis(m);
+        ScopingAnalysis* scoping = new ScopingAnalysis(m, true);
 
         std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, m, m->body, fn));
         bm->setattr("__doc__", si->getDocString(), NULL);
@@ -353,38 +353,78 @@ Box* evalOrExec(CLFunction* cl, Box* globals, Box* boxedLocals) {
     return astInterpretFunctionEval(cf, globals, boxedLocals);
 }
 
-template <typename AST_Type>
-CLFunction* compileForEvalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, std::string fn) {
+CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body, std::string fn) {
     LOCK_REGION(codegen_rwlock.asWrite());
 
     Timer _t("for evalOrExec()");
 
     ScopingAnalysis* scoping = new ScopingAnalysis(source, false);
 
-    std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, source, body, std::move(fn)));
+    std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, source, std::move(body), std::move(fn)));
     CLFunction* cl_f = new CLFunction(0, 0, false, false, std::move(si));
 
     return cl_f;
 }
 
-static CLFunction* compileExec(llvm::StringRef source, llvm::StringRef fn) {
+// TODO: CPython parses execs as Modules, not as Suites.  This is probably not too hard to change,
+// but is non-trivial since we will later decide some things (ex in scoping_analysis) based off
+// the type of the root ast node.
+static AST_Suite* parseExec(llvm::StringRef source) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
     const char* code = source.data();
     AST_Module* parsedModule = parse_string(code);
     AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings));
-    parsedSuite->body = parsedModule->body;
+    parsedSuite->body = std::move(parsedModule->body);
+    return parsedSuite;
+}
 
+static CLFunction* compileExec(AST_Suite* parsedSuite, llvm::StringRef fn) {
     return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn);
 }
 
+static AST_Expression* parseEval(llvm::StringRef source) {
+    const char* code = source.data();
+
+    // TODO error message if parse fails or if it isn't an expr
+    // TODO should have a cleaner interface that can parse the Expression directly
+    // TODO this memory leaks
+
+    // Hack: we need to support things like `eval(" 2")`.
+    // This is over-accepting since it will accept things like `eval("\n 2")`
+    while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
+        code++;
+
+    AST_Module* parsedModule = parse_string(code);
+    if (parsedModule->body.size() == 0)
+        raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
+
+    RELEASE_ASSERT(parsedModule->body.size() == 1, "");
+    RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");
+    AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
+    parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
+    return parsedExpr;
+}
+
+static CLFunction* compileEval(AST_Expression* parsedExpr, llvm::StringRef fn) {
+    // We need body (list of statements) to compile.
+    // Obtain this by simply making a single statement which contains the expression.
+    AST_Return* stmt = new AST_Return();
+    stmt->value = parsedExpr->body;
+    std::vector<AST_stmt*> body = { stmt };
+
+    return compileForEvalOrExec(parsedExpr, std::move(body), fn);
+}
+
 Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     Box* flags = _args[0];
-    Box* dont_inherit = _args[0];
-    RELEASE_ASSERT(flags == boxInt(0), "");
+    Box* dont_inherit = _args[1];
     RELEASE_ASSERT(dont_inherit == boxInt(0), "");
 
+    RELEASE_ASSERT(flags->cls == int_cls, "");
+    int64_t iflags = static_cast<BoxedInt*>(flags)->n;
+
     // source is allowed to be an AST, unicode, or anything that supports the buffer protocol
     if (source->cls == unicode_cls) {
         source = PyUnicode_AsUTF8String(source);
@@ -410,15 +450,47 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     llvm::StringRef filename_str = static_cast<BoxedString*>(fn)->s;
     llvm::StringRef type_str = static_cast<BoxedString*>(type)->s;
 
-    RELEASE_ASSERT(isSubclass(source->cls, str_cls), "");
-    llvm::StringRef source_str = static_cast<BoxedString*>(source)->s;
+    if (iflags & ~(/*PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | */ PyCF_ONLY_AST)) {
+        raiseExcHelper(ValueError, "compile(): unrecognised flags");
+    }
+
+    bool only_ast = (bool)(iflags & PyCF_ONLY_AST);
+    iflags &= ~PyCF_ONLY_AST;
+    RELEASE_ASSERT(iflags == 0, "");
+
+    AST* parsed;
+
+    if (PyAST_Check(source)) {
+        parsed = unboxAst(source);
+    } else {
+        RELEASE_ASSERT(isSubclass(source->cls, str_cls), "");
+        llvm::StringRef source_str = static_cast<BoxedString*>(source)->s;
+
+        if (type_str == "exec") {
+            parsed = parseExec(source_str);
+        } else if (type_str == "eval") {
+            parsed = parseEval(source_str);
+        } else if (type_str == "single") {
+            fatalOrError(NotImplemented, "unimplemented");
+            throwCAPIException();
+        } else {
+            raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
+        }
+    }
+
+    if (only_ast)
+        return boxAst(parsed);
 
     CLFunction* cl;
     if (type_str == "exec") {
-        cl = compileExec(source_str, filename_str);
+        // TODO: CPython parses execs as Modules
+        if (parsed->type != AST_TYPE::Suite)
+            raiseExcHelper(TypeError, "expected Suite node, got %s", boxAst(parsed)->cls->tp_name);
+        cl = compileExec(static_cast<AST_Suite*>(parsed), filename_str);
     } else if (type_str == "eval") {
-        fatalOrError(NotImplemented, "unimplemented");
-        throwCAPIException();
+        if (parsed->type != AST_TYPE::Expression)
+            raiseExcHelper(TypeError, "expected Expression node, got %s", boxAst(parsed)->cls->tp_name);
+        cl = compileEval(static_cast<AST_Expression*>(parsed), filename_str);
     } else if (type_str == "single") {
         fatalOrError(NotImplemented, "unimplemented");
         throwCAPIException();
@@ -429,14 +501,30 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     return codeForCLFunction(cl);
 }
 
-Box* eval(Box* boxedCode) {
-    Box* boxedLocals = fastLocalsToBoxedLocals();
-    BoxedModule* module = getCurrentModule();
-    Box* globals = getGlobals();
+Box* eval(Box* boxedCode, Box* globals, Box* locals) {
+    if (globals == None)
+        globals = NULL;
+
+    if (locals == None)
+        locals = NULL;
 
+    if (locals == NULL) {
+        locals = globals;
+    }
+
+    if (locals == NULL) {
+        locals = fastLocalsToBoxedLocals();
+    }
+
+    if (globals == NULL)
+        globals = getGlobals();
+
+    BoxedModule* module = getCurrentModule();
     if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
         globals = module;
 
+    assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
+
     if (boxedCode->cls == unicode_cls) {
         boxedCode = PyUnicode_AsUTF8String(boxedCode);
         if (!boxedCode)
@@ -444,36 +532,16 @@ Box* eval(Box* boxedCode) {
         // cf.cf_flags |= PyCF_SOURCE_IS_UTF8
     }
 
-    // TODO error message if parse fails or if it isn't an expr
-    // TODO should have a cleaner interface that can parse the Expression directly
-    // TODO this memory leaks
-    RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name);
-    const char* code = static_cast<BoxedString*>(boxedCode)->s.data();
-
-    // Hack: we need to support things like `eval(" 2")`.
-    // This is over-accepting since it will accept things like `eval("\n 2")`
-    while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
-        code++;
-
-    AST_Module* parsedModule = parse_string(code);
-    if (parsedModule->body.size() == 0)
-        raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
-
-    RELEASE_ASSERT(parsedModule->body.size() == 1, "");
-    RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");
-    AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
-    parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
-
-    // We need body (list of statements) to compile.
-    // Obtain this by simply making a single statement which contains the expression.
-    AST_Return* stmt = new AST_Return();
-    stmt->value = parsedExpr->body;
-    std::vector<AST_stmt*> body = { stmt };
-
-    assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
-
-    CLFunction* cl = compileForEvalOrExec(parsedExpr, body, "<string>");
-    return evalOrExec(cl, globals, boxedLocals);
+    CLFunction* cl;
+    if (boxedCode->cls == str_cls) {
+        AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s);
+        cl = compileEval(parsed, "<string>");
+    } else if (boxedCode->cls == code_cls) {
+        cl = clfunctionFromCode(boxedCode);
+    } else {
+        abort();
+    }
+    return evalOrExec(cl, globals, locals);
 }
 
 Box* exec(Box* boxedCode, Box* globals, Box* locals) {
@@ -530,7 +598,8 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        cl = compileExec(static_cast<BoxedString*>(boxedCode)->s, "<string>");
+        AST_Suite* parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s);
+        cl = compileExec(parsed, "<string>");
     } else if (boxedCode->cls == code_cls) {
         cl = clfunctionFromCode(boxedCode);
     } else {

src/codegen/irgen/hooks.h

@@ -38,7 +38,7 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals);
-extern "C" Box* eval(Box* boxedCode);
+extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
 extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
 }
 

src/core/types.h

@@ -261,7 +261,7 @@ public:
 
     Box* getDocString();
 
-    SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body, std::string fn);
+    SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, std::vector<AST_stmt*> body, std::string fn);
 };
 
 typedef std::vector<CompiledFunction*> FunctionList;

src/runtime/builtin_modules/ast.cpp

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cmath>
+#include <langinfo.h>
+#include <sstream>
+
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+#include "codegen/unwinding.h"
+#include "core/types.h"
+#include "gc/collector.h"
+#include "runtime/file.h"
+#include "runtime/inline/boxing.h"
+#include "runtime/int.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+static BoxedClass* AST_cls;
+
+class BoxedAST : public Box {
+public:
+    AST* ast;
+
+    BoxedAST() {}
+};
+
+static std::unordered_map<int, BoxedClass*> type_to_cls;
+
+Box* boxAst(AST* ast) {
+    assert(ast);
+    BoxedClass* cls = type_to_cls[ast->type];
+    assert(cls);
+
+    BoxedAST* rtn = new (cls) BoxedAST();
+    assert(rtn->cls == cls);
+    rtn->ast = ast;
+    return rtn;
+}
+
+AST* unboxAst(Box* b) {
+    assert(isSubclass(b->cls, AST_cls));
+    AST* rtn = static_cast<BoxedAST*>(b)->ast;
+    assert(rtn);
+    return rtn;
+}
+
+extern "C" int PyAST_Check(PyObject* o) noexcept {
+    return isSubclass(o->cls, AST_cls);
+}
+
+void setupAST() {
+    BoxedModule* ast_module = createModule("_ast", "__builtin__");
+
+    ast_module->giveAttr("PyCF_ONLY_AST", boxInt(PyCF_ONLY_AST));
+
+// ::create takes care of registering the class as a GC root.
+#define MAKE_CLS(name, base_cls)                                                                                       \
+    BoxedClass* name##_cls = BoxedHeapClass::create(type_cls, base_cls, /* gchandler = */ NULL, 0, 0,                  \
+                                                    sizeof(BoxedAST), false, STRINGIFY(name));                         \
+    ast_module->giveAttr(STRINGIFY(name), name##_cls);                                                                 \
+    type_to_cls[AST_TYPE::name] = name##_cls;                                                                          \
+    name##_cls->giveAttr("__module__", boxString("_ast"));                                                             \
+    name##_cls->freeze()
+
+    AST_cls
+        = BoxedHeapClass::create(type_cls, object_cls, /* gchandler = */ NULL, 0, 0, sizeof(BoxedAST), false, "AST");
+    // ::create takes care of registering the class as a GC root.
+    AST_cls->giveAttr("__module__", boxString("_ast"));
+    AST_cls->freeze();
+
+    // TODO(kmod) you can call the class constructors, such as "ast.AST()", so we need new/init
+    // TODO(kmod) there is more inheritance than "they all inherit from AST"
+
+    MAKE_CLS(alias, AST_cls);
+    MAKE_CLS(arguments, AST_cls);
+    MAKE_CLS(Assert, AST_cls);
+    MAKE_CLS(Assign, AST_cls);
+    MAKE_CLS(Attribute, AST_cls);
+    MAKE_CLS(AugAssign, AST_cls);
+    MAKE_CLS(BinOp, AST_cls);
+    MAKE_CLS(BoolOp, AST_cls);
+    MAKE_CLS(Call, AST_cls);
+    MAKE_CLS(ClassDef, AST_cls);
+    MAKE_CLS(Compare, AST_cls);
+    MAKE_CLS(comprehension, AST_cls);
+    MAKE_CLS(Delete, AST_cls);
+    MAKE_CLS(Dict, AST_cls);
+    MAKE_CLS(Exec, AST_cls);
+    MAKE_CLS(ExceptHandler, AST_cls);
+    MAKE_CLS(ExtSlice, AST_cls);
+    MAKE_CLS(Expr, AST_cls);
+    MAKE_CLS(For, AST_cls);
+    MAKE_CLS(FunctionDef, AST_cls);
+    MAKE_CLS(GeneratorExp, AST_cls);
+    MAKE_CLS(Global, AST_cls);
+    MAKE_CLS(If, AST_cls);
+    MAKE_CLS(IfExp, AST_cls);
+    MAKE_CLS(Import, AST_cls);
+    MAKE_CLS(ImportFrom, AST_cls);
+    MAKE_CLS(Index, AST_cls);
+    MAKE_CLS(keyword, AST_cls);
+    MAKE_CLS(Lambda, AST_cls);
+    MAKE_CLS(List, AST_cls);
+    MAKE_CLS(ListComp, AST_cls);
+    MAKE_CLS(Module, AST_cls);
+    MAKE_CLS(Num, AST_cls);
+    MAKE_CLS(Name, AST_cls);
+    MAKE_CLS(Pass, AST_cls);
+    MAKE_CLS(Pow, AST_cls);
+    MAKE_CLS(Print, AST_cls);
+    MAKE_CLS(Raise, AST_cls);
+    MAKE_CLS(Repr, AST_cls);
+    MAKE_CLS(Return, AST_cls);
+    MAKE_CLS(Slice, AST_cls);
+    MAKE_CLS(Str, AST_cls);
+    MAKE_CLS(Subscript, AST_cls);
+    MAKE_CLS(TryExcept, AST_cls);
+    MAKE_CLS(TryFinally, AST_cls);
+    MAKE_CLS(Tuple, AST_cls);
+    MAKE_CLS(UnaryOp, AST_cls);
+    MAKE_CLS(With, AST_cls);
+    MAKE_CLS(While, AST_cls);
+    MAKE_CLS(Yield, AST_cls);
+    MAKE_CLS(Store, AST_cls);
+    MAKE_CLS(Load, AST_cls);
+    MAKE_CLS(Param, AST_cls);
+    MAKE_CLS(Not, AST_cls);
+    MAKE_CLS(In, AST_cls);
+    MAKE_CLS(Is, AST_cls);
+    MAKE_CLS(IsNot, AST_cls);
+    MAKE_CLS(Or, AST_cls);
+    MAKE_CLS(And, AST_cls);
+    MAKE_CLS(Eq, AST_cls);
+    MAKE_CLS(NotEq, AST_cls);
+    MAKE_CLS(NotIn, AST_cls);
+    MAKE_CLS(GtE, AST_cls);
+    MAKE_CLS(Gt, AST_cls);
+    MAKE_CLS(Mod, AST_cls);
+    MAKE_CLS(Add, AST_cls);
+    MAKE_CLS(Continue, AST_cls);
+    MAKE_CLS(Lt, AST_cls);
+    MAKE_CLS(LtE, AST_cls);
+    MAKE_CLS(Break, AST_cls);
+    MAKE_CLS(Sub, AST_cls);
+    MAKE_CLS(Del, AST_cls);
+    MAKE_CLS(Mult, AST_cls);
+    MAKE_CLS(Div, AST_cls);
+    MAKE_CLS(USub, AST_cls);
+    MAKE_CLS(BitAnd, AST_cls);
+    MAKE_CLS(BitOr, AST_cls);
+    MAKE_CLS(BitXor, AST_cls);
+    MAKE_CLS(RShift, AST_cls);
+    MAKE_CLS(LShift, AST_cls);
+    MAKE_CLS(Invert, AST_cls);
+    MAKE_CLS(UAdd, AST_cls);
+    MAKE_CLS(FloorDiv, AST_cls);
+    MAKE_CLS(DictComp, AST_cls);
+    MAKE_CLS(Set, AST_cls);
+    MAKE_CLS(Ellipsis, AST_cls);
+    MAKE_CLS(Expression, AST_cls);
+    MAKE_CLS(SetComp, AST_cls);
+    MAKE_CLS(Suite, AST_cls);
+
+
+#undef MAKE_CLS
+
+    // Uncommenting this makes `import ast` work, which may or may not be desired.
+    // For now it seems like making the import fail is better than having the module not work properly.
+    // ast_module->giveAttr("__version__", boxInt(82160));
+}
+}

src/runtime/builtin_modules/builtins.cpp

@@ -1211,8 +1211,9 @@ void setupBuiltins() {
     PyType_Ready(&PyBuffer_Type);
     builtins_module->giveAttr("buffer", &PyBuffer_Type);
 
-    builtins_module->giveAttr(
-        "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false), "eval"));
+    builtins_module->giveAttr("eval",
+                              new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 3, 2, false, false),
+                                                               "eval", { NULL, NULL }));
     builtins_module->giveAttr("callable",
                               new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)callable, UNKNOWN, 1), "callable"));
 

src/runtime/types.cpp

@@ -2323,6 +2323,7 @@ void setupRuntime() {
     setupGC();
     setupImport();
     setupPyston();
+    setupAST();
 
     PyType_Ready(&PyByteArrayIter_Type);
     PyType_Ready(&PyCapsule_Type);

src/runtime/types.h

@@ -75,6 +75,7 @@ void setupBuiltins();
 void setupPyston();
 void setupThread();
 void setupImport();
+void setupAST();
 void setupSysEnd();
 
 BoxedDict* getSysModulesDict();
@@ -834,6 +835,9 @@ Box* unwrapAttrWrapper(Box* b);
 Box* attrwrapperKeys(Box* b);
 void attrwrapperDel(Box* b, const std::string& attr);
 
+Box* boxAst(AST* ast);
+AST* unboxAst(Box* b);
+
 #define SystemError ((BoxedClass*)PyExc_SystemError)
 #define StopIteration ((BoxedClass*)PyExc_StopIteration)
 #define NameError ((BoxedClass*)PyExc_NameError)

test/tests/ast_test.py

+import _ast
+
+def ast_parse(source, filename='<unknown>', mode='exec'):
+    return compile(source, filename, mode, _ast.PyCF_ONLY_AST)
+
+
+# TODO(kmod) unfortunately we don't expose any of the data members yet, and we don't even display
+# the types correctly, so just parse the code and make sure that we didn't crash
+ast_parse("print 1")
+ast_parse("print 1", "t.py", "exec")
+ast_parse("1", "t.py", "eval")
+
+c = compile(ast_parse("print 1", "t.py", "exec"), "u.py", "exec")
+print c.co_filename
+exec c
+
+try:
+    c = compile(ast_parse("print 1", "t.py", "exec"), "u.py", "eval")
+except Exception as e:
+    print type(e)
+
+c = compile(ast_parse("print 2", "t.py", "exec"), "u.py", "exec")
+print eval(c)

test/tests/compile_eval.py

+c = compile("a", "test.py", "eval")
+print type(c), c.co_filename, c.co_name
+
+print
+a = 0
+print eval(c)
+
+print
+a = 0
+g = {'a':1}
+print eval(c, g)
+
+print
+a = 0
+g = {'a':1}
+l = {'a':2}
+print eval(c, g, l)
+
+print
+g = {}
+exec """
+c = compile("a", "test.py", "eval")
+""" in g
+a = 0
+print eval(g['c'])
+
+print
+a = 0
+g = {'a':1, '_c':c}
+exec "print eval(_c)" in g

test/unittests/analysis.cpp

@@ -27,7 +27,7 @@ TEST_F(AnalysisTest, augassign) {
     AST_Module* module = caching_parse_file(fn.c_str());
     assert(module);
 
-    ScopingAnalysis *scoping = new ScopingAnalysis(module);
+    ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
@@ -57,7 +57,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_Module* module = caching_parse_file(fn.c_str());
     assert(module);
 
-    ScopingAnalysis *scoping = new ScopingAnalysis(module);
+    ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);