Add support for using the CPython parser

CMakeLists.txt

@@ -287,9 +287,9 @@ endmacro()
 
 # tests         testname directory arguments
 add_pyston_test(defaults tests --order-by-mtime -t50)
-add_pyston_test(force_llvm tests -a=-n -a=-x -t50)
+add_pyston_test(force_llvm tests -a=-n -a=-X -t50)
 if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
-  add_pyston_test(max_compilation_tier tests -a=-O -a=-x -t50)
+  add_pyston_test(max_compilation_tier tests -a=-O -a=-X -t50)
 endif()
 add_pyston_test(defaults cpython --exit-code-only --skip-failing -t50)
 add_pyston_test(defaults integration --exit-code-only --skip-failing -t600)

Makefile

@@ -779,7 +779,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t50 $(TEST_DIR)/cpython $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t600 $(TEST_DIR)/integration $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-X -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 
 .PHONY: run$1 dbg$1
@@ -825,6 +825,7 @@ perf_report:
 
 .PHONY: run run_% dbg_% debug_% perf_%
 run: run_dbg
+dbg: dbg_dbg
 run_%: run_dbg_%
 	@true
 dbg_%: dbg_dbg_%

src/capi/errors.cpp

@@ -256,7 +256,35 @@ finally:
 }
 
 static void print_error_text(PyObject* f, int offset, const char* text) noexcept {
-    Py_FatalError("unimplemented");
+    const char* nl;
+    if (offset >= 0) {
+        if (offset > 0 && offset == strlen(text) && text[offset - 1] == '\n')
+            offset--;
+        for (;;) {
+            nl = strchr(text, '\n');
+            if (nl == NULL || nl - text >= offset)
+                break;
+            offset -= (int)(nl + 1 - text);
+            text = nl + 1;
+        }
+        while (*text == ' ' || *text == '\t') {
+            text++;
+            offset--;
+        }
+    }
+    PyFile_WriteString("    ", f);
+    PyFile_WriteString(text, f);
+    if (*text == '\0' || text[strlen(text) - 1] != '\n')
+        PyFile_WriteString("\n", f);
+    if (offset == -1)
+        return;
+    PyFile_WriteString("    ", f);
+    offset--;
+    while (offset > 0) {
+        PyFile_WriteString(" ", f);
+        offset--;
+    }
+    PyFile_WriteString("^\n", f);
 }
 
 extern "C" void PyErr_Display(PyObject* exception, PyObject* value, PyObject* tb) noexcept {
@@ -469,4 +497,45 @@ extern "C" void PyErr_PrintEx(int set_sys_last_vars) noexcept {
 extern "C" void PyErr_Print() noexcept {
     PyErr_PrintEx(1);
 }
+
+/* com_fetch_program_text will attempt to load the line of text that
+   the exception refers to.  If it fails, it will return NULL but will
+   not set an exception.
+
+   XXX The functionality of this function is quite similar to the
+   functionality in tb_displayline() in traceback.c.
+*/
+
+extern "C" PyObject* PyErr_ProgramText(const char* filename, int lineno) noexcept {
+    FILE* fp;
+    int i;
+    char linebuf[1000];
+
+    if (filename == NULL || *filename == '\0' || lineno <= 0)
+        return NULL;
+    fp = fopen(filename, "r" PY_STDIOTEXTMODE);
+    if (fp == NULL)
+        return NULL;
+    for (i = 0; i < lineno; i++) {
+        char* pLastChar = &linebuf[sizeof(linebuf) - 2];
+        do {
+            *pLastChar = '\0';
+            if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, fp, NULL) == NULL)
+                break;
+            /* fgets read *something*; if it didn't get as
+               far as pLastChar, it must have found a newline
+               or hit the end of the file; if pLastChar is \n,
+               it obviously found a newline; else we haven't
+               yet seen a newline, so must continue */
+        } while (*pLastChar != '\0' && *pLastChar != '\n');
+    }
+    fclose(fp);
+    if (i == lineno) {
+        char* p = linebuf;
+        while (*p == ' ' || *p == '\t' || *p == '\014')
+            p++;
+        return PyString_FromString(p);
+    }
+    return NULL;
+}
 }

src/codegen/irgen/hooks.cpp

@@ -387,18 +387,8 @@ static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, b
     const char* code = source.data();
     AST_Module* parsedModule = parse_string(code, future_flags);
 
-    if (interactive) {
-        for (int i = 0; i < parsedModule->body.size(); ++i) {
-            AST_stmt* s = parsedModule->body[i];
-            if (s->type != AST_TYPE::Expr)
-                continue;
-
-            AST_Expr* expr = (AST_Expr*)s;
-            AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
-            print_expr->args.push_back(expr->value);
-            expr->value = print_expr;
-        }
-    }
+    if (interactive)
+        makeModuleInteractive(parsedModule);
 
     return parsedModule;
 }

src/codegen/parser.cpp

@@ -14,6 +14,8 @@
 
 #include "codegen/parser.h"
 
+#include "cpython_ast.h"
+
 #include <cassert>
 #include <cstdio>
 #include <cstdlib>
@@ -32,6 +34,7 @@
 #include "core/stats.h"
 #include "core/types.h"
 #include "core/util.h"
+#include "runtime/types.h"
 
 //#undef VERBOSITY
 //#define VERBOSITY(x) 2
@@ -1046,6 +1049,21 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
 AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
     Timer _t("parsing");
 
+    if (ENABLE_CPYTHON_PARSER) {
+        ASSERT(!inherited_flags, "unimplemented");
+        FILE* fp = fopen(fn, "r");
+        PyCompilerFlags cf;
+        cf.cf_flags = 0;
+        PyArena* arena = PyArena_New();
+        assert(arena);
+        mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
+        if (!mod)
+            throwCAPIException();
+        auto rtn = cpythonToPystonAST(mod);
+        PyArena_Free(arena);
+        return rtn;
+    }
+
     if (ENABLE_PYPA_PARSER) {
         AST_Module* rtn = pypa_parse(fn, inherited_flags);
         RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
@@ -1073,7 +1091,9 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
 }
 
 const char* getMagic() {
-    if (ENABLE_PYPA_PARSER)
+    if (ENABLE_CPYTHON_PARSER)
+        return "a\nCO";
+    else if (ENABLE_PYPA_PARSER)
         return "a\ncO";
     else
         return "a\nco";
@@ -1116,9 +1136,22 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
     file_data.insert(file_data.end(), (char*)&checksum, (char*)&checksum + CHECKSUM_LENGTH);
     checksum = 0;
 
-    if (ENABLE_PYPA_PARSER || inherited_flags) {
-        module = pypa_parse(fn, inherited_flags);
-        RELEASE_ASSERT(module, "unknown parse error");
+    if (ENABLE_CPYTHON_PARSER || ENABLE_PYPA_PARSER || inherited_flags) {
+        if (ENABLE_CPYTHON_PARSER) {
+            ASSERT(!inherited_flags, "unimplemented");
+            FILE* fp = fopen(fn, "r");
+            PyCompilerFlags cf;
+            cf.cf_flags = 0;
+            PyArena* arena = PyArena_New();
+            assert(arena);
+            mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
+            if (!mod)
+                throwCAPIException();
+            module = cpythonToPystonAST(mod);
+        } else {
+            module = pypa_parse(fn, inherited_flags);
+            RELEASE_ASSERT(module, "unknown parse error");
+        }
 
         if (!cache_fp)
             return std::vector<char>();

src/core/ast.cpp

@@ -2232,4 +2232,17 @@ void flatten(AST_expr* root, std::vector<AST*>& output, bool expand_scopes) {
 
     root->accept(&visitor);
 }
+
+void makeModuleInteractive(AST_Module* m) {
+    for (int i = 0; i < m->body.size(); ++i) {
+        AST_stmt* s = m->body[i];
+        if (s->type != AST_TYPE::Expr)
+            continue;
+
+        AST_Expr* expr = (AST_Expr*)s;
+        AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
+        print_expr->args.push_back(expr->value);
+        expr->value = print_expr;
+    }
+}
 }

src/core/ast.h

@@ -447,6 +447,7 @@ public:
     virtual void accept_stmt(StmtVisitor* v);
 
     AST_Expr() : AST_stmt(AST_TYPE::Expr) {}
+    AST_Expr(AST_expr* value) : AST_stmt(AST_TYPE::Expr), value(value) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expr;
 };
@@ -1425,6 +1426,10 @@ template <class T, class R> void findNodes(const R& roots, std::vector<T*>& outp
     }
 }
 
+// Take a normally-parsed module, and convert it (inplace) to a form that will print out any bare expressions.
+// This is used for "single" mode or the repl.
+void makeModuleInteractive(AST_Module* m);
+
 llvm::StringRef getOpSymbol(int op_type);
 BoxedString* getOpName(int op_type);
 int getReverseCmpOp(int op_type, bool& success);

src/core/options.cpp

@@ -41,6 +41,7 @@ bool USE_STRIPPED_STDLIB = true; // always true
 bool ENABLE_INTERPRETER = true;
 bool ENABLE_BASELINEJIT = true;
 bool ENABLE_PYPA_PARSER = true;
+bool ENABLE_CPYTHON_PARSER = false;
 bool USE_REGALLOC_BASIC = true;
 bool PAUSE_AT_ABORT = false;
 bool ENABLE_TRACEBACKS = true;
@@ -90,9 +91,11 @@ bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;
 extern "C" {
 int Py_FrozenFlag = 1;
 int Py_IgnoreEnvironmentFlag = 0;
+int Py_InteractiveFlag = 0;
 int Py_InspectFlag = 0;
 int Py_NoSiteFlag = 0;
 int Py_OptimizeFlag = 0;
 int Py_VerboseFlag = 0;
+int Py_UnicodeFlag = 0;
 }
 }

src/core/options.h

@@ -38,8 +38,9 @@ extern int SPECULATION_THRESHOLD;
 extern int MAX_OBJECT_CACHE_ENTRIES;
 
 extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
-    CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
-    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
+    CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, ENABLE_CPYTHON_PARSER,
+    USE_REGALLOC_BASIC, PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS,
+    FORCE_LLVM_CAPI_THROWS;
 
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,

src/jit.cpp

@@ -193,9 +193,10 @@ int handleArg(char code) {
         SHOW_DISASM = true;
     else if (code == 'I')
         FORCE_INTERPRETER = true;
-    else if (code == 'i')
+    else if (code == 'i') {
         Py_InspectFlag = true;
-    else if (code == 'n') {
+        Py_InteractiveFlag = true;
+    } else if (code == 'n') {
         ENABLE_INTERPRETER = false;
     } else if (code == 'a') {
         ASSEMBLY_LOGGING = true;
@@ -207,6 +208,8 @@ int handleArg(char code) {
         Stats::setEnabled(true);
     } else if (code == 'S') {
         Py_NoSiteFlag = 1;
+    } else if (code == 'U') {
+        Py_UnicodeFlag++;
     } else if (code == 'u') {
         unbuffered = true;
     } else if (code == 'r') {
@@ -215,6 +218,8 @@ int handleArg(char code) {
         USE_REGALLOC_BASIC = false;
     } else if (code == 'x') {
         ENABLE_PYPA_PARSER = false;
+    } else if (code == 'X') {
+        ENABLE_CPYTHON_PARSER = true;
     } else if (code == 'E') {
         Py_IgnoreEnvironmentFlag = 1;
     } else if (code == 'P') {
@@ -325,7 +330,7 @@ static int main(int argc, char** argv) {
 
         // Suppress getopt errors so we can throw them ourselves
         opterr = 0;
-        while ((code = getopt(argc, argv, "+:OqdIibpjtrsRSvnxEac:FuPTGm:")) != -1) {
+        while ((code = getopt(argc, argv, "+:OqdIibpjtrsRSUvnxXEac:FuPTGm:")) != -1) {
             if (code == 'c') {
                 assert(optarg);
                 command = optarg;
@@ -497,42 +502,19 @@ static int main(int argc, char** argv) {
         }
 
         if (Py_InspectFlag || !(command || fn || module)) {
+
+            PyObject* v = PyImport_ImportModule("readline");
+            if (!v)
+                PyErr_Clear();
+
             printf("Pyston v%d.%d (rev " STRINGIFY(GITREV) ")", PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR);
             printf(", targeting Python %d.%d.%d\n", PYTHON_VERSION_MAJOR, PYTHON_VERSION_MINOR, PYTHON_VERSION_MICRO);
 
             Py_InspectFlag = 0;
 
-            if (!main_module) {
-                main_module = createModule(boxString("__main__"), "<stdin>");
-            } else {
-                // main_module->fn = "<stdin>";
-            }
-
-            for (;;) {
-                char* line = readline(">> ");
-                if (!line)
-                    break;
-
-                add_history(line);
-
-                try {
-                    AST_Module* m = parse_string(line, /* future_flags = */ 0);
-
-                    Timer _t("repl");
-
-                    if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
-                        AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
-                        AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
-                        print_expr->args.push_back(e->value);
-                        e->value = print_expr;
-                    }
-
-                    compileAndRunModule(m, main_module);
-                } catch (ExcInfo e) {
-                    setCAPIException(e);
-                    PyErr_Print();
-                }
-            }
+            PyCompilerFlags cf;
+            cf.cf_flags = 0;
+            rtncode = PyRun_InteractiveLoopFlags(stdin, "<stdin>", &cf);
         }
 
         threading::finishMainThread();

src/runtime/builtin_modules/sys.cpp

@@ -151,6 +151,16 @@ extern "C" PyObject* PySys_GetObject(const char* name) noexcept {
     return sys_module->getattr(internStringMortal(name));
 }
 
+extern "C" FILE* PySys_GetFile(char* name, FILE* def) noexcept {
+    FILE* fp = NULL;
+    PyObject* v = PySys_GetObject(name);
+    if (v != NULL && PyFile_Check(v))
+        fp = PyFile_AsFile(v);
+    if (fp == NULL)
+        fp = def;
+    return fp;
+}
+
 static void mywrite(const char* name, FILE* fp, const char* format, va_list va) noexcept {
     PyObject* file;
     PyObject* error_type, *error_value, *error_traceback;

src/runtime/capi.cpp

@@ -18,12 +18,23 @@
 
 #include "Python.h"
 
+#include "codegen/cpython_ast.h"
+#include "grammar.h"
+#include "node.h"
+#include "token.h"
+#include "parsetok.h"
+#include "errcode.h"
+#include "ast.h"
+#undef BYTE
+#undef STRING
+
 #include "llvm/Support/ErrorHandling.h" // For llvm_unreachable
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
 #include "capi/typeobject.h"
 #include "capi/types.h"
+#include "codegen/irgen/hooks.h"
 #include "codegen/unwinding.h"
 #include "core/threading.h"
 #include "core/types.h"
@@ -1164,6 +1175,253 @@ static int dev_urandom_python(char* buffer, Py_ssize_t size) noexcept {
 }
 }
 
+extern "C" int Py_FdIsInteractive(FILE* fp, const char* filename) noexcept {
+    if (isatty((int)fileno(fp)))
+        return 1;
+    if (!Py_InteractiveFlag)
+        return 0;
+    return (filename == NULL) || (strcmp(filename, "<stdin>") == 0) || (strcmp(filename, "???") == 0);
+}
+
+extern "C" int PyRun_InteractiveOneFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
+    PyObject* m, *d, *v, *w;
+    mod_ty mod;
+    PyArena* arena;
+
+    char _buf[1] = "";
+    char* ps1 = _buf, * ps2 = _buf;
+    int errcode = 0;
+
+    v = PySys_GetObject("ps1");
+    if (v != NULL) {
+        v = PyObject_Str(v);
+        if (v == NULL)
+            PyErr_Clear();
+        else if (PyString_Check(v))
+            ps1 = PyString_AsString(v);
+    }
+    w = PySys_GetObject("ps2");
+    if (w != NULL) {
+        w = PyObject_Str(w);
+        if (w == NULL)
+            PyErr_Clear();
+        else if (PyString_Check(w))
+            ps2 = PyString_AsString(w);
+    }
+    arena = PyArena_New();
+    if (arena == NULL) {
+        Py_XDECREF(v);
+        Py_XDECREF(w);
+        return -1;
+    }
+    mod = PyParser_ASTFromFile(fp, filename, Py_single_input, ps1, ps2, flags, &errcode, arena);
+    Py_XDECREF(v);
+    Py_XDECREF(w);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        if (errcode == E_EOF) {
+            PyErr_Clear();
+            return E_EOF;
+        }
+        PyErr_Print();
+        return -1;
+    }
+    m = PyImport_AddModule("__main__");
+    if (m == NULL) {
+        PyArena_Free(arena);
+        return -1;
+    }
+
+    // Pyston change:
+    // d = PyModule_GetDict(m);
+    // v = run_mod(mod, filename, d, d, flags, arena);
+    assert(PyModule_Check(m));
+    AST_Module* pyston_module = cpythonToPystonAST(mod);
+    makeModuleInteractive(pyston_module);
+
+    bool failed = false;
+    try {
+        compileAndRunModule(pyston_module, static_cast<BoxedModule*>(m));
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        failed = true;
+    }
+
+    PyArena_Free(arena);
+    if (failed) {
+        PyErr_Print();
+        return -1;
+    }
+    Py_DECREF(v);
+    if (Py_FlushLine())
+        PyErr_Clear();
+    return 0;
+}
+
+/* Set the error appropriate to the given input error code (see errcode.h) */
+
+static void err_input(perrdetail* err) noexcept {
+    PyObject* v, *w, *errtype;
+    PyObject* u = NULL;
+    const char* msg = NULL;
+    errtype = PyExc_SyntaxError;
+    switch (err->error) {
+        case E_ERROR:
+            return;
+        case E_SYNTAX:
+            errtype = PyExc_IndentationError;
+            if (err->expected == INDENT)
+                msg = "expected an indented block";
+            else if (err->token == INDENT)
+                msg = "unexpected indent";
+            else if (err->token == DEDENT)
+                msg = "unexpected unindent";
+            else {
+                errtype = PyExc_SyntaxError;
+                msg = "invalid syntax";
+            }
+            break;
+        case E_TOKEN:
+            msg = "invalid token";
+            break;
+        case E_EOFS:
+            msg = "EOF while scanning triple-quoted string literal";
+            break;
+        case E_EOLS:
+            msg = "EOL while scanning string literal";
+            break;
+        case E_INTR:
+            if (!PyErr_Occurred())
+                PyErr_SetNone(PyExc_KeyboardInterrupt);
+            goto cleanup;
+        case E_NOMEM:
+            PyErr_NoMemory();
+            goto cleanup;
+        case E_EOF:
+            msg = "unexpected EOF while parsing";
+            break;
+        case E_TABSPACE:
+            errtype = PyExc_TabError;
+            msg = "inconsistent use of tabs and spaces in indentation";
+            break;
+        case E_OVERFLOW:
+            msg = "expression too long";
+            break;
+        case E_DEDENT:
+            errtype = PyExc_IndentationError;
+            msg = "unindent does not match any outer indentation level";
+            break;
+        case E_TOODEEP:
+            errtype = PyExc_IndentationError;
+            msg = "too many levels of indentation";
+            break;
+        case E_DECODE: {
+            PyObject* type, *value, *tb;
+            PyErr_Fetch(&type, &value, &tb);
+            if (value != NULL) {
+                u = PyObject_Str(value);
+                if (u != NULL) {
+                    msg = PyString_AsString(u);
+                }
+            }
+            if (msg == NULL)
+                msg = "unknown decode error";
+            Py_XDECREF(type);
+            Py_XDECREF(value);
+            Py_XDECREF(tb);
+            break;
+        }
+        case E_LINECONT:
+            msg = "unexpected character after line continuation character";
+            break;
+        default:
+            fprintf(stderr, "error=%d\n", err->error);
+            msg = "unknown parsing error";
+            break;
+    }
+    v = Py_BuildValue("(ziiz)", err->filename, err->lineno, err->offset, err->text);
+    w = NULL;
+    if (v != NULL)
+        w = Py_BuildValue("(sO)", msg, v);
+    Py_XDECREF(u);
+    Py_XDECREF(v);
+    PyErr_SetObject(errtype, w);
+    Py_XDECREF(w);
+cleanup:
+    if (err->text != NULL) {
+        PyObject_FREE(err->text);
+        err->text = NULL;
+    }
+}
+#if 0
+/* compute parser flags based on compiler flags */
+#define PARSER_FLAGS(flags)                                                                                            \
+    ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0)
+#endif
+#if 1
+/* Keep an example of flags with future keyword support. */
+#define PARSER_FLAGS(flags)                                                                                            \
+    ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? PyPARSE_DONT_IMPLY_DEDENT : 0)                         \
+                | (((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION) ? PyPARSE_PRINT_IS_FUNCTION : 0)                     \
+                | (((flags)->cf_flags & CO_FUTURE_UNICODE_LITERALS) ? PyPARSE_UNICODE_LITERALS : 0))                   \
+             : 0)
+#endif
+
+extern "C" grammar _PyParser_Grammar;
+extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start, char* ps1, char* ps2,
+                                       PyCompilerFlags* flags, int* errcode, PyArena* arena) noexcept {
+    mod_ty mod;
+    PyCompilerFlags localflags;
+    perrdetail err;
+    int iflags = PARSER_FLAGS(flags);
+
+    node* n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, start, ps1, ps2, &err, &iflags);
+    if (flags == NULL) {
+        localflags.cf_flags = 0;
+        flags = &localflags;
+    }
+    if (n) {
+        flags->cf_flags |= iflags & PyCF_MASK;
+        mod = PyAST_FromNode(n, flags, filename, arena);
+        PyNode_Free(n);
+        return mod;
+    } else {
+        err_input(&err);
+        if (errcode)
+            *errcode = err.error;
+        return NULL;
+    }
+}
+
+extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
+    PyObject* v;
+    int ret;
+    PyCompilerFlags local_flags;
+
+    if (flags == NULL) {
+        flags = &local_flags;
+        local_flags.cf_flags = 0;
+    }
+    v = PySys_GetObject("ps1");
+    if (v == NULL) {
+        PySys_SetObject("ps1", v = PyString_FromString(">> "));
+        Py_XDECREF(v);
+    }
+    v = PySys_GetObject("ps2");
+    if (v == NULL) {
+        PySys_SetObject("ps2", v = PyString_FromString("... "));
+        Py_XDECREF(v);
+    }
+    for (;;) {
+        ret = PyRun_InteractiveOneFlags(fp, filename, flags);
+        // PRINT_TOTAL_REFS();
+        if (ret == E_EOF)
+            return 0;
+        if (ret == E_NOMEM)
+            return -1;
+    }
+}
+
 static const char* progname = "pyston";
 extern "C" void Py_SetProgramName(char* pn) noexcept {
     if (pn && *pn)
@@ -1642,6 +1900,13 @@ Box* BoxedCApiFunction::tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPa
     return rtn;
 }
 
+/* Warning with explicit origin */
+extern "C" int PyErr_WarnExplicit(PyObject* category, const char* text, const char* filename_str, int lineno,
+                                  const char* module_str, PyObject* registry) noexcept {
+    Py_FatalError("unimplemented");
+}
+
+
 /* extension modules might be compiled with GC support so these
    functions must always be available */
 

src/runtime/file.cpp

@@ -130,6 +130,98 @@ static PyObject* err_iterbuffered(void) noexcept {
     return NULL;
 }
 
+/*
+** Py_UniversalNewlineFgets is an fgets variation that understands
+** all of \r, \n and \r\n conventions.
+** The stream should be opened in binary mode.
+** If fobj is NULL the routine always does newline conversion, and
+** it may peek one char ahead to gobble the second char in \r\n.
+** If fobj is non-NULL it must be a PyFileObject. In this case there
+** is no readahead but in stead a flag is used to skip a following
+** \n on the next read. Also, if the file is open in binary mode
+** the whole conversion is skipped. Finally, the routine keeps track of
+** the different types of newlines seen.
+** Note that we need no error handling: fgets() treats error and eof
+** identically.
+*/
+extern "C" char* Py_UniversalNewlineFgets(char* buf, int n, FILE* stream, PyObject* fobj) noexcept {
+    char* p = buf;
+    int c;
+    int newlinetypes = 0;
+    int skipnextlf = 0;
+    int univ_newline = 1;
+
+    if (fobj) {
+        if (!PyFile_Check(fobj)) {
+            errno = ENXIO; /* What can you do... */
+            return NULL;
+        }
+        univ_newline = ((BoxedFile*)fobj)->f_univ_newline;
+        if (!univ_newline)
+            return fgets(buf, n, stream);
+        newlinetypes = ((BoxedFile*)fobj)->f_newlinetypes;
+        skipnextlf = ((BoxedFile*)fobj)->f_skipnextlf;
+    }
+    FLOCKFILE(stream);
+    c = 'x'; /* Shut up gcc warning */
+    while (--n > 0 && (c = GETC(stream)) != EOF) {
+        if (skipnextlf) {
+            skipnextlf = 0;
+            if (c == '\n') {
+                /* Seeing a \n here with skipnextlf true
+                ** means we saw a \r before.
+                */
+                newlinetypes |= NEWLINE_CRLF;
+                c = GETC(stream);
+                if (c == EOF)
+                    break;
+            } else {
+                /*
+                ** Note that c == EOF also brings us here,
+                ** so we're okay if the last char in the file
+                ** is a CR.
+                */
+                newlinetypes |= NEWLINE_CR;
+            }
+        }
+        if (c == '\r') {
+            /* A \r is translated into a \n, and we skip
+            ** an adjacent \n, if any. We don't set the
+            ** newlinetypes flag until we've seen the next char.
+            */
+            skipnextlf = 1;
+            c = '\n';
+        } else if (c == '\n') {
+            newlinetypes |= NEWLINE_LF;
+        }
+        *p++ = c;
+        if (c == '\n')
+            break;
+    }
+    if (c == EOF && skipnextlf)
+        newlinetypes |= NEWLINE_CR;
+    FUNLOCKFILE(stream);
+    *p = '\0';
+    if (fobj) {
+        ((BoxedFile*)fobj)->f_newlinetypes = newlinetypes;
+        ((BoxedFile*)fobj)->f_skipnextlf = skipnextlf;
+    } else if (skipnextlf) {
+        /* If we have no file object we cannot save the
+        ** skipnextlf flag. We have to readahead, which
+        ** will cause a pause if we're reading from an
+        ** interactive stream, but that is very unlikely
+        ** unless we're doing something silly like
+        ** execfile("/dev/tty").
+        */
+        c = GETC(stream);
+        if (c != '\n')
+            ungetc(c, stream);
+    }
+    if (p == buf)
+        return NULL;
+    return buf;
+}
+
 static BoxedFile* dircheck(BoxedFile* f) {
 #if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
     struct stat buf;
@@ -1298,6 +1390,10 @@ extern "C" int PyFile_SetEncoding(PyObject* f, const char* enc) noexcept {
     return PyFile_SetEncodingAndErrors(f, enc, NULL);
 }
 
+extern "C" PyObject* PyFile_GetEncoding(PyObject* f) noexcept {
+    return static_cast<BoxedFile*>(f)->f_encoding;
+}
+
 extern "C" int PyFile_SetEncodingAndErrors(PyObject* f, const char* enc, char* errors) noexcept {
     BoxedFile* file = static_cast<BoxedFile*>(f);
     PyObject* str, *oerrors;

src/runtime/long.cpp

@@ -212,7 +212,7 @@ extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) n
 
     BoxedLong* rtn = new BoxedLong();
     int r = 0;
-    if (str[strlen(str) - 1] == 'L') {
+    if (str[strlen(str) - 1] == 'L' || str[strlen(str) - 1] == 'l') {
         std::string without_l(str, strlen(str) - 1);
         r = mpz_init_set_str(rtn->n, without_l.c_str(), base);
     } else {