Skip to content

P
Pyston
Commit f8cec61e authored by Kevin Modzelewski's avatar Kevin Modzelewski Committed by Kevin Modzelewski
Browse files
Options

Add support for using the CPython parser

parent b0cf2322
Hide whitespace changes
Inline Side-by-side
Showing with 1142 additions and 60 deletions
CMakeLists.txt
View file @ f8cec61e
......@@ -287,9 +287,9 @@ endmacro()
# tests testname directory arguments
add_pyston_test(defaults tests --order-by-mtime -t50)
add_pyston_test(force_llvm tests -a=-n -a=-x -t50)
add_pyston_test(force_llvm tests -a=-n -a=-X -t50)
if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
add_pyston_test(max_compilation_tier tests -a=-O -a=-x -t50)
add_pyston_test(max_compilation_tier tests -a=-O -a=-X -t50)
endif()
add_pyston_test(defaults cpython --exit-code-only --skip-failing -t50)
add_pyston_test(defaults integration --exit-code-only --skip-failing -t600)
......
Makefile
View file @ f8cec61e
......@@ -779,7 +779,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1
@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t50 $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t600 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -a=-X -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
.PHONY: run$1 dbg$1
......@@ -825,6 +825,7 @@ perf_report:
.PHONY: run run_% dbg_% debug_% perf_%
run: run_dbg
dbg: dbg_dbg
run_%: run_dbg_%
@true
dbg_%: dbg_dbg_%
......
src/capi/errors.cpp
View file @ f8cec61e
......@@ -256,7 +256,35 @@ finally:
}
static void print_error_text(PyObject* f, int offset, const char* text) noexcept {
Py_FatalError("unimplemented");
const char* nl;
if (offset >= 0) {
if (offset > 0 && offset == strlen(text) && text[offset - 1] == '\n')
offset--;
for (;;) {
nl = strchr(text, '\n');
if (nl == NULL || nl - text >= offset)
break;
offset -= (int)(nl + 1 - text);
text = nl + 1;
}
while (*text == ' ' || *text == '\t') {
text++;
offset--;
}
}
PyFile_WriteString(" ", f);
PyFile_WriteString(text, f);
if (*text == '\0' || text[strlen(text) - 1] != '\n')
PyFile_WriteString("\n", f);
if (offset == -1)
return;
PyFile_WriteString(" ", f);
offset--;
while (offset > 0) {
PyFile_WriteString(" ", f);
offset--;
}
PyFile_WriteString("^\n", f);
}
extern "C" void PyErr_Display(PyObject* exception, PyObject* value, PyObject* tb) noexcept {
......@@ -469,4 +497,45 @@ extern "C" void PyErr_PrintEx(int set_sys_last_vars) noexcept {
extern "C" void PyErr_Print() noexcept {
PyErr_PrintEx(1);
}
/* com_fetch_program_text will attempt to load the line of text that
the exception refers to. If it fails, it will return NULL but will
not set an exception.
XXX The functionality of this function is quite similar to the
functionality in tb_displayline() in traceback.c.
*/
extern "C" PyObject* PyErr_ProgramText(const char* filename, int lineno) noexcept {
FILE* fp;
int i;
char linebuf[1000];
if (filename == NULL || *filename == '\0' || lineno <= 0)
return NULL;
fp = fopen(filename, "r" PY_STDIOTEXTMODE);
if (fp == NULL)
return NULL;
for (i = 0; i < lineno; i++) {
char* pLastChar = &linebuf[sizeof(linebuf) - 2];
do {
*pLastChar = '\0';
if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, fp, NULL) == NULL)
break;
/* fgets read *something*; if it didn't get as
far as pLastChar, it must have found a newline
or hit the end of the file; if pLastChar is \n,
it obviously found a newline; else we haven't
yet seen a newline, so must continue */
} while (*pLastChar != '\0' && *pLastChar != '\n');
}
fclose(fp);
if (i == lineno) {
char* p = linebuf;
while (*p == ' ' || *p == '\t' || *p == '\014')
p++;
return PyString_FromString(p);
}
return NULL;
}
}
src/codegen/cpython_ast.cpp
View file @ f8cec61e
......@@ -14,10 +14,624 @@
#include "codegen/cpython_ast.h"
#include "llvm/ADT/STLExtras.h"
#include "core/types.h"
#include "runtime/types.h"
namespace pyston {
AST_Module* cpythonToPystonAst(mod_ty mod) {
Py_FatalError("unimplemented");
}
#if 0
#Helpful script:
with open("Include/Python-ast.h") as f:
lines = f.readlines()
#type, lines = "stmt", lines[72 : 181]
#type, lines = "expr", lines[196 : 309]
type, lines = "slice", lines[319:332]
names = []
while lines:
l = lines.pop(0).strip()
if l.startswith("struct {"):
continue
elif l.startswith("}"):
name = l[2:-1]
print "case %s_kind: {" % name
print "auto r = new AST_%s();" % name
print "auto v = %s->v.%s;" % (type, name)
for n in names:
print "r->%s = convert(v.%s);" % (n, n)
print "return r;"
print "}"
names = []
continue
elif l.endswith(';'):
n = l.split()[1][:-1]
if n.startswith('*'):
n = n[1:]
names.append(n)
#endif
class Converter {
private:
InternedStringPool* pool = NULL;
public:
template <typename T, typename P> std::vector<P> convert(asdl_seq* seq) {
std::vector<P> rtn;
if (!seq)
return rtn;
for (int i = 0; i < seq->size; i++) {
rtn.push_back(convert((T)seq->elements[i]));
}
return rtn;
}
template <typename T, typename P> std::vector<P> convert(asdl_int_seq* seq) {
std::vector<P> rtn;
if (!seq)
return rtn;
for (int i = 0; i < seq->size; i++) {
rtn.push_back(convert((T)seq->elements[i]));
}
return rtn;
}
template <typename T, typename P> void convertAll(asdl_seq* seq, std::vector<P>& vec) { vec = convert<T, P>(seq); }
InternedString convert(identifier ident) {
assert(pool);
if (!ident)
return pool->get("");
return pool->get(static_cast<BoxedString*>(ident)->s());
}
AST_arguments* convert(arguments_ty ident) {
auto r = new AST_arguments();
convertAll<expr_ty>(ident->args, r->args);
convertAll<expr_ty>(ident->defaults, r->defaults);
r->vararg = convert(ident->vararg);
r->kwarg = convert(ident->kwarg);
return r;
}
#define CASE(N) \
case N: \
return AST_TYPE::N
AST_TYPE::AST_TYPE convert(expr_context_ty context) {
switch (context) {
CASE(Load);
CASE(Store);
CASE(Del);
CASE(Param);
default:
RELEASE_ASSERT(0, "unhandled context type: %d", context);
}
}
AST_TYPE::AST_TYPE convert(operator_ty op) {
switch (op) {
CASE(Add);
CASE(Sub);
CASE(Mult);
CASE(Div);
CASE(Mod);
CASE(Pow);
CASE(LShift);
CASE(RShift);
CASE(BitOr);
CASE(BitXor);
CASE(BitAnd);
CASE(FloorDiv);
}
}
AST_TYPE::AST_TYPE convert(boolop_ty op) {
switch (op) {
CASE(Add);
CASE(Or);
}
}
AST_TYPE::AST_TYPE convert(unaryop_ty op) {
switch (op) {
CASE(Invert);
CASE(Not);
CASE(UAdd);
CASE(USub);
}
}
AST_TYPE::AST_TYPE convert(cmpop_ty op) {
switch (op) {
CASE(Eq);
CASE(NotEq);
CASE(Lt);
CASE(LtE);
CASE(Gt);
CASE(GtE);
CASE(Is);
CASE(IsNot);
CASE(In);
CASE(NotIn);
}
}
#undef CASE
AST_keyword* convert(keyword_ty keyword) {
auto r = new AST_keyword();
r->arg = convert(keyword->arg);
r->value = convert(keyword->value);
return r;
}
AST_comprehension* convert(comprehension_ty comprehension) {
auto r = new AST_comprehension();
r->target = convert(comprehension->target);
r->iter = convert(comprehension->iter);
r->ifs = convert<expr_ty, AST_expr*>(comprehension->ifs);
return r;
}
AST_slice* convert(slice_ty slice) {
switch (slice->kind) {
case Slice_kind: {
auto r = new AST_Slice();
auto v = slice->v.Slice;
r->lower = convert(v.lower);
r->upper = convert(v.upper);
r->step = convert(v.step);
return r;
}
case ExtSlice_kind: {
auto r = new AST_ExtSlice();
auto v = slice->v.ExtSlice;
r->dims = convert<slice_ty, AST_slice*>(v.dims);
return r;
}
case Index_kind: {
auto r = new AST_Index();
auto v = slice->v.Index;
r->value = convert(v.value);
return r;
}
case Ellipsis_kind:
return new AST_Ellipsis();
}
}
AST_expr* _convert(expr_ty expr) {
switch (expr->kind) {
case BoolOp_kind: {
auto r = new AST_BoolOp();
auto v = expr->v.BoolOp;
r->op_type = convert(v.op);
r->values = convert<expr_ty, AST_expr*>(v.values);
return r;
}
case BinOp_kind: {
auto r = new AST_BinOp();
auto v = expr->v.BinOp;
r->left = convert(v.left);
r->op_type = convert(v.op);
r->right = convert(v.right);
return r;
}
case UnaryOp_kind: {
auto r = new AST_UnaryOp();
auto v = expr->v.UnaryOp;
r->op_type = convert(v.op);
r->operand = convert(v.operand);
return r;
}
case Lambda_kind: {
auto r = new AST_Lambda();
auto v = expr->v.Lambda;
r->args = convert(v.args);
r->body = convert(v.body);
return r;
}
case IfExp_kind: {
auto r = new AST_IfExp();
auto v = expr->v.IfExp;
r->test = convert(v.test);
r->body = convert(v.body);
r->orelse = convert(v.orelse);
return r;
}
case Dict_kind: {
auto r = new AST_Dict();
auto v = expr->v.Dict;
r->keys = convert<expr_ty, AST_expr*>(v.keys);
r->values = convert<expr_ty, AST_expr*>(v.values);
return r;
}
case Set_kind: {
auto r = new AST_Set();
auto v = expr->v.Set;
r->elts = convert<expr_ty, AST_expr*>(v.elts);
return r;
}
case ListComp_kind: {
auto r = new AST_ListComp();
auto v = expr->v.ListComp;
r->elt = convert(v.elt);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case SetComp_kind: {
auto r = new AST_SetComp();
auto v = expr->v.SetComp;
r->elt = convert(v.elt);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case DictComp_kind: {
auto r = new AST_DictComp();
auto v = expr->v.DictComp;
r->key = convert(v.key);
r->value = convert(v.value);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case GeneratorExp_kind: {
auto r = new AST_GeneratorExp();
auto v = expr->v.GeneratorExp;
r->elt = convert(v.elt);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case Yield_kind: {
auto r = new AST_Yield();
auto v = expr->v.Yield;
r->value = convert(v.value);
return r;
}
case Compare_kind: {
auto r = new AST_Compare();
auto v = expr->v.Compare;
r->left = convert(v.left);
r->ops = convert<cmpop_ty, AST_TYPE::AST_TYPE>(v.ops);
r->comparators = convert<expr_ty, AST_expr*>(v.comparators);
return r;
}
case Call_kind: {
auto r = new AST_Call();
auto v = expr->v.Call;
r->func = convert(v.func);
r->args = convert<expr_ty, AST_expr*>(v.args);
r->keywords = convert<keyword_ty, AST_keyword*>(v.keywords);
r->starargs = convert(v.starargs);
r->kwargs = convert(v.kwargs);
return r;
}
case Repr_kind: {
auto r = new AST_Repr();
auto v = expr->v.Repr;
r->value = convert(v.value);
return r;
}
case Attribute_kind: {
auto r = new AST_Attribute();
auto v = expr->v.Attribute;
r->value = convert(v.value);
r->attr = convert(v.attr);
r->ctx_type = convert(v.ctx);
return r;
}
case Subscript_kind: {
auto r = new AST_Subscript();
auto v = expr->v.Subscript;
r->value = convert(v.value);
r->slice = convert(v.slice);
r->ctx_type = convert(v.ctx);
return r;
}
case Name_kind: {
auto v = expr->v.Name;
auto r = new AST_Name(convert(v.id), convert(v.ctx), 0);
return r;
}
case List_kind: {
auto r = new AST_List();
auto v = expr->v.List;
r->elts = convert<expr_ty, AST_expr*>(v.elts);
r->ctx_type = convert(v.ctx);
return r;
}
case Tuple_kind: {
auto r = new AST_Tuple();
auto v = expr->v.Tuple;
r->elts = convert<expr_ty, AST_expr*>(v.elts);
r->ctx_type = convert(v.ctx);
return r;
}
case Num_kind: {
PyObject* o = expr->v.Num.n;
if (o->cls == int_cls) {
auto r = new AST_Num();
r->num_type = AST_Num::INT;
r->n_int = unboxInt(o);
return r;
}
if (o->cls == float_cls) {
auto r = new AST_Num();
r->num_type = AST_Num::FLOAT;
r->n_float = unboxFloat(o);
return r;
}
if (o->cls == long_cls) {
auto r = new AST_Num();
r->num_type = AST_Num::LONG;
// XXX This is pretty silly:
auto s = _PyLong_Format(o, 10, 0, 0);
RELEASE_ASSERT(s, "");
r->n_long = PyString_AsString(s);
return r;
}
if (o->cls == complex_cls) {
auto r = new AST_Num();
r->num_type = AST_Num::COMPLEX;
double real = PyComplex_RealAsDouble(o);
double imag = PyComplex_ImagAsDouble(o);
RELEASE_ASSERT(real != -1.0 || !PyErr_Occurred(), "");
RELEASE_ASSERT(imag != -1.0 || !PyErr_Occurred(), "");
r->n_float = imag;
if (real == 0.0)
return r;
// TODO very silly:
auto freal = new AST_Num();
freal->n_float = real;
freal->num_type = AST_Num::FLOAT;
auto binop = new AST_BinOp();
binop->op_type = AST_TYPE::Add;
binop->left = freal;
binop->right = r;
return binop;
}
RELEASE_ASSERT(0, "unhandled num type: %s\n", o->cls->tp_name);
}
case Str_kind: {
PyObject* o = expr->v.Str.s;
if (o->cls == unicode_cls) {
o = PyUnicode_AsUTF8String(o);
RELEASE_ASSERT(o, "");
auto r = new AST_Str();
r->str_data = static_cast<BoxedString*>(o)->s();
r->str_type = AST_Str::UNICODE;
return r;
}
if (o->cls == str_cls) {
return new AST_Str(static_cast<BoxedString*>(o)->s());
}
RELEASE_ASSERT(0, "unhandled str type: %s\n", o->cls->tp_name);
}
default:
RELEASE_ASSERT(0, "unhandled kind: %d\n", expr->kind);
};
Py_FatalError("unimplemented");
}
AST_expr* convert(expr_ty expr) {
if (!expr)
return NULL;
auto r = _convert(expr);
r->lineno = expr->lineno;
r->col_offset = expr->col_offset;
return r;
}
AST_ExceptHandler* convert(excepthandler_ty eh) {
assert(eh->kind == ExceptHandler_kind);
auto r = new AST_ExceptHandler();
auto v = eh->v.ExceptHandler;
r->type = convert(v.type);
r->name = convert(v.name);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
return r;
}
AST_alias* convert(alias_ty alias) { return new AST_alias(convert(alias->name), convert(alias->asname)); }
AST_stmt* _convert(stmt_ty stmt) {
switch (stmt->kind) {
case FunctionDef_kind: {
auto r = new AST_FunctionDef();
auto v = stmt->v.FunctionDef;
r->name = convert(v.name);
r->args = convert(v.args);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->decorator_list = convert<expr_ty, AST_expr*>(v.decorator_list);
return r;
}
case ClassDef_kind: {
auto r = new AST_ClassDef();
auto v = stmt->v.ClassDef;
r->name = convert(v.name);
r->bases = convert<expr_ty, AST_expr*>(v.bases);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->decorator_list = convert<expr_ty, AST_expr*>(v.decorator_list);
return r;
}
case Return_kind: {
auto r = new AST_Return();
auto v = stmt->v.Return;
r->value = convert(v.value);
return r;
}
case Delete_kind: {
auto r = new AST_Delete();
auto v = stmt->v.Delete;
r->targets = convert<expr_ty, AST_expr*>(v.targets);
return r;
}
case Assign_kind: {
auto r = new AST_Assign();
auto v = stmt->v.Assign;
r->targets = convert<expr_ty, AST_expr*>(v.targets);
r->value = convert(v.value);
return r;
}
case AugAssign_kind: {
auto r = new AST_AugAssign();
auto v = stmt->v.AugAssign;
r->target = convert(v.target);
r->op_type = convert(v.op);
r->value = convert(v.value);
return r;
}
case Print_kind: {
auto r = new AST_Print();
auto v = stmt->v.Print;
r->dest = convert(v.dest);
r->values = convert<expr_ty, AST_expr*>(v.values);
r->nl = v.nl;
return r;
}
case For_kind: {
auto r = new AST_For();
auto v = stmt->v.For;
r->target = convert(v.target);
r->iter = convert(v.iter);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->orelse = convert<stmt_ty, AST_stmt*>(v.orelse);
return r;
}
case While_kind: {
auto r = new AST_While();
auto v = stmt->v.While;
r->test = convert(v.test);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->orelse = convert<stmt_ty, AST_stmt*>(v.orelse);
return r;
}
case If_kind: {
auto r = new AST_If();
auto v = stmt->v.If;
r->test = convert(v.test);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->orelse = convert<stmt_ty, AST_stmt*>(v.orelse);
return r;
}
case With_kind: {
auto r = new AST_With();
auto v = stmt->v.With;
r->context_expr = convert(v.context_expr);
r->optional_vars = convert(v.optional_vars);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
return r;
}
case Raise_kind: {
auto r = new AST_Raise();
auto v = stmt->v.Raise;
r->arg0 = convert(v.type);
r->arg1 = convert(v.inst);
r->arg2 = convert(v.tback);
return r;
}
case TryExcept_kind: {
auto r = new AST_TryExcept();
auto v = stmt->v.TryExcept;
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->handlers = convert<excepthandler_ty, AST_ExceptHandler*>(v.handlers);
r->orelse = convert<stmt_ty, AST_stmt*>(v.orelse);
return r;
}
case TryFinally_kind: {
auto r = new AST_TryFinally();
auto v = stmt->v.TryFinally;
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->finalbody = convert<stmt_ty, AST_stmt*>(v.finalbody);
return r;
}
case Assert_kind: {
auto r = new AST_Assert();
auto v = stmt->v.Assert;
r->test = convert(v.test);
r->msg = convert(v.msg);
return r;
}
case Import_kind: {
auto r = new AST_Import();
auto v = stmt->v.Import;
r->names = convert<alias_ty, AST_alias*>(v.names);
return r;
}
case ImportFrom_kind: {
auto r = new AST_ImportFrom();
auto v = stmt->v.ImportFrom;
r->module = convert(v.module);
r->names = convert<alias_ty, AST_alias*>(v.names);
r->level = v.level;
return r;
}
case Exec_kind: {
auto r = new AST_Exec();
auto v = stmt->v.Exec;
r->body = convert(v.body);
r->globals = convert(v.globals);
r->locals = convert(v.locals);
return r;
}
case Global_kind: {
auto r = new AST_Global();
auto v = stmt->v.Global;
r->names = convert<identifier, InternedString>(v.names);
return r;
}
case Expr_kind: {
auto r = new AST_Expr();
auto v = stmt->v.Expr;
r->value = convert(v.value);
return r;
}
case Pass_kind:
return new AST_Pass();
case Break_kind:
return new AST_Break();
case Continue_kind:
return new AST_Continue();
};
}
AST_stmt* convert(stmt_ty stmt) {
auto r = _convert(stmt);
r->lineno = stmt->lineno;
r->col_offset = stmt->col_offset;
return r;
}
AST_Module* convert(mod_ty mod) {
switch (mod->kind) {
case Module_kind:
case Interactive_kind: {
AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
assert(!this->pool);
this->pool = rtn->interned_strings.get();
convertAll<stmt_ty>(mod->v.Interactive.body, rtn->body);
return rtn;
}
default:
RELEASE_ASSERT(0, "unhandled kind: %d\n", mod->kind);
}
}
};
AST_Module* cpythonToPystonAST(mod_ty mod) {
Converter c;
return c.convert(mod);
}
}
src/codegen/irgen/hooks.cpp
View file @ f8cec61e
......@@ -387,18 +387,8 @@ static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, b
const char* code = source.data();
AST_Module* parsedModule = parse_string(code, future_flags);
if (interactive) {
for (int i = 0; i < parsedModule->body.size(); ++i) {
AST_stmt* s = parsedModule->body[i];
if (s->type != AST_TYPE::Expr)
continue;
AST_Expr* expr = (AST_Expr*)s;
AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(expr->value);
expr->value = print_expr;
}
}
if (interactive)
makeModuleInteractive(parsedModule);
return parsedModule;
}
......
src/codegen/parser.cpp
View file @ f8cec61e
......@@ -14,6 +14,8 @@
#include "codegen/parser.h"
#include "cpython_ast.h"
#include <cassert>
#include <cstdio>
#include <cstdlib>
......@@ -32,6 +34,7 @@
#include "core/stats.h"
#include "core/types.h"
#include "core/util.h"
#include "runtime/types.h"
//#undef VERBOSITY
//#define VERBOSITY(x) 2
......@@ -1046,6 +1049,21 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
Timer _t("parsing");
if (ENABLE_CPYTHON_PARSER) {
ASSERT(!inherited_flags, "unimplemented");
FILE* fp = fopen(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = 0;
PyArena* arena = PyArena_New();
assert(arena);
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
if (!mod)
throwCAPIException();
auto rtn = cpythonToPystonAST(mod);
PyArena_Free(arena);
return rtn;
}
if (ENABLE_PYPA_PARSER) {
AST_Module* rtn = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
......@@ -1073,7 +1091,9 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
}
const char* getMagic() {
if (ENABLE_PYPA_PARSER)
if (ENABLE_CPYTHON_PARSER)
return "a\nCO";
else if (ENABLE_PYPA_PARSER)
return "a\ncO";
else
return "a\nco";
......@@ -1116,9 +1136,22 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
file_data.insert(file_data.end(), (char*)&checksum, (char*)&checksum + CHECKSUM_LENGTH);
checksum = 0;
if (ENABLE_PYPA_PARSER || inherited_flags) {
module = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(module, "unknown parse error");
if (ENABLE_CPYTHON_PARSER || ENABLE_PYPA_PARSER || inherited_flags) {
if (ENABLE_CPYTHON_PARSER) {
ASSERT(!inherited_flags, "unimplemented");
FILE* fp = fopen(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = 0;
PyArena* arena = PyArena_New();
assert(arena);
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
if (!mod)
throwCAPIException();
module = cpythonToPystonAST(mod);
} else {
module = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(module, "unknown parse error");
}
if (!cache_fp)
return std::vector<char>();
......
src/core/ast.cpp
View file @ f8cec61e
......@@ -2232,4 +2232,17 @@ void flatten(AST_expr* root, std::vector<AST*>& output, bool expand_scopes) {
root->accept(&visitor);
}
void makeModuleInteractive(AST_Module* m) {
for (int i = 0; i < m->body.size(); ++i) {
AST_stmt* s = m->body[i];
if (s->type != AST_TYPE::Expr)
continue;
AST_Expr* expr = (AST_Expr*)s;
AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(expr->value);
expr->value = print_expr;
}
}
}
src/core/ast.h
View file @ f8cec61e
......@@ -447,6 +447,7 @@ public:
virtual void accept_stmt(StmtVisitor* v);
AST_Expr() : AST_stmt(AST_TYPE::Expr) {}
AST_Expr(AST_expr* value) : AST_stmt(AST_TYPE::Expr), value(value) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expr;
};
......@@ -1425,6 +1426,10 @@ template <class T, class R> void findNodes(const R& roots, std::vector<T*>& outp
}
}
// Take a normally-parsed module, and convert it (inplace) to a form that will print out any bare expressions.
// This is used for "single" mode or the repl.
void makeModuleInteractive(AST_Module* m);
llvm::StringRef getOpSymbol(int op_type);
BoxedString* getOpName(int op_type);
int getReverseCmpOp(int op_type, bool& success);
......
src/core/options.cpp
View file @ f8cec61e
......@@ -41,6 +41,7 @@ bool USE_STRIPPED_STDLIB = true; // always true
bool ENABLE_INTERPRETER = true;
bool ENABLE_BASELINEJIT = true;
bool ENABLE_PYPA_PARSER = true;
bool ENABLE_CPYTHON_PARSER = false;
bool USE_REGALLOC_BASIC = true;
bool PAUSE_AT_ABORT = false;
bool ENABLE_TRACEBACKS = true;
......@@ -90,9 +91,11 @@ bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;
extern "C" {
int Py_FrozenFlag = 1;
int Py_IgnoreEnvironmentFlag = 0;
int Py_InteractiveFlag = 0;
int Py_InspectFlag = 0;
int Py_NoSiteFlag = 0;
int Py_OptimizeFlag = 0;
int Py_VerboseFlag = 0;
int Py_UnicodeFlag = 0;
}
}
src/core/options.h
View file @ f8cec61e
......@@ -38,8 +38,9 @@ extern int SPECULATION_THRESHOLD;
extern int MAX_OBJECT_CACHE_ENTRIES;
extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, ENABLE_CPYTHON_PARSER,
USE_REGALLOC_BASIC, PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS,
FORCE_LLVM_CAPI_THROWS;
extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
......
src/jit.cpp
View file @ f8cec61e
......@@ -193,9 +193,10 @@ int handleArg(char code) {
SHOW_DISASM = true;
else if (code == 'I')
FORCE_INTERPRETER = true;
else if (code == 'i')
else if (code == 'i') {
Py_InspectFlag = true;
else if (code == 'n') {
Py_InteractiveFlag = true;
} else if (code == 'n') {
ENABLE_INTERPRETER = false;
} else if (code == 'a') {
ASSEMBLY_LOGGING = true;
......@@ -207,6 +208,8 @@ int handleArg(char code) {
Stats::setEnabled(true);
} else if (code == 'S') {
Py_NoSiteFlag = 1;
} else if (code == 'U') {
Py_UnicodeFlag++;
} else if (code == 'u') {
unbuffered = true;
} else if (code == 'r') {
......@@ -215,6 +218,8 @@ int handleArg(char code) {
USE_REGALLOC_BASIC = false;
} else if (code == 'x') {
ENABLE_PYPA_PARSER = false;
} else if (code == 'X') {
ENABLE_CPYTHON_PARSER = true;
} else if (code == 'E') {
Py_IgnoreEnvironmentFlag = 1;
} else if (code == 'P') {
......@@ -325,7 +330,7 @@ static int main(int argc, char** argv) {
// Suppress getopt errors so we can throw them ourselves
opterr = 0;
while ((code = getopt(argc, argv, "+:OqdIibpjtrsRSvnxEac:FuPTGm:")) != -1) {
while ((code = getopt(argc, argv, "+:OqdIibpjtrsRSUvnxXEac:FuPTGm:")) != -1) {
if (code == 'c') {
assert(optarg);
command = optarg;
......@@ -497,42 +502,19 @@ static int main(int argc, char** argv) {
}
if (Py_InspectFlag || !(command || fn || module)) {
PyObject* v = PyImport_ImportModule("readline");
if (!v)
PyErr_Clear();
printf("Pyston v%d.%d (rev " STRINGIFY(GITREV) ")", PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR);
printf(", targeting Python %d.%d.%d\n", PYTHON_VERSION_MAJOR, PYTHON_VERSION_MINOR, PYTHON_VERSION_MICRO);
Py_InspectFlag = 0;
if (!main_module) {
main_module = createModule(boxString("__main__"), "<stdin>");
} else {
// main_module->fn = "<stdin>";
}
for (;;) {
char* line = readline(">> ");
if (!line)
break;
add_history(line);
try {
AST_Module* m = parse_string(line, /* future_flags = */ 0);
Timer _t("repl");
if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(e->value);
e->value = print_expr;
}
compileAndRunModule(m, main_module);
} catch (ExcInfo e) {
setCAPIException(e);
PyErr_Print();
}
}
PyCompilerFlags cf;
cf.cf_flags = 0;
rtncode = PyRun_InteractiveLoopFlags(stdin, "<stdin>", &cf);
}
threading::finishMainThread();
......
src/runtime/builtin_modules/sys.cpp
View file @ f8cec61e
......@@ -151,6 +151,16 @@ extern "C" PyObject* PySys_GetObject(const char* name) noexcept {
return sys_module->getattr(internStringMortal(name));
}
extern "C" FILE* PySys_GetFile(char* name, FILE* def) noexcept {
FILE* fp = NULL;
PyObject* v = PySys_GetObject(name);
if (v != NULL && PyFile_Check(v))
fp = PyFile_AsFile(v);
if (fp == NULL)
fp = def;
return fp;
}
static void mywrite(const char* name, FILE* fp, const char* format, va_list va) noexcept {
PyObject* file;
PyObject* error_type, *error_value, *error_traceback;
......
src/runtime/capi.cpp
View file @ f8cec61e
......@@ -18,12 +18,23 @@
#include "Python.h"
#include "codegen/cpython_ast.h"
#include "grammar.h"
#include "node.h"
#include "token.h"
#include "parsetok.h"
#include "errcode.h"
#include "ast.h"
#undef BYTE
#undef STRING
#include "llvm/Support/ErrorHandling.h" // For llvm_unreachable
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "capi/typeobject.h"
#include "capi/types.h"
#include "codegen/irgen/hooks.h"
#include "codegen/unwinding.h"
#include "core/threading.h"
#include "core/types.h"
......@@ -1164,6 +1175,253 @@ static int dev_urandom_python(char* buffer, Py_ssize_t size) noexcept {
}
}
extern "C" int Py_FdIsInteractive(FILE* fp, const char* filename) noexcept {
if (isatty((int)fileno(fp)))
return 1;
if (!Py_InteractiveFlag)
return 0;
return (filename == NULL) || (strcmp(filename, "<stdin>") == 0) || (strcmp(filename, "???") == 0);
}
extern "C" int PyRun_InteractiveOneFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
PyObject* m, *d, *v, *w;
mod_ty mod;
PyArena* arena;
char _buf[1] = "";
char* ps1 = _buf, * ps2 = _buf;
int errcode = 0;
v = PySys_GetObject("ps1");
if (v != NULL) {
v = PyObject_Str(v);
if (v == NULL)
PyErr_Clear();
else if (PyString_Check(v))
ps1 = PyString_AsString(v);
}
w = PySys_GetObject("ps2");
if (w != NULL) {
w = PyObject_Str(w);
if (w == NULL)
PyErr_Clear();
else if (PyString_Check(w))
ps2 = PyString_AsString(w);
}
arena = PyArena_New();
if (arena == NULL) {
Py_XDECREF(v);
Py_XDECREF(w);
return -1;
}
mod = PyParser_ASTFromFile(fp, filename, Py_single_input, ps1, ps2, flags, &errcode, arena);
Py_XDECREF(v);
Py_XDECREF(w);
if (mod == NULL) {
PyArena_Free(arena);
if (errcode == E_EOF) {
PyErr_Clear();
return E_EOF;
}
PyErr_Print();
return -1;
}
m = PyImport_AddModule("__main__");
if (m == NULL) {
PyArena_Free(arena);
return -1;
}
// Pyston change:
// d = PyModule_GetDict(m);
// v = run_mod(mod, filename, d, d, flags, arena);
assert(PyModule_Check(m));
AST_Module* pyston_module = cpythonToPystonAST(mod);
makeModuleInteractive(pyston_module);
bool failed = false;
try {
compileAndRunModule(pyston_module, static_cast<BoxedModule*>(m));
} catch (ExcInfo e) {
setCAPIException(e);
failed = true;
}
PyArena_Free(arena);
if (failed) {
PyErr_Print();
return -1;
}
Py_DECREF(v);
if (Py_FlushLine())
PyErr_Clear();
return 0;
}
/* Set the error appropriate to the given input error code (see errcode.h) */
static void err_input(perrdetail* err) noexcept {
PyObject* v, *w, *errtype;
PyObject* u = NULL;
const char* msg = NULL;
errtype = PyExc_SyntaxError;
switch (err->error) {
case E_ERROR:
return;
case E_SYNTAX:
errtype = PyExc_IndentationError;
if (err->expected == INDENT)
msg = "expected an indented block";
else if (err->token == INDENT)
msg = "unexpected indent";
else if (err->token == DEDENT)
msg = "unexpected unindent";
else {
errtype = PyExc_SyntaxError;
msg = "invalid syntax";
}
break;
case E_TOKEN:
msg = "invalid token";
break;
case E_EOFS:
msg = "EOF while scanning triple-quoted string literal";
break;
case E_EOLS:
msg = "EOL while scanning string literal";
break;
case E_INTR:
if (!PyErr_Occurred())
PyErr_SetNone(PyExc_KeyboardInterrupt);
goto cleanup;
case E_NOMEM:
PyErr_NoMemory();
goto cleanup;
case E_EOF:
msg = "unexpected EOF while parsing";
break;
case E_TABSPACE:
errtype = PyExc_TabError;
msg = "inconsistent use of tabs and spaces in indentation";
break;
case E_OVERFLOW:
msg = "expression too long";
break;
case E_DEDENT:
errtype = PyExc_IndentationError;
msg = "unindent does not match any outer indentation level";
break;
case E_TOODEEP:
errtype = PyExc_IndentationError;
msg = "too many levels of indentation";
break;
case E_DECODE: {
PyObject* type, *value, *tb;
PyErr_Fetch(&type, &value, &tb);
if (value != NULL) {
u = PyObject_Str(value);
if (u != NULL) {
msg = PyString_AsString(u);
}
}
if (msg == NULL)
msg = "unknown decode error";
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(tb);
break;
}
case E_LINECONT:
msg = "unexpected character after line continuation character";
break;
default:
fprintf(stderr, "error=%d\n", err->error);
msg = "unknown parsing error";
break;
}
v = Py_BuildValue("(ziiz)", err->filename, err->lineno, err->offset, err->text);
w = NULL;
if (v != NULL)
w = Py_BuildValue("(sO)", msg, v);
Py_XDECREF(u);
Py_XDECREF(v);
PyErr_SetObject(errtype, w);
Py_XDECREF(w);
cleanup:
if (err->text != NULL) {
PyObject_FREE(err->text);
err->text = NULL;
}
}
#if 0
/* compute parser flags based on compiler flags */
#define PARSER_FLAGS(flags) \
((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0)
#endif
#if 1
/* Keep an example of flags with future keyword support. */
#define PARSER_FLAGS(flags) \
((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? PyPARSE_DONT_IMPLY_DEDENT : 0) \
| (((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION) ? PyPARSE_PRINT_IS_FUNCTION : 0) \
| (((flags)->cf_flags & CO_FUTURE_UNICODE_LITERALS) ? PyPARSE_UNICODE_LITERALS : 0)) \
: 0)
#endif
extern "C" grammar _PyParser_Grammar;
extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start, char* ps1, char* ps2,
PyCompilerFlags* flags, int* errcode, PyArena* arena) noexcept {
mod_ty mod;
PyCompilerFlags localflags;
perrdetail err;
int iflags = PARSER_FLAGS(flags);
node* n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, start, ps1, ps2, &err, &iflags);
if (flags == NULL) {
localflags.cf_flags = 0;
flags = &localflags;
}
if (n) {
flags->cf_flags |= iflags & PyCF_MASK;
mod = PyAST_FromNode(n, flags, filename, arena);
PyNode_Free(n);
return mod;
} else {
err_input(&err);
if (errcode)
*errcode = err.error;
return NULL;
}
}
extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
PyObject* v;
int ret;
PyCompilerFlags local_flags;
if (flags == NULL) {
flags = &local_flags;
local_flags.cf_flags = 0;
}
v = PySys_GetObject("ps1");
if (v == NULL) {
PySys_SetObject("ps1", v = PyString_FromString(">> "));
Py_XDECREF(v);
}
v = PySys_GetObject("ps2");
if (v == NULL) {
PySys_SetObject("ps2", v = PyString_FromString("... "));
Py_XDECREF(v);
}
for (;;) {
ret = PyRun_InteractiveOneFlags(fp, filename, flags);
// PRINT_TOTAL_REFS();
if (ret == E_EOF)
return 0;
if (ret == E_NOMEM)
return -1;
}
}
static const char* progname = "pyston";
extern "C" void Py_SetProgramName(char* pn) noexcept {
if (pn && *pn)
......@@ -1642,6 +1900,13 @@ Box* BoxedCApiFunction::tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPa
return rtn;
}
/* Warning with explicit origin */
extern "C" int PyErr_WarnExplicit(PyObject* category, const char* text, const char* filename_str, int lineno,
const char* module_str, PyObject* registry) noexcept {
Py_FatalError("unimplemented");
}
/* extension modules might be compiled with GC support so these
functions must always be available */
......
src/runtime/file.cpp
View file @ f8cec61e
......@@ -130,6 +130,98 @@ static PyObject* err_iterbuffered(void) noexcept {
return NULL;
}
/*
** Py_UniversalNewlineFgets is an fgets variation that understands
** all of \r, \n and \r\n conventions.
** The stream should be opened in binary mode.
** If fobj is NULL the routine always does newline conversion, and
** it may peek one char ahead to gobble the second char in \r\n.
** If fobj is non-NULL it must be a PyFileObject. In this case there
** is no readahead but in stead a flag is used to skip a following
** \n on the next read. Also, if the file is open in binary mode
** the whole conversion is skipped. Finally, the routine keeps track of
** the different types of newlines seen.
** Note that we need no error handling: fgets() treats error and eof
** identically.
*/
extern "C" char* Py_UniversalNewlineFgets(char* buf, int n, FILE* stream, PyObject* fobj) noexcept {
char* p = buf;
int c;
int newlinetypes = 0;
int skipnextlf = 0;
int univ_newline = 1;
if (fobj) {
if (!PyFile_Check(fobj)) {
errno = ENXIO; /* What can you do... */
return NULL;
}
univ_newline = ((BoxedFile*)fobj)->f_univ_newline;
if (!univ_newline)
return fgets(buf, n, stream);
newlinetypes = ((BoxedFile*)fobj)->f_newlinetypes;
skipnextlf = ((BoxedFile*)fobj)->f_skipnextlf;
}
FLOCKFILE(stream);
c = 'x'; /* Shut up gcc warning */
while (--n > 0 && (c = GETC(stream)) != EOF) {
if (skipnextlf) {
skipnextlf = 0;
if (c == '\n') {
/* Seeing a \n here with skipnextlf true
** means we saw a \r before.
*/
newlinetypes |= NEWLINE_CRLF;
c = GETC(stream);
if (c == EOF)
break;
} else {
/*
** Note that c == EOF also brings us here,
** so we're okay if the last char in the file
** is a CR.
*/
newlinetypes |= NEWLINE_CR;
}
}
if (c == '\r') {
/* A \r is translated into a \n, and we skip
** an adjacent \n, if any. We don't set the
** newlinetypes flag until we've seen the next char.
*/
skipnextlf = 1;
c = '\n';
} else if (c == '\n') {
newlinetypes |= NEWLINE_LF;
}
*p++ = c;
if (c == '\n')
break;
}
if (c == EOF && skipnextlf)
newlinetypes |= NEWLINE_CR;
FUNLOCKFILE(stream);
*p = '\0';
if (fobj) {
((BoxedFile*)fobj)->f_newlinetypes = newlinetypes;
((BoxedFile*)fobj)->f_skipnextlf = skipnextlf;
} else if (skipnextlf) {
/* If we have no file object we cannot save the
** skipnextlf flag. We have to readahead, which
** will cause a pause if we're reading from an
** interactive stream, but that is very unlikely
** unless we're doing something silly like
** execfile("/dev/tty").
*/
c = GETC(stream);
if (c != '\n')
ungetc(c, stream);
}
if (p == buf)
return NULL;
return buf;
}
static BoxedFile* dircheck(BoxedFile* f) {
#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
struct stat buf;
......@@ -1298,6 +1390,10 @@ extern "C" int PyFile_SetEncoding(PyObject* f, const char* enc) noexcept {
return PyFile_SetEncodingAndErrors(f, enc, NULL);
}
extern "C" PyObject* PyFile_GetEncoding(PyObject* f) noexcept {
return static_cast<BoxedFile*>(f)->f_encoding;
}
extern "C" int PyFile_SetEncodingAndErrors(PyObject* f, const char* enc, char* errors) noexcept {
BoxedFile* file = static_cast<BoxedFile*>(f);
PyObject* str, *oerrors;
......
src/runtime/long.cpp
View file @ f8cec61e
......@@ -212,7 +212,7 @@ extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) n
BoxedLong* rtn = new BoxedLong();
int r = 0;
if (str[strlen(str) - 1] == 'L') {
if (str[strlen(str) - 1] == 'L' || str[strlen(str) - 1] == 'l') {
std::string without_l(str, strlen(str) - 1);
r = mpz_init_set_str(rtn->n, without_l.c_str(), base);
} else {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7