Commit f8cec61e authored by Kevin Modzelewski's avatar Kevin Modzelewski Committed by Kevin Modzelewski

Add support for using the CPython parser

parent b0cf2322
......@@ -287,9 +287,9 @@ endmacro()
# tests testname directory arguments
add_pyston_test(defaults tests --order-by-mtime -t50)
add_pyston_test(force_llvm tests -a=-n -a=-x -t50)
add_pyston_test(force_llvm tests -a=-n -a=-X -t50)
if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
add_pyston_test(max_compilation_tier tests -a=-O -a=-x -t50)
add_pyston_test(max_compilation_tier tests -a=-O -a=-X -t50)
endif()
add_pyston_test(defaults cpython --exit-code-only --skip-failing -t50)
add_pyston_test(defaults integration --exit-code-only --skip-failing -t600)
......
......@@ -779,7 +779,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1
@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t50 $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t600 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -a=-X -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
.PHONY: run$1 dbg$1
......@@ -825,6 +825,7 @@ perf_report:
.PHONY: run run_% dbg_% debug_% perf_%
run: run_dbg
dbg: dbg_dbg
run_%: run_dbg_%
@true
dbg_%: dbg_dbg_%
......
......@@ -256,7 +256,35 @@ finally:
}
static void print_error_text(PyObject* f, int offset, const char* text) noexcept {
Py_FatalError("unimplemented");
const char* nl;
if (offset >= 0) {
if (offset > 0 && offset == strlen(text) && text[offset - 1] == '\n')
offset--;
for (;;) {
nl = strchr(text, '\n');
if (nl == NULL || nl - text >= offset)
break;
offset -= (int)(nl + 1 - text);
text = nl + 1;
}
while (*text == ' ' || *text == '\t') {
text++;
offset--;
}
}
PyFile_WriteString(" ", f);
PyFile_WriteString(text, f);
if (*text == '\0' || text[strlen(text) - 1] != '\n')
PyFile_WriteString("\n", f);
if (offset == -1)
return;
PyFile_WriteString(" ", f);
offset--;
while (offset > 0) {
PyFile_WriteString(" ", f);
offset--;
}
PyFile_WriteString("^\n", f);
}
extern "C" void PyErr_Display(PyObject* exception, PyObject* value, PyObject* tb) noexcept {
......@@ -469,4 +497,45 @@ extern "C" void PyErr_PrintEx(int set_sys_last_vars) noexcept {
extern "C" void PyErr_Print() noexcept {
PyErr_PrintEx(1);
}
/* com_fetch_program_text will attempt to load the line of text that
the exception refers to. If it fails, it will return NULL but will
not set an exception.
XXX The functionality of this function is quite similar to the
functionality in tb_displayline() in traceback.c.
*/
extern "C" PyObject* PyErr_ProgramText(const char* filename, int lineno) noexcept {
FILE* fp;
int i;
char linebuf[1000];
if (filename == NULL || *filename == '\0' || lineno <= 0)
return NULL;
fp = fopen(filename, "r" PY_STDIOTEXTMODE);
if (fp == NULL)
return NULL;
for (i = 0; i < lineno; i++) {
char* pLastChar = &linebuf[sizeof(linebuf) - 2];
do {
*pLastChar = '\0';
if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, fp, NULL) == NULL)
break;
/* fgets read *something*; if it didn't get as
far as pLastChar, it must have found a newline
or hit the end of the file; if pLastChar is \n,
it obviously found a newline; else we haven't
yet seen a newline, so must continue */
} while (*pLastChar != '\0' && *pLastChar != '\n');
}
fclose(fp);
if (i == lineno) {
char* p = linebuf;
while (*p == ' ' || *p == '\t' || *p == '\014')
p++;
return PyString_FromString(p);
}
return NULL;
}
}
This diff is collapsed.
......@@ -387,18 +387,8 @@ static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, b
const char* code = source.data();
AST_Module* parsedModule = parse_string(code, future_flags);
if (interactive) {
for (int i = 0; i < parsedModule->body.size(); ++i) {
AST_stmt* s = parsedModule->body[i];
if (s->type != AST_TYPE::Expr)
continue;
AST_Expr* expr = (AST_Expr*)s;
AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(expr->value);
expr->value = print_expr;
}
}
if (interactive)
makeModuleInteractive(parsedModule);
return parsedModule;
}
......
......@@ -14,6 +14,8 @@
#include "codegen/parser.h"
#include "cpython_ast.h"
#include <cassert>
#include <cstdio>
#include <cstdlib>
......@@ -32,6 +34,7 @@
#include "core/stats.h"
#include "core/types.h"
#include "core/util.h"
#include "runtime/types.h"
//#undef VERBOSITY
//#define VERBOSITY(x) 2
......@@ -1046,6 +1049,21 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
Timer _t("parsing");
if (ENABLE_CPYTHON_PARSER) {
ASSERT(!inherited_flags, "unimplemented");
FILE* fp = fopen(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = 0;
PyArena* arena = PyArena_New();
assert(arena);
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
if (!mod)
throwCAPIException();
auto rtn = cpythonToPystonAST(mod);
PyArena_Free(arena);
return rtn;
}
if (ENABLE_PYPA_PARSER) {
AST_Module* rtn = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
......@@ -1073,7 +1091,9 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
}
const char* getMagic() {
if (ENABLE_PYPA_PARSER)
if (ENABLE_CPYTHON_PARSER)
return "a\nCO";
else if (ENABLE_PYPA_PARSER)
return "a\ncO";
else
return "a\nco";
......@@ -1116,9 +1136,22 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
file_data.insert(file_data.end(), (char*)&checksum, (char*)&checksum + CHECKSUM_LENGTH);
checksum = 0;
if (ENABLE_PYPA_PARSER || inherited_flags) {
if (ENABLE_CPYTHON_PARSER || ENABLE_PYPA_PARSER || inherited_flags) {
if (ENABLE_CPYTHON_PARSER) {
ASSERT(!inherited_flags, "unimplemented");
FILE* fp = fopen(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = 0;
PyArena* arena = PyArena_New();
assert(arena);
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
if (!mod)
throwCAPIException();
module = cpythonToPystonAST(mod);
} else {
module = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(module, "unknown parse error");
}
if (!cache_fp)
return std::vector<char>();
......
......@@ -2232,4 +2232,17 @@ void flatten(AST_expr* root, std::vector<AST*>& output, bool expand_scopes) {
root->accept(&visitor);
}
void makeModuleInteractive(AST_Module* m) {
for (int i = 0; i < m->body.size(); ++i) {
AST_stmt* s = m->body[i];
if (s->type != AST_TYPE::Expr)
continue;
AST_Expr* expr = (AST_Expr*)s;
AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(expr->value);
expr->value = print_expr;
}
}
}
......@@ -447,6 +447,7 @@ public:
virtual void accept_stmt(StmtVisitor* v);
AST_Expr() : AST_stmt(AST_TYPE::Expr) {}
AST_Expr(AST_expr* value) : AST_stmt(AST_TYPE::Expr), value(value) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expr;
};
......@@ -1425,6 +1426,10 @@ template <class T, class R> void findNodes(const R& roots, std::vector<T*>& outp
}
}
// Take a normally-parsed module, and convert it (inplace) to a form that will print out any bare expressions.
// This is used for "single" mode or the repl.
void makeModuleInteractive(AST_Module* m);
llvm::StringRef getOpSymbol(int op_type);
BoxedString* getOpName(int op_type);
int getReverseCmpOp(int op_type, bool& success);
......
......@@ -41,6 +41,7 @@ bool USE_STRIPPED_STDLIB = true; // always true
bool ENABLE_INTERPRETER = true;
bool ENABLE_BASELINEJIT = true;
bool ENABLE_PYPA_PARSER = true;
bool ENABLE_CPYTHON_PARSER = false;
bool USE_REGALLOC_BASIC = true;
bool PAUSE_AT_ABORT = false;
bool ENABLE_TRACEBACKS = true;
......@@ -90,9 +91,11 @@ bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;
extern "C" {
int Py_FrozenFlag = 1;
int Py_IgnoreEnvironmentFlag = 0;
int Py_InteractiveFlag = 0;
int Py_InspectFlag = 0;
int Py_NoSiteFlag = 0;
int Py_OptimizeFlag = 0;
int Py_VerboseFlag = 0;
int Py_UnicodeFlag = 0;
}
}
......@@ -38,8 +38,9 @@ extern int SPECULATION_THRESHOLD;
extern int MAX_OBJECT_CACHE_ENTRIES;
extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, ENABLE_CPYTHON_PARSER,
USE_REGALLOC_BASIC, PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS,
FORCE_LLVM_CAPI_THROWS;
extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
......
......@@ -193,9 +193,10 @@ int handleArg(char code) {
SHOW_DISASM = true;
else if (code == 'I')
FORCE_INTERPRETER = true;
else if (code == 'i')
else if (code == 'i') {
Py_InspectFlag = true;
else if (code == 'n') {
Py_InteractiveFlag = true;
} else if (code == 'n') {
ENABLE_INTERPRETER = false;
} else if (code == 'a') {
ASSEMBLY_LOGGING = true;
......@@ -207,6 +208,8 @@ int handleArg(char code) {
Stats::setEnabled(true);
} else if (code == 'S') {
Py_NoSiteFlag = 1;
} else if (code == 'U') {
Py_UnicodeFlag++;
} else if (code == 'u') {
unbuffered = true;
} else if (code == 'r') {
......@@ -215,6 +218,8 @@ int handleArg(char code) {
USE_REGALLOC_BASIC = false;
} else if (code == 'x') {
ENABLE_PYPA_PARSER = false;
} else if (code == 'X') {
ENABLE_CPYTHON_PARSER = true;
} else if (code == 'E') {
Py_IgnoreEnvironmentFlag = 1;
} else if (code == 'P') {
......@@ -325,7 +330,7 @@ static int main(int argc, char** argv) {
// Suppress getopt errors so we can throw them ourselves
opterr = 0;
while ((code = getopt(argc, argv, "+:OqdIibpjtrsRSvnxEac:FuPTGm:")) != -1) {
while ((code = getopt(argc, argv, "+:OqdIibpjtrsRSUvnxXEac:FuPTGm:")) != -1) {
if (code == 'c') {
assert(optarg);
command = optarg;
......@@ -497,42 +502,19 @@ static int main(int argc, char** argv) {
}
if (Py_InspectFlag || !(command || fn || module)) {
PyObject* v = PyImport_ImportModule("readline");
if (!v)
PyErr_Clear();
printf("Pyston v%d.%d (rev " STRINGIFY(GITREV) ")", PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR);
printf(", targeting Python %d.%d.%d\n", PYTHON_VERSION_MAJOR, PYTHON_VERSION_MINOR, PYTHON_VERSION_MICRO);
Py_InspectFlag = 0;
if (!main_module) {
main_module = createModule(boxString("__main__"), "<stdin>");
} else {
// main_module->fn = "<stdin>";
}
for (;;) {
char* line = readline(">> ");
if (!line)
break;
add_history(line);
try {
AST_Module* m = parse_string(line, /* future_flags = */ 0);
Timer _t("repl");
if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(e->value);
e->value = print_expr;
}
compileAndRunModule(m, main_module);
} catch (ExcInfo e) {
setCAPIException(e);
PyErr_Print();
}
}
PyCompilerFlags cf;
cf.cf_flags = 0;
rtncode = PyRun_InteractiveLoopFlags(stdin, "<stdin>", &cf);
}
threading::finishMainThread();
......
......@@ -151,6 +151,16 @@ extern "C" PyObject* PySys_GetObject(const char* name) noexcept {
return sys_module->getattr(internStringMortal(name));
}
extern "C" FILE* PySys_GetFile(char* name, FILE* def) noexcept {
FILE* fp = NULL;
PyObject* v = PySys_GetObject(name);
if (v != NULL && PyFile_Check(v))
fp = PyFile_AsFile(v);
if (fp == NULL)
fp = def;
return fp;
}
static void mywrite(const char* name, FILE* fp, const char* format, va_list va) noexcept {
PyObject* file;
PyObject* error_type, *error_value, *error_traceback;
......
......@@ -18,12 +18,23 @@
#include "Python.h"
#include "codegen/cpython_ast.h"
#include "grammar.h"
#include "node.h"
#include "token.h"
#include "parsetok.h"
#include "errcode.h"
#include "ast.h"
#undef BYTE
#undef STRING
#include "llvm/Support/ErrorHandling.h" // For llvm_unreachable
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "capi/typeobject.h"
#include "capi/types.h"
#include "codegen/irgen/hooks.h"
#include "codegen/unwinding.h"
#include "core/threading.h"
#include "core/types.h"
......@@ -1164,6 +1175,253 @@ static int dev_urandom_python(char* buffer, Py_ssize_t size) noexcept {
}
}
extern "C" int Py_FdIsInteractive(FILE* fp, const char* filename) noexcept {
if (isatty((int)fileno(fp)))
return 1;
if (!Py_InteractiveFlag)
return 0;
return (filename == NULL) || (strcmp(filename, "<stdin>") == 0) || (strcmp(filename, "???") == 0);
}
extern "C" int PyRun_InteractiveOneFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
PyObject* m, *d, *v, *w;
mod_ty mod;
PyArena* arena;
char _buf[1] = "";
char* ps1 = _buf, * ps2 = _buf;
int errcode = 0;
v = PySys_GetObject("ps1");
if (v != NULL) {
v = PyObject_Str(v);
if (v == NULL)
PyErr_Clear();
else if (PyString_Check(v))
ps1 = PyString_AsString(v);
}
w = PySys_GetObject("ps2");
if (w != NULL) {
w = PyObject_Str(w);
if (w == NULL)
PyErr_Clear();
else if (PyString_Check(w))
ps2 = PyString_AsString(w);
}
arena = PyArena_New();
if (arena == NULL) {
Py_XDECREF(v);
Py_XDECREF(w);
return -1;
}
mod = PyParser_ASTFromFile(fp, filename, Py_single_input, ps1, ps2, flags, &errcode, arena);
Py_XDECREF(v);
Py_XDECREF(w);
if (mod == NULL) {
PyArena_Free(arena);
if (errcode == E_EOF) {
PyErr_Clear();
return E_EOF;
}
PyErr_Print();
return -1;
}
m = PyImport_AddModule("__main__");
if (m == NULL) {
PyArena_Free(arena);
return -1;
}
// Pyston change:
// d = PyModule_GetDict(m);
// v = run_mod(mod, filename, d, d, flags, arena);
assert(PyModule_Check(m));
AST_Module* pyston_module = cpythonToPystonAST(mod);
makeModuleInteractive(pyston_module);
bool failed = false;
try {
compileAndRunModule(pyston_module, static_cast<BoxedModule*>(m));
} catch (ExcInfo e) {
setCAPIException(e);
failed = true;
}
PyArena_Free(arena);
if (failed) {
PyErr_Print();
return -1;
}
Py_DECREF(v);
if (Py_FlushLine())
PyErr_Clear();
return 0;
}
/* Set the error appropriate to the given input error code (see errcode.h) */
static void err_input(perrdetail* err) noexcept {
PyObject* v, *w, *errtype;
PyObject* u = NULL;
const char* msg = NULL;
errtype = PyExc_SyntaxError;
switch (err->error) {
case E_ERROR:
return;
case E_SYNTAX:
errtype = PyExc_IndentationError;
if (err->expected == INDENT)
msg = "expected an indented block";
else if (err->token == INDENT)
msg = "unexpected indent";
else if (err->token == DEDENT)
msg = "unexpected unindent";
else {
errtype = PyExc_SyntaxError;
msg = "invalid syntax";
}
break;
case E_TOKEN:
msg = "invalid token";
break;
case E_EOFS:
msg = "EOF while scanning triple-quoted string literal";
break;
case E_EOLS:
msg = "EOL while scanning string literal";
break;
case E_INTR:
if (!PyErr_Occurred())
PyErr_SetNone(PyExc_KeyboardInterrupt);
goto cleanup;
case E_NOMEM:
PyErr_NoMemory();
goto cleanup;
case E_EOF:
msg = "unexpected EOF while parsing";
break;
case E_TABSPACE:
errtype = PyExc_TabError;
msg = "inconsistent use of tabs and spaces in indentation";
break;
case E_OVERFLOW:
msg = "expression too long";
break;
case E_DEDENT:
errtype = PyExc_IndentationError;
msg = "unindent does not match any outer indentation level";
break;
case E_TOODEEP:
errtype = PyExc_IndentationError;
msg = "too many levels of indentation";
break;
case E_DECODE: {
PyObject* type, *value, *tb;
PyErr_Fetch(&type, &value, &tb);
if (value != NULL) {
u = PyObject_Str(value);
if (u != NULL) {
msg = PyString_AsString(u);
}
}
if (msg == NULL)
msg = "unknown decode error";
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(tb);
break;
}
case E_LINECONT:
msg = "unexpected character after line continuation character";
break;
default:
fprintf(stderr, "error=%d\n", err->error);
msg = "unknown parsing error";
break;
}
v = Py_BuildValue("(ziiz)", err->filename, err->lineno, err->offset, err->text);
w = NULL;
if (v != NULL)
w = Py_BuildValue("(sO)", msg, v);
Py_XDECREF(u);
Py_XDECREF(v);
PyErr_SetObject(errtype, w);
Py_XDECREF(w);
cleanup:
if (err->text != NULL) {
PyObject_FREE(err->text);
err->text = NULL;
}
}
#if 0
/* compute parser flags based on compiler flags */
#define PARSER_FLAGS(flags) \
((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0)
#endif
#if 1
/* Keep an example of flags with future keyword support. */
#define PARSER_FLAGS(flags) \
((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? PyPARSE_DONT_IMPLY_DEDENT : 0) \
| (((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION) ? PyPARSE_PRINT_IS_FUNCTION : 0) \
| (((flags)->cf_flags & CO_FUTURE_UNICODE_LITERALS) ? PyPARSE_UNICODE_LITERALS : 0)) \
: 0)
#endif
extern "C" grammar _PyParser_Grammar;
extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start, char* ps1, char* ps2,
PyCompilerFlags* flags, int* errcode, PyArena* arena) noexcept {
mod_ty mod;
PyCompilerFlags localflags;
perrdetail err;
int iflags = PARSER_FLAGS(flags);
node* n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, start, ps1, ps2, &err, &iflags);
if (flags == NULL) {
localflags.cf_flags = 0;
flags = &localflags;
}
if (n) {
flags->cf_flags |= iflags & PyCF_MASK;
mod = PyAST_FromNode(n, flags, filename, arena);
PyNode_Free(n);
return mod;
} else {
err_input(&err);
if (errcode)
*errcode = err.error;
return NULL;
}
}
extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
PyObject* v;
int ret;
PyCompilerFlags local_flags;
if (flags == NULL) {
flags = &local_flags;
local_flags.cf_flags = 0;
}
v = PySys_GetObject("ps1");
if (v == NULL) {
PySys_SetObject("ps1", v = PyString_FromString(">> "));
Py_XDECREF(v);
}
v = PySys_GetObject("ps2");
if (v == NULL) {
PySys_SetObject("ps2", v = PyString_FromString("... "));
Py_XDECREF(v);
}
for (;;) {
ret = PyRun_InteractiveOneFlags(fp, filename, flags);
// PRINT_TOTAL_REFS();
if (ret == E_EOF)
return 0;
if (ret == E_NOMEM)
return -1;
}
}
static const char* progname = "pyston";
extern "C" void Py_SetProgramName(char* pn) noexcept {
if (pn && *pn)
......@@ -1642,6 +1900,13 @@ Box* BoxedCApiFunction::tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPa
return rtn;
}
/* Warning with explicit origin */
extern "C" int PyErr_WarnExplicit(PyObject* category, const char* text, const char* filename_str, int lineno,
const char* module_str, PyObject* registry) noexcept {
Py_FatalError("unimplemented");
}
/* extension modules might be compiled with GC support so these
functions must always be available */
......
......@@ -130,6 +130,98 @@ static PyObject* err_iterbuffered(void) noexcept {
return NULL;
}
/*
** Py_UniversalNewlineFgets is an fgets variation that understands
** all of \r, \n and \r\n conventions.
** The stream should be opened in binary mode.
** If fobj is NULL the routine always does newline conversion, and
** it may peek one char ahead to gobble the second char in \r\n.
** If fobj is non-NULL it must be a PyFileObject. In this case there
** is no readahead but in stead a flag is used to skip a following
** \n on the next read. Also, if the file is open in binary mode
** the whole conversion is skipped. Finally, the routine keeps track of
** the different types of newlines seen.
** Note that we need no error handling: fgets() treats error and eof
** identically.
*/
extern "C" char* Py_UniversalNewlineFgets(char* buf, int n, FILE* stream, PyObject* fobj) noexcept {
char* p = buf;
int c;
int newlinetypes = 0;
int skipnextlf = 0;
int univ_newline = 1;
if (fobj) {
if (!PyFile_Check(fobj)) {
errno = ENXIO; /* What can you do... */
return NULL;
}
univ_newline = ((BoxedFile*)fobj)->f_univ_newline;
if (!univ_newline)
return fgets(buf, n, stream);
newlinetypes = ((BoxedFile*)fobj)->f_newlinetypes;
skipnextlf = ((BoxedFile*)fobj)->f_skipnextlf;
}
FLOCKFILE(stream);
c = 'x'; /* Shut up gcc warning */
while (--n > 0 && (c = GETC(stream)) != EOF) {
if (skipnextlf) {
skipnextlf = 0;
if (c == '\n') {
/* Seeing a \n here with skipnextlf true
** means we saw a \r before.
*/
newlinetypes |= NEWLINE_CRLF;
c = GETC(stream);
if (c == EOF)
break;
} else {
/*
** Note that c == EOF also brings us here,
** so we're okay if the last char in the file
** is a CR.
*/
newlinetypes |= NEWLINE_CR;
}
}
if (c == '\r') {
/* A \r is translated into a \n, and we skip
** an adjacent \n, if any. We don't set the
** newlinetypes flag until we've seen the next char.
*/
skipnextlf = 1;
c = '\n';
} else if (c == '\n') {
newlinetypes |= NEWLINE_LF;
}
*p++ = c;
if (c == '\n')
break;
}
if (c == EOF && skipnextlf)
newlinetypes |= NEWLINE_CR;
FUNLOCKFILE(stream);
*p = '\0';
if (fobj) {
((BoxedFile*)fobj)->f_newlinetypes = newlinetypes;
((BoxedFile*)fobj)->f_skipnextlf = skipnextlf;
} else if (skipnextlf) {
/* If we have no file object we cannot save the
** skipnextlf flag. We have to readahead, which
** will cause a pause if we're reading from an
** interactive stream, but that is very unlikely
** unless we're doing something silly like
** execfile("/dev/tty").
*/
c = GETC(stream);
if (c != '\n')
ungetc(c, stream);
}
if (p == buf)
return NULL;
return buf;
}
static BoxedFile* dircheck(BoxedFile* f) {
#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
struct stat buf;
......@@ -1298,6 +1390,10 @@ extern "C" int PyFile_SetEncoding(PyObject* f, const char* enc) noexcept {
return PyFile_SetEncodingAndErrors(f, enc, NULL);
}
extern "C" PyObject* PyFile_GetEncoding(PyObject* f) noexcept {
return static_cast<BoxedFile*>(f)->f_encoding;
}
extern "C" int PyFile_SetEncodingAndErrors(PyObject* f, const char* enc, char* errors) noexcept {
BoxedFile* file = static_cast<BoxedFile*>(f);
PyObject* str, *oerrors;
......
......@@ -212,7 +212,7 @@ extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) n
BoxedLong* rtn = new BoxedLong();
int r = 0;
if (str[strlen(str) - 1] == 'L') {
if (str[strlen(str) - 1] == 'L' || str[strlen(str) - 1] == 'l') {
std::string without_l(str, strlen(str) - 1);
r = mpz_init_set_str(rtn->n, without_l.c_str(), base);
} else {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment