Commit 948416e6 authored by Kevin Modzelewski's avatar Kevin Modzelewski

Add some more builtin-type functionality

- fix list sorting gc bug (we did not hook std::get_temporary_buffer)
- eval(" 2")
- better SystemExit handling
- file.writelines
- list assign non-sequential slice
parent 06ffe979
......@@ -866,13 +866,18 @@ Value ASTInterpreter::visit_delete(AST_Delete* node) {
continue;
} else if (vst == ScopeInfo::VarScopeType::NAME) {
assert(frame_info.boxedLocals != NULL);
assert(frame_info.boxedLocals->cls == dict_cls);
auto& d = static_cast<BoxedDict*>(frame_info.boxedLocals)->d;
auto it = d.find(boxString(target->id.str()));
if (it == d.end()) {
assertNameDefined(0, target->id.c_str(), NameError, false /* local_var_msg */);
if (frame_info.boxedLocals->cls == dict_cls) {
auto& d = static_cast<BoxedDict*>(frame_info.boxedLocals)->d;
auto it = d.find(boxString(target->id.str()));
if (it == d.end()) {
assertNameDefined(0, target->id.c_str(), NameError, false /* local_var_msg */);
}
d.erase(it);
} else if (frame_info.boxedLocals->cls == attrwrapper_cls) {
attrwrapperDel(frame_info.boxedLocals, target->id.str());
} else {
RELEASE_ASSERT(0, "%s", frame_info.boxedLocals->cls->tp_name);
}
d.erase(it);
} else {
assert(vst == ScopeInfo::VarScopeType::FAST);
......
......@@ -382,7 +382,17 @@ Box* eval(Box* boxedCode) {
// TODO this memory leaks
RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name);
const char* code = static_cast<BoxedString*>(boxedCode)->s.data();
// Hack: we need to support things like `eval(" 2")`.
// This is over-accepting since it will accept things like `eval("\n 2")`
while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
code++;
AST_Module* parsedModule = parse_string(code);
if (parsedModule->body.size() == 0)
raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
RELEASE_ASSERT(parsedModule->body.size() == 1, "");
RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");
AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
......
......@@ -591,4 +591,9 @@ struct CallattrFlags {
};
}
namespace std {
template <> std::pair<pyston::Box**, std::ptrdiff_t> get_temporary_buffer<pyston::Box*>(std::ptrdiff_t count) noexcept;
template <> void return_temporary_buffer<pyston::Box*>(pyston::Box** p);
}
#endif
......@@ -30,6 +30,16 @@
//#undef VERBOSITY
//#define VERBOSITY(x) 2
namespace std {
template <> std::pair<pyston::Box**, std::ptrdiff_t> get_temporary_buffer<pyston::Box*>(std::ptrdiff_t count) noexcept {
void* r = pyston::gc::gc_alloc(sizeof(pyston::Box*) * count, pyston::gc::GCKind::CONSERVATIVE);
return std::make_pair((pyston::Box**)r, count);
}
template <> void return_temporary_buffer<pyston::Box*>(pyston::Box** p) {
pyston::gc::gc_free(p);
}
}
namespace pyston {
namespace gc {
......
......@@ -56,15 +56,31 @@ extern void setEncodingAndErrors();
// return code in `*retcode`. does not touch `*retcode* if it returns false.
static bool handle_toplevel_exn(const ExcInfo& e, int* retcode) {
if (e.matches(SystemExit)) {
Box* code = e.value->getattr("code");
Box* value = e.value;
if (!code || code == None)
if (value && PyExceptionInstance_Check(value)) {
Box* code = getattr(value, "code");
if (code)
value = code;
}
if (!value || value == None)
*retcode = 0;
else if (isSubclass(code->cls, int_cls))
*retcode = static_cast<BoxedInt*>(code)->n;
else
else if (isSubclass(value->cls, int_cls))
*retcode = static_cast<BoxedInt*>(value)->n;
else {
*retcode = 1;
PyObject* sys_stderr = PySys_GetObject("stderr");
if (sys_stderr != NULL && sys_stderr != Py_None) {
PyFile_WriteObject(value, sys_stderr, Py_PRINT_RAW);
} else {
PyObject_Print(value, stderr, Py_PRINT_RAW);
fflush(stderr);
}
PySys_WriteStderr("\n");
}
return true;
}
e.printExcAndTraceback();
......@@ -85,7 +101,7 @@ static int main(int argc, char** argv) {
// Suppress getopt errors so we can throw them ourselves
opterr = 0;
while ((code = getopt(argc, argv, "+:OqdIibpjtrsSvnxc:FuP")) != -1) {
while ((code = getopt(argc, argv, "+:OqdIibpjtrsSvnxEc:FuP")) != -1) {
if (code == 'O')
FORCE_OPTIMIZE = true;
else if (code == 't')
......@@ -118,6 +134,8 @@ static int main(int argc, char** argv) {
USE_REGALLOC_BASIC = false;
} else if (code == 'x') {
ENABLE_PYPA_PARSER = false;
} else if (code == 'E') {
Py_IgnoreEnvironmentFlag = 1;
} else if (code == 'P') {
PAUSE_AT_ABORT = true;
} else if (code == 'F') {
......
......@@ -480,7 +480,15 @@ static PyObject* file_write(BoxedFile* f, Box* arg) noexcept {
if (!f->writable)
return err_mode("writing");
if (f->f_binary) {
if (PyObject_GetBuffer(arg, &pbuf, 0))
// In CPython, this branch calls PyArg_ParseTuple for all types, but we never created
// the "args" tuple so we have to do some of the work that ParseTuple does.
// Mostly it's easy since we've already unpacked the args, but there is some unicode-specific
// code in it that is better not to duplicate.
// So, if it's unicode, just make the tuple for now and send it through PyArg_ParseTuple.
if (PyUnicode_Check(arg)) {
if (!PyArg_ParseTuple(BoxedTuple::create({ arg }), "s*", &pbuf))
return NULL;
} else if (PyObject_GetBuffer(arg, &pbuf, 0))
return NULL;
s = (const char*)pbuf.buf;
......@@ -537,6 +545,124 @@ static PyObject* file_write(BoxedFile* f, Box* arg) noexcept {
return Py_None;
}
static PyObject* file_writelines(BoxedFile* f, PyObject* seq) noexcept {
#define CHUNKSIZE 1000
PyObject* list, *line;
PyObject* it; /* iter(seq) */
PyObject* result;
int index, islist;
Py_ssize_t i, j, nwritten, len;
assert(seq != NULL);
if (f->f_fp == NULL)
return err_closed();
if (!f->writable)
return err_mode("writing");
result = NULL;
list = NULL;
islist = PyList_Check(seq);
if (islist)
it = NULL;
else {
it = PyObject_GetIter(seq);
if (it == NULL) {
PyErr_SetString(PyExc_TypeError, "writelines() requires an iterable argument");
return NULL;
}
/* From here on, fail by going to error, to reclaim "it". */
list = PyList_New(CHUNKSIZE);
if (list == NULL)
goto error;
}
/* Strategy: slurp CHUNKSIZE lines into a private list,
checking that they are all strings, then write that list
without holding the interpreter lock, then come back for more. */
for (index = 0;; index += CHUNKSIZE) {
if (islist) {
Py_XDECREF(list);
list = PyList_GetSlice(seq, index, index + CHUNKSIZE);
if (list == NULL)
goto error;
j = PyList_GET_SIZE(list);
} else {
for (j = 0; j < CHUNKSIZE; j++) {
line = PyIter_Next(it);
if (line == NULL) {
if (PyErr_Occurred())
goto error;
break;
}
PyList_SetItem(list, j, line);
}
/* The iterator might have closed the file on us. */
if (f->f_fp == NULL) {
err_closed();
goto error;
}
}
if (j == 0)
break;
/* Check that all entries are indeed strings. If not,
apply the same rules as for file.write() and
convert the results to strings. This is slow, but
seems to be the only way since all conversion APIs
could potentially execute Python code. */
for (i = 0; i < j; i++) {
PyObject* v = PyList_GET_ITEM(list, i);
if (!PyString_Check(v)) {
const char* buffer;
int res;
if (f->f_binary) {
res = PyObject_AsReadBuffer(v, (const void**)&buffer, &len);
} else {
res = PyObject_AsCharBuffer(v, &buffer, &len);
}
if (res) {
PyErr_SetString(PyExc_TypeError, "writelines() argument must be a sequence of strings");
goto error;
}
line = PyString_FromStringAndSize(buffer, len);
if (line == NULL)
goto error;
Py_DECREF(v);
PyList_SET_ITEM(list, i, line);
}
}
/* Since we are releasing the global lock, the
following code may *not* execute Python code. */
f->f_softspace = 0;
FILE_BEGIN_ALLOW_THREADS(f)
errno = 0;
for (i = 0; i < j; i++) {
line = PyList_GET_ITEM(list, i);
len = PyString_GET_SIZE(line);
nwritten = fwrite(PyString_AS_STRING(line), 1, len, f->f_fp);
if (nwritten != len) {
FILE_ABORT_ALLOW_THREADS(f)
PyErr_SetFromErrno(PyExc_IOError);
clearerr(f->f_fp);
goto error;
}
}
FILE_END_ALLOW_THREADS(f)
if (j < CHUNKSIZE)
break;
}
Py_INCREF(Py_None);
result = Py_None;
error:
Py_XDECREF(list);
Py_XDECREF(it);
return result;
#undef CHUNKSIZE
}
Box* fileWrite(BoxedFile* self, Box* val) {
assert(self->cls == file_cls);
......@@ -1469,6 +1595,7 @@ PyDoc_STRVAR(isatty_doc, "isatty() -> true or false. True if the file is connec
PyMethodDef file_methods[] = {
{ "seek", (PyCFunction)file_seek, METH_VARARGS, seek_doc },
{ "readlines", (PyCFunction)file_readlines, METH_VARARGS, readlines_doc },
{ "writelines", (PyCFunction)file_writelines, METH_O, NULL },
{ "isatty", (PyCFunction)file_isatty, METH_NOARGS, isatty_doc },
};
......
......@@ -274,7 +274,14 @@ static Box* getParent(Box* globals, int level, std::string& buf) {
if (modname == NULL || modname->cls != str_cls)
return None;
Box* modpath = getattrInternal(globals, path_str, NULL);
Box* modpath = NULL;
try {
modpath = getattrInternal(globals, path_str, NULL);
} catch (ExcInfo e) {
if (!e.matches(AttributeError))
raiseRaw(e);
}
if (modpath != NULL) {
/* __path__ is set, so modname is already the package name */
if (modname->size() > PATH_MAX) {
......@@ -516,7 +523,15 @@ extern "C" PyObject* PyImport_ImportModuleLevel(const char* name, PyObject* glob
}
static void ensureFromlist(Box* module, Box* fromlist, std::string& buf, bool recursive) {
if (getattrInternal(module, path_str, NULL) == NULL) {
Box* pathlist = NULL;
try {
pathlist = getattrInternal(module, path_str, NULL);
} catch (ExcInfo e) {
if (!e.matches(AttributeError))
raiseRaw(e);
}
if (pathlist == NULL) {
// If it's not a package, then there's no sub-importing to do
return;
}
......
......@@ -246,6 +246,138 @@ static void sliceIndex(Box* b, int64_t* out) {
*out = static_cast<BoxedInt*>(b)->n;
}
// Copied from CPython's list_ass_subscript
int list_ass_ext_slice(BoxedList* self, PyObject* item, PyObject* value) {
Py_ssize_t start, stop, step, slicelength;
if (PySlice_GetIndicesEx((PySliceObject*)item, Py_SIZE(self), &start, &stop, &step, &slicelength) < 0) {
return -1;
}
RELEASE_ASSERT(step != 1, "should have handled this elsewhere");
// if (step == 1)
// return list_ass_slice(self, start, stop, value);
/* Make sure s[5:2] = [..] inserts at the right place:
before 5, not before 2. */
if ((step < 0 && start < stop) || (step > 0 && start > stop))
stop = start;
if (value == NULL) {
/* delete slice */
PyObject** garbage;
size_t cur;
Py_ssize_t i;
if (slicelength <= 0)
return 0;
if (step < 0) {
stop = start + 1;
start = stop + step * (slicelength - 1) - 1;
step = -step;
}
assert((size_t)slicelength <= PY_SIZE_MAX / sizeof(PyObject*));
garbage = (PyObject**)PyMem_MALLOC(slicelength * sizeof(PyObject*));
if (!garbage) {
PyErr_NoMemory();
return -1;
}
/* drawing pictures might help understand these for
loops. Basically, we memmove the parts of the
list that are *not* part of the slice: step-1
items for each item that is part of the slice,
and then tail end of the list that was not
covered by the slice */
for (cur = start, i = 0; cur < (size_t)stop; cur += step, i++) {
Py_ssize_t lim = step - 1;
garbage[i] = PyList_GET_ITEM(self, cur);
if (cur + step >= self->size) {
lim = self->size - cur - 1;
}
memmove(self->elts->elts + cur - i, self->elts->elts + cur + 1, lim * sizeof(PyObject*));
}
cur = start + slicelength * step;
if (cur < self->size) {
memmove(self->elts->elts + cur - slicelength, self->elts->elts + cur,
(self->size - cur) * sizeof(PyObject*));
}
self->size -= slicelength;
// list_resize(self, Py_SIZE(self));
for (i = 0; i < slicelength; i++) {
Py_DECREF(garbage[i]);
}
PyMem_FREE(garbage);
return 0;
} else {
/* assign slice */
PyObject* ins, *seq;
PyObject** garbage, **seqitems, **selfitems;
Py_ssize_t cur, i;
/* protect against a[::-1] = a */
if (self == value) {
abort();
// seq = list_slice((PyListObject*)value, 0,
// PyList_GET_SIZE(value));
} else {
seq = PySequence_Fast(value, "must assign iterable "
"to extended slice");
}
if (!seq)
return -1;
if (PySequence_Fast_GET_SIZE(seq) != slicelength) {
PyErr_Format(PyExc_ValueError, "attempt to assign sequence of "
"size %zd to extended slice of "
"size %zd",
PySequence_Fast_GET_SIZE(seq), slicelength);
Py_DECREF(seq);
return -1;
}
if (!slicelength) {
Py_DECREF(seq);
return 0;
}
garbage = (PyObject**)PyMem_MALLOC(slicelength * sizeof(PyObject*));
if (!garbage) {
Py_DECREF(seq);
PyErr_NoMemory();
return -1;
}
selfitems = self->elts->elts;
seqitems = PySequence_Fast_ITEMS(seq);
for (cur = start, i = 0; i < slicelength; cur += step, i++) {
garbage[i] = selfitems[cur];
ins = seqitems[i];
Py_INCREF(ins);
selfitems[cur] = ins;
}
for (i = 0; i < slicelength; i++) {
Py_DECREF(garbage[i]);
}
PyMem_FREE(garbage);
Py_DECREF(seq);
return 0;
}
}
extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
LOCK_REGION(self->lock.asWrite());
......@@ -258,6 +390,12 @@ extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
sliceIndex(slice->stop, &stop);
sliceIndex(slice->step, &step);
if (step != 1) {
int r = list_ass_ext_slice(self, slice, v);
if (r)
throwCAPIException();
return None;
}
RELEASE_ASSERT(step == 1, "step sizes must be 1 for now");
// Logic from PySequence_GetSlice:
......@@ -566,6 +704,18 @@ extern "C" int PyList_Sort(PyObject* v) noexcept {
return 0;
}
extern "C" Box* PyList_GetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
assert(isSubclass(a->cls, list_cls));
BoxedList* self = static_cast<BoxedList*>(a);
try {
// Lots of extra copies here; we can do better if we need to:
return listGetitemSlice(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
} catch (ExcInfo e) {
setCAPIException(e);
return NULL;
}
}
Box* listContains(BoxedList* self, Box* elt) {
LOCK_REGION(self->lock.asRead());
......@@ -775,6 +925,46 @@ Box* listNe(BoxedList* self, Box* rhs) {
return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::NotEq);
}
Box* listLt(BoxedList* self, Box* rhs) {
if (rhs->cls != list_cls) {
return NotImplemented;
}
LOCK_REGION(self->lock.asRead());
return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::Lt);
}
Box* listLe(BoxedList* self, Box* rhs) {
if (rhs->cls != list_cls) {
return NotImplemented;
}
LOCK_REGION(self->lock.asRead());
return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::LtE);
}
Box* listGt(BoxedList* self, Box* rhs) {
if (rhs->cls != list_cls) {
return NotImplemented;
}
LOCK_REGION(self->lock.asRead());
return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::Gt);
}
Box* listGe(BoxedList* self, Box* rhs) {
if (rhs->cls != list_cls) {
return NotImplemented;
}
LOCK_REGION(self->lock.asRead());
return _listCmp(self, static_cast<BoxedList*>(rhs), AST_TYPE::GtE);
}
extern "C" PyObject* _PyList_Extend(PyListObject* self, PyObject* b) noexcept {
BoxedList* l = (BoxedList*)self;
assert(isSubclass(l->cls, list_cls));
......@@ -831,6 +1021,10 @@ void setupList() {
list_cls->giveAttr("__eq__", new BoxedFunction(boxRTFunction((void*)listEq, UNKNOWN, 2)));
list_cls->giveAttr("__ne__", new BoxedFunction(boxRTFunction((void*)listNe, UNKNOWN, 2)));
list_cls->giveAttr("__lt__", new BoxedFunction(boxRTFunction((void*)listLt, UNKNOWN, 2)));
list_cls->giveAttr("__le__", new BoxedFunction(boxRTFunction((void*)listLe, UNKNOWN, 2)));
list_cls->giveAttr("__gt__", new BoxedFunction(boxRTFunction((void*)listGt, UNKNOWN, 2)));
list_cls->giveAttr("__ge__", new BoxedFunction(boxRTFunction((void*)listGe, UNKNOWN, 2)));
list_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)listRepr, STR, 1)));
list_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)listNonzero, BOXED_BOOL, 1)));
......
......@@ -4409,11 +4409,7 @@ extern "C" Box* getGlobal(Box* globals, const std::string* name) {
}
extern "C" Box* importFrom(Box* _m, const std::string* name) {
assert(isSubclass(_m->cls, module_cls));
BoxedModule* m = static_cast<BoxedModule*>(_m);
Box* r = getattrInternal(m, *name, NULL);
Box* r = getattrInternal(_m, *name, NULL);
if (r)
return r;
......
......@@ -1431,6 +1431,10 @@ Box* attrwrapperKeys(Box* b) {
return AttrWrapper::keys(b);
}
void attrwrapperDel(Box* b, const std::string& attr) {
AttrWrapper::delitem(b, boxString(attr));
}
Box* objectNewNoArgs(BoxedClass* cls) {
assert(isSubclass(cls->cls, type_cls));
assert(typeLookup(cls, "__new__", NULL) == typeLookup(object_cls, "__new__", NULL)
......
......@@ -790,6 +790,7 @@ Box* objectSetattr(Box* obj, Box* attr, Box* value);
Box* makeAttrWrapper(Box* b);
Box* unwrapAttrWrapper(Box* b);
Box* attrwrapperKeys(Box* b);
void attrwrapperDel(Box* b, const std::string& attr);
#define SystemError ((BoxedClass*)PyExc_SystemError)
#define StopIteration ((BoxedClass*)PyExc_StopIteration)
......
......@@ -7,7 +7,7 @@ with open('/dev/null')as ignore:
# We don't (yet?) require exact stderr or return code compatibility w/
# python. So we just check that we succeed or fail as appropriate.
def run(args):
print 0 == subprocess.call([me] + args, stderr=ignore)
print subprocess.call([me] + args, stderr=ignore)
run(["-c", "print 2 + 2"])
run(["-c", "import sys; print sys.argv", "hello", "world"])
......@@ -16,3 +16,6 @@ with open('/dev/null')as ignore:
run(["-c"])
run(["-c", "-c"])
run(["-c", "this should not work"])
run(["-c", ";"])
run(["-cprint 1"])
# expected: fail
try:
eval("\n 2")
print "bad, should have thrown an exception"
except SyntaxError:
print "good, threw exception"
......@@ -157,3 +157,9 @@ def wrap():
inner2()
wrap()
try:
eval(" ")
print "worked?"
except SyntaxError:
pass
......@@ -4,6 +4,12 @@ print a"""
exec ""
try:
exec ";"
print "worked?"
except SyntaxError:
pass
# Exec of a unicode encodes as utf8:
exec u"print repr('\u0180')"
print repr(eval(u"'\u0180'"))
......@@ -4,13 +4,15 @@ import tempfile
fd, fn = tempfile.mkstemp()
with open(fn, "wb") as f:
f.write("hello world!")
f.write("hello world!\n")
f.write(u"hello world2")
with open(fn) as f:
print repr(f.read())
with open(fn, "w") as f:
f.write("hello world!")
f.writelines(["hi", "world"])
with open(fn) as f:
print repr(f.read())
......
# This throws an exception in the import machinery when we try to access __path__,
# but that should get caught.
# Also, email.MIMEText isn't even a Module or a subclass of Module...
from email.MIMEText import MIMEText
from email.MIMEMultipart import MIMEMultipart
# TODO check similar cases with descriptors
......@@ -153,3 +153,21 @@ print l
l = range(5)
l[2:4] = tuple(range(2))
print l
l = [None]*4
try:
l[::-1] = range(5)
except ValueError as e:
print e
l[::-1] = range(4)
print l
del l[::2]
print l
for i in xrange(3):
for j in xrange(3):
for k in xrange(3):
l1 = [i]
l2 = [j, k]
print l1 < l2, l1 <= l2, l1 > l2, l1 >= l2
# regression test: list sorting had a gc bug
import gc
class C(object):
def __init__(self, n):
self.n = range(n)
def __eq__(self, rhs):
# print "eq"
gc.collect()
return self.n == rhs.n
def __lt__(self, rhs):
# print "lt"
gc.collect()
return self.n < rhs.n
def keyfunc(c):
return c
def f():
for i in xrange(10):
print i
l = [C(i % 5) for i in xrange(10)]
l.sort(key=keyfunc)
f()
......@@ -29,6 +29,7 @@ def f3():
s = "hello world"
t = (1.0, "asdf")
print sorted(locals().items())
print sorted(vars().items())
f3()
def f4(t):
......@@ -38,6 +39,7 @@ def f4(t):
else:
y = 2
print sorted(locals().items())
print sorted(vars().items())
f4(0)
f4(1)
......@@ -52,6 +54,7 @@ def f5():
print a
print b
print sorted(locals().items())
print sorted(vars().items())
i()
h()
g()
......
# should_error
# no-collect-stats
try:
raise SystemExit, "hello"
except Exception as e:
pass
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment