Merge branch 'selfhost'

from_cpython/Include/dictobject.h

@@ -123,7 +123,8 @@ PyAPI_DATA(PyTypeObject*) dictvalues_cls;
 #define PyDictValues_Type (*dictvalues_cls)
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-PyAPI_FUNC(bool) PyDict_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PyDict_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PyDict_Check(op) _PyDict_Check((PyObject*)(op))
 #if 0
 #define PyDict_Check(op) \
                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS)

from_cpython/Include/listobject.h

@@ -49,7 +49,8 @@ PyAPI_DATA(PyTypeObject*) list_cls;
 #define PyList_Type (*list_cls)
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-PyAPI_FUNC(bool) PyList_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PyList_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PyList_Check(op) _PyList_Check((PyObject*)(op))
 #if 0
 #define PyList_Check(op) \
 		PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LIST_SUBCLASS)
@@ -71,9 +72,9 @@ PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *) PYSTON_NOEXCEP
 
 /* Macro, trading safety for speed */
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-#define PyList_GET_ITEM(op, i) PyList_GetItem(op, i)
-#define PyList_SET_ITEM(op, i, v) PyList_SetItem(op, i, v)
-#define PyList_GET_SIZE(op)    PyList_Size(op)
+#define PyList_GET_ITEM(op, i) PyList_GetItem((PyObject*)(op), (i))
+#define PyList_SET_ITEM(op, i, v) PyList_SetItem((PyObject*)(op), (i), (v))
+#define PyList_GET_SIZE(op)    PyList_Size((PyObject*)(op))
 //#define PyList_GET_ITEM(op, i) (((PyListObject *)(op))->ob_item[i])
 //#define PyList_SET_ITEM(op, i, v) (((PyListObject *)(op))->ob_item[i] = (v))
 //#define PyList_GET_SIZE(op)    Py_SIZE(op)

from_cpython/Include/longobject.h

@@ -21,7 +21,8 @@ PyAPI_DATA(PyTypeObject*) long_cls;
 #define PyLong_Type (*long_cls)
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-PyAPI_FUNC(bool) PyLong_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PyLong_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PyLong_Check(op) _PyLong_Check((PyObject*)(op))
 #if 0
 #define PyLong_Check(op) \
 		PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)

from_cpython/Include/object.h

@@ -499,7 +499,8 @@ PyAPI_DATA(PyTypeObject*) type_cls;
 //PyAPI_DATA(PyTypeObject) PySuper_Type; /* built-in 'super' */
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-PyAPI_FUNC(bool) PyType_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PyType_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PyType_Check(op) _PyType_Check((PyObject*)(op))
 #if 0
 #define PyType_Check(op) \
     PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)

from_cpython/Include/sliceobject.h

@@ -39,7 +39,8 @@ PyAPI_DATA(PyTypeObject*) ellipsis_cls;
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
 //#define PySlice_Check(op) (Py_TYPE(op) == &PySlice_Type)
-PyAPI_FUNC(bool) PySlice_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PySlice_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PySlice_Check(op) _PySlice_Check((PyObject*)(op))
 
 PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop,
                                   PyObject* step) PYSTON_NOEXCEPT;

from_cpython/Include/tupleobject.h

@@ -42,7 +42,8 @@ PyAPI_DATA(PyTypeObject*) tuple_cls;
 #define PyTuple_Type (*tuple_cls)
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-PyAPI_FUNC(bool) PyTuple_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PyTuple_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PyTuple_Check(op) _PyTuple_Check((PyObject*)(op))
 #if 0
 #define PyTuple_Check(op) \
                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TUPLE_SUBCLASS)

from_cpython/Include/unicodeobject.h

@@ -435,7 +435,8 @@ PyAPI_DATA(PyTypeObject*) unicode_cls;
 //PyAPI_DATA(PyTypeObject) PyUnicode_Type;
 
 // Pyston changes: these aren't direct macros any more [they potentially could be though]
-PyAPI_FUNC(bool) PyUnicode_Check(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(bool) _PyUnicode_Check(PyObject*) PYSTON_NOEXCEPT;
+#define PyUnicode_Check(op) _PyUnicode_Check((PyObject*)(op))
 #if 0
 #define PyUnicode_Check(op) \
                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)

src/runtime/builtin_modules/builtins.cpp

@@ -613,7 +613,7 @@ BoxedClass* BaseException, *Exception, *StandardError, *AssertionError, *Attribu
     *NameError, *KeyError, *IndexError, *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError,
     *RuntimeError, *ImportError, *StopIteration, *Warning, *SyntaxError, *OverflowError, *DeprecationWarning,
     *MemoryError, *LookupError, *EnvironmentError, *ArithmeticError, *BufferError, *KeyboardInterrupt, *SystemExit,
-    *SystemError, *NotImplementedError, *PendingDeprecationWarning;
+    *SystemError, *NotImplementedError, *PendingDeprecationWarning, *EOFError;
 
 Box* PyExc_RecursionErrorInst;
 Box* PyExc_MemoryErrorInst;
@@ -1038,6 +1038,7 @@ void setupBuiltins() {
     SystemError = makeBuiltinException(StandardError, "SystemError");
     NotImplementedError = makeBuiltinException(RuntimeError, "NotImplementedError");
     PendingDeprecationWarning = makeBuiltinException(Warning, "PendingDeprecationWarning");
+    EOFError = makeBuiltinException(StandardError, "EOFError");
 
     EnvironmentError->gc_visit = BoxedEnvironmentError::gcHandler;
     EnvironmentError->giveAttr(

src/runtime/builtin_modules/thread.cpp

@@ -45,6 +45,22 @@ Box* startNewThread(Box* target, Box* args) {
     return boxInt(thread_id ^ 0x12345678901L);
 }
 
+#define CHECK_STATUS(name)                                                                                             \
+    if (status != 0) {                                                                                                 \
+        perror(name);                                                                                                  \
+        error = 1;                                                                                                     \
+    }
+
+/*
+ * As of February 2002, Cygwin thread implementations mistakenly report error
+ * codes in the return value of the sem_ calls (like the pthread_ functions).
+ * Correct implementations return -1 and put the code in errno. This supports
+ * either.
+ */
+static int fix_status(int status) {
+    return (status == -1) ? errno : status;
+}
+
 static BoxedClass* thread_lock_cls;
 class BoxedThreadLock : public Box {
 private:
@@ -55,12 +71,38 @@ public:
 
     DEFAULT_CLASS(thread_lock_cls);
 
-    static Box* acquire(Box* _self) {
+    static Box* acquire(Box* _self, Box* _waitflag) {
         RELEASE_ASSERT(_self->cls == thread_lock_cls, "");
         BoxedThreadLock* self = static_cast<BoxedThreadLock*>(_self);
 
-        pthread_mutex_lock(&self->lock);
-        return None;
+        RELEASE_ASSERT(_waitflag->cls == int_cls, "");
+        int waitflag = static_cast<BoxedInt*>(_waitflag)->n;
+
+        // Copied + adapted from CPython:
+        int success;
+        auto thelock = &self->lock;
+        int status, error = 0;
+
+        {
+            threading::GLAllowThreadsReadRegion _allow_threads;
+
+            do {
+                if (waitflag)
+                    status = fix_status(pthread_mutex_lock(thelock));
+                else
+                    status = fix_status(pthread_mutex_trylock(thelock));
+            } while (status == EINTR); /* Retry if interrupted by a signal */
+        }
+
+        if (waitflag) {
+            CHECK_STATUS("mutex_lock");
+        } else if (status != EBUSY) {
+            CHECK_STATUS("mutex_trylock");
+        }
+
+        success = (status == 0) ? 1 : 0;
+
+        return boxBool(status == 0);
     }
 
     static Box* release(Box* _self) {
@@ -70,32 +112,57 @@ public:
         pthread_mutex_unlock(&self->lock);
         return None;
     }
+
+    static Box* exit(Box* _self, Box* arg1, Box* arg2, Box** args) { return release(_self); }
 };
 
 Box* allocateLock() {
     return new BoxedThreadLock();
 }
 
+static BoxedClass* thread_local_cls;
+class BoxedThreadLocal : public Box {
+public:
+    BoxedThreadLocal() {}
+
+    DEFAULT_CLASS(thread_local_cls);
+};
+
 Box* getIdent() {
     return boxInt(pthread_self());
 }
 
+Box* stackSize() {
+    Py_FatalError("unimplemented");
+}
+
 void setupThread() {
     thread_module = createModule("thread", "__builtin__");
 
     thread_module->giveAttr("start_new_thread", new BoxedFunction(boxRTFunction((void*)startNewThread, BOXED_INT, 2)));
     thread_module->giveAttr("allocate_lock", new BoxedFunction(boxRTFunction((void*)allocateLock, UNKNOWN, 0)));
     thread_module->giveAttr("get_ident", new BoxedFunction(boxRTFunction((void*)getIdent, BOXED_INT, 0)));
+    thread_module->giveAttr("stack_size", new BoxedFunction(boxRTFunction((void*)stackSize, BOXED_INT, 0)));
 
     thread_lock_cls = new BoxedHeapClass(object_cls, NULL, 0, sizeof(BoxedThreadLock), false);
     thread_lock_cls->giveAttr("__name__", boxStrConstant("lock"));
     thread_lock_cls->giveAttr("__module__", boxStrConstant("thread"));
-    thread_lock_cls->giveAttr("acquire", new BoxedFunction(boxRTFunction((void*)BoxedThreadLock::acquire, NONE, 1)));
+    thread_lock_cls->giveAttr(
+        "acquire",
+        new BoxedFunction(boxRTFunction((void*)BoxedThreadLock::acquire, NONE, 2, 1, false, false), { boxInt(1) }));
     thread_lock_cls->giveAttr("release", new BoxedFunction(boxRTFunction((void*)BoxedThreadLock::release, NONE, 1)));
     thread_lock_cls->giveAttr("acquire_lock", thread_lock_cls->getattr("acquire"));
     thread_lock_cls->giveAttr("release_lock", thread_lock_cls->getattr("release"));
+    thread_lock_cls->giveAttr("__enter__", thread_lock_cls->getattr("acquire"));
+    thread_lock_cls->giveAttr("__exit__", new BoxedFunction(boxRTFunction((void*)BoxedThreadLock::exit, NONE, 4)));
     thread_lock_cls->freeze();
 
+    thread_local_cls = new BoxedHeapClass(object_cls, NULL, 0, sizeof(BoxedThreadLocal), false);
+    thread_local_cls->giveAttr("__name__", boxStrConstant("_local"));
+    thread_local_cls->giveAttr("__module__", boxStrConstant("thread"));
+    thread_local_cls->freeze();
+    thread_module->giveAttr("_local", thread_local_cls);
+
     BoxedClass* ThreadError
         = new BoxedHeapClass(Exception, NULL, Exception->attrs_offset, Exception->tp_basicsize, false);
     ThreadError->giveAttr("__name__", boxStrConstant("error"));

src/runtime/capi.cpp

@@ -34,12 +34,10 @@ namespace pyston {
 BoxedClass* method_cls;
 
 #define MAKE_CHECK(NAME, cls_name)                                                                                     \
-    extern "C" bool Py##NAME##_Check(PyObject* op) noexcept { return isSubclass(op->cls, cls_name); }
-#define MAKE_CHECK2(NAME, cls_name)                                                                                    \
     extern "C" bool _Py##NAME##_Check(PyObject* op) noexcept { return isSubclass(op->cls, cls_name); }
 
-MAKE_CHECK2(Int, int_cls)
-MAKE_CHECK2(String, str_cls)
+MAKE_CHECK(Int, int_cls)
+MAKE_CHECK(String, str_cls)
 MAKE_CHECK(Long, long_cls)
 MAKE_CHECK(List, list_cls)
 MAKE_CHECK(Tuple, tuple_cls)

src/runtime/dict.cpp

@@ -237,7 +237,44 @@ extern "C" PyObject* PyDict_GetItem(PyObject* dict, PyObject* key) noexcept {
 }
 
 extern "C" int PyDict_Next(PyObject* op, Py_ssize_t* ppos, PyObject** pkey, PyObject** pvalue) noexcept {
-    Py_FatalError("unimplemented");
+    assert(op->cls == dict_cls);
+    BoxedDict* self = static_cast<BoxedDict*>(op);
+
+    // Callers of PyDict_New() provide a pointer to some storage for this function to use, in
+    // the form of a Py_ssize_t* -- ie they allocate a Py_ssize_t on their stack, and let us use
+    // it.
+    //
+    // We want to store an unordered_map::iterator in that.  In my glibc it would fit, but to keep
+    // things a little bit more portable, allocate separate storage for the iterator, and store the
+    // pointer to this storage in the Py_ssize_t slot.
+    //
+    // Results in lots of indirection unfortunately.  If it becomes an issue we can try to switch
+    // to storing the iterator directly in the stack slot.
+
+    typedef BoxedDict::DictMap::iterator iterator;
+
+    static_assert(sizeof(Py_ssize_t) == sizeof(iterator*), "");
+    iterator** it_ptr = reinterpret_cast<iterator**>(ppos);
+
+    // Clients are supposed to zero-initialize *ppos:
+    if (*it_ptr == NULL) {
+        *it_ptr = (iterator*)malloc(sizeof(iterator));
+        ** it_ptr = self->d.begin();
+    }
+
+    iterator* it = *it_ptr;
+
+    if (*it == self->d.end()) {
+        free(it);
+        return 0;
+    }
+
+    *pkey = (*it)->first;
+    *pvalue = (*it)->second;
+
+    ++(*it);
+
+    return 1;
 }
 
 extern "C" PyObject* PyDict_GetItemString(PyObject* dict, const char* key) noexcept {

src/runtime/list.cpp

@@ -39,10 +39,6 @@ extern "C" int PyList_Append(PyObject* op, PyObject* newitem) noexcept {
     return 0;
 }
 
-extern "C" int PyList_SetItem(PyObject* op, Py_ssize_t i, PyObject* newitem) noexcept {
-    Py_FatalError("unimplemented");
-}
-
 extern "C" Box* listRepr(BoxedList* self) {
     LOCK_REGION(self->lock.asRead());
 
@@ -180,6 +176,16 @@ extern "C" Box* listGetitem(BoxedList* self, Box* slice) {
     }
 }
 
+static void _listSetitem(BoxedList* self, int64_t n, Box* v) {
+    if (n < 0)
+        n = self->size + n;
+
+    if (n < 0 || n >= self->size) {
+        raiseExcHelper(IndexError, "list index out of range");
+    }
+
+    self->elts->elts[n] = v;
+}
 
 extern "C" Box* listSetitemInt(BoxedList* self, BoxedInt* slice, Box* v) {
     // I think r lock is ok here, since we don't change the list structure:
@@ -188,17 +194,22 @@ extern "C" Box* listSetitemInt(BoxedList* self, BoxedInt* slice, Box* v) {
     assert(self->cls == list_cls);
     assert(isSubclass(slice->cls, int_cls));
     int64_t n = slice->n;
-    if (n < 0)
-        n = self->size + n;
 
-    if (n < 0 || n >= self->size) {
-        raiseExcHelper(IndexError, "list index out of range");
-    }
+    _listSetitem(self, n, v);
 
-    self->elts->elts[n] = v;
     return None;
 }
 
+extern "C" int PyList_SetItem(PyObject* op, Py_ssize_t i, PyObject* newitem) noexcept {
+    assert(op->cls == list_cls);
+    try {
+        _listSetitem(static_cast<BoxedList*>(op), i, newitem);
+    } catch (ExcInfo e) {
+        abort();
+    }
+    return 0;
+}
+
 Box* listIAdd(BoxedList* self, Box* _rhs);
 
 // Analogue of _PyEval_SliceIndex
@@ -554,12 +565,21 @@ extern "C" Box* listNew(Box* cls, Box* container) {
 }
 
 extern "C" PyObject* PyList_New(Py_ssize_t size) noexcept {
-    // This function is supposed to return a list of `size` NULL elements.
-    // That will probably trip an assert somewhere if we try to create that (ex
-    // I think the GC will expect them to be real objects so they can be relocated).
-    RELEASE_ASSERT(size == 0, "");
     try {
-        return new BoxedList();
+        BoxedList* l = new BoxedList();
+        if (size) {
+            // This function is supposed to return a list of `size` NULL elements.
+            // That will probably trip an assert somewhere if we try to create that (ex
+            // I think the GC will expect them to be real objects so they can be relocated),
+            // so put None in instead
+            l->ensure(size);
+
+            for (Py_ssize_t i = 0; i < size; i++) {
+                l->elts->elts[i] = None;
+            }
+            l->size = size;
+        }
+        return l;
     } catch (ExcInfo e) {
         abort();
     }

test/tests/thread_test.py

@@ -19,3 +19,30 @@ while not done:
 
 print "done!"
 
+l = allocate_lock()
+print l.acquire()
+print l.acquire(0)
+print l.release()
+print l.acquire(0)
+
+
+lock = allocate_lock()
+state = 0
+def run2():
+    global state
+
+    print lock.acquire(0)
+    state = 1
+    print lock.acquire()
+    lock.release()
+    state = 2
+
+with lock:
+    start_new_thread(run2, ())
+
+    while state != 1:
+        time.sleep(0)
+while state != 2:
+    time.sleep(0)
+
+print "done!"

tools/determine_missing.py

+import os.path
+import sys
+import subprocess
+
+PYSTON_DIR = os.path.join(os.path.dirname(__file__), '..')
+STDMODULE_DIR = os.path.join(PYSTON_DIR, "from_cpython/Modules")
+
+def find_module_source(module_name):
+    for pattern in ("%smodule.c", "%s.c"):
+        src = pattern % module_name
+        if os.path.exists(os.path.join(STDMODULE_DIR, src)):
+            return src
+    print "couldn't find", module_name
+    return None
+
+def main():
+    already_supported = ('__builtin__', '_collections', '_functools', '_md5', '_random', '_sha', '_sha256', '_sha512', '_sre', '_struct', 'binascii', 'datetime', 'errno', 'fcntl', 'gc', 'itertools', 'math', 'operator', 'posix', 'pwd', 'resource', 'select', 'signal', 'sys', 'thread', 'time')
+    for module_name in sys.builtin_module_names:
+        if module_name.startswith("__"):
+            continue
+        if module_name in already_supported:
+            continue
+
+        src = find_module_source(module_name)
+        if not src:
+            continue
+
+        args = ["make", "EXTRA_STDMODULE_SRCS=%s" % src, "ERROR_LIMIT=0"]
+        print ' '.join(args), "> %s.log" % module_name
+        log_fn = "%s.log" % module_name
+        f = open(log_fn, 'w')
+        p = subprocess.Popen(args, cwd=PYSTON_DIR, stdout=f, stderr=subprocess.STDOUT)
+        code = p.wait()
+        f.close()
+        if code == 0:
+            print module_name, "worked!"
+        else:
+            print module_name, "didn't work",
+            s = open(log_fn).read()
+            if "undefined reference to" in s:
+                print "Compiled but didn't link, %d link errors" % (s.count('\n') - 3)
+            else:
+                print s.split('\n')[-3]
+
+if __name__ == "__main__":
+    main()