Merge pull request #529 from kmod/metaserver_merge

misc library work

from_cpython/Include/Python.h

@@ -74,6 +74,7 @@
 #include "setobject.h"
 #include "methodobject.h"
 #include "moduleobject.h"
+#include "funcobject.h"
 #include "classobject.h"
 #include "cobject.h"
 #include "fileobject.h"

from_cpython/Include/funcobject.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Function object interface */
+
+#ifndef Py_FUNCOBJECT_H
+#define Py_FUNCOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Function objects and code objects should not be confused with each other:
+ *
+ * Function objects are created by the execution of the 'def' statement.
+ * They reference a code object in their func_code attribute, which is a
+ * purely syntactic object, i.e. nothing more than a compiled version of some
+ * source code lines.  There is one code object per source code "fragment",
+ * but each code object can be referenced by zero or many function objects
+ * depending only on how many times the 'def' statement in the source was
+ * executed so far.
+ */
+
+// Pyston change: not our object format
+#if 0
+typedef struct {
+    PyObject_HEAD
+    PyObject *func_code;	/* A code object */
+    PyObject *func_globals;	/* A dictionary (other mappings won't do) */
+    PyObject *func_defaults;	/* NULL or a tuple */
+    PyObject *func_closure;	/* NULL or a tuple of cell objects */
+    PyObject *func_doc;		/* The __doc__ attribute, can be anything */
+    PyObject *func_name;	/* The __name__ attribute, a string object */
+    PyObject *func_dict;	/* The __dict__ attribute, a dict or NULL */
+    PyObject *func_weakreflist;	/* List of weak references */
+    PyObject *func_module;	/* The __module__ attribute, can be anything */
+
+    /* Invariant:
+     *     func_closure contains the bindings for func_code->co_freevars, so
+     *     PyTuple_Size(func_closure) == PyCode_GetNumFree(func_code)
+     *     (func_closure may be NULL if PyCode_GetNumFree(func_code) == 0).
+     */
+} PyFunctionObject;
+#endif
+
+// Pyston change: not a static object any more
+//PyAPI_DATA(PyTypeObject) PyFunction_Type;
+PyAPI_DATA(PyTypeObject*) function_cls;
+#define PyFunction_Type (*function_cls)
+
+#define PyFunction_Check(op) (Py_TYPE(op) == &PyFunction_Type)
+
+PyAPI_FUNC(PyObject *) PyFunction_New(PyObject *, PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFunction_GetCode(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFunction_GetGlobals(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFunction_GetModule(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFunction_GetDefaults(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PyFunction_SetDefaults(PyObject *, PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFunction_GetClosure(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *) PYSTON_NOEXCEPT;
+
+// Pyston change: no longer macros
+#if 0
+/* Macros for direct access to these values. Type checks are *not*
+   done, so use with care. */
+#define PyFunction_GET_CODE(func) \
+        (((PyFunctionObject *)func) -> func_code)
+#define PyFunction_GET_GLOBALS(func) \
+	(((PyFunctionObject *)func) -> func_globals)
+#define PyFunction_GET_MODULE(func) \
+	(((PyFunctionObject *)func) -> func_module)
+#define PyFunction_GET_DEFAULTS(func) \
+	(((PyFunctionObject *)func) -> func_defaults)
+#define PyFunction_GET_CLOSURE(func) \
+	(((PyFunctionObject *)func) -> func_closure)
+#endif
+#define PyFunction_GET_CODE(func) (PyFunction_GetCode((PyObject *)(func)))
+#define PyFunction_GET_GLOBALS(func) (PyFunction_GetGlobals((PyObject *)(func)))
+#define PyFunction_GET_MODULE(func) (PyFunction_GetModule((PyObject *)(func)))
+#define PyFunction_GET_DEFAULTS(func) (PyFunction_GetDefaults((PyObject *)(func)))
+#define PyFunction_GET_CLOSURE(func) (PyFunction_GetClosure((PyObject *)(func)))
+
+// Pyston change: not a static object any more
+#if 0
+/* The classmethod and staticmethod types lives here, too */
+PyAPI_DATA(PyTypeObject) PyClassMethod_Type;
+PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
+#endif
+PyAPI_DATA(PyTypeObject*) classmethod_cls;
+#define PyClassMethod_Type (*classmethod_cls)
+PyAPI_DATA(PyTypeObject*) staticmethod_cls;
+#define PyStaticMethod_Type (*staticmethod_cls)
+
+PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *) PYSTON_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_FUNCOBJECT_H */

from_cpython/Include/object.h

@@ -1044,6 +1044,9 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void) PYSTON_NOEXCEPT;
 
 #define PyTrash_UNWIND_LEVEL 50
 
+// Pyston change: I don't think we need this since destructors
+// are run differently
+#if 0
 /* Note the workaround for when the thread state is NULL (issue #17703) */
 #define Py_TRASHCAN_SAFE_BEGIN(op) \
     do { \
@@ -1064,6 +1067,9 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void) PYSTON_NOEXCEPT;
         else \
             _PyTrash_thread_deposit_object((PyObject*)op); \
     } while (0);
+#endif
+#define Py_TRASHCAN_SAFE_BEGIN(op) do {
+#define Py_TRASHCAN_SAFE_END(op) } while (0);
 
 #ifdef __cplusplus
 }

from_cpython/Include/pyconfig.h

@@ -60,6 +60,8 @@
 #define HAVE_TIMES 1
 #define HAVE_STRUCT_TM_TM_ZONE 1
 #define HAVE_MKTIME 1
+#define HAVE_PROTOTYPES 1
+#define STDC_HEADERS 1
 
 #define TIME_WITH_SYS_TIME
 #define HAVE_GETTIMEOFDAY 1

from_cpython/Lib/modulefinder.py

@@ -17,12 +17,13 @@ else:
     # remain compatible with Python  < 2.3
     READ_MODE = "r"
 
-LOAD_CONST = chr(dis.opname.index('LOAD_CONST'))
-IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME'))
-STORE_NAME = chr(dis.opname.index('STORE_NAME'))
-STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL'))
-STORE_OPS = [STORE_NAME, STORE_GLOBAL]
-HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT)
+# Pyston change: comment these out
+# LOAD_CONST = chr(dis.opname.index('LOAD_CONST'))
+# IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME'))
+# STORE_NAME = chr(dis.opname.index('STORE_NAME'))
+# STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL'))
+# STORE_OPS = [STORE_NAME, STORE_GLOBAL]
+# HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT)
 
 # Modulefinder does a good job at simulating Python's, but it can not
 # handle __path__ modifications packages make at runtime.  Therefore there

libpypa @ 7f14a666

-Subproject commit 91428d4e7a72c53a058682f3a8d1993a90efdccd
+Subproject commit 7f14a66674f6d65c717199512e7ca705110c8da1

src/asm_writing/assembler.cpp

@@ -714,7 +714,7 @@ void Assembler::cmp(Register reg, Immediate imm) {
     int reg_idx = reg.regnum;
 
     int rex = REX_W;
-    if (reg_idx > 8) {
+    if (reg_idx >= 8) {
         rex |= REX_B;
         reg_idx -= 8;
     }

src/runtime/builtin_modules/thread.cpp

@@ -177,9 +177,9 @@ public:
         return None;
     }
 
-    static int setattr(Box* obj, char* name, Box* val) noexcept {
+    static int setattro(Box* obj, Box* name, Box* val) noexcept {
         try {
-            setattrPyston(obj, boxString(name), val);
+            setattrPyston(obj, name, val);
         } catch (ExcInfo e) {
             setCAPIException(e);
             return -1;
@@ -187,18 +187,21 @@ public:
         return 0;
     }
 
-    static Box* getattr(Box* obj, char* name) noexcept {
+    static Box* getattro(Box* obj, Box* name) noexcept {
+        assert(name->cls == str_cls);
+        llvm::StringRef s = static_cast<BoxedString*>(name)->s;
+
         Box* tls_obj = getThreadLocalObject(obj);
-        if (!strcmp(name, "__dict__"))
+        if (s == "__dict__")
             return tls_obj;
 
         try {
-            return getitem(tls_obj, boxString(name));
+            return getitem(tls_obj, name);
         } catch (ExcInfo e) {
         }
 
         try {
-            Box* r = getattrInternalGeneric(obj, name, NULL, false, false, NULL, NULL);
+            Box* r = getattrInternalGeneric(obj, s, NULL, false, false, NULL, NULL);
             if (r)
                 return r;
         } catch (ExcInfo e) {
@@ -206,7 +209,7 @@ public:
             return NULL;
         }
 
-        PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", Py_TYPE(obj)->tp_name, name);
+        PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", Py_TYPE(obj)->tp_name, s.data());
         return NULL;
     }
 
@@ -249,18 +252,20 @@ void setupThread() {
     thread_lock_cls->giveAttr("__exit__", new BoxedFunction(boxRTFunction((void*)BoxedThreadLock::exit, NONE, 4)));
     thread_lock_cls->freeze();
 
-    thread_local_cls
-        = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedThreadLocal), false, "_local");
+    thread_local_cls = new (0) BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedThreadLocal), false,
+                                              static_cast<BoxedString*>(boxString("_local")));
     thread_local_cls->giveAttr("__module__", boxStrConstant("thread"));
     thread_local_cls->giveAttr("__hash__",
                                new BoxedFunction(boxRTFunction((void*)BoxedThreadLocal::hash, BOXED_INT, 1)));
     thread_local_cls->giveAttr("__setattr__",
                                new BoxedFunction(boxRTFunction((void*)BoxedThreadLocal::setattrPyston, UNKNOWN, 3)));
-    thread_local_cls->freeze();
     thread_module->giveAttr("_local", thread_local_cls);
 
-    thread_local_cls->tp_setattr = BoxedThreadLocal::setattr;
-    thread_local_cls->tp_getattr = BoxedThreadLocal::getattr;
+    thread_local_cls->tp_setattro = BoxedThreadLocal::setattro;
+    thread_local_cls->tp_getattro = BoxedThreadLocal::getattro;
+    add_operators(thread_local_cls);
+    thread_local_cls->finishInitialization();
+    thread_local_cls->freeze();
 
     BoxedClass* ThreadError
         = BoxedHeapClass::create(type_cls, Exception, NULL, Exception->attrs_offset, Exception->tp_weaklistoffset,

src/runtime/capi.cpp

@@ -1241,6 +1241,26 @@ extern "C" void PyEval_InitThreads(void) noexcept {
     // nothing to do here
 }
 
+extern "C" void PyEval_AcquireThread(PyThreadState* tstate) noexcept {
+    Py_FatalError("Unimplemented");
+}
+
+extern "C" void PyEval_ReleaseThread(PyThreadState* tstate) noexcept {
+    Py_FatalError("Unimplemented");
+}
+
+extern "C" PyThreadState* PyThreadState_Get(void) noexcept {
+    Py_FatalError("Unimplemented");
+}
+
+extern "C" PyThreadState* PyEval_SaveThread(void) noexcept {
+    Py_FatalError("Unimplemented");
+}
+
+extern "C" void PyEval_RestoreThread(PyThreadState* tstate) noexcept {
+    Py_FatalError("Unimplemented");
+}
+
 extern "C" char* PyModule_GetName(PyObject* m) noexcept {
     PyObject* d;
     PyObject* nameobj;

src/runtime/dict.cpp

@@ -437,6 +437,18 @@ Box* dictContains(BoxedDict* self, Box* k) {
     return boxBool(self->d.count(k) != 0);
 }
 
+/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */
+extern "C" int PyDict_Contains(PyObject* op, PyObject* key) noexcept {
+    BoxedDict* mp = (BoxedDict*)op;
+    try {
+        return mp->getOrNull(key) ? 1 : 0;
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return -1;
+    }
+}
+
+
 Box* dictNonzero(BoxedDict* self) {
     return boxBool(self->d.size());
 }

src/runtime/dict.h

@@ -58,6 +58,7 @@ Box* dictViewKeysIter(Box* self);
 Box* dictViewValuesIter(Box* self);
 Box* dictViewItemsIter(Box* self);
 void dictMerge(BoxedDict* self, Box* other);
+Box* dictUpdate(BoxedDict* self, BoxedTuple* args, BoxedDict* kwargs);
 }
 
 #endif

src/runtime/import.cpp

@@ -231,10 +231,9 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
             return SearchResult("", SearchResult::SEARCH_ERROR);
 
         if (importer != None) {
-            auto path_pass = path_list ? path_list : None;
             Box* loader = callattr(importer, &find_module_str,
-                                   CallattrFlags({.cls_only = false, .null_on_nonexistent = false }), ArgPassSpec(2),
-                                   boxString(full_name), path_pass, NULL, NULL, NULL);
+                                   CallattrFlags({.cls_only = false, .null_on_nonexistent = false }), ArgPassSpec(1),
+                                   boxString(full_name), NULL, NULL, NULL, NULL);
             if (loader != None)
                 return SearchResult(loader);
         }
@@ -489,7 +488,7 @@ Box* importModuleLevel(const std::string& name, Box* globals, Box* from_imports,
     std::string _name = name;
 
     Box* head;
-    bool again = loadNext(parent, level < 0 ? NULL : parent, _name, buf, &head);
+    bool again = loadNext(parent, level < 0 ? None : parent, _name, buf, &head);
     if (head == NULL)
         return NULL;
 

src/runtime/types.cpp

@@ -31,6 +31,7 @@
 #include "gc/collector.h"
 #include "runtime/capi.h"
 #include "runtime/classobj.h"
+#include "runtime/dict.h"
 #include "runtime/file.h"
 #include "runtime/ics.h"
 #include "runtime/iterobject.h"
@@ -1541,14 +1542,21 @@ public:
                 for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
                     self->b->setattr(p.first(), attrs->attr_list->attrs[p.second], NULL);
                 }
-            } else if (_container->cls == dict_cls) {
+            } else {
+                // The update rules are too complicated to be worth duplicating here;
+                // just create a new dict object and defer to dictUpdate.
+                // Hopefully this does not happen very often.
+                if (!isSubclass(_container->cls, dict_cls)) {
+                    BoxedDict* new_container = new BoxedDict();
+                    dictUpdate(new_container, BoxedTuple::create({ _container }), new BoxedDict());
+                    _container = new_container;
+                }
+                assert(isSubclass(_container->cls, dict_cls));
                 BoxedDict* container = static_cast<BoxedDict*>(_container);
 
                 for (const auto& p : container->d) {
                     AttrWrapper::setitem(self, p.first, p.second);
                 }
-            } else {
-                RELEASE_ASSERT(0, "not implemented: %s", _container->cls->tp_name);
             }
         };
 

src/runtime/types.h

@@ -223,6 +223,7 @@ protected:
 
     friend void setupRuntime();
     friend void setupSysEnd();
+    friend void setupThread();
 };
 
 class BoxedHeapClass : public BoxedClass {
@@ -256,6 +257,7 @@ private:
 
     friend void setupRuntime();
     friend void setupSys();
+    friend void setupThread();
 
     DEFAULT_CLASS_VAR(type_cls, sizeof(SlotOffset));
 };

test/tests/dir.py

@@ -110,3 +110,7 @@ class TestClass3: # old-style
 
 print sorted([d for d in dir(TestClass3) if not d.startswith('_')])
 print sorted([d for d in dir(TestClass3()) if not d.startswith('_')])
+
+c = C1()
+c.__dict__.update([('a', 1), ('b', 2)])
+print c.a, c.b

test/tests/modulefinder_test.py

+# Make sure we can at least support people who want to register themselves with modulefinder,
+# even if we don't actually support using modulefinder to find modules.
+
+import modulefinder
+modulefinder.AddPackagePath("foo", "bar")

test/tests/no_newline.py

-# fail-if: '-x' not in EXTRA_JIT_ARGS
 class C(object):
     """
     """
\ No newline at end of file

test/tests/no_newline2.py

+a = ["hello world"]
\ No newline at end of file

test/tests/sys_meta_path.py

@@ -20,13 +20,40 @@ class Finder(object):
         print "find", fullname, path
         return Loader()
 
+class Finder2(object):
+    def __new__(cls, path):
+        print "new2", path
+        return object.__new__(cls)
+
+    def __init__(self, path):
+        self.path = path
+
+    def find_module(self, fullname):
+        print "find2", self.path, fullname
+        return None
+
+# Fill the importer cache, so that we don't have to worry about the exact
+# sys.path:
 try:
     import a.b.test
 except ImportError, e:
-    print "caught import error"
     # The error CPython gives here is "No module named a.b.test".  Both we and PyPy think this
     # is wrong and that the error should be "No module named a".
     # So unfortunately we can't print out the error message.
+    print "caught import error 1"
+
+sys.path_hooks.append(Finder2)
+try:
+    import a.b.test
+except ImportError, e:
+    print "caught import error 2"
+
+sys.path.append("/my_magic_directory")
+
+try:
+    import a.b.test
+except ImportError, e:
+    print "caught import error 3"
 
 sys.meta_path.append(Finder())
 import a

test/tests/tabs_and_spaces.py

+def f():
+    if True:
+    	1
+        2

test/tests/threading_local.py

@@ -4,9 +4,21 @@ import threading
 a = threading.local()
 a.x = "hello world"
 
+class CustomThreadingLocal(threading.local):
+    n = 0
+    def __init__(self):
+        print "__init__", self.n
+        self.a = self.n
+        self.n += 1
+        print self.a, self.n
+print CustomThreadingLocal().a
+print CustomThreadingLocal().a
+
 def f():
     a.x = "goodbye world"
     print a.x
+    print CustomThreadingLocal().a
+    print CustomThreadingLocal().a
 
 def test():
     thread = threading.Thread(target=f)