Merge pull request #907 from Daetalus/pyinotify

Add "pyinotify" test, with a minor modification to Pyston

.gitmodules

@@ -37,3 +37,6 @@
 [submodule "test/lib/decorator"]
 	path = test/lib/decorator
 	url = https://github.com/micheles/decorator
+[submodule "test/lib/pyinotify"]
+	path = test/lib/pyinotify
+	url = https://github.com/seb-m/pyinotify.git

from_cpython/Include/Python.h

@@ -115,6 +115,9 @@ PyAPI_FUNC(PyObject*) PyObject_GetHcAttrString(PyObject*, const char*) PYSTON_NO
 PyAPI_FUNC(int) PyObject_SetHcAttrString(PyObject*, const char*, PyObject*) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyObject_DelHcAttrString(PyObject*, const char*) PYSTON_NOEXCEPT;
 
+// Workaround: call this instead of setting tp_dict.
+PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
+
 #include "codecs.h"
 #include "pyerrors.h"
 

from_cpython/Modules/_ctypes/_ctypes.c

@@ -425,7 +425,9 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt
         return NULL;
     }
     Py_DECREF(result->tp_dict);
-    result->tp_dict = (PyObject *)dict;
+    // Pyston change:
+    //result->tp_dict = (PyObject *)dict;
+    PyType_SetDict(result, dict);
     dict->format = _ctypes_alloc_format_string(NULL, "B");
     if (dict->format == NULL) {
         Py_DECREF(result);
@@ -995,7 +997,9 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
     Py_DECREF(result->tp_dict);
-    result->tp_dict = (PyObject *)stgdict;
+    // Pyston change:
+    //result->tp_dict = (PyObject *)stgdict;
+    PyType_SetDict(result, stgdict);
 
     return (PyObject *)result;
 }
@@ -1461,7 +1465,9 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
     Py_DECREF(result->tp_dict);
-    result->tp_dict = (PyObject *)stgdict;
+    // Pyston change:
+    //result->tp_dict = (PyObject *)stgdict;
+    PyType_SetDict(result, stgdict);
 
     /* Special case for character arrays.
        A permanent annoyance: char arrays are also strings!
@@ -1885,7 +1891,9 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject
         return NULL;
     }
     Py_DECREF(result->tp_dict);
-    result->tp_dict = (PyObject *)stgdict;
+    // Pyston change:
+    //result->tp_dict = (PyObject *)stgdict;
+    PyType_SetDict(result, stgdict);
 
     return (PyObject *)result;
 }
@@ -2014,7 +2022,9 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
     Py_DECREF(result->tp_dict);
-    result->tp_dict = (PyObject *)stgdict;
+    // Pyston change:
+    //result->tp_dict = (PyObject *)stgdict;
+    PyType_SetDict(result, stgdict);
 
     /* Install from_param class methods in ctypes base classes.
        Overrides the PyCSimpleType_from_param generic method.
@@ -2044,21 +2054,14 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
             break;
         }
 
-        // TODO: Pyston change:
-        // For now, don't run this code path because we don't support some of
-        // the descriptor CAPI functions.
-        // We do this to be able to run `import ctypes`, but this will need to be
-        // enabled again once we want CType to actually work.
-        // if (ml) {
-        if (false) {
+
+        if (ml) {
 #if (PYTHON_API_VERSION >= 1012)
             PyObject *meth;
             int x;
-            /*
             meth = PyDescr_NewClassMethod(result, ml);
             if (!meth)
                 return NULL;
-            */
 #else
 #error
             PyObject *meth, *func;
@@ -2402,7 +2405,9 @@ PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
     Py_DECREF(result->tp_dict);
-    result->tp_dict = (PyObject *)stgdict;
+    // Pyston change:
+    //result->tp_dict = (PyObject *)stgdict;
+    PyType_SetDict(result, stgdict);
 
     if (-1 == make_funcptrtype_dict(stgdict)) {
         Py_DECREF(result);

src/runtime/descr.cpp

@@ -684,9 +684,14 @@ extern "C" PyObject* PyDescr_NewGetSet(PyTypeObject* x, struct PyGetSetDef* y) n
     return new (capi_getset_cls) BoxedGetsetDescriptor(y->get, (void (*)(Box*, Box*, void*))y->set, y->closure);
 }
 
-extern "C" PyObject* PyDescr_NewClassMethod(PyTypeObject* x, PyMethodDef* y) noexcept {
-    Py_FatalError("unimplemented");
-    return NULL;
+extern "C" PyObject* PyDescr_NewClassMethod(PyTypeObject* type, PyMethodDef* method) noexcept {
+    // Pyston change: we don't have a separate capi classmethod descriptor type, we just use the normal
+    // one but with the METH_CLASS flag set.
+    if (!(method->ml_flags & METH_CLASS)) {
+        method = new PyMethodDef(*method);
+        method->ml_flags |= METH_CLASS;
+    }
+    return new pyston::BoxedMethodDescriptor(method, type);
 }
 
 extern "C" PyObject* PyDescr_NewMethod(PyTypeObject* type, PyMethodDef* method) noexcept {

src/runtime/objmodel.cpp

@@ -1010,28 +1010,36 @@ Box* typeLookup(BoxedClass* cls, BoxedString* attr, GetattrRewriteArgs* rewrite_
         obj_saved->addAttrGuard(offsetof(BoxedClass, tp_mro), (intptr_t)mro);
 
         for (auto base : *mro) {
-            rewrite_args->out_success = false;
-            if (base == cls) {
-                // Small optimization: don't have to load the class again since it was given to us in
-                // a register.
-                assert(rewrite_args->obj == obj_saved);
-            } else {
-                rewrite_args->obj = rewrite_args->rewriter->loadConst((intptr_t)base, Location::any());
-                // We are passing a constant object, and objects are not allowed to change shape
-                // (at least the kind of "shape" that Box::getattr is referring to)
-                rewrite_args->obj_shape_guarded = true;
+            if (rewrite_args) {
+                rewrite_args->out_success = false;
+                if (base == cls) {
+                    // Small optimization: don't have to load the class again since it was given to us in
+                    // a register.
+                    assert(rewrite_args->obj == obj_saved);
+                } else {
+                    rewrite_args->obj = rewrite_args->rewriter->loadConst((intptr_t)base, Location::any());
+                    // We are passing a constant object, and objects are not allowed to change shape
+                    // (at least the kind of "shape" that Box::getattr is referring to)
+                    rewrite_args->obj_shape_guarded = true;
+                }
             }
             val = base->getattr(attr, rewrite_args);
+
+            if (rewrite_args && !rewrite_args->out_success)
+                rewrite_args = NULL;
+
             if (val)
                 return val;
         }
 
-        assert(!rewrite_args->out_rtn);
-        rewrite_args->out_return_convention = GetattrRewriteArgs::NO_RETURN;
+        if (rewrite_args) {
+            assert(rewrite_args->out_success);
+            assert(!rewrite_args->out_rtn);
+            rewrite_args->out_return_convention = GetattrRewriteArgs::NO_RETURN;
+        }
         return NULL;
     } else {
         assert(attr->interned_state != SSTATE_NOT_INTERNED);
-
         assert(cls->tp_mro);
         assert(cls->tp_mro->cls == tuple_cls);
         for (auto b : *static_cast<BoxedTuple*>(cls->tp_mro)) {

src/runtime/types.cpp

@@ -1425,7 +1425,16 @@ static void typeSubSetDict(Box* obj, Box* val, void* context) {
     }
 
     if (obj->cls->instancesHaveHCAttrs()) {
-        obj->setDictBacked(val);
+        RELEASE_ASSERT(PyDict_Check(val) || val->cls == attrwrapper_cls, "%s", val->cls->tp_name);
+
+        auto new_attr_list
+            = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
+        new_attr_list->attrs[0] = val;
+
+        HCAttrs* hcattrs = obj->getHCAttrsPtr();
+
+        hcattrs->hcls = HiddenClass::dict_backed;
+        hcattrs->attr_list = new_attr_list;
         return;
     }
 
@@ -1433,6 +1442,11 @@ static void typeSubSetDict(Box* obj, Box* val, void* context) {
     abort();
 }
 
+extern "C" void PyType_SetDict(PyTypeObject* type, PyObject* dict) {
+    typeSubSetDict(type, dict, NULL);
+    type->tp_dict = dict;
+}
+
 Box* dict_descr = NULL;
 
 void BoxedInstanceMethod::gcHandler(GCVisitor* v, Box* b) {

test/integration/pyinotify_test.py

+import os
+import sys
+import subprocess
+import time
+
+ENV_NAME = "pyinotify_test_env_" + os.path.basename(sys.executable)
+
+if not os.path.exists(ENV_NAME) or os.stat(sys.executable).st_mtime > os.stat(ENV_NAME + "/bin/python").st_mtime:
+    print "Creating virtualenv to install testing dependencies..."
+    VIRTUALENV_SCRIPT = os.path.dirname(__file__) + "/../lib/virtualenv/virtualenv.py"
+
+    try:
+        args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, ENV_NAME]
+        print "Running", args
+        subprocess.check_call(args)
+    except:
+        print "Error occurred; trying to remove partially-created directory"
+        ei = sys.exc_info()
+        try:
+            subprocess.check_call(["rm", "-rf", ENV_NAME])
+        except Exception as e:
+            print e
+        raise ei[0], ei[1], ei[2]
+
+
+PYINOTIFY_DIR = os.path.dirname(__file__) + "/../lib/pyinotify"
+python_exe = os.path.abspath(ENV_NAME + "/bin/python")
+
+print "\nRunning pyinotify\n"
+
+script = """
+import pyinotify
+import sys
+
+wm = pyinotify.WatchManager()  # Watch Manager
+mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE  # watched events
+
+output = []
+
+class EventHandler(pyinotify.ProcessEvent):
+    def process_IN_CREATE(self, event):
+        output.append("Changing: " + event.pathname)
+
+    def process_IN_DELETE(self, event):
+        output.append("Changing: " + event.pathname)
+        if len(output) >= 2:
+            for line in output:
+                sys.stdout.write(line)
+                sys.stdout.write("\\n")
+            sys.stdout.close()
+            sys.exit()
+
+handler = EventHandler()
+notifier = pyinotify.Notifier(wm, handler)
+wdd = wm.add_watch('/tmp', mask, rec=True)
+
+notifier.loop()
+"""
+
+p = subprocess.Popen(["python", "-c", script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+# Give pyinotify some extra time to start up:
+time.sleep(1)
+
+# Create a file create and a file delete event
+subprocess.call(["touch", "/tmp/generate_file_event_for_pyinotify"])
+subprocess.call(["rm", "/tmp/generate_file_event_for_pyinotify"])
+
+# Veiry pyinotify watched the file create and delete
+while True:
+    line = p.stdout.readline()
+    if line != '':
+        assert "Changing" in line
+        print line.rstrip()
+    else:
+        break
+
+print
+print "PASSED"

test/tests/ctypes_test.py

+# skip-if: True
+from ctypes import *
+s = "tmp"
+ap = create_string_buffer(s)
+
+print type(ap)
+print type(c_void_p.from_param(ap))
+print type(cast(ap, c_char_p))