version up: ZODB3 3.10.7

component/egg-patch/ZODB3-3.10.5.patch

-# https://mail.zope.org/pipermail/zodb-dev/2014-February/015182.html
-diff --git a/src/ZODB/FileStorage/FileStorage.py b/src/ZODB/FileStorage/FileStorage.py
-index d45cbbf..d662bf4 100644
---- a/src/ZODB/FileStorage/FileStorage.py
-+++ b/src/ZODB/FileStorage/FileStorage.py
-@@ -683,6 +683,7 @@ def tpc_vote(self, transaction):
-                 # Hm, an error occurred writing out the data. Maybe the
-                 # disk is full. We don't want any turd at the end.
-                 self._file.truncate(self._pos)
-+                self._files.flush()
-                 raise
-             self._nextpos = self._pos + (tl + 8)
- 
-@@ -737,6 +738,7 @@ def _finish_finish(self, tid):
-     def _abort(self):
-         if self._nextpos:
-             self._file.truncate(self._pos)
-+            self._files.flush()
-             self._nextpos=0
-             self._blob_tpc_abort()
- 
-@@ -1996,6 +1998,15 @@ def __init__(self, file_name):
-         self._out = []
-         self._cond = threading.Condition()
- 
-+    def flush(self):
-+        """Empty read buffers.
-+
-+        This is required if they may contain data of rolled back transactions.
-+        """
-+        with self.write_lock():
-+            for f in self._files:
-+                f.flush()
-+
-     @contextlib.contextmanager
-     def write_lock(self):
-         with self._cond:
-
-# https://github.com/zopefoundation/ZODB/pull/15/files
-diff -ur a/src/ZODB/FileStorage/FileStorage.py b/src/ZODB/FileStorage/FileStorage.py
---- a/src/ZODB/FileStorage/FileStorage.py
-+++ b/src/ZODB/FileStorage/FileStorage.py
-@@ -430,7 +430,7 @@
-                 if h.tid == serial:
-                     break
-                 pos = h.prev
--                if not pos:
-+                if h.tid < serial or not pos:
-                     raise POSKeyError(oid)
-             if h.plen:
-                 return self._file.read(h.plen)

component/egg-patch/ZODB3-persistent-ghostify-slots.patch

-From d387a425941b37b99355077657edf7a2f117cf47 Mon Sep 17 00:00:00 2001
-From: Kirill Smelkov <kirr@nexedi.com>
-Date: Thu, 21 Jul 2016 22:34:55 +0300
-Subject: [PATCH] persistent: On deactivate release in-slots objects too
-
-( This is backport of https://github.com/zopefoundation/persistent/pull/44
-  to ZODB-3.10 )
-
-On ._p_deactivate() and ._p_invalidate(), when an object goes to ghost
-state, objects referenced by all its attributes, except related to
-persistence machinery, are released, this way freeing memory (if they
-were referenced only from going-to-ghost object).
-
-That's the idea - an object in ghost state is simply a stub, which loads
-its content on first access (via hooking into get/set attr) while
-occupying minimal memory in not-yet-loaded state.
-
-However the above is not completely true right now, as currently on
-ghostification only object's .__dict__ is released, while in-slots objects
-are retained attached to ghost object staying in RAM:
-
-    ---- 8< ----
-    from ZODB import DB
-    from persistent import Persistent
-    import gc
-
-    db = DB(None)
-    jar = db.open()
-
-    class C:
-        def __init__(self, v):
-            self.v = v
-        def __del__(self):
-            print 'released (%s)' % self.v
-
-    class P1(Persistent):
-        pass
-
-    class P2(Persistent):
-        __slots__ = ('aaa')
-
-    p1 = P1()
-    jar.add(p1)
-    p1.aaa = C(1)
-
-    p2 = P2()
-    jar.add(p2)
-    p2.aaa = C(2)
-
-    p1._p_invalidate()
-    # "released (1)" is printed
-
-    p2._p_invalidate()
-    gc.collect()
-    # "released (2)" is NOT printed     <--
-    ---- 8< ----
-
-So teach ghostify() & friends to release objects in slots to free-up
-memory when an object goes to ghost state.
-
-NOTE PyErr_Occurred() added after ghostify() calls because
-pickle_slotnames() can raise an error, but we do not want to change
-ghostify() prototype for backward compatibility reason - as it is used
-in cPersistenceCAPIstruct.
-
-( I hit this bug with wendelin.core which uses proxies to load
-  data from DB to virtual memory manager and then deactivate proxy right
-  after load has been completed:
-
-  https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L239
-  https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L295 )
----
- src/persistent/cPersistence.c          | 41 +++++++++++++++++++++++++++++++++-
- src/persistent/tests/testPersistent.py | 24 ++++++++++++++++++++
- 2 files changed, 64 insertions(+), 1 deletion(-)
-
-diff --git a/src/persistent/cPersistence.c b/src/persistent/cPersistence.c
-index b4a185c..28d1f9a 100644
---- a/src/persistent/cPersistence.c
-+++ b/src/persistent/cPersistence.c
-@@ -75,6 +75,7 @@ fatal_1350(cPersistentObject *self, const char *caller, const char *detail)
- #endif
- 
- static void ghostify(cPersistentObject*);
-+static PyObject * pickle_slotnames(PyTypeObject *cls);
- 
- /* Load the state of the object, unghostifying it.  Upon success, return 1.
-  * If an error occurred, re-ghostify the object and return -1.
-@@ -141,7 +142,7 @@ accessed(cPersistentObject *self)
- static void
- ghostify(cPersistentObject *self)
- {
--  PyObject **dictptr;
-+  PyObject **dictptr, *slotnames;
- 
-   /* are we already a ghost? */
-   if (self->state == cPersistent_GHOST_STATE)
-@@ -171,6 +172,8 @@ ghostify(cPersistentObject *self)
-     _estimated_size_in_bytes(self->estimated_size);
-   ring_del(&self->ring);
-   self->state = cPersistent_GHOST_STATE;
-+
-+  /* clear __dict__ */
-   dictptr = _PyObject_GetDictPtr((PyObject *)self);
-   if (dictptr && *dictptr)
-     {
-@@ -178,6 +181,38 @@ ghostify(cPersistentObject *self)
-       *dictptr = NULL;
-     }
- 
-+  /* clear all slots besides _p_* */
-+  slotnames = pickle_slotnames(Py_TYPE(self));
-+  if (slotnames && slotnames != Py_None)
-+  {
-+    int i;
-+
-+    for (i = 0; i < PyList_GET_SIZE(slotnames); i++)
-+    {
-+        PyObject *name;
-+        char *cname;
-+        int is_special;
-+
-+        name = PyList_GET_ITEM(slotnames, i);
-+        if (PyBytes_Check(name))
-+        {
-+            cname = PyBytes_AS_STRING(name);
-+            is_special = !strncmp(cname, "_p_", 3);
-+            if (is_special) /* skip persistent */
-+            {
-+                continue;
-+            }
-+        }
-+
-+        /* NOTE: this skips our delattr hook */
-+        if (PyObject_GenericSetAttr((PyObject *)self, name, NULL) < 0)
-+            /* delattr of non-set slot will raise AttributeError - we
-+             * simply ignore. */
-+            PyErr_Clear();
-+    }
-+  }
-+  Py_XDECREF(slotnames);
-+
-   /* We remove the reference to the just ghosted object that the ring
-    * holds.  Note that the dictionary of oids->objects has an uncounted
-    * reference, so if the ring's reference was the only one, this frees
-@@ -261,6 +296,8 @@ Per__p_deactivate(cPersistentObject *self)
-          called directly. Methods that override this need to
-          do the same! */
-       ghostify(self);
-+      if (PyErr_Occurred())
-+        return NULL;
-     }
- 
-   Py_INCREF(Py_None);
-@@ -289,6 +326,8 @@ Per__p_invalidate(cPersistentObject *self)
-       if (Per_set_changed(self, NULL) < 0)
-         return NULL;
-       ghostify(self);
-+      if (PyErr_Occurred())
-+        return NULL;
-     }
-   Py_INCREF(Py_None);
-   return Py_None;
-diff --git a/src/persistent/tests/testPersistent.py b/src/persistent/tests/testPersistent.py
-index 51e0382..fdb8b67 100644
---- a/src/persistent/tests/testPersistent.py
-+++ b/src/persistent/tests/testPersistent.py
-@@ -180,6 +180,30 @@ class PersistenceTest(unittest.TestCase):
-         self.assertEqual(obj._p_changed, None)
-         self.assertEqual(obj._p_state, GHOST)
- 
-+    def test__p_invalidate_from_changed_w_slots(self):
-+        from persistent import Persistent
-+        class Derived(Persistent):
-+            __slots__ = ('myattr1', 'myattr2')
-+            def __init__(self):
-+                self.myattr1 = 'value1'
-+                self.myattr2 = 'value2'
-+        obj = Derived()
-+        jar = self._makeJar()
-+        jar.add(obj)
-+        obj._p_activate()
-+        obj._p_changed = True
-+        jar._loaded = []
-+        jar._registered = []
-+        self.assertEqual(Derived.myattr1.__get__(obj), 'value1')
-+        self.assertEqual(Derived.myattr2.__get__(obj), 'value2')
-+        obj._p_invalidate()
-+        self.assertIs(obj._p_changed, None)
-+        self.assertEqual(list(jar._loaded), [])
-+        self.assertRaises(AttributeError, lambda: Derived.myattr1.__get__(obj))
-+        self.assertRaises(AttributeError, lambda: Derived.myattr2.__get__(obj))
-+        self.assertEqual(list(jar._loaded), [])
-+        self.assertEqual(list(jar._registered), [])
-+
-     def test_initial_serial(self):
-         NOSERIAL = "\000" * 8
-         obj = self._makeOne()
--- 
-2.9.2.701.gf965a18.dirty
-

software/neoppod/software-common.cfg

@@ -42,11 +42,6 @@ recipe = zc.recipe.egg
 eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
   ${python-mysqlclient:egg}
   ZODB3
-patch-binary = ${patch:location}/bin/patch
-ZODB3-patches =
-  ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch#c5fe331b1e3a930446f93ab4f6e97c6e
-  ${:_profile_base_location_}/../../component/egg-patch/ZODB3-persistent-ghostify-slots.patch#3a66e9c018d7269bd522d5b0a746f510
-ZODB3-patch-options = -p1
 
 [slapos-deps-eggs]
 recipe = zc.recipe.egg
@@ -105,8 +100,7 @@ md5sum = 81ab5e842ecf8385b12d735585497cc8
 
 [versions]
 slapos.recipe.template = 2.9
-# patched egg
-ZODB3 = 3.10.5+SlapOSPatched002
+ZODB3 = 3.10.7
 # Required by slapos.toolbox = 0.58
 slapos.toolbox = 0.58
 PyRSS2Gen = 1.1

software/neoppod/software.cfg

@@ -34,11 +34,13 @@ eggs = erp5.util
 interpreter = ${:_buildout_section_name_}
 
 [neoppod]
+patch-binary = ${patch:location}/bin/patch
+ZODB3-patch-options = -p1
 ZODB3-patches +=
   ${neoppod-repository:location}/ZODB3.patch
 
 [versions]
-ZODB3 = 3.10.5+SlapOSPatched003
+ZODB3 = 3.10.7+SlapOSPatched001
 erp5.util = 0.4.45
 # To match ERP5
 transaction = 1.1.1