ZBigArray: in-ZODB stored BigArray

This is like to BigArray, like ZBigFile is to BigFile (4174b84a
"bigfile: BigFile backend to store data in ZODB")

bigarray/array_zodb.py

+# -*- coding: utf-8 -*-
+# Wendelin.bigarray | ZODB-Persistent BigArray
+# Copyright (C) 2014-2015  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+
+"""BigArray that can be persisted ZODB-way
+
+TODO text
+"""
+
+from wendelin.bigarray import BigArray
+from wendelin.bigfile.file_zodb import ZBigFile, LivePersistent
+
+
+# TODO document that first access data must be either before commit or Connection.add
+
+class ZBigArray(BigArray,
+                # Live: don't allow us to go to ghost
+                # (not to loose ._v_fileh which works as DataManager)
+                LivePersistent):
+    __slots__ = (
+        'zfile',            # ZBigFile serving this array
+    )
+
+
+    # XXX default blksize hardcoded
+    def __init__(self, shape, dtype, order='C', blksize=2*1024*1024):
+        LivePersistent.__init__(self)
+        # NOTE BigArray is cooperative to us - it names all helping (= not needing
+        # to be saved) data members starting with _v_. Were it otherwise, we
+        # could not inherit from BigArray and would need to keep its instance
+        # in e.g. ._v_array helper and redirect all BigArray method from us to it.
+        BigArray._init0(self, shape, dtype, order)
+        self.zfile = ZBigFile(blksize)
+        self._v_fileh = None
+
+
+    def __setstate__(self, state):
+        super(ZBigArray, self).__setstate__(state)
+        self._v_fileh = None
+
+
+    # open fileh lazily, so that when we open it, zfile was already associated
+    # with Connection (i.e. zfile._p_jar is not None). This is needed for
+    # ZBigFile working.
+    @property
+    def _fileh(self):
+        if self._v_fileh is None:
+            self._v_fileh = self.zfile.fileh_open()
+        return self._v_fileh

bigarray/tests/test_arrayzodb.py

+# Wendeling.core.bigarray | Tests for ZBigArray
+# Copyright (C) 2014-2015  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Open Source Initiative approved licenses and Convey
+# the resulting work. Corresponding source of such a combination shall include
+# the source code for all other software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+from wendelin.bigarray.array_zodb import ZBigArray
+from wendelin.bigfile.tests.common_zodb import dbopen, dbclose
+from wendelin.bigfile.tests.test_filezodb import kkey, cacheInfo
+from persistent import UPTODATE
+import transaction
+from numpy import dtype, uint8, all
+
+def test_zbigarray(tmpdir):
+    root = dbopen('%s/1.fs' % tmpdir)
+    root['zarray'] = ZBigArray((16*1024*1024,), uint8)
+    transaction.commit()
+
+    dbclose(root)
+    del root
+
+
+    root = dbopen('%s/1.fs' % tmpdir)
+    A = root['zarray']
+
+    assert isinstance(A, ZBigArray)
+    assert A.shape  == (16*1024*1024,)
+    assert A.dtype  == dtype(uint8)
+
+    assert all(A[:] == 0)
+
+    a = A[:]
+    a[1] = 1
+    a[3] = 3
+    a[5] = 5
+    a[-1] = 99
+
+    b = A[:]
+    assert (b[0],b[1]) == (0,1)
+    assert (b[2],b[3]) == (0,3)
+    assert (b[4],b[5]) == (0,5)
+    assert all(b[6:-1] == 0)
+    assert b[-1] == 99
+
+    # abort - should forget all changes
+    transaction.abort()
+    assert all(a[:] == 0)
+    assert all(b[:] == 0)
+    assert all(A[:] == 0)
+
+    # now modify again and commit
+    a[33] = 33
+    a[-2] = 98
+    assert all(b[:33] == 0)
+    assert b[33] == 33
+    assert all(b[33+1:-2] == 0)
+    assert b[-2] == 98
+    assert b[-1] == 0
+
+    transaction.commit()
+
+    # reload DB & array
+    dbclose(root)
+    del root, a,b, A
+
+
+    root = dbopen('%s/1.fs' % tmpdir)
+    A = root['zarray']
+
+    assert isinstance(A, ZBigArray)
+    assert A.shape  == (16*1024*1024,)
+    assert A.dtype  == dtype(uint8)
+
+    a = A[:]
+    assert all(a[:33] == 0)
+    assert a[33] == 33
+    assert all(a[33+1:-2] == 0)
+    assert a[-2] == 98
+    assert a[-1] == 0
+
+
+    # like ZBigFile ZBigArray should survive Persistent cache clearing and not
+    # go to ghost state (else logic to propagate changes from pages to objects
+    # would subtly brake after Persistent cache gc)
+    db = root._p_jar.db()
+    ci = cacheInfo(db)
+    assert ci[kkey(ZBigArray)] == 1
+    assert A._p_state == UPTODATE
+    db.cacheMinimize()
+    ci = cacheInfo(db)
+    assert ci[kkey(ZBigArray)] == 1
+    assert A._p_state == UPTODATE   # it would be GHOST without LivePersistent protection
+
+    a[-1] = 99  # would not propagate to file without ZBigFile preventing itself to go to ghost
+    transaction.commit()
+
+
+    # reload & verify changes
+    dbclose(root)
+    del root, a, A, db
+
+
+    root = dbopen('%s/1.fs' % tmpdir)
+    A = root['zarray']
+
+    assert isinstance(A, ZBigArray)
+    assert A.shape  == (16*1024*1024,)
+    assert A.dtype  == dtype(uint8)
+
+    a = A[:]
+    assert all(a[:33] == 0)
+    assert a[33] == 33
+    assert all(a[33+1:-2] == 0)
+    assert a[-2] == 98
+    assert a[-1] == 99