X Restructure: found bug that it was not marking objects as modified

As a result corrupt btrees were actually persisted to storage.

wcfs/internal/xbtree_test.py

@@ -456,8 +456,16 @@ def test_restructure():
     zconn = db.open()
 
     X = []  # X[i] -> XBlk corresponding to block #i
-    for i in range(12):
-        X.append(XBlk('abcdefghijkl'[i]))
+    xv = 'abcdefghijkl'
+    for i in range(len(xv)):
+        X.append(XBlk(xv[i]))
+    def xdecode(v):
+        assert len(v) == 1
+        assert v in xv
+        return X[xv.index(v)]
+    def xencode(x):
+        assert isinstance(x, XBlk)
+        return x.data
 
     # assertB wraps global assertB to automatically fill in X[k] values for specified keys.
     def assertB(znode, *keyv):
@@ -488,7 +496,7 @@ def test_restructure():
             assert ztree[k] == v
 
         if isinstance(newtopo, str):
-            newStructure = xbtree.TopoDecode(newtopo)
+            newStructure = xbtree.TopoDecode(newtopo, xdecode)
         else:
             assert isinstance(newtopo, xbtree.Tree)
             newStructure = newtopo
@@ -506,6 +514,9 @@ def test_restructure():
     # S returns topo-encoded keys-only structure of ztree
     def S(ztree):
         return xbtree.TopoEncode(xbtree.StructureOf(ztree, onlyKeys=True))
+    # Sv returns topo-encoded structure of ztree with values
+    def Sv(ztree):
+        return xbtree.TopoEncode(xbtree.StructureOf(ztree), xencode)
 
     # Z0 creates new empty tree
     def Z0():
@@ -716,6 +727,23 @@ def test_restructure():
     R(z, 'T/B1,3,5,7,11')
 
 
+
+    # make sure changed objects are marked as such and so included into commit
+    z = Z(0,2)
+    transaction.commit()
+    R(z, 'T1/B0:a-B2:c')
+    transaction.commit()
+    assert Sv(z) == 'T1/B0:a-B2:c'
+    zconn.cacheMinimize()           # force z state to be reloaded from storage
+    assert Sv(z) == 'T1/B0:a-B2:c'  # will fail if T or B is not marked as changed
+
+    R(z, 'T/B0:a,2:c')              # one more similar check
+    transaction.commit()
+    assert Sv(z) == 'T/B0:a,2:c'
+    zconn.cacheMinimize()
+    assert Sv(z) == 'T/B0:a,2:c'
+
+
     # ---- tests on automatically generated topologies ----
     #
     # ( we make sure that Restructure can make the restructurement and that

wcfs/testprog/treegen.py

@@ -97,8 +97,9 @@ from persistent.mapping import PersistentMapping
 XLOTree = xbtree_test.XLOTree
 XLOTree.__module__ = 'BTrees.LOBTree'
 XLOTree.__name__   = 'LOBTree'
-from BTrees import LOBTree
-LOBTree.LOBTree = XLOTree
+import BTrees.LOBTree
+BTrees.LOBTree.LOBTree = XLOTree
+from BTrees.LOBTree import LOBTree
 
 
 # ZCtx represent treegen-level connection to ZODB.
@@ -152,7 +153,7 @@ def Trees(zstor, r):
     zctx = ZCtx(zstor)
     defer(zctx.close)
 
-    ztree = zctx.root['treegen/tree'] = XLOTree()
+    ztree = zctx.root['treegen/tree'] = LOBTree()
     head = commit('treegen/tree: init')
     xprint("tree.srv start @%s root=%s" % (ashex(head), ashex(ztree._p_oid)))
     treetxtPrev = zctx.ztreetxt(ztree)
@@ -240,7 +241,7 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
     print("# allstructs %s %s" % (kv1txt, kv2txt))
 
     # create the tree
-    ztree = zctx.root['ztree'] = XLOTree()
+    ztree = zctx.root['ztree'] = LOBTree()
     commit('init')
 
     # initial kv1 and kv2 states with topologies prepared as ZODB would do natively
@@ -380,7 +381,7 @@ def commit(description, skipIfEmpty=False): # -> tid | None
     assert isinstance(zconn, Connection)
     assert len(zconn._registered_objects) > 0
     obj = zconn._registered_objects[0]
-    assert obj._p_state == CHANGED
+    assert obj._p_state == CHANGED, obj._p_state
     txn.commit()
     return obj._p_serial
 

wcfs/δbtail_test.go

@@ -199,6 +199,9 @@ func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
 
 	reply, err := tg.pyout.ReadString('\n')
 	if err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
 		return zodb.InvalidTid, err
 	}
 	reply = strings.TrimSuffix(reply, "\n")
@@ -225,6 +228,9 @@ func (tg *AllStructsSrv) AllStructs(kv1, kv2 map[Key]string, maxdepth, maxsplit,
 	for {
 		reply, err := tg.pyout.ReadString('\n')
 		if err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
 			return nil, err
 		}
 		reply = strings.TrimSuffix(reply, "\n")