.

wcfs/wcfs_test.py

@@ -164,9 +164,9 @@ class DFile:
 #
 # Database root and wcfs connection are represented by .root and .wc correspondingly.
 #
-# The primary way to access wcfs is by opening BigFiles and Watches.
-# A BigFile opened under tDB is represented as tFile       - see .open for details.
-# A Watch   opened under tDB is represented as tWatchLink  - see .openwatch for details.
+# The primary way to access wcfs is by opening BigFiles and WatcheLinks.
+# A BigFile   opened under tDB is represented as tFile      - see .open for details.
+# A WatchLink opened under tDB is represented as tWatchLink - see .openwatch for details.
 #
 # The database can be mutated (via !wcfs codepath) with .change + .commit .
 # Current database head is represented by .head .
@@ -514,6 +514,7 @@ class tFile:
         for wlink, pinok in pinokByWLink.items():
             pinokByWLink[wlink] = (t.zf, pinok)
 
+        # access 1 bye on the block and verify that wcfs sends us correct pins
         blkview = t._blk(blk)
         assert t.cached()[blk] == cached
 
@@ -557,6 +558,7 @@ class tFile:
 
         assert t.cached()[blk] > 0
 
+        # verify full data of the block
         # XXX assert individually for every block's page? (easier debugging?)
         assert blkview == dataok
 
@@ -830,108 +832,7 @@ class tSrvReq:
     def at(req):      return req._parse()[2]
 
 
-# ---- helpers to query dFtail/accessed history ----
-
-# _blkData returns expected zf[blk] data and revision as of @at database state.
-#
-# If the block is hole - (b'', at0) is returned.  XXX -> @z64?
-# XXX ret for when the file did not existed at all? blk was after file size?
-@func(tDB)
-def _blkData(t, zf, blk, at): # -> (data, rev)
-    if at is None:
-        at = t.head
-
-    # XXX dup wrt _pinAt
-
-    # all changes to zf
-    vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
-
-    # changes to zf[blk] <= at
-    blkhistoryat = [_ for _ in vdf if blk in _.ddata and _.rev <= at]
-    if len(blkhistoryat) == 0:
-        # blk did not existed @at       # XXX verify whether file was existing at all
-        data = b''
-        rev  = t.dFtail[0].rev  # was hole - at0    XXX -> pin to z64
-    else:
-        _ = blkhistoryat[-1]
-        data = _.ddata[blk]
-        rev  = _.rev
-        assert rev <= at
-
-    return data, rev
-
-# _blkRev returns expected zf[blk] revision as of @at database state.
-@func(tDB)
-def _blkRev(t, zf, blk, at): # -> rev
-    _, rev = t._blkData(zf, blk, at)
-    return rev
-
-# XXX vvv -> not what we need, think again
-# _blkHeadAccessed returns whether block state corresponding to zf[blk] at
-# current head was accessed.
-#
-# for example - if head/<zf>[blk] was accessed and later changed, the answer is "no".
-# buf if head/<zf>[blk] was accessed again after change, and was no longer
-# changed, the answer is "yes"
-# XXX text
-# XXX -> _blkLastRevAccessed ?  _blkRevLastAccessed ? _blkHeadRevAccessed ?
-"""
-@func(tDB)
-def _blkHeadAccessed(t, zf, blk):
-    zfAccessed = t._accessed.get(zf, {})
-    zfAccessed.get(blk) vs t._blkRev(zf, blk, t.head)
-"""
-
-# _pinAt returns which blocks needs to be pinned for zf@at compared to zf@head
-# according to wcfs invalidation protocol.
-#
-# It does not take into account whether blocks are in cache or not and computes
-# pins as if all blocks @head were accesses - i.e. considering all file changes
-# in (at, head] range.
-#
-# The caller has to take accessed/not-accessed effect into account on its own
-# (see tDB._accessed & friends)     XXX rework?
-@func(tDB)
-def _pinAt(t, zf, at):  # -> pin = {} blk -> rev
-    # XXX dup in _blkData
-
-    # all changes to zf
-    vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
-
-    # {} blk -> at for changes ∈ (at, head]
-    pin = {}
-    for df in [_ for _ in vdf if _.rev > at]:
-        for blk in df.ddata:
-            if blk in pin:
-                continue
-            # history of blk changes <= at
-            blkhistoryat = [_.rev for _ in vdf if blk in _.ddata and _.rev <= at]
-            if len(blkhistoryat) == 0:
-                pinrev = t.dFtail[0].rev    # was hole - at0  XXX -> pin to z64?
-            else:
-                pinrev = blkhistoryat[-1]
-            assert pinrev <= at
-            pin[blk] = pinrev
-
-    return pin
-
-# iter_revv iterates through all possible at_i -> at_j -> at_k ... sequences.
-# at_i < at_j       NOTE all sequences go till head.
-@func(tDB)
-def iter_revv(t, start=z64, level=0):
-    dFtail = [_ for _ in t.dFtail if _.rev > start]
-    #print(' '*level, 'iter_revv', t.hat(start), [t.hat(_.rev) for _ in dFtail])
-    if len(dFtail) == 0:
-        yield []
-        return
-
-    for dF in dFtail:
-        #print(' '*level, 'QQQ', t.hat(dF.rev))
-        for tail in t.iter_revv(start=dF.rev, level=level+1):
-            #print(' '*level, 'zzz', tail)
-            yield ([dF.rev] + tail)
-
-# ---- watch setup/adjust ----
+# ---- infrastructure: watch setup/adjust ----
 
 # watch sets up or adjusts a watch for file@at.
 #
@@ -1144,7 +1045,109 @@ def _expectPin(twlink, ctx, zf, expect): # -> []tSrvReq
     return reqv
 
 
-# ---- test access to data ----
+# ---- infrastructure: helpers to query dFtail/accessed history ----
+
+# _blkData returns expected zf[blk] data and revision as of @at database state.
+#
+# If the block is hole - (b'', at0) is returned.  XXX -> @z64?
+# XXX ret for when the file did not existed at all? blk was after file size?
+@func(tDB)
+def _blkData(t, zf, blk, at): # -> (data, rev)
+    if at is None:
+        at = t.head
+
+    # XXX dup wrt _pinAt
+
+    # all changes to zf
+    vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
+
+    # changes to zf[blk] <= at
+    blkhistoryat = [_ for _ in vdf if blk in _.ddata and _.rev <= at]
+    if len(blkhistoryat) == 0:
+        # blk did not existed @at       # XXX verify whether file was existing at all
+        data = b''
+        rev  = t.dFtail[0].rev  # was hole - at0    XXX -> pin to z64
+    else:
+        _ = blkhistoryat[-1]
+        data = _.ddata[blk]
+        rev  = _.rev
+        assert rev <= at
+
+    return data, rev
+
+# _blkRev returns expected zf[blk] revision as of @at database state.
+@func(tDB)
+def _blkRev(t, zf, blk, at): # -> rev
+    _, rev = t._blkData(zf, blk, at)
+    return rev
+
+# XXX vvv -> not what we need, think again
+# _blkHeadAccessed returns whether block state corresponding to zf[blk] at
+# current head was accessed.
+#
+# for example - if head/<zf>[blk] was accessed and later changed, the answer is "no".
+# buf if head/<zf>[blk] was accessed again after change, and was no longer
+# changed, the answer is "yes"
+# XXX text
+# XXX -> _blkLastRevAccessed ?  _blkRevLastAccessed ? _blkHeadRevAccessed ?
+"""
+@func(tDB)
+def _blkHeadAccessed(t, zf, blk):
+    zfAccessed = t._accessed.get(zf, {})
+    zfAccessed.get(blk) vs t._blkRev(zf, blk, t.head)
+"""
+
+# _pinAt returns which blocks needs to be pinned for zf@at compared to zf@head
+# according to wcfs invalidation protocol.
+#
+# It does not take into account whether blocks are in cache or not and computes
+# pins as if all blocks @head were accesses - i.e. considering all file changes
+# in (at, head] range.
+#
+# The caller has to take accessed/not-accessed effect into account on its own
+# (see tDB._accessed & friends)     XXX rework?
+@func(tDB)
+def _pinAt(t, zf, at):  # -> pin = {} blk -> rev
+    # XXX dup in _blkData
+
+    # all changes to zf
+    vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
+
+    # {} blk -> at for changes ∈ (at, head]
+    pin = {}
+    for df in [_ for _ in vdf if _.rev > at]:
+        for blk in df.ddata:
+            if blk in pin:
+                continue
+            # history of blk changes <= at
+            blkhistoryat = [_.rev for _ in vdf if blk in _.ddata and _.rev <= at]
+            if len(blkhistoryat) == 0:
+                pinrev = t.dFtail[0].rev    # was hole - at0  XXX -> pin to z64?
+            else:
+                pinrev = blkhistoryat[-1]
+            assert pinrev <= at
+            pin[blk] = pinrev
+
+    return pin
+
+# iter_revv iterates through all possible at_i -> at_j -> at_k ... sequences.
+# at_i < at_j       NOTE all sequences go till head.
+@func(tDB)
+def iter_revv(t, start=z64, level=0):
+    dFtail = [_ for _ in t.dFtail if _.rev > start]
+    #print(' '*level, 'iter_revv', t.hat(start), [t.hat(_.rev) for _ in dFtail])
+    if len(dFtail) == 0:
+        yield []
+        return
+
+    for dF in dFtail:
+        #print(' '*level, 'QQQ', t.hat(dF.rev))
+        for tail in t.iter_revv(start=dF.rev, level=level+1):
+            #print(' '*level, 'zzz', tail)
+            yield ([dF.rev] + tail)
+
+
+# ---- actual tests to access data ----
 
 # test_wcfs exercises wcfs functionality.
 @func