Commit 5e0c0818 authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent 73abafd0
...@@ -164,9 +164,9 @@ class DFile: ...@@ -164,9 +164,9 @@ class DFile:
# #
# Database root and wcfs connection are represented by .root and .wc correspondingly. # Database root and wcfs connection are represented by .root and .wc correspondingly.
# #
# The primary way to access wcfs is by opening BigFiles and Watches. # The primary way to access wcfs is by opening BigFiles and WatcheLinks.
# A BigFile opened under tDB is represented as tFile - see .open for details. # A BigFile opened under tDB is represented as tFile - see .open for details.
# A Watch opened under tDB is represented as tWatchLink - see .openwatch for details. # A WatchLink opened under tDB is represented as tWatchLink - see .openwatch for details.
# #
# The database can be mutated (via !wcfs codepath) with .change + .commit . # The database can be mutated (via !wcfs codepath) with .change + .commit .
# Current database head is represented by .head . # Current database head is represented by .head .
...@@ -514,6 +514,7 @@ class tFile: ...@@ -514,6 +514,7 @@ class tFile:
for wlink, pinok in pinokByWLink.items(): for wlink, pinok in pinokByWLink.items():
pinokByWLink[wlink] = (t.zf, pinok) pinokByWLink[wlink] = (t.zf, pinok)
# access 1 bye on the block and verify that wcfs sends us correct pins
blkview = t._blk(blk) blkview = t._blk(blk)
assert t.cached()[blk] == cached assert t.cached()[blk] == cached
...@@ -557,6 +558,7 @@ class tFile: ...@@ -557,6 +558,7 @@ class tFile:
assert t.cached()[blk] > 0 assert t.cached()[blk] > 0
# verify full data of the block
# XXX assert individually for every block's page? (easier debugging?) # XXX assert individually for every block's page? (easier debugging?)
assert blkview == dataok assert blkview == dataok
...@@ -830,108 +832,7 @@ class tSrvReq: ...@@ -830,108 +832,7 @@ class tSrvReq:
def at(req): return req._parse()[2] def at(req): return req._parse()[2]
# ---- helpers to query dFtail/accessed history ---- # ---- infrastructure: watch setup/adjust ----
# _blkData returns expected zf[blk] data and revision as of @at database state.
#
# If the block is hole - (b'', at0) is returned. XXX -> @z64?
# XXX ret for when the file did not existed at all? blk was after file size?
@func(tDB)
def _blkData(t, zf, blk, at): # -> (data, rev)
if at is None:
at = t.head
# XXX dup wrt _pinAt
# all changes to zf
vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
# changes to zf[blk] <= at
blkhistoryat = [_ for _ in vdf if blk in _.ddata and _.rev <= at]
if len(blkhistoryat) == 0:
# blk did not existed @at # XXX verify whether file was existing at all
data = b''
rev = t.dFtail[0].rev # was hole - at0 XXX -> pin to z64
else:
_ = blkhistoryat[-1]
data = _.ddata[blk]
rev = _.rev
assert rev <= at
return data, rev
# _blkRev returns expected zf[blk] revision as of @at database state.
@func(tDB)
def _blkRev(t, zf, blk, at): # -> rev
_, rev = t._blkData(zf, blk, at)
return rev
# XXX vvv -> not what we need, think again
# _blkHeadAccessed returns whether block state corresponding to zf[blk] at
# current head was accessed.
#
# for example - if head/<zf>[blk] was accessed and later changed, the answer is "no".
# buf if head/<zf>[blk] was accessed again after change, and was no longer
# changed, the answer is "yes"
# XXX text
# XXX -> _blkLastRevAccessed ? _blkRevLastAccessed ? _blkHeadRevAccessed ?
"""
@func(tDB)
def _blkHeadAccessed(t, zf, blk):
zfAccessed = t._accessed.get(zf, {})
zfAccessed.get(blk) vs t._blkRev(zf, blk, t.head)
"""
# _pinAt returns which blocks needs to be pinned for zf@at compared to zf@head
# according to wcfs invalidation protocol.
#
# It does not take into account whether blocks are in cache or not and computes
# pins as if all blocks @head were accesses - i.e. considering all file changes
# in (at, head] range.
#
# The caller has to take accessed/not-accessed effect into account on its own
# (see tDB._accessed & friends) XXX rework?
@func(tDB)
def _pinAt(t, zf, at): # -> pin = {} blk -> rev
# XXX dup in _blkData
# all changes to zf
vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
# {} blk -> at for changes ∈ (at, head]
pin = {}
for df in [_ for _ in vdf if _.rev > at]:
for blk in df.ddata:
if blk in pin:
continue
# history of blk changes <= at
blkhistoryat = [_.rev for _ in vdf if blk in _.ddata and _.rev <= at]
if len(blkhistoryat) == 0:
pinrev = t.dFtail[0].rev # was hole - at0 XXX -> pin to z64?
else:
pinrev = blkhistoryat[-1]
assert pinrev <= at
pin[blk] = pinrev
return pin
# iter_revv iterates through all possible at_i -> at_j -> at_k ... sequences.
# at_i < at_j NOTE all sequences go till head.
@func(tDB)
def iter_revv(t, start=z64, level=0):
dFtail = [_ for _ in t.dFtail if _.rev > start]
#print(' '*level, 'iter_revv', t.hat(start), [t.hat(_.rev) for _ in dFtail])
if len(dFtail) == 0:
yield []
return
for dF in dFtail:
#print(' '*level, 'QQQ', t.hat(dF.rev))
for tail in t.iter_revv(start=dF.rev, level=level+1):
#print(' '*level, 'zzz', tail)
yield ([dF.rev] + tail)
# ---- watch setup/adjust ----
# watch sets up or adjusts a watch for file@at. # watch sets up or adjusts a watch for file@at.
# #
...@@ -1144,7 +1045,109 @@ def _expectPin(twlink, ctx, zf, expect): # -> []tSrvReq ...@@ -1144,7 +1045,109 @@ def _expectPin(twlink, ctx, zf, expect): # -> []tSrvReq
return reqv return reqv
# ---- test access to data ---- # ---- infrastructure: helpers to query dFtail/accessed history ----
# _blkData returns expected zf[blk] data and revision as of @at database state.
#
# If the block is hole - (b'', at0) is returned. XXX -> @z64?
# XXX ret for when the file did not existed at all? blk was after file size?
@func(tDB)
def _blkData(t, zf, blk, at): # -> (data, rev)
if at is None:
at = t.head
# XXX dup wrt _pinAt
# all changes to zf
vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
# changes to zf[blk] <= at
blkhistoryat = [_ for _ in vdf if blk in _.ddata and _.rev <= at]
if len(blkhistoryat) == 0:
# blk did not existed @at # XXX verify whether file was existing at all
data = b''
rev = t.dFtail[0].rev # was hole - at0 XXX -> pin to z64
else:
_ = blkhistoryat[-1]
data = _.ddata[blk]
rev = _.rev
assert rev <= at
return data, rev
# _blkRev returns expected zf[blk] revision as of @at database state.
@func(tDB)
def _blkRev(t, zf, blk, at): # -> rev
_, rev = t._blkData(zf, blk, at)
return rev
# XXX vvv -> not what we need, think again
# _blkHeadAccessed returns whether block state corresponding to zf[blk] at
# current head was accessed.
#
# for example - if head/<zf>[blk] was accessed and later changed, the answer is "no".
# buf if head/<zf>[blk] was accessed again after change, and was no longer
# changed, the answer is "yes"
# XXX text
# XXX -> _blkLastRevAccessed ? _blkRevLastAccessed ? _blkHeadRevAccessed ?
"""
@func(tDB)
def _blkHeadAccessed(t, zf, blk):
zfAccessed = t._accessed.get(zf, {})
zfAccessed.get(blk) vs t._blkRev(zf, blk, t.head)
"""
# _pinAt returns which blocks needs to be pinned for zf@at compared to zf@head
# according to wcfs invalidation protocol.
#
# It does not take into account whether blocks are in cache or not and computes
# pins as if all blocks @head were accesses - i.e. considering all file changes
# in (at, head] range.
#
# The caller has to take accessed/not-accessed effect into account on its own
# (see tDB._accessed & friends) XXX rework?
@func(tDB)
def _pinAt(t, zf, at): # -> pin = {} blk -> rev
# XXX dup in _blkData
# all changes to zf
vdf = [_.byfile[zf] for _ in t.dFtail if zf in _.byfile]
# {} blk -> at for changes ∈ (at, head]
pin = {}
for df in [_ for _ in vdf if _.rev > at]:
for blk in df.ddata:
if blk in pin:
continue
# history of blk changes <= at
blkhistoryat = [_.rev for _ in vdf if blk in _.ddata and _.rev <= at]
if len(blkhistoryat) == 0:
pinrev = t.dFtail[0].rev # was hole - at0 XXX -> pin to z64?
else:
pinrev = blkhistoryat[-1]
assert pinrev <= at
pin[blk] = pinrev
return pin
# iter_revv iterates through all possible at_i -> at_j -> at_k ... sequences.
# at_i < at_j NOTE all sequences go till head.
@func(tDB)
def iter_revv(t, start=z64, level=0):
dFtail = [_ for _ in t.dFtail if _.rev > start]
#print(' '*level, 'iter_revv', t.hat(start), [t.hat(_.rev) for _ in dFtail])
if len(dFtail) == 0:
yield []
return
for dF in dFtail:
#print(' '*level, 'QQQ', t.hat(dF.rev))
for tail in t.iter_revv(start=dF.rev, level=level+1):
#print(' '*level, 'zzz', tail)
yield ([dF.rev] + tail)
# ---- actual tests to access data ----
# test_wcfs exercises wcfs functionality. # test_wcfs exercises wcfs functionality.
@func @func
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment