Skip to content

W
wendelin.core
Commit e9c4b619 authored by Kirill Smelkov's avatar Kirill Smelkov
6 Browse files
Options

X rebuild: tests: Random testing 

Rework py allstructs to emit just set of random trees for one kv.
Rework TestΔBTailAllStructs to iterate through such obtained sets and
generate all pairs and triplets. Previously it was only pairs which is
not enough to verify rebuild properly.
parent 3e9164b3
Hide whitespace changes
Inline Side-by-side
Showing with 139 additions and 124 deletions
wcfs/internal/xbtree/testprog/treegen.py
View file @ e9c4b619
......@@ -25,8 +25,8 @@ It is used as helper for ΔBtail tests.
The following subcommands are provided:
- `trees` transition ZODB tree through requested tree states,
- `allstructs` generates topologies for subset of all possible tree changes in
between two trees specified by two key->value dicts.
- `allstructs` generate subset of all possible tree topologies for a tree
specified by key->value dict.
Because python/pkg_resources startup is very slow(*) all subcommands can be
used either in CLI or in server mode, where requests are continuously read from
......@@ -50,10 +50,10 @@ zconn.root()['treegen/tree'].
Trees protocol specification:
S: tree.srv start @<head> root=<tree-root-oid>
C: <tree>
S: <tid>
C: <tree>
S: <tid>
C: <tree₁>
S: <tid₁>
C: <tree₂>
S: <tid₂>
...
session example:
......@@ -69,43 +69,43 @@ session example:
allstructs
----------
`treegen allstructs` generates topologies for subset of all possible tree
changes in between two trees specified by two key->value dicts.
`treegen allstructs` generates subset of all possible tree topologies for a tree
specified by key->value dict.
For every kv the following tree topologies are considered: 1) native (the one
that ZODB would usually create natively via regular usage), and 2) n random
ones. Then tree topologies are emitted corresponding to tree1->tree2 and
tree1<-tree2 transitions for all combinations of (tree1, tree2) pairs.
Given kv the following tree topologies are considered: 1) native (the one
that ZODB would usually create natively via regular usage), and 2) n-1 random
ones. Then those tree topologies are emitted.
The output of `treegen allstructs` is valid input for `treegen trees`.
Allstructs protocol specification:
S: # allstructs.srv start
C: <maxdepth> <maxsplit> <n>(/<seed>) <kv1> <kv2>
S: # allstructs <kv1> <kv2>
C: <maxdepth> <maxsplit> <n>(/<seed>) <kv>
S: # allstructs <kv>
S: # maxdepth=<maxdepth> maxsplit=<maxsplit> n=<n> seed=<seed>
S: <tree1₀>
S: <tree2₀>
S: <tree1₁>
S: <tree₀>
S: <tree₁>
S: <tree₂>
...
S: # ----
session example:
# allstructs.srv start
1 1 10 1:a 2:b
# allstructs 1:a 2:b
# maxdepth=1 maxsplit=1 n=10 seed=1591369961
T/B1:a
T/B2:b
T/T/B1:a
T/B2:b
T/B1:a
T/T/B2:b
T/T/B1:a
T/T/B2:b
T/B1:a
1 1 10 1:a,2:b,3:c
# allstructs 1:a,2:b,3:c
# maxdepth=1 maxsplit=1 n=10 seed=1624901326
T2/B1:a-B2:b,3:c
T3/B1:a,2:b-B3:c
T2/T-T3/B1:a-B2:b-B3:c
T/T3/B1:a,2:b-B3:c
T/T2/B1:a-B2:b,3:c
T/B1:a,2:b,3:c
T3/T2-T/B1:a-B2:b-B3:c
T2/T-T/B1:a-B2:b,3:c
T/T/B1:a,2:b,3:c
T3/T-T/B1:a,2:b-B3:c
# ----
--------
......@@ -258,8 +258,8 @@ def TreesSrv(zstor, r):
def AllStructsSrv(r):
xprint('# allstructs.srv start')
for req in xreadlines(r):
# maxdepth maxsplit n(/seed) kv1 kv2
maxdepth, maxsplit, n, kv1txt, kv2txt = req.split()
# maxdepth maxsplit n(/seed) kv
maxdepth, maxsplit, n, kvtxt = req.split()
maxdepth = int(maxdepth)
maxsplit = int(maxsplit)
seed = None
......@@ -267,39 +267,32 @@ def AllStructsSrv(r):
n, seeds = n.split('/')
seed = int(seeds)
n = int(n)
if kv1txt == 'ø': kv1txt = ''
if kv2txt == 'ø': kv2txt = ''
if kvtxt == 'ø': kvtxt = ''
AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed)
AllStructs(kvtxt, maxdepth, maxsplit, n, seed)
xprint('# ----')
# AllStructs generates topologies for subset of all possible tree changes in
# between kv1 and kv2. See top-level documentation for details.
# AllStructs generates subset of all possible topologies for a tree specified by kv dict.
# See top-level documentation for details.
@func
def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
def AllStructs(kvtxt, maxdepth, maxsplit, n, seed=None):
zstor = MappingStorage() # in RAM storage to create native ZODB topologies
zctx = ZCtx(zstor)
defer(zctx.close)
kv1 = kvDecode(kv1txt, zctx.vdecode)
kv2 = kvDecode(kv2txt, zctx.vdecode)
kv = kvDecode(kvtxt, zctx.vdecode)
print("# allstructs %s %s" % (kv1txt, kv2txt))
print("# allstructs %s" % kvtxt)
# create the tree
ztree = zctx.root['ztree'] = LOBTree()
commit('init')
# initial kv1 and kv2 states with topologies prepared as ZODB would do natively
patch(ztree, diff({}, kv1), verify=kv1)
if kv1 == {}: ztree._p_changed = True # to avoid empty commit - see TreesSrv
commit('kv1')
t1struct0 = xbtree.StructureOf(ztree)
patch(ztree, diff(kv1, kv2), verify=kv2)
if kv2 == kv1: ztree._p_changed = True
commit('kv2')
t2struct0 = xbtree.StructureOf(ztree)
# initial kv state with topology prepared as ZODB would do natively
patch(ztree, diff({}, kv), verify=kv)
if kv == {}: ztree._p_changed = True # to avoid empty commit - see TreesSrv
commit('kv')
tstruct0 = xbtree.StructureOf(ztree)
# seed
if seed is None:
......@@ -308,31 +301,15 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
random.seed(seed)
print("# maxdepth=%d maxsplit=%d n=%d seed=%d" % (maxdepth, maxsplit, n, seed))
# make 2·n random samples from all tree topologies that can represent kv1 and kv2
t1structv = rsample(xbtree.AllStructs(kv1.keys(), maxdepth, maxsplit, kv=kv1), n)
t2structv = rsample(xbtree.AllStructs(kv2.keys(), maxdepth, maxsplit, kv=kv2), n)
# all tree1 and tree2 topologies jumps in between we are going to emit:
# native + n random ones.
if t1struct0 in t1structv: t1structv.remove(t1struct0) # avoid dups
if t2struct0 in t2structv: t2structv.remove(t2struct0)
t1structv.insert(0, t1struct0)
t2structv.insert(0, t2struct0)
# XXX rework allstructs to accept only 1 kv and emit n structs for that one kv only
# -> iterate through the pairs/triplets in the caller (TestΔBTailAllStructs)
# emit topologies for tree1->tree2 and tree1<-tree2 transitions for all
# combinations of tree1 and tree2.
t12travel = list(bitravel2Way(t1structv, t2structv))
for i,tstruct in enumerate(t12travel):
if i%2 == 0:
assert tstruct in t1structv
else:
assert tstruct in t2structv
# emit native + n-1 random samples from all tree topologies that can represent kv
tstructv = rsample(xbtree.AllStructs(kv.keys(), maxdepth, maxsplit, kv=kv), n-1)
if tstruct0 in tstructv: tstructv.remove(tstruct0) # avoid dups
tstructv.insert(0, tstruct0)
for tstruct in tstructv:
print(zctx.TopoEncode(tstruct))
# rsample returns k random samples from seq.
# it differs from random.sample in that it does not keep whole list(seq) in memory.
def rsample(seq, k): # -> [] of items; len <= k
......@@ -352,31 +329,6 @@ def rsample(seq, k): # -> [] of items; len <= k
sample[j] = item
return sample
# bitravel2Way generates travel path through all A<->B edges such
# that all edges a->b and a<-b are traveled and exactly once.
#
# The travel starts from A[0].
def bitravel2Way(A, B): # -> i[] of node
na = len(A); assert na > 0
nb = len(B); assert nb > 0
yield A[0] # A₀
for j in range(nb):
yield B[j] # A₀ -> Bj
for i in range(1,na):
yield A[i] # Ai <- Bj
yield B[j] # Ai -> Bj
yield A[0] # A₀ <- Bj
def test_bitravel2Way():
a,b,c = 'a','b','c'
A = [a,b,c]
B = [1, 2]
got = list(bitravel2Way(A, B))
want = [a,1,b,1,c,1,a,2,b,2,c,2,a]
assert got == want, (got, want)
test_bitravel2Way()
# kvEncode encodes key->value mapping into text.
# e.g. {1:'a', 2:'b'} -> '1:a,2:b'
......
wcfs/internal/xbtree/δbtail_test.go
View file @ e9c4b619
......@@ -49,6 +49,7 @@ import (
"flag"
"fmt"
"io"
"math"
"math/rand"
"os"
"os/exec"
......@@ -235,9 +236,9 @@ func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
}
// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv1, kv2 map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s %s ", maxdepth, maxsplit, n, seed, kvtxt(kv1), kvtxt(kv2))
defer xerr.Contextf(&err, "allstructs.srv: %s", req)
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
_, err = io.WriteString(tg.pyin, req + "\n")
if err != nil {
......@@ -1383,10 +1384,15 @@ func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*t
// ΔBTestEntry represents one entry in ΔBTail tests.
type ΔBTestEntry struct {
tree string // next tree topology
kadjOK KAdjMatrix // adjacency matrix against previous case (optional)
tree string // next tree topology
kadjOK KAdjMatrix // adjacency matrix against previous case (optional)
flags ΔBTestFlags
}
type ΔBTestFlags int
const ΔBTest_SkipUpdate ΔBTestFlags = 1 // skip verifying Update for this test entry
const ΔBTest_SkipRebuild ΔBTestFlags = 2 // skip verifying rebuild for this test entry
// ΔBTest converts xtest into ΔBTestEntry.
// xtest can be string|ΔBTestEntry.
func ΔBTest(xtest interface{}) ΔBTestEntry {
......@@ -1395,6 +1401,7 @@ func ΔBTest(xtest interface{}) ΔBTestEntry {
case string:
test.tree = xtest
test.kadjOK = nil
test.flags = 0
case ΔBTestEntry:
test = xtest
default:
......@@ -1406,9 +1413,8 @@ func ΔBTest(xtest interface{}) ΔBTestEntry {
// tTreeEnv is tree-based testing environment.
//
// It combines TreeSrv and client side access to ZODB with committed trees. XXX
//
// Create it with tNewTreeEnv().
// It combines TreeSrv and client side access to ZODB with committed trees.
// It should be created it via tNewTreeEnv().
type tTreeEnv struct {
*testing.T
......@@ -1417,13 +1423,13 @@ type tTreeEnv struct {
zstor zodb.IStorage
db *zodb.DB
// all committed trees XXX name
// all committed trees
commitv []*tTreeCommit
}
// tTreeCommit represent test commit changing a tree.
type tTreeCommit struct {
tree string // the tree in toplogy-encoding
tree string // the tree in topology-encoding
prev *tTreeCommit // previous commit
at zodb.Tid // commit revision
δZ *zodb.EventCommit // raw ZODB changes; δZ.tid == at
......@@ -1497,7 +1503,7 @@ func (t *tTreeEnv) Head() *tTreeCommit {
return t.commitv[len(t.commitv)-1]
}
// CommitTree calls tg.Commit and returns tTreeCommit corresponding to committed transaction.
// CommitTree calls t.treeSrv.Commit and returns tTreeCommit corresponding to committed transaction.
func (t *tTreeEnv) CommitTree(tree string) *tTreeCommit {
// TODO X = FatalIf
X := exc.Raiseif
......@@ -1651,10 +1657,12 @@ func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) {
}
// ΔBTail.Update
xverifyΔBTail_Update(t.T, subj, t.db, t.Root(), t1,t2)
if test.flags & ΔBTest_SkipUpdate == 0 {
xverifyΔBTail_Update(t.T, subj, t.db, t.Root(), t1,t2)
}
// ΔBTail.rebuild
if t0 != nil {
if t0 != nil && (test.flags & ΔBTest_SkipRebuild == 0) {
xverifyΔBTail_rebuild(t.T, t.db, t.Root(), t0,t1,t2)
}
......@@ -1984,7 +1992,7 @@ func TestΔBTailAllStructs(t *testing.T) {
n := N(10,10,100)
nkeys := N(3, 5, 10)
// server to generate AllStructs(kv1, kv2, ...)
// server to generate AllStructs(kv, ...)
sg, err := StartAllStructsSrv(); X(err)
defer func() {
err := sg.Close(); X(err)
......@@ -1998,55 +2006,110 @@ func TestΔBTailAllStructs(t *testing.T) {
rng := rand.New(rand.NewSource(seed))
t.Logf("# maxdepth=%d maxsplit=%d nkeys=%d n=%d seed=%d", maxdepth, maxsplit, nkeys, n, seed)
// generate (kv1, kv2) pairs randomly
// generate (kv1, kv2, kv3) randomly
// keysv1 and keysv2 are random shuffle of IntSets
// keysv1, keysv2 and keysv3 are random shuffle of IntSets
var keysv1 [][]int
var keysv2 [][]int
var keysv3 [][]int
for keys := range IntSets(nkeys) {
keysv1 = append(keysv1, keys)
keysv2 = append(keysv2, keys)
keysv3 = append(keysv3, keys)
}
v := keysv1
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv2
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv3
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
// generate cases: keysv1[i] -> keysv2[i] with values generated
// randomly along the way.
// given random (kv1, kv2, kv3) generate corresponding set of random tree
// topology sets (T1, T2, T3). Then iterate through T1->T2->T3->T1...
// elements such that all right-directed triplets are visited and only once.
// Test Update and rebuild on the generated tree sequences.
vv := "abcdefgh"
randv := func() string {
i := rng.Intn(len(vv))
return vv[i:i+1]
}
// XXX rework to be effective with testing rebuild:
// - AllStructs accept only 1 kv
// - we call it 3 times for kv1 kv2 kv3
// - iterate via triTravelXXXway ... here instead of in treegen.py
// the number of pairs is 3·n^2
// the number of triplets is n^3
//
// limit n for emitted triplets, so that the amount of work for Update
// and rebuild tests is approximately of the same order.
nrebuild := int(math.Ceil(math.Pow(3*float64(n*n), 1./3)))
// in non-short mode rebuild tests are exercising more keys variants, plus every test case
// takes more time. Compensate for that as well.
if !testing.Short() {
nrebuild -= 3
}
testq := make(chan ΔBTestEntry)
go func() {
defer close(testq)
for i := range keysv1 {
keys1 := keysv1[i]
keys2 := keysv2[i]
keys3 := keysv3[i]
kv1 := map[Key]string{}
kv2 := map[Key]string{}
kv3 := map[Key]string{}
for _, k := range keys1 { kv1[Key(k)] = randv() }
for _, k := range keys2 { kv2[Key(k)] = randv() }
for _, k := range keys3 { kv3[Key(k)] = randv() }
// given (kv1, kv2) - test on automatically generated (tree1 -> tree2)
reqSeed := rng.Int63()
treev, err := sg.AllStructs(kv1, kv2, maxdepth, maxsplit, n, reqSeed)
treev1, err1 := sg.AllStructs(kv1, maxdepth, maxsplit, n, rng.Int63())
treev2, err2 := sg.AllStructs(kv2, maxdepth, maxsplit, n, rng.Int63())
treev3, err3 := sg.AllStructs(kv3, maxdepth, maxsplit, n, rng.Int63())
err := xerr.Merge(err1, err2, err3)
if err != nil {
t.Fatal(err)
}
for _, tree := range treev {
testq <- ΔBTestEntry{tree, nil}
emit := func(tree string, flags ΔBTestFlags) {
// skip emitting this entry if both Update and
// Rebuild are requested to be skipped.
if flags == (ΔBTest_SkipUpdate | ΔBTest_SkipRebuild) {
return
}
testq <- ΔBTestEntry{tree, nil, flags}
}
URSkipIf := func(ucond, rcond bool) ΔBTestFlags {
var flags ΔBTestFlags
if ucond {
flags |= ΔBTest_SkipUpdate
}
if rcond {
flags |= ΔBTest_SkipRebuild
}
return flags
}
for j := range treev1 {
for k := range treev2 {
for l := range treev3 {
// limit rebuild to subset of tree topologies,
// because #(triplets) grow as n^3. See nrebuild
// definition above for details.
norebuild := (j >= nrebuild ||
k >= nrebuild ||
l >= nrebuild)
// C_{l-1} -> Aj (pair first seen on k=0)
emit(treev1[j], URSkipIf(k != 0, norebuild))
// Aj -> Bk (pair first seen on l=0)
emit(treev2[k], URSkipIf(l != 0, norebuild))
// Bk -> Cl (pair first seen on j=0)
emit(treev3[l], URSkipIf(j != 0, norebuild))
}
}
}
}
}()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7