Commit 8f6e2b1e authored by Kirill Smelkov's avatar Kirill Smelkov

X rebuild: tests: Don't access ZODB in XGetδKV

Preload data when tTreeCommit is created and use that data in
the inner loops instead of accessing ZODB from those inner loops.

130s -> 95s
parent 324241eb
......@@ -403,7 +403,7 @@ func XGetTree(db *zodb.DB, at zodb.Tid, root zodb.Oid) RBucketSet {
xwalkDFS(ctx, KeyMin, KeyMax, ztree, func(rb *RBucket) {
rbucketv = append(rbucketv, rb)
})
if len(rbucketv) == 0 { // empty tree -> [-∞, ∞){}
if len(rbucketv) == 0 { // empty tree -> [-∞,∞){}
etree := &RTree{
oid: root,
parent: nil,
......@@ -474,24 +474,15 @@ func _xwalkDFS(ctx context.Context, lo, hi_ Key, ztree *Tree, rparent *RTree, bv
}
}
// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to db@at1..at2 snapshots.
func XGetδKV(db *zodb.DB, at1, at2 zodb.Tid, δkvOid map[Key]ΔValue) map[Key]Δstring {
defer exc.Contextf("%s: get δkv %s..%s %v", db.Storage().URL(), at1, at2, δkvOid)
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn1, err := db.Open(ctx, &zodb.ConnOptions{At: at1}); X(err)
zconn2, err := db.Open(ctx, &zodb.ConnOptions{At: at2}); X(err)
δkv := map[Key]Δstring{}
// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
func XGetδKV(t1, t2 *tTreeCommit, δkvOid map[Key]ΔValue) map[Key]Δstring {
δkv := make(map[Key]Δstring, len(δkvOid))
for k, δvOid := range δkvOid {
δkv[k] = Δstring{
Old: xzgetBlkData(ctx, zconn1, δvOid.Old),
New: xzgetBlkData(ctx, zconn2, δvOid.New),
Old: t1.xgetBlkData(δvOid.Old),
New: t2.xgetBlkData(δvOid.New),
}
}
return δkv
}
......@@ -529,7 +520,7 @@ type KAdjMatrix map[Key]SetKey
// Map returns kadj·keys .
func (kadj KAdjMatrix) Map(keys SetKey) SetKey {
res := SetKey{}
res := make(SetKey, len(keys))
for k := range keys {
to, ok := kadj[k]
if !ok {
......@@ -691,7 +682,7 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
// this t.Run allocates and keeps too much memory in -verylong
// also it is not so useful as above "Update/t1->t2"
//t.Run(fmt.Sprintf(" track=%s", keys), func(t *testing.T) {
xverifyΔBTail_Update1(t, subj, db, treeRoot, t1.at,t2.at, t1.xkv,t2.xkv, t2.δZ, t2.δxkv, keys, kadj12)
xverifyΔBTail_Update1(t, subj, db, treeRoot, t1,t2, keys, kadj12)
//})
}
})
......@@ -699,10 +690,13 @@ func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb
// xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial
// tracked state defined by initialTrackedKeys.
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, at1,at2 zodb.Tid, xkv1,xkv2 RBucketSet, δZ *zodb.EventCommit, d12 map[Key]Δstring, initialTrackedKeys SetKey, kadj KAdjMatrix) {
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *tTreeCommit, initialTrackedKeys SetKey, kadj KAdjMatrix) {
X := exc.Raiseif
//t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
δZ := t2.δZ
d12 := t2.δxkv
var TrackedδZ SetKey = nil
var kadjTrackedδZ SetKey = nil
var δT, δTok map[Key]Δstring = nil, nil
......@@ -733,18 +727,18 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// δbtail @at1 with initial tracked set
δbtail := NewΔBtail(at1, db)
δbtail := NewΔBtail(t1.at, db)
xtrackKeys(δbtail, treeRoot, initialTrackedKeys)
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed)
TrackedδZ = SetKey{}
for k := range initialTrackedKeys {
leaf1 := xkv1.Get(k)
leaf1 := t1.xkv.Get(k)
oid1 := leaf1.oid
if oid1 == zodb.InvalidOid { // embedded bucket
oid1 = leaf1.parent.oid
}
leaf2 := xkv2.Get(k)
leaf2 := t2.xkv.Get(k)
oid2 := leaf2.oid
if oid2 == zodb.InvalidOid { // embedded bucket
oid2 = leaf2.parent.oid
......@@ -782,8 +776,8 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// trackSet1 = xkv1[tracked1]
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
trackSet1, tkeyCov1 := t1.xkv.trackSetWithCov(initialTrackedKeys)
trackSet2, tkeyCov2 := t2.xkv.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
// verify δbtail.trackSet against @at1
δbtail.assertTrack(t, "1", ø, trackSet1)
......@@ -845,7 +839,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// δT <- δB
δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid
δT = XGetδKV(db, at1,at2, δToid) // {} k -> δ(ZBlk(oid).data)
δT = XGetδKV(t1,t2, δToid) // {} k -> δ(ZBlk(oid).data)
// δT must be subset of d12.
// changed keys, that are
......@@ -988,7 +982,7 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t0, t1, xat,
/*trackSet=*/ø,
/*vδT=ø*/)
xverifyΔBTail_rebuild_TR(t, db, δbtail, t1, treeRoot, xat,
xverifyΔBTail_rebuild_TR(t, δbtail, t1, treeRoot, xat,
// after Track(keys1)
keys1,
/*trackSet=*/ ø,
......@@ -1083,7 +1077,7 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
// t.Run is expensive at this level of nest
//t.Run(" T"+keys2.String()+";R", func(t *testing.T) {
δbtail_ := δbtail.Clone()
xverifyΔBTail_rebuild_TR(t, db, δbtail_, t2, treeRoot, xat,
xverifyΔBTail_rebuild_TR(t, δbtail_, t2, treeRoot, xat,
// after Track(keys2)
keys2,
/*trackSet*/ t2.xkv.trackSet(keys1R2),
......@@ -1140,7 +1134,7 @@ func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid,
}
δToid, ok := δB.ΔByRoot[treeRoot]
if ok {
δT = XGetδKV(δbtail.db, ti.at, tj.at, δToid)
δT = XGetδKV(ti, tj, δToid)
}
if δB.Rev != tj.at {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.at)
......@@ -1154,7 +1148,7 @@ func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid,
}
// xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys SetKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet, vδTok ...map[Key]Δstring) {
func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys SetKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
ø := PPTreeSubSet{}
......@@ -1183,6 +1177,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
l := len(vδTok)
var vatOK []zodb.Tid
var vδTok_ []map[Key]Δstring
at2t := map[zodb.Tid]*tTreeCommit{tj.at: tj}
t0 := tj
for i := 0; i<l; i++ {
// empty vδTok entries means they should be absent in vδT
......@@ -1191,6 +1186,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
}
t0 = t0.prev
at2t[t0.at] = t0
}
vδTok = vδTok_
δTtail, ok := δbtail.vδTbyRoot[treeRoot]
......@@ -1205,7 +1201,7 @@ func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *tTreeCommit,
atPrev := t0.at
for _, δToid := range vδToid {
vat = append(vat, δToid.Rev)
δT := XGetδKV(δbtail.db, atPrev, δToid.Rev, δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
vδT = append(vδT, δT)
atPrev = δToid.Rev
}
......@@ -1382,6 +1378,7 @@ type tTreeCommit struct {
δZ *zodb.EventCommit // raw ZODB changes; δZ.tid == at
xkv RBucketSet // full tree state as of @at
δxkv map[Key]Δstring // full tree-diff against parent
blkDataTab map[zodb.Oid]string // full snapshot of all ZBlk data @at
}
// tZODBCacheEverything is workaround for ZODB/go not implementing real
......@@ -1393,6 +1390,68 @@ func (_ *tZODBCacheEverything) PCacheClassify(_ zodb.IPersistent) zodb.PCachePol
return zodb.PCachePinObject | zodb.PCacheKeepState
}
// XGetBlkDataTab loads all ZBlk from db@at.
//
// it returns {} oid -> blkdata.
func XGetBlkDataTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]string {
defer exc.Contextf("%s: @%s: get blkdatatab", db.Storage().URL(), at)
X := exc.Raiseif
blkDataTab := map[zodb.Oid]string{}
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at}); X(err)
xzroot, err := zconn.Get(ctx, 0); X(err)
zroot, ok := xzroot.(*zodb.Map)
if !ok {
exc.Raisef("root: expected %s, got %s", xzodb.TypeOf(zroot), xzodb.TypeOf(xzroot))
}
err = zroot.PActivate(ctx); X(err)
defer zroot.PDeactivate()
xzblkdir, ok := zroot.Data["treegen/values"]
if !ok {
exc.Raisef("root['treegen/values'] missing")
}
zblkdir, ok := xzblkdir.(*zodb.Map)
if !ok {
exc.Raisef("root['treegen/values']: expected %s, got %s", xzodb.TypeOf(zblkdir), xzodb.TypeOf(xzblkdir))
}
err = zblkdir.PActivate(ctx); X(err)
defer zblkdir.PDeactivate()
for k, xzblk := range zblkdir.Data {
zblk, ok := xzblk.(zodb.IPersistent)
if !ok {
exc.Raisef("root['treegen/values'][%q]: expected %s, got %s", k, xzodb.TypeOf(zblk), xzodb.TypeOf(xzblk))
}
oid := zblk.POid()
data := xzgetBlkData(ctx, zconn, oid)
blkDataTab[oid] = data
}
return blkDataTab
}
// xgetBlkData loads blk data for ZBlk<oid> @t.at
//
// For speed the load is done via preloaded t.blkDataTab instead of access to the DB.
func (t *tTreeCommit) xgetBlkData(oid zodb.Oid) string {
if oid == VDEL {
return DEL
}
data, ok := t.blkDataTab[oid]
if !ok {
exc.Raisef("getBlkData ZBlk<%s> @%s: no such ZBlk", oid, t.at)
}
return data
}
// testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq.
func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
X := exc.Raiseif
......@@ -1478,6 +1537,7 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
at: δZ.Tid,
δZ: δZ,
xkv: xkv,
blkDataTab: XGetBlkDataTab(db, δZ.Tid),
}
}
......@@ -1487,6 +1547,7 @@ func testΔBTail(t *testing.T, testq chan ΔBTestEntry) {
prev: nil,
at: tg.head,
xkv: XGetTree(db, tg.head, tg.treeRoot),
blkDataTab: XGetBlkDataTab(db, tg.head),
δZ: nil,
δxkv: nil,
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment