Commit a8595565 authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: tests: Treegen functionality

treegen.go and treegen.py together provide a way

- to commit a particular BTree topology into ZODB, and
- to generate set of random tree topologies that all correspond to particular {k->v} dict.

this will be used in upcoming ΔBtail and ΔFtail tests.

See treegen.py documentation for details.

Some preliminary history:

kirr/wendelin.core@9eca74ec    X Teach AllStructs to emit topologies with values
kirr/wendelin.core@1b962f03    X Restructure: found bug that it was not marking objects as modified
kirr/wendelin.core@2139af2c    X treegen: Verify that tree actually saved to storage is what was requested
kirr/wendelin.core@b5e39d4a    X wcfs/treegen: allstructs: Do not keep all tree structures in memory
kirr/wendelin.core@e9c4b619    X rebuild: tests: Random testing
kirr/wendelin.core@c32055fc    X wcfs/xbtree: ΔBtail tests += ø -> Tree; Tree -> ø
kirr/wendelin.core@4300d88a    X wcfs/xbtreetest/treegen.py: Fix it on ZODB4
parent 37fb6d28
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
import (
"fmt"
"sort"
"strings"
)
// KVTxt returns string representation of {} kv.
func KVTxt(kv map[Key]string) string {
if len(kv) == 0 {
return "ø"
}
keyv := []Key{}
for k := range kv { keyv = append(keyv, k) }
sort.Slice(keyv, func(i,j int) bool { return keyv[i] < keyv[j] })
sv := []string{}
for _, k := range keyv {
v := kv[k]
if strings.ContainsAny(v, " \n\t,:") {
panicf("[%v]=%q: invalid value", k, v)
}
sv = append(sv, fmt.Sprintf("%v:%s", k, v))
}
return strings.Join(sv, ",")
}
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
import (
"testing"
)
func TestKVTxt(t *testing.T) {
kv := map[Key]string{3:"hello", 1:"zzz", 4:"world"}
got := KVTxt(kv)
want := "1:zzz,3:hello,4:world"
if got != want {
t.Fatalf("error:\ngot: %q\nwant: %q", got, want)
}
}
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package xbtreetest
// treegen.go provides functionality:
//
// - to commit a particular BTree topology into ZODB, and
// - to generate set of random tree topologies that all correspond to particular {k->v} dict.
//
// treegen.py is used as helper for both tasks.
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"lab.nexedi.com/kirr/go123/my"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
argv []string
pysrv *exec.Cmd // spawned `treegen ...`
pyin io.WriteCloser // input to pysrv
pyoutRaw io.ReadCloser // output from pysrv
pyout *bufio.Reader // buffered ^^^
}
// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
*TreeGenSrv
zurl string
treeRoot zodb.Oid // oid of the tree treegen works on
head zodb.Tid // last made commit
}
// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
*TreeGenSrv
}
// startTreeGenSrv spawns `treegen ...` server.
func startTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
defer xerr.Contextf(&err, "treesrv %v: start", argv)
// spawn `treegen ...`
tg := &TreeGenSrv{argv: argv}
tg.pysrv = exec.Command(filepath.Dir(my.File())+"/treegen.py", argv...)
tg.pyin, err = tg.pysrv.StdinPipe()
if err != nil {
return nil, "", err
}
tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
if err != nil {
return nil, "", err
}
tg.pyout = bufio.NewReader(tg.pyoutRaw)
tg.pysrv.Stderr = os.Stderr // no redirection
err = tg.pysrv.Start()
if err != nil {
return nil, "", err
}
// wait for hello message and return it
defer func() {
if err != nil {
tg.Close() // ignore error
}
}()
defer xerr.Context(&err, "handshake")
hello, err = tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, "", err
}
hello = strings.TrimSuffix(hello, "\n")
return tg, hello, nil
}
// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
defer xerr.Contextf(&err, "treegen %v: close", tg.argv)
err1 := tg.pyin.Close()
err2 := tg.pyoutRaw.Close()
err3 := tg.pysrv.Wait()
return xerr.Merge(err1, err2, err3)
}
// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
tgSrv, hello, err := startTreeGenSrv("trees", zurl)
if err != nil {
return nil, err
}
tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
// tree.srv start @<at> tree=<root>
defer xerr.Contextf(&err, "invalid hello %q", hello)
startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
m := startRe.FindStringSubmatch(hello)
if m == nil {
return nil, fmt.Errorf("unexpected format")
}
tg.head, err = zodb.ParseTid(m[1]) // <at>
if err != nil {
return nil, fmt.Errorf("tid: %s", err)
}
tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
if err != nil {
return nil, fmt.Errorf("root: %s", err)
}
return tg, nil
}
// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
defer xerr.Context(&err, "allstructs.srv: start")
tgSrv, hello, err := startTreeGenSrv("allstructs")
if err != nil {
return nil, err
}
sg := &AllStructsSrv{TreeGenSrv: tgSrv}
defer func() {
if err != nil {
tgSrv.Close() // ignore error
}
}()
defer xerr.Contextf(&err, "invalid hello %q", hello)
if hello != "# allstructs.srv start" {
return nil, fmt.Errorf("unexpected format")
}
return sg, nil
}
// Commit creates new commit with underlying tree changed to specified tree topology.
//
// The following topologies are treated specially:
//
// - "ø" requests deletion of the tree, and
// - "øf" requests deletion of ZBigFile, that is referencing the tree.
// - "t<kv> D<data-kv>" changes both tree natively, and ZBlk data objects.
//
// see treegen.py for details.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)
_, err = io.WriteString(tg.pyin, tree + "\n")
if err != nil {
return zodb.InvalidTid, err
}
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return zodb.InvalidTid, err
}
reply = strings.TrimSuffix(reply, "\n")
tid, err := zodb.ParseTid(reply)
if err != nil {
return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
}
tg.head = tid
return tid, nil
}
// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, KVTxt(kv))
defer xerr.Contextf(&err, "allstructs.srv: %s ", req)
_, err = io.WriteString(tg.pyin, req + "\n")
if err != nil {
return nil, err
}
structv := []string{}
for {
reply, err := tg.pyout.ReadString('\n')
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
reply = strings.TrimSuffix(reply, "\n")
if reply == "# ----" {
return structv, nil // end of response
}
if strings.HasPrefix(reply, "#") {
continue // comment
}
structv = append(structv, reply)
}
}
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2020-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Program treegen provides infrastructure to generate ZODB BTree states.
It will be used as helper for ΔBtail and ΔFtail tests.
The following subcommands are provided:
- `trees` transition ZODB tree through requested tree states,
- `allstructs` generate subset of all possible tree topologies for a tree
specified by key->value dict.
Because python/pkg_resources startup is very slow(*) all subcommands can be
used either in CLI or in server mode, where requests are continuously read from
stdin.
Subcommands documentation follows:
trees
-----
`treegen trees <zurl>` transitions ZODB LOBTree through requested tree states.
Tree states are specified on stdin as topology-encoded strings(+), 1 state per 1 line.
For every request the tree is changed to have specified keys, values and
topology, and modifications are committed to database. For every made commit
corresponding transaction ID is printed to stdout.
The tree, that `treegen trees` generates and works on, is accessible via
zconn.root()['treegen/tree'].
Trees protocol specification:
S: tree.srv start @<head> root=<tree-root-oid>
C: <tree₁>
S: <tid₁>
C: <tree₂>
S: <tid₂>
...
session example:
S: tree.srv start @03d85dd71ed0d2ee root=000000000000000b
C: T/B1:a
S: 03d85dd84fed7844
C: T2/B1:a-B3:c
S: 03d85dd871718899
...
The following topology is treated specially:
- "ø" requests deletion of the tree.
allstructs
----------
`treegen allstructs` generates subset of all possible tree topologies for a tree
specified by key->value dict.
Given kv the following tree topologies are considered: 1) native (the one
that ZODB would usually create natively via regular usage), and 2) n-1 random
ones. Then those tree topologies are emitted.
The output of `treegen allstructs` is valid input for `treegen trees`.
Allstructs protocol specification:
S: # allstructs.srv start
C: <maxdepth> <maxsplit> <n>(/<seed>) <kv>
S: # allstructs <kv>
S: # maxdepth=<maxdepth> maxsplit=<maxsplit> n=<n> seed=<seed>
S: <tree₀>
S: <tree₁>
S: <tree₂>
...
S: # ----
session example:
# allstructs.srv start
1 1 10 1:a,2:b,3:c
# allstructs 1:a,2:b,3:c
# maxdepth=1 maxsplit=1 n=10 seed=1624901326
T2/B1:a-B2:b,3:c
T3/B1:a,2:b-B3:c
T2/T-T3/B1:a-B2:b-B3:c
T/T3/B1:a,2:b-B3:c
T/T2/B1:a-B2:b,3:c
T/B1:a,2:b,3:c
T3/T2-T/B1:a-B2:b-B3:c
T2/T-T/B1:a-B2:b,3:c
T/T/B1:a,2:b,3:c
T3/T-T/B1:a,2:b-B3:c
# ----
ΔFtail support
--------------
In addition to pure-tree mode, the following topologies are treated specially
by `treegen trees`:
- "øf" requests deletion of ZBigFile, that is referencing the tree,
- "t<kv> D<data-kv>" changes both tree natively, and ZBlk data objects
referenced via root['treegen/values'].
--------
(*) 300-500ms, see https://github.com/pypa/setuptools/issues/510.
(+) see wcfs/internal/xbtree.py
"""
from __future__ import print_function, absolute_import
import sys
from golang import func, defer, panic
from golang import time
from ZODB import DB
from ZODB.Connection import Connection
# XXX vvv way interferes with recover / sys.exc_info on py2
# TODO -> fix recover on py2
"""
try:
from ZODB.Connection import TransactionMetaData
except ImportError: # ZODB4
TransactionMetaData = None
"""
import ZODB.Connection
TransactionMetaData = getattr(ZODB.Connection, 'TransactionMetaData', None)
from ZODB.MappingStorage import MappingStorage
import transaction
import itertools
import random
import six
from wendelin.wcfs.internal import xbtree, xbtree_test
from wendelin.bigfile.file_zodb import ZBlk, ZBigFile
from zodbtools.util import storageFromURL, ashex
from persistent import CHANGED
from persistent.mapping import PersistentMapping
# XXX hack: set LOBTree.LOBTree -> XLOTree so that nodes are split often for
# regular tree updates (XLOTree is LOBTree with small .max_*_size). Do it this
# way so that generated database looks as if regular LOBTree was used. We use
# the hack because we cannot tune LOBTree directly.
XLOTree = xbtree_test.XLOTree
XLOTree.__module__ = 'BTrees.LOBTree'
XLOTree.__name__ = 'LOBTree'
import BTrees.LOBTree
BTrees.LOBTree.LOBTree = XLOTree
from BTrees.LOBTree import LOBTree
# ZCtx represent treegen-level connection to ZODB.
# It wraps zconn + provides treegen-specif integration.
class ZCtx(object):
# .zconn
# .root
# .valdict = zconn.root['treegen/values'] = {} v -> ZBlk(v)
# ZCtx(zstor) opens connection to zstor and initializes .valdict.
def __init__(zctx, zstor):
zctx.db = DB(zstor)
zctx.zconn = zctx.db.open()
zctx.root = zctx.zconn.root()
# init root['treegen/values'] = {} v -> ZBlk(v)
valdict = zctx.root.get('treegen/values', None)
if valdict is None:
valdict = zctx.root['treegen/values'] = PersistentMapping()
valv = b'abcdefghij'
for v in valv:
zblk = valdict.get(v, None)
if zblk is not None and zblk.loadblkdata() == v:
continue
zblk = ZBlk()
zblk.setblkdata(v)
valdict[v] = zblk
zctx.valdict = valdict
commit('treegen/values: init %r' % valv, skipIfEmpty=True)
# close releases resources associated with zctx.
def close(zctx):
zctx.zconn.close()
zctx.db.close()
# vdecode(vtxt) -> vobj decodes value text into value object, e.g. 'a' -> ZBlk(a)
# vencode(vobj) -> vtxt encodes value object into value text, e.g. ZBlk(a) -> 'a'
def vdecode(zctx, vtxt): # -> vobj
return zctx.valdict[vtxt]
def vencode(zctx, vobj): # -> vtxt
for (k,v) in zctx.valdict.items():
if v is vobj:
return k
raise KeyError("%r not found in value registry" % (vobj,))
# TreesSrv transitions ZODB tree through requested tree states.
# See top-level documentation for details.
@func
def TreesSrv(zstor, r):
zctx = ZCtx(zstor)
defer(zctx.close)
ztree = zctx.root['treegen/tree'] = LOBTree()
zfile = zctx.root['treegen/file'] = ZBigFile(blksize=4) # for ΔFtail tests
zfile.blktab = ztree
zdummy = zctx.root['treegen/dummy'] = PersistentMapping() # anything for ._p_changed=True
head = commit('treegen/tree: init')
xprint("tree.srv start @%s root=%s" % (ashex(head), ashex(ztree._p_oid)))
treetxtPrev = zctx.ztreetxt(ztree)
for treetxt in xreadlines(r):
subj = "treegen/tree: %s" % treetxt
# ø commands to delete the tree
if treetxt == "ø":
head = commitDelete(ztree, subj)
xprint("%s" % ashex(head))
continue
# øf command to delete the file
if treetxt == "øf":
head = commitDelete(zfile, subj)
xprint("%s" % ashex(head))
continue
# make sure we continue with undeleted ztree/zfile
if deleted(ztree):
undelete(ztree)
if deleted(zfile):
undelete(zfile)
# t... D... commands to natively commit updates to tree and values
if treetxt.startswith('t'):
t, D = treetxt.split()
assert D.startswith('D')
kv = kvDecode(t[1:], zctx.vdecode)
zv = _kvDecode(D[1:], kdecode=lambda ktxt: ktxt, vdecode=lambda vtxt: vtxt)
patch(ztree, diff(ztree, kv), kv)
# ~ patch(valdict, diff(valdict,zv)) but sets zblk.value on change
valdict = zctx.root['treegen/values']
vkeys = set(valdict.keys())
vkeys.update(zv.keys())
for k in vkeys:
zblk = valdict.get(k)
v1 = None
if zblk is not None:
v1 = zblk.loadblkdata()
v2 = zv.get(k)
if v1 != v2:
if v1 is None:
zblk = ZBlk()
valdict[k] = zblk
if v2 is not None:
zblk.setblkdata(v2)
zblk._p_changed = True
elif v2 is None:
del valdict[k]
zdummy._p_changed = True # alayws non-empty commit
head = commit(subj)
xprint("%s" % ashex(head))
continue
# everything else is considerd to be a tree topology
# mark something as changed if the same topology is requested twice.
# this ensures we can actually make a non-empty commit
if treetxt == treetxtPrev:
zdummy._p_changed = True
treetxtPrev = treetxt
tree = zctx.TopoDecode(treetxt)
# treekv[k]=v for all k
# do tree.keys() via walkBFS
treekv = {}
for level in xbtree._walkBFS(tree):
for node in level:
if isinstance(node, xbtree.Bucket):
if node.valuev is None:
panic("%s: tree must be {key->value}, not set" % treetxt)
assert len(node.keyv) == len(node.valuev)
for (k,v) in zip(node.keyv, node.valuev):
treekv[k] = v
# change ztree to requested kv
d = diff(ztree, treekv)
patch(ztree, d, treekv)
# restructure ztree to requested topology
xbtree.Restructure(ztree, tree)
# commit tree to storage
head = commit(subj)
# verify what was persisted to storage is indeed what we wanted to persist
zctx.zconn.cacheMinimize()
treetxt_onstor = zctx.ztreetxt(ztree)
if treetxt_onstor != treetxt:
panic("BUG: tree wrongly saved to storage:\nsaved: %s\nrequested: %s" %
(treetxt_onstor, treetxt))
# verify saved tree consistency
xbtree.zcheck(ztree)
# ok
xprint("%s" % ashex(head))
# AllStructsSrv is server version of AllStructs.
@func
def AllStructsSrv(r):
xprint('# allstructs.srv start')
for req in xreadlines(r):
# maxdepth maxsplit n(/seed) kv
maxdepth, maxsplit, n, kvtxt = req.split()
maxdepth = int(maxdepth)
maxsplit = int(maxsplit)
seed = None
if '/' in n:
n, seeds = n.split('/')
seed = int(seeds)
n = int(n)
if kvtxt == 'ø': kvtxt = ''
AllStructs(kvtxt, maxdepth, maxsplit, n, seed)
xprint('# ----')
# AllStructs generates subset of all possible topologies for a tree specified by kv dict.
# See top-level documentation for details.
@func
def AllStructs(kvtxt, maxdepth, maxsplit, n, seed=None):
zstor = MappingStorage() # in RAM storage to create native ZODB topologies
zctx = ZCtx(zstor)
defer(zctx.close)
kv = kvDecode(kvtxt, zctx.vdecode)
print("# allstructs %s" % kvtxt)
# create the tree
ztree = zctx.root['ztree'] = LOBTree()
commit('init')
# initial kv state with topology prepared as ZODB would do natively
patch(ztree, diff({}, kv), verify=kv)
if kv == {}: ztree._p_changed = True # to avoid empty commit - see TreesSrv
commit('kv')
tstruct0 = xbtree.StructureOf(ztree)
# seed
if seed is None:
seed = time.now()
seed = int(seed)
random.seed(seed)
print("# maxdepth=%d maxsplit=%d n=%d seed=%d" % (maxdepth, maxsplit, n, seed))
# emit native + n-1 random samples from all tree topologies that can represent kv
tstructv = rsample(xbtree.AllStructs(kv.keys(), maxdepth, maxsplit, kv=kv), n-1)
if tstruct0 in tstructv: tstructv.remove(tstruct0) # avoid dups
tstructv.insert(0, tstruct0)
for tstruct in tstructv:
print(zctx.TopoEncode(tstruct))
# rsample returns k random samples from seq.
# it differs from random.sample in that it does not keep whole list(seq) in memory.
def rsample(seq, k): # -> [] of items; len <= k
# based on https://stackoverflow.com/a/35671225/9456786
# https://en.wikipedia.org/wiki/Reservoir_sampling
if k <= 0:
raise ValueError("negative sample size")
it = iter(seq)
sample = list(itertools.islice(it, k))
random.shuffle(sample)
i = k
for item in it:
i += 1
j = random.randrange(i) # [0,i)
if j < k:
sample[j] = item
return sample
# kvEncode encodes key->value mapping into text.
# e.g. {1:'a', 2:'b'} -> '1:a,2:b'
def kvEncode(kvDict, vencode): # -> kvText
retv = []
for k in sorted(kvDict.keys()):
v = kvDict[k]
retv.append('%d:%s' % (k, vencode(v)))
return ','.join(retv)
# kvDecode decodes key->value mapping from text.
# e.g. '1:a,2:b' -> {1:'a', 2:'b'}
def kvDecode(kvText, vdecode): # -> kvDict
return _kvDecode(kvText, int, vdecode)
def _kvDecode(kvText, kdecode, vdecode): # -> kvDict
if kvText in ("", "ø"):
return {}
kv = {}
for item in kvText.split(','):
ktxt, vtxt = item.split(':')
k = kdecode(ktxt)
v = vdecode(vtxt)
if k in kv:
raise ValueError("key %s present multiple times" % k)
kv[k] = v
return kv
# diff computes difference in between mappings d1 and d2.
DEL = 'ø'
def diff(d1, d2): # -> [] of (k,v) to change; DEL means del[k]
delta = []
keys = set(d1.keys())
keys.update(d2.keys())
for k in sorted(keys):
v1 = d1.get(k, DEL)
v2 = d2.get(k, DEL)
if v1 is not v2:
delta.append((k,v2))
return delta
# patch changes mapping d according to diff.
# diff = [] of (k,v) to change; DEL means del[k]
def patch(d, diff, verify):
for (k,v) in diff:
if v == DEL:
del d[k]
else:
d[k] = v
if verify is None:
return
keys = set(d.keys())
keyok = set(verify.keys())
if keys != keyok:
panic("patch: verify: different keys: %s" % keys.symmetric_difference(keyok))
for k in keys:
if d[k] is not verify[k]:
panic("patch: verify: [%d] different: got %r; want %r" % (k, d[k], verify[k]))
# commit commits current transaction with description.
def commit(description, skipIfEmpty=False): # -> tid | None
txn = transaction.get()
if skipIfEmpty and len(txn._resources) == 0:
return None
txn.description = description
# XXX hack to retrieve committed transaction ID via ._p_serial of object changed in this transaction
assert len(txn._resources) == 1, txn._resources
zconn = txn._resources[0]
assert isinstance(zconn, Connection)
assert len(zconn._registered_objects) > 0
# NOTE objects in zconn._registered_objects are not necessarily in CHANGED state:
# https://github.com/zopefoundation/ZODB/blob/5.6.0-19-gdad778016/src/ZODB/Connection.py#L516-L530
changed_objects = [obj for obj in zconn._registered_objects if obj._p_state == CHANGED]
assert len(changed_objects) > 0
obj = changed_objects[0]
txn.commit()
return obj._p_serial
# commitDelete commits deletion of obj with description.
def commitDelete(obj, description): # -> tid
txn = transaction.get()
zstor = obj._p_jar._db.storage
# deleteObject works only at IStorage level, and at that low level
# zstor requires ZODB.IStorageTransactionMetaData not txn (ITransaction)
if TransactionMetaData is not None:
txn_stormeta = TransactionMetaData(txn.user, description, txn.extension)
else:
txn_stormeta = txn # ZODB4
zstor.tpc_begin(txn_stormeta)
zstor.deleteObject(obj._p_oid, obj._p_serial, txn_stormeta)
zstor.tpc_vote(txn_stormeta)
_ = []
zstor.tpc_finish(txn_stormeta, lambda tid: _.append(tid))
assert len(_) == 1, _
tid = _[0]
# object in the database is now current as of committed tid
# adjust obj's serial so that the next change to it (possibly recreating
# the object), does not fail with ConflictError.
obj._p_serial = tid
# reset transaction to a new one
transaction.begin()
obj._v_deleted = True
return tid
# deleted reports whether obj was deleted via commitDelete.
def deleted(obj): # -> bool
return getattr(obj, '_v_deleted', False)
# undelete forces recreation for obj that was previously deleted via commitDelete.
def undelete(obj):
obj._p_changed = True
del obj._v_deleted
# ztreetxt returns text representation of a ZODB tree.
@func(ZCtx)
def ztreetxt(zctx, ztree): # -> txt
assert ztree._p_jar is zctx.zconn
return zctx.TopoEncode(xbtree.StructureOf(ztree))
# TopoEncode and TopoDecode perform trees encode/decode with values registry
# taken from zctx.valdict.
@func(ZCtx)
def TopoEncode(zctx, tree):
return xbtree.TopoEncode(tree, zctx.vencode)
@func(ZCtx)
def TopoDecode(zctx, text):
return xbtree.TopoDecode(text, zctx.vdecode)
# xreadlines iterates through lines in r skipping comments.
def xreadlines(r):
while 1:
l = r.readline()
if l == '':
break # EOF
l = l.rstrip() # trim trailing \n
if l.startswith('#'):
continue # skip comments
yield l
@func
def cmd_allstructs(argv):
if argv == ['-h']:
print("Usage: cat requests |treegen allstructs", file=sys.stderr)
print(" treegen allstructs <request>", file=sys.stderr)
sys.exit(0)
r = sys.stdin
if len(argv) != 0:
r = six.StringIO(' '.join(argv))
AllStructsSrv(r)
@func
def cmd_trees(argv):
if argv in ([], ['-h']):
print("Usage: cat trees |treegen trees <zurl>", file=sys.stderr)
print(" treegen trees <zurl> <tree>+", file=sys.stderr)
sys.exit(1 if argv==[] else 0)
zurl = argv[0]
zstor = storageFromURL(zurl)
defer(zstor.close)
r = sys.stdin
treev = argv[1:]
if len(treev) != 0:
r = six.StringIO('\n'.join(treev))
TreesSrv(zstor, r)
# xprint prints msg to stdout and flushes it.
def xprint(msg):
print(msg)
sys.stdout.flush()
cmdRegistry = {
'allstructs': cmd_allstructs,
'trees': cmd_trees,
}
def main():
if len(sys.argv) < 2:
print("Usage: treegen <command> ...", file=sys.stderr)
sys.exit(1)
cmd = cmdRegistry.get(sys.argv[1], None)
if cmd is None:
print("E: treegen: unknown command %r" % sys.argv[1])
sys.exit(1)
argv = sys.argv[2:]
cmd(argv)
if __name__ == '__main__':
main()
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package xbtreetest provides infrastructure for testing LOBTree with ZBlk values.
package xbtreetest
import (
"fmt"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
)
// XXX instead of generics
type Tree = blib.Tree
type Bucket = blib.Bucket
type Node = blib.Node
type TreeEntry = blib.TreeEntry
type BucketEntry = blib.BucketEntry
type Key = blib.Key
type KeyRange = blib.KeyRange
const KeyMax = blib.KeyMax
const KeyMin = blib.KeyMin
func panicf(format string, argv ...interface{}) {
panic(fmt.Sprintf(format, argv...))
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment