# -*- coding: utf-8 -*-
# Copyright (C) 2020  Nexedi SA and Contributors.
#                     Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.

from __future__ import print_function, absolute_import

from wendelin.wcfs.internal import xbtree
from BTrees.LOBTree import LOBTree
from BTrees.IIBTree import IITreeSet, IISet
from BTrees.tests import testBTrees
from BTrees import check as zbcheck
from BTrees.check import BTREE_EMPTY, BTREE_ONE, BTREE_NORMAL
from ZODB.MappingStorage import MappingStorage
from ZODB import DB
from persistent import Persistent
import transaction
from pytest import raises
inf = float('inf')

# T/B are shorthands for Tree and Bucket without values.
# Bv is shorthand for Bucket with values.
T = xbtree.Tree
B = lambda *keyv: xbtree.Bucket(keyv, None)
Bv = lambda keyv, *valuev: xbtree.Bucket(keyv, valuev)

# buildDegenerateZTree builds ztree with known degenerate topology, see:
# https://github.com/zopefoundation/ZODB/commit/6cd24e99f89b
# https://github.com/zopefoundation/BTrees/blob/4.7.2-1-g078ba60/BTrees/tests/testBTrees.py#L20-L57
def buildDegenerateZTree():
    ztree, keys = testBTrees.DegenerateBTree("testBasicOps")._build_degenerate_tree()
    assert keys == [1, 3, 5, 7, 11]

    assert xbtree.StructureOf(ztree) == T([4],
                                          T([2],
                                               T([], B(1)), T([], B(3))),
                                          T([],
                                               T([6, 10],
                                                   T([], T([], B(5))),
                                                   T([], B(7)),
                                                   T([], B(11))) ))
    return ztree


def test_structureOf():
    # empty tree
    t = LOBTree()
    assert xbtree.StructureOf(t) == T([], Bv([]))

    # tree with 1 k->v
    t = LOBTree()
    t[10] = 'hello'
    assert xbtree.StructureOf(t) == T([], Bv([10], 'hello'))

    # known degenerate topology
    t = buildDegenerateZTree()
    assert xbtree.StructureOf(t) == T([4],
                                      T([2],
                                           T([], B(1)), T([], B(3))),
                                      T([],
                                           T([6, 10],
                                               T([], T([], B(5))),
                                               T([], B(7)),
                                               T([], B(11))) ))

def test_topoEncoding():
    def X(tree):
        topo = xbtree.TopoEncode(tree)
        t2   = xbtree.TopoDecode(topo)
        assert t2 == tree
        return topo

    assert X(T([], B()))            == 'T/B'
    assert X(T([], B(1)))           == 'T/B1'
    assert X(T([], B(1,3)))         == 'T/B1,3'
    assert X(T([], T([], B())))     == 'T/T/B'

    assert X(T([3],
               T([], B(1)),
               T([],
                 T([5], B(), B(7,8,9)))))   == "T3/T-T/B1-T5/B-B7,8,9"

    # degenerate btree from ZODB
    assert X(T([4],
               T([2],
                    T([], B(1)), T([], B(3))),
               T([],
                    T([6, 10],
                        T([], T([], B(5))),
                        T([], B(7)),
                        T([], B(11))) )))   == "T4/T2-T/T-T-T6,10/B1-B3-T-T-T/T-B7-B11/B5"

    # tree with key->value
    assert X(T([], Bv([])))             == 'T/B:'
    assert X(T([], Bv([1],   4)))       == 'T/B1:4'
    assert X(T([], Bv([1,2], 4,5)))     == 'T/B1:4,2:5'
    assert X(T([3],
               Bv([1],   10),
               Bv([4,5], 11,12)))       == 'T3/B1:10-B4:11,5:12'

    # TopoEncode/TopoDecode on autogenerated topologies.
    for tree in xbtree.AllStructs([1,3,7,8], 1,2, allowEmptyBuckets=True):
        t2 = xbtree.TopoDecode(xbtree.TopoEncode(tree))
        assert t2 == tree


def test_allStructs():
    # X = AllStructs(..., allowEmptyBuckets=True)
    # Y = AllStructs(..., allowEmptyBuckets=False)
    # XY = X = Y + assert X == Y
    def X(keys, maxdepth, maxsplit, allowEmptyBuckets=True, kv=None):
        return list(xbtree.AllStructs(keys, maxdepth, maxsplit, allowEmptyBuckets, kv))
    def Y(keys, maxdepth, maxsplit, kv=None):
        return X(keys, maxdepth, maxsplit, allowEmptyBuckets=False, kv=kv)
    def XY(keys, maxdepth, maxsplit, kv=None):
        x = X(keys, maxdepth, maxsplit, kv=kv)
        y = Y(keys, maxdepth, maxsplit, kv=kv)
        assert x == y
        return x

    assert XY([],   0, 0)   == [ T([], B()) ]

    assert XY([1],  0, 0)   == [ T([], B(1)) ]

    assert XY([1,3], 0, 0)  == [ T([], B(1,3)) ]

    assert XY([],   0, 1)   == [ T([], B()) ] # nothing to split


    assert X([],    1, 0)   == [ T([], B()),
                                 T([],
                                   T([], B())) ]
    assert Y([],    1, 0)   == [ T([], B()) ]

    assert X([],    2, 0)   == [ T([], B()),
                                 T([],
                                   T([], B())),
                                 T([],
                                   T([],
                                     T([], B()))) ]
    assert Y([],    2, 0)   == [ T([], B()) ]


    assert XY([1,3], 0, 0)  == [ T([], B(1,3)) ]

    assert X([1,3], 0, 1)   == [ T([], B(1,3)),
                                 # nsplit=1
                                 T([0], B(), B(1,3)),
                                 T([1], B(), B(1,3)),
                                 T([2], B(1), B(3)),
                                 T([3], B(1), B(3)),
                                 T([4], B(1,3), B()),
                               ]
    assert Y([1,3], 0, 1)   == [ T([], B(1,3)),
                                 # nsplit=1
                                 T([2], B(1), B(3)),
                                 T([3], B(1), B(3)),
                               ]

    assert XY([1,3], 1, 0)  == [ T([], B(1,3)),
                                 # depth=1
                                 T([],
                                   T([], B(1,3)))
                               ]

    assert X([1,3], 1, 1) == [
                               # T/
                               T([], B(1,3)),           # nsplit=0
                               T([],                    # nsplit=0,0
                                 T([], B(1,3))),
                               T([],                    # nsplit=0,1
                                 T([0], B(), B(1,3))),
                               T([],
                                 T([1], B(), B(1,3))),
                               T([],
                                 T([2], B(1), B(3))),
                               T([],
                                 T([3], B(1), B(3))),
                               T([],
                                 T([4], B(1,3), B())),

                               # T0/
                               T([0], B(), B(1,3)),     # nsplit=1
                               T([0],                   # nsplit=1,(0,0)
                                 T([], B()),
                                 T([], B(1,3))),
                               T([0],                   # nsplit=1,(0,1)
                                 T([],  B()),
                                 T([1], B(),  B(1,3))),
                               T([0],
                                 T([],  B()),
                                 T([2], B(1), B(3))),
                               T([0],
                                 T([],  B()),
                                 T([3], B(1), B(3))),
                               T([0],
                                 T([],  B()),
                                 T([4], B(1,3), B())),
                                                        # nsplit=1,(1,*) -> ø

                               # T1/
                               T([1], B(), B(1,3)),     # nsplit=1
                               T([1],                   # nsplit=1,(0,0)
                                 T([], B()),
                                 T([], B(1,3))),
                               T([1],
                                 T([],  B()),           # nsplit=1,(0,1)
                                 T([2], B(1), B(3))),
                               T([1],
                                 T([],  B()),
                                 T([3], B(1), B(3))),
                               T([1],
                                 T([],  B()),
                                 T([4], B(1,3), B())),
                               T([1],                   # nsplit=1,(1,0)
                                 T([0], B(),  B()),
                                 T([],  B(1,3))),
                               T([1],                   # nsplit=1,(1,1)
                                 T([0], B(),  B()),
                                 T([2], B(1), B(3))),
                               T([1],
                                 T([0], B(),  B()),
                                 T([3], B(1), B(3))),
                               T([1],
                                 T([0], B(),  B()),
                                 T([4], B(1,3), B())),

                               # T2/
                               T([2], B(1), B(3)),      # nsplit=1
                               T([2],                   # nsplit=1,(0,0)
                                 T([], B(1)),
                                 T([], B(3))),
                               T([2],                   # nsplit=1,(0,1)
                                 T([],  B(1)),
                                 T([3], B(),  B(3))),
                               T([2],
                                 T([],  B(1)),
                                 T([4], B(3), B())),
                               T([2],                   # nsplit=1,(1,0)
                                 T([0], B(),  B(1)),
                                 T([],  B(3))),
                               T([2],                   # nsplit=1,(1,1)
                                 T([0], B(),  B(1)),
                                 T([3], B(),  B(3))),
                               T([2],
                                 T([0], B(),  B(1)),
                                 T([4], B(3), B())),
                               T([2],                   # nsplit=1,(1,0)
                                 T([1], B(),  B(1)),
                                 T([],  B(3))),
                               T([2],                   # nsplit=1,(1,1)
                                 T([1], B(),  B(1)),
                                 T([3], B(),  B(3))),
                               T([2],
                                 T([1], B(),  B(1)),
                                 T([4], B(3), B())),

                               # T3/
                               T([3], B(1), B(3)),      # nsplit=1
                               T([3],                   # nsplit=1,(0,0)
                                 T([], B(1)),
                                 T([], B(3))),
                               T([3],                   # nsplit=1,(0,1)
                                 T([],  B(1)),
                                 T([4], B(3), B())),
                               T([3],                   # nsplit=1,(1,0)
                                 T([0], B(),  B(1)),
                                 T([],  B(3))),
                               T([3],                   # nsplit=1,(1,1)
                                 T([0], B(),  B(1)),
                                 T([4], B(3), B())),
                               T([3],                   # nsplit=1,(1,0)
                                 T([1], B(),  B(1)),
                                 T([],  B(3))),
                               T([3],                   # nsplit=1,(1,1)
                                 T([1], B(),  B(1)),
                                 T([4], B(3), B())),
                               T([3],                   # nsplit=1,(1,0)
                                 T([2], B(1), B()),
                                 T([],  B(3))),
                               T([3],                   # nsplit=1,(1,1)
                                 T([2], B(1), B()),
                                 T([4], B(3), B())),

                               # T4/
                               T([4], B(1,3), B()),     # nsplit=1
                               T([4],                   # nsplit=1,(0,0)
                                 T([], B(1,3)),
                                 T([], B())),
                                                        # nsplit=1,(0,1) -> ø
                               T([4],                   # nsplit=1,(1,0)
                                 T([0], B(),  B(1,3)),
                                 T([],  B())),
                               T([4],
                                 T([1], B(),  B(1,3)),
                                 T([],  B())),
                               T([4],
                                 T([2], B(1), B(3)),
                                 T([],  B())),
                               T([4],
                                 T([3], B(1), B(3)),
                                 T([],  B())),
                                                        # nsplit=1,(1,1) -> ø
                              ]

    assert Y([1,3], 1, 1) == [
                               # T/
                               T([], B(1,3)),           # nsplit=0
                               T([],                    # nsplit=0,0
                                 T([], B(1,3))),
                               T([],                    # nsplit=0,1
                                 T([2], B(1), B(3))),
                               T([],
                                 T([3], B(1), B(3))),

                               # T0/
                               #    nothing - leftmost bucket is always empty

                               # T1/
                               #    nothing - leftmost bucket is always empty

                               # T2/
                               T([2], B(1), B(3)),      # nsplit=1
                               T([2],                   # nsplit=1,(0,0)
                                 T([], B(1)),
                                 T([], B(3))),

                               # T3/
                               T([3], B(1), B(3)),      # nsplit=1
                               T([3],                   # nsplit=1,(0,0)
                                 T([], B(1)),
                                 T([], B(3))),

                               # T4/
                               #    nothing - rightmost bucket is always empty
                              ]


    # TODO test for maxsplit=2 / maxdepth=2 vvv
    def TY(keys, maxdepth, maxsplit, kv=None):
        yv = Y(keys, maxdepth, maxsplit, kv=kv)
        return list([xbtree.TopoEncode(_, vencode=lambda v: v) for _ in yv])

    assert TY([1,3], 1, 1) == [
                                'T/B1,3',
                                'T/T/B1,3',
                                'T/T2/B1-B3',
                                'T/T3/B1-B3',
                                'T2/B1-B3',
                                'T2/T-T/B1-B3',
                                'T3/B1-B3',
                                'T3/T-T/B1-B3',
                              ]

    # with values
    assert TY([1,3], 1,1, kv={1:'a',3:'c'}) == [
                                'T/B1:a,3:c',
                                'T/T/B1:a,3:c',
                                'T/T2/B1:a-B3:c',
                                'T/T3/B1:a-B3:c',
                                'T2/B1:a-B3:c',
                                'T2/T-T/B1:a-B3:c',
                                'T3/B1:a-B3:c',
                                'T3/T-T/B1:a-B3:c',
                              ]


# XBlk simulates ZBlk without xbtree_test.py depending on file_zodb.py
class XBlk(Persistent):
    def __init__(self, data):
        self.data = data
    def __str__(self):
        return 'X%s' % self.data
    __repr__ = __str__

# XLOTree is like LOBTree but with small max tree and bucket node sizes.
# It's tree and bucket nodes are split often on regular tree updates.
class XLOTree(LOBTree):
    #_bucket_type = XLOBucket
    max_leaf_size     = 2
    max_internal_size = 2

zbcheck._type2kind[XLOTree] = (zbcheck.TYPE_BTREE, True)
zbcheck._btree2bucket[XLOTree] = XLOTree._bucket_type

def crack_btree(ztree):
    assert xbtree._zclassify(ztree).is_ztree, ztree
    return zbcheck.crack_btree(ztree, is_mapping=True)

def crack_bucket(zbucket):
    assert xbtree._zclassify(zbucket).is_zbucket, zbucket
    return zbcheck.crack_bucket(zbucket, is_mapping=True)

# assertT asserts that znode is normal tree node + has specified keys and children.
#
# by default children are checked exactly via "is"
# if a child is represented as 'T' or 'B' - it is only verified to be of tree
# or bucket type correspondingly.
def assertT(znode, keyv, *children): # -> [] of children marked with 'T'/'B'
    _ = xbtree._zclassify(znode)
    assert _.is_ztree
    kind, keys, kids = zbcheck.crack_btree(znode, _.is_map)
    assert kind == BTREE_NORMAL
    assert keys == keyv
    assert len(kids) == len(children)
    retv = []
    for (child, childOK) in zip(kids, children):
        if childOK == 'T':
            assert type(child) is type(znode)
            retv.append(child)
        elif childOK == 'B':
            assert type(child) is znode._bucket_type
            retv.append(child)
        else:
            assert child is childOK
    return retv

# assertB asserts that znode is bucket node with specified keys and values
def assertB(znode, *kvv):
    _ = xbtree._zclassify(znode)
    assert _.is_zbucket
    keys, values = zbcheck.crack_bucket(znode, _.is_map)
    if not _.is_map:
        assert values == []
        assert keys == kvv
    else:
        assert len(keys) == len(values)
        assert len(keys) == len(kvv)
        for (i,(k,v)) in enumerate(zip(keys, values)):
            kok, vok = kvv[i]
            assert k == kok
            assert v is vok

def test_restructure():
    # do restructure tests under ZODB because without ZODB connection it is not
    # always possible to __setstate__ for e.g. .../T/B. We also want to make
    # sure Restructure correctly marks modified nodes as changed so that the
    # changes are actually persisted to storage on commit.
    zstor = MappingStorage()
    db = DB(zstor)
    zconn = db.open()

    X = []  # X[i] -> XBlk corresponding to block #i
    xv = 'abcdefghijkl'
    for i in range(len(xv)):
        X.append(XBlk(xv[i]))
    def xdecode(v):
        assert len(v) == 1
        assert v in xv
        return X[xv.index(v)]
    def xencode(x):
        assert isinstance(x, XBlk)
        return x.data

    # assertB wraps global assertB to automatically fill in X[k] values for specified keys.
    def assertB(znode, *keyv):
        globals()['assertB'](znode, *[(k,X[k]) for k in keyv])

    # Z prepares XLOTree ztree with given keys via usual way.
    # the tree is setup as {} k -> X[k].
    def Z(*keys):
        ztree = XLOTree()
        zconn.add(ztree)
        for k in keys:
            ztree[k] = X[k]

        # check all keys via iterating (this verifies firstbucket and B->next pointers)
        keys2 = set(ztree.keys())
        assert keys2 == set(keys)
        # check all keys by [] access
        for k in keys:
            assert ztree[k] is X[k]

        return ztree

    # R restructures ztree to have specified new topology.
    # The result is committed unless dontcommit=Y specified.
    def R(ztree, newtopo, dontcommit=False):
        # verify ztree consistency
        items = list(ztree.items())
        for (k,v) in items:
            assert ztree[k] == v

        if isinstance(newtopo, str):
            newStructure = xbtree.TopoDecode(newtopo, xdecode)
        else:
            assert isinstance(newtopo, xbtree.Tree)
            newStructure = newtopo
        xbtree.Restructure(ztree, newStructure)

        if not dontcommit:
            transaction.commit()
        # force objects state to be reloaded from storage.
        # this leads further checks to also verify if Restructure modified a
        # node, but did not marked it as changed. If this bug is indeed there -
        # then the modifications will be lost after live cache clearance.
        zconn.cacheMinimize()

        assert xbtree.StructureOf(ztree, onlyKeys=True) == \
               newStructure.copy(onlyKeys=True)

        # verify iteration produces the same [] of (key, v)
        assert list(ztree.items()) == items
        # verify [k] gives the same v   (for all k)
        for (k,v) in items:
            assert ztree[k] == v

    # S returns topo-encoded keys-only structure of ztree.
    # Sv returns topo-encoded structure of ztree with values.
    def S(ztree):
        return xbtree.TopoEncode(xbtree.StructureOf(ztree, onlyKeys=True))
    def Sv(ztree):
        return xbtree.TopoEncode(xbtree.StructureOf(ztree), xencode)

    # Z0 creates new empty tree
    def Z0():
        z = Z()
        assert crack_btree(z) == (BTREE_EMPTY, [], [])
        return z

    # ---- tests with manual verification of resulting topology and nodes ----

    # ø -> T/B
    z = Z0()
    R(z, 'T/B')
    assert crack_btree(z) == (BTREE_EMPTY, [], [])

    with raises(ValueError, match="new keys != old keys"):
        R(z, 'T/B1')

    # ø -> T/T/B    (don't - we don't emit topologies with empty buckets for
    #                tests since ZODB breaks on them)
    """
    z = Z0()
    R(z, 'T/T/B')
    t, = assertT(z, [], 'T')
    b, = assertT(t, [], 'B')
    assertB(b)
    """

    # ø -> T/T-T/B-B    (don't - see ^^^)
    """
    z = Z0()
    R(z, 'T0/T-T/B-B')
    Tl, Tr = assertT(z, [0], 'T','T')
    bl, = assertT(Tl, [], 'B')
    br, = assertT(Tr, [], 'B')
    assertB(bl)
    assertB(br)
    """

    # tree with 1 k->v (not yet committed bucket)
    z = Z(1)
    assert crack_btree(z) == (BTREE_ONE, ((1, X[1]),), None)
    R(z, 'T/B1', dontcommit=True)
    assert crack_btree(z) == (BTREE_ONE, ((1, X[1]),), None)
    R(z, 'T/T/B1', dontcommit=True)
    t, = assertT(z, [], 'T')
    b1, = assertT(t, [], 'B')
    assertB(b1, 1)
    assert b1._p_oid is not None
    R(z, 'T/B1', dontcommit=True)
    assertT(z, [], b1)
    assertB(b1, 1)


    # tree with 2 k->v (not-yet committed bucket)
    z = Z(1,3)
    assert crack_btree(z) == (BTREE_ONE, ((1, X[1], 3, X[3]),), None)
    R(z, 'T2/B1-B3', dontcommit=True)
    b1, b3 = assertT(z, [2], 'B','B')
    assert b1._p_oid is None
    assert b3._p_oid is None
    assertB(b1, 1)
    assertB(b3, 3)
    R(z, 'T/B1,3')
    # buckets were not yet assigned oid -> collapsed back into T
    assert crack_btree(z) == (BTREE_ONE, ((1, X[1], 3, X[3]),), None)

    R(z, 'T3/B1-B3', dontcommit=True)
    b1, b3 = assertT(z, [3], 'B','B')
    assert b1._p_oid is None
    assert b3._p_oid is None
    assertB(b1, 1)
    assertB(b3, 3)

    transaction.commit()    # force buckets to be assigned oid
    assert b1._p_oid is not None
    assert b3._p_oid is not None

    # restructure back - buckets not collapsed back into T
    R(z, 'T/B1,3')
    b13, = assertT(z, [], 'B')
    assertB(b13, 1,3)


    # add 1 key -> B splits -> B + B
    assert S(z) == 'T/B1,3'
    z[5] = X[5]
    assert S(z) == 'T3/B1-B3,5'
    b1, b35 = assertT(z, [3], 'B','B')
    assertB(b1,  1)
    assertB(b35, 3,5)

    # -> T2/T-T/B1-B3,5         (add intermediate T-T level)
    R(z, 'T2/T-T/B1-B3,5')
    tl, tr = assertT(z, [2], 'T','T')
    assertT(tl, [], b1)
    assertT(tr, [], b35)
    assertB(b1,  1)
    assertB(b35, 3,5)

    # -> T2/T-T/B1-T/B3,5       (add intermediate T level in right arm)
    R(z, 'T2/T-T/B1-T/B3,5')
    assertT(z, [2], tl, tr)
    assertT(tl, [], b1)
    trr, = assertT(tr, [], 'T')
    assert isinstance(trr, XLOTree)
    assertT(trr, [], b35)
    assertB(b1,  1)
    assertB(b35, 3,5)

    # -> T2,4/B1-B3-B5          (kill intermediate trees, split B35->B3+B5)
    R(z, 'T2,4/B1-B3-B5')
    b3, = assertT(z, [2,4], b1,'B',b35)
    b5 = b35; del b35
    assertB(b1, 1)
    assertB(b3, 3)
    assertB(b5, 5)

    # -> T2/T-T4/B1-B3-B5       (add intermediate T-T4 level)
    R(z, 'T2/T-T4/B1-B3-B5')
    tl, tr = assertT(z, [2], 'T','T')
    assertT(tl, [],  b1)
    assertT(tr, [4], b3,b5)
    assertB(b1, 1)
    assertB(b3, 3)
    assertB(b5, 5)

    # -> T2/T-T/B1-T4/B3-B5     (add intermediate level in right arm)
    R(z, 'T2/T-T/B1-T4/B3-B5')
    tr, = assertT(z, [2], tl,'T')
    assertT(tl, [], b1)
    trr, = assertT(tr, [], 'T')
    assertT(trr, [4], b3,b5)
    assertB(b1, 1)
    assertB(b3, 3)
    assertB(b5, 5)

    # -> T/B1,3,5               (collapse into T/B)
    R(z, 'T/B1,3,5')
    assertT(z, [], b1)
    b135 = b1
    assertB(b135, 1,3,5)


    # grow the tree with four more keys (6,7,8,9) till top-level tree node splits
    assert S(z) == 'T/B1,3,5'
    z[6] = X[6]
    assert S(z) == 'T5/B1,3-B5,6'
    z[7] = X[7]
    assert S(z) == 'T5,6/B1,3-B5-B6,7'
    z[8] = X[8]
    assert S(z) == 'T6/T5-T7/B1,3-B5-B6-B7,8'

    # rotate keys in T and reflow B to the left
    tl, tr  = assertT(z,  [6], 'T','T')
    b13, b5 = assertT(tl, [5], 'B','B')
    b6, b78 = assertT(tr, [7], 'B','B')
    assertB(b13, 1,3)
    assertB(b5,  5)
    assertB(b6,  6)
    assertB(b78, 7,8)
    R(z, 'T7/T4,6-T/B1,3-B5-B6-B7,8')
    assertT(z,  [7],   tl,tr)
    assertT(tl, [4,6], b13,b5,b6)
    assertT(tr, [],    b78)
    assertB(b13, 1,3)
    assertB(b5,  5)
    assertB(b6,  6)
    assertB(b78, 7,8)

    # migrate keys in between buckets
    R(z, 'T6/T3-T8/B1-B3,5-B6,7-B8')
    assertT(z,  [6], tl,tr)
    assertT(tl, [3], b13,b5)
    assertT(tr, [8], b6,b78)
    b1  = b13; del b13
    b35 = b5;  del b5
    b67 = b6;  del b6
    b8  = b78; del b78
    assertB(b1,  1)
    assertB(b35, 3,5)
    assertB(b67, 6,7)
    assertB(b8,  8)


    # ---- new structure given with values ----
    z = Z(0,2)
    R(z, T([1], Bv([0],X[0]), Bv([2],X[2])))
    b0, b2 = assertT(z, [1], 'B','B')
    assertB(b0, 0)
    assertB(b2, 2)
    assert b0[0] is X[0]
    assert b2[2] is X[2]

    # [2] changes value from X[2] to X[3]
    with raises(ValueError, match=r"target bucket changes \[2\]"):
        R(z, T([1], Bv([0],X[0]), Bv([2],X[3])))


    # ---- tricky cases

    z = Z(0,1,2,3)
    R(z, 'T2/T1-T3/B0-B1-T-T/B2-B3')
    R(z, 'T2/T1-T/T-T-B2,3/B0-B1')

    # degenerate topology from ZODB example
    z = Z(1,3,5,7,11)
    R(z, 'T4/T2-T/T-T-T6,10/B1-B3-T-T-T/T-B7-B11/B5')
    R(z, 'T/B1,3,5,7,11')

    # verify that changed objects are marked as such and so included into commit
    # (just R also partly verifies this on every call)
    z = Z(0,2,3)
    transaction.commit()
    def Rz(newtopo):
        R(z, newtopo, dontcommit=True)
        transaction.commit()
        assert Sv(z) == newtopo
        zconn.cacheMinimize()    # force z state to be reloaded from storage
        assert Sv(z) == newtopo  # will fail if T or B is not marked as changed
        xbtree.zcheck(z)

    Rz('T/B0:a,2:c,3:d')
    Rz('T1/B0:a-B2:c,3:d')
    Rz('T2/B0:a-B2:c,3:d')
    Rz('T3/B0:a,2:c-B3:d')
    Rz('T2/T-T3/B0:a-B2:c-B3:d')

    # make sure that only modified nodes are marked as changed.
    z = Z(0,1,2,3)
    R(z, 'T1/T-T2/B0-B1-B2,3')
    tl, tr  = assertT(z,  [1], 'T','T')
    b0,     = assertT(tl, [],  'B')
    b1, b23 = assertT(tr, [2], 'B','B')
    assertB(b0,  0)
    assertB(b1,  1)
    assertB(b23, 2,3)
    assert z._p_changed     == False
    assert tl._p_changed    == False
    assert tr._p_changed    == False
    assert b0._p_changed    == False
    assert b1._p_changed    == False
    assert b23._p_changed   == False

    R(z, 'T1/T-T3/B0-B1,2-B3', dontcommit=True) # reflow right arm
    assertT(z,  [1], tl, tr)
    assertT(tl, [],  b0)
    assertT(tr, [3], b1, b23)   # changed
    assertB(b0,  0)
    assertB(b1,  1,2)           # changed
    assertB(b23, 3)             # changed
    assert z._p_changed     == False
    assert tl._p_changed    == False
    assert tr._p_changed    == True
    assert b0._p_changed    == False
    assert b1._p_changed    == True
    assert b23._p_changed   == True
    transaction.commit()



    # ---- tests on automatically generated topologies ----
    #
    # ( we make sure that Restructure can make the restructurement and that
    #   after restructure a tree remains valid without any error introduced )
    for nkeys in range(5):  # XXX !slow -> ↑
        for xkeyv in xbtree._iterSplitByN(-1, 5+1, nkeys):
            keyv = xkeyv[1:-1]  # -1, ..., N -> ...
            #print(keyv)
            z = Z(*keyv)
            # d  s   Nvariants  Ttest
            # 3  2   35·10³     40s
            # 3  1   18·10³     20s
            # 2  2    8·10³      8s
            # 2  1    3·103      4s
            # 1  1    1·10³      1s
            for tree in xbtree.AllStructs(keyv, 2, 1): # XXX !slow -> d=3, s=2
                #print('\t%s' % xbtree.TopoEncode(tree))
                R(z, tree)


def test_walkBFS():
    R = xbtree._Range

    # T/B
    b = B()
    t = T([], b)
    walkv = list(xbtree.__walkBFS(t))
    assert len(walkv) == 2          # (t) (b)

    _ = walkv[0]
    assert len(_) == 1
    assert _[0].range == R(-inf, inf)
    assert _[0].node  is t

    _ = walkv[1]
    assert len(_) == 1
    assert _[0].range == R(-inf, inf)
    assert _[0].node  is b


    # T0/T-T/B-B
    bl = B(); br = B()
    tl = T([], bl)
    tr = T([], br)
    t = T([0], tl, tr)

    walkv = list(xbtree.__walkBFS(t))
    assert len(walkv) == 3          # (t) (tl, tr), (bl, br)

    _ = walkv[0]
    assert len(_) == 1
    assert _[0].range == R(-inf, inf)
    assert _[0].node  is t

    _ = walkv[1]
    assert len(_) == 2
    assert _[0].range == R(-inf, 0)
    assert _[0].node  is tl

    assert _[1].range == R(0, inf)
    assert _[1].node  is tr

    _ = walkv[2]
    assert len(_) == 2
    assert _[0].range == R(-inf, 0)
    assert _[0].node  is bl
    assert _[1].range == R(0, inf)
    assert _[1].node  is br

    # XXX more tests?

def test_zwalkBFS():
    zt = buildDegenerateZTree()

    # assign oid to created objects to force btrees not to embed bucket state
    zstor = MappingStorage()
    db = DB(zstor)
    zconn = db.open()
    zroot = zconn.root()
    zroot['x'] = zt
    transaction.commit()

    def assertT(znode, keyv, *children):
        assert isinstance(znode, IITreeSet)
        return globals()['assertT'](znode, keyv, *children)

    # assertB asserts that znode is bucket + has specified keys
    def assertB(znode, *keyv):
        assert isinstance(znode, IISet)
        globals()['assertB'](znode, *keyv)

    R = xbtree._Range

    zwalkv = list(xbtree.__zwalkBFS(zt))
    assert len(zwalkv) == 6         # [-∞,∞)T4,
                                    #   [-∞,4)T2,  [4,∞)T
                                    #     [-∞,2)T, [2,4)T, [4,∞)T6,10
                                    #       [-∞,2)B1, [2,4)B3, [4,6)T, [6,10)T, [10,∞]T
                                    #         [4,6)T, [6,10)B7, [10,∞)B11
                                    #           [4,6)B5

    _ = zwalkv[5]   # [4,6)B5
    assert len(_) == 1
    assert _[0].range   == R(4,6)
    b5   = _[0].node;   assertB(b5, 5)

    _ = zwalkv[4]   # [4,6)T, [6,10)B7, [10,∞)B11
    assert len(_) == 3
    assert _[0].range   == R(4,6)
    assert _[1].range   == R(6,10)
    assert _[2].range   == R(10,inf)
    t4_b5= _[0].node;   assertT(t4_b5, [], b5)
    b7   = _[1].node;   assertB(b7, 7)
    b11  = _[2].node;   assertB(b11, 11)

    _ = zwalkv[3]   # [-∞,2)B1, [2,4)B3, [4,6)T, [6,10)T, [10,∞]T
    assert len(_) == 5
    assert _[0].range   == R(-inf,2)
    assert _[1].range   == R(2,4)
    assert _[2].range   == R(4,6)
    assert _[3].range   == R(6,10)
    assert _[4].range   == R(10,inf)
    b1       = _[0].node;   assertB(b1, 1)
    b3       = _[1].node;   assertB(b3, 3)
    t3_t4_b5 = _[2].node;   assertT(t3_t4_b5, [], t4_b5)
    t3_b7    = _[3].node;   assertT(t3_b7,    [], b7)
    t3_b11   = _[4].node;   assertT(t3_b11,   [], b11)

    _ = zwalkv[2]   # [-∞,2)T, [2,4)T, [4,∞)T6,10
    assert len(_) == 3
    assert _[0].range   == R(-inf,2)
    assert _[1].range   == R(2,4)
    assert _[2].range   == R(4,inf)
    t2_b1 = _[0].node;  assertT(t2_b1,  [], b1)
    t2_b3 = _[1].node;  assertT(t2_b3,  [], b3)
    t2_610= _[2].node;  assertT(t2_610, [6,10], t3_t4_b5, t3_b7, t3_b11)

    _ = zwalkv[1]   # [-∞,4)T2, [4,∞)T
    assert len(_) == 2
    assert _[0].range   == R(-inf, 4)
    assert _[1].range   == R(4, inf)
    t1_2       = _[0].node;   assertT(t1_2,      [2], t2_b1, t2_b3)
    t1_t2_610  = _[1].node;   assertT(t1_t2_610, [],  t2_610)

    _ = zwalkv[0]   # [-∞,∞)T4,
    assert len(_) == 1
    assert _[0].range   == R(-inf, inf)
    assertT(_[0].node, [4], t1_2, t1_t2_610)



def test_keyvSliceBy():
    X = xbtree._keyvSliceBy
    assert X([],  0,0)  == []
    assert X([1], 0,0)  == []
    assert X([1], 0,1)  == []
    assert X([1], 1,1)  == []
    assert X([1], 1,2)  == [1]

    assert X([1,3,5,10,17], 3,10) == [3,5]

def test_iterSplitByN():
    def X(lo, hi, nsplit):
        return tuple(xbtree._iterSplitByN(lo, hi, nsplit))

    assert X(0,0, 0)    == ( [0,0], )
    assert X(0,0, 1)    == ()
    assert X(0,1, 0)    == ( [0,1], )
    assert X(0,1, 1)    == ()

    assert X(0,2, 0)    == ( [0,2], )
    assert X(0,2, 1)    == ( [0,1,2], )
    assert X(0,2, 2)    == ()

    assert X(0,3, 0)    == ( [0,3], )
    assert X(0,3, 1)    == ( [0,1,3], [0,2,3] )
    assert X(0,3, 2)    == ( [0,1,2,3], )
    assert X(0,3, 3)    == ()

    assert X(0,4, 0)    == ( [0,4], )
    assert X(0,4, 1)    == ( [0,1,4], [0,2,4], [0,3,4] )
    assert X(0,4, 2)    == ( [0,1,2,4], [0,1,3,4], [0,2,3,4] )
    assert X(0,4, 3)    == ( [0,1,2,3,4], )
    assert X(0,4, 4)    == ()

    assert X(0,5, 0)    == ( [0,5], )
    assert X(0,5, 1)    == ( [0,1,5], [0,2,5], [0,3,5], [0,4,5] )
    assert X(0,5, 2)    == ( [0,1,2,5], [0,1,3,5], [0,1,4,5], [0,2,3,5], [0,2,4,5], [0,3,4,5] )
    assert X(0,5, 3)    == ( [0,1,2,3,5], [0,1,2,4,5], [0,1,3,4,5], [0,2,3,4,5] )
    assert X(0,5, 4)    == ( [0,1,2,3,4,5], )
    assert X(0,5, 5)    == ()

def test_iterSplitKeyvByN():
    keyv = [1,3,4]
    def X(lo, hi, nsplit):
        return tuple(xbtree._iterSplitKeyvByN(lo, hi, keyv, nsplit))

    assert X(0,7, 0)    == ( [0,7], )
    assert X(0,7, 1)    == ( [0,2,7], [0,3,7], [0,4,7] )
    assert X(0,7, 2)    == ( [0,2,4,7], [0,3,4,7] )
    assert X(0,7, 3)    == ()
