Commit 2a2b9e30 authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: xbtree/xbtreetest: Fix it on Python3

Treegen.py was using bytes valv and iterating it to get tree keys to
be set, but on py3 iterating bytes yields integer not bytes and so with
py3 e.g. TestΔBTail was failing as

    --- FAIL: TestΔBTail (1.21s)
    panic: root['treegen/values']: key ['a']: expected str, got int64 [recovered]
            panic: file:///tmp/TestΔBTail1569960846/001/1.fs: @03fb8a9c91bba733: get blktab: root['treegen/values']: key ['a']: expected str, got int64 [recovered]

-> Fix it by reworking treegen.py to operate in strings domain and
   adjusting treeenv.go loader to use pickle.AsString correspondingly.

   On py3 the implementation depends on nexedi/pygolang!21,
   but on py2 it works both with and without pygolang bstr patches.

Preliminary history:

    vnmabus/wendelin.core@eca099bc
    vnmabus/wendelin.core@5a2b639c
    vnmabus/wendelin.core@fadc5a07

but there we were operating in mixed string/bytes mode with cluttering
the code with .encode('ascii') calls and potentially missing to handle
some logic right because on py3 str == bytes returns False, not raises,
and with mixed types it is easy to miss some logic where conversion is
needed if the outcome depends on values comparison.

Switching to work in strings domain uniformly avoids those problem in
principle and by doing only one thing locally.
Co-authored-by: Carlos Ramos Carreño's avatarCarlos Ramos Carreño <carlos.ramos@nexedi.com>
parent bf816c40
......@@ -303,9 +303,9 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
defer zblkdir.PDeactivate()
zblkdir.Iter()(func(xname, xzblk any) bool {
name, ok := xname.(pickle.ByteString)
if !ok {
exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname)
name, err := pickle.AsString(xname)
if err != nil {
exc.Raisef("root['treegen/values']: key [%q]: %s", xname, err)
}
zblk, ok := xzblk.(zodb.IPersistent)
......@@ -315,7 +315,7 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
oid := zblk.POid()
data := xzgetBlkData(ctx, zconn, oid)
blkTab[oid] = ZBlkInfo{string(name), data}
blkTab[oid] = ZBlkInfo{name, data}
return true
})
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2020-2021 Nexedi SA and Contributors.
# Copyright (C) 2020-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -133,7 +133,7 @@ by `treegen trees`:
from __future__ import print_function, absolute_import
import sys
from golang import func, defer, panic
from golang import func, defer, panic, b
from golang import time
from ZODB import DB
from ZODB.Connection import Connection
......@@ -174,6 +174,21 @@ BTrees.LOBTree.LOBTree = XLOTree
from BTrees.LOBTree import LOBTree
# Treegen works in strings domain. To help this:
#
# - loadblkstr loads ZBlk data as string.
# - setblkstr sets ZBlk data from string.
#
# We and ΔBtail/ΔFtail tests store into ZBlks only small set of letters with
# optional digit suffix (e.g. "c" and "d4"), so using strings to handle ZBlk
# data is ok and convenient. Besides ZBlk everything else in treegen uses
# strings natively.
def loadblkstr(zblk): # -> bstr
return b(zblk.loadblkdata())
def setblkstr(zblk, strdata):
zblk.setblkdata(b(strdata))
# ZCtx represent treegen-level connection to ZODB.
# It wraps zconn + provides treegen-specif integration.
class ZCtx(object):
......@@ -191,13 +206,13 @@ class ZCtx(object):
valdict = zctx.root.get('treegen/values', None)
if valdict is None:
valdict = zctx.root['treegen/values'] = PersistentMapping()
valv = b'abcdefghij'
valv = 'abcdefghij'
for v in valv:
zblk = valdict.get(v, None)
if zblk is not None and zblk.loadblkdata() == v:
if zblk is not None and loadblkstr(zblk) == v:
continue
zblk = ZBlk()
zblk.setblkdata(v)
setblkstr(zblk, v)
valdict[v] = zblk
zctx.valdict = valdict
commit('treegen/values: init %r' % valv, skipIfEmpty=True)
......@@ -270,14 +285,14 @@ def TreesSrv(zstor, r):
zblk = valdict.get(k)
v1 = None
if zblk is not None:
v1 = zblk.loadblkdata()
v1 = loadblkstr(zblk)
v2 = zv.get(k)
if v1 != v2:
if v1 is None:
zblk = ZBlk()
valdict[k] = zblk
if v2 is not None:
zblk.setblkdata(v2)
setblkstr(zblk, v2)
zblk._p_changed = True
elif v2 is None:
del valdict[k]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment