gen-testdata

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2024  Nexedi SA and Contributors.
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""generate reference fs1 database and index for tests"""

from __future__ import print_function

from ZODB.FileStorage import FileStorage
from ZODB.FileStorage.FileStorage import FILESTORAGE_MAGIC, TxnHeader, DataHeader, TRANS_HDR_LEN
from ZODB import DB
from ZODB.Connection import TransactionMetaData
from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_all_zodb_pickle_kinds, current_zkind
from os import stat, remove, makedirs
from os.path import exists, relpath
from shutil import copyfile, rmtree
from golang.gcompat import qq

import struct

# convert numeric oid to/from str
def p64(num):
    return struct.pack('>Q', num)

def unpack64(packed):
    return struct.unpack('>Q', packed)[0]

def hex64(packed):
    return '0x%016x' % unpack64(packed)

# bchar returns bytes corresponding to bytes_data[i].
# it works as bytes_data[i] on py2, but on py3, contrary to builtin behaviour, also returns bytes instead of int.
def bchar(bytes_data, i): # -> bytes
    if i >= len(bytes_data):
        raise IndexError('index out of range')
    return bytes_data[i:i+1]


def main2():
    zkind = current_zkind()
    prefix = "testdata/%s" % zkind
    if exists(prefix):
        rmtree(prefix)
    makedirs(prefix)

    outfs = "%s/1.fs" % prefix
    gen_testdb(outfs)

    # dump to go what to expect
    stor = FileStorage(outfs, read_only=True)
    with open("ztestdata_expect_%s_test.go" % zkind, "w") as f:
        def emit(v):
            print(v, file=f)
        emit("// Code generated by %s; DO NOT EDIT." % relpath(__file__))
        emit("package fs1\n")
        emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")

        emit("func init() {")

        # index
        emit("\tconst _1fs_indexTopPos = %i" % stor._pos)
        emit("\tvar _1fs_indexEntryv = []indexEntry{")
        for k, v in stor._index.iteritems():
            emit("\t\t{%8i, %8i}," % (unpack64(k), v))
        emit("\t}")

        # database records
        emit("\n\tvar _1fs_dbEntryv = []dbEntry{")
        txnLenPrev = 0
        for txn in stor.iterator():     # txn is TransactionRecord
            # txn.extension is already depickled dict - we want to put raw data from file
            # also we need to access txn record length which is not provided by higher-level iterator
            # do deep-dive into FileStorage
            th = stor._read_txn_header(txn._tpos)
            assert th.tid == txn.tid
            assert th.tlen == txn._tend - txn._tpos

            # fs1/go keeps in RAM whole txn length, not len-8 as it is on disk
            txnLen = th.tlen + 8

            emit("\t\t{")

            # -> TxnHeader
            emit("\t\t\tTxnHeader{")
            emit("\t\t\t\tPos:\t  %i," % txn._tpos)
            emit("\t\t\t\tLenPrev:  %i," % txnLenPrev)
            emit("\t\t\t\tLen:\t  %i," % txnLen)
            emit("\t\t\t\tTxnInfo:\tzodb.TxnInfo{")
            emit("\t\t\t\t\tTid:\t%s," % hex64(txn.tid))
            emit("\t\t\t\t\tStatus:\t'%s'," % txn.status)
            emit("\t\t\t\t\tUser:\t\t[]byte(%s)," % qq(txn.user))
            emit("\t\t\t\t\tDescription:\t[]byte(%s)," % qq(txn.description))
            emit("\t\t\t\t\tExtension:\t[]byte(%s)," % qq(th.ext))
            emit("\t\t\t\t},")
            emit("\t\t\t},")

            txnLenPrev = txnLen

            # -> DataHeader + payload
            emit("\n\t\t\t[]txnEntry{")

            for drec in txn:            # drec is itemof(TransactionRecordIterator) = Record
                # same as with txn - not everything is possible to get via
                # higher-level api
                dh = stor._read_data_header(drec.pos)
                assert dh.oid == drec.oid
                assert dh.tid == drec.tid
                assert dh.tloc == txn._tpos

                emit("\t\t\t\t{")
                emit("\t\t\t\t\tDataHeader{")
                emit("\t\t\t\t\t\tPos:\t%i," % drec.pos)
                emit("\t\t\t\t\t\tOid:\t%i," % unpack64(drec.oid))
                emit("\t\t\t\t\t\tTid:\t%s," % hex64(drec.tid))

                emit("\t\t\t\t\t\tPrevRevPos:\t%i," % dh.prev)
                emit("\t\t\t\t\t\tTxnPos:\t%i," % txn._tpos)
                assert drec.version == ''
                emit("\t\t\t\t\t\tDataLen:\t%i," % dh.plen)
                emit("\t\t\t\t\t},")
                plen = dh.plen
                if plen == 0:
                    rawdata = p64(dh.back) # back-pointer or 0 (= delete)
                    if drec.data is None:
                        data = "/* deleted */ nil"
                        datatid = "/* deleted */ 0"
                    else:
                        data = "[]byte(%s)" % qq(drec.data)
                        datatid = "/* copy from */ " + hex64(drec.data_txn)
                else:
                    rawdata = drec.data
                    data = "/* same as ^^^ */ sameAsRaw"
                    datatid = "/* no copy */ 0"

                emit("\t\t\t\t\t[]byte(%s)," % qq(rawdata))
                emit("\t\t\t\t\t%s," % data)
                emit("\t\t\t\t\t%s," % datatid)
                emit("\t\t\t\t},")

            emit("\t\t\t},")
            emit("\t\t},")
        emit("\t}")

        emit("\n\tztestdataReg.Register(%s, %s, &_TestDataOK{_1fs_indexTopPos, _1fs_indexEntryv, _1fs_dbEntryv})" % (qq(zkind), qq(prefix)))
        emit("}")
    stor.close()


    # prepare file with voted (not fully committed) tail
    voted = "%s/1voted.fs" % prefix
    copyfile(outfs, voted)
    def _():
        vstor = FileStorage(voted)
        vdb   = DB(vstor)
        vconn = vdb.open()
        vroot = vconn.root()
        vroot._p_activate() # to know its current serial

        txn = precommit(u"author", u"description", {'aaa': 'bbb'})
        txn_stormeta = TransactionMetaData(txn.user, txn.description, txn.extension)
        vstor.tpc_begin(txn_stormeta)
        vstor.store(vroot._p_oid, vroot._p_serial, b'000 data 000', '', txn_stormeta)
        vstor.tpc_vote(txn_stormeta)
        # NO tpc_finish here so that status remain 'c' (voted) instead of ' ' (committed)
    _()

    st = stat(outfs)
    l  = st.st_size
    vf = open(voted, 'rb')
    vf.seek(l)
    voted_tail = vf.read()
    assert bchar(voted_tail, -1+8+8+1) == b'c'  # voted, not finished (' ')

    with open("%s/1voted.tail" % prefix, "wb") as vt:
        vt.write(voted_tail)

    remove(voted)
    remove(voted+".index")
    remove(voted+".tmp")
    remove(voted+".lock")


    # prepare file with whiteout (deletion of previously non-existing object)
    whiteout = "%s/whiteout.fs" % prefix
    # as of 20210317 FileStorage.deleteObject verifies that object exists
    # -> prepare magic/transaction/data records manually
    with open(whiteout, "wb") as f:
        oid = p64(1)
        tid = p64(0x17)

        # file header
        f.write(FILESTORAGE_MAGIC)
        tpos = f.tell()

        # data record (see FileStorage.deleteObject)
        dh = DataHeader(oid, tid, 0, tpos, 0, 0)
        drec = dh.asString() + p64(0)

        # emit txn header (see FileStorage.tpc_vote)
        tlen = TRANS_HDR_LEN + 0 + 0 + 0 + len(drec) # empty u,d,e
        th = TxnHeader(tid, tlen, ' ', 0, 0, 0)
        th.user  = b''
        th.descr = b''
        th.ext   = b''
        f.write(th.asString())

        # emit data record
        f.write(drec)

        # emit txn tail
        f.write(p64(tlen))


    # prepare empty.fs
    empty = "%s/empty.fs" % prefix
    with open(empty, "wb") as f:
        f.write(FILESTORAGE_MAGIC)


def main():
    run_with_all_zodb_pickle_kinds(main2)

if __name__ == '__main__':
    main()