gen-testdata

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright (C) 2017  Nexedi SA and Contributors.
#                     Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""generate reference fs1 database and index for tests"""

from ZODB.FileStorage import FileStorage
from zodbtools.test.gen_testdata import gen_testdb
from zodbtools.util import escapeqq

import struct

# convert numeric oid to/from str
def p64(num):
    return struct.pack('>Q', num)

def unpack64(packed):
    return struct.unpack('>Q', packed)[0]

def hex64(packed):
    return '0x%016x' % unpack64(packed)


def main():
    outfs = "testdata/1.fs"
    gen_testdb(outfs)

    # dump to go what to expect
    stor = FileStorage(outfs, read_only=True)
    with open("ztestdata_expect_test.go", "w") as f:
        def emit(v):
            print >>f, v
        emit("// Code generated by %s; DO NOT EDIT." % __file__)
        emit("package fs1\n")
        emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")

        # index
        emit("const _1fs_indexTopPos = %i" % stor._pos)
        emit("var _1fs_indexEntryv = [...]indexEntry{")
        for k, v in stor._index.iteritems():
            emit("\t{%8i, %8i}," % (unpack64(k), v))
        emit("}")

        # database records
        emit("\nvar _1fs_dbEntryv = [...]dbEntry{")
        txnLenPrev = 0
        for txn in stor.iterator():     # txn is TransactionRecord
            # txn.extension is already depickled dict - we want to put raw data from file
            # also we need to access txn record length which is not provided by higher-level iterator
            # do deep-dive into FileStorage
            th = stor._read_txn_header(txn._tpos)
            assert th.tid == txn.tid
            assert th.tlen == txn._tend - txn._tpos

            # fs1/go keeps in RAM whole txn length, not len-8 as it is on disk
            txnLen = th.tlen + 8

            emit("\t{")

            # -> TxnHeader
            emit("\t\tTxnHeader{")
            emit("\t\t\tPos:\t  %i," % txn._tpos)
            emit("\t\t\tLenPrev:  %i," % txnLenPrev)
            emit("\t\t\tLen:\t  %i," % txnLen)
            emit("\t\t\tTxnInfo:\tzodb.TxnInfo{")
            emit("\t\t\t\tTid:\t%s," % hex64(txn.tid))
            emit("\t\t\t\tStatus:\t'%s'," % txn.status)
            emit("\t\t\t\tUser:\t\t[]byte(%s)," % escapeqq(txn.user))
            emit("\t\t\t\tDescription:\t[]byte(%s)," % escapeqq(txn.description))
            emit("\t\t\t\tExtension:\t[]byte(%s)," % escapeqq(th.ext))
            emit("\t\t\t},")
            emit("\t\t},")

            txnLenPrev = txnLen

            # -> DataHeader + payload
            emit("\n\t\t[]txnEntry{")

            for drec in txn:            # drec is itemof(TransactionRecordIterator) = Record
                # same as with txn - not everything is possible to get via
                # higher-level api
                dh = stor._read_data_header(drec.pos)
                assert dh.oid == drec.oid
                assert dh.tid == drec.tid
                assert dh.tloc == txn._tpos

                emit("\t\t\t{")
                emit("\t\t\t\tDataHeader{")
                emit("\t\t\t\t\tPos:\t%i," % drec.pos)
                emit("\t\t\t\t\tOid:\t%i," % unpack64(drec.oid))
                emit("\t\t\t\t\tTid:\t%s," % hex64(drec.tid))

                emit("\t\t\t\t\tPrevRevPos:\t%i," % dh.prev)
                emit("\t\t\t\t\tTxnPos:\t%i," % txn._tpos)
                assert drec.version == ''
                emit("\t\t\t\t\tDataLen:\t%i," % dh.plen)
                emit("\t\t\t\t},")
                plen = dh.plen
                if plen == 0:
                    rawdata = p64(dh.back) # back-pointer or 0 (= delete)
                    if drec.data is None:
                        data = "/* deleted */ nil"
                        datatid = "/* deleted */ 0"
                    else:
                        data = "[]byte(%s)" % escapeqq(drec.data)
                        datatid = "/* copy from */ " + hex64(drec.data_txn)
                else:
                    rawdata = drec.data
                    data = "/* same as ^^^ */ sameAsRaw"
                    datatid = "/* no copy */ 0"

                emit("\t\t\t\t[]byte(%s)," % escapeqq(rawdata))
                emit("\t\t\t\t%s," % data)
                emit("\t\t\t\t%s," % datatid)
                emit("\t\t\t},")

            emit("\t\t},")
            emit("\t},")
        emit("}")


if __name__ == '__main__':
    main()