test/gen_testdata: Generate transactions with both " " and "p" status

Until now we were generating only regular transactions with " " status
and this does not cover e.g. restore case when it needs to replicate
packed transaction: instead of recreating it bit-to-bit exactly as
original with "p" status, restore recreates it with " " status, breaking
restore promise.

Adjusting testdata this way exposes that bug in restore:

    ======================================== FAILURES ========================================
    ________________________________________ test_zodbrestore[!zext] ________________________________________

    tmpdir = local('/tmp/pytest-of-kirr/pytest-17/test_zodbrestore__zext_0'), zext = <function _ at 0x7fd6b7a03750>

        @func
        def test_zodbrestore(tmpdir, zext):
            zkind = '_!zext' if zext.disabled else ''

            # restore from testdata/1.zdump.ok and verify it gives result that is
            # bit-to-bit identical to testdata/1.fs
            tdata = dirname(__file__) + "/testdata"
            @func
            def _():
                zdump = open("%s/1%s.zdump.raw.ok" % (tdata, zkind), 'rb')
                defer(zdump.close)

                stor = storageFromURL('%s/2.fs' % tmpdir)
                defer(stor.close)

                zodbrestore(stor, zdump)
            _()

            zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/1%s.fs" % (tdata, zkind)))
            zfs2 = readfile("%s/2.fs" % tmpdir)
    >       assert zfs1 == zfs2
    E       assert 'FS21\x02\x85...0\x00\x00\xb2' == 'FS21\x02\x85\...0\x00\x00\xb2'
    E         Skipping 49 identical leading characters in diff, use -v to show
    E         Skipping 22871 identical trailing characters in diff, use -v to show
    E         - 0\x00\x00tp\x00\x08\x00\t\x00\x00user0.15step 0.15\x00\x00\x00\x00\x00\x00\x00\x03\x02\x85\xcb\xac\x83i\xd0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00"\x80\x02c__main__
    E         ?           ^
    E         + 0\x00\x00t \x00\x08\x00\t\x00\x00user0.15step 0.15\x00\x00\x00\x00\x00\x00\x00\x03\x02\x85\xcb\xac\x83i\xd0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00...
    E
    E         ...Full output truncated (39 lines hidden), use '-vv' to show

    test_restore.py:53: AssertionError

Having "p" transactions in the testdata will also make sure that all tools
should handle such transactions well.

The problem of restore not handling "p" status properly was reported by Jérome
at nexedi/zodbtools!24.

In the next patch we will fix that problem.

/reviewed-by @jerome
/reviewed-on nexedi/zodbtools!24

zodbtools/test/gen_testdata.py

@@ -187,10 +187,14 @@ def _gen_testdb(outfs_path, zext):
     random.seed(0)
 
     namev = [_ for _ in "abcdefg"]
-    Niter = 2
+    Niter = 3
     for i in range(Niter):
         stor = FileStorage(outfs_path, create=(i == 0))
         db   = DB(stor)
+        if i == 1:
+            # change several transactions created during first pass to have "p" status
+            # (this also removes some transactions completely)
+            db.pack()
         conn = db.open()
         root = conn.root()
         assert root._p_oid == p64(0), repr(root._p_oid)
@@ -247,10 +251,6 @@ def _gen_testdb(outfs_path, zext):
         stor.tpc_begin(txn_stormeta)
         stor.deleteObject(obj._p_oid, obj_tid_lastchange, txn_stormeta)
         stor.tpc_vote(txn_stormeta)
-        # TODO different txn status vvv
-        # XXX vvv it does the thing, but py fs iterator treats this txn as EOF
-        #if i != Niter-1:
-        #    stor.tpc_finish(txn_stormeta)
         stor.tpc_finish(txn_stormeta)
 
         # close db & rest not to get conflict errors after we touched stor

zodbtools/test/test_analyze.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2019-2022 Nexedi SA and Contributors.
+# Copyright (C) 2019-2023 Nexedi SA and Contributors.
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -40,7 +40,7 @@ def test_zodbanalyze(tmpdir, capsys):
             csv=False,
         )
         captured = capsys.readouterr()
-        assert "Processed 68 records in 59 transactions" in captured.out
+        assert "Processed 68 records in 65 transactions" in captured.out
         assert captured.err == ""
 
     # csv output
@@ -57,8 +57,8 @@ def test_zodbanalyze(tmpdir, capsys):
     captured = capsys.readouterr()
     assert (
         """Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
-persistent.mapping.PersistentMapping,10,1578,45.633314%,157.800000,1,213,9,1365
-__main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577
+persistent.mapping.PersistentMapping,3,639,23.194192%,213.000000,1,213,2,426
+__main__.Object,63,2116,76.805808%,33.587302,9,303,54,1813
 """
         == captured.out
     )

zodbtools/test/testdata/.gitignore

 /*.lock
 /*.tmp
+/*.old