diff --git a/software/erp5/test/setup.py b/software/erp5/test/setup.py index b5a670e6205f75cb14a19d6afeb02ea0d38254a4..2f78eb8f32ab7adbe1f035bebe4b1ed08af3b1a7 100644 --- a/software/erp5/test/setup.py +++ b/software/erp5/test/setup.py @@ -51,6 +51,8 @@ setup(name=name, 'cryptography', 'pexpect', 'pyOpenSSL', + 'ZEO', + 'zodburi', ], test_suite='test', ) diff --git a/software/erp5/test/test/test_zodb_zeo.py b/software/erp5/test/test/test_zodb_zeo.py new file mode 100644 index 0000000000000000000000000000000000000000..f022b9a17334b4e8fb7e345a0d590afc5c259ef3 --- /dev/null +++ b/software/erp5/test/test/test_zodb_zeo.py @@ -0,0 +1,96 @@ +import contextlib +import subprocess +import json + +import zodburi +from ZODB.DB import DB +from slapos.testing.utils import CrontabMixin + + +from . import ERP5InstanceTestCase, default, matrix, setUpModule, ERP5PY3 + +_ = setUpModule + + +class ZEOTestCase(ERP5InstanceTestCase): + __test_matrix__ = matrix((default,)) + + @classmethod + def getInstanceSoftwareType(cls) -> str: + return "zodb-zeo" + + @classmethod + def _getInstanceParameterDict(cls) -> dict: + return { + "tcpv4-port": 8000, + "computer-memory-percent-threshold": 100, + "name": cls.__name__, + "monitor-passwd": "secret", + "zodb-dict": {"root": {}}, + } + + @classmethod + def getInstanceParameterDict(cls) -> dict: + return {"_": json.dumps(cls._getInstanceParameterDict())} + + def setUp(self) -> None: + self.storage_dict = json.loads( + self.computer_partition.getConnectionParameterDict()["_"] + )["storage-dict"] + + def db(self) -> contextlib.AbstractContextManager[DB]: + root = self.storage_dict["root"] + zeo_uri = f"zeo://{root['server']}?storage={root['storage']}" + storage_factory, dbkw = zodburi.resolve_uri(zeo_uri) + return contextlib.closing(DB(storage_factory(), **dbkw)) + + +class TestRepozo(ZEOTestCase, CrontabMixin): + __partition_reference__ = "rpz" + + def test_backup_and_restore(self) -> None: + def check_state(): + (self.computer_partition_root_path / ".timestamp").unlink() + self.waitForInstance() + if ERP5PY3: + with self.db() as db: + with db.transaction() as cnx: + self.assertEqual(cnx.root.state, "before backup") + + if ERP5PY3: + # as it is not possible to connect to a python2 ZEO server + # from a python3 client, we check more when the server is python3 + with self.db() as db: + with db.transaction() as cnx: + cnx.root.state = "before backup" + + check_state() + self._executeCrontabAtDate("tidstorage", "2000-01-01 UTC") + dat, fsz, index = sorted( + [ + p.name + for p in ( + self.computer_partition_root_path / "srv" / "backup" / "zodb" / "root" + ).glob("*") + ] + ) + self.assertRegex(dat, r'2000-01-01-00-\d\d-\d\d.dat') + self.assertRegex(fsz, r'2000-01-01-00-\d\d-\d\d.fsz') + self.assertRegex(index, r'2000-01-01-00-\d\d-\d\d.index') + + if ERP5PY3: + with self.db() as db: + with db.transaction() as cnx: + cnx.root.state = "after backup" + db.close() + + restore_script = self.computer_partition_root_path / "srv" / "runner-import-restore" + self.assertTrue(restore_script.exists()) + status, restore_output = subprocess.getstatusoutput(str(restore_script)) + self.assertEqual(status, 1) + self.assertIn("Zeo is already running", restore_output) + + with self.slap.instance_supervisor_rpc as supervisor: + supervisor.stopAllProcesses() + restore_output = subprocess.check_output(restore_script) + check_state() diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index caf6e181a223de13de5be60346f9514e726fe135..bc5a0a215c2c658b75e4dbead30a428c112af7af 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -14,6 +14,7 @@ extends = ../../component/python-pynacl/buildout.cfg ../../component/python-backports-lzma/buildout.cfg ../../component/selenium/buildout.cfg + ../../component/ZODB/buildout.cfg ../../stack/slapos.cfg ../../stack/nxdtest.cfg @@ -356,6 +357,7 @@ setup = ${recurls-repository:location} [python-interpreter] eggs += + ${BTrees:egg} ${lxml-python:egg} ${python-PyYAML:egg} ${slapos.core-setup:egg} @@ -365,6 +367,7 @@ eggs += beautifulsoup4 caucase erp5.util + ${persistent:egg} ${python-pynacl:egg} ${python-cryptography:egg} ${python-mysqlclient:egg} @@ -526,15 +529,23 @@ recurls = slapos.core = # Various needed versions -Pillow = 10.2.0+SlapOSPatched001 +BTrees = 6.1 forcediphttpsadapter = 1.0.1 image = 1.5.25 +mysqlclient = 2.1.1 +paho-mqtt = 1.5.0 +pcpp = 1.30 +persistent = 6.1 +Pillow = 10.2.0+SlapOSPatched001 plantuml = 0.3.0:whl pypdf = 3.6.0:whl pysftp = 0.2.9 requests-toolbelt = 0.8.0 testfixtures = 6.11.0 -mysqlclient = 2.1.1 -paho-mqtt = 1.5.0 -pcpp = 1.30 +transaction = 5.0 xmltodict = 0.13.0 +ZEO = 6.0.0 +ZODB = 6.0.0 +zodbpickle = 4.1.1 +zope.deferredimport = 5.0 +zope.proxy = 6.1 diff --git a/stack/erp5/buildout.hash.cfg b/stack/erp5/buildout.hash.cfg index bffae7acad6c9c840fa784046af452daa281ff7c..a2c32ce548abb5398851fcc33b5add66977eac21 100644 --- a/stack/erp5/buildout.hash.cfg +++ b/stack/erp5/buildout.hash.cfg @@ -78,7 +78,7 @@ md5sum = 1333d2fc21f64da4010a4eafea59d141 [template-zeo] filename = instance-zeo.cfg.in -md5sum = 3190fb6b2380ffbef40db62e1d4ba4d0 +md5sum = 702afb430227eebe4312a618da7ef7cb [template-zeo-conf] filename = zeo.conf.in diff --git a/stack/erp5/instance-zeo.cfg.in b/stack/erp5/instance-zeo.cfg.in index 91b4b5e5e5aecf64a8df2ca1900cb90498a6ab02..e37907c23eb9a57a52d2503390f9aafd49f63bdf 100644 --- a/stack/erp5/instance-zeo.cfg.in +++ b/stack/erp5/instance-zeo.cfg.in @@ -80,6 +80,45 @@ config-port = {{ "${" ~ zeo_section_name ~ ":port}" }} {% set tidstorage_repozo_path = '' -%} {% else -%} +[repozo-backup-script] +repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo + +# BBB on python3 we don't use Products.TIDStorage but repozo directly. +[repozo-backup-script:python3] +recipe = slapos.recipe.template +inline = + #!/bin/sh + zodb_directory="${directory:zodb}" + zodb_backup_directory="{{ default_backup_path }}" + repozo="${tidstorage:repozo-binary}" + EXIT_CODE=0 + + {% for family, zodb in six.iteritems(zodb_dict) -%} + {% for name, zodb in zodb -%} + storage_name="{{ name }}" + zodb_path="$storage_name.fs" + [ ! -d "$zodb_backup_directory/$storage_name" ]] && mkdir "$zodb_backup_directory/$storage_name" + echo "Backing up $storage_name ..." + $repozo \ + --backup \ + --kill-old-on-full \ + --gzip \ + --quick \ + --repository="$zodb_backup_directory/$storage_name" \ + --file="$zodb_directory/$zodb_path" + + CURRENT_EXIT_CODE=$? + if [ ! "$CURRENT_EXIT_CODE"="0" ]; then + EXIT_CODE="$CURRENT_EXIT_CODE" + echo "$storage_name Backup restoration failed." + fi + {% endfor -%} + {% endfor -%} + exit $EXIT_CODE +repozo-wrapper = ${:output} +mode = 755 +output = ${buildout:bin-directory}/repozo-backup + [tidstorage] recipe = slapos.cookbook:tidstorage known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }} @@ -116,7 +155,7 @@ recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = tidstorage time = {{ dumps(backup_periodicity) }} -command = ${tidstorage:repozo-wrapper} +command = ${repozo-backup-script:repozo-wrapper} # Used for ERP5 resiliency or (more probably) # webrunner resiliency with erp5 inside. @@ -137,8 +176,9 @@ mode = 770 [{{ section("resiliency-after-import-script") }}] # Generate after import script used by importer instance of webrunner -recipe = collective.recipe.template -input = inline: #!/bin/sh +recipe = slapos.recipe.template +inline = + #!/bin/sh # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE # OR ZODB DATA WILL BE ERASED. @@ -146,8 +186,6 @@ input = inline: #!/bin/sh # zodb location. It is launched by the clone (importer) instance of webrunner # in the end of the import script. - # Depending on the output, it will create a file containing - # the status of the restoration (success or failure). zodb_directory="${directory:zodb}" zodb_backup_directory="{{ default_backup_path }}" repozo="${tidstorage:repozo-binary}"