Commit 0a09d51e authored by Levin Zimmermann's avatar Levin Zimmermann Committed by Kirill Smelkov

lib/zodb/zstor_2zurl: Add comprehensive tests

This patch adds comprehensive tests for 'wendelin.lib.zodb.zstor_2zurl'.
Before this patch only one related test existed ('test_zurlstable').
This test only lightly checked correct functionality of 'zstor_2zurl'.
Therefore we added the new tests 'test_zstor_2zurl' and
'test_zurlsamedb'.

The new tests only cover existing functionality.

---

Co-authored-by: kirr

/reviewed-by @kirr
/reviewed-on nexedi/wendelin.core!13
parent 28a7db7f
# Wendelin.core.bigfile | Tests for ZODB utilities and critical properties of ZODB itself # Wendelin.core.bigfile | Tests for ZODB utilities and critical properties of ZODB itself
# Copyright (C) 2014-2021 Nexedi SA and Contributors. # Copyright (C) 2014-2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -17,13 +17,14 @@ ...@@ -17,13 +17,14 @@
# #
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl, zmajor, _zhasNXDPatch from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl, zmajor, _zhasNXDPatch, dbstoropen
from wendelin.lib.testing import getTestDB from wendelin.lib.testing import getTestDB
from wendelin.lib import testing from wendelin.lib import testing
from persistent import Persistent, UPTODATE, GHOST, CHANGED from persistent import Persistent, UPTODATE, GHOST, CHANGED
from ZODB import DB, POSException from ZODB import DB, POSException
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.MappingStorage import MappingStorage from ZODB.MappingStorage import MappingStorage
from ZODB.DemoStorage import DemoStorage
from BTrees.IOBTree import IOBTree from BTrees.IOBTree import IOBTree
import transaction import transaction
from transaction import TransactionManager from transaction import TransactionManager
...@@ -31,6 +32,9 @@ from golang import defer, func ...@@ -31,6 +32,9 @@ from golang import defer, func
from pytest import raises from pytest import raises
import pytest; xfail = pytest.mark.xfail import pytest; xfail = pytest.mark.xfail
from ZEO.ClientStorage import ClientStorage as ZEOStorage from ZEO.ClientStorage import ClientStorage as ZEOStorage
from neo.client.Storage import Storage as NEOStorage
import os
from six.moves.urllib.parse import quote_plus
from wendelin.lib.tests.testprog import zopenrace, zloadrace from wendelin.lib.tests.testprog import zopenrace, zloadrace
...@@ -407,6 +411,142 @@ def test_zurlstable(): ...@@ -407,6 +411,142 @@ def test_zurlstable():
assert zurl == zurl0 assert zurl == zurl0
# test that ZODB database opened via storage's zurl, provides access to the same data.
@func
def test_zurlsamedb():
stor1 = testdb.getZODBStorage()
defer(stor1.close)
# skip on FileStorage - ZODB/py fails with LockError on attempt to create
# two FileStorage's connected to the same data.
if isinstance(stor1, FileStorage):
pytest.skip("skipping on FileStorage")
zurl = zstor_2zurl(stor1)
stor2 = dbstoropen(zurl)
defer(stor2.close)
db1 = DB(stor1)
db2 = DB(stor2)
# get/set retrieves or sets root['X'] = x.
@func
def set(db, x):
conn = db.open(); defer(conn.close)
root = conn.root()
root['X'] = x
transaction.commit()
@func
def get(db):
zsync(db.storage)
conn = db.open(); defer(conn.close)
root = conn.root()
return root['X']
# stor1/stor2 should have the same data
set(db1, 1)
assert get(db2) == 1
set(db1, 'abc')
assert get(db2) == 'abc'
# ensure zstor_2zurl returns expected zurl.
def test_zstor_2zurl(tmpdir, neo_ssl_dict):
# fs1 returns new FileStorage located in tmpdir.
def fs1(name):
return FileStorage("%s/%s" % (tmpdir, name))
# zeo returns new ZEO client for specified storage name and server address.
#
# server_addr can be either:
# - str (specifying address of UNIX socket), or
# - (host, addr) pair - specifying TCP address.
#
# NOTE the client is returned without waiting until server is connected.
def zeo(storage_name, server_addr):
if testing.TestDB_ZEO('').z5:
return ZEOStorage(server_addr, storage=storage_name, wait=False)
# It's better to use a mock storage for zeo == 4, because
# we would have to wait for a long time. See here the
# respective part in ZEO4 source code:
#
# https://github.com/zopefoundation/ZEO/blob/4/src/ZEO/ClientStorage.py#L423-L430
#
# ..compared to ZEO5 which omits the else clause:
#
# https://github.com/zopefoundation/ZEO/blob/5.3.0/src/ZEO/ClientStorage.py#L279-L286
zeo_storage = type(
"ClientStorage",
(object,),
{
"_addr": server_addr,
"_storage": storage_name,
"close": lambda self: None,
"getName": lambda self: self._storage
}
)()
type(zeo_storage).__module__ = "ZEO.ClientStorage"
type(zeo_storage).__name__ = "ClientStorage"
return zeo_storage
# neo returns new NEO client for specified cluster name and master address.
# NOTE, similarly to ZEO, the client is returned without waiting until server nodes are connected.
def neo(cluster_name, master_addr, ssl=0):
kwargs = dict(master_nodes=master_addr, name=cluster_name)
if ssl:
kwargs.update(neo_ssl_dict)
return NEOStorage(**kwargs)
# demo returns new DemoStorage with specified base and delta.
def demo(base, delta):
return DemoStorage(base=base, changes=delta)
# assert_zurl_is_correct verifies that zstor_2zurl(zstor) returns zurl_ok.
# zstor is closed after this test.
@func
def assert_zurl_is_correct(zstor, zurl_ok):
defer(zstor.close)
assert zstor_2zurl(zstor) == zurl_ok
# sslp is the ssl encryption uri part of an encrypted NEO node
q = quote_plus
sslp = ";".join(("%s=%s" % (q(k), q(v)) for k, v in sorted(neo_ssl_dict.items())))
_ = assert_zurl_is_correct
_(fs1("test.fs"), "file://%s/test.fs" % tmpdir) # FileStorage
_(zeo("1", "/path/to/zeo.sock"), "zeo:///path/to/zeo.sock") # ZEO/unix
_(zeo("test", "/path/to/zeo.sock"), "zeo:///path/to/zeo.sock?storage=test") # + non-default storage name
_(zeo("1", ("127.0.0.1", 1234)), "zeo://127.0.0.1:1234") # ZEO/ip4
_(zeo("test", ("127.0.0.1", 1234)), "zeo://127.0.0.1:1234?storage=test") # + non-default storage name
_(neo("test", "127.0.0.1:1234"), "neo://127.0.0.1:1234/test") # NEO/ip4
_(neo("test", "127.0.0.1:1234", 1), "neos://%s@127.0.0.1:1234/test" % sslp) # + ssl
_(demo(zeo("base", ("1.2.3.4", 5)), # DemoStorage
fs1("delta.fs")), "demo:(zeo://1.2.3.4:5?storage=base)/(file://%s/delta.fs)" % tmpdir)
# Test exceptions
# invalid storage
with raises(ValueError, match="in-RAM storages are not supported"):
zstor_2zurl(MappingStorage())
# invalid object
with raises(NotImplementedError):
zstor_2zurl("I am not a storage.")
# neo_ssl_dict returns the path of precomputed static ssl certificate
# files.
@pytest.fixture
def neo_ssl_dict():
ssl_files_base_path = "%s%s%s%s" % (
os.path.dirname(__file__), os.sep, "testdata", os.sep
)
return {
k: "%s%s" % (ssl_files_base_path, v)
for k, v in dict(ca="ca.crt", key="node.key", cert="node.crt").items()
}
# ---- tests for critical properties of ZODB ---- # ---- tests for critical properties of ZODB ----
# verify race in between Connection.open and invalidations. # verify race in between Connection.open and invalidations.
......
-----BEGIN CERTIFICATE-----
MIIC/TCCAeWgAwIBAgIUOkZ7N15eWCwcDK3XGsGrFZFXNT8wDQYJKoZIhvcNAQEL
BQAwDTELMAkGA1UEAwwCQ0EwIBcNMjIxMTE2MTM0OTQ4WhgPMjI5NjA4MzExMzQ5
NDhaMA0xCzAJBgNVBAMMAkNBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
AQEAofbtYCkTLsptpaol4DqjNEz7er3INVp255T7j6pth1Z12255KlLUojTJBYqD
doeek3XVGY+mZ4Hxt00pyzUIbpKw7HEltkvINdBPBIxO8q2fM5fkSOUkfFQkkpxo
O7rQLMFCFZ3qiDsZ7sZp16nW3wfwlR9lsHPwH5buMxRnJ9KvltdEHmNcsfPH3/x6
gbxIpKdpOWQXtWb4CoFpgYIPmn6SE/hpHWV0LBkXEv/l4Pp7LnYyts1F2uO32SLT
zJ8U1Q7JQe6C0BejoO2g9y5xtHLKjmwE9mWu8iD9qH4MYjU9+O/Z1Djnb1E+Jvjb
zahvdNmTwxfFF62FaR66f8EBawIDAQABo1MwUTAdBgNVHQ4EFgQUQCW5f7bArbvZ
SQsNeYPl89xKeWMwHwYDVR0jBBgwFoAUQCW5f7bArbvZSQsNeYPl89xKeWMwDwYD
VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAn9Ei+DAOc92Iau3XtEcn
wLvVB4+KYG75TzwBWFDn0o+ycBJ1Ie8IyK/pEe+Pw6SvsSKEMpzH+R4h3vcbDMTP
bJaaL4GPosfZCpMkCD3RprvHCW+wJYTWjjEKBswOTLabSAyMNvyT+51qfp6Hlr3M
Dbl/qwHvSmGehVH/EASQwOnHAO037hbUYilra24TW+qTQSK9A0x3395QZUIHVUVG
6YrSUdeYzZRUjYyEOjxMkkr0WDF5O9+QDETYhZjtmD/OE6O2Tuqayf3y15yApynq
KMi6Eti47GUXmvfZDTyIosRNtVp8SUq1yCFV+641iEb9GakUkmc9G8d6/WhI6kmw
nQ==
-----END CERTIFICATE-----
#!/bin/sh -e
# Same as the one used in NEO/py tests:
# https://lab.nexedi.com/nexedi/neoppod/blob/bff5c82f/neo/tests/create-certs
DAYS=100000 # = ~270 years
at_exit() { rm -f "$CAkey"; }
trap at_exit 0
CAkey=`mktemp`
openssl req -new -x509 -nodes -keyout "$CAkey" -out ca.crt -subj /CN=CA -days $DAYS
openssl req -new -nodes -keyout node.key -subj /CN=node |
openssl x509 -CA ca.crt -CAkey "$CAkey" -req -out node.crt -set_serial 1 -days $DAYS
-----BEGIN CERTIFICATE-----
MIICkjCCAXoCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwIBcNMjIx
MTE2MTM0OTQ4WhgPMjI5NjA4MzExMzQ5NDhaMA8xDTALBgNVBAMMBG5vZGUwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrewmDGrWefg+4GOjqZhkrQ0ht
Kxhh0wZhuJmpcjayc8TE+BbJJ64vOhdmGO5dZ3Ssj4EJkxaX0zG573xn6iJAmRJv
WCo/ufFzCqMeOXc0JejpZD47sNEd5A9pOe8MBeT8HEt08st7Mh+HNE9SPUePo1y8
31FRX2DTj0xLxtK+KLSWsrhEu7jotR0XY0K1YYj47Xu8EJmOdmdK7bRosVnoVPa3
naaKlUt1k2wgzEMOzGsHDPIXLo3Xpe9o1tHSM7phG7prZLWv6r1FUwbHSQVzMQXi
HsBpHkgMkoH7YN2MtHrjulzYjjtzYOjqvWP+MIUUXTo0PqpiPoeVlxCjVHFxAgMB
AAEwDQYJKoZIhvcNAQELBQADggEBABdzqOXg0L+dUmFklg6zQlbq8Ca7q6OBpe50
aV0b6lQQ1euVcPWoBmxC0giTPsBrWCcn0jpCmBEZ3xUerM7Qih1POyKrM8uQ5m3T
3eifgDE7krwzRDNqsMPh1AC7PpABbRCHXRxdqe4tLjMmI8jXNtsDTuNwjnXSZD5W
WJjLQ19WXy8V6ffvwjVSwmnu83qY4fQ9vmgG+0liOyXx4tlaH6NrZUMGuPUWJ5Qz
n6UBeegOi1HhovjBdXhTZgVRCB2TCwNaGA7EiDuAKFglKlw4SBoLSCnV1PJdc/u6
SBdbRDKBm4zAOfYHs5h86+3+Okj/VIhZlQ0oPofCRxi2fWG0El8=
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrewmDGrWefg+4
GOjqZhkrQ0htKxhh0wZhuJmpcjayc8TE+BbJJ64vOhdmGO5dZ3Ssj4EJkxaX0zG5
73xn6iJAmRJvWCo/ufFzCqMeOXc0JejpZD47sNEd5A9pOe8MBeT8HEt08st7Mh+H
NE9SPUePo1y831FRX2DTj0xLxtK+KLSWsrhEu7jotR0XY0K1YYj47Xu8EJmOdmdK
7bRosVnoVPa3naaKlUt1k2wgzEMOzGsHDPIXLo3Xpe9o1tHSM7phG7prZLWv6r1F
UwbHSQVzMQXiHsBpHkgMkoH7YN2MtHrjulzYjjtzYOjqvWP+MIUUXTo0PqpiPoeV
lxCjVHFxAgMBAAECggEAINhzTPpruUz6I9AZpPQjdAbyM37ir7Ik8cjTHgKzeWD1
WqkObD8+fOtKmhLCbX+oeHkQRU44MKKSTsQRdzIAl5D6V/tyNDygg+GQtvRY4dM1
FqdNyE8o7unFDem74PisEKI2zdLHqPRD17jsTJtSPrASzlaUE9sKB9vVyFnv+d4v
xaTXQzPm0SOc+gNDiyUyEo8yYqESf1Ypz8Lq0iHBQzXd2gPBWwKjjm69xeA7RjLV
H7WoYOPdXI1OcIDr/FmSkD3Di1vo9Ar5FwB7t99Y9rwkdJvgAIqqttUIO0qJ35RP
0IiFSGrGfU9Gk8/bSNRrdWF1keAAstI9Vy8LtWyDlQKBgQDbzQ4br+PK/wzGsgt5
niJ2K7mD2JAuNk3QidAewhxCjG1IxvE4NaqPLWG7A9Bya+1wzPGEgV9q7eu8j8s3
UsFyaYd5EkFj8uejdhR8HLAfiem66lVSYFFMDSO3w7SeEXognvQzkBlON32q9HRs
eg+kW8jWzriv77kdV0SVSWxX4wKBgQDHuMPbP/ZV8HXSAjm2z1LfAcrcdoJf3/dL
cU0CD80f55PhlDehMq0irS1DI4Jt++L3zzqmFaF9kvrjF7UQU3ypxz1iGQLtkPat
rv2Nnb98afBpmqSQ9cKJpB2fUsKYjtKFZ3Tlk+lgK46m58bAj9xZhBafU4uDftcR
QgFrdH7JmwKBgCde7IcDE395/fTdUXbgX3XhUSKPuQu+CGQxO0k3Jc65zp15nR2f
Z0SsMVRJU54bPPW2kBBXCBtkMJSfARyQwzvWMUD7hyD7EojSDstIbDfKAUbM0Kmn
vQgtVleFQ6YIpWKQdleCGTZMdEMZ9Hy1xHWBR1FKyAQjoXni9tLyyuhXAoGAYuV2
9mZh0IRVkzxO39shqhP1I937CQaaKd1cyJySci316EnAki3J1dXzB7/ZoiSMazFO
VVd4s/nIChN/MSpmHJvwJVdFP+X8gFyaaTmnzC+ZQHG286D+R7HgiTgt6+UtEan9
FIF9moV3CebTYB7ceecSXKq99sskLAOHOHJjJekCgYBM+rvFRWF+pfXwySnP97Me
+377ViGsKf3tGy1tXpuNhc1AC/iyIUelrhOcWrwkMK2kYnT6VWqqpg8rVJMolHLE
UJRDGrmEW31HUjd+YjFIa9HrWmUP3JToct0YE+E4wHyRVaL2LDnjHO8lKHUS0nQT
1cNYtzNncApsNw4rdOg6Sg==
-----END PRIVATE KEY-----
...@@ -397,6 +397,7 @@ setup( ...@@ -397,6 +397,7 @@ setup(
'test': [ 'test': [
'pytest', 'pytest',
'scipy', 'scipy',
'neoppod', # lib/tests/test_zodb.py
'ZEO', # lib/tests/test_zodb.py 'ZEO', # lib/tests/test_zodb.py
], ],
}, },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment