Commit 01a01c8c authored by Kirill Smelkov's avatar Kirill Smelkov

client: Add support for zodburi

zodburi[1] provides a way to open ZODB storages by URL/URI. It already
has support for file:// zeo:// zconfig:// memory:// etc schemes out of
the box and third-party-to-ZODB storages can add support for their
schemes via providing zodburi.resolvers entrypoint.

For example relstorage and newtdb do this.

Let's also teach NEO to open itself via neo:// URI schema.

[1] http://docs.pylonsproject.org/projects/zodburi
[2] https://github.com/zodb/relstorage/blob/2.1a1-15-g68c8cf1/relstorage/zodburi_resolver.py
[3] https://github.com/newtdb/db/blob/0.5.2-1-gbd36e90/src/newt/db/zodburi.py
parent 3bf2a0c6
#
# Copyright (C) 2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""NEO URI resolver for zodburi
URI format:
neo://name@master1,master2,...,masterN?options
"""
import ZODB.config
import ZConfig
from cStringIO import StringIO
from collections import OrderedDict
from urlparse import urlsplit, parse_qsl
# neo_zconf_options returns set of zconfig options supported by NEO storage
def neo_zconf_options():
neo_schema = """<schema>
<import package="ZODB" />
<import package="neo.client" />
</schema>"""
neo_schema = StringIO(neo_schema)
neo_schema = ZConfig.loadSchemaFile(neo_schema)
neo_storage_zconf = neo_schema.gettype('NeoStorage')
options = {k for k, _ in neo_storage_zconf}
assert 'master_nodes' in options
assert 'name' in options
return options
# canonical_opt_name returns "oPtion_nAme" as "option-name"
def canonical_opt_name(name):
return name.lower().replace('_', '-')
# worker entrypoint for resolve_uri and tests
def _resolve_uri(uri):
scheme, netloc, path, query, frag = urlsplit(uri)
if scheme != "neo":
raise ValueError("invalid uri: %s : expected neo:// scheme" % uri)
if path != "":
raise ValueError("invalid uri: %s : non-empty path" % uri)
if frag != "":
raise ValueError("invalid uri: %s : non-empty fragment" % uri)
# extract master list and name from netloc
name, masterloc = netloc.split('@', 1)
master_list = masterloc.split(',')
neokw = OrderedDict()
neokw['master_nodes'] = ' '.join(master_list)
neokw['name'] = name
# get options from query: only those that are defined by NEO schema go to
# storage - rest are returned as database options
dbkw = {}
neo_options = neo_zconf_options()
for k, v in OrderedDict(parse_qsl(query)).items():
if k in neo_options:
neokw[k] = v
else:
# it might be option for storage, but not in canonical form e.g.
# read_only -> read-only (zodburi world settled on using "_" and
# ZConfig world on "-" as separators)
k2 = canonical_opt_name(k)
if k2 in neo_options:
neokw[k2] = v
# else keep this kv as db option
else:
dbkw[k] = v
# now we have everything. Let ZConfig do actual work for validation options
# and borning the storage
neozconf = """%import neo.client
<NEOStorage>
"""
for k, v in neokw.items():
neozconf += " %s\t%s\n" % (k, v)
neozconf += "</NEOStorage>\n"
return neozconf, dbkw
# resolve_uri resolves uri according to neo:// schema.
# see module docstring for uri format.
def resolve_uri(uri):
neozconf, dbkw = _resolve_uri(uri)
def factory():
return ZODB.config.storageFromString(neozconf)
return factory, dbkw
......@@ -66,6 +66,7 @@ UNIT_TEST_MODULES = [
# client application
'neo.tests.client.testClientApp',
'neo.tests.client.testMasterHandler',
'neo.tests.client.testZODBURI',
# light functional tests
'neo.tests.threaded.test',
'neo.tests.threaded.testImporter',
......
#
# Copyright (C) 2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import unittest
from neo.client.zodburi import _resolve_uri
testv = [
# [] of (uri, zconf_ok, dbkw_ok)
("neo://dbname@master",
"""\
master_nodes\tmaster
name\tdbname
""",
{}),
("neo://db2@master1:port1,master2:port2,master3:port3",
"""\
master_nodes\tmaster1:port1 master2:port2 master3:port3
name\tdb2
""",
{}),
("neo://db3@master1,master2:port2?read_only=true",
"""\
master_nodes\tmaster1 master2:port2
name\tdb3
read-only\ttrue
""",
{}),
("neo://db4@[2001:67c:1254:2a::1]:1234,master2:port2?read_only=false"
"&compress=true&logfile=xxx&alpha=111&dynamic_master_list=zzz&ca=qqq"
"&cert=rrr&key=sss&beta=222",
"""\
master_nodes\t[2001:67c:1254:2a::1]:1234 master2:port2
name\tdb4
read-only\tfalse
compress\ttrue
logfile\txxx
dynamic_master_list\tzzz
ca\tqqq
cert\trrr
key\tsss
""",
{"alpha": "111", "beta": "222"}),
]
class ZODBURITests(unittest.TestCase):
def test_zodburi(self):
# invalid schema / path / fragment
self.assertRaises(ValueError, _resolve_uri, "http://db@master")
self.assertRaises(ValueError, _resolve_uri, "neo://db@master/path")
self.assertRaises(ValueError, _resolve_uri, "neo://db@master#frag")
# db @ master not fully specified
self.assertRaises(ValueError, _resolve_uri, "neo://master")
# verify zodburi resolver produces expected zconfig
for uri, zconf_ok, dbkw_ok in testv:
zconf_ok = "%import neo.client\n<NEOStorage>\n" + zconf_ok + \
"</NEOStorage>\n"
zconf, dbkw = _resolve_uri(uri)
self.assertMultiLineEqual(zconf, zconf_ok)
self.assertEqual(dbkw, dbkw_ok)
if __name__ == '__main__':
unittest.main()
......@@ -84,6 +84,9 @@ setup(
'neosimple=neo.scripts.simple:main',
'stat_zodb=neo.tests.stat_zodb:main',
],
'zodburi.resolvers': [
'neo = neo.client.zodburi:resolve_uri [client]',
],
},
extras_require = extras_require,
package_data = {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment