demo: URI resolver

This patch adds `demo:` URI scheme to create DemoStorage from an URI.

While several existing resolvers already handle ?demostorage argument to
wrap itself with DemoStorage and in-RAM MappingStorage for changes, this
approach has the following drawbacks:

- every resolver must do it
- it is not possible to create DemoStorage with non-MappingStorage for changes.

My particular motivation here is Wendelin.core 2: it spawn WCFS
filesystem server to serve array data from ZODB storage, and passes
storage URL to spawned wcfs process, so that wcfs could connect and
retrieve data from the same ZODB storage that client process is
using. When original ERP5 client is using DemoStorage, both `base` and
`changes` must be persisted because if changes would be in-RAM
MappingStorage, WCFS could not access that data because it runs as a
separate process.

To build a DemoStorage URI we follow XRI Cross-references approach to
embed URIs for base and changes into combining demo URI:

    demo:(base_uri)/(δ_uri)

https://en.wikipedia.org/wiki/Extensible_Resource_Identifier provides
some related details and examples.

I choose fragments as the place for ZODB.DB arguments:

    demo:(base_uri)/(δ_uri)#dbkw...

The reason fragments - instead of parameters - are used, is because
DB arguments are _local_. Even with different DB arguments the URI
refers to the same storage, and for wendelin.core 2 it is important to
be able to determine whether two client processes actually use the same
underlying ZODB storage, even if those two clients use different local
parameters, like connection_pool_size and similar. Keeping such local
arguments in fragments makes it easy to determine the underlying URI of
the storage - by dropping fragments. It also follows logic from RFC 3986,
which says that "fragment ... may be ... some view on representations of
the primary resource": https://tools.ietf.org/html/rfc3986#section-3.5

Thanks beforehand,
Kirill

/cc @tseaver, @azmeuk, @jimfulton

P.S.

It was already suggested several times to add DemoStorageURIResolver and
deprecate ?demostorage in existing URI resolvers:

https://github.com/Pylons/zodburi/pull/25#issuecomment-485506959
https://github.com/Pylons/zodburi/pull/25#issuecomment-511480572
https://github.com/Pylons/zodburi/pull/25#issuecomment-622931793

docs/index.rst

@@ -43,8 +43,8 @@ URI Schemes
 -----------
 
 The URI schemes currently recognized in the ``zodbconn.uri`` setting
-are ``file://``, ``zeo://``, ``zconfig://``, ``memory://``
-.  Documentation for these URI scheme syntaxes are below.
+are ``file://``, ``zeo://``, ``zconfig://``, ``memory://`` and ``demo:``.
+Documentation for these URI scheme syntaxes are below.
 
 In addition to those schemes, the relstorage_ package adds support for
 ``postgres://``.
@@ -356,6 +356,29 @@ An example that combines a dbname with a query string::
 
    memory://storagename?connection_cache_size=100&database_name=fleeb
 
+
+``demo:`` URI scheme
+~~~~~~~~~~~~~~~~~~~~
+
+The ``demo:`` URI scheme can be passed as ``zodbconn.uri`` to create a
+DemoStorage database factory. DemoStorage provides an overlay combining base
+and δ ("delta", or in other words, "changes") storages.
+The URI scheme contains two parts, base and δ::
+
+    demo:(base_uri)/(δ_uri)
+
+an optional fragment specifies arguments for ``ZODB.DB.DB`` constructor::
+
+    demo:(base_uri)/(δ_uri)#dbkw
+
+Example
++++++++
+
+An example that combines ZEO with local FileStorage for changes::
+
+    demo:(zeo://localhost:9001?storage=abc)/(file:///path/to/Changes.fs)
+
+
 More Information
 ----------------
 

setup.py

@@ -61,6 +61,7 @@ setup(name='zodburi',
       file = zodburi.resolvers:file_storage_resolver
       zconfig = zodburi.resolvers:zconfig_resolver
       memory = zodburi.resolvers:mapping_storage_resolver
+      demo = zodburi.resolvers:demo_storage_resolver
       """,
       extras_require = {
         'testing': testing_extras,

zodburi/__init__.py

@@ -8,12 +8,17 @@ def resolve_uri(uri):
     returns a storage matching the spec defined in the uri.  dbkw is a dict of
     keyword arguments that may be passed to ZODB.DB.DB.
     """
+    factory, dbkw = _resolve_uri(uri)
+    return factory, _get_dbkw(dbkw)
+
+# _resolve_uri serves resolve_uri: it returns factory and original raw dbkw.
+def _resolve_uri(uri):
     scheme = uri[:uri.find(':')]
     for ep in iter_entry_points('zodburi.resolvers'):
         if ep.name == scheme:
             resolver = ep.load()
             factory, dbkw = resolver(uri)
-            return factory, _get_dbkw(dbkw)
+            return factory, dbkw
     else:
         raise KeyError('No resolver found for uri: %s' % uri)
 

zodburi/resolvers.py

+# -*- coding: utf-8 -*-
 from io import BytesIO
 import os
+import re
 
 from ZConfig import loadConfig
 from ZConfig import loadSchemaFile
@@ -15,6 +17,7 @@ from zodburi.datatypes import convert_int
 from zodburi.datatypes import convert_tuple
 from zodburi._compat import parse_qsl
 from zodburi._compat import urlsplit
+from zodburi import _resolve_uri
 
 # Capability test for older Pythons (2.x < 2.7.4, 3.x < 3.2.4)
 (scheme, netloc, path, query, frag) = urlsplit('scheme:///path/#frag')
@@ -205,7 +208,49 @@ class ZConfigURIResolver(object):
         return factory.open, dbkw
 
 
+class DemoStorageURIResolver(object):
+
+    # demo:(base_uri)/(δ_uri)#dbkw...
+    # URI format follows XRI Cross-references to refer to base and δ
+    # (see https://en.wikipedia.org/wiki/Extensible_Resource_Identifier)
+    _uri_re = re.compile(r'^demo:\((?P<base>.*)\)/\((?P<changes>.*)\)(?P<frag>#.*)?$')
+
+    def __call__(self, uri):
+        def baduri(why):
+            bad = 'demo: invalid uri %r' % uri
+            if why:
+                bad += ": " + why
+            raise ValueError(bad)
+
+        m = self._uri_re.match(uri)
+        if m is None:
+            baduri('')
+
+        base_uri = m.group('base')
+        delta_uri = m.group('changes')
+
+        basef, base_dbkw = _resolve_uri(base_uri)
+        if base_dbkw:
+            baduri('DB arguments in base')
+        deltaf, delta_dbkw = _resolve_uri(delta_uri)
+        if delta_dbkw:
+            baduri('DB arguments in changes')
+
+        frag = m.group('frag')
+        dbkw = {}
+        if frag:
+            dbkw = dict(parse_qsl(frag[1:]))
+
+        def factory():
+            base = basef()
+            delta = deltaf()
+            return DemoStorage(base=base, changes=delta)
+
+        return factory, dbkw
+
+
 client_storage_resolver = ClientStorageURIResolver()
 file_storage_resolver = FileStorageURIResolver()
 zconfig_resolver = ZConfigURIResolver()
 mapping_storage_resolver = MappingStorageURIResolver()
+demo_storage_resolver = DemoStorageURIResolver()

zodburi/tests/test_resolvers.py

@@ -600,6 +600,54 @@ class TestMappingStorageURIResolver(Base, unittest.TestCase):
         self.assertEqual(storage.__name__, 'storagename')
 
 
+class TestDemoStorageURIResolver(unittest.TestCase):
+
+    def _getTargetClass(self):
+        from zodburi.resolvers import DemoStorageURIResolver
+        return DemoStorageURIResolver
+
+    def _makeOne(self):
+        klass = self._getTargetClass()
+        return klass()
+
+    def test_fsoverlay(self):
+        import os.path, tempfile, shutil
+        tmpdir = tempfile.mkdtemp()
+        def _():
+            shutil.rmtree(tmpdir)
+        self.addCleanup(_)
+
+        resolver = self._makeOne()
+        basef   = os.path.join(tmpdir, 'base.fs')
+        changef = os.path.join(tmpdir, 'changes.fs')
+        self.assertFalse(os.path.exists(basef))
+        self.assertFalse(os.path.exists(changef))
+        factory, dbkw = resolver('demo:(file://%s)/(file://%s?quota=200)' % (basef, changef))
+        self.assertEqual(dbkw, {})
+        demo = factory()
+        from ZODB.DemoStorage import DemoStorage
+        from ZODB.FileStorage import FileStorage
+        self.assertTrue(isinstance(demo, DemoStorage))
+        self.assertTrue(isinstance(demo.base, FileStorage))
+        self.assertTrue(isinstance(demo.changes, FileStorage))
+        self.assertTrue(os.path.exists(basef))
+        self.assertTrue(os.path.exists(changef))
+        self.assertEqual(demo.changes._quota, 200)
+
+    def test_parse_frag(self):
+        resolver = self._makeOne()
+        factory, dbkw = resolver('demo:(memory://111)/(memory://222)#foo=bar&abc=def')
+        self.assertEqual(dbkw, {'foo': 'bar', 'abc': 'def'})
+        demo = factory()
+        from ZODB.DemoStorage import DemoStorage
+        from ZODB.MappingStorage import MappingStorage
+        self.assertTrue(isinstance(demo, DemoStorage))
+        self.assertTrue(isinstance(demo.base, MappingStorage))
+        self.assertEqual(demo.base.__name__, '111')
+        self.assertTrue(isinstance(demo.changes, MappingStorage))
+        self.assertEqual(demo.changes.__name__, '222')
+
+
 class TestEntryPoints(unittest.TestCase):
 
     def test_it(self):
@@ -610,6 +658,7 @@ class TestEntryPoints(unittest.TestCase):
             ('zeo', resolvers.ClientStorageURIResolver),
             ('file', resolvers.FileStorageURIResolver),
             ('zconfig', resolvers.ZConfigURIResolver),
+            ('demo', resolvers.DemoStorageURIResolver),
         ]
         for name, cls in expected:
             target = load_entry_point('zodburi', 'zodburi.resolvers', name)