Merge rev 30116 from 3.4 branch.

Port from ZODB 3.2.

DemoStorage:  Added implementations for registerDB() and new_oid().  As
Tres discovered the hard way, wrapping a ZEO client storage as a
DemoStorage base storage yields insane behavior otherwise.

BaseStorage.new_oid():  Rewrite to eliminate recursion, and hence also the
need for the undocumented and irregular `last=` argument.

Other:  removed the `last=` argument to new_oid() every place that felt
compelled to spread that insanity ;-).  Seriously, it served no purpose
at all elsewhere, and looks like people just cut 'n pasted in fear.

NEWS.txt

@@ -7,6 +7,14 @@ What's new in ZODB3 3.4a4?
 ==========================
 Release date: DD-MMM-YYYY
 
+DemoStorage
+-----------
+
+Appropriate implementations of the storage API's ``registerDB()`` and
+``new_oid()`` methods were added, delegating to the base storage.  This was
+needed to support wrapping a ZEO client storage as a ``DemoStorage`` base
+storage, as some new Zope tests want to do.
+
 Tools
 -----
 
@@ -14,6 +22,16 @@ Tools
 actually went in several months go, but wasn't noted here at the time.
 Thanks to Dmitry Vasiliev for contributing code and tests.
 
+BaseStorage
+-----------
+
+``new_oid()``'s undocumented ``last=`` argument was removed.  It was used
+only for internal recursion, and injured code sanity elsewhere because not
+all storages included it in their ``new_oid()``'s signature.  Straightening
+this out required adding ``last=`` everywhere, or removing it everywhere.
+Since recursion isn't actually needed, and there was no other use for
+``last=``, removing it everywhere was the obvious choice.
+
 
 What's new in ZODB3 3.4a3?
 ==========================

src/ZEO/ServerStub.py

@@ -259,11 +259,8 @@ class StorageServer:
     def modifiedInVersion(self, oid):
         return self.rpc.call('modifiedInVersion', oid)
 
-    def new_oid(self, last=None):
-        if last is None:
+    def new_oid(self):
         return self.rpc.call('new_oid')
-        else:
-            return self.rpc.call('new_oid', last)
 
     def store(self, oid, serial, data, version, trans):
         return self.rpc.call('store', oid, serial, data, version, trans)

src/ZODB/BaseStorage.py

@@ -19,6 +19,7 @@ import cPickle
 import threading
 import time
 import logging
+from struct import pack as _structpack, unpack as _structunpack
 
 from persistent.TimeStamp import TimeStamp
 
@@ -96,10 +97,15 @@ class BaseStorage(UndoLogCompatible):
         t=time.time()
         t=self._ts=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,)))
         self._tid = `t`
+
+        # ._oid is the highest oid in use (0 is always in use -- it's
+        # a reserved oid for the root object).  Our new_oid() method
+        # increments it by 1, and returns the result.  It's really a
+        # 64-bit integer stored as an 8-byte big-endian string.
         if base is None:
-            self._oid=z64
+            self._oid = z64
         else:
-            self._oid=base._oid
+            self._oid = base._oid
 
     def abortVersion(self, src, transaction):
         if transaction is not self._transaction:
@@ -138,24 +144,22 @@ class BaseStorage(UndoLogCompatible):
     def modifiedInVersion(self, oid):
         return ''
 
-    def new_oid(self, last=None):
-        # 'last' is only for internal use, not part of the public API
+    def new_oid(self):
         if self._is_read_only:
             raise POSException.ReadOnlyError()
-        if last is None:
         self._lock_acquire()
         try:
-                last=self._oid
-                d=ord(last[-1])
-                if d < 255: last=last[:-1]+chr(d+1)
-                else:       last=self.new_oid(last[:-1])
-                self._oid=last
+            last = self._oid
+            d = ord(last[-1])
+            if d < 255:  # fast path for the usual case
+                last = last[:-1] + chr(d+1)
+            else:        # there's a carry out of the last byte
+                last_as_long, = _structunpack(">Q", last)
+                last = _structpack(">Q", last_as_long + 1)
+            self._oid = last
             return last
-            finally: self._lock_release()
-        else:
-            d=ord(last[-1])
-            if d < 255: return last[:-1]+chr(d+1)+'\0'*(8-len(last))
-            else:       return self.new_oid(last[:-1])
+        finally:
+            self._lock_release()
 
     # Update the maximum oid in use, under protection of a lock.  The
     # maximum-in-use attribute is changed only if possible_new_max_oid is

src/ZODB/DemoStorage.py

@@ -81,16 +81,17 @@ and call it to monitor the storage.
 """
 
 import base64, time
-from ZODB import POSException, BaseStorage
+from ZODB import POSException
 from ZODB.utils import z64, oid_repr
+from ZODB.BaseStorage import BaseStorage
 from persistent.TimeStamp import TimeStamp
 from cPickle import loads
 from BTrees import OOBTree
 
-class DemoStorage(BaseStorage.BaseStorage):
+class DemoStorage(BaseStorage):
 
     def __init__(self, name='Demo Storage', base=None, quota=None):
-        BaseStorage.BaseStorage.__init__(self, name, base)
+        BaseStorage.__init__(self, name, base)
 
         # We use a BTree because the items are sorted!
         self._data = OOBTree.OOBTree()
@@ -105,6 +106,23 @@ class DemoStorage(BaseStorage.BaseStorage):
             raise POSException.StorageError, (
                 "Demo base storage has version data")
 
+    # While we officially don't support wrapping a non-read-only base
+    # storage, it has proved useful for test suites to wrap a ClientStorage
+    # in DemoStorage.  The least we can do to help support that case is
+    # to arrange for invalidations to get delivered to the base storage.
+    def registerDB(self, db, limit):
+        if self._base is not None: # delegate
+            self._base.registerDB(db, limit)
+
+    # When DemoStorage needs to create a new oid, and there is a base
+    # storage, it must use that storage's new_oid() method.  Else
+    # DemoStorage may end up assigning "new" oids that are already in use
+    # by the base storage, leading to a variety of "impossible" problems.
+    def new_oid(self):
+        if self._base is None:
+            return BaseStorage.new_oid(self)
+        else:
+            return self._base.new_oid()
 
     def __len__(self):
         base=self._base

src/ZODB/FileStorage/FileStorage.py

@@ -1406,8 +1406,10 @@ class FileStorage(BaseStorage.BaseStorage,
         index = self._index
         oid = index.minKey(next)
 
+        oid_as_long, = unpack(">Q", oid)
+        next_oid = pack(">Q", oid_as_long + 1)
         try:
-            next_oid = index.minKey(self.new_oid(oid))
+            next_oid = index.minKey(next_oid)
         except ValueError: # "empty tree" error
             next_oid = None
 

src/ZODB/interfaces.py

@@ -391,7 +391,7 @@ class IStorage(Interface):
     def history(oid, version, length=1, filter=None):
         """TODO"""
 
-    def new_oid(last=None):
+    def new_oid():
         """TODO"""
 
     def set_max_oid(possible_new_max_oid):