Commit 3a8f6f03 authored by Julien Muchembled's avatar Julien Muchembled

Drop support for ZODB3

parent 414573b9
graft tools
include neo.conf CHANGELOG.rst TODO ZODB3.patch
include neo.conf CHANGELOG.rst TODO
......@@ -52,7 +52,7 @@ Requirements
- MySQLdb: https://github.com/PyMySQL/mysqlclient-python
- For client nodes: ZODB 3.10.x or later
- For client nodes: ZODB 4.4.5 or later
Installation
============
......
This diff is collapsed.
......@@ -25,51 +25,7 @@ def patch():
H = lambda f: md5(f.func_code.co_code).hexdigest()
# Allow serial to be returned as late as tpc_finish
#
# This makes possible for storage to allocate serial inside tpc_finish,
# removing the requirement to serialise second commit phase (tpc_vote
# to tpc_finish/tpc_abort).
h = H(Connection.tpc_finish)
def tpc_finish(self, transaction):
"""Indicate confirmation that the transaction is done."""
def callback(tid):
if self._mvcc_storage:
# Inter-connection invalidation is not needed when the
# storage provides MVCC.
return
d = dict.fromkeys(self._modified)
self._db.invalidate(tid, d, self)
# It's important that the storage calls the passed function
# while it still has its lock. We don't want another thread
# to be able to read any updated data until we've had a chance
# to send an invalidation message to all of the other
# connections!
# <patch>
serial = self._storage.tpc_finish(transaction, callback)
if serial is not None:
assert isinstance(serial, bytes), repr(serial)
for oid_iterator in (self._modified, self._creating):
for oid in oid_iterator:
obj = self._cache.get(oid, None)
# Ignore missing objects and don't update ghosts.
if obj is not None and obj._p_changed is not None:
obj._p_changed = 0
obj._p_serial = serial
# </patch>
self._tpc_cleanup()
global OLD_ZODB
OLD_ZODB = h in (
'ab9b1b8d82c40e5fffa84f7bc4ea3a8b', # Python 2.7
)
if OLD_ZODB:
Connection.tpc_finish = tpc_finish
elif hasattr(Connection, '_handle_serial'): # merged upstream ?
if hasattr(Connection, '_handle_serial'): # merged upstream ?
assert hasattr(Connection, '_warn_about_returned_serial')
# sync() is used to provide a "network barrier", which is required for
......
......@@ -25,9 +25,6 @@ except ImportError:
from cPickle import dumps, loads
_protocol = 1
from ZODB.POSException import UndoError, ConflictError, ReadConflictError
from . import OLD_ZODB
if OLD_ZODB:
from ZODB.ConflictResolution import ResolvedSerial
from persistent.TimeStamp import TimeStamp
from neo.lib import logging
......@@ -641,9 +638,6 @@ class Application(ThreadedApplication):
# - If possible, recover from master failure.
if txn_context.error:
raise NEOStorageError(txn_context.error)
if OLD_ZODB:
return [(oid, ResolvedSerial)
for oid in txn_context.resolved_dict]
return txn_context.resolved_dict
def tpc_abort(self, transaction):
......
......@@ -609,6 +609,10 @@ class ImporterDatabaseManager(DatabaseManager):
except TypeError: # loadBefore returned None
return False
except POSKeyError:
# loadBefore does not distinguish between an oid:
# - that does not exist at any serial
# - that was deleted
# - whose creation was undone
assert not o or o[3] is None, o
return o
if serial != tid:
......@@ -617,22 +621,15 @@ class ImporterDatabaseManager(DatabaseManager):
u_tid = u64(serial)
if u_tid <= self.zodb_tid and o:
return o
if value:
value = zodb.repickle(value)
checksum = util.makeChecksum(value)
else:
# CAVEAT: Although we think loadBefore should not return an empty
# value for a deleted object (BBB: fixed in ZODB4),
# there's no need to distinguish this case in the above
# except clause because it would be crazy to import a
# NEO DB using this backend.
checksum = None
if not next_serial:
next_serial = db._getNextTID(db._getPartition(u_oid), u_oid, u_tid)
if next_serial:
next_serial = p64(next_serial)
return (serial, next_serial,
0, checksum, value, zodb.getDataTid(z_oid, u_tid))
return (serial, next_serial, 0,
util.makeChecksum(value),
value,
zodb.getDataTid(z_oid, u_tid))
def getTransaction(self, tid, all=False):
u64 = util.u64
......
......@@ -517,6 +517,15 @@ class TransactionalResource(object):
return lambda *_: None
return self.__getattribute__(attr)
try:
from ZODB.Connection import TransactionMetaData
except ImportError: # BBB: ZODB < 5
def getTransactionMetaData(txn, conn):
return txn
else:
def getTransactionMetaData(txn, conn):
return txn.data(conn)
class Patch(object):
"""
......
......@@ -330,6 +330,15 @@ class Node(object):
def filterConnection(self, *peers):
return ConnectionFilter(self.getConnectionList(*peers))
@contextmanager
def patchDeferred(self, method):
deferred = []
with Patch(method.__self__, **{method.__name__:
lambda orig, *args, **kw: deferred.append(
partial(orig, *args, **kw))}) as p:
yield p
self.em.wakeup(*deferred)
class ServerNode(Node):
_server_class_dict = {}
......
......@@ -36,7 +36,7 @@ from neo.lib.handler import DelayEvent, EventHandler
from neo.lib import logging
from neo.lib.protocol import (CellStates, ClusterStates, NodeStates, NodeTypes,
Packets, Packet, uuid_str, ZERO_OID, ZERO_TID, MAX_TID)
from .. import Patch, TransactionalResource
from .. import Patch, TransactionalResource, getTransactionMetaData
from . import ClientApplication, ConnectionFilter, LockLock, NEOCluster, \
NEOThreadedTest, RandomConflictDict, Serialized, ThreadId, with_cluster
from neo.lib.util import add64, makeChecksum, p64, u64
......@@ -1312,10 +1312,7 @@ class Test(NEOThreadedTest):
# Check that the storage hasn't answered to the store,
# which means that a lock is still taken for r['x'] by t2.
self.tic()
try:
txn = txn.data(c1)
except (AttributeError, KeyError): # BBB: ZODB < 5
pass
txn = getTransactionMetaData(txn, c1)
txn_context = cluster.client._txn_container.get(txn)
empty = txn_context.queue.empty()
ll()
......
......@@ -19,18 +19,22 @@ from cStringIO import StringIO
from itertools import izip_longest
import os, random, shutil, threading, time, unittest
import transaction, ZODB
from persistent import Persistent
from neo.client.exception import NEOPrimaryMasterLost
from neo.lib import logging
from neo.lib.protocol import MAX_TID
from neo.lib.util import cached_property, p64, u64
from neo.master.transactions import TransactionManager
from neo.storage.database import getAdapterKlass, importer, manager
from neo.storage.database.importer import \
Repickler, TransactionRecord, WriteBack
from .. import expectedFailure, getTempDirectory, random_tree, Patch
from .. import expectedFailure, getTempDirectory, random_tree, \
Patch, TransactionalResource, getTransactionMetaData
from . import NEOCluster, NEOThreadedTest
from ZODB import serialize
from ZODB.DB import TransactionalUndo
from ZODB.FileStorage import FileStorage
from ZODB.POSException import POSKeyError
class Equal:
......@@ -371,6 +375,45 @@ class ImporterTests(NEOThreadedTest):
t.begin()
finalCheck(r)
def testDeleteAndUndo(self):
fs_path, cfg = self.getFS()
c = ZODB.DB(FileStorage(fs_path)).open()
s = c.db().storage
r = c.root()
tid = r._p_serial
r[''] = delete = Persistent()
transaction.commit()
self.assertEqual(delete._p_oid, p64(1))
del r['']
TransactionalResource(transaction, 0, commit=lambda txn:
s.deleteObject(delete._p_oid, delete._p_serial,
getTransactionMetaData(txn, c)))
transaction.commit()
r[''] = undo = Persistent()
transaction.commit()
c.db().undo(s.undoLog(last=1)[0]['id'])
transaction.commit()
self.assertEqual(undo._p_oid, p64(2))
def check():
self.assertIsNone(s.loadBefore(delete._p_oid, tid))
for oid in delete._p_oid, undo._p_oid, p64(3):
self.assertRaises(POSKeyError, s.loadBefore, oid, MAX_TID)
check() # FileStorage
c.db().close()
importer = {'zodb': [('root', cfg)]}
with NEOCluster(importer=importer) as cluster:
storage = cluster.storage
dm = storage.dm
with storage.patchDeferred(dm._finished):
with storage.patchDeferred(dm.doOperation):
cluster.start()
s = cluster.getZODBStorage()
check() # before import
self.tic()
check() # imported, Importer getObject
self.tic()
check() # imported, direct getObject
if __name__ == "__main__":
unittest.main()
......@@ -44,7 +44,7 @@ get3rdParty(x, '3rdparty/' + x, 'https://lab.nexedi.com/nexedi/erp5'
'/raw/14b0fcdcc31c5791646f9590678ca028f5d221f5/product/ERP5Type/' + x,
'abb7970856540fd02150edd1fa9a3a3e8d0074ec526ab189684ef7ea9b41825f')
zodb_require = ['ZODB3>=3.10dev']
zodb_require = ['ZODB>=4.4.5']
extras_require = {
'admin': [],
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment