Commit 0ad980c8 by Julien Muchembled

Drop compatibility with Python < 2.6 and ZODB < 3.9

Also simplify some code by using some new Python syntax.
1 parent 0f049140
Change History
==============
1.0 (unreleased)
----------------
- Drop compatibility with Python < 2.6 and ZODB < 3.9
0.10 (2011-10-17)
-----------------
......
......@@ -54,19 +54,13 @@ Requirements
- Linux 2.6 or later
- Python 2.4 or later
- For python 2.4: `ctypes <http://python.net/crew/theller/ctypes/>`_
(packaged with later python versions)
Note that setup.py does not define any dependency to 'ctypes' so you will
have to install it explicitely.
- Python 2.6.x or 2.7.x
- For storage nodes:
- MySQLdb: http://sourceforge.net/projects/mysql-python
- For client nodes: ZODB 3.10.x but it should work with ZODB >= 3.4
- For client nodes: ZODB 3.10.x but it should work with ZODB 3.9.x
Installation
============
......
......@@ -35,18 +35,6 @@ def check_read_only(func):
return func(self, *args, **kw)
return wraps(func)(wrapped)
def old_history_api(func):
try:
if ZODB.interfaces.IStorage['history'].positional[1] != 'version':
return func # ZODB >= 3.9
except KeyError: # ZODB < 3.8
pass
def history(self, oid, version=None, *args, **kw):
if version is None:
return func(self, oid, *args, **kw)
raise ValueError('Versions are not supported')
return wraps(func)(history)
class Storage(BaseStorage.BaseStorage,
ConflictResolution.ConflictResolvingStorage):
"""Wrapper class for neoclient."""
......@@ -72,8 +60,8 @@ class Storage(BaseStorage.BaseStorage,
# - transaction isolation is not done
# ZODB.interfaces.IStorageIteration,
ZODB.interfaces.IStorageUndoable,
getattr(ZODB.interfaces, 'IExternalGC', None), # XXX ZODB < 3.9
getattr(ZODB.interfaces, 'ReadVerifyingStorage', None), # XXX ZODB 3.9
ZODB.interfaces.IExternalGC,
getattr(ZODB.interfaces, 'ReadVerifyingStorage', None), # BBB ZODB 3.9
)))
def __init__(self, master_nodes, name, read_only=False,
......@@ -227,7 +215,6 @@ class Storage(BaseStorage.BaseStorage,
def registerDB(self, db, limit=None):
self.app.registerDB(db, limit)
@old_history_api
def history(self, oid, *args, **kw):
try:
return self.app.history(oid, *args, **kw)
......
......@@ -28,8 +28,7 @@ if 1:
"""Indicate confirmation that the transaction is done."""
def callback(tid):
# BBB: _mvcc_storage not supported on older ZODB
if getattr(self, '_mvcc_storage', False):
if self._mvcc_storage:
# Inter-connection invalidation is not needed when the
# storage provides MVCC.
return
......@@ -73,8 +72,8 @@ if 1:
def _flush_invalidations():
acquire = self._db._a
try:
self._db._r() # this is a RLock
except (AssertionError, RuntimeError): # old Python uses assert
self._db._r() # this is a RLock
except RuntimeError:
acquire = lambda: None
try:
del self._flush_invalidations
......@@ -87,12 +86,9 @@ if 1:
Connection_open(self, *args, **kw)
finally:
del self._flush_invalidations
try:
Connection_open = Connection._setDB
Connection._setDB = open
except AttributeError: # recent ZODB
Connection_open = Connection.open
Connection.open = open
Connection_open = Connection.open
Connection.open = open
# Storage.sync usually implements a "network barrier" (at least
# in NEO, but ZEO should be fixed to do the same), which is quite
......@@ -107,7 +103,7 @@ if 1:
def afterCompletion(self, *ignored):
try:
self._readCurrent.clear()
except AttributeError: # old ZODB (e.g. ZODB 3.4)
except AttributeError: # BBB: ZODB < 3.10
pass
self._flush_invalidations()
#Connection.afterCompletion = afterCompletion
......@@ -118,8 +114,7 @@ if 1:
Wrapper to DB instance that properly initialize Connection objects
with NEO storages.
It forces the connection to always create a new instance of the
storage, for compatibility with ZODB 3.4, and because we don't
implement IMVCCStorage completely.
storage, because we don't implement IMVCCStorage completely.
"""
def __new__(cls, db, connection):
......@@ -132,23 +127,11 @@ if 1:
def __getattr__(self, attr):
result = getattr(self._db, attr)
if attr in ('storage', '_storage'):
result = result.new_instance()
if attr == 'storage':
self.storage = result = result.new_instance()
self._connection._db = self._db
setattr(self, attr, result)
return result
try:
Connection_setDB = Connection._setDB
except AttributeError: # recent ZODB
Connection_init = Connection.__init__
Connection.__init__ = lambda self, db, *args, **kw: \
Connection_init(self, _DB(db, self), *args, **kw)
else: # old ZODB (e.g. ZODB 3.4)
Connection._setDB = lambda self, odb, *args, **kw: \
Connection_setDB(self, _DB(odb, self), *args, **kw)
from ZODB.DB import DB
DB_invalidate = DB.invalidate
DB.invalidate = lambda self, tid, oids, *args, **kw: \
DB_invalidate(self, tid, dict.fromkeys(oids, None), *args, **kw)
Connection_init = Connection.__init__
Connection.__init__ = lambda self, db, *args, **kw: \
Connection_init(self, _DB(db, self), *args, **kw)
......@@ -835,9 +835,7 @@ class Application(object):
# key, so that all cells with the same (smallest) key has
# identical chance to be chosen.
shuffle(cell_list)
# BBB: min(..., key=...) requires Python >= 2.5
cell_list.sort(key=getCellSortKey)
storage_conn = getConnForCell(cell_list[0])
storage_conn = getConnForCell(min(cell_list, key=getCellSortKey))
storage_conn.ask(Packets.AskObjectUndoSerial(ttid,
snapshot_tid, undone_tid, oid_list), queue=queue)
......
......@@ -26,7 +26,7 @@ class _ThreadedPoll(Thread):
def __init__(self, em, **kw):
Thread.__init__(self, **kw)
self.em = em
self.setDaemon(True)
self.daemon = True
self._stop = Event()
def run(self):
......
......@@ -151,11 +151,9 @@ class ConnectionPool(object):
def getConnForNode(self, node):
"""Return a locked connection object to a given node
If no connection exists, create a new one"""
if not node.isRunning():
return None
uuid = node.getUUID()
self.connection_lock_acquire()
try:
if node.isRunning():
uuid = node.getUUID()
self.connection_lock_acquire()
try:
# Already connected to node
return self.connection_dict[uuid]
......@@ -164,15 +162,12 @@ class ConnectionPool(object):
# must drop some unused connections
self._dropConnections()
# Create new connection to node
while True:
conn = self._initNodeConnection(node)
if conn is not None:
self.connection_dict[uuid] = conn
return conn
else:
return None
finally:
self.connection_lock_release()
conn = self._initNodeConnection(node)
if conn is not None:
self.connection_dict[uuid] = conn
return conn
finally:
self.connection_lock_release()
@profiler_decorator
def removeConnection(self, node):
......
......@@ -15,7 +15,6 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import neo.lib.python
import logging as logging_std
FMT = ('%(asctime)s %(levelname)-9s %(name)-10s'
......
......@@ -17,7 +17,6 @@
import traceback
import signal
import ctypes
import imp
import os
import sys
......@@ -41,20 +40,14 @@ ENABLED = False
# SIGUSR2:
# Triggers a pdb prompt on process' controlling TTY.
libc = ctypes.cdll.LoadLibrary('libc.so.6')
errno = ctypes.c_int.in_dll(libc, 'errno')
def decorate(func):
def decorator(sig, frame):
# Save errno value, to restore it after sig handler returns
old_errno = errno.value
try:
func(sig, frame)
except:
# Prevent exception from exiting signal handler, so mistakes in
# "debug" module don't kill process.
traceback.print_exc()
errno.value = old_errno
return wraps(func)(decorator)
@decorate
......
......@@ -15,25 +15,17 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
r"""This is an epoll(4) interface available in Linux 2.6. This requires
ctypes <http://python.net/crew/theller/ctypes/>."""
"""This is an epoll(4) interface available in Linux 2.6."""
from ctypes import cdll, Union, Structure, \
c_void_p, c_int, byref
try:
from ctypes import c_uint32, c_uint64
except ImportError:
from ctypes import c_uint, c_ulonglong
c_uint32 = c_uint
c_uint64 = c_ulonglong
from ctypes import CDLL, get_errno, Union, Structure
from ctypes import c_void_p, c_int, byref, c_uint32, c_uint64
from os import close
from errno import EINTR, EAGAIN
libc = cdll.LoadLibrary('libc.so.6')
libc = CDLL('libc.so.6', use_errno=True)
epoll_create = libc.epoll_create
epoll_wait = libc.epoll_wait
epoll_ctl = libc.epoll_ctl
errno = c_int.in_dll(libc, 'errno')
EPOLLIN = 0x001
EPOLLPRI = 0x002
......@@ -69,7 +61,7 @@ class Epoll(object):
def __init__(self):
self.efd = epoll_create(10)
if self.efd == -1:
raise OSError(errno.value, 'epoll_create failed')
raise OSError(get_errno(), 'epoll_create failed')
self.maxevents = 1024 # XXX arbitrary
epoll_event_array = EpollEvent * self.maxevents
......@@ -85,8 +77,8 @@ class Epoll(object):
n = epoll_wait(self.efd, byref(self.events), self.maxevents,
timeout)
if n == -1:
e = errno.value
# XXX: Why 0 ? Maybe due to partial workaround in neo.lib.debug.
e = get_errno()
# XXX: Why 0 ?
if e in (0, EINTR, EAGAIN):
continue
else:
......@@ -111,7 +103,7 @@ class Epoll(object):
ev.data.fd = fd
ret = epoll_ctl(self.efd, EPOLL_CTL_ADD, fd, byref(ev))
if ret == -1:
raise OSError(errno.value, 'epoll_ctl failed')
raise OSError(get_errno(), 'epoll_ctl failed')
def modify(self, fd, readable, writable):
ev = EpollEvent()
......@@ -124,13 +116,13 @@ class Epoll(object):
ev.events = events
ret = epoll_ctl(self.efd, EPOLL_CTL_MOD, fd, byref(ev))
if ret == -1:
raise OSError(errno.value, 'epoll_ctl failed')
raise OSError(get_errno(), 'epoll_ctl failed')
def unregister(self, fd):
ev = EpollEvent()
ret = epoll_ctl(self.efd, EPOLL_CTL_DEL, fd, byref(ev))
if ret == -1:
raise OSError(errno.value, 'epoll_ctl failed')
raise OSError(get_errno(), 'epoll_ctl failed')
def __del__(self):
efd = self.efd
......
# Copyright (C) 2011 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import sys, types
if sys.version_info < (2, 5):
import __builtin__, imp
def all(iterable):
"""
Return True if bool(x) is True for all values x in the iterable.
"""
for x in iterable:
if not x:
return False
return True
__builtin__.all = all
def any(iterable):
"""
Return True if bool(x) is True for any x in the iterable.
"""
for x in iterable:
if x:
return True
return False
__builtin__.any = any
import md5, sha
sys.modules['hashlib'] = hashlib = imp.new_module('hashlib')
hashlib.md5 = md5.new
hashlib.sha1 = sha.new
import struct
class Struct(object):
def __init__(self, fmt):
self._fmt = fmt
self.size = struct.calcsize(fmt)
def pack(self, *args):
return struct.pack(self._fmt, *args)
def unpack(self, *args):
return struct.unpack(self._fmt, *args)
struct.Struct = Struct
sys.modules['functools'] = functools = imp.new_module('functools')
def wraps(wrapped):
"""Simple backport of functools.wraps from Python >= 2.5"""
def decorator(wrapper):
wrapper.__module__ = wrapped.__module__
wrapper.__name__ = wrapped.__name__
wrapper.__doc__ = wrapped.__doc__
wrapper.__dict__.update(wrapped.__dict__)
return wrapper
return decorator
functools.wraps = wraps
#!/usr/bin/env python
#
# neoadmin - run an administrator node of NEO
#
# Copyright (C) 2009 Nexedi SA
......
#!/usr/bin/env python
#
# neoadmin - run an administrator node of NEO
#
# Copyright (C) 2009 Nexedi SA
......
#!/usr/bin/env python
#
# neomaster - run a master node of NEO
#
# Copyright (C) 2006 Nexedi SA
......
#! /usr/bin/env python2.4
#!/usr/bin/env python
#
# neomaster - run a master node of NEO
#
......
#! /usr/bin/env python2.4
#!/usr/bin/env python
#
# neostorage - run a storage node of NEO
#
......
#! /usr/bin/env python
#!/usr/bin/env python
#
# Copyright (C) 2009 Nexedi SA
#
......
......@@ -263,8 +263,7 @@ class BTreeDatabaseManager(DatabaseManager):
uncommitted_data = self._uncommitted_data
def deleter_callback(tree, key_list):
for tid in key_list:
checksum = tree[tid][0] # BBB: recent ZODB provides pop()
del tree[tid] #
checksum = tree.pop(tid)[0]
if checksum:
index = data[checksum][2]
index.remove((oid, tid))
......@@ -277,7 +276,7 @@ class BTreeDatabaseManager(DatabaseManager):
checksum_list = []
checksum_set = set()
for oid in key_list:
tserial = tree[oid]; del tree[oid] # BBB: recent ZODB provides pop()
tserial = tree.pop(oid)
for tid, (checksum, _) in tserial.items():
if checksum:
index = data[checksum][2]
......
......@@ -557,9 +557,7 @@ class MySQLDatabaseManager(DatabaseManager):
oid = u64(oid)
sql = " FROM obj WHERE partition=%d AND oid=%d AND serial=%d" \
% (getPartition(oid), oid, tid)
hash_list = q("SELECT hash" + sql)
if hash_list: # BBB: Python < 2.6
checksum_set.update(*hash_list)
checksum_set.update(*q("SELECT hash" + sql))
q("DELETE" + sql)
checksum_set.discard(None)
self._pruneData(checksum_set)
......
......@@ -81,13 +81,10 @@ class ClusterDict(dict):
self._r, self._w = os.pipe()
# shm_open(3) would be better but Python doesn't provide it.
# See also http://nikitathespider.com/python/shm/
f = tempfile.TemporaryFile()
try:
with tempfile.TemporaryFile() as f:
f.write(dumps(self.copy(), -1))
f.flush()
self._shared = mmap.mmap(f.fileno(), f.tell())
finally:
f.close()
self.release()
def __del__(self):
......
......@@ -634,7 +634,7 @@ class NEOFunctionalTest(NeoTestBase):
except:
exc_list.append(sys.exc_info())
thread = threading.Thread(None, excWrapper, args=args, kwargs=kwargs)
thread.setDaemon(True)
thread.daemon = True
thread.start()
thread.join(timeout)
self.assertFalse(thread.isAlive(), 'Run timeout')
......
......@@ -107,12 +107,9 @@ class Serialized(object):
@classmethod
def background(cls):
cls._lock_lock.acquire()
try:
with cls._lock_lock:
if cls._lock_list:
cls._lock_list.popleft().release()
finally:
cls._lock_lock.release()
class SerializedEventManager(EventManager):
......@@ -186,7 +183,7 @@ class ServerNode(Node):
def __init__(self, cluster, address, **kw):
self._init_args = (cluster, address), dict(kw)
threading.Thread.__init__(self)
self.setDaemon(True)
self.daemon = True
h, p = address
self.node_type = getattr(NodeTypes,
SERVER_TYPE[VIRTUAL_IP.index(h)].upper())
......@@ -394,8 +391,7 @@ class ConnectionFilter(object):
return queue
def __call__(self, revert=1):
self.lock.acquire()
try:
with self.lock:
self.filter_dict.clear()
self._retry()
if revert:
......@@ -403,8 +399,6 @@ class ConnectionFilter(object):
assert not queue
del conn._addPacket
del self.conn_list[:]
finally:
self.lock.release()
def _retry(self):
for conn, queue in self.conn_list:
......@@ -423,20 +417,14 @@ class ConnectionFilter(object):
self(0)
def add(self, filter, *patches):
self.lock.acquire()
try:
with self.lock:
self.filter_dict[filter] = patches
finally:
self.lock.release()
def remove(self, *filters):
self.lock.acquire()
try:
with self.lock:
for filter in filters:
del self.filter_dict[filter]
self._retry()
finally:
self.lock.release()
def __contains__(self, filter):
return filter in self.filter_dict
......@@ -665,7 +653,7 @@ class NEOThreadedTest(NeoTestBase):
def __init__(self, func, *args, **kw):
threading.Thread.__init__(self)
self.__target = func, args, kw
self.setDaemon(True)
self.daemon = True
self.start()
def run(self):
......
......@@ -9,17 +9,18 @@ Framework :: ZODB
Intended Audience :: Developers
License :: OSI Approved :: GNU General Public License (GPL)
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
Topic :: Database
Topic :: Software Development :: Libraries :: Python Modules
"""
if not os.path.exists('mock.py'):
import cStringIO, md5, urllib, zipfile
import cStringIO, hashlib, urllib, zipfile
mock_py = zipfile.ZipFile(cStringIO.StringIO(urllib.urlopen(
'http://downloads.sf.net/sourceforge/python-mock/pythonmock-0.1.0.zip'
).read())).read('mock.py')
if md5.md5(mock_py).hexdigest() != '79f42f390678e5195d9ce4ae43bd18ec':
if hashlib.md5(mock_py).hexdigest() != '79f42f390678e5195d9ce4ae43bd18ec':
raise EnvironmentError("MD5 checksum mismatch downloading 'mock.py'")
open('mock.py', 'w').write(mock_py)
......@@ -36,7 +37,7 @@ extras_require['tests'] = ['zope.testing', 'psutil',
setup(
name = 'neoppod',
version = '0.10',
version = '1.0-dev',
description = __doc__.strip(),
author = 'NEOPPOD',
author_email = 'neo-dev@erp5.org',
......@@ -62,8 +63,6 @@ setup(
'stat_zodb=neo.tests.stat_zodb:main',
],
},
# Raah!!! I wish I could write something like:
# install_requires = ['python>=2.5|ctypes'],
extras_require = extras_require,
package_data = {
'neo.client': [
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!