Commit 0789da45 authored by Jim Fulton's avatar Jim Fulton

There is a new pool-timeout database configuration option to specify that

connections unused after the given time interval should be garbage
colection.  This will provide a means of dealing with extra
connections that are created in rare circumstances and that would
consume an unreasonable amount of memory.
parent 96a9be81
...@@ -63,6 +63,12 @@ New Features ...@@ -63,6 +63,12 @@ New Features
wastage. Now, when connections are placed on the stack, they sink wastage. Now, when connections are placed on the stack, they sink
below existing connections that have more active objects. below existing connections that have more active objects.
- There is a new pool-timeout database configuration option to specify that
connections unused after the given time interval should be garbage
colection. This will provide a means of dealing with extra
connections that are created in rare circumstances and that would
consume an unreasonable amount of memory.
3.9.0a8 (2008-12-15) 3.9.0a8 (2008-12-15)
==================== ====================
......
...@@ -12,8 +12,7 @@ ...@@ -12,8 +12,7 @@
# #
############################################################################## ##############################################################################
"""Database objects """Database objects
"""
$Id$"""
import warnings import warnings
...@@ -70,7 +69,7 @@ class AbstractConnectionPool(object): ...@@ -70,7 +69,7 @@ class AbstractConnectionPool(object):
connectionDebugInfo() can still gather statistics. connectionDebugInfo() can still gather statistics.
""" """
def __init__(self, size, timeout=None): def __init__(self, size, timeout):
# The largest # of connections we expect to see alive simultaneously. # The largest # of connections we expect to see alive simultaneously.
self._size = size self._size = size
...@@ -95,8 +94,7 @@ class AbstractConnectionPool(object): ...@@ -95,8 +94,7 @@ class AbstractConnectionPool(object):
def setTimeout(self, timeout): def setTimeout(self, timeout):
old = self._timeout old = self._timeout
self._timeout = timeout self._timeout = timeout
if timeout is not None and old != timeout and ( if timeout < old:
old is None or old > timeout):
self._reduce_size() self._reduce_size()
def getSize(self): def getSize(self):
...@@ -112,7 +110,7 @@ class AbstractConnectionPool(object): ...@@ -112,7 +110,7 @@ class AbstractConnectionPool(object):
class ConnectionPool(AbstractConnectionPool): class ConnectionPool(AbstractConnectionPool):
# XXX WTF, passing time.time() as a default? # XXX WTF, passing time.time() as a default?
def __init__(self, size, timeout=time.time()): def __init__(self, size, timeout=1<<31):
super(ConnectionPool, self).__init__(size, timeout) super(ConnectionPool, self).__init__(size, timeout)
# A stack of connections available to hand out. This is a subset # A stack of connections available to hand out. This is a subset
...@@ -245,7 +243,7 @@ class KeyedConnectionPool(AbstractConnectionPool): ...@@ -245,7 +243,7 @@ class KeyedConnectionPool(AbstractConnectionPool):
# see the comments in ConnectionPool for method descriptions. # see the comments in ConnectionPool for method descriptions.
def __init__(self, size, timeout=time.time()): def __init__(self, size, timeout=1<<31):
super(KeyedConnectionPool, self).__init__(size, timeout) super(KeyedConnectionPool, self).__init__(size, timeout)
self.pools = {} self.pools = {}
...@@ -307,7 +305,6 @@ class KeyedConnectionPool(AbstractConnectionPool): ...@@ -307,7 +305,6 @@ class KeyedConnectionPool(AbstractConnectionPool):
return tuple(result) return tuple(result)
def toTimeStamp(dt): def toTimeStamp(dt):
utc_struct = dt.utctimetuple() utc_struct = dt.utctimetuple()
# if this is a leapsecond, this will probably fail. That may be a good # if this is a leapsecond, this will probably fail. That may be a good
...@@ -379,6 +376,7 @@ class DB(object): ...@@ -379,6 +376,7 @@ class DB(object):
def __init__(self, storage, def __init__(self, storage,
pool_size=7, pool_size=7,
pool_timeout=1<<31,
cache_size=400, cache_size=400,
cache_size_bytes=0, cache_size_bytes=0,
historical_pool_size=3, historical_pool_size=3,
...@@ -416,7 +414,7 @@ class DB(object): ...@@ -416,7 +414,7 @@ class DB(object):
self._r = x.release self._r = x.release
# pools and cache sizes # pools and cache sizes
self.pool = ConnectionPool(pool_size) self.pool = ConnectionPool(pool_size, pool_timeout)
self.historical_pool = KeyedConnectionPool(historical_pool_size, self.historical_pool = KeyedConnectionPool(historical_pool_size,
historical_timeout) historical_timeout)
self._cache_size = cache_size self._cache_size = cache_size
......
...@@ -250,6 +250,11 @@ ...@@ -250,6 +250,11 @@
and exceeding twice pool-size connections causes a critical and exceeding twice pool-size connections causes a critical
message to be logged. message to be logged.
</description> </description>
<key name="pool-timeout" datatype="time-interval"/>
<description>
The minimum interval that an unused (non-historical)
connection should be kept.
</description>
<key name="historical-pool-size" datatype="integer" default="3"/> <key name="historical-pool-size" datatype="integer" default="3"/>
<description> <description>
The expected maximum total number of historical connections The expected maximum total number of historical connections
......
...@@ -92,6 +92,10 @@ class ZODBDatabase(BaseConfig): ...@@ -92,6 +92,10 @@ class ZODBDatabase(BaseConfig):
def open(self, databases=None): def open(self, databases=None):
section = self.config section = self.config
storage = section.storage.open() storage = section.storage.open()
options = {}
if section.pool_timeout is not None:
options['pool_timeout'] = section.pool_timeout
try: try:
return ZODB.DB( return ZODB.DB(
storage, storage,
...@@ -104,7 +108,7 @@ class ZODBDatabase(BaseConfig): ...@@ -104,7 +108,7 @@ class ZODBDatabase(BaseConfig):
historical_timeout=section.historical_timeout, historical_timeout=section.historical_timeout,
database_name=section.database_name, database_name=section.database_name,
databases=databases, databases=databases,
) **options)
except: except:
storage.close() storage.close()
raise raise
......
...@@ -19,6 +19,7 @@ import ZEO.ClientStorage ...@@ -19,6 +19,7 @@ import ZEO.ClientStorage
import ZODB.config import ZODB.config
import ZODB.POSException import ZODB.POSException
import ZODB.tests.util import ZODB.tests.util
from zope.testing import doctest
class ConfigTestBase(ZODB.tests.util.TestCase): class ConfigTestBase(ZODB.tests.util.TestCase):
...@@ -128,11 +129,35 @@ class ZEOConfigTest(ConfigTestBase): ...@@ -128,11 +129,35 @@ class ZEOConfigTest(ConfigTestBase):
os.path.abspath('blobs')) os.path.abspath('blobs'))
self.assertRaises(ClientDisconnected, self._test, cfg) self.assertRaises(ClientDisconnected, self._test, cfg)
def db_connection_pool_timeout():
"""
Test that the database pool timeout option works:
>>> db = ZODB.config.databaseFromString('''
... <zodb>
... <mappingstorage/>
... </zodb>
... ''')
>>> db.pool._timeout == 1<<31
True
>>> db = ZODB.config.databaseFromString('''
... <zodb>
... pool-timeout 600
... <mappingstorage/>
... </zodb>
... ''')
>>> db.pool._timeout == 600
True
"""
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(ZODBConfigTest)) suite.addTest(unittest.makeSuite(ZODBConfigTest))
suite.addTest(unittest.makeSuite(ZEOConfigTest)) suite.addTest(unittest.makeSuite(ZEOConfigTest))
suite.addTest(doctest.DocTestSuite())
return suite return suite
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment