Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZODB
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
ZODB
Commits
3a90b8d0
Commit
3a90b8d0
authored
Sep 10, 2008
by
Jim Fulton
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Merged Dieter's branch that ads an option to drop a zeo cache rather
than verifying it.
parent
f741a067
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
128 additions
and
10 deletions
+128
-10
src/CHANGES.txt
src/CHANGES.txt
+6
-0
src/ZEO/ClientStorage.py
src/ZEO/ClientStorage.py
+47
-9
src/ZEO/component.xml
src/ZEO/component.xml
+10
-0
src/ZEO/tests/testZEO.py
src/ZEO/tests/testZEO.py
+57
-1
src/ZODB/component.xml
src/ZODB/component.xml
+7
-0
src/ZODB/config.py
src/ZODB/config.py
+1
-0
No files found.
src/CHANGES.txt
View file @
3a90b8d0
...
@@ -8,6 +8,12 @@ Change History
...
@@ -8,6 +8,12 @@ Change History
New Features
New Features
------------
------------
- New `ClientStorage` configuration option `drop_cache_rather_verify`.
If this option is true then the ZEO client cache is dropped instead of
the long (unoptimized) verification. For large caches, setting this
option can avoid effective downtimes in the order of hours when
the connection to the ZEO server was interrupted for a longer time.
- The connection now estimates the object size based on its pickle size
- The connection now estimates the object size based on its pickle size
and informs the cache about size changes.
and informs the cache about size changes.
...
...
src/ZEO/ClientStorage.py
View file @
3a90b8d0
...
@@ -122,6 +122,7 @@ class ClientStorage(object):
...
@@ -122,6 +122,7 @@ class ClientStorage(object):
wait_for_server_on_startup
=
None
,
# deprecated alias for wait
wait_for_server_on_startup
=
None
,
# deprecated alias for wait
wait
=
None
,
wait_timeout
=
None
,
wait
=
None
,
wait_timeout
=
None
,
read_only
=
0
,
read_only_fallback
=
0
,
read_only
=
0
,
read_only_fallback
=
0
,
drop_cache_rather_verify
=
False
,
username
=
''
,
password
=
''
,
realm
=
None
,
username
=
''
,
password
=
''
,
realm
=
None
,
blob_dir
=
None
,
shared_blob_dir
=
False
):
blob_dir
=
None
,
shared_blob_dir
=
False
):
"""ClientStorage constructor.
"""ClientStorage constructor.
...
@@ -196,6 +197,9 @@ class ClientStorage(object):
...
@@ -196,6 +197,9 @@ class ClientStorage(object):
realm -- not documented.
realm -- not documented.
drop_cache_rather_verify -- a flag indicating that the cache
should be dropped rather than expensively verified.
blob_dir -- directory path for blob data. 'blob data' is data that
blob_dir -- directory path for blob data. 'blob data' is data that
is retrieved via the loadBlob API.
is retrieved via the loadBlob API.
...
@@ -218,6 +222,14 @@ class ClientStorage(object):
...
@@ -218,6 +222,14 @@ class ClientStorage(object):
if
debug
:
if
debug
:
log2
(
"ClientStorage(): debug argument is no longer used"
)
log2
(
"ClientStorage(): debug argument is no longer used"
)
# Remember some parameters for "_setupCache"
self
.
_var_
=
var
self
.
_storage_
=
storage
self
.
_client_
=
client
self
.
_cache_size_
=
cache_size
self
.
_drop_cache_rather_verify
=
drop_cache_rather_verify
# wait defaults to True, but wait_for_server_on_startup overrides
# wait defaults to True, but wait_for_server_on_startup overrides
# if not None
# if not None
if
wait_for_server_on_startup
is
not
None
:
if
wait_for_server_on_startup
is
not
None
:
...
@@ -342,13 +354,7 @@ class ClientStorage(object):
...
@@ -342,13 +354,7 @@ class ClientStorage(object):
else
:
else
:
self
.
fshelper
=
None
self
.
fshelper
=
None
# Decide whether to use non-temporary files
self
.
_setupCache
()
if
client
is
not
None
:
dir
=
var
or
os
.
getcwd
()
cache_path
=
os
.
path
.
join
(
dir
,
"%s-%s.zec"
%
(
client
,
storage
))
else
:
cache_path
=
None
self
.
_cache
=
self
.
ClientCacheClass
(
cache_path
,
size
=
cache_size
)
self
.
_rpc_mgr
=
self
.
ConnectionManagerClass
(
addr
,
self
,
self
.
_rpc_mgr
=
self
.
ConnectionManagerClass
(
addr
,
self
,
tmin
=
min_disconnect_poll
,
tmin
=
min_disconnect_poll
,
...
@@ -363,6 +369,19 @@ class ClientStorage(object):
...
@@ -363,6 +369,19 @@ class ClientStorage(object):
if
not
self
.
_rpc_mgr
.
attempt_connect
():
if
not
self
.
_rpc_mgr
.
attempt_connect
():
self
.
_rpc_mgr
.
connect
()
self
.
_rpc_mgr
.
connect
()
def
_setupCache
(
self
):
'''create and open the cache.'''
# Decide whether to use non-temporary files
storage
=
self
.
_storage_
client
=
self
.
_client_
cache_size
=
self
.
_cache_size_
if
client
is
not
None
:
dir
=
self
.
_var_
or
os
.
getcwd
()
cache_path
=
os
.
path
.
join
(
dir
,
"%s-%s.zec"
%
(
client
,
storage
))
else
:
cache_path
=
None
self
.
_cache
=
self
.
ClientCacheClass
(
cache_path
,
size
=
cache_size
)
def
_wait
(
self
,
timeout
=
None
):
def
_wait
(
self
,
timeout
=
None
):
if
timeout
is
not
None
:
if
timeout
is
not
None
:
deadline
=
time
.
time
()
+
timeout
deadline
=
time
.
time
()
+
timeout
...
@@ -1225,6 +1244,23 @@ class ClientStorage(object):
...
@@ -1225,6 +1244,23 @@ class ClientStorage(object):
elif
ltid
and
ltid
!=
utils
.
z64
:
elif
ltid
and
ltid
!=
utils
.
z64
:
self
.
_cache
.
setLastTid
(
ltid
)
self
.
_cache
.
setLastTid
(
ltid
)
# From this point on, we do not have complete information about
# the missed transactions. The reason is that cache
# verification only checks objects in the client cache and
# there may be objects in the object caches that aren't in the
# client cach that would need verification too. We avoid that
# problem by just invalidating the objects in the object caches.
if
self
.
_db
is
not
None
:
self
.
_db
.
invalidateCache
()
if
self
.
_cache
and
self
.
_drop_cache_rather_verify
:
log2
(
"dropping cache"
)
self
.
_cache
.
close
()
self
.
_setupCache
()
# creates a new cache
self
.
_server
=
server
self
.
_ready
.
set
()
return
"cache dropped"
log2
(
"Verifying cache"
)
log2
(
"Verifying cache"
)
for
oid
,
tid
in
self
.
_cache
.
contents
():
for
oid
,
tid
in
self
.
_cache
.
contents
():
server
.
verify
(
oid
,
tid
)
server
.
verify
(
oid
,
tid
)
...
@@ -1381,7 +1417,8 @@ class TransactionIterator(object):
...
@@ -1381,7 +1417,8 @@ class TransactionIterator(object):
class
ClientStorageTransactionInformation
(
ZODB
.
BaseStorage
.
TransactionRecord
):
class
ClientStorageTransactionInformation
(
ZODB
.
BaseStorage
.
TransactionRecord
):
def
__init__
(
self
,
storage
,
txiter
,
tid
,
status
,
user
,
description
,
extension
):
def
__init__
(
self
,
storage
,
txiter
,
tid
,
status
,
user
,
description
,
extension
):
self
.
_storage
=
storage
self
.
_storage
=
storage
self
.
_txiter
=
txiter
self
.
_txiter
=
txiter
self
.
_completed
=
False
self
.
_completed
=
False
...
@@ -1394,7 +1431,8 @@ class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
...
@@ -1394,7 +1431,8 @@ class ClientStorageTransactionInformation(ZODB.BaseStorage.TransactionRecord):
self
.
extension
=
extension
self
.
extension
=
extension
def
__iter__
(
self
):
def
__iter__
(
self
):
riid
=
self
.
_storage
.
_server
.
iterator_record_start
(
self
.
_txiter
.
_iid
,
self
.
tid
)
riid
=
self
.
_storage
.
_server
.
iterator_record_start
(
self
.
_txiter
.
_iid
,
self
.
tid
)
return
self
.
_storage
.
_setup_iterator
(
RecordIterator
,
riid
)
return
self
.
_storage
.
_setup_iterator
(
RecordIterator
,
riid
)
...
...
src/ZEO/component.xml
View file @
3a90b8d0
...
@@ -102,6 +102,16 @@
...
@@ -102,6 +102,16 @@
<metadefault>
$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)
</metadefault>
<metadefault>
$INSTANCE/var/ZEO.pid (or $clienthome/ZEO.pid)
</metadefault>
</key>
</key>
<!-- DM 2006-06-12: added option -->
<key
name=
"drop-cache-rather-verify"
datatype=
"boolean"
required=
"no"
default=
"false"
>
<description>
indicates that the cache should be dropped rather than
verified when the verification optimization is not
available (e.g. when the ZEO server restarted).
</description>
</key>
</sectiontype>
</sectiontype>
</component>
</component>
src/ZEO/tests/testZEO.py
View file @
3a90b8d0
...
@@ -60,6 +60,8 @@ logger = logging.getLogger('ZEO.tests.testZEO')
...
@@ -60,6 +60,8 @@ logger = logging.getLogger('ZEO.tests.testZEO')
class
DummyDB
:
class
DummyDB
:
def
invalidate
(
self
,
*
args
):
def
invalidate
(
self
,
*
args
):
pass
pass
def
invalidateCache
(
*
unused
):
pass
class
OneTimeTests
(
unittest
.
TestCase
):
class
OneTimeTests
(
unittest
.
TestCase
):
...
@@ -145,6 +147,58 @@ class MiscZEOTests:
...
@@ -145,6 +147,58 @@ class MiscZEOTests:
self
.
assertNotEquals
(
ZODB
.
utils
.
z64
,
storage3
.
lastTransaction
())
self
.
assertNotEquals
(
ZODB
.
utils
.
z64
,
storage3
.
lastTransaction
())
storage3
.
close
()
storage3
.
close
()
def
checkDropCacheRatherVerifyImplementation
(
self
):
# As it is quite difficult to set things up such that the verification
# optimizations do not step in, we emulate both the cache
# as well as the server.
from
ZODB.TimeStamp
import
TimeStamp
class
CacheEmulator
(
object
):
# the settings below would be inconsitent for a normal cache
# but they are sufficient for our test setup
def
__len__
(
self
):
return
1
# claim not to be empty
def
contents
(
self
):
return
()
# do not invalidate anything
def
getLastTid
(
self
):
return
def
close
(
self
):
pass
class
ServerEmulator
(
object
):
def
verify
(
*
unused
):
pass
def
endZeoVerify
(
*
unused
):
pass
def
lastTransaction
(
*
unused
):
pass
storage
=
self
.
_storage
storage
.
_cache
=
cache
=
CacheEmulator
()
server
=
ServerEmulator
()
# test the standard behaviour
self
.
assertEqual
(
storage
.
verify_cache
(
server
),
"full verification"
)
# test the "drop cache rather verify" behaviour
storage
.
_drop_cache_rather_verify
=
True
self
.
assertEqual
(
storage
.
verify_cache
(
server
),
"cache dropped"
)
# verify that we got a new cache
self
.
assert_
(
cache
!=
storage
.
_cache
)
class
ConfigurationTests
(
unittest
.
TestCase
):
def
checkDropCacheRatherVerifyConfiguration
(
self
):
from
ZODB.config
import
storageFromString
# the default is to do verification and not drop the cache
cs
=
storageFromString
(
'''
<zeoclient>
server localhost:9090
wait false
</zeoclient>
'''
)
self
.
assertEqual
(
cs
.
_drop_cache_rather_verify
,
False
)
cs
.
close
()
# now for dropping
cs
=
storageFromString
(
'''
<zeoclient>
server localhost:9090
wait false
drop-cache-rather-verify true
</zeoclient>
'''
)
self
.
assertEqual
(
cs
.
_drop_cache_rather_verify
,
True
)
cs
.
close
()
class
GenericTests
(
class
GenericTests
(
# Base class for all ZODB tests
# Base class for all ZODB tests
StorageTestBase
.
StorageTestBase
,
StorageTestBase
.
StorageTestBase
,
...
@@ -955,7 +1009,9 @@ transaction, we'll get a result:
...
@@ -955,7 +1009,9 @@ transaction, we'll get a result:
test_classes
=
[
FileStorageTests
,
FileStorageRecoveryTests
,
test_classes
=
[
FileStorageTests
,
FileStorageRecoveryTests
,
MappingStorageTests
,
DemoStorageTests
,
MappingStorageTests
,
DemoStorageTests
,
BlobAdaptedFileStorageTests
,
BlobWritableCacheTests
]
BlobAdaptedFileStorageTests
,
BlobWritableCacheTests
,
ConfigurationTests
,
]
def
test_suite
():
def
test_suite
():
suite
=
unittest
.
TestSuite
()
suite
=
unittest
.
TestSuite
()
...
...
src/ZODB/component.xml
View file @
3a90b8d0
...
@@ -163,6 +163,13 @@
...
@@ -163,6 +163,13 @@
that are accepted by this server.
that are accepted by this server.
</description>
</description>
</key>
</key>
<!-- DM 2008-05-15: added -->
<key
name=
"drop-cache-rather-verify"
datatype=
"boolean"
default=
"off"
>
<description>
A flag indicating whether the client cache should be dropped
instead of an expensive verification.
</description>
</key>
</sectiontype>
</sectiontype>
<sectiontype
name=
"demostorage"
datatype=
".DemoStorage"
<sectiontype
name=
"demostorage"
datatype=
".DemoStorage"
...
...
src/ZODB/config.py
View file @
3a90b8d0
...
@@ -167,6 +167,7 @@ class ZEOClient(BaseConfig):
...
@@ -167,6 +167,7 @@ class ZEOClient(BaseConfig):
wait
=
self
.
config
.
wait
,
wait
=
self
.
config
.
wait
,
read_only
=
self
.
config
.
read_only
,
read_only
=
self
.
config
.
read_only
,
read_only_fallback
=
self
.
config
.
read_only_fallback
,
read_only_fallback
=
self
.
config
.
read_only_fallback
,
drop_cache_rather_verify
=
self
.
config
.
drop_cache_rather_verify
,
username
=
self
.
config
.
username
,
username
=
self
.
config
.
username
,
password
=
self
.
config
.
password
,
password
=
self
.
config
.
password
,
realm
=
self
.
config
.
realm
)
realm
=
self
.
config
.
realm
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment