Commit 96172040 authored by root's avatar root

Use tagged version of src/ZEO

parent 291d1897
This diff is collapsed.
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""RPC stubs for interface exported by ClientStorage."""
class ClientStorage:
"""An RPC stub class for the interface exported by ClientStorage.
This is the interface presented by ClientStorage to the
StorageServer; i.e. the StorageServer calls these methods and they
are executed in the ClientStorage.
See the ClientStorage class for documentation on these methods.
It is currently important that all methods here are asynchronous
(meaning they don't have a return value and the caller doesn't
wait for them to complete), *and* that none of them cause any
calls from the client to the storage. This is due to limitations
in the zrpc subpackage.
The on-the-wire names of some of the methods don't match the
Python method names. That's because the on-the-wire protocol was
fixed for ZEO 2 and we don't want to change it. There are some
aliases in ClientStorage.py to make up for this.
"""
def __init__(self, rpc):
"""Constructor.
The argument is a connection: an instance of the
zrpc.connection.Connection class.
"""
self.rpc = rpc
def beginVerify(self):
self.rpc.callAsync('beginVerify')
def invalidateVerify(self, args):
self.rpc.callAsync('invalidateVerify', args)
def endVerify(self):
self.rpc.callAsync('endVerify')
def invalidateTransaction(self, tid, args):
self.rpc.callAsyncNoPoll('invalidateTransaction', tid, args)
def serialnos(self, arg):
self.rpc.callAsync('serialnos', arg)
def info(self, arg):
self.rpc.callAsync('info', arg)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Log a transaction's commit info during two-phase commit.
A storage server allows multiple clients to commit transactions, but
must serialize them as the actually execute at the server. The
concurrent commits are achieved by logging actions up until the
tpc_vote(). At that point, the entire transaction is committed on the
real storage.
"""
import cPickle
import tempfile
class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".log")
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
self.stores = 0
self.read = 0
def size(self):
return self.file.tell()
def store(self, oid, serial, data, version):
self.pickler.dump((oid, serial, data, version))
self.stores += 1
def get_loader(self):
self.read = 1
self.file.seek(0)
return self.stores, cPickle.Unpickler(self.file)
def close(self):
if self.file:
self.file.close()
self.file = None
BTrees
ThreadedAsync
ZConfig
ZODB
persistent
transaction
zdaemon
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A debugging version of the server that records network activity."""
import struct
import time
from ZEO.StorageServer import StorageServer, log
from ZEO.zrpc.server import ManagedServerConnection
# a bunch of codes
NEW_CONN = 1
CLOSE_CONN = 2
DATA = 3
ERROR = 4
class DebugManagedServerConnection(ManagedServerConnection):
def __init__(self, sock, addr, obj, mgr):
# mgr is the DebugServer instance
self.mgr = mgr
self.__super_init(sock, addr, obj)
record_id = mgr._record_connection(addr)
self._record = lambda code, data: mgr._record(record_id, code, data)
self.obj.notifyConnected(self)
def close(self):
self._record(CLOSE_CONN, "")
ManagedServerConnection.close(self)
# override the lowest-level of asyncore's connection
def recv(self, buffer_size):
try:
data = self.socket.recv(buffer_size)
if not data:
# a closed connection is indicated by signaling
# a read condition, and having recv() return 0.
self.handle_close()
return ''
else:
self._record(DATA, data)
return data
except socket.error, why:
# winsock sometimes throws ENOTCONN
self._record(ERROR, why)
if why[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN]:
self.handle_close()
return ''
else:
raise socket.error, why
class DebugServer(StorageServer):
ZEOStorageClass = DebugZEOStorage
ManagedServerConnectionClass = DebugManagerConnection
def __init__(self, *args, **kwargs):
StorageServer.__init__(*args, **kwargs)
self._setup_record(kwargs["record"])
self._conn_counter = 1
def _setup_record(self, path):
try:
self._recordfile = open(path, "ab")
except IOError, msg:
self._recordfile = None
log("failed to open recordfile %s: %s" % (path, msg))
def _record_connection(self, addr):
cid = self._conn_counter
self._conn_counter += 1
self._record(cid, NEW_CONN, str(addr))
return cid
def _record(self, conn, code, data):
s = struct.pack(">iii", code, time.time(), len(data)) + data
self._recordfile.write(s)
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Exceptions for ZEO."""
from ZODB.POSException import StorageError
class ClientStorageError(StorageError):
"""An error occured in the ZEO Client Storage."""
class UnrecognizedResult(ClientStorageError):
"""A server call returned an unrecognized result."""
class ClientDisconnected(ClientStorageError):
"""The database storage is disconnected from the storage."""
class AuthError(StorageError):
"""The client provided invalid authentication credentials."""
=======
ZEO 2.0
=======
What's ZEO?
-----------
ZEO stands for Zope Enterprise Objects. ZEO is an add-on for Zope
that allows multiple processes to connect to a single ZODB storage.
Those processes can live on different machines, but don't need to.
ZEO 2 has many improvements over ZEO 1, and is incompatible with ZEO 1;
if you upgrade an existing ZEO 1 installation, you must upgrade the
server and all clients simultaneous. If you received ZEO 2 as part of
the ZODB 3 distribution, the ZEO 1 sources are provided in a separate
directory (ZEO1). Some documentation for ZEO is available in the ZODB 3
package in the Doc subdirectory. ZEO depends on the ZODB software; it
can be used with the version of ZODB distributed with Zope 2.5.1 or
later. More information about ZEO can be found in the ZODB Wiki:
http://www.zope.org/Wikis/ZODB
What's here?
------------
This list of filenames is mostly for ZEO developers::
ClientCache.py client-side cache implementation
ClientStorage.py client-side storage implementation
ClientStub.py RPC stubs for callbacks from server to client
CommitLog.py buffer used during two-phase commit on the server
Exceptions.py definitions of exceptions
ICache.py interface definition for the client-side cache
ServerStub.py RPC stubs for the server
StorageServer.py server-side storage implementation
TransactionBuffer.py buffer used for transaction data in the client
__init__.py near-empty file to make this directory a package
simul.py command-line tool to simulate cache behavior
start.py command-line tool to start the storage server
stats.py command-line tool to process client cache traces
tests/ unit tests and other test utilities
util.py utilities used by the server startup tool
version.txt text file indicating the ZEO version
zrpc/ subpackage implementing Remote Procedure Call (RPC)
This diff is collapsed.
This diff is collapsed.
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A TransactionBuffer store transaction updates until commit or abort.
A transaction may generate enough data that it is not practical to
always hold pending updates in memory. Instead, a TransactionBuffer
is used to store the data until a commit or abort.
"""
# A faster implementation might store trans data in memory until it
# reaches a certain size.
import cPickle
import tempfile
from threading import Lock
class TransactionBuffer:
# Valid call sequences:
#
# ((store | invalidate)* begin_iterate next* clear)* close
#
# get_size can be called any time
# The TransactionBuffer is used by client storage to hold update
# data until the tpc_finish(). It is normally used by a single
# thread, because only one thread can be in the two-phase commit
# at one time.
# It is possible, however, for one thread to close the storage
# while another thread is in the two-phase commit. We must use
# a lock to guard against this race, because unpredictable things
# can happen in Python if one thread closes a file that another
# thread is reading. In a debug build, an assert() can fail.
# XXX If an operation is performed on a closed TransactionBuffer,
# it has no effect and does not raise an exception. The only time
# this should occur is when a ClientStorage is closed in one
# thread while another thread is in its tpc_finish(). It's not
# clear what should happen in this case. If the tpc_finish()
# completes without error, the Connection using it could have
# inconsistent data. This should have minimal effect, though,
# because the Connection is connected to a closed storage.
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".tbuf")
self.lock = Lock()
self.closed = 0
self.count = 0
self.size = 0
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
def close(self):
self.lock.acquire()
try:
self.closed = 1
try:
self.file.close()
except OSError:
pass
finally:
self.lock.release()
def store(self, oid, version, data):
self.lock.acquire()
try:
self._store(oid, version, data)
finally:
self.lock.release()
def _store(self, oid, version, data):
"""Store oid, version, data for later retrieval"""
if self.closed:
return
self.pickler.dump((oid, version, data))
self.count += 1
# Estimate per-record cache size
self.size = self.size + len(data) + 31
if version:
# Assume version data has same size as non-version data
self.size = self.size + len(version) + len(data) + 12
def invalidate(self, oid, version):
self.lock.acquire()
try:
if self.closed:
return
self.pickler.dump((oid, version, None))
self.count += 1
finally:
self.lock.release()
def clear(self):
"""Mark the buffer as empty"""
self.lock.acquire()
try:
if self.closed:
return
self.file.seek(0)
self.count = 0
self.size = 0
finally:
self.lock.release()
def __iter__(self):
self.lock.acquire()
try:
if self.closed:
return
self.file.flush()
self.file.seek(0)
return TBIterator(self.file, self.count)
finally:
self.lock.release()
class TBIterator(object):
def __init__(self, f, count):
self.file = f
self.count = count
self.unpickler = cPickle.Unpickler(f)
def __iter__(self):
return self
def next(self):
"""Return next tuple of data or None if EOF"""
if self.count == 0:
self.file.seek(0)
self.size = 0
raise StopIteration
oid_ver_data = self.unpickler.load()
self.count -= 1
return oid_ver_data
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO -- Zope Enterprise Objects.
See the file README.txt in this directory for an overview.
ZEO is now part of ZODB; ZODB's home on the web is
http://www.zope.org/Wikis/ZODB
"""
# The next line must use double quotes, so replace.py recognizes it.
version = "2.3a3"
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
_auth_modules = {}
def get_module(name):
if name == 'sha':
from auth_sha import StorageClass, SHAClient, Database
return StorageClass, SHAClient, Database
elif name == 'digest':
from auth_digest import StorageClass, DigestClient, DigestDatabase
return StorageClass, DigestClient, DigestDatabase
else:
return _auth_modules.get(name)
def register_module(name, storage_class, client, db):
if _auth_modules.has_key(name):
raise TypeError, "%s is already registred" % name
_auth_modules[name] = storage_class, client, db
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Digest authentication for ZEO
This authentication mechanism follows the design of HTTP digest
authentication (RFC 2069). It is a simple challenge-response protocol
that does not send passwords in the clear, but does not offer strong
security. The RFC discusses many of the limitations of this kind of
protocol.
Guard the password database as if it contained plaintext passwords.
It stores the hash of a username and password. This does not expose
the plaintext password, but it is sensitive nonetheless. An attacker
with the hash can impersonate the real user. This is a limitation of
the simple digest scheme.
HTTP is a stateless protocol, and ZEO is a stateful protocol. The
security requirements are quite different as a result. The HTTP
protocol uses a nonce as a challenge. The ZEO protocol requires a
separate session key that is used for message authentication. We
generate a second nonce for this purpose; the hash of nonce and
user/realm/password is used as the session key. XXX I'm not sure if
this is a sound approach; SRP would be preferred.
"""
import os
import random
import sha
import struct
import time
from ZEO.auth.base import Database, Client
from ZEO.StorageServer import ZEOStorage
from ZEO.Exceptions import AuthError
def get_random_bytes(n=8):
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
s = f.read(n)
f.close()
else:
L = [chr(random.randint(0, 255)) for i in range(n)]
s = "".join(L)
return s
def hexdigest(s):
return sha.new(s).hexdigest()
class DigestDatabase(Database):
def __init__(self, filename, realm=None):
Database.__init__(self, filename, realm)
# Initialize a key used to build the nonce for a challenge.
# We need one key for the lifetime of the server, so it
# is convenient to store in on the database.
self.noncekey = get_random_bytes(8)
def _store_password(self, username, password):
dig = hexdigest("%s:%s:%s" % (username, self.realm, password))
self._users[username] = dig
def session_key(h_up, nonce):
# The hash itself is a bit too short to be a session key.
# HMAC wants a 64-byte key. We don't want to use h_up
# directly because it would never change over time. Instead
# use the hash plus part of h_up.
return sha.new("%s:%s" % (h_up, nonce)).digest() + h_up[:44]
class StorageClass(ZEOStorage):
def set_database(self, database):
assert isinstance(database, DigestDatabase)
self.database = database
self.noncekey = database.noncekey
def _get_time(self):
# Return a string representing the current time.
t = int(time.time())
return struct.pack("i", t)
def _get_nonce(self):
# RFC 2069 recommends a nonce of the form
# H(client-IP ":" time-stamp ":" private-key)
dig = sha.sha()
dig.update(str(self.connection.addr))
dig.update(self._get_time())
dig.update(self.noncekey)
return dig.hexdigest()
def auth_get_challenge(self):
"""Return realm, challenge, and nonce."""
self._challenge = self._get_nonce()
self._key_nonce = self._get_nonce()
return self.auth_realm, self._challenge, self._key_nonce
def auth_response(self, resp):
# verify client response
user, challenge, response = resp
# Since zrpc is a stateful protocol, we just store the nonce
# we sent to the client. It will need to generate a new
# nonce for a new connection anyway.
if self._challenge != challenge:
raise ValueError, "invalid challenge"
# lookup user in database
h_up = self.database.get_password(user)
# regeneration resp from user, password, and nonce
check = hexdigest("%s:%s" % (h_up, challenge))
if check == response:
self.connection.setSessionKey(session_key(h_up, self._key_nonce))
return self.finish_auth(check == response)
extensions = [auth_get_challenge, auth_response]
class DigestClient(Client):
extensions = ["auth_get_challenge", "auth_response"]
def start(self, username, realm, password):
_realm, challenge, nonce = self.stub.auth_get_challenge()
if _realm != realm:
raise AuthError("expected realm %r, got realm %r"
% (_realm, realm))
h_up = hexdigest("%s:%s:%s" % (username, realm, password))
resp_dig = hexdigest("%s:%s" % (h_up, challenge))
result = self.stub.auth_response((username, challenge, resp_dig))
if result:
return session_key(h_up, nonce)
else:
return None
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Base classes for defining an authentication protocol.
Database -- abstract base class for password database
Client -- abstract base class for authentication client
"""
import os
import sha
class Client:
# Subclass should override to list the names of methods that
# will be called on the server.
extensions = []
def __init__(self, stub):
self.stub = stub
for m in self.extensions:
setattr(self.stub, m, self.stub.extensionMethod(m))
def sort(L):
"""Sort a list in-place and return it."""
L.sort()
return L
class Database:
"""Abstracts a password database.
This class is used both in the authentication process (via
get_password()) and by client scripts that manage the password
database file.
The password file is a simple, colon-separated text file mapping
usernames to password hashes. The hashes are SHA hex digests
produced from the password string.
"""
realm = None
def __init__(self, filename, realm=None):
"""Creates a new Database
filename: a string containing the full pathname of
the password database file. Must be readable by the user
running ZEO. Must be writeable by any client script that
accesses the database.
realm: the realm name (a string)
"""
self._users = {}
self.filename = filename
self.load()
if realm:
if self.realm and self.realm != realm:
raise ValueError, ("Specified realm %r differs from database "
"realm %r" % (realm or '', self.realm))
else:
self.realm = realm
def save(self, fd=None):
filename = self.filename
if not fd:
fd = open(filename, 'w')
if self.realm:
print >> fd, "realm", self.realm
for username in sort(self._users.keys()):
print >> fd, "%s: %s" % (username, self._users[username])
def load(self):
filename = self.filename
if not filename:
return
if not os.path.exists(filename):
return
fd = open(filename)
L = fd.readlines()
if not L:
return
if L[0].startswith("realm "):
line = L.pop(0).strip()
self.realm = line[len("realm "):]
for line in L:
username, hash = line.strip().split(":", 1)
self._users[username] = hash.strip()
def _store_password(self, username, password):
self._users[username] = self.hash(password)
def get_password(self, username):
"""Returns password hash for specified username.
Callers must check for LookupError, which is raised in
the case of a non-existent user specified."""
if not self._users.has_key(username):
raise LookupError, "No such user: %s" % username
return self._users[username]
def hash(self, s):
return sha.new(s).hexdigest()
def add_user(self, username, password):
if self._users.has_key(username):
raise LookupError, "User %s already exists" % username
self._store_password(username, password)
def del_user(self, username):
if not self._users.has_key(username):
raise LookupError, "No such user: %s" % username
del self._users[username]
def change_password(self, username, password):
if not self._users.has_key(username):
raise LookupError, "No such user: %s" % username
self._store_password(username, password)
"""HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError, "clear() method not available in HMAC."
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
This diff is collapsed.
<component>
<sectiontype name="zeo">
<description>
The content of a ZEO section describe operational parameters
of a ZEO server except for the storage(s) to be served.
</description>
<key name="address" datatype="socket-address"
required="yes">
<description>
The address at which the server should listen. This can be in
the form 'host:port' to signify a TCP/IP connection or a
pathname string to signify a Unix domain socket connection (at
least one '/' is required). A hostname may be a DNS name or a
dotted IP address. If the hostname is omitted, the platform's
default behavior is used when binding the listening socket (''
is passed to socket.bind() as the hostname portion of the
address).
</description>
</key>
<key name="read-only" datatype="boolean"
required="no"
default="false">
<description>
Flag indicating whether the server should operate in read-only
mode. Defaults to false. Note that even if the server is
operating in writable mode, individual storages may still be
read-only. But if the server is in read-only mode, no write
operations are allowed, even if the storages are writable. Note
that pack() is considered a read-only operation.
</description>
</key>
<key name="invalidation-queue-size" datatype="integer"
required="no"
default="100">
<description>
The storage server keeps a queue of the objects modified by the
last N transactions, where N == invalidation_queue_size. This
queue is used to speed client cache verification when a client
disconnects for a short period of time.
</description>
</key>
<key name="monitor-address" datatype="socket-address"
required="no">
<description>
The address at which the monitor server should listen. If
specified, a monitor server is started. The monitor server
provides server statistics in a simple text format. This can
be in the form 'host:port' to signify a TCP/IP connection or a
pathname string to signify a Unix domain socket connection (at
least one '/' is required). A hostname may be a DNS name or a
dotted IP address. If the hostname is omitted, the platform's
default behavior is used when binding the listening socket (''
is passed to socket.bind() as the hostname portion of the
address).
</description>
</key>
<key name="transaction-timeout" datatype="integer"
required="no">
<description>
The maximum amount of time to wait for a transaction to commit
after acquiring the storage lock, specified in seconds. If the
transaction takes too long, the client connection will be closed
and the transaction aborted.
</description>
</key>
<key name="authentication-protocol" required="no">
<description>
The name of the protocol used for authentication. The
only protocol provided with ZEO is "digest," but extensions
may provide other protocols.
</description>
</key>
<key name="authentication-database" required="no">
<description>
The path of the database containing authentication credentials.
</description>
</key>
<key name="authentication-realm" required="no">
<description>
The authentication realm of the server. Some authentication
schemes use a realm to identify the logical set of usernames
that are accepted by this server.
</description>
</key>
</sectiontype>
</component>
#!python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""%(program)s -- create a ZEO instance.
Usage: %(program)s home [port]
Given an "instance home directory" <home> and some configuration
options (all of which have default values), create the following:
<home>/etc/zeo.conf -- ZEO config file
<home>/var/ -- Directory for data files: Data.fs etc.
<home>/log/ -- Directory for log files: zeo.log and zeoctl.log
<home>/bin/runzeo -- the zeo server runner
<home>/bin/zeoctl -- start/stop script (a shim for zeoctl.py)
The script will not overwrite existing files; instead, it will issue a
warning if an existing file is found that differs from the file that
would be written if it didn't exist.
"""
# WARNING! Several templates and functions here are reused by ZRS.
# So be careful with changes.
import os
import sys
import stat
import getopt
zeo_conf_template = """\
# ZEO configuration file
%%define INSTANCE %(instance_home)s
<zeo>
address %(port)d
read-only false
invalidation-queue-size 100
# monitor-address PORT
# transaction-timeout SECONDS
</zeo>
<filestorage 1>
path $INSTANCE/var/Data.fs
</filestorage>
<eventlog>
level info
<logfile>
path $INSTANCE/log/zeo.log
</logfile>
</eventlog>
<runner>
program $INSTANCE/bin/runzeo
socket-name $INSTANCE/etc/%(package)s.zdsock
daemon true
forever false
backoff-limit 10
exit-codes 0, 2
directory $INSTANCE
default-to-interactive true
# user zope
python %(python)s
zdrun %(zodb3_home)s/zdaemon/zdrun.py
# This logfile should match the one in the %(package)s.conf file.
# It is used by zdctl's logtail command, zdrun/zdctl doesn't write it.
logfile $INSTANCE/log/%(package)s.log
</runner>
"""
zeoctl_template = """\
#!/bin/sh
# %(PACKAGE)s instance control script
# The following two lines are for chkconfig. On Red Hat Linux (and
# some other systems), you can copy or symlink this script into
# /etc/rc.d/init.d/ and then use chkconfig(8) to automatically start
# %(PACKAGE)s at boot time.
# chkconfig: 345 90 10
# description: start a %(PACKAGE)s server
PYTHON="%(python)s"
ZODB3_HOME="%(zodb3_home)s"
CONFIG_FILE="%(instance_home)s/etc/%(package)s.conf"
PYTHONPATH="$ZODB3_HOME"
export PYTHONPATH
ZEOCTL="$ZODB3_HOME/ZEO/zeoctl.py"
exec "$PYTHON" "$ZEOCTL" -C "$CONFIG_FILE" ${1+"$@"}
"""
runzeo_template = """\
#!/bin/sh
# %(PACKAGE)s instance start script
PYTHON="%(python)s"
ZODB3_HOME="%(zodb3_home)s"
CONFIG_FILE="%(instance_home)s/etc/%(package)s.conf"
PYTHONPATH="$ZODB3_HOME"
export PYTHONPATH
RUNZEO="$ZODB3_HOME/ZEO/runzeo.py"
exec "$PYTHON" "$RUNZEO" -C "$CONFIG_FILE" ${1+"$@"}
"""
def main():
ZEOInstanceBuilder().run()
print "All done."
class ZEOInstanceBuilder:
def run(self):
try:
opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
except getopt.error, msg:
print msg
sys.exit(2)
program = os.path.basename(sys.argv[0])
if opts:
# There's only the help options, so just dump some help:
msg = __doc__ % {"program": program}
print msg
sys.exit()
if len(args) not in [1, 2]:
print "Usage: %s home [port]" % program
sys.exit(2)
instance_home = args[0]
if not os.path.isabs(instance_home):
instance_home = os.path.abspath(instance_home)
for entry in sys.path:
if os.path.exists(os.path.join(entry, 'ZODB')):
zodb3_home = entry
break
else:
print "Can't find the Zope home (not in sys.path)"
sys.exit(2)
if args[1:]:
port = int(args[1])
else:
port = 9999
params = self.get_params(zodb3_home, instance_home, port)
self.create(instance_home, params)
def get_params(self, zodb3_home, instance_home, port):
return {
"package": "zeo",
"PACKAGE": "ZEO",
"zodb3_home": zodb3_home,
"instance_home": instance_home,
"port": port,
"python": sys.executable,
}
def create(self, home, params):
makedir(home)
makedir(home, "etc")
makedir(home, "var")
makedir(home, "log")
makedir(home, "bin")
makefile(zeo_conf_template, home, "etc", "zeo.conf", **params)
makexfile(zeoctl_template, home, "bin", "zeoctl", **params)
makexfile(runzeo_template, home, "bin", "runzeo", **params)
def which(program):
strpath = os.getenv("PATH")
binpath = strpath.split(os.pathsep)
for dir in binpath:
path = os.path.join(dir, program)
if os.path.isfile(path) and os.access(path, os.X_OK):
if not os.path.isabs(path):
path = os.path.abspath(path)
return path
raise IOError, "can't find %r on path %r" % (program, strpath)
def makedir(*args):
path = ""
for arg in args:
path = os.path.join(path, arg)
mkdirs(path)
return path
def mkdirs(path):
if os.path.isdir(path):
return
head, tail = os.path.split(path)
if head and tail and not os.path.isdir(head):
mkdirs(head)
os.mkdir(path)
print "Created directory", path
def makefile(template, *args, **kwds):
path = makedir(*args[:-1])
path = os.path.join(path, args[-1])
data = template % kwds
if os.path.exists(path):
f = open(path)
olddata = f.read().strip()
f.close()
if olddata:
if olddata != data.strip():
print "Warning: not overwriting existing file %r" % path
return path
f = open(path, "w")
f.write(data)
f.close()
print "Wrote file", path
return path
def makexfile(template, *args, **kwds):
path = makefile(template, *args, **kwds)
umask = os.umask(022)
os.umask(umask)
mode = 0777 & ~umask
if stat.S_IMODE(os.stat(path)[stat.ST_MODE]) != mode:
os.chmod(path, mode)
print "Changed mode for %s to %o" % (path, mode)
return path
if __name__ == "__main__":
main()
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Monitor behavior of ZEO server and record statistics.
$Id: monitor.py,v 1.6 2004/04/25 11:34:15 gintautasm Exp $
"""
import asyncore
import socket
import time
import types
import logging
import ZEO
class StorageStats:
"""Per-storage usage statistics."""
def __init__(self):
self.loads = 0
self.stores = 0
self.commits = 0
self.aborts = 0
self.active_txns = 0
self.clients = 0
self.verifying_clients = 0
self.lock_time = None
self.conflicts = 0
self.conflicts_resolved = 0
self.start = time.ctime()
def parse(self, s):
# parse the dump format
lines = s.split("\n")
for line in lines:
field, value = line.split(":", 1)
if field == "Server started":
self.start = value
elif field == "Clients":
self.clients = int(value)
elif field == "Clients verifying":
self.verifying_clients = int(value)
elif field == "Active transactions":
self.active_txns = int(value)
elif field == "Commit lock held for":
# This assumes
self.lock_time = time.time() - int(value)
elif field == "Commits":
self.commits = int(value)
elif field == "Aborts":
self.aborts = int(value)
elif field == "Loads":
self.loads = int(value)
elif field == "Stores":
self.stores = int(value)
elif field == "Conflicts":
self.conflicts = int(value)
elif field == "Conflicts resolved":
self.conflicts_resolved = int(value)
def dump(self, f):
print >> f, "Server started:", self.start
print >> f, "Clients:", self.clients
print >> f, "Clients verifying:", self.verifying_clients
print >> f, "Active transactions:", self.active_txns
if self.lock_time:
howlong = time.time() - self.lock_time
print >> f, "Commit lock held for:", int(howlong)
print >> f, "Commits:", self.commits
print >> f, "Aborts:", self.aborts
print >> f, "Loads:", self.loads
print >> f, "Stores:", self.stores
print >> f, "Conflicts:", self.conflicts
print >> f, "Conflicts resolved:", self.conflicts_resolved
class StatsClient(asyncore.dispatcher):
def __init__(self, sock, addr):
asyncore.dispatcher.__init__(self, sock)
self.buf = []
self.closed = 0
def close(self):
self.closed = 1
# The socket is closed after all the data is written.
# See handle_write().
def write(self, s):
self.buf.append(s)
def writable(self):
return len(self.buf)
def readable(self):
# XXX what goes here?
return 0
def handle_write(self):
s = "".join(self.buf)
self.buf = []
n = self.socket.send(s)
if n < len(s):
self.buf.append(s[:n])
if self.closed and not self.buf:
asyncore.dispatcher.close(self)
class StatsServer(asyncore.dispatcher):
StatsConnectionClass = StatsClient
def __init__(self, addr, stats):
asyncore.dispatcher.__init__(self)
self.addr = addr
self.stats = stats
if type(self.addr) == types.TupleType:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
logger = logging.getLogger('ZEO.monitor')
logger.info("listening on %s", repr(self.addr))
self.bind(self.addr)
self.listen(5)
def writable(self):
return 0
def readable(self):
return 1
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error:
return
f = self.StatsConnectionClass(sock, addr)
self.dump(f)
f.close()
def dump(self, f):
print >> f, "ZEO monitor server version %s" % ZEO.version
print >> f, time.ctime()
print >> f
L = self.stats.keys()
L.sort()
for k in L:
stats = self.stats[k]
print >> f, "Storage:", k
stats.dump(f)
print >> f
#!python
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Start the ZEO storage server.
Usage: %s [-C URL] [-a ADDRESS] [-f FILENAME] [-h]
Options:
-C/--configuration URL -- configuration file or URL
-a/--address ADDRESS -- server address of the form PORT, HOST:PORT, or PATH
(a PATH must contain at least one "/")
-f/--filename FILENAME -- filename for FileStorage
-t/--timeout TIMEOUT -- transaction timeout in seconds (default no timeout)
-h/--help -- print this usage message and exit
-m/--monitor ADDRESS -- address of monitor server ([HOST:]PORT or PATH)
Unless -C is specified, -a and -f are required.
"""
# The code here is designed to be reused by other, similar servers.
# For the forseeable future, it must work under Python 2.1 as well as
# 2.2 and above.
import os
import sys
import signal
import socket
import logging
import ZConfig, ZConfig.datatypes
import ZEO
from zdaemon.zdoptions import ZDOptions
logger = logging.getLogger('ZEO.runzeo')
_pid = str(os.getpid())
def log(msg, level=logging.INFO, exc_info=False):
"""Internal: generic logging function."""
message = "(%s) %s" % (_pid, msg)
logger.log(level, message, exc_info=exc_info)
def parse_address(arg):
# XXX Not part of the official ZConfig API
obj = ZConfig.datatypes.SocketAddress(arg)
return obj.family, obj.address
class ZEOOptionsMixin:
storages = None
def handle_address(self, arg):
self.family, self.address = parse_address(arg)
def handle_monitor_address(self, arg):
self.monitor_family, self.monitor_address = parse_address(arg)
def handle_filename(self, arg):
from ZODB.config import FileStorage # That's a FileStorage *opener*!
class FSConfig:
def __init__(self, name, path):
self._name = name
self.path = path
self.create = 0
self.read_only = 0
self.stop = None
self.quota = None
def getSectionName(self):
return self._name
if not self.storages:
self.storages = []
name = str(1 + len(self.storages))
conf = FileStorage(FSConfig(name, arg))
self.storages.append(conf)
def add_zeo_options(self):
self.add(None, None, "a:", "address=", self.handle_address)
self.add(None, None, "f:", "filename=", self.handle_filename)
self.add("family", "zeo.address.family")
self.add("address", "zeo.address.address",
required="no server address specified; use -a or -C")
self.add("read_only", "zeo.read_only", default=0)
self.add("invalidation_queue_size", "zeo.invalidation_queue_size",
default=100)
self.add("transaction_timeout", "zeo.transaction_timeout",
"t:", "timeout=", float)
self.add("monitor_address", "zeo.monitor_address.address",
"m:", "monitor=", self.handle_monitor_address)
self.add('auth_protocol', 'zeo.authentication_protocol',
None, 'auth-protocol=', default=None)
self.add('auth_database', 'zeo.authentication_database',
None, 'auth-database=')
self.add('auth_realm', 'zeo.authentication_realm',
None, 'auth-realm=')
class ZEOOptions(ZDOptions, ZEOOptionsMixin):
logsectionname = "eventlog"
schemadir = os.path.dirname(ZEO.__file__)
def __init__(self):
ZDOptions.__init__(self)
self.add_zeo_options()
self.add("storages", "storages",
required="no storages specified; use -f or -C")
class ZEOServer:
def __init__(self, options):
self.options = options
def main(self):
self.setup_default_logging()
self.check_socket()
self.clear_socket()
try:
self.open_storages()
self.setup_signals()
self.create_server()
self.loop_forever()
finally:
self.close_storages()
self.clear_socket()
def setup_default_logging(self):
if self.options.config_logger is not None:
return
# No log file is configured; default to stderr.
logger = logging.getLogger()
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logger.addHandler(handler)
def check_socket(self):
if self.can_connect(self.options.family, self.options.address):
self.options.usage("address %s already in use" %
repr(self.options.address))
def can_connect(self, family, address):
s = socket.socket(family, socket.SOCK_STREAM)
try:
s.connect(address)
except socket.error:
return 0
else:
s.close()
return 1
def clear_socket(self):
if isinstance(self.options.address, type("")):
try:
os.unlink(self.options.address)
except os.error:
pass
def open_storages(self):
self.storages = {}
for opener in self.options.storages:
log("opening storage %r using %s"
% (opener.name, opener.__class__.__name__))
self.storages[opener.name] = opener.open()
def setup_signals(self):
"""Set up signal handlers.
The signal handler for SIGFOO is a method handle_sigfoo().
If no handler method is defined for a signal, the signal
action is not changed from its initial value. The handler
method is called without additional arguments.
"""
if os.name != "posix":
return
if hasattr(signal, 'SIGXFSZ'):
signal.signal(signal.SIGXFSZ, signal.SIG_IGN) # Special case
init_signames()
for sig, name in signames.items():
method = getattr(self, "handle_" + name.lower(), None)
if method is not None:
def wrapper(sig_dummy, frame_dummy, method=method):
method()
signal.signal(sig, wrapper)
def create_server(self):
from ZEO.StorageServer import StorageServer
self.server = StorageServer(
self.options.address,
self.storages,
read_only=self.options.read_only,
invalidation_queue_size=self.options.invalidation_queue_size,
transaction_timeout=self.options.transaction_timeout,
monitor_address=self.options.monitor_address,
auth_protocol=self.options.auth_protocol,
auth_database=self.options.auth_database, # XXX option spelling
auth_realm=self.options.auth_realm)
def loop_forever(self):
import ThreadedAsync.LoopCallback
ThreadedAsync.LoopCallback.loop()
def handle_sigterm(self):
log("terminated by SIGTERM")
sys.exit(0)
def handle_sigint(self):
log("terminated by SIGINT")
sys.exit(0)
def handle_sighup(self):
log("restarted by SIGHUP")
sys.exit(1)
def handle_sigusr2(self):
# XXX this used to reinitialize zLOG. How do I achieve
# the same effect with Python's logging package?
# Should we restart as with SIGHUP?
log("received SIGUSR2, but it was not handled!", level=logging.WARNING)
def close_storages(self):
for name, storage in self.storages.items():
log("closing storage %r" % name)
try:
storage.close()
except: # Keep going
log("failed to close storage %r" % name,
level=logging.EXCEPTION, exc_info=True)
# Signal names
signames = None
def signame(sig):
"""Return a symbolic name for a signal.
Return "signal NNN" if there is no corresponding SIG name in the
signal module.
"""
if signames is None:
init_signames()
return signames.get(sig) or "signal %d" % sig
def init_signames():
global signames
signames = {}
for name, sig in signal.__dict__.items():
k_startswith = getattr(name, "startswith", None)
if k_startswith is None:
continue
if k_startswith("SIG") and not k_startswith("SIG_"):
signames[sig] = name
# Main program
def main(args=None):
options = ZEOOptions()
options.realize(args)
s = ZEOServer(options)
s.main()
if __name__ == "__main__":
main()
<schema>
<!-- note that zeoctl.xml is a closely related schema which should
match this schema, but should require the "runner" section -->
<description>
This schema describes the configuration of the ZEO storage server
process.
</description>
<!-- Use the storage types defined by ZODB. -->
<import package="ZODB"/>
<!-- Use the ZEO server information structure. -->
<import package="ZEO"/>
<import package="ZConfig.components.logger"/>
<!-- runner control -->
<import package="zdaemon"/>
<section type="zeo" name="*" required="yes" attribute="zeo" />
<section type="runner" name="*" required="no" attribute="runner" />
<multisection name="+" type="ZODB.storage"
attribute="storages"
required="yes">
<description>
One or more storages that are provided by the ZEO server. The
section names are used as the storage names, and must be unique
within each ZEO storage server. Traditionally, these names
represent small integers starting at '1'.
</description>
</multisection>
<section name="*" type="eventlog" attribute="eventlog" required="no" />
</schema>
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment