first merge of ICP support

ZServer/ICPServer.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+# Medusa ICP server
+#
+# Why would you want to use this?
+# see http://www.zope.org/Members/htrd/icp/intro
+
+import sys, string, os, socket, errno, struct
+
+import asyncore
+
+from medusa import counter
+
+
+ICP_OP_QUERY = 1
+ICP_OP_HIT = 2
+ICP_OP_MISS = 3
+ICP_OP_ERR = 4
+ICP_OP_MISS_NOFETCH = 21
+ICP_OP_DENIED = 22
+
+class BaseICPServer(asyncore.dispatcher):
+
+    REQUESTS_PER_LOOP = 4
+   
+    def __init__ (self,ip,port):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket (socket.AF_INET, socket.SOCK_DGRAM)
+        self.set_reuse_addr()
+        self.bind((ip,port))
+        if ip=='':
+            addr = 'any'
+        else:
+            addr = ip
+        self.log_info('ICP server started\n\tAddress: %s\n\tPort: %s' % (addr,port) )
+        
+    def handle_read(self):
+        for i in range(self.REQUESTS_PER_LOOP):
+            try:
+                request, whence = self.socket.recvfrom(16384)
+            except socket.error,e:
+                if e[0]==errno.EWOULDBLOCK:
+                    break
+                else:
+                    raise
+            else:
+                if self.check_whence(whence):
+                    reply = self.calc_reply(request)
+                    if reply:
+                        self.socket.sendto(reply,whence)
+
+    def readable(self):
+        return 1
+
+    def writable(self):
+        return 0
+
+    def handle_write (self):
+        self.log_info ('unexpected write event', 'warning')
+
+    def handle_error (self):      # don't close the socket on error
+        (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+        self.log_info('Problem in ICP (%s:%s %s)' % (t, v, tbinfo),
+                      'error')
+
+    def check_whence(self,whence):
+        return 1
+
+    def calc_reply(self,request):
+        if len(request)>20:
+            opcode,version,length,number,options,opdata,junk = struct.unpack('!BBHIIII',request[:20])
+            if version==2:
+                if opcode==ICP_OP_QUERY:
+                    if len(request)!=length:
+                        out_opcode = ICP_OP_ERR
+                    else:
+                        url = request[24:]
+                        if url[-1:]=='\x00':
+                            url = url[:-1]
+                        out_opcode = self.check_url(url)
+                    return struct.pack('!BBHIIII',out_opcode,2,20,number,0,0,0)
+                    
+    def check_url(self,url):
+        # derived classes replace this with a more
+        # useful policy
+        return ICP_OP_MISS
+
+
+class ICPServer(BaseICPServer):
+    # Products that want to do special ICP handling should .append their hooks into
+    # this list. Each hook is called in turn with the URL as a parameter, and
+    # they must return an ICP_OP code from above or None. The first
+    # non-None return is used as the ICP response
+    hooks = []
+    
+    def check_url(self,url):
+        for hook in self.hooks:
+            r = hook(url)
+            if r is not None:
+                return r
+        return ICP_OP_MISS

doc/CHANGES.txt

@@ -47,8 +47,10 @@ Zope Changes
         possible to enable case-sensitive indexing. The new 
         configuration options are available through the addForm
         of the Vocabulary object.
-        
 
+      - ICP server support. For more information see
+        http://www.zope.org/Members/htrd/icp/intro
+        
     Bugs:
 
       - Collector #32: Use difflib instead of ndiff

lib/python/ZServer/ICPServer.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+# Medusa ICP server
+#
+# Why would you want to use this?
+# see http://www.zope.org/Members/htrd/icp/intro
+
+import sys, string, os, socket, errno, struct
+
+import asyncore
+
+from medusa import counter
+
+
+ICP_OP_QUERY = 1
+ICP_OP_HIT = 2
+ICP_OP_MISS = 3
+ICP_OP_ERR = 4
+ICP_OP_MISS_NOFETCH = 21
+ICP_OP_DENIED = 22
+
+class BaseICPServer(asyncore.dispatcher):
+
+    REQUESTS_PER_LOOP = 4
+   
+    def __init__ (self,ip,port):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket (socket.AF_INET, socket.SOCK_DGRAM)
+        self.set_reuse_addr()
+        self.bind((ip,port))
+        if ip=='':
+            addr = 'any'
+        else:
+            addr = ip
+        self.log_info('ICP server started\n\tAddress: %s\n\tPort: %s' % (addr,port) )
+        
+    def handle_read(self):
+        for i in range(self.REQUESTS_PER_LOOP):
+            try:
+                request, whence = self.socket.recvfrom(16384)
+            except socket.error,e:
+                if e[0]==errno.EWOULDBLOCK:
+                    break
+                else:
+                    raise
+            else:
+                if self.check_whence(whence):
+                    reply = self.calc_reply(request)
+                    if reply:
+                        self.socket.sendto(reply,whence)
+
+    def readable(self):
+        return 1
+
+    def writable(self):
+        return 0
+
+    def handle_write (self):
+        self.log_info ('unexpected write event', 'warning')
+
+    def handle_error (self):      # don't close the socket on error
+        (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+        self.log_info('Problem in ICP (%s:%s %s)' % (t, v, tbinfo),
+                      'error')
+
+    def check_whence(self,whence):
+        return 1
+
+    def calc_reply(self,request):
+        if len(request)>20:
+            opcode,version,length,number,options,opdata,junk = struct.unpack('!BBHIIII',request[:20])
+            if version==2:
+                if opcode==ICP_OP_QUERY:
+                    if len(request)!=length:
+                        out_opcode = ICP_OP_ERR
+                    else:
+                        url = request[24:]
+                        if url[-1:]=='\x00':
+                            url = url[:-1]
+                        out_opcode = self.check_url(url)
+                    return struct.pack('!BBHIIII',out_opcode,2,20,number,0,0,0)
+                    
+    def check_url(self,url):
+        # derived classes replace this with a more
+        # useful policy
+        return ICP_OP_MISS
+
+
+class ICPServer(BaseICPServer):
+    # Products that want to do special ICP handling should .append their hooks into
+    # this list. Each hook is called in turn with the URL as a parameter, and
+    # they must return an ICP_OP code from above or None. The first
+    # non-None return is used as the ICP response
+    hooks = []
+    
+    def check_url(self,url):
+        for hook in self.hooks:
+            r = hook(url)
+            if r is not None:
+                return r
+        return ICP_OP_MISS

z2.py

@@ -154,6 +154,18 @@ Options:
 
     Multiple -m options can be provided to run multiple servers.
 
+  --icp port
+
+    The ICP port. ICP can be used to distribute load between back-end
+    zope servers, if you are using an ICP-aware front-end proxy such
+    as Squid.
+
+    The port can be preeceeded by an ip address follwed by a colon
+    to specify an address to listen on. This allows different servers
+    to listen on different addresses.
+
+    Multiple --icp options can be provided to run multiple servers.
+
   -l path
 
     Path to the ZServer log file. If this is a relative path then the
@@ -288,6 +300,9 @@ PCGI_FILE='Zope.cgi'
 ## Monitor configuration
 MONITOR_PORT=0
 
+## ICP configuration
+ICP_PORT=0
+
 # Module to be published, which must be Main or Zope
 MODULE='Zope'
 
@@ -341,7 +356,9 @@ try:
         raise 'Invalid python version', sys.version.split()[0]
 
     opts, args = getopt.getopt(sys.argv[1:],
-                               'hz:Z:t:i:a:d:u:w:W:f:p:m:Sl:2DP:rF:L:XM:')
+                               'hz:Z:t:i:a:d:u:w:W:f:p:m:Sl:2DP:rF:L:XM:'
+                               ['icp=',
+                               ])
 
     DEBUG=0
     READ_ONLY=0
@@ -384,7 +401,7 @@ try:
             DEBUG=1
         elif o=='-S': sys.ZMANAGED=1
         elif o=='-X':
-            MONITOR_PORT=HTTP_PORT=FTP_PORT=FCGI_PORT=0
+            MONITOR_PORT=HTTP_PORT=FTP_PORT=FCGI_PORT=ICP_PORT=0
             PCGI_FILE=''
         elif o=='-m':
             MONITOR_PORT=server_info(MONITOR_PORT, v)
@@ -397,6 +414,8 @@ try:
         elif o=='-P':
             HTTP_PORT=server_info(HTTP_PORT, v, 80)
             FTP_PORT=server_info(FTP_PORT, v, 21)
+        elif o=='--icp':
+            ICP_PORT=server_info(ICP_PORT, v)
 
         elif o=='-p':
             if v=='-': v=''
@@ -666,6 +685,12 @@ try:
                     hostname=address,
                     port=port)
 
+    if ICP_PORT:
+        if type(ICP_PORT) is type(0): ICP_PORT=((IP_ADDRESS, ICP_PORT),)
+        from ZServer.ICPServer import ICPServer
+        for address, port in ICP_PORT:
+            ICPServer(address,port)
+
     # Try to set uid to "-u" -provided uid.
     # Try to set gid to  "-u" user's primary group. 
     # This will only work if this script is run by root.