merged toby-stiff-cache-branch and toby-unicode-branch

doc/CHANGES.txt

@@ -56,6 +56,8 @@ Zope Changes
 
       - Collector #304: several catalog optimisations
         
+      - New implementation of ZODB object cache.
+
     Bugs:
 
       - Fixed bug reported on maillist during EWOULDBLOCK when using FTP server

lib/Components/BTree/BTree.c

@@ -13,7 +13,7 @@
 
 static char BTree_module_documentation[] = 
 ""
-"\n$Id: BTree.c,v 1.31 2002/03/21 15:48:53 htrd Exp $"
+"\n$Id: BTree.c,v 1.32 2002/03/27 10:13:59 htrd Exp $"
 ;
 
 #define PERSISTENT
@@ -1105,7 +1105,7 @@ bucket__p___reinit__(Bucket *self, PyObject *args)
 	}
       if(HasInstDict(self) && (dict=INSTANCE_DICT(self))) PyDict_Clear(dict);
       self->len=0;
-      self->state=cPersistent_GHOST_STATE;
+      PER_GHOSTIFY(self);
     }
 
   Py_INCREF(Py_None);
@@ -1163,7 +1163,7 @@ BTree__p___reinit__(BTree *self, PyObject *args)
 
       if(_BTree_clear(self) < 0) return NULL;
       if(HasInstDict(self) && (dict=INSTANCE_DICT(self))) PyDict_Clear(dict);
-      self->state=cPersistent_GHOST_STATE;
+      PER_GHOSTIFY(self);
     }
 
   Py_INCREF(Py_None);

lib/Components/BTree/intSet.c

@@ -13,7 +13,7 @@
 
 static char intSet_module_documentation[] = 
 ""
-"\n$Id: intSet.c,v 1.22 2002/03/21 15:48:53 htrd Exp $"
+"\n$Id: intSet.c,v 1.23 2002/03/27 10:13:59 htrd Exp $"
 ;
 
 #include <limits.h>
@@ -368,7 +368,7 @@ intSet__p___reinit__(intSet *self, PyObject *args)
      && HasInstDict(self) && (dict=INSTANCE_DICT(self)))
     {
       PyDict_Clear(dict);
-      self->state=cPersistent_GHOST_STATE;
+      PER_GHOSTIFY(self);
     }
 
   Py_INCREF(Py_None);

lib/Components/ExtensionClass/src/Record.c

@@ -13,7 +13,7 @@
  ****************************************************************************/
 static char Record_module_documentation[] = 
 ""
-"\n$Id: Record.c,v 1.15 2002/02/11 23:40:40 gvanrossum Exp $"
+"\n$Id: Record.c,v 1.16 2002/03/27 10:14:00 htrd Exp $"
 ;
 
 #ifdef PERSISTENCE
@@ -150,7 +150,7 @@ Record__p_deactivate(Record *self, PyObject *args)
   Record_deal(self);
   self->schema=NULL;
   self->data=NULL;
-  self->state=cPersistent_GHOST_STATE;
+  PER_GHOSTIFY(self);
   Py_INCREF(Py_None);
   return Py_None;
 }

lib/python/App/CacheManager.py

@@ -13,8 +13,8 @@
 __doc__='''Cache management support
 
 
-$Id: CacheManager.py,v 1.23 2002/02/07 17:37:10 andreasjung Exp $'''
-__version__='$Revision: 1.23 $'[11:-2]
+$Id: CacheManager.py,v 1.24 2002/03/27 10:14:00 htrd Exp $'''
+__version__='$Revision: 1.24 $'[11:-2]
 
 import Globals, time, sys
 
@@ -36,6 +36,12 @@ class CacheManager:
             return len(Globals.Bobobase._jar.cache)
         else: return db.cacheSize()
 
+    def cache_detail_length(self):
+      try: db=self._p_jar.db()
+      except:
+          return ()
+      else: return db.cacheDetailSize()
+
     def database_size(self):
         try: db=self._p_jar.db()
         except:
@@ -156,7 +162,7 @@ class CacheManager:
             response=REQUEST['RESPONSE']
             response.redirect(REQUEST['URL1']+'/manage_cacheGC')
 
-    def manage_minimize(self,value,REQUEST):
+    def manage_minimize(self,value=1,REQUEST=None):
         "Perform a full sweep through the cache"
         try: db=self._p_jar.db()
         except:
@@ -206,7 +212,6 @@ class CacheManager:
             # sort the list.
             lst = map(lambda dict: ((dict['conn_no'], dict['oid']), dict),
                       detail)
-            lst.sort()
             # format as text.
             res = [
                 '# Table shows connection number, oid, refcount, state, '

lib/python/App/dtml/cacheGC.dtml

@@ -3,39 +3,6 @@
 
 <br />
 <table width="100%" cellspacing="0" cellpadding="2" border="0">
-<tr class="section-bar">
-  <td colspan="2" align="left">
-  <div class="form-label">
-  Full Sweep
-  </div>
-  </td>
-</tr>
-<tr>
-  <td align="left" valign="top">
-  <div class="form-text">
-  Make a single pass through the cache, removing any objects that are no
-  longer referenced, and deactivating objects that have not been
-  accessed in the number of seconds given in the input box to the right
-  of this text.
-  </div>
-  </td>
-  <td>
-  <form action="<dtml-var URL1>/manage_full_sweep" method="get">
-  <input type="text" name="value:int" value="<dtml-var 
-   cache_age html_quote>" size="6" />
-  <br />
-  <div class="form-element">
-  <input type="submit" name="submit" value="Full Sweep" />
-  </div>
-  </form>
-  </td>
-</tr>
-
-<tr>
-  <td colspan="2">
-  <br />&nbsp;
-  </td>
-</tr>
 
 <tr class="section-bar">
   <td colspan="2" align="left">
@@ -47,16 +14,10 @@
 <tr>
   <td align="left" valign="top">
   <div class="form-text">
-  Make multiple passes through the cache, removing any objects that are no
-  longer referenced, and deactivating objects that have not been
-  accessed in the number of seconds given in the input box to the right
-  of this text.
+  Remove all objects from the ZODB in-memory cache
   </td>
   <td>
   <form action="<dtml-var URL1>/manage_minimize" method=GET>
-  <input type="text" name="value:int" value="<dtml-var 
-   cache_age html_quote>" size="6" />
-  <br />
   <div class="form-element">
   <input type="submit" name="submit" value="Minimize" />
   </div>

lib/python/App/dtml/cacheParameters.dtml

@@ -18,7 +18,7 @@
 <tr>
   <td align="left">
   <div class="form-label">
-  Total number of objects in all caches
+  Total number of objects in memory from all caches
   </div>
   </td>
   <td>
@@ -31,7 +31,7 @@
 <tr>
   <td align="left">
   <div class="form-label">
-  Target Size
+  Target number of objects in memory per cache
   </div>
   </td>
   <td>
@@ -45,22 +45,6 @@
   </td>
 </tr>
 
-<tr>
-  <td align="left">
-  <div class="form-label">
-  Target max time between accesses
-  </div>
-  </td>
-  <td>
-  <form action="<dtml-var URL1>/manage_cache_age" method="get">
-  <input type="text" name="value:int" value="<dtml-var 
-   cache_age html_quote>" size="6" />
-  <span class="form-element">
-  <input type="submit" name="submit" value="Change">
-  </div>
-  </form>
-  </td>
-</tr>
 
 <dtml-in cacheStatistics>
 <tr>
@@ -72,6 +56,34 @@
   </td>
 </tr>
 </dtml-in>
+
+<tr>
+  <td align="left" colspan=2>
+  <div class="form-label">
+  Total number of objects in each cache:
+  </div>
+  </td>
+</tr>
+
+<tr class="list-header">
+    <th><div class="list-item">Cache Name</div></th>
+    <th><div class="list-item">Number of object in memory</div></th>
+    <th><div class="list-item">Number of ghost objects</div></th>
+</tr>
+<dtml-in cache_detail_length mapping>
+<dtml-if name="sequence-odd"><tr class="row-normal">
+<dtml-else><tr class="row-hilite"></dtml-if>
+    <td><div class="form-text"><dtml-var connection html_quote></div></td>
+    <td><div class="form-text"><dtml-var ngsize></div></td>
+    <td><div class="form-text"><dtml-var size></div></td>
+</tr>
+</dtml-in>
+<tr class="row-hilite">
+    <td><div class="list-item">Total</div></td>
+    <td><div class="list-item"><dtml-var cache_length></div></td>
+    <td><div class="list-item"></div></td>
+</tr>
+
 </table>
 
 <dtml-if show_cache_detail>

lib/python/App/dtml/manage_page_header.dtml

 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
 <html>
 <head>
+<dtml-unless management_page_charset>
+<dtml-call "REQUEST.set('management_page_charset','iso-8859-1')">
+</dtml-unless>
+<meta http-equiv="content-type" content="text/html;charset=<dtml-var management_page_charset>">
+<dtml-call "RESPONSE.setHeader('content-type','text/html;charset='+management_page_charset)">
 <title><dtml-if title><dtml-var title></dtml-if></title>
 <dtml-let ag="REQUEST.get('HTTP_USER_AGENT', '')"
      is_nav4="ag[:9] == 'Mozilla/4' and _.string.find(ag, 'MSIE') < 0"

lib/python/DocumentTemplate/DT_Util.py

@@ -10,11 +10,11 @@
 # FOR A PARTICULAR PURPOSE
 # 
 ##############################################################################
-'''$Id: DT_Util.py,v 1.85 2001/11/28 15:50:55 matt Exp $''' 
-__version__='$Revision: 1.85 $'[11:-2]
+'''$Id: DT_Util.py,v 1.86 2002/03/27 10:14:02 htrd Exp $''' 
+__version__='$Revision: 1.86 $'[11:-2]
 
 import re, os
-from html_quote import html_quote # for import by other modules, dont remove!
+from html_quote import html_quote, ustr # for import by other modules, dont remove!
 from RestrictedPython.Guards import safe_builtins
 from RestrictedPython.Utilities import utility_builtins
 from RestrictedPython.Eval import RestrictionCapableEval
@@ -41,10 +41,9 @@ def int_param(params,md,name,default=0, st=type('')):
 try:
     import ExtensionClass
     from cDocumentTemplate import InstanceDict, TemplateDict, \
-         render_blocks, safe_callable
+         render_blocks, safe_callable, join_unicode
 except: from pDocumentTemplate import InstanceDict, TemplateDict, \
-        render_blocks, safe_callable
-
+        render_blocks, safe_callable, join_unicode
 
 functype = type(int_param)
 class NotBindable:

lib/python/DocumentTemplate/DT_Var.py

@@ -145,10 +145,10 @@ Evaluating expressions without rendering results
    
 
 ''' # '
-__rcs_id__='$Id: DT_Var.py,v 1.51 2002/03/18 20:15:27 andreasjung Exp $'
-__version__='$Revision: 1.51 $'[11:-2]
+__rcs_id__='$Id: DT_Var.py,v 1.52 2002/03/27 10:14:02 htrd Exp $'
+__version__='$Revision: 1.52 $'[11:-2]
 
-from DT_Util import parse_params, name_param, str
+from DT_Util import parse_params, name_param, str, ustr
 import os, string, re,  sys
 from urllib import quote, quote_plus
 from cgi import escape
@@ -253,7 +253,7 @@ class Var:
 
         # finally, pump it through the actual string format...
         fmt=self.fmt
-        if fmt=='s': val=str(val)
+        if fmt=='s': val=ustr(val)
         else: val = ('%'+self.fmt) % (val,)
 
         # next, look for upper, lower, etc

lib/python/DocumentTemplate/cDocumentTemplate.c

@@ -12,7 +12,7 @@
  ****************************************************************************/
 static char cDocumentTemplate_module_documentation[] = 
 ""
-"\n$Id: cDocumentTemplate.c,v 1.43 2002/03/21 15:48:54 htrd Exp $"
+"\n$Id: cDocumentTemplate.c,v 1.44 2002/03/27 10:14:02 htrd Exp $"
 ;
 
 #include "ExtensionClass.h"
@@ -22,7 +22,7 @@ static PyObject *py___call__, *py___roles__, *py_AUTHENTICATED_USER;
 static PyObject *py_hasRole, *py__proxy_roles, *py_Unauthorized;
 static PyObject *py_Unauthorized_fmt, *py_guarded_getattr;
 static PyObject *py__push, *py__pop, *py_aq_base, *py_renderNS;
-static PyObject *py___class__, *html_quote;
+static PyObject *py___class__, *html_quote, *ustr;
 
 /* ----------------------------------------------------- */
 
@@ -42,6 +42,8 @@ typedef struct {
 
 staticforward PyExtensionClass InstanceDictType;
 
+staticforward PyObject *_join_unicode(PyObject *prejoin);
+
 static PyObject *
 InstanceDict___init__(InstanceDictobject *self, PyObject *args)
 {
@@ -685,22 +687,43 @@ render_blocks_(PyObject *blocks, PyObject *rendered,
 	      if (PyString_Check(t)) t=PyObject_GetItem(md, t);
 	      else t=PyObject_CallObject(t, mda);
 
-              if (t == NULL || (! PyString_Check(t)))
+              if (t == NULL) return -1;
+
+              if (! ( PyString_Check(t) || PyUnicode_Check(t) ) )
                 {
-                  if (t) ASSIGN(t, PyObject_Str(t));
+                  ASSIGN(t, PyObject_CallFunction(ustr, "O", t));
                   UNLESS(t) return -1;
                 }
 
-              if (PyString_Check(t) 
-                  && PyTuple_GET_SIZE(block) == 3) /* html_quote */
+              if (PyTuple_GET_SIZE(block) == 3) /* html_quote */
                 {
-                  if (strchr(PyString_AS_STRING(t), '&')
-                      || strchr(PyString_AS_STRING(t), '<')
-                      || strchr(PyString_AS_STRING(t), '>')
-                      || strchr(PyString_AS_STRING(t), '"')
-                      )
-                    ASSIGN(t, PyObject_CallFunction(html_quote, "O", t));
-		    if (t == NULL) return -1;
+                  int skip_html_quote;
+                  if (PyString_Check(t))
+                    {
+                      if (strchr(PyString_AS_STRING(t), '&') ||
+                          strchr(PyString_AS_STRING(t), '<') ||
+                          strchr(PyString_AS_STRING(t), '>') ||
+                          strchr(PyString_AS_STRING(t), '"')     )
+                        {
+                          /* string includes html problem characters, so
+                             we cant skip the quoting process */
+                          skip_html_quote = 0;
+                        }
+                      else
+                        {
+                          skip_html_quote = 1;
+                        }
+                    }
+                  else
+                    {
+                      /* never skip the quoting for unicode strings */
+                      skip_html_quote = 0;
+                    }
+                  if (!skip_html_quote)
+                    {
+                      ASSIGN(t, PyObject_CallFunction(html_quote, "O", t));
+                      if (t == NULL) return -1;
+                    }
                 }
                   
               block = t;
@@ -787,7 +810,7 @@ render_blocks_(PyObject *blocks, PyObject *rendered,
               return -1;
             }
         }
-      else if (PyString_Check(block))
+      else if (PyString_Check(block) || PyUnicode_Check(block))
 	{
 	  Py_INCREF(block);
 	}
@@ -830,7 +853,7 @@ render_blocks(PyObject *self, PyObject *args)
   else if (l==1)
     ASSIGN(rendered, PySequence_GetItem(rendered,0));
   else
-    ASSIGN(rendered, PyObject_CallFunction(join,"OO",rendered,py_));
+    ASSIGN(rendered, _join_unicode(rendered));
 
   return rendered;
 
@@ -852,11 +875,65 @@ safe_callable(PyObject *self, PyObject *args)
     return PyInt_FromLong(1);
   else
     return PyInt_FromLong(0);
-}  
+}
+
+static PyObject *
+_join_unicode(PyObject *prejoin)
+{
+    PyObject *joined;
+    joined = PyObject_CallFunction(join,"OO",prejoin,py_);
+    if(!joined && PyErr_ExceptionMatches(PyExc_UnicodeError))
+    {
+        int i,l;
+        PyObject *list;
+        PyErr_Clear();
+        list = PySequence_List(prejoin);
+        if(!list)
+        {
+            return NULL;
+        }
+        l = PyList_Size(list);
+        for(i=0;i<l;++i)
+        {
+            PyObject *item = PyList_GetItem(list,i);
+            if(PyString_Check(item))
+            {
+                PyObject *unicode = PyUnicode_DecodeLatin1(PyString_AsString(item),PyString_Size(item),NULL);
+                if(unicode)
+                {
+                    PyList_SetItem(list,i,unicode);
+                }
+                else
+                {
+                    Py_DECREF(list);
+                    return NULL;
+                }
+           }
+       }
+       joined = PyObject_CallFunction(join,"OO",list,py_);
+       Py_DECREF(list);
+    }
+    return joined;
+}
+
+static PyObject *
+join_unicode(PyObject *self, PyObject *args)
+{
+  PyObject *ob;
+
+  UNLESS(PyArg_ParseTuple(args,"O", &ob)) return NULL;
+  return _join_unicode(ob);
+}
+
 
 static struct PyMethodDef Module_Level__methods[] = {
   {"render_blocks", (PyCFunction)render_blocks,	METH_VARARGS,
    ""},
+  {"join_unicode", (PyCFunction)join_unicode,	METH_VARARGS,
+   "join a list of plain strings into a single plain string,"
+   "a list of unicode strings into a single unicode strings,"
+   "or a list containing a mix into a single unicode string with"
+   "the plain strings converted from latin-1"},
   {"safe_callable", (PyCFunction)safe_callable,	METH_VARARGS,
    "callable() with a workaround for a problem with ExtensionClasses\n"
    "and __call__()."},
@@ -871,6 +948,8 @@ initcDocumentTemplate(void)
   DictInstanceType.ob_type=&PyType_Type;
 
   UNLESS (html_quote = PyImport_ImportModule("html_quote")) return;
+  ASSIGN(ustr, PyObject_GetAttrString(html_quote, "ustr"));
+  UNLESS (ustr) return;
   ASSIGN(html_quote, PyObject_GetAttrString(html_quote, "html_quote"));
   UNLESS (html_quote) return;
 

lib/python/DocumentTemplate/html_quote.py

 # split off into its own module for aliasing without circrefs
 
 from cgi import escape
+from ustr import ustr
 
 def html_quote(v, name='(Unknown name)', md={}):
-    return escape(str(v), 1)
+    return escape(ustr(v), 1)
 

lib/python/DocumentTemplate/pDocumentTemplate.py

@@ -13,8 +13,8 @@
 __doc__='''Python implementations of document template some features
 
 
-$Id: pDocumentTemplate.py,v 1.34 2002/02/07 17:47:42 andreasjung Exp $'''
-__version__='$Revision: 1.34 $'[11:-2]
+$Id: pDocumentTemplate.py,v 1.35 2002/03/27 10:14:02 htrd Exp $'''
+__version__='$Revision: 1.35 $'[11:-2]
 
 import  sys, types
 
@@ -41,6 +41,7 @@ def safe_callable(ob):
 
 
 StringType=type('')
+UnicodeType=type(u'')
 TupleType=type(())
 
 
@@ -187,7 +188,7 @@ def render_blocks(blocks, md):
                 section=section[0]
                 if type(section) is StringType: section=md[section]
                 else: section=section(md)
-                section=str(section)
+                section=ustr(section)
             else:
                 # if
                 cache={}
@@ -220,7 +221,7 @@ def render_blocks(blocks, md):
 
                 finally: md._pop()
 
-        elif type(section) is not StringType:
+        elif type(section) is not StringType and type(section) is not UnicodeType:
             section=section(md)
 
         if section: rendered.append(section)
@@ -228,5 +229,21 @@ def render_blocks(blocks, md):
     l=len(rendered)
     if l==0: return ''
     elif l==1: return rendered[0]
-    return ''.join(rendered)
-    return rendered
+    return join_unicode(rendered)
+
+def join_unicode(rendered):
+    """join a list of plain strings into a single plain string,
+    a list of unicode strings into a single unicode strings,
+    or a list containing a mix into a single unicode string with
+    the plain strings converted from latin-1
+    """
+    try:
+        return ''.join(rendered)
+    except UnicodeError:
+        # A mix of unicode string and non-ascii plain strings.
+        # Fix up the list, treating normal strings as latin-1
+        rendered = list(rendered)
+        for i in range(len(rendered)):
+            if type(rendered[i]) is StringType:
+                rendered[i] = unicode(rendered[i],'latin-1')
+        return u''.join(rendered)

lib/python/DocumentTemplate/tests/testDTMLUnicode.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+"""Document Template Tests
+"""
+
+__rcs_id__='$Id: testDTMLUnicode.py,v 1.2 2002/03/27 10:14:02 htrd Exp $'
+__version__='$Revision: 1.2 $'[11:-2]
+
+import sys, os
+import unittest
+
+from DocumentTemplate.DT_HTML import HTML, String
+from ExtensionClass import Base
+
+class force_str:
+    # A class whose string representation is not always a plain string:
+    def __init__(self,s):
+        self.s = s
+    def __str__(self):
+        return self.s
+
+class DTMLUnicodeTests (unittest.TestCase):
+
+    doc_class = HTML
+
+    def testAA(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = 'helloworld'
+        res = html(a='hello',b='world')
+        assert res == expected, `res`
+
+    def testUU(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'helloworld'
+        res = html(a=u'hello',b=u'world')
+        assert res == expected, `res`
+
+    def testAU(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'helloworld'
+        res = html(a='hello',b=u'world')
+        assert res == expected, `res`
+
+    def testAB(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = 'hello\xc8'
+        res = html(a='hello',b=chr(200))
+        assert res == expected, `res`
+
+    def testUB(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'hello\xc8'
+        res = html(a=u'hello',b=chr(200))
+        assert res == expected, `res`
+
+    def testUB2(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'\u07d0\xc8'
+        res = html(a=unichr(2000),b=chr(200))
+        assert res == expected, `res`
+
+    def testUnicodeStr(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'\u07d0\xc8'
+        res = html(a=force_str(unichr(2000)),b=chr(200))
+        assert res == expected, `res`
+
+    def testUqB(self):
+        html=self.doc_class('<dtml-var a html_quote><dtml-var b>')
+        expected = u'he&gt;llo\xc8'
+        res = html(a=u'he>llo',b=chr(200))
+        assert res == expected, `res`
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest( unittest.makeSuite( DTMLUnicodeTests ) )
+    return suite
+
+def main():
+    unittest.TextTestRunner().run(test_suite())
+
+if __name__ == '__main__':
+    main()

lib/python/DocumentTemplate/tests/testustr.py

+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+"""Document Template Tests
+"""
+
+__rcs_id__='$Id: testustr.py,v 1.2 2002/03/27 10:14:02 htrd Exp $'
+__version__='$Revision: 1.2 $'[11:-2]
+
+import sys, os
+import unittest
+
+from DocumentTemplate.ustr import ustr
+from ExtensionClass import Base
+
+class force_str:
+    # A class whose string representation is not always a plain string:
+    def __init__(self,s):
+        self.s = s
+    def __str__(self):
+        return self.s
+
+class UnicodeTests (unittest.TestCase):
+
+    def testPlain(self):
+        a = ustr('hello')
+        assert a=='hello', `a`
+        a = ustr(force_str('hello'))
+        assert a=='hello', `a`
+        a = ustr(chr(200))
+        assert a==chr(200), `a`
+        a = ustr(force_str(chr(200)))
+        assert a==chr(200), `a`
+        a = ustr(22)
+        assert a=='22', `a`
+        a = ustr([1,2,3])
+        assert a=='[1, 2, 3]', `a`
+
+    def testUnicode(self):
+        a = ustr(u'hello')
+        assert a=='hello', `a`
+        a = ustr(force_str(u'hello'))
+        assert a=='hello', `a`
+        a = ustr(unichr(200))
+        assert a==unichr(200), `a`
+        a = ustr(force_str(unichr(200)))
+        assert a==unichr(200), `a`
+
+    def testExceptions(self):
+        a = ustr(ValueError(unichr(200)))
+        assert a==unichr(200), `a`
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest( unittest.makeSuite( UnicodeTests ) )
+    return suite
+
+def main():
+    unittest.TextTestRunner().run(test_suite())
+
+if __name__ == '__main__':
+    main()

lib/python/DocumentTemplate/ustr.py

+from types import StringType, UnicodeType, InstanceType
+
+nasty_exception_str = Exception.__str__.im_func
+
+def ustr(v):
+    """convert an object to a plain string or unicode string
+    """
+    string_types = (StringType,UnicodeType)
+    if type(v) in string_types:
+        return v
+    else:
+        fn = getattr(v,'__str__',None)
+        if fn is not None:
+            # An object that wants to present its own string representation,
+            # but we dont know what type of string. We cant use any built-in
+            # function like str() or unicode() to retrieve it because
+            # they all constrain the type which potentially raises an exception.
+            # To avoid exceptions we have to call __str__ direct.
+            if getattr(fn,'im_func',None)==nasty_exception_str:
+                # Exception objects have been optimised into C, and their
+                # __str__ function fails when given a unicode object.
+                # Unfortunately this scenario is all too common when
+                # migrating to unicode, because of code which does:
+                # raise ValueError(something_I_wasnt_expecting_to_be_unicode)
+                return _exception_str(v)
+            else:
+                # Trust the object to do this right
+                v = fn()
+                if type(v) in string_types:
+                    return v
+                else:
+                    raise ValueError('__str__ returned wrong type')
+        # Drop through for non-instance types, and instances that
+        # do not define a special __str__
+        return str(v)
+
+
+def _exception_str(self):
+    if not self.args:
+        return ''
+    elif len(self.args) == 1:
+        return ustr(self.args[0])
+    else:
+        return str(self.args)

lib/python/OFS/SimpleItem.py

@@ -17,8 +17,8 @@ Aqueduct database adapters, etc.
 This module can also be used as a simple template for implementing new
 item types. 
 
-$Id: SimpleItem.py,v 1.93 2002/02/07 17:20:59 andreasjung Exp $'''
-__version__='$Revision: 1.93 $'[11:-2]
+$Id: SimpleItem.py,v 1.94 2002/03/27 10:14:03 htrd Exp $'''
+__version__='$Revision: 1.94 $'[11:-2]
 
 import re, sys, Globals, App.Management, Acquisition, App.Undo
 import AccessControl.Role, AccessControl.Owned, App.Common
@@ -30,6 +30,7 @@ from ComputedAttribute import ComputedAttribute
 from AccessControl import getSecurityManager
 from Traversable import Traversable
 from Acquisition import aq_base
+from DocumentTemplate.ustr import ustr
 import time
 
 import marshal
@@ -165,16 +166,15 @@ class Item(Base, Resource, CopySource, App.Management.Tabs, Traversable,
                 raise error_type, error_value, tb
 
             if not error_message:
-                if type(error_value) is InstanceType:
-                    try:
-                        s=str(error_value)
-                    except:
-                        pass
-                    else:
-                        if tagSearch(s) is not None:
-                            error_message=error_value
-                elif (type(error_value) is StringType
-                      and tagSearch(error_value) is not None):
+                try:
+                    s = ustr(error_value)
+                except:
+                    s = error_value
+                try:
+                    match = tagSearch(s)
+                except TypeError:
+                    match = None
+                if match is not None:
                     error_message=error_value
 
             if client is None: client=self
@@ -197,7 +197,8 @@ class Item(Base, Resource, CopySource, App.Management.Tabs, Traversable,
                     v = s(**kwargs)
                 else:
                     v = HTML.__call__(s, client, REQUEST, **kwargs)
-            except: v = error_value or "Sorry, an error occurred"
+            except:
+                v = error_value or "Sorry, an error occurred"
             raise error_type, v, tb
         finally:
             if hasattr(self, '_v_eek'): del self._v_eek

lib/python/OFS/dtml/properties.dtml

+<dtml-call "REQUEST.set('management_page_charset','UTF-8')">
+<dtml-var "u''">
 <dtml-var manage_page_header>
 <dtml-with "_(management_view='Properties')">
 <dtml-var manage_tabs>
@@ -42,7 +44,7 @@ property values, edit the values and click &quot;Save Changes&quot;.
 <tr>
   <td align="left" valign="top" width="16">
   <dtml-if "'d' in _['sequence-item'].get('mode', 'awd')">
-  <input type="checkbox" name="_ids:list" value="<dtml-var id html_quote>" 
+  <input type="checkbox" name="_ids:utf8:list" value="<dtml-var id html_quote>"
    id="cb-<dtml-var id>">
   <dtml-else>
   </dtml-if>
@@ -64,22 +66,22 @@ property values, edit the values and click &quot;Save Changes&quot;.
    value="<dtml-if "hasProperty(id)"><dtml-var 
    "('%s' % getProperty(id))" html_quote></dtml-if>">
   <dtml-elif "type in ('float', 'date')">
-  <input type="text" name="<dtml-var id>:<dtml-var type>" size="35"
+  <input type="text" name="<dtml-var id>:utf8:<dtml-var type>" size="35"
    value="<dtml-var "getProperty(id)" html_quote>">
-  <dtml-elif "type=='string'">
-  <input type="text" name="<dtml-var id>:string" size="35"
+  <dtml-elif "type in ['string','ustring']">
+  <input type="text" name="<dtml-var id>:utf8:<dtml-var type>" size="35"
    value="<dtml-var "getProperty(id)" html_quote>">
   <dtml-elif "type=='boolean'">
   <input type="checkbox" name="<dtml-var id>:boolean" size="35"
    <dtml-if "getProperty(id)">CHECKED</dtml-if>>
-  <dtml-elif "type=='tokens'">
-  <input type="text" name="<dtml-var id>:tokens" size="35"
+  <dtml-elif "type in ['tokens','utokens']">
+  <input type="text" name="<dtml-var id>:utf8:<dtml-var type>" size="35"
    value="<dtml-in "getProperty(id)"><dtml-var sequence-item html_quote> </dtml-in>">
-  <dtml-elif "type=='text'">
-  <textarea name="<dtml-var id>:text" rows="6" cols="35"><dtml-var 
+  <dtml-elif "type in ['text','utext']">
+  <textarea name="<dtml-var id>:utf8:<dtml-var type>" rows="6" cols="35"><dtml-var
    "getProperty(id)" html_quote></textarea>
-  <dtml-elif "type=='lines'">
-  <textarea name="<dtml-var id>:lines" rows="6" cols="35"><dtml-in
+  <dtml-elif "type in ['lines','ulines']">
+  <textarea name="<dtml-var id>:utf8:<dtml-var type>" rows="6" cols="35"><dtml-in
   "getProperty(id)"><dtml-var sequence-item html_quote><dtml-if
    sequence-end><dtml-else><dtml-var "'\n'"></dtml-if></dtml-in></textarea>
 
@@ -87,7 +89,7 @@ property values, edit the values and click &quot;Save Changes&quot;.
 
     <dtml-if "hasProperty(select_variable)">
     <div class="form-element">
-    <select name="<dtml-var id>">
+    <select name="<dtml-var id>:utf8:text">
       <dtml-in "getProperty(select_variable)">
          <option
           <dtml-if "_['sequence-item']==getProperty(id)">SELECTED</dtml-if>
@@ -97,7 +99,7 @@ property values, edit the values and click &quot;Save Changes&quot;.
     </div>
     <dtml-elif "_.has_key(select_variable)">
     <div class="form-element">
-    <select name="<dtml-var id>">
+    <select name="<dtml-var id>:utf8:text">
       <dtml-in "_[select_variable]">
          <option
           <dtml-if "_['sequence-item']==getProperty(id)">SELECTED</dtml-if>
@@ -115,7 +117,7 @@ property values, edit the values and click &quot;Save Changes&quot;.
 
     <dtml-if "hasProperty(select_variable)">
     <div class="form-element">
-    <select name="<dtml-var id>:list" multiple
+    <select name="<dtml-var id>:utf8:list" multiple
       size="<dtml-var "_.min(7, _.len(getProperty(select_variable)))">">
       <dtml-in "getProperty(select_variable)">
          <option<dtml-if 
@@ -127,7 +129,7 @@ property values, edit the values and click &quot;Save Changes&quot;.
     </div>
     <dtml-elif "_.has_key(select_variable)">
     <div class="form-element">
-    <select name="<dtml-var id>:list" multiple
+    <select name="<dtml-var id>:utf8:list" multiple
       size="<dtml-var "_.min(7, _.len(_[select_variable]))">">
       <dtml-in "_[select_variable]">
          <option<dtml-if 
@@ -211,7 +213,7 @@ property and click the &quot;Add&quot; button.
   </div>
   </td>
   <td align="left" valign="top">
-  <input type="text" name="id" size="30" value=""/>
+  <input type="text" name="id:utf8:string" size="30" value=""/>
   </td>
   <td align="left" valign="top" class="form-label">
   Type
@@ -226,8 +228,12 @@ property and click the &quot;Add&quot; button.
       <option>lines</option>
       <option>long</option>
       <option selected>string</option>
+      <option>ustring</option>
       <option>text</option>
       <option>tokens</option>
+      <option>utext</option>
+      <option>utokens</option>
+      <option>ulines</option>
       <option>selection</option>
       <option>multiple selection</option>
     </select>
@@ -241,7 +247,7 @@ property and click the &quot;Add&quot; button.
   </div>
   </td>
   <td colspan=2 align="left" valign="top">
-  <input type="text" name="value" size="30" />
+  <input type="text" name="value:utf8:string" size="30" />
   </td>
   <td align="right" valign="top">
   <div class="form-element">

lib/python/Products/OFSP/help/Database-Management_Cache-Parameters.stx

@@ -4,28 +4,49 @@ Database Management - Cache Parameters: Zope database cache.
 
     This view allows you to view Zope database cache statistics and
     set cache parameters. The Zope database cache operates by keeping
-    frequently used objects in memory to improve performance. A large
+    recently used objects in memory to improve performance. A large
     cache improves object access speed, but increases memory usage. A
     small cache reduces memory usage but may slow down object access
     speed.
 
-    Information
-
-
-      'Total number of objects in the database' -- Indicates the
-      number of *persistent* objects in the Zope database.
-
-      'Total number of objects in all the caches combined' --
-      Indicates the number of objects which are currently cached in
-      memory.
-
-Controls
-
-    'Target size' -- Indicates ideal number of objects to have in
-    memory at any given time. This controls the size of the Zope
-    database cache.
-
-    'Target maximum time between accesses' -- Indicates the amount of
-    time after which Zope should remove an object from memory if it
-    hasn't been accessed. This controls how quickly the Zope database
-    cache removes objects that aren't being used.
+  Information
+
+    'Total number of objects in the database' -- Indicates the
+    number of persistent objects in the Zope database. All of these
+    objects are stored on disk, some of them are in memory too.
+
+    'Total number of objects in memory from all caches' --
+    Indicates the number of objects which are currently
+    cached in memory. This is a good indication of the amount
+    of memory used by Zope.
+
+    'Target number of objects in memory per cache' -- Indicates
+    the target number of objects to have in each cache at any
+    given time. Zope will allow the number of objects to grow
+    above this number while processing a request, but will always
+    reduce the level to this number at the end of each request.
+
+     This parameter is one factor affecting the amount of memory
+     use by Zope. It controls the amount of memory used per cache,
+     the other factor is the number of caches. In general,
+     Zope uses one cache per worker thread (specified by the '-t'
+     switch on the command line)
+
+    'Total number of objects in each cache' -- This table displays
+    one row for each object cache.
+
+     'Number of objects in memory' --  This value should not stay
+     larger than the configured target size for longer than one
+     transaction.
+
+      Note that the total number at the bottom of this column
+      should be the same as the figure in the top half of the
+      page. It may be slightly different because there is a small
+      time interval between the calculation of the two totals.
+
+     'Number of ghost objects' -- Ghost objects are those
+     which only have a tiny memory footprint because their full
+     state has not been loaded. You generally do not need to
+     worry about the number of ghost objects because they are
+     so small.
+    

lib/python/Products/OFSP/help/Database-Management_Flush-Cache.stx

@@ -6,18 +6,7 @@ Database Management - Flush Cache: Zope Database cache flush.
 
   Controls
 
-    'Full Sweep' -- Allows you to remove objects from the cache. The 
-    associated field (with the default value of 60 seconds) indicates 
-    the number of seconds within which an object must have been 
-    accessed in order *not* to be deactivated by  the flush operation.
+    'Minimize' -- Allows you to remove all persistent objects from
+    memory. They will be reloaded again on demand, when they
+    are next accessed.
 
-    'Minimize' -- Allows you to remove objects from
-    the cache. The field (with the  default value of 60 seconds)
-    indicates the number of seconds within which an object must have
-    been accessed in order not to be deactivated by the flush
-    operation.
-
-      The minimize operation differs from the full sweep in that it
-      removes all objects which are no longer referenced from the
-      root, while the full sweep merely removes most objects. The
-      minimize operation takes longer than full sweep.

lib/python/Products/OFSP/help/Properties.stx

@@ -15,21 +15,26 @@ Properties - Define properties.
 
       'int' -- An integer number, for example, '12'. 
 
-      'lines' -- A list of strings, one per line. 
+      'lines', 'ulines' -- A list of strings, one per line. 
 
       'long' -- A long integer, for example '12232322322323232323423'. 
 
-      'string' -- A string of characters, for example 'This is a string'. 
+      'string', 'ustring' -- A string of characters, for example 'This is a string'. 
 
-      'text' -- A multi-line string, for example a paragraph. 
+      'text', 'utext' -- A multi-line string, for example a paragraph. 
 
-      'tokens' -- A list of strings separated by white space, for example
+      'tokens', 'utokens' -- A list of strings separated by white space, for example
       'one two three'. 
 
       'selection' -- A string selected by a pop-up menu. 
 
       'multiple selection' -- A list of strings selected by a selection list.
 
+      Some of these textual properties come in two forms. The 'u'-prefixed
+      form stores Unicode text. The form without the prefix stores a python
+      plain string object, which is commonly assumed to contain latin-1
+      text.
+
   Controls
 
      Editing Properties

lib/python/ZPublisher/Converters.py

@@ -10,7 +10,7 @@
 # FOR A PARTICULAR PURPOSE
 # 
 ##############################################################################
-__version__='$Revision: 1.14 $'[11:-2]
+__version__='$Revision: 1.15 $'[11:-2]
 
 import re
 from types import ListType, TupleType, UnicodeType
@@ -113,6 +113,41 @@ def field2date(v):
 def field2boolean(v):
     return v
 
+class _unicode_converter:
+    def __call__(self,v):
+        # Convert a regular python string. This probably doesnt do what you want,
+        # whatever that might be. If you are getting exceptions below, you
+        # probably missed the encoding tag from a form field name. Use:
+        #       <input name="description:utf8:ustring" .....
+        # rather than
+        #       <input name="description:ustring" .....
+        if hasattr(v,'read'): v=v.read()
+        v = unicode(v)
+        return self.convert_unicode(v)
+
+    def convert_unicode(self,v):
+        raise NotImplementedError('convert_unicode')
+
+class field2ustring(_unicode_converter):
+    def convert_unicode(self,v):
+        return v
+field2ustring = field2ustring()
+
+class field2utokens(_unicode_converter):
+    def convert_unicode(self,v):
+        return v.split()
+field2utokens = field2utokens()
+
+class field2utext(_unicode_converter):
+    def convert_unicode(self,v):
+        return unicode(field2text(v.encode('utf8')),'utf8')
+field2utext = field2utext()
+
+class field2ulines(_unicode_converter):
+    def convert_unicode(self,v):
+        return field2utext.convert_unicode(v).split('\n')
+field2ulines = field2ulines()
+
 type_converters = {
     'float':    field2float,
     'int':      field2int,
@@ -123,7 +158,11 @@ type_converters = {
     'tokens':   field2tokens,
     'lines':    field2lines,
     'text':     field2text,
-    'boolean':     field2boolean,
+    'boolean':  field2boolean,
+    'ustring':  field2ustring,
+    'utokens':  field2utokens,
+    'ulines':   field2ulines,
+    'utext':    field2utext,
     }
 
 get_converter=type_converters.get

lib/python/ZPublisher/HTTPRequest.py

@@ -11,9 +11,9 @@
 # 
 ##############################################################################
 
-__version__='$Revision: 1.62 $'[11:-2]
+__version__='$Revision: 1.63 $'[11:-2]
 
-import re, sys, os,  urllib, time, whrandom, cgi
+import re, sys, os,  urllib, time, whrandom, cgi, codecs
 from BaseRequest import BaseRequest
 from HTTPResponse import HTTPResponse
 from cgi import FieldStorage, escape
@@ -384,6 +384,7 @@ class HTTPRequest(BaseRequest):
                         item=item.value
 
                 flags=0
+                character_encoding = ''
 
                 # Loop through the different types and set
                 # the appropriate flags
@@ -431,6 +432,8 @@ class HTTPRequest(BaseRequest):
                             flags=flags|RECORDS
                         elif type_name == 'ignore_empty':
                             if not item: flags=flags|EMPTY
+                        elif has_codec(type_name):
+                            character_encoding = type_name
     
                         l=key.rfind(':')
                         if l < 0: break
@@ -456,7 +459,17 @@ class HTTPRequest(BaseRequest):
                     # defer conversion
                     if flags&CONVERTED:
                         try:
-                            item=converter(item)
+                            if character_encoding:
+                                # We have a string with a specified character encoding.
+                                # This gets passed to the converter either as unicode, if it can
+                                # handle it, or crunched back down to latin-1 if it can not.
+                                item = unicode(item,character_encoding)
+                                if hasattr(converter,'convert_unicode'):
+                                    item = converter.convert_unicode(item)
+                                else:
+                                    item = converter(item.encode('latin1'))
+                            else:
+                                item=converter(item)
                         except:
                             if (not item and not (flags&DEFAULT) and
                                 defaults.has_key(key)):
@@ -965,6 +978,13 @@ class HTTPRequest(BaseRequest):
                 return name, password
 
 
+def has_codec(x):
+    try:
+        codecs.lookup(x)
+    except LookupError:
+        return 0
+    else:
+        return 1
 
 
 base64=None

lib/python/ZPublisher/HTTPResponse.py

@@ -12,12 +12,12 @@
 ##############################################################################
 '''CGI Response Output formatter
 
-$Id: HTTPResponse.py,v 1.54 2002/03/15 22:34:17 evan Exp $'''
-__version__='$Revision: 1.54 $'[11:-2]
+$Id: HTTPResponse.py,v 1.55 2002/03/27 10:14:04 htrd Exp $'''
+__version__='$Revision: 1.55 $'[11:-2]
 
 import types, os, sys, re
 from string import translate, maketrans
-from types import StringType, InstanceType, LongType
+from types import StringType, InstanceType, LongType, UnicodeType
 from BaseResponse import BaseResponse
 from zExceptions import Unauthorized
 
@@ -241,7 +241,16 @@ class HTTPResponse(BaseResponse):
             if hasattr(body,'asHTML'):
                 body=body.asHTML()
 
-        body=str(body)
+        if type(body) is UnicodeType:
+            body = self._encode_unicode(body)
+        elif type(body) is StringType:
+            pass
+        else:
+            try:
+                body = str(body)
+            except UnicodeError:
+                body = _encode_unicode(unicode(body))
+
         l=len(body)
         if ((l < 200) and body[:1]=='<' and body.find('>')==l-1 and 
             bogus_str_search(body) is not None):
@@ -276,6 +285,16 @@ class HTTPResponse(BaseResponse):
         self.insertBase()
         return self
 
+    def _encode_unicode(self,body,charset_re=re.compile(r'text/[0-9a-z]+\s*;\s*charset=([-_0-9a-z]+)(?:(?:\s*;)|\Z)',re.IGNORECASE)):
+        # Encode the Unicode data as requested
+        if self.headers.has_key('content-type'):
+            match = charset_re.match(self.headers['content-type'])
+            if match:
+                encoding = match.group(1)
+                return body.encode(encoding)
+        # Use the default character encoding
+        return body.encode('latin1','replace')
+
     def setBase(self,base):
         'Set the base URL for the returned document.'
         if base[-1:] != '/':
@@ -594,23 +613,28 @@ class HTTPResponse(BaseResponse):
                 (str(t),
                  'Zope has exited normally.<p>' + self._traceback(t, v, tb)),
                 is_error=1)
-        elif type(b) is not types.StringType or tag_search(b) is None:
-            body = self.setBody(
-                (str(t),
-                'Sorry, a site error occurred.<p>'
-                 + self._traceback(t, v, tb)),
-                 is_error=1)
-        elif b.strip().lower()[:6]=='<html>' or \
-              b.strip().lower()[:14]=='<!doctype html':
-            # error is an HTML document, not just a snippet of html
-            body = self.setBody(b + self._traceback(t, '(see above)', tb),
-                              is_error=1)
         else:
-            body = self.setBody((str(t),
-                               b + self._traceback(t,'(see above)', tb)),
-                              is_error=1)
-        del tb
-        return body
+            try:
+                match = tag_search(b)
+            except TypeError:
+                match = None
+            if match is None:
+                body = self.setBody(
+                    (str(t),
+                    'Sorry, a site error occurred.<p>'
+                     + self._traceback(t, v, tb)),
+                     is_error=1)
+            elif b.strip().lower()[:6]=='<html>' or \
+                  b.strip().lower()[:14]=='<!doctype html':
+                # error is an HTML document, not just a snippet of html
+                body = self.setBody(b + self._traceback(t, '(see above)', tb),
+                                  is_error=1)
+            else:
+                body = self.setBody((str(t),
+                                   b + self._traceback(t,'(see above)', tb)),
+                                  is_error=1)
+            del tb
+            return body
 
     _wrote=None
 
@@ -705,3 +729,4 @@ class HTTPResponse(BaseResponse):
 
         self.stdout.write(data)
 
+