- implemented some unicode encoding conflict resolvers

- configuration through ZCML
- additional useful logging

lib/python/Products/PageTemplates/Expressions.py

@@ -18,8 +18,9 @@ for Python expressions, string literals, and paths.
 $Id$
 """
 
-import sys
+import logging
 
+from zope.component import getUtility
 from zope.interface import implements
 from zope.tales.tales import Context, Iterator
 from zope.tales.expressions import PathExpr, StringExpr, NotExpr
@@ -34,13 +35,17 @@ import OFS.interfaces
 from MultiMapping import MultiMapping
 from Acquisition import aq_base
 from zExceptions import NotFound, Unauthorized
+
 from Products.Five.browser.providerexpression import Z2ProviderExpression
 from Products.PageTemplates import ZRPythonExpr
 from Products.PageTemplates.DeferExpr import LazyExpr
 from Products.PageTemplates.GlobalTranslationService import getGlobalTranslationService
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
 
 SecureModuleImporter = ZRPythonExpr._SecureModuleImporter()
 
+LOG = logging.getLogger('Expressions')
+
 # BBB 2005/05/01 -- remove after 12 months
 import zope.deprecation
 from zope.deprecation import deprecate
@@ -183,9 +188,9 @@ class ZopeContext(Context):
         """
 
         text = self.evaluate(expr)
-#        print expr, repr(text), text.__class__
 
         if text is self.getDefault() or text is None:
+            # XXX: should be unicode???
             return text
 
         if isinstance(text, unicode):
@@ -193,26 +198,20 @@ class ZopeContext(Context):
             return text
 
         elif isinstance(text, str):
-            # waahhh...a standard string
-            # trying to be somewhat smart...this must be
-            # replaced with some kind of configurable
-            # UnicodeEncodingConflictResolver (what a name)
+            # bahh...non-unicode string..we need to convert it to unicode
+            resolver = getUtility(IUnicodeEncodingConflictResolver)
 
             try:
-                return unicode(text)
-
-            except UnicodeDecodeError:
-                # check for management_page_charset property, default to Python's
-                # default encoding
-
-                encoding = getattr(self.contexts['context'], 'management_page_charset', sys.getdefaultencoding())
-
-                try:
-                    return unicode(text, encoding)
-                except UnicodeDecodeError:
-                    # errors='replace' sucks...it's fine for now
-                    return unicode(text, encoding, 'replace')
-
+                return resolver.resolve(self.contexts['context'], text, expr)
+            except UnicodeDecodeError,e:
+                LOG.error("""UnicodeDecodeError detected for expression "%s"\n"""
+                          """Resolver class: %s\n"""
+                          """Exception text: %s\n"""
+                          """Template: %s\n"""
+                          """Rendered text: %r"""  % \
+                          (expr, resolver.__class__, e, 
+                            self.contexts['template'].absolute_url(1), text))
+                raise 
         else:
 
             # This is a weird culprit ...calling unicode() on non-string

lib/python/Products/PageTemplates/configure.zcml

-<configure xmlns="http://namespaces.zope.org/zope"
-           xmlns:browser="http://namespaces.zope.org/browser"
-           xmlns:five="http://namespaces.zope.org/five">
-
+<configure xmlns="http://namespaces.zope.org/zope">
 
   <utility
-      provides="zope.component.interfaces.IFactory"
-      component="Products.PageTemplates.unicodeconflictresolver.UnicodeEncodingResolverFactory"
+      provides="Products.PageTemplates.interfaces.IUnicodeEncodingConflictResolver"
+      component="Products.PageTemplates.unicodeconflictresolver.DefaultUnicodeEncodingConflictResolver"
       />
 
 </configure>

lib/python/Products/PageTemplates/interfaces.py

-
+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
 
 from zope.interface import Interface
 
 class IUnicodeEncodingConflictResolver(Interface):
+    """ A utility that tries to convert a non-unicode string into
+       a Python unicode by implementing some policy in order
+       to figure out a possible encoding - either through the
+       calling context, the location or the system environment
+    """
 
-    def resolve(context, text):
+    def resolve(context, text, expression):
         """ Returns 'text' as unicode string. 
-            'context' is the current context object
+            'context' is the current context object.
+            'expression' is the original expression (can be used for 
+            logging purposes)
         """
 
 

lib/python/Products/PageTemplates/unicodeconflictresolver.py

-###########################################################################
-# TextIndexNG V 3                
-# The next generation TextIndex for Zope
+##############################################################################
 #
-# This software is governed by a license. See
-# LICENSE.txt for the terms of this license.
-###########################################################################
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
 
+import sys
+from zope.interface import implements
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
 
+default_encoding = sys.getdefaultencoding()
 
-from zope.component.interfaces import IFactory
-from zope.interface import implements, implementedBy
+class DefaultUnicodeEncodingConflictResolver:
+    """ This resolver implements the old-style behavior and will 
+        raise an exception in case of the string 'text' can't be converted
+        propertly to unicode.
+    """
 
-from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
+    implements(IUnicodeEncodingConflictResolver)
 
-class UnicodeEncodingResolver:
+    def resolve(self, context, text, expression):
+        return unicode(text)
+
+DefaultUnicodeEncodingConflictResolver = DefaultUnicodeEncodingConflictResolver()
 
-    implements(IUnicodeEncodingConflictResolver)
 
-    def __init__(self, context, text):
-        self.context = context
-        self.text = text
+class Z2UnicodeEncodingConflictResolver:
+    """ This resolver tries to lookup the encoding from the 
+        'management_page_charset' property and defaults to 
+        sys.getdefaultencoding().
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
 
-    def resolve(self, context, text):
-        return unicode(self.text, errors='replace')
+    def __init__(self, mode='strict'):
+        self.mode = mode
 
-class UnicodeEncodingResolverFactory:
-    
-    implements(IFactory)
+    def resolve(self, context, text, expression):
 
-    def __call__(self, context, text):
-        return UnicodeEncodingResolver(context, text)
+        try:
+            return unicode(text)
+        except UnicodeDecodeError:
+            encoding = getattr(context, 'managment_page_charset', default_encoding)
+            return unicode(text, encoding, self.mode)
 
-    def getInterfaces(self):
-        return implementedBy(UnicodeEncodingResolverFactory)
 
-UnicodeEncodingResolverFactory = UnicodeEncodingResolverFactory() 
+StrictUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('strict')
+ReplacingUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('replace')
+IgnoringUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('ignore')