support for a configurable resolver for UnicodeDecodeErrors

doc/CHANGES.txt

@@ -48,6 +48,10 @@ Zope Changes
         'output_encodings' property that controls the conversion from/to unicode
         for WebDAV/FTP operations.
 
+      - the ZPT implementation has now a configurable option in order how to deal 
+        with UnicodeDecodeErrors. A custom UnicodeEncodingConflictResolver can
+        be configured through ZCML (see Products/PageTemplates/(configure.zcml,
+        unicodeconflictresolver.py, interfaces.py)
 
     Bugs Fixed
 

lib/python/Products/PageTemplates/Expressions.py

@@ -17,6 +17,10 @@ for Python expressions, string literals, and paths.
 
 $Id$
 """
+
+import logging
+
+from zope.component import getUtility
 from zope.interface import implements
 from zope.tales.tales import Context, Iterator
 from zope.tales.expressions import PathExpr, StringExpr, NotExpr
@@ -31,13 +35,17 @@ import OFS.interfaces
 from MultiMapping import MultiMapping
 from Acquisition import aq_base
 from zExceptions import NotFound, Unauthorized
+
 from Products.Five.browser.providerexpression import Z2ProviderExpression
 from Products.PageTemplates import ZRPythonExpr
 from Products.PageTemplates.DeferExpr import LazyExpr
 from Products.PageTemplates.GlobalTranslationService import getGlobalTranslationService
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
 
 SecureModuleImporter = ZRPythonExpr._SecureModuleImporter()
 
+LOG = logging.getLogger('Expressions')
+
 # BBB 2005/05/01 -- remove after 12 months
 import zope.deprecation
 from zope.deprecation import deprecate
@@ -173,6 +181,44 @@ class ZopeContext(Context):
             domain, msgid, mapping=mapping,
             context=context, default=default)
 
+
+    def evaluateText(self, expr):
+        """ customized version in order to get rid of unicode
+            errors for all and ever
+        """
+
+        text = self.evaluate(expr)
+
+        if text is self.getDefault() or text is None:
+            # XXX: should be unicode???
+            return text
+
+        if isinstance(text, unicode):
+            # we love unicode, nothing to do
+            return text
+
+        elif isinstance(text, str):
+            # bahh...non-unicode string..we need to convert it to unicode
+            resolver = getUtility(IUnicodeEncodingConflictResolver)
+
+            try:
+                return resolver.resolve(self.contexts['context'], text, expr)
+            except UnicodeDecodeError,e:
+                LOG.error("""UnicodeDecodeError detected for expression "%s"\n"""
+                          """Resolver class: %s\n"""
+                          """Exception text: %s\n"""
+                          """Template: %s\n"""
+                          """Rendered text: %r"""  % \
+                          (expr, resolver.__class__, e, 
+                            self.contexts['template'].absolute_url(1), text))
+                raise 
+        else:
+
+            # This is a weird culprit ...calling unicode() on non-string
+            # objects
+            return unicode(text)
+
+
 class ZopeEngine(zope.app.pagetemplate.engine.ZopeEngine):
 
     _create_context = ZopeContext

lib/python/Products/PageTemplates/configure.zcml

+<configure xmlns="http://namespaces.zope.org/zope">
+
+  <utility
+      provides="Products.PageTemplates.interfaces.IUnicodeEncodingConflictResolver"
+      component="Products.PageTemplates.unicodeconflictresolver.DefaultUnicodeEncodingConflictResolver"
+      />
+
+</configure>

lib/python/Products/PageTemplates/interfaces.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+from zope.interface import Interface
+
+class IUnicodeEncodingConflictResolver(Interface):
+    """ A utility that tries to convert a non-unicode string into
+       a Python unicode by implementing some policy in order
+       to figure out a possible encoding - either through the
+       calling context, the location or the system environment
+    """
+
+    def resolve(context, text, expression):
+        """ Returns 'text' as unicode string. 
+            'context' is the current context object.
+            'expression' is the original expression (can be used for 
+            logging purposes)
+        """
+
+

lib/python/Products/PageTemplates/unicodeconflictresolver.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import sys
+from zope.interface import implements
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
+
+default_encoding = sys.getdefaultencoding()
+
+class DefaultUnicodeEncodingConflictResolver:
+    """ This resolver implements the old-style behavior and will 
+        raise an exception in case of the string 'text' can't be converted
+        propertly to unicode.
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
+
+    def resolve(self, context, text, expression):
+        return unicode(text)
+
+DefaultUnicodeEncodingConflictResolver = DefaultUnicodeEncodingConflictResolver()
+
+
+class Z2UnicodeEncodingConflictResolver:
+    """ This resolver tries to lookup the encoding from the 
+        'management_page_charset' property and defaults to 
+        sys.getdefaultencoding().
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
+
+    def __init__(self, mode='strict'):
+        self.mode = mode
+
+    def resolve(self, context, text, expression):
+
+        try:
+            return unicode(text)
+        except UnicodeDecodeError:
+            encoding = getattr(context, 'managment_page_charset', default_encoding)
+            return unicode(text, encoding, self.mode)
+
+
+StrictUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('strict')
+ReplacingUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('replace')
+IgnoringUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('ignore')