merged UnicodeEncodingConflictResolver

lib/python/Products/PageTemplates/Expressions.py

@@ -17,6 +17,10 @@ for Python expressions, string literals, and paths.
 
 $Id$
 """
+
+import logging
+
+from zope.component import getUtility
 from zope.interface import implements
 from zope.tales.tales import Context, Iterator
 from zope.tales.expressions import PathExpr, StringExpr, NotExpr
@@ -31,13 +35,17 @@ import OFS.interfaces
 from MultiMapping import MultiMapping
 from Acquisition import aq_base
 from zExceptions import NotFound, Unauthorized
+
 from Products.Five.browser.providerexpression import Z2ProviderExpression
 from Products.PageTemplates import ZRPythonExpr
 from Products.PageTemplates.DeferExpr import LazyExpr
 from Products.PageTemplates.GlobalTranslationService import getGlobalTranslationService
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
 
 SecureModuleImporter = ZRPythonExpr._SecureModuleImporter()
 
+LOG = logging.getLogger('Expressions')
+
 # BBB 2005/05/01 -- remove after 12 months
 import zope.deprecation
 from zope.deprecation import deprecate
@@ -173,6 +181,44 @@ class ZopeContext(Context):
             domain, msgid, mapping=mapping,
             context=context, default=default)
 
+
+    def evaluateText(self, expr):
+        """ customized version in order to get rid of unicode
+            errors for all and ever
+        """
+
+        text = self.evaluate(expr)
+
+        if text is self.getDefault() or text is None:
+            # XXX: should be unicode???
+            return text
+
+        if isinstance(text, unicode):
+            # we love unicode, nothing to do
+            return text
+
+        elif isinstance(text, str):
+            # bahh...non-unicode string..we need to convert it to unicode
+            resolver = getUtility(IUnicodeEncodingConflictResolver)
+
+            try:
+                return resolver.resolve(self.contexts['context'], text, expr)
+            except UnicodeDecodeError,e:
+                LOG.error("""UnicodeDecodeError detected for expression "%s"\n"""
+                          """Resolver class: %s\n"""
+                          """Exception text: %s\n"""
+                          """Template: %s\n"""
+                          """Rendered text: %r"""  % \
+                          (expr, resolver.__class__, e, 
+                            self.contexts['template'].absolute_url(1), text))
+                raise 
+        else:
+
+            # This is a weird culprit ...calling unicode() on non-string
+            # objects
+            return unicode(text)
+
+
 class ZopeEngine(zope.app.pagetemplate.engine.ZopeEngine):
 
     _create_context = ZopeContext

lib/python/Products/PageTemplates/ZopePageTemplate.py

@@ -40,7 +40,8 @@ from Products.PageTemplates.PageTemplateFile import PageTemplateFile
 from Products.PageTemplates.PageTemplateFile import guess_type
 from Products.PageTemplates.Expressions import SecureModuleImporter
 
-from Products.PageTemplates.utils import encodingFromXMLPreamble, charsetFromMetaEquiv, convertToUnicode
+from Products.PageTemplates.utils import encodingFromXMLPreamble, \
+         charsetFromMetaEquiv, convertToUnicode
             
 
 preferred_encodings = ['utf-8', 'iso-8859-15']
@@ -103,7 +104,8 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
     security.declareProtected(view_management_screens,
                               'read', 'ZScriptHTML_tryForm')
 
-    def __init__(self, id, text=None, content_type='text/html', strict=True, output_encoding='utf-8'):
+    def __init__(self, id, text=None, content_type='text/html', strict=True, 
+                 output_encoding='utf-8'):
         self.id = id
         self.expand = 0                                                               
         self.ZBindings_edit(self._default_bindings)
@@ -139,16 +141,13 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
             charset = charsetFromMetaEquiv(text)
 
             if is_unicode:
-
                 if charset:
                     encoding = None
                     output_encoding = charset
                 else:
                     encoding = None
                     output_encoding = 'iso-8859-15'
-
             else:
-
                 if charset:
                     encoding = charset
                     output_encoding = charset
@@ -287,7 +286,9 @@ class ZopePageTemplate(Script, PageTemplate, Historical, Cacheable,
     def write(self, text):
 
         if not isinstance(text, unicode):
-            text, encoding = convertToUnicode(text, self.content_type, preferred_encodings)
+            text, encoding = convertToUnicode(text, 
+                                              self.content_type,
+                                              preferred_encodings)
             self.output_encoding = encoding
 
         self.ZCacheable_invalidate()

lib/python/Products/PageTemplates/configure.zcml

+<configure xmlns="http://namespaces.zope.org/zope">
+
+  <utility
+      provides="Products.PageTemplates.interfaces.IUnicodeEncodingConflictResolver"
+      component="Products.PageTemplates.unicodeconflictresolver.DefaultUnicodeEncodingConflictResolver"
+      />
+
+</configure>

lib/python/Products/PageTemplates/interfaces.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+from zope.interface import Interface
+
+class IUnicodeEncodingConflictResolver(Interface):
+    """ A utility that tries to convert a non-unicode string into
+       a Python unicode by implementing some policy in order
+       to figure out a possible encoding - either through the
+       calling context, the location or the system environment
+    """
+
+    def resolve(context, text, expression):
+        """ Returns 'text' as unicode string. 
+            'context' is the current context object.
+            'expression' is the original expression (can be used for 
+            logging purposes)
+        """
+
+

lib/python/Products/PageTemplates/unicodeconflictresolver.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Corporation and Contributors. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import sys
+from zope.interface import implements
+from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
+
+default_encoding = sys.getdefaultencoding()
+
+class DefaultUnicodeEncodingConflictResolver:
+    """ This resolver implements the old-style behavior and will 
+        raise an exception in case of the string 'text' can't be converted
+        propertly to unicode.
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
+
+    def resolve(self, context, text, expression):
+        return unicode(text)
+
+DefaultUnicodeEncodingConflictResolver = DefaultUnicodeEncodingConflictResolver()
+
+
+class Z2UnicodeEncodingConflictResolver:
+    """ This resolver tries to lookup the encoding from the 
+        'management_page_charset' property and defaults to 
+        sys.getdefaultencoding().
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
+
+    def __init__(self, mode='strict'):
+        self.mode = mode
+
+    def resolve(self, context, text, expression):
+
+        try:
+            return unicode(text)
+        except UnicodeDecodeError:
+            encoding = getattr(context, 'managment_page_charset', default_encoding)
+            return unicode(text, encoding, self.mode)
+
+
+StrictUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('strict')
+ReplacingUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('replace')
+IgnoringUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('ignore')