added a PreferredCharset resolver that also

deal with Collector #2180

lib/python/Products/PageTemplates/configure.zcml

@@ -2,7 +2,7 @@
 
   <utility
       provides="Products.PageTemplates.interfaces.IUnicodeEncodingConflictResolver"
-      component="Products.PageTemplates.unicodeconflictresolver.DefaultUnicodeEncodingConflictResolver"
+      component="Products.PageTemplates.unicodeconflictresolver.PreferredCharsetResolver"
       />
 
 </configure>

lib/python/Products/PageTemplates/unicodeconflictresolver.py

@@ -12,7 +12,10 @@
 ##############################################################################
 
 import sys
+
 from zope.interface import implements
+from zope.i18n.interfaces import IUserPreferredCharsets
+
 from Products.PageTemplates.interfaces import IUnicodeEncodingConflictResolver
 
 default_encoding = sys.getdefaultencoding()
@@ -50,7 +53,44 @@ class Z2UnicodeEncodingConflictResolver:
             encoding = getattr(context, 'management_page_charset', default_encoding)
             return unicode(text, encoding, self.mode)
 
+class PreferredCharsetResolver:
+    """ A resolver that tries uses the HTTP_ACCEPT_CHARSET
+        header.
+    """
+
+    implements(IUnicodeEncodingConflictResolver)
+
+    def resolve(self, context, text, expression):
+
+        request = context.REQUEST
+
+        if not hasattr(request, '__zpt_available_charsets'):
+            charsets = IUserPreferredCharsets(request).getPreferredCharsets()
+
+            # add management_page_charset as one fallback
+            management_charset = getattr(context, 'management_page_charset', None)
+            if management_charset:
+                charsets.append(management_charset)
+
+            # add Python's default encoding as last fallback
+            charsets.append(default_encoding)               
+
+            # cache list of charsets
+            request.__zpt_available_charsets = charsets
+
+        else:
+            charsets = request.__zpt_available_charsets
+
+        for enc in charsets:
+            try:
+                return unicode(text, enc)
+            except UnicodeDecodeError:
+                pass
+
+        return text
+
 
 StrictUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('strict')
 ReplacingUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('replace')
 IgnoringUnicodeEncodingConflictResolver = Z2UnicodeEncodingConflictResolver('ignore')
+PreferredCharsetResolver = PreferredCharsetResolver()