I18n interpolation now tries to deal with the case where there is a mix

of Unicode and non-ascii string that are incompatible (because the encoding of the latter is unknown) by substituting a representation of the non-ascii string. I18n interpolation doesn't fail anymore if a i18n:name is not provided, the ${string} in the translation is just left as is. Collector #696: tal:replace of a non-string (a number for examlpe) associated with a i18n:name failed to be interpolated properly. Improved tests for i18n and added tests for interpolate().

I18n interpolation now tries to deal with the case where there is a mix
of Unicode and non-ascii string that are incompatible (because the encoding of the latter is unknown) by substituting a representation of the non-ascii string. I18n interpolation doesn't fail anymore if a i18n:name is not provided, the ${string} in the translation is just left as is. Collector #696: tal:replace of a non-string (a number for examlpe) associated with a i18n:name failed to be interpolated properly. Improved tests for i18n and added tests for interpolate().
d0311171 · Florent Guillaume · 02338f11 · d0311171 · d0311171 · d0311171
Commit d0311171 authored Jan 30, 2003 by Florent Guillaume
7 changed files
--- a/doc/CHANGES.txt
+++ b/doc/CHANGES.txt
@@ -35,6 +35,18 @@ Zope Changes

    Bugs Fixed

+     - I18n interpolation now tries to deal with the case where there is
+       a mix of Unicode and non-ascii string that are incompatible
+       (because the encoding of the latter is unknown) by substituting a
+       representation of the non-ascii string.
+
+     - I18n interpolation doesn't fail anymore if a i18n:name is not
+       provided, the ${string} in the translation is just left as is.
+
+     - Collector #696: tal:replace of a non-string (a number for
+       examlpe) associated with a i18n:name failed to be interpolated
+       properly.
+
     - Collector #771: ZCatalog failed to index DTML Document if the name
       of a catalog metadata was identical with the name of an acquired
       object.

--- a/lib/python/Products/PageTemplates/tests/input/CheckI18nTranslateHooked.html
+++ b/lib/python/Products/PageTemplates/tests/input/CheckI18nTranslateHooked.html
@@ -7,6 +7,9 @@
     <span tal:replace="string:Lomax" i18n:name="name" /> was born in
     <span tal:replace="string:Antarctica" i18n:name="country" />.
   </p>
+   <p i18n:translate="hmm">
+     I'm <span tal:replace="python:25" i18n:name="age">Age</span>
+   </p>
 </head>
 </body>
 </html>
--- a/lib/python/Products/PageTemplates/tests/output/CheckI18nTranslateHooked.html
+++ b/lib/python/Products/PageTemplates/tests/output/CheckI18nTranslateHooked.html
 <html>
 <body>
 <head>
-   <p>[foo](bar)</p>
-   <a href="foo" alt="[default](alttext)">link</a>
-   <p>[dom](${name} was born in ${country}.)</p>
+   <p>[foo](bar/{})</p>
+   <a href="foo" alt="[default](alttext/{})">link</a>
+   <p>[dom](${name} was born in ${country}./{'country':'Antarctica','name':'Lomax'})</p>
+   <p>[default](hmm/{'age':25})</p>
 </head>
 </body>
 </html>
--- a/lib/python/Products/PageTemplates/tests/testHTMLTests.py
+++ b/lib/python/Products/PageTemplates/tests/testHTMLTests.py
@@ -28,8 +28,14 @@ class Folder(util.Base):
    pass

 class TestTranslationService:
-    def translate(self, domain, msgid, *args, **kw):
-        return "[%s](%s)" % (domain, msgid)
+    def translate(self, domain, msgid, mapping=None, *args, **kw):
+        maps = []
+        if mapping is not None:
+            # Get a deterministic, sorted representation of dicts.
+            for k, v in mapping.items():
+                maps.append('%s:%s' % (`k`, `v`))
+            maps.sort()
+        return "[%s](%s/{%s})" % (domain, msgid, ','.join(maps))


 class UnitTestSecurityPolicy:

--- a/lib/python/TAL/DummyEngine.py
+++ b/lib/python/TAL/DummyEngine.py
@@ -240,7 +240,7 @@ class DummyDomain:
        # substrings.  Then upcase everything but the placeholders, then glue
        # things back together.
        def repl(m, mapping=mapping):
-            return mapping[m.group(m.lastindex).lower()]
+            return ustr(mapping[m.group(m.lastindex).lower()])
        cre = re.compile(r'\$(?:([_A-Z]\w*)|\{([_A-Z]\w*)\})')
        return cre.sub(repl, msgid.upper())


--- a/lib/python/TAL/TALInterpreter.py
+++ b/lib/python/TAL/TALInterpreter.py
@@ -52,7 +52,11 @@ _interp_regex = re.compile(r'(?<!\$)(\$(?:%(n)s|{%(n)s}))' %({'n': NAME_RE}))
 _get_var_regex = re.compile(r'%(n)s' %({'n': NAME_RE}))

 def interpolate(text, mapping):
-    """Interpolate ${keyword} substitutions."""
+    """Interpolate ${keyword} substitutions.
+
+    This is called when no translation is provided by the translation
+    service.
+    """
    if not mapping:
        return text
    # Find all the spots we want to substitute.
@@ -60,7 +64,17 @@ def interpolate(text, mapping):
    # Now substitute with the variables in mapping.
    for string in to_replace:
        var = _get_var_regex.findall(string)[0]
-        text = text.replace(string, mapping.get(var))
+        if mapping.has_key(var):
+            # Call ustr because we may have an integer for instance.
+            subst = ustr(mapping[var])
+            try:
+                text = text.replace(string, subst)
+            except UnicodeError:
+                # subst contains high-bit chars...
+                # As we have no way of knowing the correct encoding,
+                # substitue something instead of raising an exception.
+                subst = `subst`[1:-1]
+                text = text.replace(string, subst)
    return text



--- a/lib/python/TAL/tests/test_talinterpreter.py
+++ b/lib/python/TAL/tests/test_talinterpreter.py
@@ -11,6 +11,7 @@ from StringIO import StringIO
 from TAL.TALDefs import METALError
 from TAL.HTMLTALParser import HTMLTALParser
 from TAL.TALInterpreter import TALInterpreter
+from TAL.TALInterpreter import interpolate
 from TAL.DummyEngine import DummyEngine


@@ -74,6 +75,15 @@ class OutputPresentationTestCase(TestCaseBase):
        EXPECTED = u"""dj-vu""" "\n"
        self.compare(INPUT, EXPECTED)

+    def check_i18n_replace_number(self):
+        INPUT = """
+        <p i18n:translate="foo ${bar}">
+        <span tal:replace="python:123" i18n:name="bar">para</span>
+        </p>"""
+        EXPECTED = u"""
+        <p>FOO 123</p>""" "\n"
+        self.compare(INPUT, EXPECTED)
+
    def check_entities(self):
        INPUT = ('<img tal:define="foo nothing" '
                 'alt="&a; &#1; &#x0a; &a &#45 &; &#0a; <>" />')
@@ -88,10 +98,55 @@ class OutputPresentationTestCase(TestCaseBase):
        interp()
        self.assertEqual(sio.getvalue(), EXPECTED)

+class InterpolateTestCase(TestCaseBase):
+    def check_syntax_ok(self):
+        text = "foo ${bar_0MAN} $baz_zz bee"
+        mapping = {'bar_0MAN': 'fish', 'baz_zz': 'moo'}
+        expected = "foo fish moo bee"
+        self.assertEqual(interpolate(text, mapping), expected)
+
+    def check_syntax_bad(self):
+        text = "foo $_bar_man} $ ${baz bee"
+        mapping = {'_bar_man': 'fish', 'baz': 'moo'}
+        expected = text
+        self.assertEqual(interpolate(text, mapping), expected)
+
+    def check_missing(self):
+        text = "foo ${bar} ${baz}"
+        mapping = {'bar': 'fish'}
+        expected = "foo fish ${baz}"
+        self.assertEqual(interpolate(text, mapping), expected)
+
+    def check_redundant(self):
+        text = "foo ${bar}"
+        mapping = {'bar': 'fish', 'baz': 'moo'}
+        expected = "foo fish"
+        self.assertEqual(interpolate(text, mapping), expected)
+
+    def check_numeric(self):
+        text = "foo ${bar}"
+        mapping = {'bar': 123}
+        expected = "foo 123"
+        self.assertEqual(interpolate(text, mapping), expected)
+
+    def check_unicode(self):
+        text = u"foo ${bar}"
+        mapping = {u'bar': u'baz'}
+        expected = u"foo baz"
+        self.assertEqual(interpolate(text, mapping), expected)
+
+    def check_unicode_mixed_unknown_encoding(self):
+        # This test assumes that sys.getdefaultencoding is ascii...
+        text = u"foo ${bar}"
+        mapping = {u'bar': 'd\xe9j\xe0'}
+        expected = u"foo d\\xe9j\\xe0"
+        self.assertEqual(interpolate(text, mapping), expected)
+
 def test_suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(MacroErrorsTestCase, "check_"))
    suite.addTest(unittest.makeSuite(OutputPresentationTestCase, "check_"))
+    suite.addTest(unittest.makeSuite(InterpolateTestCase, "check_"))
    return suite