Commit 5c1b57b6 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Jérome Perrin

py2/py3: unicode->str conversion in Python 2 only.

parent 9603c774
......@@ -8,7 +8,7 @@ to use re) that would analyze text content of the doc
and return a dictionary of properties.
"""
#Proxify to allow discover of metadata when publishing document
import six
information = context.getContentInformation()
result = {}
......@@ -16,7 +16,7 @@ property_id_list = context.propertyIds()
for k, v in information.items():
key = k.lower()
if v:
if isinstance(v, unicode): v = v.encode('utf-8')
if six.PY2 and isinstance(v, six.text_type): v = v.encode('utf-8')
if key in property_id_list:
if key == 'reference':
pass # XXX - We can not trust reference on getContentInformation
......
......@@ -7,6 +7,7 @@ To use, write your own method (probably External Method, since it is most likely
to use re) that would analyze text content of the doc
and return a dictionary of properties.
"""
import six
information = context.getContentInformation()
result = {}
......@@ -14,7 +15,7 @@ property_id_list = context.propertyIds()
for k, v in information.items():
key = k.lower()
if v:
if isinstance(v, unicode): v = v.encode('utf-8')
if six.PY2 and isinstance(v, six.text_type): v = v.encode('utf-8')
if key in property_id_list:
if key == 'reference':
pass # XXX - We can not trust reference on getContentInformation
......@@ -26,7 +27,7 @@ for k, v in information.items():
result['contributor'] = p.getRelativeUrl()
elif key == 'keywords':
if isinstance(v, (list, tuple)):
v = [isinstance(x, unicode) and x.encode('utf-8') or x for x in v]
v = [x.encode('utf-8') if six.PY2 and isinstance(x, text_type) else x for x in v]
else:
v = v.split()
result['subject_list'] = v
......
......@@ -46,6 +46,7 @@ from Products.ERP5Type.Utils import guessEncodingFromText
from lxml import html as etree_html
from lxml import etree
import six
class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContentHistoryMixin,
TextContent, File):
......@@ -91,16 +92,20 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContent
mapping = method(**kw)
is_str = isinstance(text, str)
if is_str:
if six.PY2 and is_str:
text = text.decode('utf-8')
class UnicodeMapping:
def __getitem__(self, item):
v = mapping[item]
if six.PY2:
if isinstance(v, str):
v = v.decode('utf-8')
elif not isinstance(v, unicode):
elif not isinstance(v, six.text_type):
v = str(v).decode('utf-8')
else:
if not isinstance(v, str):
v = str(v)
return v
unicode_mapping = UnicodeMapping()
......
......@@ -111,7 +111,7 @@ class CrawlableMixin:
# For now take into acount only a and img tags
if attribute_name not in ('href',):
continue
if isinstance(link, six.text_type):
if six.PY2 and isinstance(link, six.text_type):
link = link.encode('utf-8')
href_list.append(link)
return href_list
......@@ -129,7 +129,7 @@ class CrawlableMixin:
path_part = '/'.join(path_part.split('/')[:-1])
base_url = urlunsplit((splitted_url[0], splitted_url[1], path_part, None,
None))
if isinstance(base_url, six.text_type):
if six.PY2 and isinstance(base_url, six.text_type):
base_url = base_url.encode('utf-8')
return base_url
......@@ -145,7 +145,7 @@ class CrawlableMixin:
# in www.example.com or www.3.example.com
# keep only the example.com part
reference_domain = ''.join(reference_domain.split('.')[-2:])
if isinstance(reference_domain, six.text_type):
if six.PY2 and isinstance(reference_domain, six.text_type):
reference_domain = reference_domain.encode('utf-8')
url_list = []
base_url = self.getContentBaseURL()
......@@ -159,7 +159,7 @@ class CrawlableMixin:
if not url:
continue
url_domain = urlsplit(url)[1]
if isinstance(url_domain, six.text_type):
if six.PY2 and isinstance(url_domain, six.text_type):
url_domain = url_domain.encode('utf-8')
if url_domain and ''.join(url_domain.split('.')[-2:]) != reference_domain:
continue
......
......@@ -2712,7 +2712,10 @@ class ListBoxListRenderer(ListBoxRenderer):
title_listboxline = ListBoxLine()
title_listboxline.markTitleLine()
for c in self.getSelectedColumnList():
if six.PY2:
title_listboxline.addColumn(c[0], c[1].encode(self.getEncoding()))
else:
title_listboxline.addColumn(c[0], c[1])
listboxline_list.append(title_listboxline)
# Obtain the list of lines.
......@@ -2738,7 +2741,7 @@ class ListBoxListRenderer(ListBoxRenderer):
listboxline.checkLine(uid in checked_uid_set)
for (original_value, processed_value), (sql, title) in zip(line.getValueList(), self.getSelectedColumnList()):
if isinstance(original_value, six.text_type):
if six.PY2 and isinstance(original_value, six.text_type):
value = original_value.encode(self.getEncoding())
else:
value = original_value
......@@ -2756,7 +2759,7 @@ class ListBoxListRenderer(ListBoxRenderer):
stat_listboxline.markStatLine()
for (original_value, processed_value), (sql, title) in zip(self.getStatValueList(), self.getSelectedColumnList()):
if isinstance(original_value, six.text_type):
if six.PY2 and isinstance(original_value, six.text_type):
value = original_value.encode(self.getEncoding())
else:
value = original_value
......
......@@ -345,7 +345,10 @@ class OOoTemplate(ZopePageTemplate):
('style', 'draw:style-name', 'fr1')):
options_dict.setdefault(name, options_dict.pop(old_name, default))
if six.PY2:
picture = self._resolvePath(options_dict.pop('path').encode())
else:
picture = self._resolvePath(options_dict.pop('path'))
# If this is not a File, build a new file with this content
if not isinstance(picture, File):
......
......@@ -1757,7 +1757,7 @@ def legacyNormalizeUrl(url, base_url=None):
# Remove trailing '?'
# http://www.example.com/? -> http://www.example.com/
url = re_cleanup_tail.sub('', url)
if isinstance(url, six.text_type):
if six.PY2 and isinstance(url, six.text_type):
url = url.encode('utf-8')
return url
......@@ -1779,7 +1779,7 @@ def urlnormNormaliseUrl(url, base_url=None):
if base_url and not (url_protocol or url_domain):
# Make relative URL absolute
url = urljoin(base_url, url)
if isinstance(url, six.text_type):
if six.PY2 and isinstance(url, six.text_type):
url = url.encode('utf-8')
return url
......
......@@ -20,7 +20,10 @@ def formToXML(form, prologue=1):
else:
value = 'false'
sub_element = SubElement(form_as_xml, id)
if six.PY2:
sub_element.text = str(value).decode(encoding)
else:
sub_element.text = str(value)
groups = SubElement(form_as_xml, 'groups')
# export form groups
for group in form.get_groups(include_empty=1):
......@@ -36,8 +39,7 @@ def formToXML(form, prologue=1):
fields.append(field_element)
values_element = SubElement(field_element, 'values')
items = field.values.items()
items.sort()
items = sorted(field.values.items())
for key, value in items:
if value is None:
continue
......@@ -57,19 +59,27 @@ def formToXML(form, prologue=1):
if not isinstance(value, six.string_types):
value = str(value)
value_element = SubElement(values_element, key)
if six.PY2:
value_element.text = str(value).decode(encoding)
else:
value_element.text = str(value)
tales_element = SubElement(field_element, 'tales')
items = field.tales.items()
items.sort()
items = sorted(field.tales.items())
for key, value in items:
if value:
tale_element = SubElement(tales_element, key)
if six.PY2:
tale_element.text = str(value._text).decode(encoding)
else:
tale_element.text = str(value._text)
messages = SubElement(field_element, 'messages')
for message_key in field.get_error_names():
message_element = SubElement(messages, 'message', name=message_key)
if six.PY2:
message_element.text = field.get_error_message(message_key).decode(encoding)
else:
message_element.text = field.get_error_message(message_key)
# Special attribute for ProxyFields *delegated_list*
delegated_list = getattr(field, 'delegated_list', [])
if delegated_list:
......@@ -78,8 +88,12 @@ def formToXML(form, prologue=1):
[SubElement(delegated_list_element, delegated) for delegated in delegated_list]
if form.unicode_mode:
return etree.tostring(form_as_xml, encoding='utf-8',
xml = etree.tostring(form_as_xml, encoding='utf-8',
xml_declaration=True, pretty_print=True)
else:
return etree.tostring(form_as_xml, encoding=form.stored_encoding,
xml = etree.tostring(form_as_xml, encoding=form.stored_encoding,
xml_declaration=True, pretty_print=True)
if six.PY2:
return xml
else:
return xml.decode()
......@@ -517,7 +517,7 @@ class SelectionValidator(StringBaseValidator):
# will remain integers.
# XXX it is impossible with the UI currently to fill in unicode
# items, but it's possible to do it with the TALES tab
if field.get_value('unicode') and isinstance(item_value, six.text_type):
if six.PY2 and field.get_value('unicode') and isinstance(item_value, six.text_type):
str_value = item_value.encode(field.get_form_encoding())
else:
str_value = str(item_value)
......@@ -908,10 +908,15 @@ fullwidth_minus_character_list = (
)
def normalizeFullWidthNumber(value):
try:
if six.PY2:
value = unicodedata.normalize('NFKD', value.decode('UTF8'))
else:
value = unicodedata.normalize('NFKD', value)
if value[0] in fullwidth_minus_character_list:
value = u'-' + value[1:]
value = value.encode('ASCII', 'ignore')
if six.PY3:
value = value.decode()
except UnicodeDecodeError:
pass
return value
......@@ -41,7 +41,7 @@ def convert_to_xml_compatible_string(value):
"""
if not value:
return ''
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
value = value.decode('utf-8')
# remove control characters as described in the example from
......@@ -260,7 +260,7 @@ class Widget:
"""
if attr_dict is None:
attr_dict = {}
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
if value is None:
......@@ -287,7 +287,7 @@ class Widget:
if attr_dict is None:
attr_dict = {}
attr_dict['{%s}value-type' % OFFICE_URI] = 'string'
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
if value is None:
......@@ -330,7 +330,7 @@ class Widget:
"""
if attr_dict is None:
attr_dict = {}
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
if value is None:
......@@ -618,7 +618,7 @@ class CheckBoxWidget(Widget):
attr_dict = {}
if isinstance(value, int):
value = str(value)
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
......@@ -723,7 +723,7 @@ class TextAreaWidget(Widget):
render_prefix, attr_dict, local_name):
if attr_dict is None:
attr_dict = {}
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
......@@ -995,7 +995,7 @@ class SingleItemsWidget(ItemsWidget):
if attr_dict is None:
attr_dict = {}
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
......@@ -1146,7 +1146,10 @@ class MultiItemsWidget(ItemsWidget):
if value is None:
return None
value_list = self.render_items_odf(field, value, REQUEST)
if six.PY2:
value = ', '.join(value_list).decode('utf-8')
else:
value = ', '.join(value_list)
return Widget.render_odg(self, field, value, as_string, ooo_builder,
REQUEST, render_prefix, attr_dict, local_name)
......@@ -1178,7 +1181,7 @@ class MultiItemsWidget(ItemsWidget):
if attr_dict is None:
attr_dict = {}
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
......@@ -1712,7 +1715,10 @@ class DateTimeWidget(Widget):
# get the field value
if not value and field.get_value('default_now'):
value = DateTime()
if six.PY2:
text_node.text = self.format_value(field, value, mode='pdf').decode('utf-8')
else:
text_node.text = self.format_value(field, value, mode='pdf')
text_node.attrib.update(attr_dict)
if as_string:
return etree.tostring(text_node)
......@@ -1754,7 +1760,10 @@ class DateTimeWidget(Widget):
"""
if not value and field.get_value('default_now'):
value = DateTime()
if six.PY2:
value_as_text = self.format_value(field, value, mode='pdf').decode('utf-8')
else:
value_as_text = self.format_value(field, value, mode='pdf')
return Widget.render_odg_view(self, field, value_as_text, as_string,
ooo_builder, REQUEST, render_prefix,
attr_dict, local_name)
......@@ -1928,7 +1937,7 @@ class IntegerWidget(TextWidget) :
if attr_dict is None:
attr_dict = {}
attr_dict['{%s}value-type' % OFFICE_URI] = 'float'
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
......@@ -2128,7 +2137,10 @@ class FloatWidget(TextWidget):
if attr_dict is None:
attr_dict = {}
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
if six.PY2:
text_node.text = self.format_value(field, value).decode('utf-8')
else:
text_node.text = self.format_value(field, value)
text_node.attrib.update(attr_dict)
if as_string:
return etree.tostring(text_node)
......@@ -2146,7 +2158,7 @@ class FloatWidget(TextWidget):
if attr_dict is None:
attr_dict = {}
attr_dict['{%s}value-type' % OFFICE_URI] = 'float'
if isinstance(value, str):
if six.PY2 and isinstance(value, str):
#required by lxml
value = value.decode('utf-8')
text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
......
......@@ -3,6 +3,7 @@ from __future__ import absolute_import
from . import XMLObjects
from Products.Formulator.TALESField import TALESMethod
from Products.Formulator.MethodField import Method
import six
def XMLToForm(s, form, override_encoding=None):
"""Takes an xml string and changes formulator form accordingly.
......@@ -127,7 +128,7 @@ def XMLToForm(s, form, override_encoding=None):
form.remove_group('Default')
def encode(text, encoding):
if encoding is None:
if six.PY3 or encoding is None:
return text
else:
return text.encode(encoding)
......@@ -56,11 +56,11 @@ class StringValidatorTestCase(ValidatorTestCase):
self.assertEqual('<html>', result)
def test_encoding(self):
utf8_string = 'M\303\274ller' # this is a M&uuml;ller
unicode_string = unicode(utf8_string, 'utf-8')
utf8_bytes = b'M\303\274ller' # this is a M&uuml;ller
unicode_string = utf8_bytes.decode('utf-8')
result = self.v.validate(
TestField('f', max_length=0, truncate=0, required=0, unicode=1),
'f', {'f' : utf8_string})
'f', {'f' : utf8_bytes})
self.assertEqual(unicode_string, result)
def test_strip_whitespace(self):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment