Commit c7cb731e authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Arnaud Fontaine

py2/py3: data property is for binary, thus getData() should return bytes.

parent 8c325d1d
Pipeline #35833 failed with stage
in 0 seconds
......@@ -32,8 +32,7 @@
import os
import subprocess
from cStringIO import StringIO
from io import BytesIO
from AccessControl import ClassSecurityInfo
from Acquisition import aq_base
......@@ -117,7 +116,7 @@ class Image(TextConvertableMixin, File, OFSImage):
content_type, width, height = getImageInfo(self.data)
if not content_type:
try:
image = PIL.Image.open(StringIO(str(self.data)))
image = PIL.Image.open(BytesIO(bytes(self.data)))
except IOError:
width = height = -1
content_type = 'application/unknown'
......@@ -381,7 +380,7 @@ class Image(TextConvertableMixin, File, OFSImage):
else:
parameter_list.append('-')
data = str(self.getData())
data = bytes(self.getData())
if self.getContentType() == "image/svg+xml":
data = transformUrlToDataURI(data)
......@@ -401,7 +400,7 @@ class Image(TextConvertableMixin, File, OFSImage):
finally:
del process
if image:
return StringIO(image)
return BytesIO(image)
raise ConversionError('Image conversion failed (%s).' % err)
def _getContentTypeAndImageData(
......
......@@ -27,7 +27,8 @@
#
##############################################################################
import re, zipfile, cStringIO
import re, zipfile
from io import BytesIO
from warnings import warn
from AccessControl import ClassSecurityInfo
from OFS.Image import Pdata
......@@ -38,7 +39,7 @@ from Products.ERP5Type.Cache import CachingMethod
from erp5.component.document.File import File
from erp5.component.document.Document import Document, \
VALID_IMAGE_FORMAT_LIST, ConversionError, NotConvertedError
from Products.ERP5Type.Utils import fill_args_from_request
from Products.ERP5Type.Utils import bytes2str, fill_args_from_request, str2bytes
# Mixin Import
from erp5.component.mixin.BaseConvertableFileMixin import BaseConvertableFileMixin
......@@ -198,10 +199,10 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
raise NotConvertedError()
if format == 'text-content':
# Extract text from the ODF file
cs = cStringIO.StringIO()
cs.write(str(self.getBaseData()))
cs = BytesIO()
cs.write(self.getBaseData())
z = zipfile.ZipFile(cs)
s = z.read('content.xml')
s = bytes2str(z.read('content.xml'))
s = self.rx_strip.sub(" ", s) # strip xml
s = self.rx_compr.sub(" ", s) # compress multiple spaces
cs.close()
......@@ -210,7 +211,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
server_proxy = DocumentConversionServerProxy(self)
orig_format = self.getBaseContentType()
generate_result = server_proxy.run_generate(self.getId(),
enc(str(self.getBaseData())),
bytes2str(enc(bytes(self.getBaseData()))),
None,
format,
orig_format)
......@@ -222,7 +223,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
response_dict = generate_result
# XXX: handle possible OOOd server failure
return response_dict['mime'], Pdata(dec(response_dict['data']))
return response_dict['mime'], Pdata(dec(str2bytes(response_dict['data'])))
# Conversion API
def _convert(self, format, frame=0, **kw): # pylint: disable=redefined-builtin
......@@ -258,7 +259,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
original_format = format
allowed_format_list = self.getTargetFormatList()
if format == 'base-data':
return self.getBaseContentType(), str(self.getBaseData())
return self.getBaseContentType(), self.getBaseData()
if format == 'pdf':
format_list = [x for x in allowed_format_list
if x.endswith('pdf')]
......@@ -301,8 +302,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
if is_html:
# Extra processing required since
# we receive a zip file
cs = cStringIO.StringIO()
cs.write(str(data))
cs = BytesIO()
cs.write(bytes(data)) # Cast explicitly to bytes for possible Pdata
z = zipfile.ZipFile(cs) # A disk file would be more RAM efficient
for f in z.infolist():
fn = f.filename
......@@ -324,7 +325,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
# create temporary image and use it to resize accordingly
temp_image = self.portal_contributions.newContent(
portal_type='Image',
file=cStringIO.StringIO(),
file=BytesIO(),
filename=self.getId(),
temp_object=1)
temp_image._setData(data)
......@@ -346,8 +347,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
format_list = [x for x in self.getTargetFormatList()
if x.startswith('html') or x.endswith('html')]
mime, data = self._getConversionFromProxyServer(format_list[0])
archive_file = cStringIO.StringIO()
archive_file.write(str(data))
archive_file = BytesIO()
archive_file.write(data)
zip_file = zipfile.ZipFile(archive_file)
must_close = 1
else:
......@@ -384,13 +385,13 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
server_proxy = DocumentConversionServerProxy(self)
response_code, response_dict, response_message = server_proxy.run_convert(
self.getFilename() or self.getId(),
enc(str(self.getData())),
bytes2str(enc(bytes(self.getData()))),
None,
None,
self.getContentType())
if response_code == 200:
# sucessfully converted document
self._setBaseData(dec(response_dict['data']))
self._setBaseData(dec(str2bytes(response_dict['data'])))
metadata = response_dict['meta']
self._base_metadata = metadata
if metadata.get('MIMEType', None) is not None:
......@@ -423,11 +424,11 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
server_proxy = DocumentConversionServerProxy(self)
response_code, response_dict, response_message = \
server_proxy.run_setmetadata(self.getId(),
enc(str(self.getBaseData())),
bytes2str(enc(bytes(self.getBaseData()))),
kw)
if response_code == 200:
# successful meta data extraction
self._setBaseData(dec(response_dict['data']))
self._setBaseData(dec(str2bytes(response_dict['data'])))
self.updateFileMetadata() # record in workflow history # XXX must put appropriate comments.
else:
# Explicitly raise the exception!
......
......@@ -29,7 +29,11 @@ if context.getPortalType() in ["Presentation"]:
portal = context.getPortalObject()
mimetype = 'text/html'
content_type = context.getContentType()
raw_data = portal.portal_transforms.convertToData(mimetype, str(context.getData() or ""), context=context, mimetype=content_type)
raw_data = portal.portal_transforms.convertToData(
mimetype,
bytes(context.getData() or b""),
context=context,
mimetype=content_type)
if raw_data is None:
raise ValueError("Failed to convert to %r" % mimetype)
if context.REQUEST is not None:
......
......@@ -32,13 +32,16 @@ import zope.interface
from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5Type.Utils import bytes2str
from erp5.component.interface.IWatermarkable import IWatermarkable
from erp5.component.document.Image import Image
from erp5.component.document.Document import ConversionError
from subprocess import Popen, PIPE
from zLOG import LOG, INFO, PROBLEM
import errno
from StringIO import StringIO
from io import BytesIO
from six.moves import range
import six
@zope.interface.implementer(IWatermarkable)
class PDFDocument(Image):
......@@ -90,8 +93,8 @@ class PDFDocument(Image):
raise ValueError("watermark_data cannot not be empty")
if not self.hasData():
raise ValueError("Cannot watermark an empty document")
self_reader = PdfFileReader(StringIO(self.getData()))
watermark_reader = PdfFileReader(StringIO(watermark_data))
self_reader = PdfFileReader(BytesIO(self.getData()))
watermark_reader = PdfFileReader(BytesIO(watermark_data))
watermark_page_count = watermark_reader.getNumPages()
output = PdfFileWriter()
......@@ -109,7 +112,7 @@ class PDFDocument(Image):
self_page.mergePage(watermark_page)
output.addPage(self_page)
outputStream = StringIO()
outputStream = BytesIO()
output.write(outputStream)
return outputStream.getvalue()
......@@ -171,7 +174,7 @@ class PDFDocument(Image):
"""
if not self.hasData():
return ''
data = str(self.getData())
data = bytes(self.getData())
try:
from PyPDF2 import PdfFileReader
from PyPDF2.utils import PdfReadError
......@@ -179,7 +182,7 @@ class PDFDocument(Image):
pass
else:
try:
if PdfFileReader(StringIO(data)).isEncrypted:
if PdfFileReader(BytesIO(data)).isEncrypted:
return ''
except PdfReadError:
return ''
......@@ -344,7 +347,7 @@ class PDFDocument(Image):
raise
result = {}
for line in command_result.splitlines():
for line in bytes2str(command_result).splitlines():
item_list = line.split(':')
key = item_list[0].strip()
value = ':'.join(item_list[1:]).strip()
......@@ -360,9 +363,9 @@ class PDFDocument(Image):
else:
try:
pdf_file = PdfFileReader(tmp)
for info_key, info_value in (pdf_file.getDocumentInfo() or {}).iteritems():
for info_key, info_value in six.iteritems(pdf_file.getDocumentInfo() or {}):
info_key = info_key.lstrip("/")
if isinstance(info_value, unicode):
if six.PY2 and isinstance(info_value, six.text_type):
info_value = info_value.encode("utf-8")
# Ignore values that cannot be pickled ( such as AAPL:Keywords )
......
......@@ -60,12 +60,12 @@ if not zip_file:
rejectSoftwarePublication(software_publication)
return
from cStringIO import StringIO
from io import BytesIO
import zipfile
from zipfile import BadZipfile
zipbuffer = StringIO()
zipbuffer.write(str(zip_file.getData()))
zipbuffer = BytesIO()
zipbuffer.write(bytes(zip_file.getData()))
try:
zip_reader = zipfile.ZipFile(zipbuffer)
except BadZipfile:
......
......@@ -38,7 +38,6 @@ from Products.ERP5Type.Base import Base, removeIContentishInterface
from OFS.Image import File as OFS_File
from Products.ERP5Type.Utils import deprecated
_MARKER = object()
class File(Document, OFS_File):
......@@ -67,7 +66,7 @@ class File(Document, OFS_File):
security.declareObjectProtected(Permissions.AccessContentsInformation)
# Default global values
data = '' # A hack required to use OFS.Image.index_html without calling OFS.Image.__init__
data = b'' # A hack required to use OFS.Image.index_html without calling OFS.Image.__init__
# Default Properties
property_sheets = ( PropertySheet.Base
......@@ -186,6 +185,8 @@ class File(Document, OFS_File):
if data is None:
return None
else:
if six.PY3 and isinstance(data, str):
return bytes(data, self._get_encoding())
return bytes(data)
# DAV Support
......
......@@ -37,6 +37,7 @@ from AccessControl import ClassSecurityInfo
from Products.ERP5Type.Globals import InitializeClass
from Products.ERP5Type import Permissions
from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
from Products.ERP5Type.Utils import str2bytes
from OFS.Image import Pdata, Image as OFSImage
from DateTime import DateTime
......@@ -130,17 +131,20 @@ class CachedConvertableMixin:
conversion_md5 = None
size = 0
elif isinstance(data, Pdata):
cached_value = aq_base(data)
size = str(cached_value) # not a size but avoids a 'del' statement
conversion_md5 = md5(size).hexdigest()
size = len(size)
cached_value = bytes(aq_base(data))
conversion_md5 = md5(cached_value).hexdigest()
size = len(cached_value)
elif isinstance(data, OFSImage):
warn('Passing an OFS.Image to setConversion is deprecated', stacklevel=1)
cached_value = bytes(data)
conversion_md5 = md5(cached_value).hexdigest()
size = len(cached_value)
elif isinstance(data, bytes):
cached_value = data
conversion_md5 = md5(str(data.data)).hexdigest()
size = len(data.data)
conversion_md5 = md5(cached_value).hexdigest()
size = len(cached_value)
elif isinstance(data, six.string_types):
cached_value = data
cached_value = str2bytes(data)
conversion_md5 = md5(cached_value).hexdigest()
size = len(cached_value)
elif isinstance(data, dict):
......
......@@ -49,7 +49,7 @@
</item>
<item>
<key> <string>property_default</string> </key>
<value> <string>python: \'\'</string> </value>
<value> <string>python: b\'\'</string> </value>
</item>
</dictionary>
</pickle>
......
......@@ -49,7 +49,7 @@
</item>
<item>
<key> <string>property_default</string> </key>
<value> <string>python: \'\'</string> </value>
<value> <string>python: b\'\'</string> </value>
</item>
</dictionary>
</pickle>
......
......@@ -37,6 +37,7 @@ from Products.ERP5OOo.OOoUtils import OOoBuilder
from Products.CMFCore.exceptions import AccessControl_Unauthorized
from Acquisition import Implicit, aq_base
from Products.ERP5Type.Globals import InitializeClass, DTMLFile, Persistent
from Products.ERP5Type.Utils import bytes2str, str2bytes
from AccessControl import ClassSecurityInfo
from OFS.role import RoleManager
from OFS.SimpleItem import Item
......@@ -290,11 +291,11 @@ class FormPrintout(Implicit, Persistent, RoleManager, Item, PropertyManager):
from erp5.component.document.Document import DocumentConversionServerProxy, enc, dec
server_proxy = DocumentConversionServerProxy(self)
extension = guess_extension(content_type).strip('.')
printout = dec(server_proxy.convertFile(enc(printout),
printout = dec(str2bytes(server_proxy.convertFile(bytes2str(enc(printout)),
extension, # source_format
extension, # destination_format
False, # zip
True)) # refresh
True))) # refresh
# End of temporary implementation
if not format:
if REQUEST is not None and not batch_mode:
......@@ -317,7 +318,7 @@ class FormPrintout(Implicit, Persistent, RoleManager, Item, PropertyManager):
REQUEST.RESPONSE.setHeader('Content-type', mime)
REQUEST.RESPONSE.setHeader('Content-disposition',
'attachment;filename="%s.%s"' % (filename, format))
return str(data)
return bytes(data)
InitializeClass(FormPrintout)
......@@ -574,7 +575,7 @@ class ODFStrategy(Implicit):
path = image_field.get_value('default')
image_node = image_list[0]
image_frame = image_node.getparent()
if path is not None:
if six.PY2 and path is not None:
path = path.encode()
picture = self.getPortalObject().restrictedTraverse(path)
picture_data = getattr(aq_base(picture), 'data', None)
......@@ -816,19 +817,20 @@ class ODFStrategy(Implicit):
\n -> line-breaks
DateTime -> Y-m-d
"""
assert six.PY2 # TODO-py3
if value is None:
value = ''
translated_value = str(value)
if isinstance(value, DateTime):
translated_value = value.strftime('%Y-%m-%d')
elif isinstance(value, bytes):
translated_value = value.decode('utf-8')
else:
translated_value = str(value)
translated_value = escape(translated_value)
tab_element_str = '<text:tab xmlns:text="%s"/>' % TEXT_URI
line_break_element_str ='<text:line-break xmlns:text="%s"/>' % TEXT_URI
translated_value = translated_value.replace('\t', tab_element_str)
translated_value = translated_value.replace('\r', '')
translated_value = translated_value.replace('\n', line_break_element_str)
translated_value = unicode(str(translated_value),'utf-8')
# create a paragraph
template = '<text:p xmlns:text="%s">%s</text:p>'
fragment_element_tree = etree.XML(template % (TEXT_URI, translated_value))
......
......@@ -44,10 +44,11 @@ from Acquisition import aq_base
from AccessControl import ClassSecurityInfo
from .OOoUtils import OOoBuilder
from zipfile import ZipFile, ZIP_DEFLATED
from six.moves import cStringIO as StringIO
from io import BytesIO
import re
import itertools
import six
from Products.ERP5Type.Utils import bytes2str
try:
from zExceptions import ResourceLockedError
......@@ -226,7 +227,7 @@ class OOoTemplate(ZopePageTemplate):
self.OLE_documents_zipstring = None
# create a zip archive and store it
if attached_files_list:
memory_file = StringIO()
memory_file = BytesIO()
try:
zf = ZipFile(memory_file, mode='w', compression=ZIP_DEFLATED)
except RuntimeError:
......@@ -436,8 +437,8 @@ class OOoTemplate(ZopePageTemplate):
draw_object.attrib.update({'{%s}href' % xml_doc.nsmap.get('xlink'): new_path})
draw_object.attrib.update(dict(office_include.attrib))
office_include.getparent().replace(office_include, draw_object)
text = etree.tostring(xml_doc, encoding='utf-8', xml_declaration=True,
pretty_print=False)
text = bytes2str(etree.tostring(xml_doc, encoding='utf-8', xml_declaration=True,
pretty_print=False))
text = re.sub('<\s*office:include_img\s+(.*?)\s*/\s*>(?s)', replaceIncludesImg, text)
return (text, attached_files_dict)
......
......@@ -44,6 +44,12 @@ ATTRIBUTE_PREFIX = ''
def identity(value):
return value
def asData(value):
assert not isinstance(value, six.text_type)
return value
def asFloat(value):
"""
Return the value as a float or a type-specific default value if it fails.
......@@ -101,7 +107,7 @@ def asString(value):
if six.PY2 and isinstance(value, unicode):
result = value.encode('utf-8')
elif six.PY3 and isinstance(value, bytes):
result = value.decode('utf-8')
result = value.decode('utf-8', 'surrogateescape')
else:
result = str(value)
except TypeError:
......@@ -195,8 +201,8 @@ type_definition = {
# which intention is store large data
# such as files of BLOBs. It uses pdata
# structure.
'data' : { 'cast' : identity,
'null' : ('', 'None', None,),
'data' : { 'cast' : asData,
'null' : (b'', b'None', None,),
'default' : None,
'isList' : 0,
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment