Commit ae7db4ae authored by Bryton Lacquement's avatar Bryton Lacquement 🚪

BigFile: fix

parent 2d94df67
...@@ -28,6 +28,7 @@ from cStringIO import StringIO ...@@ -28,6 +28,7 @@ from cStringIO import StringIO
from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.HTTPRequest import HTTPRequest
from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.Iterators import IUnboundStreamIterator
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.BTreeData import BTreeData from Products.ERP5Type.BTreeData import BTreeData
...@@ -102,6 +103,13 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -102,6 +103,13 @@ class TestBigFile(ERP5TypeTestCase):
# check that object (document) method corresponding to request returns # check that object (document) method corresponding to request returns
# result, with expected response body, status and headers # result, with expected response body, status and headers
def checkRequest(self, document, request, kw, result, body, status, header_dict): def checkRequest(self, document, request, kw, result, body, status, header_dict):
assert type(result) is str
assert type(body) is str
# - result corresponds to the content returned as a string;
# - body corresponds to the content returned inside a stream iterator.
# We can't have both at the same time.
assert not(bool(result) == bool(body) == True)
# request -> method to call # request -> method to call
method_name = request['REQUEST_METHOD'] method_name = request['REQUEST_METHOD']
if method_name == 'GET': if method_name == 'GET':
...@@ -112,18 +120,18 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -112,18 +120,18 @@ class TestBigFile(ERP5TypeTestCase):
# like in ZPublisher - returned RESPONSE means empty # like in ZPublisher - returned RESPONSE means empty
if ret is request.RESPONSE: if ret is request.RESPONSE:
ret = '' ret = ''
self.assertEqual(ret, result) elif IUnboundStreamIterator.providedBy(ret):
ret = ''.join(ret)
self.assertEqual(status, request.RESPONSE.getStatus()) self.assertEqual(status, request.RESPONSE.getStatus())
for h,v in header_dict.items(): for h,v in header_dict.items():
rv = request.RESPONSE.getHeader(h) rv = request.RESPONSE.getHeader(h)
self.assertEqual(v, rv, '%s: %r != %r' % (h, v, rv)) self.assertEqual(v, rv, '%s: %r != %r' % (h, v, rv))
if result:
# force response flush to its stdout self.assertEqual(ret, result)
request.RESPONSE.write('') elif body:
# body and headers are delimited by empty line (RFC 2616, 4.1) self.assertEqual(ret, body)
response_body = request.RESPONSE.stdout.getvalue().split('\r\n\r\n', 1)[1] else:
self.assertEqual(body, response_body) self.assertEqual(ret, '')
# basic tests for working with BigFile via its public interface # basic tests for working with BigFile via its public interface
def testBigFile_01_Basic(self): def testBigFile_01_Basic(self):
...@@ -356,4 +364,3 @@ class TestBigFile(ERP5TypeTestCase): ...@@ -356,4 +364,3 @@ class TestBigFile(ERP5TypeTestCase):
# TODO write big data to file and ensure it still works # TODO write big data to file and ensure it still works
# TODO test streaming works in chunks
...@@ -18,6 +18,7 @@ from cStringIO import StringIO ...@@ -18,6 +18,7 @@ from cStringIO import StringIO
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet from Products.ERP5Type import Permissions, PropertySheet
from Products.ERP5Type.Base import removeIContentishInterface from Products.ERP5Type.Base import removeIContentishInterface
from Products.ERP5Type.Utils import IterableAsStreamIterator
from Products.ERP5.Document.File import File, _MARKER from Products.ERP5.Document.File import File, _MARKER
from Products.ERP5Type.BTreeData import BTreeData from Products.ERP5Type.BTreeData import BTreeData
from ZPublisher.HTTPRequest import FileUpload from ZPublisher.HTTPRequest import FileUpload
...@@ -214,7 +215,7 @@ class BigFile(File): ...@@ -214,7 +215,7 @@ class BigFile(File):
RESPONSE.setHeader('Content-Type', self.content_type) RESPONSE.setHeader('Content-Type', self.content_type)
RESPONSE.setHeader('Content-Length', self.getSize()) RESPONSE.setHeader('Content-Length', self.getSize())
RESPONSE.setStatus(416) RESPONSE.setStatus(416)
return True return ''
ranges = HTTPRangeSupport.expandRanges(ranges, self.getSize()) ranges = HTTPRangeSupport.expandRanges(ranges, self.getSize())
...@@ -233,11 +234,8 @@ class BigFile(File): ...@@ -233,11 +234,8 @@ class BigFile(File):
# NOTE data cannot be None here (if it is - ranges are not satisfiable) # NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str): if isinstance(data, str):
RESPONSE.write(data[start:end]) return data[start:end]
return True return IterableAsStreamIterator(data.iterate(start, size), size)
for chunk in data.iterate(start, end-start):
RESPONSE.write(chunk)
return True
else: else:
boundary = choose_boundary() boundary = choose_boundary()
...@@ -266,33 +264,35 @@ class BigFile(File): ...@@ -266,33 +264,35 @@ class BigFile(File):
draftprefix, boundary)) draftprefix, boundary))
RESPONSE.setStatus(206) # Partial content RESPONSE.setStatus(206) # Partial content
self_content_type = self.content_type
self_getSize = self.getSize()
def generator():
for start, end in ranges: for start, end in ranges:
RESPONSE.write('\r\n--%s\r\n' % boundary) yield '\r\n--%s\r\n' % boundary
RESPONSE.write('Content-Type: %s\r\n' % yield 'Content-Type: %s\r\n' % self.content_type
self.content_type) yield 'Content-Range: bytes %d-%d/%d\r\n\r\n' % (
RESPONSE.write( start, end - 1, self_getSize)
'Content-Range: bytes %d-%d/%d\r\n\r\n' % (
start, end - 1, self.getSize()))
# NOTE data cannot be None here (if it is - ranges are not satisfiable) # NOTE data cannot be None here (if it is - ranges are not satisfiable)
if isinstance(data, str): if isinstance(data, str):
RESPONSE.write(data[start:end]) yield data[start:end]
else: else:
for chunk in data.iterate(start, end-start): for chunk in data.iterate(start, end - start):
RESPONSE.write(chunk) # BBB: Python 3.3+ yield from
yield chunk
RESPONSE.write('\r\n--%s--\r\n' % boundary) yield '\r\n--%s--\r\n' % boundary
return True return IterableAsStreamIterator(generator(), size)
security.declareProtected(Permissions.View, 'index_html') security.declareProtected(Permissions.View, 'index_html')
def index_html(self, REQUEST, RESPONSE, format=_MARKER, inline=_MARKER, **kw): def index_html(self, REQUEST, RESPONSE, format=_MARKER, inline=_MARKER, **kw):
""" """
Support streaming Support streaming
""" """
if self._range_request_handler(REQUEST, RESPONSE): response_iterable = self._range_request_handler(REQUEST, RESPONSE)
# we served a chunk of content in response to a range request. if response_iterable is not None:
return '' # we serve a chunk of content in response to a range request.
return response_iterable
web_cache_kw = kw.copy() web_cache_kw = kw.copy()
if format is not _MARKER: if format is not _MARKER:
...@@ -327,9 +327,7 @@ class BigFile(File): ...@@ -327,9 +327,7 @@ class BigFile(File):
if isinstance(data, str): if isinstance(data, str):
RESPONSE.setBase(None) RESPONSE.setBase(None)
return data return data
for chunk in data.iterate(): return IterableAsStreamIterator(data.iterate(), len(data))
RESPONSE.write(chunk)
return ''
security.declareProtected(Permissions.ModifyPortalContent,'PUT') security.declareProtected(Permissions.ModifyPortalContent,'PUT')
def PUT(self, REQUEST, RESPONSE): def PUT(self, REQUEST, RESPONSE):
......
...@@ -58,6 +58,9 @@ from Products.PageTemplates.Expressions import getEngine ...@@ -58,6 +58,9 @@ from Products.PageTemplates.Expressions import getEngine
from Products.PageTemplates.Expressions import SecureModuleImporter from Products.PageTemplates.Expressions import SecureModuleImporter
from Products.ZCatalog.Lazy import LazyMap from Products.ZCatalog.Lazy import LazyMap
from zope.interface import implementer
from ZPublisher.Iterators import IStreamIterator
try: try:
import chardet import chardet
except ImportError: except ImportError:
...@@ -1795,3 +1798,24 @@ def formatRFC822Headers(headers): ...@@ -1795,3 +1798,24 @@ def formatRFC822Headers(headers):
vallines = linesplit.split(str(value)) vallines = linesplit.split(str(value))
munged.append('%s: %s' % (key, '\r\n '.join(vallines))) munged.append('%s: %s' % (key, '\r\n '.join(vallines)))
return '\r\n'.join(munged) return '\r\n'.join(munged)
#####################################################
# WSGI/Medusa compatibility
#####################################################
@implementer(IStreamIterator)
class IterableAsStreamIterator:
def __init__(self, iterable, content_length):
self.iterable = iterable
self.content_length = content_length
def __iter__(self):
return self
def __len__(self):
return self.content_length
def next(self):
for chunk in self.iterable:
return chunk
raise StopIteration
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment