Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Nikola Balog
erp5
Commits
ae7db4ae
Commit
ae7db4ae
authored
Oct 30, 2019
by
Bryton Lacquement
🚪
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
BigFile: fix
parent
2d94df67
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
69 additions
and
40 deletions
+69
-40
bt5/erp5_big_file/TestTemplateItem/portal_components/test.erp5.testBigFile.py
...stTemplateItem/portal_components/test.erp5.testBigFile.py
+17
-10
product/ERP5/Document/BigFile.py
product/ERP5/Document/BigFile.py
+28
-30
product/ERP5Type/Utils.py
product/ERP5Type/Utils.py
+24
-0
No files found.
bt5/erp5_big_file/TestTemplateItem/portal_components/test.erp5.testBigFile.py
View file @
ae7db4ae
...
@@ -28,6 +28,7 @@ from cStringIO import StringIO
...
@@ -28,6 +28,7 @@ from cStringIO import StringIO
from
ZPublisher.HTTPRequest
import
HTTPRequest
from
ZPublisher.HTTPRequest
import
HTTPRequest
from
ZPublisher.HTTPResponse
import
HTTPResponse
from
ZPublisher.HTTPResponse
import
HTTPResponse
from
ZPublisher.Iterators
import
IUnboundStreamIterator
from
Products.ERP5Type.tests.ERP5TypeTestCase
import
ERP5TypeTestCase
from
Products.ERP5Type.tests.ERP5TypeTestCase
import
ERP5TypeTestCase
from
Products.ERP5Type.BTreeData
import
BTreeData
from
Products.ERP5Type.BTreeData
import
BTreeData
...
@@ -102,6 +103,13 @@ class TestBigFile(ERP5TypeTestCase):
...
@@ -102,6 +103,13 @@ class TestBigFile(ERP5TypeTestCase):
# check that object (document) method corresponding to request returns
# check that object (document) method corresponding to request returns
# result, with expected response body, status and headers
# result, with expected response body, status and headers
def
checkRequest
(
self
,
document
,
request
,
kw
,
result
,
body
,
status
,
header_dict
):
def
checkRequest
(
self
,
document
,
request
,
kw
,
result
,
body
,
status
,
header_dict
):
assert
type
(
result
)
is
str
assert
type
(
body
)
is
str
# - result corresponds to the content returned as a string;
# - body corresponds to the content returned inside a stream iterator.
# We can't have both at the same time.
assert
not
(
bool
(
result
)
==
bool
(
body
)
==
True
)
# request -> method to call
# request -> method to call
method_name
=
request
[
'REQUEST_METHOD'
]
method_name
=
request
[
'REQUEST_METHOD'
]
if
method_name
==
'GET'
:
if
method_name
==
'GET'
:
...
@@ -112,18 +120,18 @@ class TestBigFile(ERP5TypeTestCase):
...
@@ -112,18 +120,18 @@ class TestBigFile(ERP5TypeTestCase):
# like in ZPublisher - returned RESPONSE means empty
# like in ZPublisher - returned RESPONSE means empty
if
ret
is
request
.
RESPONSE
:
if
ret
is
request
.
RESPONSE
:
ret
=
''
ret
=
''
self
.
assertEqual
(
ret
,
result
)
elif
IUnboundStreamIterator
.
providedBy
(
ret
):
self
.
assertEqual
(
status
,
request
.
RESPONSE
.
getStatus
())
ret
=
''
.
join
(
ret
)
self
.
assertEqual
(
status
,
request
.
RESPONSE
.
getStatus
())
for
h
,
v
in
header_dict
.
items
():
for
h
,
v
in
header_dict
.
items
():
rv
=
request
.
RESPONSE
.
getHeader
(
h
)
rv
=
request
.
RESPONSE
.
getHeader
(
h
)
self
.
assertEqual
(
v
,
rv
,
'%s: %r != %r'
%
(
h
,
v
,
rv
))
self
.
assertEqual
(
v
,
rv
,
'%s: %r != %r'
%
(
h
,
v
,
rv
))
if
result
:
# force response flush to its stdout
self
.
assertEqual
(
ret
,
result
)
request
.
RESPONSE
.
write
(
''
)
elif
body
:
# body and headers are delimited by empty line (RFC 2616, 4.1)
self
.
assertEqual
(
ret
,
body
)
response_body
=
request
.
RESPONSE
.
stdout
.
getvalue
().
split
(
'
\
r
\
n
\
r
\
n
'
,
1
)[
1
]
else
:
self
.
assertEqual
(
body
,
response_body
)
self
.
assertEqual
(
ret
,
''
)
# basic tests for working with BigFile via its public interface
# basic tests for working with BigFile via its public interface
def
testBigFile_01_Basic
(
self
):
def
testBigFile_01_Basic
(
self
):
...
@@ -356,4 +364,3 @@ class TestBigFile(ERP5TypeTestCase):
...
@@ -356,4 +364,3 @@ class TestBigFile(ERP5TypeTestCase):
# TODO write big data to file and ensure it still works
# TODO write big data to file and ensure it still works
# TODO test streaming works in chunks
product/ERP5/Document/BigFile.py
View file @
ae7db4ae
...
@@ -18,6 +18,7 @@ from cStringIO import StringIO
...
@@ -18,6 +18,7 @@ from cStringIO import StringIO
from
AccessControl
import
ClassSecurityInfo
from
AccessControl
import
ClassSecurityInfo
from
Products.ERP5Type
import
Permissions
,
PropertySheet
from
Products.ERP5Type
import
Permissions
,
PropertySheet
from
Products.ERP5Type.Base
import
removeIContentishInterface
from
Products.ERP5Type.Base
import
removeIContentishInterface
from
Products.ERP5Type.Utils
import
IterableAsStreamIterator
from
Products.ERP5.Document.File
import
File
,
_MARKER
from
Products.ERP5.Document.File
import
File
,
_MARKER
from
Products.ERP5Type.BTreeData
import
BTreeData
from
Products.ERP5Type.BTreeData
import
BTreeData
from
ZPublisher.HTTPRequest
import
FileUpload
from
ZPublisher.HTTPRequest
import
FileUpload
...
@@ -214,7 +215,7 @@ class BigFile(File):
...
@@ -214,7 +215,7 @@ class BigFile(File):
RESPONSE
.
setHeader
(
'Content-Type'
,
self
.
content_type
)
RESPONSE
.
setHeader
(
'Content-Type'
,
self
.
content_type
)
RESPONSE
.
setHeader
(
'Content-Length'
,
self
.
getSize
())
RESPONSE
.
setHeader
(
'Content-Length'
,
self
.
getSize
())
RESPONSE
.
setStatus
(
416
)
RESPONSE
.
setStatus
(
416
)
return
True
return
''
ranges
=
HTTPRangeSupport
.
expandRanges
(
ranges
,
self
.
getSize
())
ranges
=
HTTPRangeSupport
.
expandRanges
(
ranges
,
self
.
getSize
())
...
@@ -233,11 +234,8 @@ class BigFile(File):
...
@@ -233,11 +234,8 @@ class BigFile(File):
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if
isinstance
(
data
,
str
):
if
isinstance
(
data
,
str
):
RESPONSE
.
write
(
data
[
start
:
end
])
return
data
[
start
:
end
]
return
True
return
IterableAsStreamIterator
(
data
.
iterate
(
start
,
size
),
size
)
for
chunk
in
data
.
iterate
(
start
,
end
-
start
):
RESPONSE
.
write
(
chunk
)
return
True
else
:
else
:
boundary
=
choose_boundary
()
boundary
=
choose_boundary
()
...
@@ -266,33 +264,35 @@ class BigFile(File):
...
@@ -266,33 +264,35 @@ class BigFile(File):
draftprefix
,
boundary
))
draftprefix
,
boundary
))
RESPONSE
.
setStatus
(
206
)
# Partial content
RESPONSE
.
setStatus
(
206
)
# Partial content
for
start
,
end
in
ranges
:
self_content_type
=
self
.
content_type
RESPONSE
.
write
(
'
\
r
\
n
--%s
\
r
\
n
'
%
boundary
)
self_getSize
=
self
.
getSize
()
RESPONSE
.
write
(
'Content-Type: %s
\
r
\
n
'
%
def
generator
():
self
.
content_type
)
for
start
,
end
in
ranges
:
RESPONSE
.
write
(
yield
'
\
r
\
n
--%s
\
r
\
n
'
%
boundary
'Content-Range: bytes %d-%d/%d
\
r
\
n
\
r
\
n
'
%
(
yield
'Content-Type: %s
\
r
\
n
'
%
self
.
content_type
start
,
end
-
1
,
self
.
getSize
()))
yield
'Content-Range: bytes %d-%d/%d
\
r
\
n
\
r
\
n
'
%
(
start
,
end
-
1
,
self_getSize
)
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
if
isinstance
(
data
,
str
):
# NOTE data cannot be None here (if it is - ranges are not satisfiable)
RESPONSE
.
write
(
data
[
start
:
end
])
if
isinstance
(
data
,
str
):
yield
data
[
start
:
end
]
else
:
else
:
for
chunk
in
data
.
iterate
(
start
,
end
-
start
):
for
chunk
in
data
.
iterate
(
start
,
end
-
start
):
RESPONSE
.
write
(
chunk
)
# BBB: Python 3.3+ yield from
yield
chunk
RESPONSE
.
write
(
'
\
r
\
n
--%s--
\
r
\
n
'
%
boundary
)
return
True
yield
'
\
r
\
n
--%s--
\
r
\
n
'
%
boundary
return
IterableAsStreamIterator
(
generator
(),
size
)
security
.
declareProtected
(
Permissions
.
View
,
'index_html'
)
security
.
declareProtected
(
Permissions
.
View
,
'index_html'
)
def
index_html
(
self
,
REQUEST
,
RESPONSE
,
format
=
_MARKER
,
inline
=
_MARKER
,
**
kw
):
def
index_html
(
self
,
REQUEST
,
RESPONSE
,
format
=
_MARKER
,
inline
=
_MARKER
,
**
kw
):
"""
"""
Support streaming
Support streaming
"""
"""
if
self
.
_range_request_handler
(
REQUEST
,
RESPONSE
):
response_iterable
=
self
.
_range_request_handler
(
REQUEST
,
RESPONSE
)
# we served a chunk of content in response to a range request.
if
response_iterable
is
not
None
:
return
''
# we serve a chunk of content in response to a range request.
return
response_iterable
web_cache_kw
=
kw
.
copy
()
web_cache_kw
=
kw
.
copy
()
if
format
is
not
_MARKER
:
if
format
is
not
_MARKER
:
...
@@ -327,9 +327,7 @@ class BigFile(File):
...
@@ -327,9 +327,7 @@ class BigFile(File):
if
isinstance
(
data
,
str
):
if
isinstance
(
data
,
str
):
RESPONSE
.
setBase
(
None
)
RESPONSE
.
setBase
(
None
)
return
data
return
data
for
chunk
in
data
.
iterate
():
return
IterableAsStreamIterator
(
data
.
iterate
(),
len
(
data
))
RESPONSE
.
write
(
chunk
)
return
''
security
.
declareProtected
(
Permissions
.
ModifyPortalContent
,
'PUT'
)
security
.
declareProtected
(
Permissions
.
ModifyPortalContent
,
'PUT'
)
def
PUT
(
self
,
REQUEST
,
RESPONSE
):
def
PUT
(
self
,
REQUEST
,
RESPONSE
):
...
...
product/ERP5Type/Utils.py
View file @
ae7db4ae
...
@@ -58,6 +58,9 @@ from Products.PageTemplates.Expressions import getEngine
...
@@ -58,6 +58,9 @@ from Products.PageTemplates.Expressions import getEngine
from
Products.PageTemplates.Expressions
import
SecureModuleImporter
from
Products.PageTemplates.Expressions
import
SecureModuleImporter
from
Products.ZCatalog.Lazy
import
LazyMap
from
Products.ZCatalog.Lazy
import
LazyMap
from
zope.interface
import
implementer
from
ZPublisher.Iterators
import
IStreamIterator
try
:
try
:
import
chardet
import
chardet
except
ImportError
:
except
ImportError
:
...
@@ -1795,3 +1798,24 @@ def formatRFC822Headers(headers):
...
@@ -1795,3 +1798,24 @@ def formatRFC822Headers(headers):
vallines
=
linesplit
.
split
(
str
(
value
))
vallines
=
linesplit
.
split
(
str
(
value
))
munged
.
append
(
'%s: %s'
%
(
key
,
'
\
r
\
n
'
.
join
(
vallines
)))
munged
.
append
(
'%s: %s'
%
(
key
,
'
\
r
\
n
'
.
join
(
vallines
)))
return
'
\
r
\
n
'
.
join
(
munged
)
return
'
\
r
\
n
'
.
join
(
munged
)
#####################################################
# WSGI/Medusa compatibility
#####################################################
@
implementer
(
IStreamIterator
)
class
IterableAsStreamIterator
:
def
__init__
(
self
,
iterable
,
content_length
):
self
.
iterable
=
iterable
self
.
content_length
=
content_length
def
__iter__
(
self
):
return
self
def
__len__
(
self
):
return
self
.
content_length
def
next
(
self
):
for
chunk
in
self
.
iterable
:
return
chunk
raise
StopIteration
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment