Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Carlos Ramos Carreño
erp5
Commits
82c6df06
Commit
82c6df06
authored
Apr 05, 2024
by
Jérome Perrin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fixup! fixup! fixup! fixup! XMLExportImport: more support pickle protocol 3
🚧
( repair python2 )
parent
51cb4518
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
255 additions
and
245 deletions
+255
-245
bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py
...TemplateItem/portal_components/test.erp5.testXMLPickle.py
+140
-81
product/ERP5Type/XMLExportImport/__init__.py
product/ERP5Type/XMLExportImport/__init__.py
+0
-7
product/ERP5Type/XMLExportImport/ppml.py
product/ERP5Type/XMLExportImport/ppml.py
+115
-157
No files found.
bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py
View file @
82c6df06
##############################################################################
##############################################################################
#
#
coding: utf-8
# Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved.
# Copyright (c) 2008 Nexedi SA and Contributors. All Rights Reserved.
# TAHARA Yusei <yusei@nexedi.com>
# TAHARA Yusei <yusei@nexedi.com>
#
#
...
@@ -26,16 +26,17 @@
...
@@ -26,16 +26,17 @@
#
#
##############################################################################
##############################################################################
import
base64
import
unittest
import
unittest
import
zodbpickle
import
zodbpickle
import
zodbpickle.fastpickle
as
pickle
import
zodbpickle.fastpickle
as
pickle
import
re
import
re
import
xml.sax
from
io
import
BytesIO
from
io
import
BytesIO
,
StringIO
from
six
import
StringIO
from
Products.ERP5Type.XMLExportImport
import
importXML
,
ppml
from
Products.ERP5Type.XMLExportImport
import
ppml
import
six
import
six
class
DummyClass
:
class
DummyClass
:
"""
"""
A dummy data class
A dummy data class
...
@@ -50,26 +51,37 @@ class XMLPickleTestCase(unittest.TestCase):
...
@@ -50,26 +51,37 @@ class XMLPickleTestCase(unittest.TestCase):
def
dump_to_xml
(
self
,
obj
):
def
dump_to_xml
(
self
,
obj
):
pickled_string
=
pickle
.
dumps
(
obj
,
protocol
=
self
.
_pickle_protocol
)
pickled_string
=
pickle
.
dumps
(
obj
,
protocol
=
self
.
_pickle_protocol
)
f
=
BytesIO
(
pickled_string
)
f
=
BytesIO
(
pickled_string
)
return
str
(
ppml
.
ToXMLUnpickler
(
f
).
load
())
xml
=
ppml
.
ToXMLUnpickler
(
f
).
load
().
__str__
()
if
six
.
PY2
and
isinstance
(
xml
,
six
.
text_type
):
def
load_from_xml
(
self
,
xml_string
):
xml
=
xml
.
encode
(
'utf-8'
)
output
=
StringIO
()
return
xml
F
=
ppml
.
xmlPickler
()
F
.
file
=
output
def
load_from_xml
(
self
,
xml_string
,
persistent_load
=
None
):
F
.
binary
=
1
# XML pickle actually only supports the case of binary = 1
assertEqual
=
self
.
assertEqual
class
DummyJar
:
content_handler
=
xml
.
sax
.
handler
.
ContentHandler
()
loaded
=
None
content_handler
.
startElement
=
F
.
unknown_starttag
"""follow interface expected by importXML"""
content_handler
.
endElement
=
F
.
unknown_endtag
def
importFile
(
self
,
file
,
clue
):
content_handler
.
characters
=
F
.
handle_data
assertEqual
(
clue
,
'ignored'
)
xml
.
sax
.
parseString
(
xml_string
,
content_handler
)
assertEqual
(
file
.
read
(
4
),
b'ZEXP'
)
unpickler
=
pickle
.
Unpickler
(
file
)
reconstructed_pickled_data
=
F
.
_stack
[
0
][
0
]
if
persistent_load
:
return
pickle
.
loads
(
reconstructed_pickled_data
)
unpickler
.
persistent_load
=
persistent_load
self
.
loaded
=
unpickler
.
load
()
jar
=
DummyJar
()
xml_string
=
'<?xml version="1.0"?>
\
n
<ZopeData>%s</ZopeData>'
%
xml_string
importXML
(
jar
,
StringIO
(
xml_string
),
clue
=
'ignored'
)
return
jar
.
loaded
def
dump_and_load
(
self
,
obj
):
def
dump_and_load
(
self
,
obj
):
return
self
.
load_from_xml
(
self
.
dump_to_xml
(
obj
))
return
self
.
load_from_xml
(
self
.
dump_to_xml
(
obj
))
def
check_and_load
(
self
,
v
):
reconstructed
=
self
.
dump_and_load
(
v
)
self
.
assertEqual
(
reconstructed
,
v
)
self
.
assertIs
(
type
(
reconstructed
),
type
(
v
))
class
TestXMLPickle
(
XMLPickleTestCase
):
class
TestXMLPickle
(
XMLPickleTestCase
):
...
@@ -101,76 +113,95 @@ class TestXMLPickle(XMLPickleTestCase):
...
@@ -101,76 +113,95 @@ class TestXMLPickle(XMLPickleTestCase):
self
.
assertIs
(
self
.
dump_and_load
(
False
),
False
)
self
.
assertIs
(
self
.
dump_and_load
(
False
),
False
)
def
test_int
(
self
):
def
test_int
(
self
):
def
check_int
(
v
):
self
.
check_and_load
(
-
0
)
reconstructed
=
self
.
dump_and_load
(
v
)
self
.
check_and_load
(
1
)
self
.
assertEqual
(
reconstructed
,
v
)
self
.
check_and_load
(
-
1
)
self
.
assertIs
(
type
(
reconstructed
),
int
)
self
.
check_and_load
(
0xff
)
check_int
(
-
0
)
self
.
check_and_load
(
0xff1
)
check_int
(
1
)
self
.
check_and_load
(
0xffff
)
check_int
(
-
1
)
self
.
check_and_load
(
2
**
128
)
check_int
(
0xff
)
# long4
check_int
(
0xff1
)
# https://github.com/python/cpython/blob/4d4a6f1b/Lib/test/pickletester.py#L2049-L2050
check_int
(
0xffff
)
self
.
check_and_load
(
12345678910111213141516178920
<<
(
256
*
8
))
check_int
(
0xffff1
)
if
six
.
PY2
:
def
test_long
(
self
):
self
.
check_and_load
(
long
(
-
0
))
self
.
check_and_load
(
long
(
1
))
self
.
check_and_load
(
long
(
-
1
))
self
.
check_and_load
(
long
(
0xff
))
self
.
check_and_load
(
long
(
0xff1
))
self
.
check_and_load
(
long
(
0xffff
))
self
.
check_and_load
(
long
(
2
**
128
))
self
.
check_and_load
(
12345678910111213141516178920
<<
(
256
*
8
))
def
test_float
(
self
):
def
test_float
(
self
):
def
check_float
(
v
):
self
.
check_and_load
(
-
0.0
)
reconstructed
=
self
.
dump_and_load
(
v
)
self
.
check_and_load
(
1.0
)
self
.
assertEqual
(
reconstructed
,
v
)
self
.
check_and_load
(
-
1.0
)
self
.
assertIs
(
type
(
reconstructed
),
float
)
self
.
check_and_load
(.
33
)
check_float
(
-
0.0
)
check_float
(
1.0
)
check_float
(
-
1.0
)
check_float
(.
1
+
.
2
)
def
test_None
(
self
):
def
test_None
(
self
):
self
.
assertIs
(
self
.
assertIs
(
self
.
dump_and_load
(
None
),
None
)
self
.
dump_and_load
(
None
),
None
)
def
test_bytes
(
self
):
def
test_bytes
(
self
):
self
.
assertEqual
(
self
.
dump_and_load
(
b"bytes"
),
b"bytes"
)
self
.
check_and_load
(
b"bytes"
)
self
.
assertEqual
(
self
.
dump_and_load
(
b"long bytes"
*
100
),
b"long bytes"
*
100
)
self
.
check_and_load
(
b"long bytes"
*
100
)
self
.
assertEqual
(
self
.
check_and_load
(
zodbpickle
.
binary
(
b"bytes"
))
self
.
dump_and_load
(
zodbpickle
.
binary
(
b"bytes"
)),
self
.
check_and_load
(
zodbpickle
.
binary
(
b""
))
zodbpickle
.
binary
(
b"bytes"
))
self
.
assertIs
(
type
(
self
.
dump_and_load
(
zodbpickle
.
binary
(
b"bytes"
))),
zodbpickle
.
binary
)
def
test_unicode
(
self
):
def
test_unicode
(
self
):
# BBB PY2
self
.
assertIs
(
type
(
self
.
dump_and_load
(
u"OK"
)),
six
.
text_type
)
self
.
assertIs
(
type
(
self
.
dump_and_load
(
u"OK"
)),
six
.
text_type
)
self
.
assertEqual
(
self
.
dump_and_load
(
u"short"
),
u"short"
)
self
.
check_and_load
(
u"short"
)
self
.
assertEqual
(
self
.
dump_and_load
(
u"unicode 👍"
),
u"unicode 👍"
)
self
.
check_and_load
(
u"unicode 👍"
)
self
.
assertEqual
(
self
.
dump_and_load
(
u"long"
*
100
),
u"long"
*
100
)
self
.
check_and_load
(
u"long"
*
100
)
self
.
assertEqual
(
self
.
dump_and_load
(
u"long…"
*
100
),
u"long…"
*
100
)
self
.
check_and_load
(
u"long…"
*
100
)
self
.
assertEqual
(
self
.
dump_and_load
(
u">"
),
u">"
)
self
.
check_and_load
(
u">"
)
self
.
assertEqual
(
self
.
dump_and_load
(
u"a
\
n
b"
),
u"a
\
n
b"
)
self
.
check_and_load
(
u"a
\
n
b"
)
self
.
check_and_load
(
u" with spaces "
)
self
.
check_and_load
(
u"
\
t
with
\
t
tabs
\
t
"
)
self
.
check_and_load
(
u""
)
def
test_str
(
self
):
self
.
assertIs
(
type
(
self
.
dump_and_load
(
"OK"
)),
str
)
self
.
check_and_load
(
"short"
)
self
.
check_and_load
(
"unicode 👍"
)
self
.
check_and_load
(
"long"
*
100
)
self
.
check_and_load
(
"long…"
*
100
)
self
.
check_and_load
(
">"
)
self
.
check_and_load
(
"a
\
n
b"
)
self
.
check_and_load
(
" with spaces "
)
self
.
check_and_load
(
"
\
t
with
\
t
tabs
\
t
"
)
self
.
check_and_load
(
""
)
def
test_dict
(
self
):
def
test_dict
(
self
):
self
.
assertEqual
(
self
.
check_and_load
({
'a'
:
1
,
'b'
:
2
})
self
.
dump_and_load
({
'a'
:
1
,
'b'
:
2
}),
{
'a'
:
1
,
'b'
:
2
})
self
.
check_and_load
({
'hé'
:
'ho'
})
self
.
check_and_load
(
dict
.
fromkeys
(
range
(
3000
)))
def
test_tuple
(
self
):
def
test_tuple
(
self
):
self
.
assertEqual
(
self
.
check_and_load
((
1
,
))
self
.
dump_and_load
((
1
,
)),
(
1
,
))
self
.
check_and_load
((
1
,
'two'
))
self
.
assertEqual
(
self
.
check_and_load
((
1
,
'two'
,
3.0
))
self
.
dump_and_load
((
1
,
'two'
)),
(
1
,
'two'
))
self
.
check_and_load
(
tuple
([
1
]
*
1000
))
self
.
assertEqual
(
self
.
check_and_load
(())
self
.
dump_and_load
((
1
,
'two'
,
3.0
)),
(
1
,
'two'
,
3.0
))
self
.
check_and_load
((
'hé'
,))
self
.
assertEqual
(
self
.
check_and_load
((
'hé'
,
'hé'
))
self
.
dump_and_load
(
tuple
([
1
]
*
1000
)),
tuple
([
1
]
*
1000
))
self
.
check_and_load
((
'hé'
,
'hé'
,
'hé'
))
self
.
assertEqual
(
self
.
check_and_load
((
'hé'
,
'hé'
,
'hé'
,
'hé'
))
self
.
dump_and_load
(()),
())
def
test_list
(
self
):
def
test_list
(
self
):
self
.
assertEqual
(
self
.
check_and_load
([
1
])
self
.
dump_and_load
([
1
]),
[
1
])
self
.
check_and_load
([])
self
.
assertEqual
(
self
.
check_and_load
([
1
]
*
1000
)
self
.
dump_and_load
([]),
[])
self
.
check_and_load
([
'hé'
])
self
.
assertEqual
(
self
.
dump_and_load
([
1
]
*
1000
),
[
1
]
*
1000
)
def
test_set
(
self
):
def
test_set
(
self
):
self
.
assertEqual
(
self
.
check_and_load
(
set
(
'abc'
))
self
.
dump_and_load
(
set
(
'abc'
)),
set
(
'abc'
))
self
.
check_and_load
(
set
(
'hé'
))
self
.
check_and_load
(
set
([]))
def
test_reference
(
self
):
def
test_reference
(
self
):
ref
=
[]
ref
=
[]
...
@@ -178,6 +209,14 @@ class TestXMLPickle(XMLPickleTestCase):
...
@@ -178,6 +209,14 @@ class TestXMLPickle(XMLPickleTestCase):
self
.
assertEqual
(
reconstructed
,
[
ref
,
ref
,
ref
])
self
.
assertEqual
(
reconstructed
,
[
ref
,
ref
,
ref
])
self
.
assertIs
(
reconstructed
[
0
],
reconstructed
[
1
])
self
.
assertIs
(
reconstructed
[
0
],
reconstructed
[
1
])
def
test_reference_long
(
self
):
# same as reference (which is using BINPUT/BINGET but with large data
# to use LONG_BINPUT/LONG_BINGET)
ref
=
[
list
()
for
_
in
range
(
256
)]
reconstructed
=
self
.
dump_and_load
([
ref
,
ref
,
ref
])
self
.
assertEqual
(
reconstructed
,
[
ref
,
ref
,
ref
])
self
.
assertIs
(
reconstructed
[
0
],
reconstructed
[
1
])
class
TestXMLPickleStringEncoding
(
XMLPickleTestCase
):
class
TestXMLPickleStringEncoding
(
XMLPickleTestCase
):
def
test_string_base64
(
self
):
def
test_string_base64
(
self
):
...
@@ -214,19 +253,39 @@ class TestXMLPickleStringEncoding(XMLPickleTestCase):
...
@@ -214,19 +253,39 @@ class TestXMLPickleStringEncoding(XMLPickleTestCase):
class
TestXMLPickleStringHeuristics
(
XMLPickleTestCase
):
class
TestXMLPickleStringHeuristics
(
XMLPickleTestCase
):
"""Heuristics to map python2 str to unicode or bytes in business templates.
"""Heuristics to map python2 str to unicode or bytes in business templates.
"""
"""
def
test_oid_base64
(
self
):
# if it looks like an oid, it's bytes
self
.
assertEqual
(
self
.
load_from_xml
(
"""
<pickle><string encoding="base64">AAAAAAAAAAE=</string></pickle>
"""
),
b"
\
x00
\
x00
\
x00
\
x00
\
x00
\
x00
\
x00
\
x01
"
)
def
test_bytes_base64
(
self
):
def
test_bytes_base64
(
self
):
# if it does not decode as utf-8 it's bytes
# if it does not decode as utf-8
,
it's bytes
self
.
assertEqual
(
self
.
assertEqual
(
self
.
load_from_xml
(
"""
self
.
load_from_xml
(
"""
<pickle><string encoding="base64">/wA=</string></pickle>
<pickle><string encoding="base64">/wA=</string></pickle>
"""
),
"""
),
b"
\
xFF
\
x00
"
)
b"
\
xFF
\
x00
"
)
def
test_long_bytes_base64
(
self
):
# if it does not decode as utf-8, it's bytes
long_bytes
=
b"
\
xFF
\
x00
"
*
256
self
.
assertEqual
(
self
.
load_from_xml
(
"""
<pickle><string encoding="base64">%s</string></pickle>
"""
%
base64
.
b64encode
(
long_bytes
).
decode
()),
long_bytes
)
def
test_string_persistent_id_base64
(
self
):
# persistent ids are loaded as bytes
persistent_ids
=
[]
def
persistent_load
(
oid
):
persistent_ids
.
append
(
oid
)
self
.
assertEqual
(
self
.
load_from_xml
(
"""
<pickle>
<persistent>
<string encoding="base64">AAAAAAAAAAE=</string>
</persistent>
</pickle>
"""
,
persistent_load
=
persistent_load
),
None
)
self
.
assertEqual
(
persistent_ids
,
[
b'
\
x00
\
x00
\
x00
\
x00
\
x00
\
x00
\
x00
\
x01
'
])
product/ERP5Type/XMLExportImport/__init__.py
View file @
82c6df06
...
@@ -363,12 +363,6 @@ def exportXML(jar, oid, file=None):
...
@@ -363,12 +363,6 @@ def exportXML(jar, oid, file=None):
p
=
getReorderedPickle
(
oid
)
p
=
getReorderedPickle
(
oid
)
write
(
XMLrecord
(
oid
,
len
(
p
),
p
,
id_mapping
))
write
(
XMLrecord
(
oid
,
len
(
p
),
p
,
id_mapping
))
write
(
'</ZopeData>
\
n
'
)
write
(
'</ZopeData>
\
n
'
)
if
0
:
try
:
print
(
file
.
getvalue
())
except
AttributeError
:
pass
import
pdb
;
pdb
.
set_trace
()
return
file
return
file
class
zopedata
:
class
zopedata
:
...
@@ -421,7 +415,6 @@ def importXML(jar, file, clue=''):
...
@@ -421,7 +415,6 @@ def importXML(jar, file, clue=''):
F
.
end_handlers
[
'record'
]
=
save_record
F
.
end_handlers
[
'record'
]
=
save_record
F
.
end_handlers
[
'ZopeData'
]
=
save_zopedata
F
.
end_handlers
[
'ZopeData'
]
=
save_zopedata
F
.
start_handlers
[
'ZopeData'
]
=
start_zopedata
F
.
start_handlers
[
'ZopeData'
]
=
start_zopedata
F
.
binary
=
1
F
.
file
=
outfile
F
.
file
=
outfile
# <patch>
# <patch>
# Our BTs XML files don't declare encoding but have accented chars in them
# Our BTs XML files don't declare encoding but have accented chars in them
...
...
product/ERP5Type/XMLExportImport/ppml.py
View file @
82c6df06
...
@@ -24,8 +24,10 @@ import struct
...
@@ -24,8 +24,10 @@ import struct
import
six
import
six
if
six
.
PY2
:
if
six
.
PY2
:
from
base64
import
encodestring
as
base64_encodebytes
,
decodestring
as
base64_decodebytes
from
base64
import
encodestring
as
base64_encodebytes
,
decodestring
as
base64_decodebytes
from
zodbpickle.pickle_2
import
decode_long
else
:
else
:
from
base64
import
encodebytes
as
base64_encodebytes
,
decodebytes
as
base64_decodebytes
from
base64
import
encodebytes
as
base64_encodebytes
,
decodebytes
as
base64_decodebytes
from
zodbpickle.pickle_3
import
decode_long
import
re
import
re
from
marshal
import
loads
as
mloads
from
marshal
import
loads
as
mloads
from
.xyap
import
NoBlanks
from
.xyap
import
NoBlanks
...
@@ -201,12 +203,14 @@ class String(Scalar):
...
@@ -201,12 +203,14 @@ class String(Scalar):
self
.
mapping
.
setImmutable
(
self
.
id
,
Immutable
(
value
=
result
))
self
.
mapping
.
setImmutable
(
self
.
id
,
Immutable
(
value
=
result
))
return
'%s%s
\
n
'
%
(
' '
*
indent
,
result
)
return
'%s%s
\
n
'
%
(
' '
*
indent
,
result
)
class
Unicode
(
String
):
class
Unicode
(
String
):
def
tag_name
(
self
):
def
tag_name
(
self
):
if
six
.
PY3
:
if
six
.
PY3
:
return
'string'
return
'string'
return
super
(
Unicode
,
self
).
tag_name
()
return
super
(
Unicode
,
self
).
tag_name
()
class
Bytes
(
String
):
class
Bytes
(
String
):
pass
pass
...
@@ -226,7 +230,7 @@ class Wrapper(object):
...
@@ -226,7 +230,7 @@ class Wrapper(object):
v
=
self
.
_v
v
=
self
.
_v
i
=
' '
*
indent
i
=
' '
*
indent
if
isinstance
(
v
,
Scalar
):
if
isinstance
(
v
,
Scalar
):
return
'%s<%s%s> %s </%s>
\
n
'
%
(
i
,
name
,
id
,
str
(
v
)[:
-
1
],
name
)
return
'%s<%s%s> %s </%s>
\
n
'
%
(
i
,
name
,
id
,
v
.
__str__
(
)[:
-
1
],
name
)
else
:
else
:
v
=
v
.
__str__
(
indent
+
2
)
v
=
v
.
__str__
(
indent
+
2
)
return
'%s<%s%s>
\
n
%s%s</%s>
\
n
'
%
(
i
,
name
,
id
,
v
,
i
,
name
)
return
'%s<%s%s>
\
n
%s%s</%s>
\
n
'
%
(
i
,
name
,
id
,
v
,
i
,
name
)
...
@@ -479,6 +483,18 @@ class MinimalMapping(IdentityMapping):
...
@@ -479,6 +483,18 @@ class MinimalMapping(IdentityMapping):
def
__str__
(
self
,
a
):
def
__str__
(
self
,
a
):
return
"Error here"
return
"Error here"
class
UnsupportedOpCode
(
AssertionError
):
"""Error when encountering an opcode that is not supposed to be used
by this implementation.
"""
def
unsupported_opcode
(
opcode
):
def
handler
(
self
):
raise
UnsupportedOpCode
(
opcode
)
return
handler
class
ToXMLUnpickler
(
Unpickler
):
class
ToXMLUnpickler
(
Unpickler
):
def
load
(
self
,
id_mapping
=
None
):
def
load
(
self
,
id_mapping
=
None
):
if
id_mapping
is
None
:
if
id_mapping
is
None
:
...
@@ -493,13 +509,6 @@ class ToXMLUnpickler(Unpickler):
...
@@ -493,13 +509,6 @@ class ToXMLUnpickler(Unpickler):
def
persistent_load
(
self
,
v
):
def
persistent_load
(
self
,
v
):
return
Persistent
(
v
,
self
.
id_mapping
)
return
Persistent
(
v
,
self
.
id_mapping
)
def
load_persid
(
self
):
pid
=
self
.
readline
()[:
-
1
]
self
.
append
(
self
.
persistent_load
(
String
(
pid
,
self
.
id_mapping
)))
if
six
.
PY2
:
dispatch
[
PERSID
]
=
load_persid
dispatch
[
PERSID
[
0
]]
=
load_persid
def
load_binpersid
(
self
):
def
load_binpersid
(
self
):
pid
=
self
.
stack
.
pop
()
pid
=
self
.
stack
.
pop
()
self
.
append
(
self
.
persistent_load
(
pid
))
self
.
append
(
self
.
persistent_load
(
pid
))
...
@@ -513,12 +522,6 @@ class ToXMLUnpickler(Unpickler):
...
@@ -513,12 +522,6 @@ class ToXMLUnpickler(Unpickler):
dispatch
[
NONE
]
=
load_none
dispatch
[
NONE
]
=
load_none
dispatch
[
NONE
[
0
]]
=
load_none
dispatch
[
NONE
[
0
]]
=
load_none
def
load_int
(
self
):
self
.
append
(
Int
(
int
(
self
.
readline
()[:
-
1
]),
self
.
id_mapping
))
if
six
.
PY2
:
dispatch
[
INT
]
=
load_int
dispatch
[
INT
[
0
]]
=
load_int
def
load_binint
(
self
):
def
load_binint
(
self
):
self
.
append
(
Int
(
mloads
(
b'i'
+
self
.
read
(
4
)),
self
.
id_mapping
))
self
.
append
(
Int
(
mloads
(
b'i'
+
self
.
read
(
4
)),
self
.
id_mapping
))
if
six
.
PY2
:
if
six
.
PY2
:
...
@@ -537,17 +540,24 @@ class ToXMLUnpickler(Unpickler):
...
@@ -537,17 +540,24 @@ class ToXMLUnpickler(Unpickler):
dispatch
[
BININT2
]
=
load_binint2
dispatch
[
BININT2
]
=
load_binint2
dispatch
[
BININT2
[
0
]]
=
load_binint2
dispatch
[
BININT2
[
0
]]
=
load_binint2
def
load_long
(
self
):
def
load_long1
(
self
):
self
.
append
(
Long
(
long_
(
self
.
readline
()[:
-
1
],
0
),
self
.
id_mapping
))
n
=
ord
(
self
.
read
(
1
))
data
=
self
.
read
(
n
)
self
.
append
(
Long
(
decode_long
(
data
),
self
.
id_mapping
))
if
six
.
PY2
:
if
six
.
PY2
:
dispatch
[
LONG
]
=
load_long
dispatch
[
LONG1
]
=
load_long1
dispatch
[
LONG
[
0
]]
=
load_long
dispatch
[
LONG1
[
0
]]
=
load_long1
def
load_float
(
self
):
def
load_long4
(
self
):
self
.
append
(
Float
(
float
(
self
.
readline
()[:
-
1
]),
self
.
id_mapping
))
n
=
mloads
(
b'i'
+
self
.
read
(
4
))
if
n
<
0
:
# Corrupt or hostile pickle -- we never write one like this
raise
UnpicklingError
(
"LONG pickle has negative byte count"
);
data
=
self
.
read
(
n
)
self
.
append
(
Long
(
decode_long
(
data
),
self
.
id_mapping
))
if
six
.
PY2
:
if
six
.
PY2
:
dispatch
[
FLOAT
]
=
load_float
dispatch
[
LONG4
]
=
load_long4
dispatch
[
FLOAT
[
0
]]
=
load_float
dispatch
[
LONG4
[
0
]]
=
load_long4
def
load_true
(
self
):
def
load_true
(
self
):
self
.
append
(
Bool
(
True
,
self
.
id_mapping
))
self
.
append
(
Bool
(
True
,
self
.
id_mapping
))
...
@@ -567,13 +577,6 @@ class ToXMLUnpickler(Unpickler):
...
@@ -567,13 +577,6 @@ class ToXMLUnpickler(Unpickler):
dispatch
[
BINFLOAT
]
=
load_binfloat
dispatch
[
BINFLOAT
]
=
load_binfloat
dispatch
[
BINFLOAT
[
0
]]
=
load_binfloat
dispatch
[
BINFLOAT
[
0
]]
=
load_binfloat
def
load_string
(
self
):
self
.
append
(
String
(
ast
.
literal_eval
(
self
.
readline
()[:
-
1
],
{
'__builtins__'
:
{}}),
self
.
id_mapping
))
# Let's be careful
if
six
.
PY2
:
dispatch
[
STRING
]
=
load_string
dispatch
[
STRING
[
0
]]
=
load_string
def
load_binstring
(
self
):
def
load_binstring
(
self
):
len
=
mloads
(
b'i'
+
self
.
read
(
4
))
len
=
mloads
(
b'i'
+
self
.
read
(
4
))
self
.
append
(
String
(
self
.
read
(
len
),
self
.
id_mapping
))
self
.
append
(
String
(
self
.
read
(
len
),
self
.
id_mapping
))
...
@@ -581,14 +584,6 @@ class ToXMLUnpickler(Unpickler):
...
@@ -581,14 +584,6 @@ class ToXMLUnpickler(Unpickler):
dispatch
[
BINSTRING
]
=
load_binstring
dispatch
[
BINSTRING
]
=
load_binstring
dispatch
[
BINSTRING
[
0
]]
=
load_binstring
dispatch
[
BINSTRING
[
0
]]
=
load_binstring
def
load_unicode
(
self
):
line
=
self
.
readline
()
self
.
append
(
Unicode
(
six
.
text_type
(
ast
.
literal_eval
(
line
[:
-
1
],
{
'__builtins__'
:
{}})),
self
.
id_mapping
))
# Let's be careful
if
six
.
PY2
:
dispatch
[
UNICODE
]
=
load_unicode
dispatch
[
UNICODE
[
0
]]
=
load_unicode
def
load_binunicode
(
self
):
def
load_binunicode
(
self
):
len
=
mloads
(
b'i'
+
self
.
read
(
4
))
len
=
mloads
(
b'i'
+
self
.
read
(
4
))
self
.
append
(
Unicode
(
six
.
text_type
(
self
.
read
(
len
),
'utf-8'
),
self
.
id_mapping
))
self
.
append
(
Unicode
(
six
.
text_type
(
self
.
read
(
len
),
'utf-8'
),
self
.
id_mapping
))
...
@@ -619,8 +614,6 @@ class ToXMLUnpickler(Unpickler):
...
@@ -619,8 +614,6 @@ class ToXMLUnpickler(Unpickler):
def
load_tuple
(
self
):
def
load_tuple
(
self
):
k
=
self
.
marker
()
k
=
self
.
marker
()
#LOG('load_tuple, k',0,k)
#LOG('load_tuple, stack[k+1:]',0,self.stack[k+1:])
self
.
stack
[
k
:]
=
[
Tuple
(
self
.
id_mapping
,
v
=
self
.
stack
[
k
+
1
:])]
self
.
stack
[
k
:]
=
[
Tuple
(
self
.
id_mapping
,
v
=
self
.
stack
[
k
+
1
:])]
if
six
.
PY2
:
if
six
.
PY2
:
dispatch
[
TUPLE
]
=
load_tuple
dispatch
[
TUPLE
]
=
load_tuple
...
@@ -742,12 +735,6 @@ class ToXMLUnpickler(Unpickler):
...
@@ -742,12 +735,6 @@ class ToXMLUnpickler(Unpickler):
idprefix
=
''
idprefix
=
''
def
load_get
(
self
):
self
.
append
(
Get
(
self
.
idprefix
+
self
.
readline
()[:
-
1
],
self
.
id_mapping
))
if
six
.
PY2
:
dispatch
[
GET
]
=
load_get
dispatch
[
GET
[
0
]]
=
load_get
def
load_binget
(
self
):
def
load_binget
(
self
):
i
=
mloads
(
b'i'
+
self
.
read
(
1
)
+
b'
\
000
\
000
\
000
'
)
i
=
mloads
(
b'i'
+
self
.
read
(
1
)
+
b'
\
000
\
000
\
000
'
)
self
.
append
(
Get
(
self
.
idprefix
+
repr
(
i
),
self
.
id_mapping
))
self
.
append
(
Get
(
self
.
idprefix
+
repr
(
i
),
self
.
id_mapping
))
...
@@ -782,6 +769,11 @@ class ToXMLUnpickler(Unpickler):
...
@@ -782,6 +769,11 @@ class ToXMLUnpickler(Unpickler):
dispatch
[
LONG_BINPUT
]
=
load_long_binput
dispatch
[
LONG_BINPUT
]
=
load_long_binput
dispatch
[
LONG_BINPUT
[
0
]]
=
load_long_binput
dispatch
[
LONG_BINPUT
[
0
]]
=
load_long_binput
for
code
in
PERSID
,
INT
,
LONG
,
FLOAT
,
STRING
,
UNICODE
,
GET
,
PUT
:
if
six
.
PY2
:
dispatch
[
code
]
=
unsupported_opcode
(
code
)
dispatch
[
code
[
0
]]
=
unsupported_opcode
(
code
)
class
LogCall
:
class
LogCall
:
def
__init__
(
self
,
func
):
def
__init__
(
self
,
func
):
self
.
func
=
func
self
.
func
=
func
...
@@ -809,7 +801,6 @@ def start_pickle(self, tag, attrs):
...
@@ -809,7 +801,6 @@ def start_pickle(self, tag, attrs):
return
[
tag
,
attrs
]
return
[
tag
,
attrs
]
def
save_int
(
self
,
tag
,
data
):
def
save_int
(
self
,
tag
,
data
):
if
self
.
binary
:
v
=
int
(
name
(
self
,
tag
,
data
))
v
=
int
(
name
(
self
,
tag
,
data
))
if
v
>=
0
:
if
v
>=
0
:
if
v
<=
0xff
:
if
v
<=
0xff
:
...
@@ -822,10 +813,7 @@ def save_int(self, tag, data):
...
@@ -822,10 +813,7 @@ def save_int(self, tag, data):
return
INT
+
name
(
self
,
tag
,
data
)
+
b'
\
n
'
return
INT
+
name
(
self
,
tag
,
data
)
+
b'
\
n
'
def
save_float
(
self
,
tag
,
data
):
def
save_float
(
self
,
tag
,
data
):
if
self
.
binary
:
return
BINFLOAT
+
struct
.
pack
(
'>d'
,
float
(
name
(
self
,
tag
,
data
)))
return
BINFLOAT
+
struct
.
pack
(
'>d'
,
float
(
name
(
self
,
tag
,
data
)))
else
:
return
FLOAT
+
name
(
self
,
tag
,
data
)
+
b'
\
n
'
def
save_put
(
self
,
v
,
attrs
):
def
save_put
(
self
,
v
,
attrs
):
id
=
attrs
.
get
(
'id'
,
''
)
id
=
attrs
.
get
(
'id'
,
''
)
...
@@ -835,14 +823,11 @@ def save_put(self, v, attrs):
...
@@ -835,14 +823,11 @@ def save_put(self, v, attrs):
id
=
id
[
prefix
+
1
:]
id
=
id
[
prefix
+
1
:]
elif
id
[
0
]
==
'i'
:
elif
id
[
0
]
==
'i'
:
id
=
id
[
1
:]
id
=
id
[
1
:]
if
self
.
binary
:
id
=
int
(
id
)
id
=
int
(
id
)
if
id
<
256
:
if
id
<
256
:
id
=
BINPUT
+
six
.
int2byte
(
id
)
id
=
BINPUT
+
six
.
int2byte
(
id
)
else
:
else
:
id
=
LONG_BINPUT
+
struct
.
pack
(
'<i'
,
id
)
id
=
LONG_BINPUT
+
struct
.
pack
(
'<i'
,
id
)
else
:
id
=
PUT
+
repr
(
id
).
encode
()
+
b'
\
n
'
return
v
+
id
return
v
+
id
return
v
return
v
...
@@ -852,20 +837,16 @@ def save_string(self, tag, data):
...
@@ -852,20 +837,16 @@ def save_string(self, tag, data):
encoding
=
a
.
get
(
'encoding'
,
'repr'
)
# JPS: repr is default encoding
encoding
=
a
.
get
(
'encoding'
,
'repr'
)
# JPS: repr is default encoding
if
encoding
!=
''
:
if
encoding
!=
''
:
v
=
unconvert
(
encoding
,
v
)
v
=
unconvert
(
encoding
,
v
)
if
self
.
binary
:
l
=
len
(
v
)
l
=
len
(
v
)
if
l
<
256
:
if
l
<
256
:
if
encoding
==
'base64'
:
if
encoding
==
'base64'
:
# TODO: zope4py3 (all this is unfinished)
# We can be here for two reasons:
# We can be here for two reasons:
# - the input was a string with \n or similar control characters
# - the input was a string with \n or similar control characters
# that are not allowed in XML, so the str was exported as base64.
# that are not allowed in XML, so the str was exported as base64.
# - the input was a _p_o
id exported from python2, in that case
# - the input was a persistent
id exported from python2, in that case
# we want to get a zodbpickle.binary back
# we want to get a zodbpickle.binary back
# XXX all this seems a bad idea, we need more context if we want
if
len
(
v
)
==
8
and
self
.
_stack
[
-
1
][
0
]
in
(
'persistent'
,
):
# to have such heuristics
# looks like a persistent id, assume it is a persistent_id -> bytes
if
len
(
v
)
==
8
:
# looks like a _p_oid, assume it is a persistent_id -> bytes
op
=
SHORT_BINBYTES
op
=
SHORT_BINBYTES
else
:
else
:
# if it's a valid UTF-8 string -> str
# if it's a valid UTF-8 string -> str
...
@@ -879,11 +860,9 @@ def save_string(self, tag, data):
...
@@ -879,11 +860,9 @@ def save_string(self, tag, data):
# not valid utf-8 -> bytes
# not valid utf-8 -> bytes
op
=
SHORT_BINBYTES
op
=
SHORT_BINBYTES
else
:
else
:
# XXX this branch seems wrong
op
=
SHORT_BINSTRING
op
=
SHORT_BINSTRING
try
:
try
:
v
.
decode
(
'ascii'
)
v
.
decode
(
'ascii'
)
# XXX zope4py3 we could also create an unpickler with encoding utf-8 ?
except
UnicodeDecodeError
:
except
UnicodeDecodeError
:
op
=
BINUNICODE
if
six
.
PY3
else
BINSTRING
op
=
BINUNICODE
if
six
.
PY3
else
BINSTRING
v
=
op
+
struct
.
pack
(
'<i'
,
l
)
+
v
v
=
op
+
struct
.
pack
(
'<i'
,
l
)
+
v
...
@@ -891,17 +870,19 @@ def save_string(self, tag, data):
...
@@ -891,17 +870,19 @@ def save_string(self, tag, data):
v
=
op
+
six
.
int2byte
(
l
)
+
v
v
=
op
+
six
.
int2byte
(
l
)
+
v
else
:
else
:
# TODO: zope4py3 see assumption above for SHORT_BINBYTES / SHORT_BINSTRING
# TODO no! check this more ...
# op = BINSTRING
if
encoding
==
'base64'
:
if
encoding
==
'base64'
:
op
=
BINBYTES
op
=
BINBYTES
# if it's a valid UTF-8 string -> str
try
:
v
.
decode
(
'utf-8'
)
op
=
BINUNICODE
if
six
.
PY3
else
BINSTRING
except
UnicodeDecodeError
:
# not valid utf-8 -> bytes
pass
else
:
else
:
op
=
BINSTRING
if
six
.
PY2
else
BINUNICODE
op
=
BINSTRING
if
six
.
PY2
else
BINUNICODE
v
=
op
+
struct
.
pack
(
'<i'
,
l
)
+
v
v
=
op
+
struct
.
pack
(
'<i'
,
l
)
+
v
else
:
v
=
STRING
+
repr
(
v
).
encode
()
+
b'
\
n
'
return
save_put
(
self
,
v
,
a
)
return
save_put
(
self
,
v
,
a
)
def
save_bytes
(
self
,
tag
,
data
):
def
save_bytes
(
self
,
tag
,
data
):
...
@@ -910,7 +891,6 @@ def save_bytes(self, tag, data):
...
@@ -910,7 +891,6 @@ def save_bytes(self, tag, data):
encoding
=
a
.
get
(
'encoding'
,
'repr'
)
encoding
=
a
.
get
(
'encoding'
,
'repr'
)
if
encoding
:
if
encoding
:
v
=
unconvert
(
encoding
,
v
)
v
=
unconvert
(
encoding
,
v
)
if
self
.
binary
:
l
=
len
(
v
)
l
=
len
(
v
)
if
l
<
256
:
if
l
<
256
:
op
=
SHORT_BINBYTES
op
=
SHORT_BINBYTES
...
@@ -918,14 +898,10 @@ def save_bytes(self, tag, data):
...
@@ -918,14 +898,10 @@ def save_bytes(self, tag, data):
else
:
else
:
op
=
BINBYTES
op
=
BINBYTES
v
=
op
+
struct
.
pack
(
'<i'
,
l
)
+
v
v
=
op
+
struct
.
pack
(
'<i'
,
l
)
+
v
else
:
# XXX used ??? seems wrong
v
=
BYTES
+
repr
(
v
)
+
'
\
n
'
return
save_put
(
self
,
v
,
a
)
return
save_put
(
self
,
v
,
a
)
def
save_unicode
(
self
,
tag
,
data
):
def
save_unicode
(
self
,
tag
,
data
):
binary
=
self
.
binary
v
=
b''
v
=
b''
a
=
data
[
1
]
a
=
data
[
1
]
if
len
(
data
)
>
2
:
if
len
(
data
)
>
2
:
...
@@ -934,11 +910,9 @@ def save_unicode(self, tag, data):
...
@@ -934,11 +910,9 @@ def save_unicode(self, tag, data):
encoding
=
a
.
get
(
'encoding'
,
'repr'
)
# JPS: repr is default encoding
encoding
=
a
.
get
(
'encoding'
,
'repr'
)
# JPS: repr is default encoding
if
encoding
!=
''
:
if
encoding
!=
''
:
v
=
unconvert
(
encoding
,
v
)
v
=
unconvert
(
encoding
,
v
)
if
binary
:
l
=
len
(
v
)
l
=
len
(
v
)
s
=
mdumps
(
l
)[
1
:]
s
=
mdumps
(
l
)[
1
:]
v
=
BINUNICODE
+
s
+
v
v
=
BINUNICODE
+
s
+
v
else
:
v
=
UNICODE
+
"'"
+
v
+
"'
\
012
"
return
save_put
(
self
,
v
,
a
)
return
save_put
(
self
,
v
,
a
)
def
save_tuple
(
self
,
tag
,
data
):
def
save_tuple
(
self
,
tag
,
data
):
...
@@ -949,26 +923,16 @@ def save_tuple(self, tag, data):
...
@@ -949,26 +923,16 @@ def save_tuple(self, tag, data):
def
save_list
(
self
,
tag
,
data
):
def
save_list
(
self
,
tag
,
data
):
L
=
data
[
2
:]
L
=
data
[
2
:]
if
self
.
binary
:
v
=
save_put
(
self
,
EMPTY_LIST
,
data
[
1
])
v
=
save_put
(
self
,
EMPTY_LIST
,
data
[
1
])
if
L
:
if
L
:
v
=
v
+
MARK
+
b''
.
join
(
L
)
+
APPENDS
v
=
v
+
MARK
+
b''
.
join
(
L
)
+
APPENDS
else
:
v
=
save_put
(
self
,
MARK
+
LIST
,
data
[
1
])
if
L
:
v
=
APPEND
.
join
(
L
)
+
APPEND
return
v
return
v
def
save_dict
(
self
,
tag
,
data
):
def
save_dict
(
self
,
tag
,
data
):
D
=
data
[
2
:]
D
=
data
[
2
:]
if
self
.
binary
:
v
=
save_put
(
self
,
EMPTY_DICT
,
data
[
1
])
v
=
save_put
(
self
,
EMPTY_DICT
,
data
[
1
])
if
D
:
if
D
:
v
=
v
+
MARK
+
b''
.
join
(
D
)
+
SETITEMS
v
=
v
+
MARK
+
b''
.
join
(
D
)
+
SETITEMS
else
:
v
=
save_put
(
self
,
MARK
+
DICT
,
data
[
1
])
if
D
:
v
=
v
+
SETITEM
.
join
(
D
)
+
SETITEM
return
v
return
v
def
save_reference
(
self
,
tag
,
data
):
def
save_reference
(
self
,
tag
,
data
):
...
@@ -977,14 +941,11 @@ def save_reference(self, tag, data):
...
@@ -977,14 +941,11 @@ def save_reference(self, tag, data):
prefix
=
id
.
rfind
(
'.'
)
prefix
=
id
.
rfind
(
'.'
)
if
prefix
>=
0
:
if
prefix
>=
0
:
id
=
id
[
prefix
+
1
:]
id
=
id
[
prefix
+
1
:]
if
self
.
binary
:
id
=
int
(
id
)
id
=
int
(
id
)
if
id
<
256
:
if
id
<
256
:
return
BINGET
+
six
.
int2byte
(
id
)
return
BINGET
+
six
.
int2byte
(
id
)
else
:
else
:
return
LONG_BINGET
+
struct
.
pack
(
'<i'
,
id
)
return
LONG_BINGET
+
struct
.
pack
(
'<i'
,
id
)
else
:
return
GET
+
repr
(
id
).
encode
()
+
b'
\
n
'
def
save_object
(
self
,
tag
,
data
):
def
save_object
(
self
,
tag
,
data
):
if
len
(
data
)
==
5
:
if
len
(
data
)
==
5
:
...
@@ -1012,10 +973,7 @@ def save_global(self, tag, data):
...
@@ -1012,10 +973,7 @@ def save_global(self, tag, data):
def
save_persis
(
self
,
tag
,
data
):
def
save_persis
(
self
,
tag
,
data
):
v
=
data
[
2
]
v
=
data
[
2
]
if
self
.
binary
:
return
v
+
BINPERSID
return
v
+
BINPERSID
else
:
return
PERSID
+
v
def
save_pickle_start
(
self
,
tag
,
attrs
):
def
save_pickle_start
(
self
,
tag
,
attrs
):
return
[
tag
,
attrs
]
return
[
tag
,
attrs
]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment