web_shadir: PY3 compatibility

bt5/erp5_web_shacache/ExtensionTemplateItem/portal_components/extension.erp5.ShaCache.py

@@ -26,8 +26,9 @@
 #
 ##############################################################################
 
-
-import hashlib, six.moves.http_client
+import hashlib
+import six
+import six.moves.http_client
 from Products.ERP5Type.UnrestrictedMethod import super_user
 
 
@@ -77,7 +78,7 @@ def File_viewAsWeb(self):
 
   # Shortcut if the file is not a Pdata.
   data=self.data
-  if isinstance(data, str):
+  if isinstance(data, bytes):
     # Do this way instead of 'return data'
     # to bypass default caching policy manager.
     RESPONSE.write(data)
@@ -85,7 +86,9 @@ def File_viewAsWeb(self):
 
   # For Pdata type, we must iterate and send chunk by chunk.
   # And no need to continue if the client closed the connection.
-  while data and not RESPONSE.stdout._channel.closed:
+  while data:
+    if six.PY2 and RESPONSE.stdout._channel.closed:
+      break
     # Send data to the client.
     RESPONSE.write(data.data)
     # Load next object without keeping previous chunks in memory.

bt5/erp5_web_shacache/TestTemplateItem/portal_components/test.erp5.ShaCacheMixin.py

@@ -52,5 +52,5 @@ class ShaCacheMixin(object):
     }
     self.shacache_url = self.shacache.absolute_url()
     self.tic()
-    self.data = 'Random Content. %s' % str(random.random()).encode()
+    self.data = ('Random Content. %s' % random.random()).encode()
     self.key = hashlib.sha512(self.data).hexdigest()

bt5/erp5_web_shacache/TestTemplateItem/portal_components/test.erp5.testShaCache.py

@@ -83,7 +83,7 @@ class TestShaCache(ShaCacheMixin, ERP5TypeTestCase):
     """
     result, data = self.postFile()
     self.assertEqual(result, six.moves.http_client.CREATED)
-    self.assertEqual(data, self.key)
+    self.assertEqual(data, self.key.encode())
 
     self.tic()
 
@@ -101,7 +101,7 @@ class TestShaCache(ShaCacheMixin, ERP5TypeTestCase):
     """
     result, data = self.postFile()
     self.assertEqual(result, six.moves.http_client.CREATED)
-    self.assertEqual(data, self.key)
+    self.assertEqual(data, self.key.encode())
 
     self.tic()
 
@@ -131,6 +131,7 @@ class TestShaCache(ShaCacheMixin, ERP5TypeTestCase):
     self.assertEqual('published', document2.getValidationState())
     self.assertEqual('archived', document.getValidationState())
 
+  @expectedFailure
   def test_put_file_twice_no_tic(self):
     self.postFile()
     self.commit()
@@ -140,5 +141,7 @@ class TestShaCache(ShaCacheMixin, ERP5TypeTestCase):
     document_list = self.portal.portal_catalog(reference=self.key)
 
     self.assertEqual(2, len(document_list))
-    expectedFailure(self.assertEqual)(sorted(['archived', 'published']),
+    # this is the expected failure
+    self.assertEqual(
+      sorted(['archived', 'published']),
       sorted(q.getValidationState() for q in document_list))

bt5/erp5_web_shadir/ExtensionTemplateItem/portal_components/extension.erp5.ShaDir.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 
+import six
 import hashlib
 from base64 import b64decode
 from binascii import a2b_hex
@@ -33,6 +34,7 @@ from json import dumps, loads
 from zExceptions import BadRequest
 from DateTime import DateTime
 from Products.ERP5Type.UnrestrictedMethod import super_user
+from Products.ERP5Type.Utils import unicode2str, str2bytes
 
 
 def WebSection_getDocumentValue(self, key, portal=None, language=None,\
@@ -67,10 +69,9 @@ def WebSection_getDocumentValue(self, key, portal=None, language=None,\
                          validation_state='published')]
 
     temp_file = self.newContent(temp_object=True, portal_type='File', id='%s.txt' % key)
-    temp_file.setData(dumps(document_list))
+    temp_file.setData(str2bytes(dumps(document_list)))
     temp_file.setContentType('application/json')
     return temp_file.getObject()
-
   return None
 
 def WebSection_setObject(self, id, ob, **kw):
@@ -78,12 +79,14 @@ def WebSection_setObject(self, id, ob, **kw):
     Make any change related to the file uploaded.
   """
   portal = self.getPortalObject()
+  ob = ob.getOriginalDocument()
   data = self.REQUEST.get('BODY')
+
   try:
     metadata, signature = loads(data)
     metadata = loads(metadata)
     # a few basic checks
-    b64decode(signature)
+    b64decode(str2bytes(signature))
     if len(a2b_hex(metadata['sha512'])) != 64:
       raise Exception('sha512: invalid length')
   except Exception as e:
@@ -126,7 +129,13 @@ def WebSection_putFactory(self, name, typ, body):
   document = portal.portal_contributions.newContent(data=body,
                                                     filename=name,
                                                     discover_metadata=False)
-  return document
+
+  # return a document for which getId() returns the name for _setObject to be
+  # called with id=name ( for WebSection_setObject ), but for which
+  # getRelativeUrl returns the relative url of the real document, for
+  # VirtualFolderMixin transactional variable cache between _setObject and
+  # _getOb
+  return document.asContext(getId=lambda: name)
 
 # The following scripts are helpers to search & clean up shadir entries.
 # XXX: Due to lack of View skin for shadir, external methods are currently
@@ -217,10 +226,10 @@ def ShaDir_search(self, filename, summary, delete=False):
     document_list.append(document)
     metadata = loads(loads(document.getData())[0])
     del metadata[u"sha512"]
-    x[';'.join('%s=%r' % (k, v.encode('utf-8') if type(v) is unicode else v)
+    x[';'.join('%s=%r' % (k, unicode2str(v))
                for k, v in sorted(metadata.iteritems()))].append(
       document.getId())
-  r = '\n'.join('%s %s' % (k, sorted(v)) for k, v in sorted(x.iteritems()))
+  r = '\n'.join('%s %s' % (k, sorted(v)) for k, v in sorted(six.iteritems(x)))
   if delete:
     r += '\n' + _deleteDocumentList(self, document_list)
   return r

bt5/erp5_web_shadir/TestTemplateItem/portal_components/test.erp5.ShaDirMixin.py

@@ -44,10 +44,9 @@ class ShaDirMixin(object):
       Initialize the ERP5 site.
     """
     self.login()
-    self.portal = self.getPortal()
 
     self.key = 'mykey' + str(random.random())
-    self.file_content = 'This is the content.'
+    self.file_content = b'This is the content.'
     self.file_sha512sum = hashlib.sha512(self.file_content).hexdigest()
     self.distribution = 'pypi'
     self.creation_date = DateTime()
@@ -62,14 +61,14 @@ class ShaDirMixin(object):
                       'expiration_date': str(self.expiration_date),
                       'distribution': self.distribution,
                       'architecture': self.architecture}),
-                      b64encode("User SIGNATURE goes here.")]
+                      b64encode(b"User SIGNATURE goes here.").decode()]
 
-    self.data = json.dumps(self.data_list)
+    self.data = json.dumps(self.data_list).encode()
     self.sha512sum = hashlib.sha512(self.data).hexdigest()
 
     self.header_dict = {
       'Content-Type': 'application/json',
-      'Authorization': 'Basic ' + b64encode('ERP5TypeTestCase:'),
+      'Authorization': 'Basic ' + b64encode(b'ERP5TypeTestCase:').decode(),
     }
 
     module = self.portal.web_site_module

bt5/erp5_web_shadir/TestTemplateItem/portal_components/test.erp5.testShaDir.py

@@ -36,6 +36,8 @@ from base64 import b64encode
 from unittest import expectedFailure
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from erp5.component.test.ShaDirMixin import ShaDirMixin
+from Products.ERP5Type.Utils import bytes2str
+
 
 class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
   """
@@ -63,7 +65,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     finally:
       connection.close()
     self.assertEqual(result.status, six.moves.http_client.CREATED)
-    self.assertEqual(data, '')
+    self.assertEqual(data, b'')
 
   def getInformation(self, key=None):
     """
@@ -103,19 +105,23 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     data_set = self.portal.portal_catalog.getResultValue(
       reference=self.key)
     self.assertEqual(self.key, data_set.getReference())
+    self.assertNotEqual(self.key, data_set.getId())
     self.assertEqual('published', data_set.getValidationState())
+    self.assertEqual(len(self.portal.data_set_module.contentValues()), 1)
 
     # Asserting Document
     document = self.portal.portal_catalog.getResultValue(
       reference=self.sha512sum)
     self.assertEqual(self.sha512sum, document.getTitle())
     self.assertEqual(self.sha512sum, document.getReference())
+    self.assertNotEqual(self.sha512sum, document.getId())
     self.assertEqual(self.data, document.getData())
     self.assertEqual(data_set, document.getFollowUpValue())
     self.assertEqual(str(self.expiration_date),
                                     str(document.getExpirationDate()))
     self.assertEqual('application/json', document.getContentType())
     self.assertEqual('Published', document.getValidationStateTitle())
+    self.assertEqual(len(self.portal.document_module.contentValues()), 1)
 
   def test_get_information(self):
     """
@@ -131,7 +137,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     information_list = json.loads(data)
 
     self.assertEqual(1, len(information_list))
-    self.assertEqual(json.dumps(information_list[0]), self.data)
+    self.assertEqual(json.dumps(information_list[0]), bytes2str(self.data))
 
   def test_post_information_more_than_once(self):
     """
@@ -162,7 +168,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     information_list = json.loads(data)
 
     self.assertEqual(1, len(information_list))
-    self.assertEqual(json.dumps(information_list[0]), self.data)
+    self.assertEqual(json.dumps(information_list[0]), bytes2str(self.data))
 
   def test_post_information_more_than_once_no_tic(self):
     """
@@ -200,7 +206,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     information_list = json.loads(data)
 
     self.assertEqual(1, len(information_list))
-    self.assertEqual(json.dumps(information_list[0]), self.data)
+    self.assertEqual(json.dumps(information_list[0]), bytes2str(self.data))
 
   def test_get_information_from_different_data_set(self):
     """
@@ -215,7 +221,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     self.postInformation()
     self.tic()
 
-    sha512_2 = hashlib.sha512(str(random.random())).hexdigest()
+    sha512_2 = hashlib.sha512(str(random.random()).encode()).hexdigest()
     key_2 = 'another_key' + str(random.random())
     data_list_2 = [json.dumps({
                       'sha512': sha512_2,
@@ -223,7 +229,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
                       'expiration_date': str(self.expiration_date),
                       'distribution': self.distribution,
                       'architecture': self.architecture}),
-                      b64encode("User SIGNATURE goes here.")]
+                      b64encode(b"User SIGNATURE goes here.").decode()]
     data_2 = json.dumps(data_list_2)
     self.postInformation(key_2, data_2)
     self.tic()
@@ -268,7 +274,7 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
             version="001",
             language="en",
             follow_up_value=person,
-            data="FILEDATA")
+            data=b"FILEDATA")
     doc.publish()
 
     self.tic()

bt5/networkcache_erp5/TestTemplateItem/portal_components/test.erp5.testShaCacheExternal.py

@@ -82,7 +82,7 @@ class TestShaCacheExternal(ShaCacheMixin, ShaSecurityMixin, ERP5TypeTestCase):
       data = result.read()
     finally:
       connection.close()
-    self.assertEqual(self.key, data)
+    self.assertEqual(self.key, data.decode())
     self.assertEqual(six.moves.http_client.CREATED, result.status)
 
     # Check Document

bt5/networkcache_erp5/TestTemplateItem/portal_components/test.erp5.testShaCacheSecurity.py

@@ -202,7 +202,7 @@ class TestShaCacheSecurity(ShaCacheMixin, ShaSecurityMixin, SecurityTestCase):
                                                            contribution_tool)
     document = self.portal.portal_contributions.newContent(
                                     filename='test.txt',
-                                    data='test content',
+                                    data=b'test content',
                                     reference='test-reference')
     document()
     document.view()

bt5/networkcache_erp5/TestTemplateItem/portal_components/test.erp5.testShaDirExternal.py

@@ -60,8 +60,8 @@ class TestShaDirExternal(ShaDirMixin, ShaSecurityMixin, ERP5TypeTestCase):
 
     # Define POST headers with Authentication
     self.content_type =  'application/json'
-    authentication_string = 'lucas:lucas'
-    base64string = base64.encodestring(authentication_string).strip()
+    authentication_string = b'lucas:lucas'
+    base64string = base64.b64encode(authentication_string).decode().strip()
     self.header_dict = {'Authorization': 'Basic %s' % base64string,
                         'Content-Type': self.content_type}
 
@@ -83,7 +83,7 @@ class TestShaDirExternal(ShaDirMixin, ShaSecurityMixin, ERP5TypeTestCase):
       data = result.read()
     finally:
       connection.close()
-    self.assertEqual('', data)
+    self.assertEqual(b'', data)
     self.assertEqual(201, result.status)
 
     # Check Data Set
@@ -121,7 +121,7 @@ class TestShaDirExternal(ShaDirMixin, ShaSecurityMixin, ERP5TypeTestCase):
       data = result.read()
     finally:
       connection.close()
-    self.assertEqual(json.dumps([json.loads(self.data)]), data)
+    self.assertEqual(json.dumps([json.loads(self.data)]), data.decode())
     self.assertEqual(200, result.status)
     self.assertEqual(self.content_type, result.getheader("content-type"))