shadir: review metadata validation

urlmd5 is useless: better store url. And if metadata contains url,
file also becomes useless. There remain so little to check that we can
do better with custom validation instead of using a JSON validator.

Also, if we wanted to set a file name on a shadir document, which is
json data, it should be different from the file name of the referenced
cached data. At least, the extension would end with '.json'

bt5/erp5_web_shadir/ExtensionTemplateItem/portal_components/extension.erp5.ShaDir.py

@@ -25,10 +25,11 @@
 #
 ##############################################################################
 
-
 import hashlib
 import json
-import validictory
+from base64 import b64decode
+from binascii import a2b_hex
+from zExceptions import BadRequest
 from Products.ERP5Type.UnrestrictedMethod import super_user
 
 
@@ -39,17 +40,10 @@ def WebSection_getDocumentValue(self, key, portal=None, language=None,\
 
      - POST /<key>
         + parameters required:
-           * file: the name of the file
-           * urlmd5: mdsum of orginal url
            * sha512: the hash (sha512) of the file content
 
-        + parameters not required:
-           * valid-until: the date which the file must be expired
-           * architecture: computer architecture
-
        Used to add information on shadir server.
 
-
      - GET /<key>
        Return list of information for a given key
        Raise HTTP error (404) if key does not exist
@@ -83,16 +77,17 @@ def WebSection_setObject(self, id, ob, **kw):
   """
   portal = self.getPortalObject()
   data = self.REQUEST.get('BODY')
-  schema = self.WebSite_getJSONSchema()
-  structure = json.loads(data)
-  # 0 elementh in structure is json in json
-  # 1 elementh is just signature
-  structure = [json.loads(structure[0]), structure[1]]
-
-  validictory.validate(structure, schema)
-
-  file_name = structure[0].get('file', None)
-  expiration_date = structure[0].get('expiration_date', None)
+  try:
+    metadata, signature = json.loads(data)
+    metadata = json.loads(metadata)
+    # a few basic checks
+    b64decode(signature)
+    if len(a2b_hex(metadata['sha512'])) != 64:
+      raise Exception('sha512: invalid length')
+  except Exception as e:
+    raise BadRequest(str(e))
+
+  expiration_date = metadata.get('expiration_date')
 
   data_set = portal.portal_catalog.getResultValue(portal_type='Data Set',
                                                   reference=id)
@@ -105,7 +100,6 @@ def WebSection_setObject(self, id, ob, **kw):
 
 
   reference = hashlib.sha512(data).hexdigest()
-  ob.setFilename(file_name)
   ob.setFollowUp(data_set.getRelativeUrl())
   ob.setContentType('application/json')
   ob.setReference(reference)

bt5/erp5_web_shadir/SkinTemplateItem/portal_skins/erp5_web_shadir/WebSite_getJSONSchema.py

-return {
-   'type': 'array',
-   'items': [
-      {'type': 'object',
-       'properties':{
-          'file':{
-             'type': 'string',
-             'required': True,
-          },
-          'urlmd5': {
-             'type': 'string',
-             'required': True,
-          },
-          'sha512': {
-             'type': 'string',
-             'required': True,
-          },
-          'creation_date': {
-             'type': 'string',
-             'required': False,
-          },
-          'expiration_date': {
-             'type': 'string',
-             'required': False,
-          },
-          'distribution': {
-             'type': 'string',
-             'required': False,
-          },
-          'architecture': {
-             'type': 'string',
-             'required': False,
-          },
-       }
-     },
-     {'type': 'string',
-      'blank': True},
-   ]
-}

bt5/erp5_web_shadir/SkinTemplateItem/portal_skins/erp5_web_shadir/WebSite_getJSONSchema.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
-        </item>
-        <item>
-            <key> <string>_params</string> </key>
-            <value> <string></string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>WebSite_getJSONSchema</string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

bt5/erp5_web_shadir/TestTemplateItem/portal_components/test.erp5.ShaDirMixin.py

@@ -27,11 +27,10 @@
 #
 ##############################################################################
 
-import base64
 import hashlib
 import json
-import platform
 import random
+from base64 import b64encode
 from DateTime import DateTime
 
 
@@ -47,32 +46,25 @@ class ShaDirMixin(object):
     self.portal = self.getPortal()
 
     self.key = 'mykey' + str(random.random())
-    self.file_name = 'file.txt'
-    self.urlmd5 = hashlib.md5(self.key).hexdigest()
-    self.file_content = 'This is the content.'
-    self.file_sha512sum = hashlib.sha512(self.file_content).hexdigest()
-    self.distribution = 'pypi'
-    self.creation_date = DateTime()
-    self.expiration_date = self.creation_date + 30
+    file_content = 'This is the content.'
+    creation_date = DateTime()
+    self.expiration_date = creation_date + 30
 
-    libc_version = '%s %s' % (platform.libc_ver()[0], platform.libc_ver()[1])
-    self.architecture = '%s %s' % (platform.machine(), libc_version)
+    self.data = json.dumps([
+      json.dumps({
+        'sha512': hashlib.sha512(file_content).hexdigest(),
+        'url': 'https://example.com/some/file.ext',
+        'creation_date': str(creation_date),
+        'expiration_date': str(self.expiration_date),
+      }),
+      b64encode("User SIGNATURE goes here."),
+    ])
 
-    self.data_list = [json.dumps({'file': self.file_name,
-                      'urlmd5': self.urlmd5,
-                      'sha512': self.file_sha512sum,
-                      'creation_date': str(self.creation_date),
-                      'expiration_date': str(self.expiration_date),
-                      'distribution': self.distribution,
-                      'architecture': self.architecture}),
-                      "User SIGNATURE goes here."]
-
-    self.data = json.dumps(self.data_list)
     self.sha512sum = hashlib.sha512(self.data).hexdigest()
 
     self.header_dict = {
       'Content-Type': 'application/json',
-      'Authorization': 'Basic %s' % (base64.encodestring('ERP5TypeTestCase:').strip())
+      'Authorization': 'Basic ' + b64encode('ERP5TypeTestCase:'),
     }
 
     module = self.portal.web_site_module

bt5/erp5_web_shadir/TestTemplateItem/portal_components/test.erp5.testShaDir.py

@@ -27,11 +27,12 @@
 #
 ##############################################################################
 
-
+import hashlib
 import httplib
 import urlparse
 import json
 import random
+from base64 import b64encode
 from unittest import expectedFailure
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from erp5.component.test.ShaDirMixin import ShaDirMixin
@@ -214,18 +215,14 @@ class TestShaDir(ShaDirMixin, ERP5TypeTestCase):
     self.postInformation()
     self.tic()
 
-    urlmd5_2 = 'anotherurlmd5' + str(random.random())
-    sha512_2 = 'anothersha512_2' + str(random.random())
     key_2 = 'another_key' + str(random.random())
-    data_list_2 = [json.dumps({'file': self.file_name,
-                      'urlmd5': urlmd5_2,
-                      'sha512': sha512_2,
-                      'creation_date': str(self.creation_date),
-                      'expiration_date': str(self.expiration_date),
-                      'distribution': self.distribution,
-                      'architecture': self.architecture}),
-                      "User SIGNATURE goes here."]
-    data_2 = json.dumps(data_list_2)
+    data_2 = json.dumps([
+      json.dumps({
+        'sha512': hashlib.sha512(str(random.random())).hexdigest(),
+        'url': 'http://example.com/another.file',
+      }),
+      b64encode("User SIGNATURE goes here."),
+    ])
     self.postInformation(key_2, data_2)
     self.tic()
 

bt5/networkcache_erp5/TestTemplateItem/portal_components/test.erp5.testShaDirExternal.py

@@ -32,6 +32,7 @@ import base64
 import json
 import os
 import httplib
+from contextlib import contextmanager
 from DateTime import DateTime
 from Products.ERP5Type.tests.ERP5TypeLiveTestCase import ERP5TypeTestCase
 from erp5.component.test.ShaDirMixin import ShaDirMixin
@@ -146,26 +147,30 @@ class TestShaDirExternal(ShaDirMixin, ShaSecurityMixin, ERP5TypeTestCase):
     self.assertEqual(302, result.status)
 
   def test_external_post_with_wrong_data(self):
-    """
-      The data which is sent to the server must follow a JSON schema.
-      If the data does not follow the schema it must return the error.
-    """
-    # Removing a required property
-    data = json.loads(self.data)
-    data[0] = json.loads(data[0])
-    data[0].pop('file')
-    data[0] = json.dumps(data[0])
-    data = json.dumps(data)
-
-    connection = httplib.HTTPConnection('%s:%s' % (self.host, self.port))
-    try:
-      connection.request('PUT', self.path, data, self.header_dict)
-      result = connection.getresponse()
-      self.tic()
-      data = result.read()
-    finally:
-      connection.close()
-    self.assertTrue("Required field 'file' is missing" in data, data)
-    self.assertEqual(500, result.status)
-    self.assertEqual('text/html; charset=utf-8',
-                                         result.getheader("content-type"))
+    @contextmanager
+    def check():
+      metadata, signature = json.loads(self.data)
+      data = [json.loads(metadata), signature]
+      yield data
+      data[0] = json.dumps(data[0])
+      data = json.dumps(data)
+      connection = httplib.HTTPConnection('%s:%s' % (self.host, self.port))
+      try:
+        connection.request('PUT', self.path, data, self.header_dict)
+        result = connection.getresponse()
+        self.tic()
+        data = result.read()
+      finally:
+        connection.close()
+      self.assertEqual(400, result.status)
+      self.assertEqual('text/html; charset=utf-8',
+                       result.getheader("content-type"))
+    with check() as data:
+      del data[0]['sha512']
+    with check() as data:
+      data[0]['sha512'] = '1234'
+    for x in 123, 'foo':
+      with check() as data:
+        data[0]['sha512'] = x
+      with check() as data:
+        data[1] = x