[json_style] Enable reports by augmenting mode='search'

-  ListBox passes all its default attributes into serizlized JSON for 'search'
-  search reads script's arguments and provide them from REQUEST

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.py

@@ -26,6 +26,56 @@ def byteify(string):
   else:
     return string
 
+def ensure_serializable(obj):
+  """Ensure obj and all sub-objects are JSON serializable."""
+  if isinstance(obj, dict):
+    for key in obj:
+      obj[key] = ensure_serializable(obj[key])
+  if isinstance(obj, DateTime):
+    return serialize_DateTime(obj)
+  if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)):
+    return serialize_datetime(obj)
+  # we don't check other isinstances - we believe that iterables don't contain unserializable objects
+  return obj
+
+date_re = re.compile(r'\d{4}-\d{2}-\d{2}')
+datetime_re = re.compile(r'\d{4}-\d{2}-\d{2} |T\d{2}:\d{2}:\d{2}.*')
+def ensure_deserialized(obj):
+  """Deserialize classes serialized by our own `ensure_serializable`.
+
+  Method `biteify` must not be called on the result because it would revert out
+  deserialization by calling __str__ on constructed classes.
+  """
+  if isinstance(obj, dict):
+    for key in obj:
+      obj[key] = ensure_deserialized(obj[key])
+  # seems that default __str__ method is good enough
+  if isinstance(obj, str):
+    if date_re.match(obj) or datetime_re.match(obj):
+      return DateTime(obj)
+  return obj
+
+def serialize_datetime(obj):
+  """Serialize date/time into string for further JSON serialization into query parameters."""
+  if isinstance(obj, datetime.datetime):
+    if obj.time():
+      # date containing time as well
+      return obj.strftime("%Y-%m-%d %H:%M:%S")
+    obj = obj.date()
+  if isinstance(obj, datetime.date):
+    return obj.strftime("%Y-%m-%d")
+  if isinstance(obj, datetime.time):
+    return obj.strftime("%H:%M:%S")
+  raise ValueError("obj {!s} is not instance of datetime.*".format(type(obj)))
+
+
+def serialize_DateTime(obj):
+  """Serialize Date/Time into string for further JSON serialization into query parameters."""
+  if obj.Time() == '00:00:00':
+    return obj.Date().replace("/", "-")
+  return obj.ISO()
+
+
 def getProtectedProperty(document, select):
   try:
     #see https://lab.nexedi.com/nexedi/erp5/blob/master/product/ERP5Form/ListBox.py#L2293
@@ -75,9 +125,11 @@ url_template_dict = {
 default_document_uri_template = url_template_dict["jio_get_template"]
 Base_translateString = context.getPortalObject().Base_translateString
 
+
 def getRealRelativeUrl(document):
   return '/'.join(portal.portal_url.getRelativeContentPath(document))
 
+
 def getFormRelativeUrl(form):
   return portal.portal_catalog(
     portal_type="ERP5 Form",
@@ -87,6 +139,7 @@ def getFormRelativeUrl(form):
     select_dict={'relative_url': None}
   )[0].relative_url
 
+
 def getFieldDefault(traversed_document, field, key, value=None):
   # REQUEST.get(field.id, field.get_value("default"))
   result = traversed_document.Field_getDefaultValue(field, key, value, REQUEST)
@@ -102,7 +155,8 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
     meta_type = field.meta_type
   if key is None:
     key = field.generate_field_key(key_prefix=key_prefix)
-
+  context.log("traversed_document={!s}, field={!s}, form={!s}, value={!s}, meta_type={!s}".format(
+    traversed_document, field, form, value, meta_type))
   result = {
     "type": meta_type,
     "title": Base_translateString(field.get_value("title")),
@@ -322,20 +376,56 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
     # implemented in javascript in the end
     # see https://lab.nexedi.com/nexedi/erp5/blob/master/bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_listbox_js.js#L163
 
-    portal_types = field.get_value('portal_types')
-    default_params = dict(field.get_value('default_params'))
+    default_params = dict(field.get_value('default_params'))  # default_params is a list of tuples
     default_params['ignore_unknown_columns'] = True
+    
+    # we abandoned Selections in RJS thus we mix selection query parameters into
+    # listbox's default parameters
     if selection_params is not None:
       default_params.update(selection_params)
-    # How to implement pagination?
-    # default_params.update(REQUEST.form)
+    
     lines = field.get_value('lines')
     list_method_name = traversed_document.Listbox_getListMethodName(field)
-    list_method_query_dict = dict(
-      portal_type=[x[1] for x in portal_types], **default_params
-    )
+    context.log("ListBox '{!s}'\n >> selection params: {!s}\n >> default params: {!s} ".format(
+      field.absolute_url(), selection_params, default_params))
+
+    # ListBoxes in report view has portal_type defined already in default_params
+    # in that case we prefer non_empty version
+    list_method_query_dict = default_params.copy()
+    if not list_method_query_dict.get("portal_type", []):
+      list_method_query_dict["portal_type"] = [x for x, _ in field.get_value('portal_types')]
     list_method_custom = None
     
+    # Search for non-editable documents - all reports goes here
+    # Reports have custom search scripts which wants parameters from the form
+    # thus we introspect such parameters and try to find them in REQUEST
+    list_method = None
+    if list_method_name and list_method_name not in ("portal_catalog", "searchFolder", "objectValues"):
+      # we avoid accessing known protected objects and builtin functions above
+      try:
+        list_method = getattr(form, list_method_name)
+      except Unauthorized:
+        # we are touching some specially protected (usually builtin) methods
+        # which we will not introspect
+        pass
+    
+    if list_method is not None and hasattr(list_method, "ZScriptHTML_tryParams"):
+      for list_method_param in list_method.ZScriptHTML_tryParams():
+        if list_method_param in REQUEST and list_method_param not in list_method_query_dict:
+          list_method_query_dict[list_method_param] = REQUEST.get(list_method_param)
+      # MIDDLE-DANGEROUS!
+      # In case of reports (later even exports) substitute None for unknown 
+      # parameters. We suppose Python syntax for parameters!
+      for list_method_param in list_method.params().split(","):
+        if "*" in list_method_param:
+          continue
+        if "=" in list_method_param:
+          continue
+        # now we have only mandatory parameters
+        list_method_param = list_method_param.strip()
+        if list_method_param not in list_method_query_dict:
+          list_method_query_dict[list_method_param] = None
+
     if (editable_column_list):
       list_method_custom = url_template_dict["custom_search_template"] % {
         "root_url": site_root.absolute_url(),
@@ -343,8 +433,11 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
         "relative_url": traversed_document.getRelativeUrl().replace("/", "%2F"),
         "form_relative_url": "%s/%s" % (getFormRelativeUrl(form), field.id),
         "list_method": list_method_name,
-        "default_param_json": urlsafe_b64encode(json.dumps(list_method_query_dict))
+        "default_param_json": urlsafe_b64encode(
+          json.dumps(ensure_serializable(list_method_query_dict)))
       }
+      # once we imprint `default_params` into query string of 'list method' we
+      # don't want them to propagate to the query as well
       list_method_query_dict = {}
     elif (list_method_name == "portal_catalog"):
       pass
@@ -356,7 +449,7 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
         "script_id": script.id,
         "relative_url": traversed_document.getRelativeUrl().replace("/", "%2F"),
         "list_method": list_method_name,
-        "default_param_json": urlsafe_b64encode(json.dumps(list_method_query_dict))
+        "default_param_json": urlsafe_b64encode(json.dumps(ensure_serializable(list_method_query_dict)))
       }
       list_method_query_dict = {}
 
@@ -387,9 +480,9 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
       "sort_column_list": sort_column_list,
       "editable_column_list": editable_column_list,
       "show_anchor": field.get_value("anchor"),
-      "portal_type": portal_types,
+      "portal_type": field.get_value('portal_types'),
       "lines": lines,
-      "default_params": default_params,
+      "default_params": ensure_serializable(default_params),
       "list_method": list_method_name,
       "query": url_template_dict["jio_search_template"] % {
         "query": make_query({
@@ -540,9 +633,15 @@ def renderForm(traversed_document, form, response_dict, key_prefix=None, selecti
       # report_class = "report_title_level_%s" % report_item.getLevel()
       report_form = report_item.getFormId()
       report_result = {'_links': {}}
-      renderForm(traversed_document, getattr(report_context, report_item.getFormId()),
-                 report_result, key_prefix=report_prefix,
-                 selection_params=report_item.selection_params)
+      # some reports save a lot of unserializable data (datetime.datetime) and
+      # key "portal_type" (don't confuse with "portal_types" in ListBox) into
+      # report_item.selection_params thus we need to take that into account in
+      # ListBox field
+      renderForm(traversed_document,
+                 getattr(report_context, report_item.getFormId()),
+                 report_result,
+                 key_prefix=report_prefix,
+                 selection_params=None)  # report_item.selection_params XXX: does it matter?
       report_result_list.append(report_result)
 
     response_dict['report_section_list'] = report_result_list
@@ -746,7 +845,6 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
         erp5_action_list.append({
           'href': '%s' % view_action['url'],
           'name': view_action['id'],
-          'icon': view_action['icon'],
           'title': Base_translateString(view_action['title'])
         })
         # Try to embed the form in the result
@@ -965,33 +1063,48 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
   elif mode == 'search':
     #################################################
     # Portal catalog search
+    
     #
     # Possible call arguments example:
     #  form_relative_url: portal_skins/erp5_web/WebSite_view/listbox
-    #  list_method: objectValues                      (Script providing listing)
+    #  list_method: objectValues                      (Script providing items)
     #  default_param_json: <base64 encoded JSON>      (Additional search params)
     #  query: <str>                                   (term for fulltext search)
     #  select_list: ['int_index', 'id', 'title', ...] (column names to select)
     #  limit: [15, 16]                                (begin_index, num_records)
     #  local_roles: TODO
+    #
+    # Default Param JSON contains
+    #  portal_type: list of Portal Types to include (singular form matches the
+    #                                                catalog column name)
     #################################################
     if REQUEST.other['method'] != "GET":
       response.setStatus(405)
       return ""
   
     # hardcoded responses for site and portal objects (which are not Documents!)
+    # we let the flow to continue because the result of a list_method call can
+    # be similar - they can in practice return anything
     if query == "__root__":
-      sql_list = [site_root]
+      search_result_iterable = [site_root]
     elif query == "__portal__":
-      sql_list = [portal]
+      search_result_iterable = [portal]
     else:
+      # otherwise gather kwargs for list_method and get whatever result it gives
+      callable_list_method = portal.portal_catalog
+      if list_method:
+        callable_list_method = getattr(traversed_document, list_method)
+
       catalog_kw = {
         "local_roles": local_roles,
         "limit": limit,
-        "sort_on": ()  # default is empty tuple
+        "sort_on": ()  # default is an empty tuple
       }
       if default_param_json is not None:
-        catalog_kw.update(byteify(json.loads(urlsafe_b64decode(default_param_json))))
+        catalog_kw.update(
+          ensure_deserialized(
+            byteify(
+              json.loads(urlsafe_b64decode(default_param_json)))))
       if query:
         catalog_kw["full_text"] = query
       if sort_on is not None:
@@ -1002,103 +1115,147 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           sort_col, sort_order = json.loads(sort_on)
           catalog_kw['sort_on'] = ((byteify(sort_col), byteify(sort_order)), )
 
-      if (list_method is None):
-        callable_list_method = portal.portal_catalog
-      else:
-        callable_list_method = getattr(traversed_document, list_method)
-
-      sql_list = callable_list_method(**catalog_kw)
-
-    result_list = []  # returned "content" of the search
+      context.log("callable_list_method(**catalog_kw): {}({!s})".format(
+        list_method, catalog_kw))
+      search_result_iterable = callable_list_method(**catalog_kw)
+    context.log('search_result_iterable: {!s}\n >> len {:d}\n >> first item: {!s}'.format(
+      search_result_iterable,
+      len(search_result_iterable),
+      (search_result_iterable[0] if len(search_result_iterable) > 0 else "search_result_iterable empty")))
 
     # Cast to list if only one element is provided
-    editable_field_dict = {}
     if select_list is None:
       select_list = []
     elif same_type(select_list, ""):
       select_list = [select_list]
+    context.log("listbox: select_list {!s}".format(select_list))
 
-    if select_list:
-      if (form_relative_url is not None):
+    # extract form field definition into `editable_field_dict`
+    editable_field_dict = {}
+    if form_relative_url is not None:
       listbox_field = portal.restrictedTraverse(form_relative_url)
       listbox_field_id = listbox_field.id
       # XXX Proxy field are not correctly handled in traversed_document of web site
       listbox_form = getattr(traversed_document, listbox_field.aq_parent.id)
       for select in select_list:
         # See Listbox.py getValueList --> getEditableField & getColumnAliasList method
-          tmp = select.replace('.', '_')
-          if listbox_form.has_field("%s_%s" % (listbox_field_id, tmp), include_disabled=1):
-            editable_field_dict[select] = listbox_form.get_field("%s_%s" % (listbox_field_id, tmp), include_disabled=1)
+        field_name = "{}_{}".format(listbox_field_id, select.replace(".", "_"))
+        if listbox_form.has_field(field_name, include_disabled=1):
+          editable_field_dict[select] = listbox_form.get_field(field_name, include_disabled=1)
 
     # handle the case when list-scripts are ignoring `limit` - paginate for them
     if limit is not None and isinstance(limit, (tuple, list)):
       start, num_items = map(int, limit)
-      if len(sql_list) <= num_items:
+      if len(search_result_iterable) <= num_items:
         # the limit was most likely taken into account thus we don't need to slice
-        start, num_items = 0, len(sql_list)
+        start, num_items = 0, len(search_result_iterable)
     else:
-      start, num_items = 0, len(sql_list)
+      start, num_items = 0, len(search_result_iterable)
+
+    # go through documents and assign values into result_dict._embedded
+    contents_list = []  # returned "content" of the search
+    result_dict.update({
+      '_query': query,
+      '_local_roles': local_roles,
+      '_limit': limit,
+      '_select_list': select_list,
+      '_embedded': {
+        'contents': contents_list
+      }
+    })
 
-    for document_index, sql_document in enumerate(sql_list):
-      if document_index < start:
+    # now fill in `contents_list` with actual information
+    # beware that search_result_iterable can hide anything inside!
+    for result_index, search_result in enumerate(search_result_iterable):
+      # skip documents out of `limit`
+      if result_index < start:
         continue
-      if document_index >= start + num_items:
+      if result_index >= start + num_items:
         break
   
-      try:
-        document = sql_document.getObject()
-      except AttributeError:
-        # XXX ERP5 Site is not an ERP5 document
-        document = sql_document
-      document_uid = sql_document.uid
-      document_result = {
-        '_links': {
+      contents_item = {}
+      contents_list.append(contents_item)
+
+      contents_uid = None
+      if hasattr(search_result, "getObject"):
+        search_result = search_result.getObject()
+        contents_uid = search_result.uid
+        # every document indexed in catalog has to have relativeUrl
+        contents_relative_url = getRealRelativeUrl(search_result)
+        # get property in secure way from documents
+        search_property_getter = getProtectedProperty
+      elif hasattr(search_result, "aq_self"):
+        # Zope products have at least ID thus we work with that
+        contents_uid = search_result.uid
+        # either we got a document with relativeUrl or we got product and use ID
+        contents_relative_url = getRealRelativeUrl(search_result) or search_result.getId()
+        # documents and products have the same way of accessing properties
+        search_property_getter = getProtectedProperty
+      else:
+        # In case of reports the `search_result` can be list of 
+        # PythonScripts.standard._Object - a reimplementation of plain dictionary
+
+        # means we are iterating over plain objects
+        # list_method must be defined because POPOs can return only that
+        contents_uid = "{}#{:d}".format(list_method, result_index)
+        # JIO requires every item to have _links.self.href so it can construct
+        # links to the document. Here we have a object in RAM (which should 
+        # never happen!) thus we provide temporary UID
+        contents_relative_url = "{}/{}".format(traversed_document.getRelativeUrl(), contents_uid)
+        # property getter must be simple __getattr__ implementation
+        search_property_getter = getattr
+
+      # _links.self.href is mandatory for JIO so it can create reference to the
+      # (listbox) item alone
+      contents_item['_links'] = {
         'self': {
           "href": default_document_uri_template % {
             "root_url": site_root.absolute_url(),
-              # XXX ERP5 Site is not an ERP5 document
-              "relative_url": getRealRelativeUrl(document) or document.getId(), 
+            "relative_url": contents_relative_url, 
             "script_id": script.id
           },
         },
       }
-      }
+      # ERP5 stores&send the list of editable elements in a hidden field called
+      # only database results can be editable so it belongs here
       if editable_field_dict:
-        document_result['listbox_uid:list'] = {
+        contents_item['listbox_uid:list'] = {
           'key': "%s_uid:list" % listbox_field_id,
-          'value': document_uid
+          'value': contents_uid
         }
+      # render whole field in contents_item or at least search result value 
       for select in select_list:
         if editable_field_dict.has_key(select):
-          REQUEST.set('cell', sql_document)
+          REQUEST.set('cell', search_result)
+  
+          # if default value is given by evaluating Tales expression then we only
+          # put "cell" to request (expected by tales) and let the field evaluate
+          default_field_value = None
+          if getattr(editable_field_dict[select].tales, "default", "") == "":
+            # if there is no tales expr we extract the value from search result
+            default_field_value = search_property_getter(search_result, select)
+          contents_item[select] = renderField(
+            traversed_document,
+            editable_field_dict[select],
+            listbox_form, 
+            value=default_field_value,
+            key='field_%s_%s' % (editable_field_dict[select].id, contents_uid))
+          REQUEST.other.pop('cell', None)
 
-          if ('default' in editable_field_dict[select].tales):
-            tmp_value = None
         else:
-            tmp_value = getProtectedProperty(document, select)
+          contents_value = search_property_getter(search_result, select)
 
-          property_value = renderField(
-            traversed_document, editable_field_dict[select], form, tmp_value,
-            key='field_%s_%s' % (editable_field_dict[select].id, document_uid))
-          REQUEST.other.pop('cell', None)
-        else:
-          property_value = getProtectedProperty(document, select)
-        if property_value is not None:
-          if same_type(property_value, DateTime()):
+          if contents_value is not None:
+            if same_type(contents_value, DateTime()):
               # Serialize DateTime
-            property_value = property_value.rfc822()
-          elif isinstance(property_value, datetime.date):
-            property_value = formatdate(time.mktime(property_value.timetuple()))
-          elif getattr(property_value, 'translate', None) is not None:
-            property_value = "%s" % property_value
-          document_result[select] = property_value
-      result_list.append(document_result)
-    result_dict['_embedded'] = {"contents": result_list}
-  
-    result_dict['_query'] = query
-    result_dict['_local_roles'] = local_roles
-    result_dict['_limit'] = limit
-    result_dict['_select_list'] = select_list
+              contents_value = contents_value.rfc822()
+            elif isinstance(contents_value, datetime.date):
+              contents_value = formatdate(time.mktime(contents_value.timetuple()))
+            elif getattr(contents_value, 'translate', None) is not None:
+              contents_value = "%s" % contents_value
+
+          contents_item[select] = contents_value
+
 
   elif mode == 'form':
     #################################################