[hal_json + web_renderjs_ui]: Generate URLs using URL columns for renderJS UI

Also, change the JSON format and update tests according to these changes

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_restricted_style/ERP5Document_getHateoas.py

@@ -24,5 +24,6 @@ return context.ERP5Document_getHateoas(
   sort_on=sort_on,
   local_roles=local_roles,
   selection_domain=selection_domain,
+  extra_param_json=extra_param_json,
   restricted=1
 )

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_restricted_style/ERP5Document_getHateoas.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>REQUEST=None, response=None, view=None, mode=\'root\', query=None, select_list=None, limit=10, local_roles=None, form=None, relative_url=None, list_method=None, default_param_json=None, form_relative_url=None, bulk_list="[]", sort_on=None, selection_domain=None</string> </value>
+            <value> <string>REQUEST=None, response=None, view=None, mode=\'root\', query=None, select_list=None, limit=10, local_roles=None, form=None, relative_url=None, list_method=None, default_param_json=None, form_relative_url=None, bulk_list="[]", sort_on=None, selection_domain=None, extra_param_json=None</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.py

 """Hello. This will be long because this godly script does almost everything.
 
-In general, it always returns a JSON reponse in HATEOAS format specification.
+In general, it always returns a JSON response in HATEOAS format specification.
 
 :param REQUEST: HttpRequest holding GET and/or POST data
 :param response:
@@ -413,6 +413,8 @@ url_template_dict = {
                        "&relative_url=%(relative_url)s&view=%(view)s",
   "traverse_generator_non_view": "%(root_url)s/%(script_id)s?mode=traverse" + \
                        "&relative_url=%(relative_url)s&view=%(view)s&form_id=%(form_id)s",
+  "traverse_generator_with_parameter": "%(root_url)s/%(script_id)s?mode=traverse" + \
+                       "&relative_url=%(relative_url)s&view=%(view)s&extra_param_json=%(extra_param_json)s",
   "traverse_template": "%(root_url)s/%(script_id)s?mode=traverse" + \
                        "{&relative_url,view}",
   "search_template": "%(root_url)s/%(script_id)s?mode=search" + \
@@ -471,7 +473,7 @@ def getFieldDefault(form, field, key, value=None):
   return value
 
 
-def renderField(traversed_document, field, form, value=None, meta_type=None, key=None, key_prefix=None, selection_params=None, request_field=True):
+def renderField(traversed_document, field, form, value=None, meta_type=None, key=None, key_prefix=None, selection_params=None, request_field=True, extra_param_json=None):
   """Extract important field's attributes into `result` dictionary."""
   if selection_params is None:
     selection_params = {}
@@ -750,6 +752,10 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
     # listbox's default parameters
     default_params.update(selection_params)
 
+    if extra_param_json is not None:
+      default_params.update(ensureDeserialized(byteify(
+        json.loads(urlsafe_b64decode(extra_param_json)))))
+
     # ListBoxes in report view has portal_type defined already in default_params
     # in that case we prefer non_empty version
     list_method_query_dict = default_params.copy()
@@ -913,7 +919,7 @@ def renderField(traversed_document, field, form, value=None, meta_type=None, key
   return result
 
 
-def renderForm(traversed_document, form, response_dict, key_prefix=None, selection_params=None):
+def renderForm(traversed_document, form, response_dict, key_prefix=None, selection_params=None, extra_param_json=None):
   """
   Render a `form` in plain python dict.
 
@@ -996,7 +1002,7 @@ def renderForm(traversed_document, form, response_dict, key_prefix=None, selecti
       if not field.get_value("enabled"):
         continue
       try:
-        response_dict[field.id] = renderField(traversed_document, field, form, key_prefix=key_prefix, selection_params=selection_params)
+        response_dict[field.id] = renderField(traversed_document, field, form, key_prefix=key_prefix, selection_params=selection_params, extra_param_json=extra_param_json)
         if field_errors.has_key(field.id):
           response_dict[field.id]["error_text"] = field_errors[field.id].error_text
       except AttributeError:
@@ -1371,8 +1377,8 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           return traversed_document.Base_redirect(keep_items={
             'portal_status_message': status_message})
 
-        renderForm(traversed_document, renderer_form, embedded_dict)
 
+        renderForm(traversed_document, renderer_form, embedded_dict, extra_param_json=extra_param_json)
         result_dict['_embedded'] = {
           '_view': embedded_dict
         }
@@ -1660,6 +1666,7 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
 
     # extract form field definition into `editable_field_dict`
     editable_field_dict = {}
+    url_column_dict = {}
     listbox_form = None
     listbox_field_id = None
     source_field_meta_type = source_field.meta_type if source_field is not None else ""
@@ -1670,6 +1677,17 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       listbox_field_id = source_field.id
       listbox_form = getattr(traversed_document, source_field.aq_parent.id)
 
+      url_column_dict = dict(source_field.get_value('url_columns'))
+
+      # support only selection_name for stat methods&url columns because any `selection` is deprecated
+      # and should be removed. Selection_name can be passed in catalog_kw by e.g. reports so it has precedence.
+      # Romain wants full backward compatibility so putting `selection` back in parameters
+      selection_name = catalog_kw.get('selection_name', source_field.get_value('selection_name'))
+      if selection_name and 'selection_name' not in catalog_kw:
+        catalog_kw['selection_name'] = selection_name
+      if 'selection' not in catalog_kw:
+        catalog_kw['selection'] = context.getPortalObject().portal_selections.getSelectionFor(selection_name, REQUEST)
+
       # field TALES expression evaluated by Base_getRelatedObjectParameter requires that
       REQUEST.other['form_id'] = listbox_form.id
 
@@ -1753,6 +1771,7 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       # put "cell" to request (expected by tales) and let the field evaluate
       REQUEST.set('cell', search_result)
       for select in select_list:
+        contents_item[select] = {}
         default_field_value = None
         # every `select` can have a template field or be just a exotic getter for a value
         if editable_field_dict.has_key(select):
@@ -1777,6 +1796,108 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
           # given search_result. This name can unfortunately mean almost anything from
           # a key name to Python Script with variable number of input parameters.
           contents_item[select] = getAttrFromAnything(search_result, select, property_getter, {'brain': search_result})
+
+        # If the contents_item has field rendering in it, better is to add an
+        # extra layer of abstraction to not get conflicts
+        if isinstance(contents_item[select], dict):
+          contents_item[select] = {
+            'field_gadget_param': contents_item[select],
+          }
+
+        # By default, we won't be generating views in the URL
+        generate_view = False
+        url_parameter_dict = {}
+        if select in url_column_dict:
+          # Check if we get URL parameters using listbox field `url_columns`
+          url_column_method = getattr(search_result, url_column_dict[select], None)
+          # If there is empty url_column_method, do nothing and continue. This
+          # will create no URL in these cases
+          if url_column_method is None:
+            continue
+          url_parameter_dict = url_column_method(
+                                      url_dict=True,
+                                      brain=search_result,
+                                      selection=catalog_kw['selection'],
+                                      selection_name=catalog_kw['selection_name'],
+                                      column_id=select)
+          # Since, now we are using url_columns for both XHTML UI and renderJS UI,
+          # its normal to get string as a result of the `url_column_method`
+          # function call. In that cases, we will do nothing. Take note, the
+          # result of `url_column_method` function call which is usable here should
+          # be dictionary in the format :-
+          # {
+          #   'command': <command_name, ex: 'raw', 'push_history'>,
+          #   'options': {
+          #               'url': <Absolute URL>,
+          #               'jio_key': <Relative URL of object>,
+          #               'view': <id of the view>,
+          #               }
+          # }
+          if isinstance(url_parameter_dict, str):
+            continue
+
+        elif getattr(search_result, 'getListItemUrlDict', None) is not None:
+          # Check if we can get URL result from the brain
+          try:
+            url_parameter_dict = search_result.getListItemUrlDict(
+                                                    select,
+                                                    result_index,
+                                                    catalog_kw['selection_name']
+                                                    )
+          except (ConflictError, RuntimeError):
+            raise
+          except:
+            log('could not evaluate the url method getListItemUrlDict with %r' % search_result,
+                level=800)
+            continue
+
+        else:
+          # Continue in case there is no url_column_dict or brain to get URL
+          continue
+
+        url_result_dict = {
+          select: url_parameter_dict
+        }
+
+        # If contents item don't have `field_gadget_param` in it, then add it
+        # to default
+        if not isinstance(contents_item[select], dict):
+          contents_item[select] = {
+            'default': contents_item[select],
+          }
+
+        contents_item[select].update({'url_value': url_result_dict[select]})
+
+        if contents_item[select]['url_value']:
+
+          # We should be generating view if there is extra params for view in
+          # view_kw. These parameters are required to create url at hateoas side
+          # using the URL template as necessary
+          if 'view_kw' in contents_item[select]['url_value']:
+            generate_view = True
+            # Get extra parameters either from url_result_dict or from brain
+            extra_url_param_dict = contents_item[select]['url_value']['view_kw'].get('extra_param_json', {})
+
+          if generate_view:
+            url_template_id = 'traverse_generator'
+            if extra_url_param_dict:
+              url_template_id = 'traverse_generator_with_parameter'
+
+            contents_item[select]['url_value']['options']['view'] =\
+              url_template_dict[url_template_id] % {
+                "root_url": site_root.absolute_url(),
+                "script_id": script.id,
+                "relative_url": contents_item[select]['url_value']['view_kw']['jio_key'].replace("/", "%2F"),
+                "view": contents_item[select]['url_value']['view_kw']['view'],
+                "extra_param_json":  urlsafe_b64encode(
+                  json.dumps(ensureSerializable(extra_url_param_dict)))
+              }
+
+          # Its better to remove the 'view_kw' from the dictionary as it doesn't
+          # serve any purpose further in the result_dict
+          if 'view_kw' in contents_item[select]['url_value']:
+            del contents_item[select]['url_value']['view_kw']
+
       # endfor select
       REQUEST.other.pop('cell', None)
       contents_list.append(contents_item)
@@ -1809,6 +1930,8 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       if count_method != "" and count_method.getMethodName() != list_method:
         count_kw = dict(catalog_kw)
         # Drop not needed parameters
+        count_kw.pop('selection', None)
+        count_kw.pop('selection_name', None)
         count_kw.pop("sort_on", None)
         count_kw.pop("limit", None)
         count_method_result = getattr(traversed_document, count_method.getMethodName())(REQUEST=REQUEST, **count_kw)
@@ -1820,14 +1943,6 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
       # XXX: we should check whether they asked for it
       stat_method = source_field.get_value('stat_method')
       stat_columns = source_field.get_value('stat_columns')
-      # support only selection_name for stat methods because any `selection` is deprecated
-      # and should be removed. Selection_name can be passed in catalog_kw by e.g. reports so it has precedence.
-      # Romain wants full backward compatibility so putting `selection` back in parameters
-      selection_name = catalog_kw.get('selection_name', source_field.get_value('selection_name'))
-      if selection_name and 'selection_name' not in catalog_kw:
-        catalog_kw['selection_name'] = selection_name
-      if 'selection' not in catalog_kw:
-        catalog_kw['selection'] = context.getPortalObject().portal_selections.getSelectionFor(selection_name, REQUEST)
 
       contents_stat = {}
       if len(stat_columns) > 0:

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.xml

@@ -56,7 +56,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>REQUEST=None, response=None, view=None, mode=\'root\', query=None, select_list=None, limit=10, local_roles=None, form=None, relative_url=None, restricted=0, list_method=None, default_param_json=None, form_relative_url=None, bulk_list="[]", sort_on=None, selection_domain=None</string> </value>
+            <value> <string>REQUEST=None, response=None, view=None, mode=\'root\', query=None, select_list=None, limit=10, local_roles=None, form=None, relative_url=None, restricted=0, list_method=None, default_param_json=None, form_relative_url=None, bulk_list="[]", sort_on=None, selection_domain=None, extra_param_json=None</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_hal_json_style/TestTemplateItem/portal_components/test.erp5.testHalJsonStyle.py

@@ -1182,8 +1182,8 @@ return context.getPortalObject().foo_module.contentValues()
     )
     result_dict = json.loads(result)
     #editalble creation date is defined at proxy form
-    self.assertEqual(result_dict['_embedded']['contents'][0]['creation_date']['type'], 'DateTimeField')
-    self.assertEqual(result_dict['_embedded']['contents'][0]['modification_date']['type'], 'DateTimeField')
+    self.assertEqual(result_dict['_embedded']['contents'][0]['creation_date']['field_gadget_param']['type'], 'DateTimeField')
+    self.assertEqual(result_dict['_embedded']['contents'][0]['modification_date']['field_gadget_param']['type'], 'DateTimeField')
     # There is a count method on this listbox
     self.assertEqual(result_dict['_embedded']['count'], 0)
 
@@ -1241,12 +1241,12 @@ return context.getPortalObject().portal_catalog(portal_type='Foo', sort_on=[('id
     )
     result_dict = json.loads(result)
     self.assertEqual(2, len(result_dict['_embedded']['contents']))
-    self.assertIn("field_listbox", result_dict['_embedded']['contents'][0]['id']['key'])
-    self.assertEqual("StringField", result_dict['_embedded']['contents'][0]['id']['type'])
-    self.assertEqual(document_list[0].getId(), result_dict['_embedded']['contents'][0]['id']['default'])
-    self.assertIn("field_listbox", result_dict['_embedded']['contents'][1]['id']['key'])
-    self.assertEqual("StringField", result_dict['_embedded']['contents'][1]['id']['type'])
-    self.assertEqual(document_list[1].getId(), result_dict['_embedded']['contents'][1]['id']['default'])
+    self.assertIn("field_listbox", result_dict['_embedded']['contents'][0]['id']['field_gadget_param']['key'])
+    self.assertEqual("StringField", result_dict['_embedded']['contents'][0]['id']['field_gadget_param']['type'])
+    self.assertEqual(document_list[0].getId(), result_dict['_embedded']['contents'][0]['id']['field_gadget_param']['default'])
+    self.assertIn("field_listbox", result_dict['_embedded']['contents'][1]['id']['field_gadget_param']['key'])
+    self.assertEqual("StringField", result_dict['_embedded']['contents'][1]['id']['field_gadget_param']['type'])
+    self.assertEqual(document_list[1].getId(), result_dict['_embedded']['contents'][1]['id']['field_gadget_param']['default'])
     # There is a count method on the listbox
     self.assertEqual(result_dict['_embedded']['count'], 0)
 

bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_listbox_html.html

@@ -77,12 +77,20 @@
                   {{#if editable}}
                     <div class="editable_div" data-column="{{column}}" data-line="{{line}}"></div>
                   {{else}}
+                    {{#if href}}
                       <a href="{{href}}" class="ui-link">
                         <div class="editable_div" data-column="{{column}}" data-line="{{line}}"></div>
                       </a>
+                     {{else}}
+                       <div class="editable_div" data-column="{{column}}" data-line="{{line}}"></div>
+                     {{/if}}
                   {{/if}}
                 {{else}}
+                  {{#if href}}
                     <a href="{{href}}" class="ui-link">{{default}}</a>
+                  {{else}}
+                    <p>{{default}}</p>
+                  {{/if}}
                 {{/if}}
               </td>
            {{/each}}
@@ -117,12 +125,20 @@
                   {{#if editable}}
                     <div class="editable_div" data-column="{{column}}" data-line="{{line}}"></div>
                   {{else}}
+                    {{#if href}}
                       <a href="{{href}}" class="ui-link">
                         <div class="editable_div" data-column="{{column}}" data-line="{{line}}"></div>
                       </a>
+                    {{else}}
+                      <div class="editable_div" data-column="{{column}}" data-line="{{line}}"></div>
+                    {{/if}}
                   {{/if}}
                 {{else}}
+                  {{#if href}}
                     <a href="{{href}}" class="ui-link">{{default}}</a>
+                  {{else}}
+                    <p>{{default}}</p>
+                  {{/if}}
                 {{/if}}
               </td>
            {{/each}}

bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_listbox_html.xml

@@ -234,7 +234,7 @@
             </item>
             <item>
                 <key> <string>serial</string> </key>
-                <value> <string>965.63630.22597.59392</string> </value>
+                <value> <string>966.44073.11272.27409</string> </value>
             </item>
             <item>
                 <key> <string>state</string> </key>
@@ -252,7 +252,7 @@
                     </tuple>
                     <state>
                       <tuple>
-                        <float>1520243676.37</float>
+                        <float>1523002868.97</float>
                         <string>UTC</string>
                       </tuple>
                     </state>

bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_listbox_js.js

@@ -468,9 +468,11 @@
           .push(function () {
             var lines = gadget.state.lines,
               promise_list = [],
+              url_promise_list = [],
               allDocs_result = gadget.state.allDocs_result,
               counter,
-              pagination_message = '';
+              pagination_message = '',
+              content_value;
 
             column_list = JSON.parse(gadget.state.column_list_json);
             // for actual allDocs_result structure see ref:gadget_erp5_jio.js
@@ -496,17 +498,35 @@
                   }
                 })
               );
+              for (j = 0; j < column_list.length; j += 1) {
+                content_value = allDocs_result.data.rows[i].value[column_list[j][0]] || "";
+                if (content_value.url_value) {
+                  if (content_value.url_value.command) {
+                    url_promise_list.push(
+                      gadget.getUrlFor(content_value.url_value)
+                    );
+                  } else {
+                    url_promise_list.push(false);
+                  }
+                }
+              }
             }
             return new RSVP.Queue()
               .push(function () {
-                return RSVP.all(promise_list);
+                return RSVP.all([
+                  RSVP.all(promise_list),
+                  RSVP.all(url_promise_list)
+                ]);
               })
-
-              .push(function (line_link_list) {
+              .push(function (result_list) {
                 var row_list = [],
                   value,
                   cell_list,
+                  url_value,
+                  index = 0,
                   listbox_tbody_template,
+                  line_link_list = result_list[0],
+                  url_column_list = result_list[1],
                   setNonEditable = function (cell) {cell.editable = false; };
                 // reset list of UIDs of editable sub-documents
                 gadget.props.listbox_uid_dict = {
@@ -520,15 +540,35 @@
                   cell_list = [];
                   for (j = 0; j < column_list.length; j += 1) {
                     value = allDocs_result.data.rows[i].value[column_list[j][0]] || "";
-                    // value can be simply just a value in case of non-editable field
-                    // thus we construct "field_json" manually and insert the value in "default"
-                    if (value.constructor !== Object) {
+                     //url column
+                    // get url value
+                    if (value.url_value) {
+                      url_value = url_column_list[index];
+                      index += 1;
+                    } else {
+                      url_value = line_link_list[i];
+                    }
+                    // We need to check for field_gadget_param and then update
+                    // value accordingly. value can be simply just a value in
+                    // case of non-editable field thus we construct "field_json"
+                    // manually and insert the value in "default"
+
+                    if (value.constructor === Object) {
+                      if (value.field_gadget_param) {
+                        value = value.field_gadget_param;
+                      } else {
+                        value = {
+                          'editable': 0,
+                          'default': value.default
+                        };
+                      }
+                    } else {
                       value = {
                         'editable': 0,
                         'default': value
                       };
                     }
-                    value.href = line_link_list[i];
+                    value.href = url_value;
                     value.editable = value.editable && gadget.state.editable;
                     value.line = i;
                     value.column = j;

bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_gadget_erp5_listbox_js.xml

@@ -236,7 +236,7 @@
             </item>
             <item>
                 <key> <string>serial</string> </key>
-                <value> <string>966.8130.3423.18312</string> </value>
+                <value> <string>966.54628.39321.22579</string> </value>
             </item>
             <item>
                 <key> <string>state</string> </key>
@@ -254,8 +254,8 @@
                     </tuple>
                     <state>
                       <tuple>
-                        <float>1520872134.75</float>
-                        <string>UTC</string>
+                        <float>1523635564.65</float>
+                        <string>GMT+2</string>
                       </tuple>
                     </state>
                   </object>