erp5_tabular_data_tool: Add portal_tabular (Tabular Data Tool)

Tabular Data Tool is a Tool to handle tabular data (2-dimentional table data)
in erp5.
The tabular data type is currently a plain pandas DataFrame. It is intended to
use with ERP5 Reports to create spread sheets (ODS/Excel).

With Tabular Data Tool, we can do those things on reporting:

- transform dict-list, portal-catalog-result and document-list into a tabular data
- joining/grouping/sorting the tabular data
- using vectorizing function to optimise time consuming reports

bt5/erp5_tabular_data_tool/DocumentTemplateItem/portal_components/tool.erp5.TabularDataTool.py

+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type import Permissions
+from Products.ERP5Type.Globals import InitializeClass
+from Products.ERP5Type.Tool.BaseTool import BaseTool
+import pandas as pd
+
+MARKER = object()
+DEFAULT_FILL_STRING = ''
+DEFAULT_FILL_NUMBER = 0.0
+DEFAULT_FILL_UID = -1
+
+class TabularDataTool(BaseTool):
+  """
+  Tabular Data Tool is a Tool to handle tabular data (2-dimentional table data) in erp5.
+  The tabular data type is currently a plain pandas DataFrame. It is intended to
+  use with ERP5 Reports to create spread sheets (ODS/Excel).
+
+  With Tabular Data Tool, we can do those things on reporting:
+  - transform dict-list, portal-catalog-result and document-list into a tabular data
+  - joining/grouping/sorting the tabular data
+  - using vectorizing function to optimise time consuming reports
+  """
+  id = 'portal_tabular'
+  title = 'Tabular Data Tool'
+  meta_type = 'ERP5 Tabular Data Tool'
+  portal_type = 'Tabular Data Tool'
+  allowed_types = ()
+
+
+  # Declarative Security
+  security = ClassSecurityInfo()
+
+  # The empty DataFrame marker
+  def _createEmptyDataFrame(self):
+    return pd.DataFrame()
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'fillna')
+  def fillna(self, tabular, fillna_on_string=MARKER, fillna_on_number=MARKER, fillna_on_uid=MARKER):
+    """
+    Fill NA/NaN values on the given tabular (a dataframe).
+
+    Keyword arguments:
+    tabular -- the dataframe to fill empty values
+    fillna_on_string -- Fill given value for empty string
+    fillna_on_number -- Fill given value for empty numbers (int or float)
+    fillna_on_uid -- Fill given value for empty uid, which overrides fillna_on_numbers for uid
+    """
+    if tabular is None:
+      return tabular
+    string_filler = DEFAULT_FILL_STRING if fillna_on_string is MARKER else fillna_on_string
+    number_filler = DEFAULT_FILL_NUMBER if fillna_on_number is MARKER else fillna_on_number
+    uid_filler = DEFAULT_FILL_UID if fillna_on_uid is MARKER else fillna_on_uid
+    fillna_setting_dict = {}
+    string_column_list = tabular.select_dtypes(include=[object]).columns.tolist()
+    string_none_column_list = []
+    if fillna_on_string is None:
+      # pandas.dataframe.fillna(None) raises error, so use .replace() instead
+      string_none_column_list = string_column_list
+      string_column_list = []
+    number_column_list = tabular.select_dtypes(include=[int,float]).columns.tolist()
+    uid_column_list = [column for column in tabular.columns.tolist() if column.endswith('_uid')]
+
+    number_column_list = list(set(number_column_list) - set(uid_column_list))
+    string_column_list = list(set(string_column_list) - set(uid_column_list))
+    fillna_setting_dict.update(dict.fromkeys(string_column_list, string_filler))
+    fillna_setting_dict.update(dict.fromkeys(number_column_list, number_filler))
+    fillna_setting_dict.update(dict.fromkeys(uid_column_list, uid_filler))
+    filled_tabular = tabular.fillna(value=fillna_setting_dict)
+    if string_none_column_list:
+      replace_dict = dict.fromkeys(string_none_column_list, {"":None})
+      filled_tabular = filled_tabular.replace(replace_dict)
+    return filled_tabular
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'getTabular')
+  def getTabular(self, data_list,
+                 fillna_on_string=MARKER,
+                 fillna_on_number=MARKER,
+                 fillna_on_uid=MARKER,
+                 additional_property_name_list=MARKER):
+    """
+    Create a tabular from dict list with filling values for convenience in ERP5.
+    Currently the tabular data type is pandas DataFrame.
+
+    Keyword arguments:
+    data_list -- A dict list or brain_list to convert it to dataframe
+    fillna_on_string -- Fill given value for empty string
+    fillna_on_numbers -- Fill given value for empty numbers (int or float)
+    fillna_on_uid -- Fill given value for empty uid, which overrides fillna_on_numbers for uid
+    additional_property_name_list -- Additional property name list of brain object
+    """
+    if data_list is None or len(data_list) == 0:
+      return self._createEmptyDataFrame()
+    df = None
+    if isinstance(data_list[0], dict):
+      df = pd.DataFrame.from_dict(data_list)
+
+    # Expect Shared.ZRDB.Results
+    elif getattr(data_list, 'dictionaries', None) is not None:
+      if additional_property_name_list is not MARKER:
+        new_data_dict_list = []
+        for (data_, record_dict) in zip(data_list, data_list.dictionaries()):
+          property_dict = self._getPropertyDict(data_, additional_property_name_list, False, False)
+          property_dict.update(record_dict)
+          new_data_dict_list.append(property_dict)
+        df = self.getTabular(new_data_dict_list)
+      else:
+        df = self.getTabular(data_list.dictionaries())
+    if df is not None:
+      return self.fillna(df, fillna_on_string=fillna_on_string,
+                         fillna_on_number=fillna_on_number,
+                         fillna_on_uid=fillna_on_uid)
+    raise ValueError("The type is not supported, {}".format(data_list))
+
+  def _getPropertyDict(self, document, property_name_list, is_category_as_uid, add_category_title):
+    portal_type = document.getPortalType()
+    base_type = self.getPortalObject().portal_types[portal_type]
+    base_category_list = base_type.getInstanceBaseCategoryList()
+    if property_name_list is MARKER:
+      property_name_list = list(base_type.getInstancePropertySet())
+    else:
+      # If property_name_list is explicitly passed, respect the name list in category
+      base_category_list = list(set(base_category_list).intersection(set(property_name_list)))
+    property_dict = {}
+    for property_name in property_name_list:
+      property_dict[property_name] = document.getProperty(property_name)
+    for base_category_id in base_category_list:
+      category_name = "{}_uid".format(base_category_id) if is_category_as_uid else base_category_id
+      property_dict[category_name] = document.getProperty(category_name)
+      if add_category_title:
+        category_title = "{}_title".format(base_category_id)
+        property_dict[category_title] = document.getProperty(category_title)
+    return property_dict
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'getFromDocumentList')
+  def getFromDocumentList(self, document_list,
+                          property_name_list=MARKER,
+                          fillna_on_string=MARKER,
+                          fillna_on_number=MARKER,
+                          fillna_on_uid=MARKER,
+                          is_category_as_uid=True,
+                          add_category_title=True):
+    """
+    Create tabular data from document list
+
+    Keyword arguments:
+    property_name_list -- Explict to specify the property name list of document
+                          If not specified all properties and categories of the portal type is applied
+    fillna_on_string -- Fill given value for empty string
+    fillna_on_numbers -- Fill given value for empty numbers (int or float)
+    fillna_on_uid -- Fill given value for empty uid, which overrides fill_on_numbers for uid
+    is_category_as_uid -- Store uid on {base_category}_uid, instead of store the url on {base_category}
+    add_category_title -- Add {category}_title columns into the tabular
+
+    Note: It takes O(N) time.
+    """
+    if document_list is None or len(document_list) == 0:
+      return self._createEmptyDataFrame()
+    # raise Attribute error if given document does not have portal_type
+    portal_type_set = set([x.getPortalType() for x in document_list])
+    if len(portal_type_set) != 1:
+      raise ValueError('Given documents do not have the same portal type:{}'.format(portal_type_set))
+    dict_list = [self._getPropertyDict(x, property_name_list,
+                                       is_category_as_uid,
+                                       add_category_title) for x in document_list]
+    return self.getTabular(dict_list,
+                           fillna_on_string=fillna_on_string,
+                           fillna_on_number=fillna_on_number,
+                           fillna_on_uid=fillna_on_uid)
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'searchResults')
+  def searchResults(self, select_dict=None,
+                   fillna_on_string=MARKER,
+                   fillna_on_number=MARKER,
+                   fillna_on_uid=MARKER, **kw):
+    """
+    Create tabular data based on the portal_catalog.searchResult()
+
+    select_dict -- If select_dict is None, specify all the column ids of catalog table.
+    """
+    if select_dict is None:
+      portal = self.getPortalObject()
+      catalog_id = portal.portal_catalog.getDefaultErp5CatalogId()
+      default_erp5_catalog = portal.portal_catalog[catalog_id]
+      column_id_set = set(default_erp5_catalog.getResultColumnIds())
+      sql_search_result_keys_set = set(default_erp5_catalog.getSqlSearchResultKeysList())
+      all_column_id_set = column_id_set - sql_search_result_keys_set
+      select_dict = dict([(column_id, None) for column_id in all_column_id_set if column_id.startswith('catalog.')])
+    search_result = portal.portal_catalog(select_dict=select_dict, **kw)
+    return self.getTabular(search_result,
+                            fillna_on_string=fillna_on_string,
+                            fillna_on_number=fillna_on_number,
+                            fillna_on_uid=fillna_on_uid)
+
+  __call__ = getTabular
+
+InitializeClass(TabularDataTool)
\ No newline at end of file

bt5/erp5_tabular_data_tool/DocumentTemplateItem/portal_components/tool.erp5.TabularDataTool.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Tool Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>TabularDataTool</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>tool.erp5.TabularDataTool</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Tool Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_tabular_data_tool/PortalTypeTemplateItem/portal_types/Tabular%20Data%20Tool.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Base Type" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>acquire_local_roles</string> </key>
+            <value> <int>1</int> </value>
+        </item>
+        <item>
+            <key> <string>content_icon</string> </key>
+            <value> <string>folder_icon.gif</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value> <string>Tabular Data Tool is for handling tabular data (2-dimentional table data) .</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>Tabular Data Tool</string> </value>
+        </item>
+        <item>
+            <key> <string>init_script</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>permission</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Base Type</string> </value>
+        </item>
+        <item>
+            <key> <string>type_class</string> </key>
+            <value> <string>TabularDataTool</string> </value>
+        </item>
+        <item>
+            <key> <string>type_interface</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>type_mixin</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_tabular_data_tool/ToolTemplateItem/portal_tabular.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Tabular Data Tool" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>portal_tabular</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Tabular Data Tool</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_tabular_data_tool/bt/dependency_list

+erp5_base
\ No newline at end of file

bt5/erp5_tabular_data_tool/bt/template_document_id_list

+tool.erp5.TabularDataTool
\ No newline at end of file

bt5/erp5_tabular_data_tool/bt/template_format_version

+1
\ No newline at end of file

bt5/erp5_tabular_data_tool/bt/template_portal_type_id_list

+Tabular Data Tool
\ No newline at end of file

bt5/erp5_tabular_data_tool/bt/template_tool_id_list

+portal_tabular
\ No newline at end of file

bt5/erp5_tabular_data_tool/bt/template_update_tool

+1
\ No newline at end of file

bt5/erp5_tabular_data_tool/bt/title

+erp5_tabular_data_tool
\ No newline at end of file

bt5/erp5_tabular_data_tool_test/TestTemplateItem/portal_components/test.erp5.testTabularDataTool.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Test Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>testTabularDataTool</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>test.erp5.testTabularDataTool</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Test Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_tabular_data_tool_test/bt/dependency_list

+erp5_trade
+erp5_full_text_mroonga_catalog
+erp5_tabular_data_tool
\ No newline at end of file

bt5/erp5_tabular_data_tool_test/bt/template_format_version

+1
\ No newline at end of file

bt5/erp5_tabular_data_tool_test/bt/template_test_id_list

+test.erp5.testTabularDataTool
\ No newline at end of file

bt5/erp5_tabular_data_tool_test/bt/title

+erp5_tabular_data_tool_test
\ No newline at end of file