Commit 072d0780 authored by Xiaowu Zhang's avatar Xiaowu Zhang

Data mapping

See merge request nexedi/wendelin!104
parents fa798a82 09f481a6
Pipeline #25390 failed with stage
in 0 seconds
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ActionInformation" module="Products.CMFCore.ActionInformation"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>action_type/object_list</string>
</tuple>
</value>
</item>
<item>
<key> <string>category</string> </key>
<value> <string>object_list</string> </value>
</item>
<item>
<key> <string>condition</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>icon</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>view</string> </value>
</item>
<item>
<key> <string>permissions</string> </key>
<value>
<tuple>
<string>View</string>
</tuple>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Action Information</string> </value>
</item>
<item>
<key> <string>priority</string> </key>
<value> <float>1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>View</string> </value>
</item>
<item>
<key> <string>visible</string> </key>
<value> <int>1</int> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Expression" module="Products.CMFCore.Expression"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>text</string> </key>
<value> <string>string:${object_url}/DataMappingModule_viewDataMappingList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ActionInformation" module="Products.CMFCore.ActionInformation"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>action_type/object_view</string>
</tuple>
</value>
</item>
<item>
<key> <string>category</string> </key>
<value> <string>object_view</string> </value>
</item>
<item>
<key> <string>condition</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>icon</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>view</string> </value>
</item>
<item>
<key> <string>permissions</string> </key>
<value>
<tuple>
<string>View</string>
</tuple>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Action Information</string> </value>
</item>
<item>
<key> <string>priority</string> </key>
<value> <float>1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>View</string> </value>
</item>
<item>
<key> <string>visible</string> </key>
<value> <int>1</int> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Expression" module="Products.CMFCore.Expression"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>text</string> </key>
<value> <string>string:${object_url}/DataMapping_view</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
# Xiaowu Zhang <xiaowu.zhang@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from BTrees.OLBTree import OLBTree
from BTrees.LOBTree import LOBTree
from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet
from erp5.component.document.Document import Document
class DataMapping(Document):
"""
data mapping is used to map complex data into a single value
the idea is if an object has complet data to process, like (object, value1, value2, value3,value4....), we can mapping (object, value1, value2, value3,value4....) into a single value X, then we process only X to make it faster
here is an use case:
we have two 2D data arrays with 5 columns, the first column is the name of object, the other fours are the differents value of this object
Data ArrayA:
| object | value1 | value2 | value3 | value4 |
| ------ | ------ | ------ | ------ | ------ |
| X | 1 | 2 | 3 | 4 |
| Y | 5 | 6 | 7 | 8 |
| Z | 9 | 10 | 11 | 12 |
Data ArrayB:
| object | value1 | value2 | value3 | value4 |
| ------ | ------ | ------ | ------ | ------ |
| X | 1 | 2 | 3 | 4 |
| Y | 5 | 8 | 7 | 8 |
| Z | 9 | 10 | 192 | 12 |
now we need to compare data array A to data array B to find which object inside A has different value.
without data mapping, we need to compare each object's 4 values, the complexity is O(2^n)
with data mapping:
we map those values:
(X, 1, 2, 3, 4) ==> 1
(Y, 5, 6, 7, 8) ==> 2
(Z, 9, 10, 11,12) ==>3
(Y, 5, 8, 7, 8) ==> 4
(Z, 9, 10, 192, 12) ==> 5
Data ArrayA:
| object |
| ------ |
| 1 |
| 2 |
| 3 |
Data ArrayB:
| object |
| ------ |
| 1 |
| 4 |
| 5 |
then compare 1D array is fast, the complexity is O(n)
"""
meta_type = 'ERP5 Data Mapping'
portal_type = 'Data Mapping'
add_permission = Permissions.AddPortalContent
# Declarative security
security = ClassSecurityInfo()
security.declareObjectProtected(Permissions.AccessContentsInformation)
# Declarative properties
property_sheets = (
PropertySheet.CategoryCore,
PropertySheet.SortIndex
)
def __init__(self, *args, **kw):
self._last_value = 0
self._object_to_index_tree = OLBTree()
self._index_to_object_tree = LOBTree()
Document.__init__(self, *args, **kw)
security.declareProtected(Permissions.AccessContentsInformation, 'addObject')
def addObject(self, obj):
if obj in self._object_to_index_tree:
return self._object_to_index_tree[obj]
self._object_to_index_tree[obj] = self._last_value
self._index_to_object_tree[self._last_value] = obj
self._last_value += 1
return self._object_to_index_tree[obj]
security.declareProtected(Permissions.AccessContentsInformation, 'getValueFromObject')
def getValueFromObject(self, obj):
if obj in self._object_to_index_tree:
return self._object_to_index_tree[obj]
else:
return None
security.declareProtected(Permissions.AccessContentsInformation, 'getObjectFromValue')
def getObjectFromValue(self, value):
if value in self._index_to_object_tree:
return self._index_to_object_tree[value]
else:
return None
security.declareProtected(Permissions.AccessContentsInformation, 'getData')
def getData(self):
data_list = []
for obj in self._object_to_index_tree.keys():
data_list.append((obj, self._object_to_index_tree[obj]))
return data_list
security.declareProtected(Permissions.AccessContentsInformation, 'getSize')
def getSize(self):
return len(self._object_to_index_tree.keys())
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Document Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>default_reference</string> </key>
<value> <string>DataMapping</string> </value>
</item>
<item>
<key> <string>default_source_reference</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>document.erp5.DataMapping</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Document Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<module>
<category_list>
<category>business_application/big_data</category>
</category_list>
<id>data_mapping_module</id>
<permission_list>
<permission type='tuple'>
<name>Access Transient Objects</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Access contents information</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Access session data</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Add portal content</name>
<role>Assignor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Add portal folders</name>
<role>Assignor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Change local roles</name>
<role>Assignor</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Copy or Move</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Delete objects</name>
<role>Assignor</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>List folder contents</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Author</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>Modify portal content</name>
<role>Assignor</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>View</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Manager</role>
</permission>
<permission type='tuple'>
<name>View History</name>
<role>Assignee</role>
<role>Assignor</role>
<role>Associate</role>
<role>Auditor</role>
<role>Author</role>
<role>Manager</role>
</permission>
</permission_list>
<portal_type>Data Mapping Module</portal_type>
<title>Data Mappings</title>
</module>
\ No newline at end of file
......@@ -40,6 +40,9 @@
<portal_type id="Data License Module">
<item>Data License</item>
</portal_type>
<portal_type id="Data Mapping Module">
<item>Data Mapping</item>
</portal_type>
<portal_type id="Data Operation Module">
<item>Data Operation</item>
</portal_type>
......
......@@ -11,15 +11,15 @@
<portal_type id="Data Array Module">
<item>business_application</item>
</portal_type>
<portal_type id="Data Array View Line">
<item>predecessor</item>
</portal_type>
<portal_type id="Data Configuration Module">
<item>business_application</item>
</portal_type>
<portal_type id="Data Descriptor Module">
<item>business_application</item>
</portal_type>
<portal_type id="Data Array View Line">
<item>predecessor</item>
</portal_type>
<portal_type id="Data Event">
<item>aggregate</item>
<item>destination</item>
......@@ -38,6 +38,9 @@
<portal_type id="Data License Module">
<item>business_application</item>
</portal_type>
<portal_type id="Data Mapping Module">
<item>business_application</item>
</portal_type>
<portal_type id="Data Operation Module">
<item>business_application</item>
</portal_type>
......@@ -72,4 +75,4 @@
<portal_type id="Progress Indicator Module">
<item>business_application</item>
</portal_type>
</base_category_list>
</base_category_list>
\ No newline at end of file
......@@ -12,6 +12,10 @@
<item>ConstraintType</item>
<item>DataArrayLine</item>
</portal_type>
<portal_type id="Data Array View Line">
<item>DataArrayLine</item>
<item>Document</item>
</portal_type>
<portal_type id="Data Bucket Stream">
<item>OffsetIndex</item>
</portal_type>
......@@ -19,10 +23,6 @@
<item>TextDocument</item>
<item>Url</item>
</portal_type>
<portal_type id="Data Array View Line">
<item>DataArrayLine</item>
<item>Document</item>
</portal_type>
<portal_type id="Data Event">
<item>DataEvent</item>
</portal_type>
......@@ -69,4 +69,4 @@
<portal_type id="Progress Indicator">
<item>OffsetIndex</item>
</portal_type>
</property_sheet_list>
</property_sheet_list>
\ No newline at end of file
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Base Type" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_property_domain_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>content_icon</string> </key>
<value> <string>folder_icon.gif</string> </value>
</item>
<item>
<key> <string>factory</string> </key>
<value> <string>addFolder</string> </value>
</item>
<item>
<key> <string>group_list</string> </key>
<value>
<tuple>
<string>module</string>
</tuple>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Data Mapping Module</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Base Type</string> </value>
</item>
<item>
<key> <string>type_class</string> </key>
<value> <string>Folder</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>short_title</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="TranslationInformation" module="Products.ERP5Type.TranslationProviderBase"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>domain_name</string> </key>
<value> <string>erp5_ui</string> </value>
</item>
<item>
<key> <string>property_name</string> </key>
<value> <string>short_title</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="TranslationInformation" module="Products.ERP5Type.TranslationProviderBase"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>domain_name</string> </key>
<value> <string>erp5_ui</string> </value>
</item>
<item>
<key> <string>property_name</string> </key>
<value> <string>title</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Base Type" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>content_icon</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>factory</string> </key>
<value> <string>addXMLObject</string> </value>
</item>
<item>
<key> <string>group_list</string> </key>
<value>
<tuple>
<string>item</string>
</tuple>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Data Mapping</string> </value>
</item>
<item>
<key> <string>init_script</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>permission</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Base Type</string> </value>
</item>
<item>
<key> <string>type_class</string> </key>
<value> <string>DataMapping</string> </value>
</item>
<item>
<key> <string>type_interface</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>type_mixin</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -23,10 +23,6 @@
<type>Data Array Line</type>
<workflow>edit_workflow</workflow>
</chain>
<chain>
<type>Data Bucket Stream</type>
<workflow>data_stream_interaction_workflow, edit_workflow, validation_workflow</workflow>
</chain>
<chain>
<type>Data Array View</type>
<workflow>edit_workflow, validation_workflow</workflow>
......@@ -35,6 +31,10 @@
<type>Data Array View Line</type>
<workflow>edit_workflow</workflow>
</chain>
<chain>
<type>Data Bucket Stream</type>
<workflow>data_stream_interaction_workflow, edit_workflow, validation_workflow</workflow>
</chain>
<chain>
<type>Data Event</type>
<workflow>edit_workflow</workflow>
......@@ -51,6 +51,10 @@
<type>Data License</type>
<workflow>edit_workflow, validation_workflow</workflow>
</chain>
<chain>
<type>Data Mapping</type>
<workflow>edit_workflow, validation_workflow</workflow>
</chain>
<chain>
<type>Data Operation</type>
<workflow>edit_workflow, validation_workflow</workflow>
......@@ -103,4 +107,4 @@
<type>Sensor</type>
<workflow>edit_workflow, validation_workflow</workflow>
</chain>
</workflow_chain>
</workflow_chain>
\ No newline at end of file
......@@ -759,4 +759,114 @@ result = [x for x in data_bucket_stream.getBucketIndexKeySequenceByIndex()]
for data_analysis_title in data_analysis_title_list:
data_analysis = getDataAnalysisByTitle(data_analysis_title)
self.assertNotEqual(data_analysis, None)
self.addCleanup(self._removeDocument, data_analysis)
\ No newline at end of file
self.addCleanup(self._removeDocument, data_analysis)
def test_17_DataMapping(self):
"""
"""
portal = self.portal
data_mapping = portal.data_mapping_module.newContent(portal_type='Data Mapping')
self.assertEqual(0, data_mapping.getSize())
data_list = [
('/usr/bin/2to3-2.7', '3ea002bead53f6bdf7', 'fade8568285eb14146a7244', 'f631570af55ee08ecef78f3'),
('/usr/bin/R', 'b4c48d52345ae2eb7ca0455db', '59441ddbc00b6521da571', 'a92be1a7acc03f3846'),
('/usr/bin/Rscript', 'e97842e556f90be5f7e5', '806725443a01bcae802','1829d887e0c3380ec8f463527'),
('/usr/bin/[', '5c348873d0e0abe26d56cc752f0', 'ebb7ae1a78018b62224','9162fdf5b9598c9e9d'),
('/usr/bin/aa-enabled', 'c2336b14b7e9d1407d9', 'c8c7af8e6d14a4d1061', '09726542c6c4ca907417c676b4'),
('/usr/bin/aa-exec', '7f66cd90da4b27703de299', 'b4bfe90f53a690deb2687342', 'ad36c9002fa84d4e86'),
('/usr/bin/aclocal-1.16', '6ba134fb4f97d79a5', '64e518309fc9544b01c164c2e48', 'cc4595fba3251aaa9b48'),
('/usr/bin/activate-global-python-argcomplete3', '4007899eba237603a', '8a2584480e9094f9e4761f7', '893ebeeca071795ecd457ed2'),
('/usr/bin/addpart', '078a10c409b8b21c3b5137d', '481766f1aa1d393e4a3a', '668374ce14a57b2dd14bb'),
('/usr/bin/ansible', 'cb8a161dabae51cf2616c4a85', '31173931d09486697c05', '00ee4c55e8b46f69a54'),
('/usr/bin/ansible-connection', 'ffbef34c8d2ae633031f4e8', '61cf10e6b6e4ff33c7ded21244', 'c670ce6af9b645c'),
('/usr/bin/ansible-test', '40f528bebb16199d2b11a43b', 'ea6d8680d833fb36e9548a', 'a2c1b309b22cd4285a'),
('/usr/bin/appres', '7ccb78e306838a87b68d2c', '7d089e41ee491fc64b2b', 'c3e24ec3fee558e9f06bd0'),
('/usr/bin/apt', '3ca60d2b26761b7a50d812', '8b624ea3f31040bd838dfc', '36b5070dd02c87b47512'),
('/usr/bin/apt-cache', 'f0e82d30aa1d8a80e', '7da7f514d8056033c4844a6f0', 'be07d8cb7140399252c51c'),
('/usr/bin/apt-cdrom', '073483cad694b013a1', 'fb7b20b9450ab7abb4bcc', '2ad240a6670f486e2e1da9daa'),
('/usr/bin/apt-config', '32856a9a4e703346d6b8', '407560704903fb078e45dac', 'ee559cd54c2a34f3ad3d4')
]
data_mapped_list = []
# each different object return a different value
for data in data_list:
data_mapped_list.append(data_mapping.addObject(data))
self.assertEqual(len(data_mapped_list), len(data_list))
self.assertEqual(len(data_list), data_mapping.getSize())
# ensure add again same data return always samething
tmp_list = []
for data in data_list:
tmp_list.append(data_mapping.addObject(data))
self.assertEqual(tmp_list, data_mapped_list)
# size still same
self.assertEqual(len(data_list), data_mapping.getSize())
# ensure we can get original value
for index in range(len(data_mapped_list)):
self.assertEqual(data_mapping.getObjectFromValue(data_mapped_list[index]), data_list[index])
# ensure we can get mapped value
for index in range(len(data_mapped_list)):
self.assertEqual(data_mapping.getValueFromObject(data_list[index]), data_mapped_list[index])
# another data list, /usr/bin/2to3-2.7, /usr/bin/appres, /usr/bin/aclocal-1.16 's value are different compare to previous data
# so 3 values are different
another_data_list = [
('/usr/bin/2to3-2.7', 'ModifiedValue', 'fade8568285eb14146a7244', 'f631570af55ee08ecef78f3'),
('/usr/bin/R', 'b4c48d52345ae2eb7ca0455db', '59441ddbc00b6521da571', 'a92be1a7acc03f3846'),
('/usr/bin/Rscript', 'e97842e556f90be5f7e5', '806725443a01bcae802','1829d887e0c3380ec8f463527'),
('/usr/bin/[', '5c348873d0e0abe26d56cc752f0', 'ebb7ae1a78018b62224','9162fdf5b9598c9e9d'),
('/usr/bin/aa-enabled', 'c2336b14b7e9d1407d9', 'c8c7af8e6d14a4d1061', '09726542c6c4ca907417c676b4'),
('/usr/bin/aa-exec', '7f66cd90da4b27703de299', 'b4bfe90f53a690deb2687342', 'ad36c9002fa84d4e86'),
('/usr/bin/aclocal-1.16', '6ba134fb4f97d79a5', 'ModifiedValue', 'cc4595fba3251aaa9b48'),
('/usr/bin/activate-global-python-argcomplete3', '4007899eba237603a', '8a2584480e9094f9e4761f7', '893ebeeca071795ecd457ed2'),
('/usr/bin/addpart', '078a10c409b8b21c3b5137d', '481766f1aa1d393e4a3a', '668374ce14a57b2dd14bb'),
('/usr/bin/ansible', 'cb8a161dabae51cf2616c4a85', '31173931d09486697c05', '00ee4c55e8b46f69a54'),
('/usr/bin/ansible-connection', 'ffbef34c8d2ae633031f4e8', '61cf10e6b6e4ff33c7ded21244', 'c670ce6af9b645c'),
('/usr/bin/ansible-test', '40f528bebb16199d2b11a43b', 'ea6d8680d833fb36e9548a', 'a2c1b309b22cd4285a'),
('/usr/bin/appres', '7ccb78e306838a87b68d2c', 'ModifiedValue', 'c3e24ec3fee558e9f06bd0'),
('/usr/bin/apt', '3ca60d2b26761b7a50d812', '8b624ea3f31040bd838dfc', '36b5070dd02c87b47512'),
('/usr/bin/apt-cache', 'f0e82d30aa1d8a80e', '7da7f514d8056033c4844a6f0', 'be07d8cb7140399252c51c'),
('/usr/bin/apt-cdrom', '073483cad694b013a1', 'fb7b20b9450ab7abb4bcc', '2ad240a6670f486e2e1da9daa'),
('/usr/bin/apt-config', '32856a9a4e703346d6b8', '407560704903fb078e45dac', 'ee559cd54c2a34f3ad3d4')
]
another_data_mapped_list = []
for data in another_data_list:
another_data_mapped_list.append(data_mapping.addObject(data))
self.assertEqual(len(another_data_mapped_list), len(data_list))
array = np.array(data_mapped_list)
another_array = np.array(another_data_mapped_list)
# simply call setdiff1d to get the different between two datas
diff_array = np.setdiff1d(another_array, array)
self.assertEqual(diff_array.size, 3)
diff_object_list = []
for value in diff_array:
diff_object_list.append(data_mapping.getObjectFromValue(value))
self.assertEqual(diff_object_list, [('/usr/bin/2to3-2.7', 'ModifiedValue', 'fade8568285eb14146a7244', 'f631570af55ee08ecef78f3'),
('/usr/bin/aclocal-1.16', '6ba134fb4f97d79a5', 'ModifiedValue', 'cc4595fba3251aaa9b48'),
('/usr/bin/appres', '7ccb78e306838a87b68d2c', 'ModifiedValue', 'c3e24ec3fee558e9f06bd0')])
# same data value as "another_data_list" but in other format
other_format_data_list = [
['/usr/bin/2to3-2.7', 'ModifiedValue', 'fade8568285eb14146a7244', 'f631570af55ee08ecef78f3'],
['/usr/bin/R', 'b4c48d52345ae2eb7ca0455db', '59441ddbc00b6521da571', 'a92be1a7acc03f3846'],
['/usr/bin/Rscript', 'e97842e556f90be5f7e5', '806725443a01bcae802','1829d887e0c3380ec8f463527']
]
other_format_data_mapped_list = []
original_size = data_mapping.getSize()
for data in other_format_data_list:
other_format_data_mapped_list.append(data_mapping.addObject(data))
self.assertEqual(original_size + 3, data_mapping.getSize())
other_format_array = np.array(other_format_data_mapped_list)
# ensure "even data values are same but format is different" is considered different
diff_array = np.setdiff1d(other_format_array, another_array)
self.assertEqual(diff_array.size, 3)
diff_object_list = []
for value in diff_array:
diff_object_list.append(data_mapping.getObjectFromValue(value))
self.assertEqual(diff_object_list, [['/usr/bin/2to3-2.7', 'ModifiedValue', 'fade8568285eb14146a7244', 'f631570af55ee08ecef78f3'],
['/usr/bin/R', 'b4c48d52345ae2eb7ca0455db', '59441ddbc00b6521da571', 'a92be1a7acc03f3846'],
['/usr/bin/Rscript', 'e97842e556f90be5f7e5', '806725443a01bcae802','1829d887e0c3380ec8f463527']])
......@@ -35,6 +35,8 @@ Data Ingestion Module | view
Data Ingestion | view
Data License Module | view
Data License | view
Data Mapping Module | view
Data Mapping | view
Data Operation Module | view
Data Operation | view
Data Order Line | view
......
......@@ -7,4 +7,5 @@ document.erp5.DataArrayViewLine
document.erp5.DataArrayLineExistenceConstraint
document.erp5.DataBucketStream
document.erp5.DataStream
document.erp5.DataTransformation
\ No newline at end of file
document.erp5.DataTransformation
document.erp5.DataMapping
\ No newline at end of file
......@@ -8,6 +8,7 @@ data_event_module
data_ingestion_batch_module
data_ingestion_module
data_license_module
data_mapping_module
data_operation_module
data_order_module
data_product_module
......
......@@ -13,6 +13,7 @@ Data Ingestion Batch Module | Data Ingestion Batch
Data Ingestion Module | Data Ingestion
Data Ingestion | Data Ingestion Line
Data License Module | Data License
Data Mapping Module | Data Mapping
Data Operation Module | Data Operation
Data Order Module | Data Order
Data Order | Data Order Line
......
......@@ -2,10 +2,10 @@ Data Acquisition Unit Module | business_application
Data Aggregation Unit Module | business_application
Data Analysis Module | business_application
Data Array Module | business_application
Data Array View Line | predecessor
Data Configuration Module | business_application
Data Descriptor Module | business_application
Data Event Module | business_application
Data Array View Line | predecessor
Data Event | aggregate
Data Event | destination
Data Event | resource
......@@ -13,6 +13,7 @@ Data Event | source
Data Ingestion Batch Module | business_application
Data Ingestion Module | business_application
Data License Module | business_application
Data Mapping Module | business_application
Data Operation Module | business_application
Data Order Module | business_application
Data Product Module | business_application
......@@ -24,4 +25,4 @@ Data Stream | successor
Data Supply Module | business_application
Data Transformation Module | business_application
Device Configuration Module | business_application
Progress Indicator Module | business_application
Progress Indicator Module | business_application
\ No newline at end of file
......@@ -9,12 +9,12 @@ Data Array
Data Array Line
Data Array Line Existence Constraint
Data Array Module
Data Array View
Data Array View Line
Data Bucket Stream
Data Configuration Module
Data Descriptor Module
Data Descriptor Type
Data Array View
Data Array View Line
Data Event
Data Event Module
Data Ingestion
......@@ -24,6 +24,8 @@ Data Ingestion Line
Data Ingestion Module
Data License
Data License Module
Data Mapping
Data Mapping Module
Data Operation
Data Operation Module
Data Order
......@@ -51,4 +53,4 @@ Ingestion Policy Tool
Progress Indicator
Progress Indicator Module
Sensor
Sensor Module
Sensor Module
\ No newline at end of file
......@@ -21,6 +21,8 @@ Data Ingestion | data_simulation_workflow
Data Ingestion | edit_workflow
Data License | edit_workflow
Data License | validation_workflow
Data Mapping | edit_workflow
Data Mapping | validation_workflow
Data Operation | edit_workflow
Data Operation | validation_workflow
Data Order Line | edit_workflow
......@@ -47,4 +49,4 @@ Ingestion Policy | validation_workflow
Progress Indicator | edit_workflow
Progress Indicator | validation_workflow
Sensor | edit_workflow
Sensor | validation_workflow
Sensor | validation_workflow
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment