Skip to content

S
slapos.core
Commit 1822be3b authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

slapos_mysql_innodb_catalog: Index consistency_error 

    We index now for very few portal types the bool(checkConsistency()) in order to search inconsistent documents.

    The indexation is limited to the scope we must monitor rather them all documents for 2 major reasons:

        - Too much useless constrants on erp5 code base, not applicable for our project (ie.: On person)
        - Performance on indexation is penalized from this extra cost, specially when index large trees (Deliveries or open orders)**

    ** Not so much, however, on a scale of millions of documents it can make a major difference, so this commits keeps up to the minimal implementation.
parent 9661e882
Pipeline #17927 passed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 685 additions and 1 deletion
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_consistency.catalog_keys.xml 0 → 100644
    View file @ 1822be3b
    <catalog_method>
    <item key="sql_clear_catalog" type="int">
    <value>1</value>
    </item>
    </catalog_method>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_consistency.xml 0 → 100644
    View file @ 1822be3b
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="SQL Method" module="erp5.portal_type"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_col</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    <item>
    <key> <string>allow_simple_one_argument_traversal</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>arguments_src</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>cache_time_</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>class_file_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>class_name_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_hook</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_id</string> </key>
    <value> <string>erp5_sql_connection</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>z0_drop_consistency</string> </value>
    </item>
    <item>
    <key> <string>max_cache_</string> </key>
    <value> <int>100</int> </value>
    </item>
    <item>
    <key> <string>max_rows_</string> </key>
    <value> <int>1000</int> </value>
    </item>
    <item>
    <key> <string>portal_type</string> </key>
    <value> <string>SQL Method</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>z0_drop_consistency</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_consistency.catalog_keys.xml 0 → 100644
    View file @ 1822be3b
    <catalog_method>
    <item key="sql_uncatalog_object" type="int">
    <value>1</value>
    </item>
    </catalog_method>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_consistency.xml 0 → 100644
    View file @ 1822be3b
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="SQL Method" module="erp5.portal_type"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>allow_simple_one_argument_traversal</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>arguments_src</string> </key>
    <value> <string>uid</string> </value>
    </item>
    <item>
    <key> <string>cache_time_</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>class_file_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>class_name_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_hook</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_id</string> </key>
    <value> <string>erp5_sql_connection</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>z0_uncatalog_consistency</string> </value>
    </item>
    <item>
    <key> <string>max_cache_</string> </key>
    <value> <int>100</int> </value>
    </item>
    <item>
    <key> <string>max_rows_</string> </key>
    <value> <int>1000</int> </value>
    </item>
    <item>
    <key> <string>portal_type</string> </key>
    <value> <string>SQL Method</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>z0_uncatalog_consistency</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_consistency.catalog_keys.xml 0 → 100644
    View file @ 1822be3b
    <catalog_method>
    <item key="sql_catalog_object_list" type="int">
    <value>1</value>
    </item>
    </catalog_method>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_consistency.sql 0 → 100644
    View file @ 1822be3b
    DELETE FROM
    consistency
    WHERE
    <dtml-in uid>
    uid=<dtml-sqlvar sequence-item type="int"><dtml-if sequence-end><dtml-else> OR </dtml-if>
    </dtml-in>
    ;
    <dtml-var "'\0'">
    INSERT INTO
    consistency
    VALUES
    <dtml-in prefix="loop" expr="_.range(_.len(uid))">
    (
    <dtml-sqlvar expr="uid[loop_item]" type="int">,
    <dtml-sqlvar expr="int(len(checkConsistency[loop_item]) > 0)" type="int">
    )
    <dtml-if sequence-end><dtml-else>,</dtml-if>
    </dtml-in>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_consistency.xml 0 → 100644
    View file @ 1822be3b
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="SQL Method" module="erp5.portal_type"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>allow_simple_one_argument_traversal</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>arguments_src</string> </key>
    <value> <string>uid\n
    checkConsistency</string> </value>
    </item>
    <item>
    <key> <string>cache_time_</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>class_file_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>class_name_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_hook</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_id</string> </key>
    <value> <string>erp5_sql_connection</string> </value>
    </item>
    <item>
    <key> <string>expression</string> </key>
    <value> <string>python: context.getPortalType() in ["Sale Invoice Transaction", "Subscription Request"]</string> </value>
    </item>
    <item>
    <key> <string>expression_cache_key</string> </key>
    <value>
    <tuple>
    <string>portal_type</string>
    </tuple>
    </value>
    </item>
    <item>
    <key> <string>filtered</string> </key>
    <value> <int>1</int> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>z_catalog_consistency</string> </value>
    </item>
    <item>
    <key> <string>max_cache_</string> </key>
    <value> <int>100</int> </value>
    </item>
    <item>
    <key> <string>max_rows_</string> </key>
    <value> <int>1000</int> </value>
    </item>
    <item>
    <key> <string>portal_type</string> </key>
    <value> <string>SQL Method</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>z_catalog_consistency</string> </value>
    </item>
    <item>
    <key> <string>type</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_consistency.catalog_keys.xml 0 → 100644
    View file @ 1822be3b
    <catalog_method>
    <item key="sql_clear_catalog" type="int">
    <value>1</value>
    </item>
    </catalog_method>
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_consistency.sql 0 → 100644
    View file @ 1822be3b
    CREATE TABLE `consistency` (
    `uid` BIGINT UNSIGNED NOT NULL,
    `consistency_error` BOOL DEFAULT 0,
    PRIMARY KEY (`uid`, `consistency_error`)
    ) ENGINE=InnoDB;
    master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_consistency.xml 0 → 100644
    View file @ 1822be3b
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="SQL Method" module="erp5.portal_type"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_col</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    <item>
    <key> <string>allow_simple_one_argument_traversal</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>arguments_src</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>cache_time_</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>class_file_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>class_name_</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_hook</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>connection_id</string> </key>
    <value> <string>erp5_sql_connection</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>z_create_consistency</string> </value>
    </item>
    <item>
    <key> <string>max_cache_</string> </key>
    <value> <int>100</int> </value>
    </item>
    <item>
    <key> <string>max_rows_</string> </key>
    <value> <int>1000</int> </value>
    </item>
    <item>
    <key> <string>portal_type</string> </key>
    <value> <string>SQL Method</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>z_create_consistency</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_mysql_innodb_catalog/CatalogResultTableTemplateItem/result_table_list.xml
    View file @ 1822be3b
    <key_list>
    <key>compute_partition</key>
    <key>consistency</key>
    <key>slapos_item</key>
    <key>software_instance_tree</key>
    </key_list>
    \ No newline at end of file
    master/bt5/slapos_mysql_innodb_catalog/TestTemplateItem/portal_components/test.erp5.testSlapOSCatalogConsistency.py 0 → 100644
    View file @ 1822be3b
    # -*- coding: utf-8 -*-
    ##############################################################################
    #
    # Copyright (C) 2013-2019 Nexedi SA and Contributors.
    #
    # This program is free software: you can Use, Study, Modify and Redistribute
    # it under the terms of the GNU General Public License version 3, or (at your
    # option) any later version, as published by the Free Software Foundation.
    #
    # You can also Link and Combine this program with other software covered by
    # the terms of any of the Free Software licenses or any of the Open Source
    # Initiative approved licenses and Convey the resulting work. Corresponding
    # source of such a combination shall include the source code for all other
    # software used.
    #
    # This program is distributed WITHOUT ANY WARRANTY; without even the implied
    # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    #
    # See COPYING file for full licensing terms.
    # See https://www.nexedi.com/licensing for rationale and options.
    #
    ##############################################################################
    from erp5.component.test.SlapOSTestCaseMixin import SlapOSTestCaseMixin
    import transaction
    def checkConsistencyWithError(self, **kw):
    return ["Inconsistent"]
    def checkConsistencyWithoutError(self, **kw):
    return []
    class TestSlapOSCatalogConsistency(SlapOSTestCaseMixin):
    def afterSetUp(self):
    SlapOSTestCaseMixin.afterSetUp(self)
    def testSaleInvoiceTransaction(self):
    from Products.ERP5Type.Base import Base
    sale_invoice_transaction = self.portal.accounting_module.newContent(
    portal_type="Sale Invoice Transaction"
    )
    original_checkConsistency = Base.checkConsistency
    Base.checkConsistency = checkConsistencyWithoutError
    try:
    transaction.commit()
    sale_invoice_transaction.immediateReindexObject()
    self.tic()
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Sale Invoice Transaction",
    consistency_error=0,
    uid=sale_invoice_transaction.getUid()
    )
    self.assertEqual(document, sale_invoice_transaction)
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Sale Invoice Transaction",
    consistency_error=1,
    uid=sale_invoice_transaction.getUid()
    )
    self.assertEqual(document, None)
    Base.checkConsistency = checkConsistencyWithError
    transaction.commit()
    sale_invoice_transaction.immediateReindexObject()
    self.tic()
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Sale Invoice Transaction",
    consistency_error=1,
    uid=sale_invoice_transaction.getUid()
    )
    self.assertEqual(document, sale_invoice_transaction)
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Sale Invoice Transaction",
    consistency_error=0,
    uid=sale_invoice_transaction.getUid(),
    )
    self.assertEqual(document, None)
    finally:
    Base.checkConsistency = original_checkConsistency
    def testSubscriptionRequest(self):
    from Products.ERP5Type.Base import Base
    subscription_request = self.portal.subscription_request_module.newContent(
    portal_type="Subscription Request"
    )
    original_checkConsistency = Base.checkConsistency
    Base.checkConsistency = checkConsistencyWithoutError
    try:
    transaction.commit()
    subscription_request.immediateReindexObject()
    self.tic()
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Subscription Request",
    consistency_error=0,
    uid=subscription_request.getUid()
    )
    self.assertEqual(document, subscription_request)
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Subscription Request",
    consistency_error=1,
    uid=subscription_request.getUid()
    )
    self.assertEqual(document, None)
    Base.checkConsistency = checkConsistencyWithError
    transaction.commit()
    subscription_request.immediateReindexObject()
    self.tic()
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Subscription Request",
    consistency_error=1,
    uid=subscription_request.getUid()
    )
    self.assertEqual(document, subscription_request)
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Subscription Request",
    consistency_error=0,
    uid=subscription_request.getUid(),
    )
    self.assertEqual(document, None)
    finally:
    Base.checkConsistency = original_checkConsistency
    def testNotIndexedOnConsistencyTable(self):
    from Products.ERP5Type.Base import Base
    person = self.portal.person_module.newContent(
    portal_type="Person"
    )
    original_checkConsistency = Base.checkConsistency
    Base.checkConsistency = checkConsistencyWithoutError
    try:
    transaction.commit()
    person.immediateReindexObject()
    self.tic()
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Person",
    consistency_error=0,
    uid=person.getUid()
    )
    self.assertEqual(document, None)
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Person",
    consistency_error=1,
    uid=person.getUid()
    )
    self.assertEqual(document, None)
    Base.checkConsistency = checkConsistencyWithError
    transaction.commit()
    person.immediateReindexObject()
    self.tic()
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Person",
    consistency_error=1,
    uid=person.getUid()
    )
    self.assertEqual(document, None)
    document = self.portal.portal_catalog.getResultValue(
    portal_type="Person",
    consistency_error=0,
    uid=person.getUid(),
    )
    self.assertEqual(document, None)
    finally:
    Base.checkConsistency = original_checkConsistency
    \ No newline at end of file
    master/bt5/slapos_mysql_innodb_catalog/TestTemplateItem/portal_components/test.erp5.testSlapOSCatalogConsistency.xml 0 → 100644
    View file @ 1822be3b
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="Test Component" module="erp5.portal_type"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_recorded_property_dict</string> </key>
    <value>
    <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
    </value>
    </item>
    <item>
    <key> <string>default_reference</string> </key>
    <value> <string>testSlapOSCatalogConsistency</string> </value>
    </item>
    <item>
    <key> <string>default_source_reference</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>description</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>test.erp5.testSlapOSCatalogConsistency</string> </value>
    </item>
    <item>
    <key> <string>portal_type</string> </key>
    <value> <string>Test Component</string> </value>
    </item>
    <item>
    <key> <string>sid</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>text_content_error_message</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    <item>
    <key> <string>text_content_warning_message</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    <item>
    <key> <string>version</string> </key>
    <value> <string>erp5</string> </value>
    </item>
    <item>
    <key> <string>workflow_history</string> </key>
    <value>
    <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    <record id="2" aka="AAAAAAAAAAI=">
    <pickle>
    <global name="PersistentMapping" module="Persistence.mapping"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>data</string> </key>
    <value>
    <dictionary/>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    <record id="3" aka="AAAAAAAAAAM=">
    <pickle>
    <global name="PersistentMapping" module="Persistence.mapping"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>data</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>component_validation_workflow</string> </key>
    <value>
    <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
    </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    <record id="4" aka="AAAAAAAAAAQ=">
    <pickle>
    <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_log</string> </key>
    <value>
    <list>
    <dictionary>
    <item>
    <key> <string>action</string> </key>
    <value> <string>validate</string> </value>
    </item>
    <item>
    <key> <string>validation_state</string> </key>
    <value> <string>validated</string> </value>
    </item>
    </dictionary>
    </list>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_mysql_innodb_catalog/bt/template_catalog_method_id_list
    View file @ 1822be3b
    erp5_mysql_innodb/z0_drop_compute_partition
    erp5_mysql_innodb/z0_drop_consistency
    erp5_mysql_innodb/z0_drop_slapos_item
    erp5_mysql_innodb/z0_drop_software_instance_tree
    erp5_mysql_innodb/z0_uncatalog_compute_partition
    erp5_mysql_innodb/z0_uncatalog_consistency
    erp5_mysql_innodb/z0_uncatalog_email
    erp5_mysql_innodb/z0_uncatalog_slapos_item
    erp5_mysql_innodb/z0_uncatalog_software_instance
    erp5_mysql_innodb/z_catalog_compute_partition_list
    erp5_mysql_innodb/z_catalog_consistency
    erp5_mysql_innodb/z_catalog_email_list
    erp5_mysql_innodb/z_catalog_item_list
    erp5_mysql_innodb/z_catalog_object_list
    ......@@ -13,6 +16,7 @@ erp5_mysql_innodb/z_catalog_slapos_item_list
    erp5_mysql_innodb/z_catalog_software_instance_list
    erp5_mysql_innodb/z_create_catalog
    erp5_mysql_innodb/z_create_compute_partition
    erp5_mysql_innodb/z_create_consistency
    erp5_mysql_innodb/z_create_item
    erp5_mysql_innodb/z_create_slapos_item
    erp5_mysql_innodb/z_create_software_instance_tree
    ......
    master/bt5/slapos_mysql_innodb_catalog/bt/template_catalog_result_table_list
    View file @ 1822be3b
    compute_partition
    software_instance_tree
    slapos_item
    \ No newline at end of file
    slapos_item
    consistency
    \ No newline at end of file
    master/bt5/slapos_mysql_innodb_catalog/bt/template_test_id_list 0 → 100644
    View file @ 1822be3b
    test.erp5.testSlapOSCatalogConsistency
    \ No newline at end of file
    master/tests/__init__.py
    View file @ 1822be3b
    ......@@ -18,6 +18,7 @@ slapos_bt_list = [
    'slapos_payzen',
    'slapos_wechat',
    'slapos_configurator',
    'slapos_mysql_innodb_catalog',
    'slapos_jio',
    'slapos_jio_ui_test'
    ]
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7