Skip to content

erp5
erp5
Commit 3b781bc8 authored by Vincent Pelletier's avatar Vincent Pelletier
Browse files
Options

ERP5Catalog.test: Move indexation-content-checking tests to a separate file 

To not receive catalog already tainted by clearing & reindexing.

In turn, this allows testing catalog reference content for equality:
catalog must be exactly the same before/after an ERP5Site_reindexAll, and
before/after a hot-reindex. So issubset is not an acceptable method.
Instead, use assertItemsEqual, and raise max diff size.

Also, tidy code a bit:
- Avoid clearing catalog only to restrict result set.
- Improve coding style.
- Simplify code (single-use variables...).
- Call assert methods producing more helpful error messages on failure.
parent ddc39b3b
Hide whitespace changes
Inline Side-by-side
Showing with 376 additions and 331 deletions
product/ERP5Catalog/tests/testERP5Catalog.py
View file @ 3b781bc8
...@@ -27,28 +27,20 @@ ...@@ -27,28 +27,20 @@
# #
############################################################################## ##############################################################################
import unittest from random import randint
import sys import sys
import unittest
from unittest import expectedFailure from unittest import expectedFailure
from Testing import ZopeTestCase
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from AccessControl import getSecurityManager from AccessControl import getSecurityManager
from AccessControl.SecurityManagement import newSecurityManager from AccessControl.SecurityManagement import newSecurityManager
from zLOG import LOG
from DateTime import DateTime from DateTime import DateTime
from Products.ERP5Type.tests.utils import LogInterceptor from OFS.ObjectManager import ObjectManager
from Products.ERP5Type.tests.utils import createZODBPythonScript, todo_erp5, \ from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
getExtraSqlConnectionStringList from Products.ERP5Type.tests.utils import LogInterceptor, createZODBPythonScript, todo_erp5, getExtraSqlConnectionStringList
from Products.ZSQLCatalog.ZSQLCatalog import HOT_REINDEXING_FINISHED_STATE,\
HOT_REINDEXING_RECORDING_STATE, HOT_REINDEXING_DOUBLE_INDEXING_STATE
from Products.CMFActivity.Errors import ActivityFlushError
from Products.PageTemplates.Expressions import getEngine from Products.PageTemplates.Expressions import getEngine
from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery, SimpleQuery from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery, SimpleQuery
from Testing import ZopeTestCase
from zLOG import LOG
from OFS.ObjectManager import ObjectManager
from random import randint
class IndexableDocument(ObjectManager): class IndexableDocument(ObjectManager):
...@@ -149,11 +141,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -149,11 +141,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
_, row_list = sql_connection().query(sql, max_rows=0) _, row_list = sql_connection().query(sql, max_rows=0)
return [x for x, in row_list] return [x for x, in row_list]
def getSQLPathListWithRolesAndUsers(self, connection_id):
sql = 'select distinct(path) from catalog, roles_and_users\
where catalog.security_uid=roles_and_users.uid'
return self.getSQLPathList(connection_id, sql)
def checkRelativeUrlInSQLPathList(self,url_list,connection_id=None): def checkRelativeUrlInSQLPathList(self,url_list,connection_id=None):
path_list = self.getSQLPathList(connection_id=connection_id) path_list = self.getSQLPathList(connection_id=connection_id)
portal_id = self.getPortalId() portal_id = self.getPortalId()
...@@ -268,50 +255,71 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -268,50 +255,71 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
def test_10_OrderedSearchFolder(self): def test_10_OrderedSearchFolder(self):
person_module = self.getPersonModule() person_module = self.getPersonModule()
person_uid_list = [
# Clear catalog person_module.newContent(id='a', portal_type='Person', title='a', description='z').getUid(),
portal_catalog = self.getCatalogTool() person_module.newContent(id='b', portal_type='Person', title='a', description='y').getUid(),
portal_catalog.manage_catalogClear() person_module.newContent(id='c', portal_type='Person', title='a', description='x').getUid(),
]
person = person_module.newContent(id='a',portal_type='Person',title='a',description='z') self.tic()
self.tic() self.assertEqual(
person = person_module.newContent(id='b',portal_type='Person',title='a',description='y') ['a','b','c'],
self.tic() [
person = person_module.newContent(id='c',portal_type='Person',title='a',description='x') x.getObject().getId()
self.tic() for x in person_module.searchFolder(
folder_object_list = [x.getObject().getId() sort_on=[('id', 'ascending')],
for x in person_module.searchFolder(sort_on=[('id','ascending')])] )
self.assertEqual(['a','b','c'],folder_object_list) ],
folder_object_list = [x.getObject().getId() )
for x in person_module.searchFolder( self.assertEqual(
sort_on=[('title','ascending'), ('description','ascending')])] ['c','b','a'],
self.assertEqual(['c','b','a'],folder_object_list) [
folder_object_list = [x.getObject().getId() x.getObject().getId()
for x in person_module.searchFolder( for x in person_module.searchFolder(
sort_on=[('title','ascending'),('description','descending')])] sort_on=[('title', 'ascending'), ('description', 'ascending')],
self.assertEqual(['a','b','c'],folder_object_list) )
],
)
self.assertEqual(
['a','b','c'],
[
x.getObject().getId()
for x in person_module.searchFolder(
sort_on=[('title', 'ascending'), ('description', 'descending')],
)
],
)
def test_11_CastStringAsInt(self): def test_11_CastStringAsInt(self):
person_module = self.getPersonModule() person_module = self.getPersonModule()
person_uid_list = [
# Clear catalog person_module.newContent(portal_type='Person', title='1').getUid(),
portal_catalog = self.getCatalogTool() person_module.newContent(portal_type='Person', title='2').getUid(),
portal_catalog.manage_catalogClear() person_module.newContent(portal_type='Person', title='12').getUid(),
]
person = person_module.newContent(id='a',portal_type='Person',title='1') self.tic()
self.tic() self.assertEqual(
person = person_module.newContent(id='b',portal_type='Person',title='2') ['1', '12', '2'],
self.tic() [
person = person_module.newContent(id='c',portal_type='Person',title='12') x.getObject().getTitle()
self.tic() for x in person_module.searchFolder(
folder_object_list = [x.getObject().getTitle() for x in person_module.searchFolder(sort_on=[('title','ascending')])] sort_on=[('title', 'ascending')],
self.assertEqual(['1','12','2'],folder_object_list) uid=person_uid_list,
folder_object_list = [x.getObject().getTitle() for x in person_module.searchFolder(sort_on=[('title','ascending','int')])] )
self.assertEqual(['1','2','12'],folder_object_list) ],
)
self.assertEqual(
['1', '2', '12'],
[
x.getObject().getTitle()
for x in person_module.searchFolder(
sort_on=[('title', 'ascending', 'int')],
uid=person_uid_list,
)
],
)
def test_12_TransactionalUidBuffer(self): def test_12_TransactionalUidBuffer(self):
portal_catalog = self.getCatalogTool() catalog = self.getCatalogTool().getSQLCatalog()
catalog = portal_catalog.getSQLCatalog()
self.assertTrue(catalog is not None) self.assertTrue(catalog is not None)
from Products.ZSQLCatalog.SQLCatalog import global_reserved_uid_lock from Products.ZSQLCatalog.SQLCatalog import global_reserved_uid_lock
# Clear out the uid buffer. # Clear out the uid buffer.
...@@ -337,75 +345,35 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -337,75 +345,35 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
uid_buffer = getUIDBuffer() uid_buffer = getUIDBuffer()
self.assertTrue(len(uid_buffer) == 0) self.assertTrue(len(uid_buffer) == 0)
def test_13_ERP5Site_reindexAll(self):
portal = self.getPortal()
self.getCategoryTool().newContent(portal_type='Base Category', title="GreatTitle1")
portal.getDefaultModule('Organisation').newContent(portal_type='Organisation', title="GreatTitle2")
self.tic()
original_path_list = self.getSQLPathList()
self.getCatalogTool().manage_catalogClear()
self.assertEqual([], self.getSQLPathList())
portal.ERP5Site_reindexAll()
self.tic()
# Check if all objects are catalogued as before
self.assertTrue(set(original_path_list).issubset(self.getSQLPathList()))
def test_14_ReindexWithBrokenCategory(self): def test_14_ReindexWithBrokenCategory(self):
"""Reindexing an object with 1 broken category must not affect other valid """Reindexing an object with 1 broken category must not affect other valid
categories""" categories"""
# Flush message queue
self.tic() self.tic()
# Create some objects
portal = self.portal
portal_category = self.getCategoryTool() portal_category = self.getCategoryTool()
group_nexedi_category = portal_category.group\ group_nexedi_category = portal_category.group.newContent(id='nexedi')
.newContent( id = 'nexedi', ) region_europe_category = portal_category.region.newContent(id='europe')
region_europe_category = portal_category.region\ organisation = self.portal.getDefaultModule('Organisation').newContent(portal_type='Organisation')
.newContent( id = 'europe', )
module = portal.getDefaultModule('Organisation')
organisation = module.newContent(portal_type='Organisation',)
organisation.setGroup('nexedi') organisation.setGroup('nexedi')
self.assertEqual(organisation.getGroupValue(), group_nexedi_category) self.assertEqual(organisation.getGroupValue(), group_nexedi_category)
organisation.setRegion('europe') organisation.setRegion('europe')
self.assertEqual(organisation.getRegionValue(), region_europe_category) self.assertEqual(organisation.getRegionValue(), region_europe_category)
organisation.setRole('not_exists') organisation.setRole('not_exists')
self.assertEqual(organisation.getRoleValue(), None) self.assertEqual(organisation.getRoleValue(), None)
# Flush message queue
self.tic() self.tic()
# Clear catalog
portal_catalog = self.getCatalogTool()
portal_catalog.manage_catalogClear()
sql_connection = self.getSQLConnection() sql_connection = self.getSQLConnection()
sql = 'SELECT COUNT(*) FROM category '\
'WHERE uid=%s and category_strict_membership = 1' %\
organisation.getUid()
result = sql_connection.manage_test(sql)
message_count = result[0]['COUNT(*)']
self.assertEqual(0, message_count)
# Commit
self.tic()
# Check catalog
organisation.reindexObject()
# Commit
self.tic()
sql = 'select count(*) from message'
result = sql_connection.manage_test(sql)
message_count = result[0]['COUNT(*)']
self.assertEqual(0, message_count)
# Check region and group categories are catalogued # Check region and group categories are catalogued
for base_cat, theorical_count in { for base_cat, theorical_count in (
'region':1, ('region', 1),
'group':1, ('group', 1),
'role':0}.items() : ('role', 0),
sql = """SELECT COUNT(*) FROM category ):
WHERE category.uid=%s and category.category_strict_membership = 1 self.assertEqual(
AND category.base_category_uid = %s""" % (organisation.getUid(), theorical_count,
portal_category[base_cat].getUid()) sql_connection.manage_test(
result = sql_connection.manage_test(sql) "SELECT COUNT(*) FROM category WHERE category.uid=%s and category.category_strict_membership = 1 AND category.base_category_uid = %s" % (organisation.getUid(), portal_category[base_cat].getUid())
cataloged_obj_count = result[0]['COUNT(*)'] )[0]['COUNT(*)'],
self.assertEqual(theorical_count, cataloged_obj_count, 'category %s is not cataloged correctly' % base_cat,
'category %s is not cataloged correctly' % base_cat) )
def test_15_getObject(self,): def test_15_getObject(self,):
# portal_catalog.getObject raises a ValueError if UID parameter is a string # portal_catalog.getObject raises a ValueError if UID parameter is a string
...@@ -1074,179 +1042,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1074,179 +1042,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
len(self.getCatalogTool()(portal_type='Organisation', limit=None))) len(self.getCatalogTool()(portal_type='Organisation', limit=None)))
ctool.default_result_limit = old_default_result_limit ctool.default_result_limit = old_default_result_limit
def test_48_ERP5Site_hotReindexAll(self):
"""
test the hot reindexing of catalog -> catalog2
then a hot reindexing detailed catalog2 -> catalog
this test use the variable environment: extra_sql_connection_string_list
"""
portal = self.portal
self.original_connection_id = 'erp5_sql_connection'
self.original_deferred_connection_id = self.new_erp5_deferred_sql_connection
self.new_connection_id = self.new_erp5_sql_connection
self.new_deferred_connection_id = 'erp5_sql_deferred_connection2'
new_connection_string = getExtraSqlConnectionStringList()[0]
# Skip this test if default connection string is not "test test".
original_connection = getattr(portal, self.original_connection_id)
connection_string = original_connection.connection_string
if (connection_string == new_connection_string):
message = 'SKIPPED: default connection string is the same as the one for hot-reindex catalog'
ZopeTestCase._print(message)
LOG('Testing... ',0,message)
portal_category = self.getCategoryTool()
portal_activities = self.getActivityTool()
self.base_category = portal_category.newContent(portal_type='Base Category',
title="GreatTitle1")
module = portal.getDefaultModule('Organisation')
self.organisation = module.newContent(portal_type='Organisation',
title="GreatTitle2")
# Flush message queue
self.tic()
addSQLConnection = portal.manage_addProduct['ZMySQLDA'] \
.manage_addZMySQLConnection
# Create new connectors
addSQLConnection(self.new_connection_id,'', new_connection_string)
new_connection = portal[self.new_connection_id]
new_connection.manage_open_connection()
addSQLConnection(self.new_deferred_connection_id,'', new_connection_string)
new_connection = portal[self.new_deferred_connection_id]
new_connection.manage_open_connection()
# the transactionless connector must not be change because this one
# create the portal_ids otherwise it create of conflicts with uid
# objects
# Create new catalog
portal_catalog = self.getCatalogTool()
self.original_catalog_id = 'erp5_mysql_innodb'
self.new_catalog_id = self.original_catalog_id + '2'
cp_data = portal_catalog.manage_copyObjects(ids=('erp5_mysql_innodb',))
new_id = portal_catalog.manage_pasteObjects(cp_data)[0]['new_id']
portal_catalog.manage_renameObject(id=new_id, new_id=self.new_catalog_id)
# Parse all methods in the new catalog in order to change the connector
new_catalog = portal_catalog[self.new_catalog_id]
source_sql_connection_id_list=list((self.original_connection_id,
self.original_deferred_connection_id))
destination_sql_connection_id_list=list((self.new_connection_id,
self.new_deferred_connection_id))
#launch the full hot reindexing
portal_catalog.manage_hotReindexAll(source_sql_catalog_id=self.original_catalog_id,
destination_sql_catalog_id=self.new_catalog_id,
source_sql_connection_id_list=source_sql_connection_id_list,
destination_sql_connection_id_list=destination_sql_connection_id_list,
update_destination_sql_catalog=True)
# Flush message queue
self.tic()
original_path_list = self.getSQLPathList(self.original_connection_id)
new_path_list = self.getSQLPathList(self.new_connection_id)
self.assertTrue(
set(original_path_list).issubset(new_path_list),
set(original_path_list).difference(new_path_list),
)
self.organisation2 = module.newContent(portal_type='Organisation',
title="GreatTitle2")
first_deleted_url = self.organisation2.getRelativeUrl()
self.tic()
path_list = [self.organisation.getRelativeUrl()]
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.original_connection_id)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_connection_id)
path_list = [first_deleted_url]
self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=self.original_connection_id)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_connection_id)
# Make sure some zsql method use the right connection_id
zsql_method = portal.portal_skins.erp5_core.Resource_zGetInventoryList
self.assertEqual(getattr(zsql_method,'connection_id'),self.new_connection_id)
self.assertEqual(portal_catalog.getHotReindexingState(),
HOT_REINDEXING_FINISHED_STATE)
# Do a hot reindex in the reverse way, but this time a more
# complicated hot reindex
portal_catalog.manage_hotReindexAll(
source_sql_catalog_id=self.new_catalog_id,
destination_sql_catalog_id=self.original_catalog_id,
source_sql_connection_id_list=destination_sql_connection_id_list,
destination_sql_connection_id_list=source_sql_connection_id_list,
update_destination_sql_catalog=True)
self.commit()
self.assertEqual(portal_catalog.getHotReindexingState(),
HOT_REINDEXING_RECORDING_STATE)
self.organisation3 = module.newContent(portal_type='Organisation',
title="GreatTitle2")
# Try something more complicated, create new object, reindex it
# and then delete it
self.deleted_organisation = module.newContent(portal_type='Organisation',
title="GreatTitle2")
self.deleted_organisation.immediateReindexObject()
self.commit()
deleted_url = self.deleted_organisation.getRelativeUrl()
module.manage_delObjects(ids=[self.deleted_organisation.getId()])
self.commit()
query = self.portal.cmf_activity_sql_connection().query
query(
'update message set processing_node=-4 where method_id in '
'("playBackRecordedObjectList", "_finishHotReindexing")',
)
hasNoProcessableMessage = lambda message_list: all(
x.processing_node == -4
for x in message_list
)
self.tic(stop_condition=hasNoProcessableMessage)
self.assertEqual(portal_catalog.getHotReindexingState(),
HOT_REINDEXING_DOUBLE_INDEXING_STATE)
# try to delete objects in double indexing state
module.manage_delObjects(ids=[self.organisation2.getId()])
self.commit()
query(
'update message set processing_node=-1 where '
'method_id="playBackRecordedObjectList"',
)
self.tic(stop_condition=hasNoProcessableMessage)
self.assertEqual(portal_catalog.getHotReindexingState(),
HOT_REINDEXING_DOUBLE_INDEXING_STATE)
# Now we have started an double indexing
self.next_deleted_organisation = module.newContent(portal_type='Organisation',
title="GreatTitle2",id='toto')
next_deleted_url = self.next_deleted_organisation.getRelativeUrl()
self.tic(stop_condition=hasNoProcessableMessage)
path_list=[next_deleted_url]
self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.new_connection_id)
self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.original_connection_id)
module.manage_delObjects(ids=[self.next_deleted_organisation.getId()])
#Create object during the double indexing to check the security object
#after the hot reindexing
self.organisation4 = module.newContent(portal_type='Organisation',
title="GreatTitle2")
self.commit()
query(
'update message set processing_node=-1 where '
'method_id="_finishHotReindexing"',
)
self.tic()
self.assertEqual(portal_catalog.getHotReindexingState(),
HOT_REINDEXING_FINISHED_STATE)
# Check Security UID object exist in roles and users
# compare the number object in the catalog
count_catalog = len(self.getSQLPathList(self.original_connection_id))
count_restricted_catalog = len(self.getSQLPathListWithRolesAndUsers(\
self.original_connection_id))
self.assertEqual(count_catalog, count_restricted_catalog)
path_list = [self.organisation3.getRelativeUrl()]
self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.new_connection_id)
self.checkRelativeUrlInSQLPathList(path_list,connection_id=self.original_connection_id)
path_list = [first_deleted_url,deleted_url,next_deleted_url]
self.checkRelativeUrlNotInSQLPathList(path_list,connection_id=self.new_connection_id)
self.checkRelativeUrlNotInSQLPathList(path_list,connection_id=self.original_connection_id)
# Make sure module are there
path_list = [module.getRelativeUrl()]
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_connection_id)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.original_connection_id)
def test_48bis_ERP5Site_hotReindexAllCheckCachedValues(self): def test_48bis_ERP5Site_hotReindexAllCheckCachedValues(self):
""" """
test the hot reindexing of catalog -> catalog2 test the hot reindexing of catalog -> catalog2
...@@ -1400,42 +1195,20 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1400,42 +1195,20 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
@todo_erp5 @todo_erp5
def test_49_IndexInOrderedSearchFolder(self): def test_49_IndexInOrderedSearchFolder(self):
person_module = self.getPersonModule() searchFolder = self.getPersonModule().searchFolder
catalog = self.getCatalogTool().objectValues()[0]
# Clear catalog
portal_catalog = self.getCatalogTool()
portal_catalog.manage_catalogClear()
catalog = portal_catalog.objectValues()[0]
person = person_module.newContent(id='a',portal_type='Person',title='a',description='z')
self.tic()
person = person_module.newContent(id='b',portal_type='Person',title='a',description='y')
self.tic() self.tic()
person = person_module.newContent(id='c',portal_type='Person',title='a',description='x') self.assertEqual(catalog.sql_catalog_index_on_order_keys, ())
self.tic()
index_columns = getattr(catalog, 'sql_catalog_index_on_order_keys', None)
self.assertNotEqual(index_columns, None)
self.assertEqual(len(index_columns), 0)
# Check catalog don't tell to use index if nothing defined # Check catalog don't tell to use index if nothing defined
sql = person_module.searchFolder(src__=1) self.assertNotIn('use index', searchFolder(src__=1))
self.assertTrue('use index' not in sql) self.assertNotIn('use index', searchFolder(src__=1, sort_on=[('id','ascending')]))
sql = person_module.searchFolder(src__=1, sort_on=[('id','ascending')]) self.assertNotIn('use index', searchFolder(src__=1, sort_on=[('title','ascending')]))
self.assertTrue('use index' not in sql)
sql = person_module.searchFolder(src__=1, sort_on=[('title','ascending')])
self.assertTrue('use index' not in sql)
# Defined that catalog must tell to use index when order by catalog.title # Defined that catalog must tell to use index when order by catalog.title
index_columns = ('catalog.title',) catalog.sql_catalog_index_on_order_keys = ('catalog.title', )
setattr(catalog, 'sql_catalog_index_on_order_keys', index_columns)
index_columns = getattr(catalog, 'sql_catalog_index_on_order_keys', None)
self.assertNotEqual(index_columns, None)
self.assertEqual(len(index_columns), 1)
# Check catalog tell to use index only when ordering by catalog.title # Check catalog tell to use index only when ordering by catalog.title
sql = person_module.searchFolder(src__=1) self.assertNotIn('use index', searchFolder(src__=1))
self.assertTrue('use index' not in sql) self.assertNotIn('use index', searchFolder(src__=1, sort_on=[('id','ascending')]))
sql = person_module.searchFolder(src__=1, sort_on=[('id','ascending')]) self.assertIn('use index', searchFolder(src__=1, sort_on=[('title','ascending')]))
self.assertTrue('use index' not in sql)
sql = person_module.searchFolder(src__=1, sort_on=[('title','ascending')])
self.assertTrue('use index' in sql)
def test_50_LocalRolesArgument(self): def test_50_LocalRolesArgument(self):
"""test local_roles= argument """test local_roles= argument
...@@ -1662,17 +1435,25 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1662,17 +1435,25 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
# Add a script to create uid list # Add a script to create uid list
catalog = self.getCatalogTool().getSQLCatalog() catalog = self.getCatalogTool().getSQLCatalog()
script_id = 'z0_zCreateUid' script_id = 'z0_zCreateUid'
script_content = "context.getPortalObject().portal_ids.generateNewIdList(id_generator='uid',\ script = createZODBPythonScript(
id_group='text_uid')" catalog,
script = createZODBPythonScript(catalog, script_id, script_id,
'*args,**kw', script_content) '*args,**kw',
sql_clear_catalog = list(catalog.sql_clear_catalog) "context.getPortalObject().portal_ids.generateNewIdList(id_generator='uid', id_group='text_uid')",
sql_clear_catalog.append(script_id) )
sql_clear_catalog.sort() sql_clear_catalog_orig = catalog.sql_clear_catalog
catalog.sql_clear_catalog = tuple(sql_clear_catalog) catalog.sql_clear_catalog = tuple(sorted(sql_clear_catalog_orig + (script_id, )))
# launch the sql_clear_catalog with the script after the drop tables and # launch the sql_clear_catalog with the script after the drop tables and
# before the recreate tables of catalog # before the recreate tables of catalog
catalog.manage_catalogClear() try:
self.commit()
catalog.manage_catalogClear()
finally:
self.abort()
catalog.sql_clear_catalog = sql_clear_catalog_orig
self.commit()
self.portal.ERP5Site_reindexAll(clear_catalog=True)
self.tic()
def test_SearchOnOwner(self): def test_SearchOnOwner(self):
# owner= can be used a search key in the catalog to have all documents for # owner= can be used a search key in the catalog to have all documents for
...@@ -2042,9 +1823,7 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -2042,9 +1823,7 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
uf._doAddUser('foo', 'foo', ['Member', ], []) uf._doAddUser('foo', 'foo', ['Member', ], [])
uf._doAddUser('ERP5TypeTestCase', 'ERP5TypeTestCase', ['Member', ], []) uf._doAddUser('ERP5TypeTestCase', 'ERP5TypeTestCase', ['Member', ], [])
self.commit() self.commit()
portal_catalog = self.getCatalogTool() self.getPortal().ERP5Site_reindexAll(clear_catalog=True)
portal_catalog.manage_catalogClear()
self.getPortal().ERP5Site_reindexAll()
self.tic() self.tic()
# Person stuff # Person stuff
......
product/ERP5Catalog/tests/testVanillaERP5Catalog.py 0 → 100644
View file @ 3b781bc8
##############################################################################
#
# Copyright (c) 2018 Nexedi SARL and Contributors. All Rights Reserved.
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import LogInterceptor, getExtraSqlConnectionStringList
from Products.ZSQLCatalog.ZSQLCatalog import HOT_REINDEXING_FINISHED_STATE, HOT_REINDEXING_RECORDING_STATE, HOT_REINDEXING_DOUBLE_INDEXING_STATE
from Testing import ZopeTestCase
from zLOG import LOG
class TestVanillaERP5Catalog(ERP5TypeTestCase, LogInterceptor):
"""
Tests for ERP5 Catalog where a pristine freshly-created-site catalog is
needed. Clearing catalog is only allowed when comparing before/after for
strict item equality.
"""
def getTitle(self):
return "VanillaERP5Catalog"
def getBusinessTemplateList(self):
return ('erp5_full_text_mroonga_catalog', 'erp5_base')
# Different variables used for this test
username = 'seb'
new_erp5_sql_connection = 'erp5_sql_connection2'
new_erp5_deferred_sql_connection = 'erp5_sql_deferred_connection2'
original_catalog_id = 'erp5_mysql_innodb'
new_catalog_id = 'erp5_mysql_innodb2'
def afterSetUp(self):
portal = self.portal
portal.acl_users._doAddUser(self.username, '', ['Manager'], [])
self.loginByUserName(self.username)
self.tic()
def beforeTearDown(self):
# restore default_catalog
portal = self.portal
portal.portal_catalog._setDefaultSqlCatalogId(self.original_catalog_id)
portal.portal_catalog.hot_reindexing_state = None
module = self.getOrganisationModule()
module.manage_delObjects(list(module.objectIds()))
module.reindexObject()
# Remove copied sql_connector and catalog
if self.new_erp5_sql_connection in portal.objectIds():
portal.manage_delObjects([self.new_erp5_sql_connection])
if self.new_erp5_deferred_sql_connection in portal.objectIds():
portal.manage_delObjects([self.new_erp5_deferred_sql_connection])
if self.new_catalog_id in portal.portal_catalog.objectIds():
portal.portal_catalog.manage_delObjects([self.new_catalog_id])
self.tic()
def getSQLPathList(self,connection_id=None, sql=None):
"""
Give the full list of path in the catalog
"""
if connection_id is None:
sql_connection = self.getSQLConnection()
else:
sql_connection = getattr(self.getPortal(), connection_id)
if sql is None:
sql = 'select distinct(path) from catalog'
_, row_list = sql_connection().query(sql, max_rows=0)
return [x for x, in row_list]
def getSQLPathListWithRolesAndUsers(self, connection_id):
sql = 'select distinct(path) from catalog, roles_and_users\
where catalog.security_uid=roles_and_users.uid'
return self.getSQLPathList(connection_id, sql)
def checkRelativeUrlInSQLPathList(self,url_list,connection_id=None):
path_list = self.getSQLPathList(connection_id=connection_id)
portal_id = self.getPortalId()
for url in url_list:
path = '/' + portal_id + '/' + url
self.assertTrue(path in path_list)
LOG('checkRelativeUrlInSQLPathList found path:',0,path)
def checkRelativeUrlNotInSQLPathList(self,url_list,connection_id=None):
path_list = self.getSQLPathList(connection_id=connection_id)
portal_id = self.getPortalId()
for url in url_list:
path = '/' + portal_id + '/' + url
self.assertTrue(path not in path_list)
LOG('checkRelativeUrlInSQLPathList not found path:',0,path)
def test_1_ERP5Site_reindexAll(self):
portal = self.getPortal()
portal.portal_categories.newContent(portal_type='Base Category', title="GreatTitle1")
portal.organisation_module.newContent(portal_type='Organisation', title="GreatTitle2")
self.tic()
original_path_list = self.getSQLPathList()
self.getCatalogTool().manage_catalogClear()
self.assertEqual([], self.getSQLPathList())
portal.ERP5Site_reindexAll()
self.tic()
# Check if all objects are catalogued as before
self.maxDiff = None
self.assertItemsEqual(original_path_list, self.getSQLPathList())
def test_2_ERP5Site_hotReindexAll(self):
"""
test the hot reindexing of catalog -> catalog2
then a hot reindexing detailed catalog2 -> catalog
this test use the variable environment: extra_sql_connection_string_list
"""
portal = self.portal
original_connection_id = 'erp5_sql_connection'
extra_connection_string_list = getExtraSqlConnectionStringList()
if not extra_connection_string_list or extra_connection_string_list[0] == getattr(portal, original_connection_id).connection_string:
self.skipTest('default connection string is the same as the one for hot-reindex catalog')
new_connection_string = extra_connection_string_list[0]
new_deferred_connection_id = 'erp5_sql_deferred_connection2'
module = portal.organisation_module
organisation = module.newContent(portal_type='Organisation', title="GreatTitle2")
self.tic()
addSQLConnection = portal.manage_addProduct['ZMySQLDA'].manage_addZMySQLConnection
# Create new connectors
addSQLConnection(self.new_erp5_sql_connection, '', new_connection_string)
portal[self.new_erp5_sql_connection].manage_open_connection()
addSQLConnection(new_deferred_connection_id, '', new_connection_string)
portal[new_deferred_connection_id].manage_open_connection()
# Note: transactionless connector must not be changed because this one
# create the portal_ids otherwise it create of conflicts with uid
# objects.
# Create new catalog
portal_catalog = portal.portal_catalog
portal_catalog.manage_renameObject(
id=portal_catalog.manage_pasteObjects(
portal_catalog.manage_copyObjects(ids=(self.original_catalog_id, )),
)[0]['new_id'],
new_id=self.new_catalog_id,
)
source_sql_connection_id_list = [original_connection_id, self.new_erp5_deferred_sql_connection]
destination_sql_connection_id_list = [self.new_erp5_sql_connection, new_deferred_connection_id]
portal_catalog.manage_hotReindexAll(
source_sql_catalog_id=self.original_catalog_id,
destination_sql_catalog_id=self.new_catalog_id,
source_sql_connection_id_list=source_sql_connection_id_list,
destination_sql_connection_id_list=destination_sql_connection_id_list,
update_destination_sql_catalog=True,
)
self.tic()
original_path_list = self.getSQLPathList(original_connection_id)
new_path_list = self.getSQLPathList(self.new_erp5_sql_connection)
self.maxDiff = None
self.assertItemsEqual(original_path_list, new_path_list)
organisation2 = module.newContent(portal_type='Organisation', title="GreatTitle2")
first_deleted_url = organisation2.getRelativeUrl()
self.tic()
path_list = [organisation.getRelativeUrl()]
self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection)
path_list = [first_deleted_url]
self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=original_connection_id)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection)
self.assertEqual(portal.portal_skins.erp5_core.Resource_zGetInventoryList.connection_id, self.new_erp5_sql_connection)
self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_FINISHED_STATE)
# Do a hot reindex in the reverse way, but this time a more
# complicated hot reindex
portal_catalog.manage_hotReindexAll(
source_sql_catalog_id=self.new_catalog_id,
destination_sql_catalog_id=self.original_catalog_id,
source_sql_connection_id_list=destination_sql_connection_id_list,
destination_sql_connection_id_list=source_sql_connection_id_list,
update_destination_sql_catalog=True,
)
self.commit()
self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_RECORDING_STATE)
organisation3 = module.newContent(portal_type='Organisation', title="GreatTitle2")
# Try something more complicated, create new object, reindex it
# and then delete it
deleted_organisation = module.newContent(portal_type='Organisation', title="GreatTitle2")
deleted_organisation.immediateReindexObject()
self.commit()
deleted_url = deleted_organisation.getRelativeUrl()
module.manage_delObjects(ids=[deleted_organisation.getId()])
self.commit()
query = self.portal.cmf_activity_sql_connection().query
query(
'update message set processing_node=-4 where method_id in '
'("playBackRecordedObjectList", "_finishHotReindexing")',
)
hasNoProcessableMessage = lambda message_list: all(
x.processing_node == -4
for x in message_list
)
self.tic(stop_condition=hasNoProcessableMessage)
self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_DOUBLE_INDEXING_STATE)
# try to delete objects in double indexing state
module.manage_delObjects(ids=[organisation2.getId()])
self.commit()
query(
'update message set processing_node=-1 where '
'method_id="playBackRecordedObjectList"',
)
self.tic(stop_condition=hasNoProcessableMessage)
self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_DOUBLE_INDEXING_STATE)
# Now we have started an double indexing
next_deleted_organisation = module.newContent(portal_type='Organisation', title="GreatTitle2",id='toto')
next_deleted_url = next_deleted_organisation.getRelativeUrl()
self.tic(stop_condition=hasNoProcessableMessage)
path_list=[next_deleted_url]
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id)
module.manage_delObjects(ids=[next_deleted_organisation.getId()])
# Create object during the double indexing to check the security object
# after the hot reindexing
module.newContent(portal_type='Organisation', title="GreatTitle2")
self.commit()
query(
'update message set processing_node=-1 where '
'method_id="_finishHotReindexing"',
)
self.tic()
self.assertEqual(portal_catalog.getHotReindexingState(), HOT_REINDEXING_FINISHED_STATE)
# Check Security UID object exist in roles and users
# compare the number object in the catalog
self.assertItemsEqual(
self.getSQLPathList(original_connection_id),
self.getSQLPathListWithRolesAndUsers(original_connection_id),
)
path_list = [organisation3.getRelativeUrl()]
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id)
path_list = [first_deleted_url, deleted_url,next_deleted_url]
self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection)
self.checkRelativeUrlNotInSQLPathList(path_list, connection_id=original_connection_id)
# Make sure module are there
path_list = [module.getRelativeUrl()]
self.checkRelativeUrlInSQLPathList(path_list, connection_id=self.new_erp5_sql_connection)
self.checkRelativeUrlInSQLPathList(path_list, connection_id=original_connection_id)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestVanillaERP5Catalog))
return suite
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7