Commit 7cfdac13 authored by Tristan Cavelier's avatar Tristan Cavelier Committed by Kazuhiko Shiozaki

implement fulltext search for title and description.

parent 108a61f4
......@@ -68,8 +68,9 @@ for path in path_list:\n
try:\n
tmp_dict = {}\n
for property in property_list:\n
if property == \'SearchableText\':\n
value = obj.SearchableText()\n
getter = getattr(obj, property, None)\n
if getter is not None and callable(getter):\n
value = getter()\n
else:\n
value = getattr(obj, \'get%s\' % UpperCase(property))()\n
tmp_dict[property] = value\n
......
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS catalog_full_text</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_deferred_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -15,7 +15,9 @@
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid\r\n
SearchableText</string> </value>
SearchableText\r\n
getTitle\r\n
getDescription</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
......@@ -79,7 +81,19 @@ VALUES\n
)<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n
</dtml-if>\n
</dtml-let>
</dtml-let>\n
<dtml-var sql_delimiter>\n
REPLACE INTO\n
catalog_full_text (`uid`, `title`, `description`)\n
VALUES\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
(\n
<dtml-sqlvar expr="uid[loop_item]" type="int">, \n
<dtml-sqlvar expr="getTitle[loop_item]" type="string" optional>,\n
<dtml-sqlvar expr="getDescription[loop_item]" type="string" optional>\n
)<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n
]]></string> </value>
</item>
......
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string># Host:\n
# Database: test\n
# Table: \'catalog_full_text\'\n
#\n
CREATE TABLE `catalog_full_text` (\n
`uid` BIGINT UNSIGNED NOT NULL,\n
`title` varchar(255) default \'\',\n
`description` text,\n
PRIMARY KEY (`uid`),\n
FULLTEXT `title` (`title`),\n
FULLTEXT `description` (`description`)\n
) ENGINE=mroonga;\n
</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<key_list>
<key>description | catalog_full_text/description/z_related_uid</key>
<key>title | catalog_full_text/title/z_related_uid</key>
</key_list>
\ No newline at end of file
<key_list>
<key>catalog_full_text</key>
<key>full_text</key>
</key_list>
\ No newline at end of file
<key_list>
<key>SearchableText | MroongaFullTextKey</key>
<key>catalog_full_text.description | MroongaFullTextKey</key>
<key>catalog_full_text.title | MroongaBooleanFullTextKey</key>
<key>description | MroongaFullTextKey</key>
<key>full_text.SearchableText | MroongaFullTextKey</key>
<key>title | MroongaBooleanFullTextKey</key>
</key_list>
\ No newline at end of file
erp5_mysql_innodb/SQLCatalog_deferFullTextIndex
erp5_mysql_innodb/SQLCatalog_deferFullTextIndexActivity
erp5_mysql_innodb/SQLCatalog_makeFullTextQuery
erp5_mysql_innodb/z0_drop_catalog_fulltext
erp5_mysql_innodb/z0_drop_content_translation
erp5_mysql_innodb/z0_drop_fulltext
erp5_mysql_innodb/z0_uncatalog_catalog_fulltext
erp5_mysql_innodb/z0_uncatalog_fulltext
erp5_mysql_innodb/z_catalog_fulltext_list
erp5_mysql_innodb/z_create_catalog_fulltext
erp5_mysql_innodb/z_create_content_translation
erp5_mysql_innodb/z_create_fulltext
\ No newline at end of file
title | catalog_full_text/title/z_related_uid
description | catalog_full_text/description/z_related_uid
\ No newline at end of file
SearchableText | MroongaFullTextKey
full_text.SearchableText | MroongaFullTextKey
\ No newline at end of file
catalog_full_text.description | MroongaFullTextKey
catalog_full_text.title | MroongaBooleanFullTextKey
description | MroongaFullTextKey
full_text.SearchableText | MroongaFullTextKey
title | MroongaBooleanFullTextKey
\ No newline at end of file
<key_list>
<key>SearchableText</key>
<key>full_text.SearchableText</key>
<key>catalog_full_text.description</key>
<key>catalog_full_text.title</key> |
<key>description</key> |
<key>full_text.SearchableText</key> |
<key>title</key> |
</key_list>
\ No newline at end of file
......@@ -68,8 +68,9 @@ for path in path_list:\n
try:\n
tmp_dict = {}\n
for property in property_list:\n
if property == \'SearchableText\':\n
value = obj.SearchableText()\n
getter = getattr(obj, property, None)\n
if getter is not None and callable(getter):\n
value = getter()\n
else:\n
value = getattr(obj, \'get%s\' % UpperCase(property))()\n
tmp_dict[property] = value\n
......
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS catalog_full_text</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_deferred_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -15,7 +15,9 @@
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid\r\n
SearchableText</string> </value>
SearchableText\r\n
getTitle\r\n
getDescription</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
......@@ -79,7 +81,19 @@ VALUES\n
)<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n
</dtml-if>\n
</dtml-let>
</dtml-let>\n
<dtml-var sql_delimiter>\n
REPLACE INTO\n
catalog_full_text (`uid`, `title`, `description`)\n
VALUES\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
(\n
<dtml-sqlvar expr="uid[loop_item]" type="int">, \n
<dtml-sqlvar expr="getTitle[loop_item]" type="string" optional>,\n
<dtml-sqlvar expr="getDescription[loop_item]" type="string" optional>\n
)<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n
]]></string> </value>
</item>
......
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string># Host:\n
# Database: test\n
# Table: \'catalog_full_text\'\n
#\n
CREATE TABLE `catalog_full_text` (\n
`uid` BIGINT UNSIGNED NOT NULL,\n
`title` varchar(255) default \'\',\n
`description` text,\n
PRIMARY KEY (`uid`),\n
FULLTEXT `title` (`title`),\n
FULLTEXT `description` (`description`)\n
) ENGINE=mroonga;\n
</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<key_list>
<key>description | catalog_full_text/description/z_related_uid</key>
<key>title | catalog_full_text/title/z_related_uid</key>
</key_list>
\ No newline at end of file
<key_list>
<key>catalog_full_text</key>
<key>full_text</key>
</key_list>
\ No newline at end of file
SearchableText
full_text.SearchableText
\ No newline at end of file
catalog_full_text.description
catalog_full_text.title
description
full_text.SearchableText
title
\ No newline at end of file
erp5_mysql_innodb/SQLCatalog_deferFullTextIndex
erp5_mysql_innodb/SQLCatalog_deferFullTextIndexActivity
erp5_mysql_innodb/SQLCatalog_makeFullTextQuery
erp5_mysql_innodb/z0_drop_catalog_fulltext
erp5_mysql_innodb/z0_drop_content_translation
erp5_mysql_innodb/z0_drop_fulltext
erp5_mysql_innodb/z0_uncatalog_catalog_fulltext
erp5_mysql_innodb/z0_uncatalog_fulltext
erp5_mysql_innodb/z_catalog_fulltext_list
erp5_mysql_innodb/z_create_catalog_fulltext
erp5_mysql_innodb/z_create_content_translation
erp5_mysql_innodb/z_create_fulltext
\ No newline at end of file
title | catalog_full_text/title/z_related_uid
description | catalog_full_text/description/z_related_uid
\ No newline at end of file
......@@ -43,21 +43,58 @@ class TestI18NSearch(ERP5TypeTestCase):
portal_type='Person',
first_name='Gabriel',
last_name='Fauré',
description='Quick brown fox jumps over the lazy dog.',
)
person2 = person_module.newContent(
portal_type='Person',
first_name='武者小路',
last_name='実篤'
last_name='実篤',
description='Slow white fox jumps over the diligent dog.',
)
self.tic()
# check if 'é' == 'e' collation works
result = person_module.searchFolder(SearchableText='Faure')
self.assertEqual(len(result), 1)
self.assertEqual(result[0].getPath(), person1.getPath())
result = person_module.searchFolder(title='Faure')
self.assertEqual(len(result), 1)
self.assertEqual(result[0].getPath(), person1.getPath())
# check if a partial string of CJK string matches
result = person_module.searchFolder(SearchableText='武者')
self.assertEqual(len(result), 1)
self.assertEqual(result[0].getPath(), person2.getPath())
result = person_module.searchFolder(title='武者')
self.assertEqual(len(result), 1)
self.assertEqual(result[0].getPath(), person2.getPath())
# check natural language mode search
result = person_module.searchFolder(SearchableText='quick fox dog')
self.assertEqual(len(result), 2)
sorted_result = sorted(list(result), key=lambda x:x.SearchableText, reverse=True)
# all of 'quick', 'fox' and 'dog' match.
self.assertEqual(result[0].getPath(), person1.getPath())
# only 'fox' and 'dog' match.
self.assertEqual(result[1].getPath(), person2.getPath())
result = person_module.searchFolder(description='quick fox dog')
self.assertEqual(len(result), 2)
sorted_result = sorted(list(result), key=lambda x:x.description, reverse=True)
# all of 'quick', 'fox' and 'dog' match.
self.assertEqual(result[0].getPath(), person1.getPath())
# only 'fox' and 'dog' match.
self.assertEqual(result[1].getPath(), person2.getPath())
# check boolean language mode search
result = person_module.searchFolder(SearchableText='+quick +fox +dog')
self.assertEqual(len(result), 1)
self.assertEqual(result[0].getPath(), person1.getPath())
result = person_module.searchFolder(description='+quick +fox +dog')
self.assertEqual(len(result), 1)
self.assertEqual(result[0].getPath(), person1.getPath())
# check fulltext search for automatically generated related keys.
self.assertTrue('MATCH' in self.portal.portal_catalog(destination_title='Faure', src__=1))
def test_suite():
suite = unittest.TestSuite()
......
......@@ -881,9 +881,10 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
by looking at the category tree.
For exemple it will generate:
destination_title | category,catalog/title/z_related_destination
default_destination_title | category,catalog/title/z_related_destination
strict_destination_title | category,catalog/title/z_related_strict_destination
destination_reference | category,catalog/reference/z_related_destination
default_destination_reference | category,catalog/reference/z_related_destination
strict_destination_reference | category,catalog/reference/z_related_strict_destination
destination_title | category,catalog_full_text/title/z_related_destination
strict_ related keys only returns documents which are strictly member of
the category.
......@@ -917,9 +918,18 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
if related:
end_key = end_key[len(related_string):]
# XXX: joining with non-catalog tables is not trivial and requires
# ZSQLCatalog's ColumnMapper cooperation, so only allow catalog
# columns.
if 'catalog' in column_map.get(end_key, ()):
# ZSQLCatalog's ColumnMapper cooperation, so only allow columns in
# catalog or catalog_full_text tables.
if end_key != 'uid' and 'catalog_full_text' in column_map.get(end_key, ()):
related_key_list.append(
prefix + key + ' | category,catalog_full_text/' +
end_key +
'/z_related_' +
('strict_' if strict else '') +
expected_base_cat_id +
('_related' if related else '')
)
elif 'catalog' in column_map.get(end_key, ()):
is_uid = end_key == 'uid'
if is_uid:
end_key = 'uid' if related else 'category_uid'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment