Commit d47df833 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

use fulltext search in title and description.

* to quickly setup catalog_full_text table, you can use the following SQL.

  REPLACE INTO catalog_full_text SELECT uid, title, description FROM catalog;

* non fulltext queries like '=abc', '>abc', '%abc%' are supported.

* now erp5_full_text_mroonga_catalog is used for unit tests thus I recommend using it instead of erp5_full_text_myisam_catalog.

* to migrate existing MyISAM full_text table into Mroonga, you can use the following SQL.

  ALTER TABLE full_text DROP KEY SearchableText,
    ENGINE = mroonga,
    ADD FULLTEXT KEY SearchableText (`SearchableText`) COMMENT 'parser "TokenBigramSplitSymbolAlpha"';

* fulltext search score is no longer provided as (column_name) but now provided as (column_name)__score__.

* (category)_title, like source_title, related keys are automatically generated. (category)_description keys as well.
parent 315d57fb
<key_list>
<key>accounting_transaction_line_node_uid | stock/node_uid/z_related_accounting_transaction_stock_line</key>
<key>accounting_transaction_line_total_price | stock/total_price/z_related_accounting_transaction_stock_line</key>
<key>accounting_transaction_mirror_section_title | catalog/title/z_related_accounting_transaction_mirror_section</key>
<key>accounting_transaction_payment_title | catalog/title/z_related_accounting_transaction_payment</key>
<key>accounting_transaction_project_title | catalog/title/z_related_accounting_transaction_project</key>
<key>accounting_transaction_section_title | catalog/title/z_related_accounting_transaction_section</key>
<key>accounting_transaction_mirror_section_title | catalog_full_text/title/z_related_accounting_transaction_mirror_section</key>
<key>accounting_transaction_payment_title | catalog_full_text/title/z_related_accounting_transaction_payment</key>
<key>accounting_transaction_project_title | catalog_full_text/title/z_related_accounting_transaction_project</key>
<key>accounting_transaction_section_title | catalog_full_text/title/z_related_accounting_transaction_section</key>
<key>preferred_gap_id | category,catalog/id/z_related_preferred_gap</key>
<key>preferred_gap_strict_membership_id | category,catalog/id/z_related_strict_membership_preferred_gap</key>
</key_list>
\ No newline at end of file
preferred_gap_id | category,catalog/id/z_related_preferred_gap
preferred_gap_strict_membership_id | category,catalog/id/z_related_strict_membership_preferred_gap
accounting_transaction_mirror_section_title | catalog/title/z_related_accounting_transaction_mirror_section
accounting_transaction_section_title | catalog/title/z_related_accounting_transaction_section
accounting_transaction_project_title | catalog/title/z_related_accounting_transaction_project
accounting_transaction_payment_title | catalog/title/z_related_accounting_transaction_payment
accounting_transaction_mirror_section_title | catalog_full_text/title/z_related_accounting_transaction_mirror_section
accounting_transaction_section_title | catalog_full_text/title/z_related_accounting_transaction_section
accounting_transaction_project_title | catalog_full_text/title/z_related_accounting_transaction_project
accounting_transaction_payment_title | catalog_full_text/title/z_related_accounting_transaction_payment
accounting_transaction_line_node_uid | stock/node_uid/z_related_accounting_transaction_stock_line
accounting_transaction_line_total_price | stock/total_price/z_related_accounting_transaction_stock_line
\ No newline at end of file
......@@ -53,6 +53,7 @@
<value> <string encoding="cdata"><![CDATA[
from DateTime import DateTime\n
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery\n
\n
# params\n
section_title = \'My Organisation\'\n
......@@ -90,7 +91,7 @@ if 1:\n
def getAccountByTitle(title):\n
account_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Account\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(account_list) == 1, \\\n
\'%d account with title "%s"\' % (len(account_list), title)\n
return account_list[0]\n
......@@ -98,7 +99,7 @@ def getAccountByTitle(title):\n
def getOrganisationByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Organisation\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d organisation with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
......@@ -109,14 +110,14 @@ euro_resource = \'currency_module/euro\'\n
def getBankAccountByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Bank Account\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d Bank Account with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
\n
product_list = [o.getObject() for o in portal.portal_catalog(\n
portal_type=\'Product\',\n
title=\'Dummy Product for testing\')]\n
title=SimpleQuery(title=\'Dummy Product for testing\', comparison_operator=\'=\'))]\n
if product_list:\n
product = product_list[0]\n
else:\n
......
......@@ -53,6 +53,7 @@
<value> <string encoding="cdata"><![CDATA[
from DateTime import DateTime\n
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery\n
\n
# params\n
section_title = \'My Organisation\'\n
......@@ -80,7 +81,7 @@ if 1:\n
def getAccountByTitle(title):\n
account_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Account\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(account_list) == 1, \\\n
\'%d account with title "%s"\' % (len(account_list), title)\n
return account_list[0]\n
......@@ -88,7 +89,7 @@ def getAccountByTitle(title):\n
def getOrganisationByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Organisation\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d organisation with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
......@@ -106,7 +107,7 @@ euro_resource = getCurrencyByReference(\'EUR\')\n
def getBankAccountByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Bank Account\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d Bank Account with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
......
......@@ -58,6 +58,7 @@ business_process = \'business_process_module/erp5_default_business_process\'\n
portal = context.getPortalObject()\n
accounting_module = portal.accounting_module\n
from DateTime import DateTime\n
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery\n
year = 2005\n
\n
# if the previous test didn\'t change input data, no need to recreate content\n
......@@ -82,7 +83,7 @@ if 1:\n
def getAccountByTitle(title):\n
account_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Account\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(account_list) == 1, \\\n
\'%d account with title "%s"\' % (len(account_list), title)\n
return account_list[0]\n
......@@ -90,7 +91,7 @@ def getAccountByTitle(title):\n
def getOrganisationByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Organisation\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d organisation with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
......@@ -99,7 +100,7 @@ section = getOrganisationByTitle(section_title)\n
def getPersonByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Person\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d person with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
......@@ -116,7 +117,7 @@ euro_resource = getCurrencyByReference(\'EUR\')\n
def getBankAccountByTitle(title):\n
document_list = [x.getObject().getRelativeUrl() for x in\n
portal.portal_catalog(portal_type=\'Bank Account\',\n
title=title)]\n
title=SimpleQuery(title=title, comparison_operator=\'=\'))]\n
assert len(document_list) == 1, \\\n
\'%d Bank Account with title "%s"\' % (len(document_list), title)\n
return document_list[0]\n
......
......@@ -51,6 +51,7 @@
<item>
<key> <string>_body</string> </key>
<value> <string>portal = context.getPortalObject()\n
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery\n
\n
# validate rules\n
for rule in portal.portal_rules.objectValues():\n
......@@ -129,7 +130,7 @@ stool.setSelectionColumns(\'account_module_selection\',\n
\n
# delete the "dummy account" we create in test_account_gap_parallel_list_field\n
dummy_account_list = portal.account_module.searchFolder(\n
title=\'Dummy Account for UI Test\')\n
title=SimpleQuery(title=\'Dummy Account for UI Test\', comparison_operator=\'=\'))\n
if dummy_account_list:\n
portal.account_module.manage_delObjects([dummy_account_list[0].getId()])\n
\n
......
......@@ -2,8 +2,8 @@
<key>child_address_SearchableText | catalog,full_text/SearchableText/z_related_child_address</key>
<key>child_telephone_SearchableText | catalog,full_text/SearchableText/z_related_child_telephone</key>
<key>default_email_text | catalog,email/url_string/z_related_default_email</key>
<key>destination_person_title | category,catalog/title/z_related_destination_person</key>
<key>destination_person_title | category,catalog_full_text/title/z_related_destination_person</key>
<key>related_resource_use_uid | category,category,catalog,catalog/uid/z_related_resource_use</key>
<key>source_organisation_title | category,catalog/title/z_related_source_organisation</key>
<key>source_person_title | category,catalog/title/z_related_source_person</key>
<key>source_organisation_title | category,catalog_full_text/title/z_related_source_organisation</key>
<key>source_person_title | category,catalog_full_text/title/z_related_source_person</key>
</key_list>
\ No newline at end of file
source_organisation_title | category,catalog/title/z_related_source_organisation
source_person_title | category,catalog/title/z_related_source_person
destination_person_title | category,catalog/title/z_related_destination_person
source_organisation_title | category,catalog_full_text/title/z_related_source_organisation
source_person_title | category,catalog_full_text/title/z_related_source_person
destination_person_title | category,catalog_full_text/title/z_related_destination_person
default_email_text | catalog,email/url_string/z_related_default_email
related_resource_use_uid | category,category,catalog,catalog/uid/z_related_resource_use
child_address_SearchableText | catalog,full_text/SearchableText/z_related_child_address
......
erp5_full_text_myisam_catalog
\ No newline at end of file
erp5_full_text_mroonga_catalog
\ No newline at end of file
erp5_full_text_myisam_catalog
\ No newline at end of file
erp5_full_text_mroonga_catalog
\ No newline at end of file
erp5_core_proxy_field_legacy
erp5_full_text_myisam_catalog
erp5_full_text_mroonga_catalog
erp5_base
erp5_workflow
\ No newline at end of file
erp5_core_proxy_field_legacy
erp5_full_text_myisam_catalog
erp5_full_text_mroonga_catalog
erp5_base
erp5_workflow
\ No newline at end of file
erp5_full_text_myisam_catalog
\ No newline at end of file
erp5_full_text_mroonga_catalog
\ No newline at end of file
erp5_core_proxy_field_legacy
erp5_full_text_myisam_catalog
erp5_full_text_mroonga_catalog
erp5_base
erp5_workflow
\ No newline at end of file
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery, AndQuery\n
\n
########### CONFIGURATION ######################################\n
# This is a query for search translated properties of Person #\n
# documents. If you want to get other results. Customise this. # \n
################################################################\n
\n
query = AndQuery(SimpleQuery(**{\'content_translation.translated_text\': value, \'comparison_operator\': \'match\'}),\n
Query(**{\'content_translation.property_name\': \'title\'})\n
)\n
\n
################################################################\n
# Above query must make SQL condition like this. #\n
################################################################\n
# MATCH(content_translation.translated_text) AGAINST({value})\n
# AND content_translation.property_name = \'title\'\n
\n
return query\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>value</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_makeContentTranslationSearchQuery</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<key_list>
<key>content_translation_title | SQLCatalog_makeContentTranslationSearchQuery</key>
<key>content_translation_title | SQLCatalog_makeTranslatedTitleQuery</key>
</key_list>
\ No newline at end of file
content_translation_title | SQLCatalog_makeContentTranslationSearchQuery
\ No newline at end of file
content_translation_title | SQLCatalog_makeTranslatedTitleQuery
\ No newline at end of file
......@@ -6,5 +6,5 @@
<key>related_source_or_source_decision_or_destination_or_destination_decision | category,catalog,catalog/uid/z_related_source_or_source_decision_or_destination_or_destination_decision</key>
<key>related_source_or_source_section_or_destination_or_destination_section | category,catalog,catalog/uid/z_related_source_or_source_section_or_destination_or_destination_section</key>
<key>related_source_section_or_destination_section | category,catalog,catalog/uid/z_related_source_section_or_destination_section</key>
<key>source_organisation_title | category,catalog/title/z_related_source_organisation</key>
<key>source_organisation_title | category,catalog_full_text/title/z_related_source_organisation</key>
</key_list>
\ No newline at end of file
source_organisation_title | category,catalog/title/z_related_source_organisation
source_organisation_title | category,catalog_full_text/title/z_related_source_organisation
event_causality_ticket_uid | category,catalog,category,catalog/uid/z_related_event_causality_ticket
related_source_or_destination | category,catalog,catalog/uid/z_related_source_or_destination
related_source_section_or_destination_section | category,catalog,catalog/uid/z_related_source_section_or_destination_section
......
......@@ -51,7 +51,7 @@
<item>
<key> <string>_body</string> </key>
<value> <string>return (\'erp5_core_proxy_field_legacy\',\n
\'erp5_full_text_myisam_catalog\',\n
\'erp5_full_text_mroonga_catalog\',\n
\'erp5_base\',\n
\'erp5_workflow\',\n
\'erp5_configurator\',\n
......
......@@ -51,7 +51,7 @@
<item>
<key> <string>_body</string> </key>
<value> <string>return (\'erp5_core_proxy_field_legacy\',\n
\'erp5_full_text_myisam_catalog\',\n
\'erp5_full_text_mroonga_catalog\',\n
\'erp5_base\',\n
\'erp5_simulation\',\n
\'erp5_dhtml_style\',\n
......
erp5_full_text_myisam_catalog
erp5_full_text_mroonga_catalog
erp5_base
erp5_web
erp5_ingestion_mysql_innodb_catalog
......
<key_list>
<key>SearchableText</key>
<key>full_text.SearchableText</key>
</key_list>
\ No newline at end of file
......@@ -68,8 +68,9 @@ for path in path_list:\n
try:\n
tmp_dict = {}\n
for property in property_list:\n
if property == \'SearchableText\':\n
value = obj.SearchableText()\n
getter = getattr(obj, property, None)\n
if getter is not None and callable(getter):\n
value = getter()\n
else:\n
value = getattr(obj, \'get%s\' % UpperCase(property))()\n
tmp_dict[property] = value\n
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>DROP TABLE IF EXISTS catalog_full_text</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_deferred_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_catalog_fulltext</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int>
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid\r\n
getTitle\r\n
getDescription</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_deferred_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_catalog_fulltext_list</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
REPLACE INTO\n
catalog_full_text (`uid`, `title`, `description`)\n
VALUES\n
<dtml-in prefix="loop" expr="_.range(_.len(uid))">\n
(\n
<dtml-sqlvar expr="uid[loop_item]" type="int">, \n
<dtml-sqlvar expr="getTitle[loop_item]" type="string" optional>,\n
<dtml-sqlvar expr="getDescription[loop_item]" type="string" optional>\n
)<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>