diff --git a/bt5/erp5_hal_json_style/TestTemplateItem/portal_components/test.erp5.testHalJsonStyle.py b/bt5/erp5_hal_json_style/TestTemplateItem/portal_components/test.erp5.testHalJsonStyle.py index 22fb9b3fe7990817331254c0b75ecd8611bdbcc7..0c66ab6e3435a9aa63b13604c572fc295406c2ad 100644 --- a/bt5/erp5_hal_json_style/TestTemplateItem/portal_components/test.erp5.testHalJsonStyle.py +++ b/bt5/erp5_hal_json_style/TestTemplateItem/portal_components/test.erp5.testHalJsonStyle.py @@ -577,7 +577,7 @@ class TestERP5Document_getHateoas_mode_traverse(ERP5HALJSONStyleSkinsMixin): self.assertEqual(result_dict['_links']['action_workflow'][0]['name'], "custom_action_no_dialog") self.assertEqual(result_dict['_links']['action_object_jump']['href'], - "urn:jio:allDocs?query=portal_type%%3A%%22Query%%22%%20AND%%20default_agent_uid%%3A%sL" % + "urn:jio:allDocs?query=portal_type%%3AQuery%%20AND%%20default_agent_uid%%3A%sL" % document.getUid()) self.assertEqual(result_dict['_links']['action_object_jump']['title'], "Queries") self.assertEqual(result_dict['_links']['action_object_jump']['name'], "jump_query") @@ -909,7 +909,7 @@ class TestERP5Document_getHateoas_mode_traverse(ERP5HALJSONStyleSkinsMixin): self.assertEqual(result_dict['_links']['action_workflow'][0]['name'], "custom_action_no_dialog") self.assertEqual(result_dict['_links']['action_object_jump']['href'], - "urn:jio:allDocs?query=portal_type%%3A%%22Query%%22%%20AND%%20default_agent_uid%%3A%sL" % + "urn:jio:allDocs?query=portal_type%%3AQuery%%20AND%%20default_agent_uid%%3A%sL" % document.getUid()) self.assertEqual(result_dict['_links']['action_object_jump']['title'], "Queries") self.assertEqual(result_dict['_links']['action_object_jump']['name'], "jump_query") @@ -2328,7 +2328,7 @@ class TestERP5Document_getHateoas_mode_worklist(ERP5HALJSONStyleSkinsMixin): self.assertTrue(work_list[0]['count'] > 0) self.assertEqual(work_list[0]['name'], 'Draft To Validate') self.assertFalse('module' in work_list[0]) - self.assertEqual(work_list[0]['href'], 'urn:jio:allDocs?query=portal_type%3A%28%22Bar%22%20OR%20%22Foo%22%29%20AND%20simulation_state%3A%22draft%22') + self.assertEqual(work_list[0]['href'], 'urn:jio:allDocs?query=portal_type%3A%28Bar%20OR%20Foo%29%20AND%20simulation_state%3Adraft') self.assertEqual(result_dict['_debug'], "worklist") @@ -2431,7 +2431,7 @@ return msg" self.assertEqual(work_list[0]['name'], 'daiyanzhen') self.assertEqual(work_list[0]['count'], 1) self.assertFalse('module' in work_list[0]) - self.assertEqual(work_list[0]['href'], 'urn:jio:allDocs?query=portal_type%3A%28%22Bar%22%20OR%20%22Foo%22%29%20AND%20simulation_state%3A%22draft%22') + self.assertEqual(work_list[0]['href'], 'urn:jio:allDocs?query=portal_type%3A%28Bar%20OR%20Foo%29%20AND%20simulation_state%3Adraft') self.assertEqual(result_dict['_debug'], "worklist") diff --git a/bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_jio_js.js b/bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_jio_js.js index 1f82e4b54e46397bd84be961982917a04613b752..690b46fcdbb809f8991d5b6f1a79955cad49c31c 100644 --- a/bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_jio_js.js +++ b/bt5/erp5_web_renderjs_ui/PathTemplateItem/web_page_module/rjs_jio_js.js @@ -7435,7 +7435,16 @@ return new Parser; return new Query(key_schema); } if (typeof object === "string") { - object = parseStringToObject(object); + try { + object = parseStringToObject(object); + } catch (error) { + if (error.hash && error.hash.expected && + error.hash.expected.length === 1 && + error.hash.expected[0] === "'QUOTE'") { + return new query_class_dict.simple({value: object}); + } + throw error; + } } if (typeof (object || {}).type === "string" && query_class_dict[object.type]) { @@ -7445,6 +7454,13 @@ return new Parser; "Argument 1 is not a search text or a parsable object"); }; + function sanitizeQueryValue(value) { + if (typeof value === "string") { + return value.replace(/((?:\\\\)*)\\$/, "$1").replace(/"/g, '\\"'); + } + return value; + } + function objectToSearchText(query) { var i = 0, query_list = null, @@ -7453,7 +7469,8 @@ return new Parser; common_key = ""; if (query.type === "simple") { return (query.key ? query.key + ": " : "") + - (query.operator || "") + ' "' + query.value + '"'; + (query.operator || "") + + ' "' + sanitizeQueryValue(query.value) + '"'; } if (query.type === "complex") { query_list = query.query_list; @@ -7484,7 +7501,7 @@ return new Parser; for (i = 0; i < query_list.length; i += 1) { string_list.push( (query_list[i].operator || "") + - ' "' + query_list[i].value + '"' + ' "' + sanitizeQueryValue(query_list[i].value) + '"' ); } } else { diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/jio.js.js b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/jio.js.js index 1f82e4b54e46397bd84be961982917a04613b752..40a21a62a3a61d322262ad79c4d3dbf1ce3daac6 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/jio.js.js +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/jio.js.js @@ -7435,7 +7435,16 @@ return new Parser; return new Query(key_schema); } if (typeof object === "string") { - object = parseStringToObject(object); + try { + object = parseStringToObject(object); + } catch (error) { + if (error.hash && error.hash.expected && + error.hash.expected.length === 1 && + error.hash.expected[0] === "'QUOTE'") { + return new query_class_dict.simple({value: object}); + } + throw error; + } } if (typeof (object || {}).type === "string" && query_class_dict[object.type]) { @@ -7445,6 +7454,13 @@ return new Parser; "Argument 1 is not a search text or a parsable object"); }; + function sanitizeQueryValue(value) { + if (typeof value === "string") { + return value.replace(/((?:\\\\)*)\\$/, "$1"); + } + return value; + } + function objectToSearchText(query) { var i = 0, query_list = null, @@ -7453,7 +7469,8 @@ return new Parser; common_key = ""; if (query.type === "simple") { return (query.key ? query.key + ": " : "") + - (query.operator || "") + ' "' + query.value + '"'; + (query.operator || "") + + ' "' + sanitizeQueryValue(query.value) + '"'; } if (query.type === "complex") { query_list = query.query_list; @@ -7484,7 +7501,7 @@ return new Parser; for (i = 0; i < query_list.length; i += 1) { string_list.push( (query_list[i].operator || "") + - ' "' + query_list[i].value + '"' + ' "' + sanitizeQueryValue(query_list[i].value) + '"' ); } } else { diff --git a/product/ERP5Type/tests/testERP5Query.py b/product/ERP5Type/tests/testERP5Query.py new file mode 100644 index 0000000000000000000000000000000000000000..ad70a50a8f1e5b78b5550061e4267742cf540900 --- /dev/null +++ b/product/ERP5Type/tests/testERP5Query.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2002-2019 Nexedi SA and Contributors. All Rights Reserved. +# Tristan Cavelier +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import unittest +from unittest import skip +from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase +from Products.ZSQLCatalog.SQLCatalog import SimpleQuery + +class TestERP5Query(ERP5TypeTestCase): + """Test query search text parsing/rendering and document matching. + + The goal is to check that stringified queries are well parsed and + rendered back to the user (mostly for ERP5JS interface) so that originaly + searched objects can strictly be retreived again with the rendered search. + This requires to buildQuery and to render asSearchTextExpression with + the current configuration of the catalog in order to detect + inconsistency, for example in the use of backslashes in the current + full text engine syntax. + """ + + def getTitle(self): + return "ERP5Query" + + def getBusinessTemplateList(self): + return ("erp5_full_text_mroonga_catalog", "erp5_base") + + def _createOrganisationOverwrite(self, **kw): + _id = kw.pop("id") + organisation = getattr(self.portal.organisation_module, _id, None) + if organisation is not None: + self.portal.organisation_module.manage_delObjects(ids=[_id]) + return self.portal.organisation_module.newContent(portal_type="Organisation", id=_id, **kw) + + def _assertQueryKwParsingRenderingMatching(self, query_kw, expected_match_list=None, XXX=False): + catalog = self.portal.portal_catalog + sql_catalog = catalog.getSQLCatalog() + parsed_query = sql_catalog.buildQuery(query_kw) + generated_sql = catalog(query=parsed_query, src__=1) + rendered_search_text = parsed_query.asSearchTextExpression(sql_catalog) + parsed_query_2 = sql_catalog.buildQuery({"search_text": rendered_search_text}) + generated_sql_2 = catalog(query=parsed_query_2, src__=1) + + self.assertEqual(XXX if XXX else generated_sql, generated_sql_2, "{!r} != {!r}\n\ntraceback_info : {!r}".format( + generated_sql, + generated_sql_2, + { + "query_kw": query_kw, + "parsed_query": parsed_query, + "rendered_search_text": rendered_search_text, + "parsed_query_2": parsed_query_2, + }, + )) + + if expected_match_list is None: + return + + expected_match_list = [r.getPath() for r in sorted(expected_match_list, key=lambda o: o.getPath())] + resulting_match_list = [r.path for r in catalog(query=parsed_query, portal_type="Organisation", uid=[o.getUid() for o in self.organisation_list], sort_on=[("path", "ascending")])] + + self.assertEqual(expected_match_list, resulting_match_list) + + def afterSetUp(self): + """ + This is ran before anything, used to set the environment + """ + self.organisation_list = [ + self._createOrganisationOverwrite( + id="test_erp5_query_000", + title="TestERP5QueryBackslashAndText", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_011", + title="TestERP5QueryBackslash\\AndText", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_012", + title="TestERP5QueryBackslash\\\\AndText", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_013", + title="TestERP5QueryBackslash\\\\\\AndText", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_021", + title="TestERP5QueryBackslash\\", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_022", + title="TestERP5QueryBackslash\\\\", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_023", + title="TestERP5QueryBackslash\\\\\\", + ), + self._createOrganisationOverwrite( + id="test_erp5_query_030", + title="TestERP5QuerySpace AndText", + ), + ] + self.organisation_dict = {o.getId()[len("test_erp5_query_"):]: o for o in self.organisation_list} + self.tic() + + def test_query_kw_parsing_rendering_and_matching_with_column(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslashAndText'}, [self.organisation_dict["000"]]) + def test_query_kw_parsing_rendering_and_matching_with_column_and_backslash(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash\\AndText'}, [self.organisation_dict["011"]]) + def test_query_kw_parsing_rendering_and_matching_with_column_and_2_backslashes(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash\\\\AndText'}, [self.organisation_dict["012"]]) + def test_query_kw_parsing_rendering_and_matching_with_column_and_3_backslashes(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash\\\\\\AndText'}, [self.organisation_dict["013"]]) + + def test_query_kw_parsing_rendering_and_matching_with_column_and_no_ending_backslash(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash'}, []) + def test_query_kw_parsing_rendering_and_matching_with_column_and_ending_backslash(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash\\'}, [self.organisation_dict["021"]]) + def test_query_kw_parsing_rendering_and_matching_with_column_and_2_ending_backslashes(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash\\\\'}, [self.organisation_dict["022"]]) + def test_query_kw_parsing_rendering_and_matching_with_column_and_3_ending_backslashes(self): + self._assertQueryKwParsingRenderingMatching({'title': 'TestERP5QueryBackslash\\\\\\'}, [self.organisation_dict["023"]]) + + def test_query_kw_parsing_and_rendering_with_column_and_operator(self): + self._assertQueryKwParsingRenderingMatching({'title': ' + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace(AndText'}, XXX=True) # + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace)AndText'}, XXX=True) # + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace:AndText'}, XXX=True) # + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace>AndText'}, XXX=True) # + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace!AndText'}, XXX=True) # + #self._assertQueryKwParsingRenderingMatching({'search_text': 'title:TestERP5QuerySpace=AndText'}, XXX=True) # + pass + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestERP5Query)) + return suite diff --git a/product/ZSQLCatalog/Operator/OperatorBase.py b/product/ZSQLCatalog/Operator/OperatorBase.py index 66151d60988bc69d9ec46dfc93bc17bbefdd87b0..5b76a9f6e1c8b8f68fbb4f6f74e0a78132e938e5 100644 --- a/product/ZSQLCatalog/Operator/OperatorBase.py +++ b/product/ZSQLCatalog/Operator/OperatorBase.py @@ -34,6 +34,7 @@ from Products.ZSQLCatalog.interfaces.operator import IOperator from Products.ZSQLCatalog.Utils import sqlquote as escapeString from zope.interface.verify import verifyClass from zope.interface import implements +import re def valueFloatRenderer(value): if isinstance(value, basestring): @@ -65,12 +66,22 @@ value_search_text_renderer = { DateTime: str, } +# Allows all ascii chars except query syntax special chars. +# In other words it forbids operator chars !<=> as first character +# plus forbids any other syntax special chars like : or space. +raw_string_validator_re = re.compile(r"^[#\$%&'\*\+,\-\./0-9;\?@A-Z\[\\\]\^_`a-z\{\|\}~][!#\$%&'\*\+,\-\./0-9;<=>\?@A-Z\[\\\]\^_`a-z\{\|\}~]*$") + def valueDefaultSearchTextRenderer(value): """ This is just repr, but always surrounding text strings with doublequotes. """ if isinstance(value, basestring): - result = '"%s"' % (value.replace('\\', '\\\\').replace('"', '\\"'), ) + if raw_string_validator_re.match(value): + result = value + else: + if value.replace("\\\\", "")[-1] == "\\": + value = value[:-1] + result = '"{}"'.format(value.replace('"', '\\"')) else: result = repr(value) return result diff --git a/product/ZSQLCatalog/SearchText/SearchTextParser.py b/product/ZSQLCatalog/SearchText/SearchTextParser.py index a20ff11963c1596ee4ffb73fd880d3a962988fcb..df5882182e28d1e6f3191c2ea38e50c7a3911509 100755 --- a/product/ZSQLCatalog/SearchText/SearchTextParser.py +++ b/product/ZSQLCatalog/SearchText/SearchTextParser.py @@ -279,7 +279,7 @@ if __name__ == '__main__': query_list = [walk(x, key) for x in node.getNodeList()] operator = node.getLogicalOperator() if operator == 'not' or len(query_list) > 1: - result = ComplexQuery(query_list, logical_operator=logical_operator) + result = ComplexQuery(query_list, logical_operator=operator) elif len(query_list) == 1: result = query_list[0] else: