Skip to content

erp5
erp5
Commit 3aeb0ab9 authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Refactoring of ZSQLCatalog. 

Introduce search key which now are reposnsible for Query generation.
Improve parsing of search strings (you must install "ply":http://www.dabeaz.com/ply/).



git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@19152 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 8d482a11
Show whitespace changes
Inline Side-by-side
Showing with 1881 additions and 444 deletions
product/ZSQLCatalog/Query/ComplexQuery.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.PythonScripts.Utility import allow_class
from Query import QueryMixin
class ComplexQuery(QueryMixin):
"""
Used in order to concatenate many queries
"""
def __init__(self, *args, **kw):
self.query_list = args
self.operator = kw.pop('operator', 'AND')
# XXX: What is that used for ?! It's utterly dangerous.
#self.__dict__.update(kw)
def getQueryList(self):
return self.query_list
def getRelatedTableMapDict(self):
result = {}
for query in self.getQueryList():
if not(isinstance(query, basestring)):
result.update(query.getRelatedTableMapDict())
return result
def asSQLExpression(self, key_alias_dict=None,
ignore_empty_string=1,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
stat__=0):
"""
Build the sql string
"""
sql_expression_list = []
select_expression_list = []
for query in self.getQueryList():
if isinstance(query, basestring):
sql_expression_list.append(query)
else:
query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
ignore_empty_string=ignore_empty_string,
keyword_search_keys=keyword_search_keys,
datetime_search_keys=datetime_search_keys,
full_text_search_keys=full_text_search_keys,
stat__=stat__)
sql_expression_list.append(query_result['where_expression'])
select_expression_list.extend(query_result['select_expression_list'])
operator = self.getOperator()
result = {'where_expression':('(%s)' % \
(' %s ' % operator).join(['(%s)' % x for x in sql_expression_list])),
'select_expression_list':select_expression_list}
return result
def getSQLKeyList(self):
"""
Returns the list of keys used by this
instance
"""
key_list=[]
for query in self.getQueryList():
if not(isinstance(query, basestring)):
key_list.extend(query.getSQLKeyList())
return key_list
allow_class(ComplexQuery)
product/ZSQLCatalog/Query/Query.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
class QueryMixin:
"""
Mixing class which implements methods which are
common to all kinds of Queries
"""
operator = None
format = None
type = None
def __call__(self, **kw):
return self.asSQLExpression(**kw)
def getOperator(self):
return self.operator
def getFormat(self):
return self.format
def getType(self):
return self.type
def getRange(self):
return self.range
def getTableAliasList(self):
return self.table_alias_list
def getSearchMode(self):
"""Search mode used for Full Text search
"""
return self.search_mode
def getSearchKey(self):
"""Search mode used for Full Text search
"""
return self.search_key
def getKey(self):
return self.key
def getValue(self):
return self.value
def getOperator(self):
return self.operator.upper().strip()
def asSearchTextExpression(self):
raise NotImplementedError
def asSQLExpression(self, key_alias_dict=None,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
ignore_empty_string=1, stat__=0):
"""
Return a dictionnary containing the keys and value types:
'where_expression': string
'select_expression_list': string
"""
raise NotImplementedError
def getSQLKeyList(self):
"""
Return a list of keys used by this query and its subqueries.
"""
raise NotImplementedError
def getRelatedTableMapDict(self):
"""
Return for each key used by this query (plus ones used by its
subqueries) the table alias mapping.
"""
raise NotImplementedError
def _quoteSQLString(self, value):
"""Return a quoted string of the value.
XXX: Left for backwards compatability!
"""
format = self.getFormat()
type = self.getType()
if format is not None and type is not None:
if type == 'date':
if hasattr(value, 'strftime'):
value = value.strftime(format)
if isinstance(value, basestring):
value = "STR_TO_DATE('%s','%s')" % (value, format)
if type == 'float':
# Make sure there is no space in float values
value = value.replace(' ','')
value = "'%s'" % value
else:
if getattr(value, 'ISO', None) is not None:
value = "'%s'" % value.toZone('UTC').ISO()
else:
value = "'%s'" % sql_quote(str(value))
return value
def _quoteSQLKey(self, key):
"""Return a quoted string of the value.
XXX: Left for backwards compatability!
"""
format = self.getFormat()
type = self.getType()
if format is not None and type is not None:
if type == 'date':
key = "STR_TO_DATE(DATE_FORMAT(%s,'%s'),'%s')" % (key, format, format)
if type == 'float':
float_format = format.replace(' ','')
if float_format.find('.') >= 0:
precision = len(float_format.split('.')[1])
key = "TRUNCATE(%s,%s)" % (key, precision)
return key
product/ZSQLCatalog/Query/SimpleQuery.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.PythonScripts.Utility import allow_class
from DateTime import DateTime
from Query import QueryMixin
from pprint import pprint
# valid search modes for queries
FULL_TEXT_SEARCH_MODE = 'FullText'
EXACT_MATCH_SEARCH_MODE = 'ExactMatch'
KEYWORD_SEARCH_MODE = 'Keyword'
DATETIME_SEARCH_MODE = 'DateTime'
def isSimpleType(value):
return isinstance(value, basestring) or \
isinstance(value, int) or \
isinstance(value, long) or \
isinstance(value, float)
# XXX Bad name JPS - NotQuery or NegativeQuery is better NegationQuery
class NegatedQuery(QueryMixin):
"""
Do a boolean negation of given query.
"""
def __init__(self, query):
self._query = query
def asSQLExpression(self, *args, **kw):
sql_expression_dict = self._query.asSQLExpression(*args, **kw)
sql_expression_dict['where_expression'] = '(NOT (%s))' % \
(sql_expression_dict['where_expression'], )
return sql_expression_dict
def getSQLKeyList(self, *args, **kw):
return self._query.getSQLKeyList(*args, **kw)
def getRelatedTableMapDict(self, *args, **kw):
return self._query.getRelatedTableMapDict(*args, **kw)
allow_class(NegatedQuery)
class SimpleQuery(QueryMixin):
"""
This allow to define constraints on a sql column
format - type date : %d/%m/%Y
type float : 1 234.12
"""
def __init__(self, format=None, operator=None, range=None, key=None,
search_mode=None, table_alias_list=None, type=None, **kw):
self.format = format
if operator is None:
operator = 'OR'
self.operator = operator
self.range = range
self.search_mode = search_mode
self.table_alias_list = table_alias_list
key_list = kw.keys()
if len(key_list) != 1:
raise KeyError, 'Query must have only one key'
self.key = key_list[0]
self.value = kw[self.key]
self.type = type
self.search_key = key
def getRelatedTableMapDict(self):
result = {}
table_alias_list = self.getTableAliasList()
if table_alias_list is not None:
result[self.getKey()] = table_alias_list
return result
def getSQLKeyList(self):
"""
Returns the list of keys used by this
instance
"""
return [self.getKey()]
def asSearchTextExpression(self):
# This will be the standard way to represent
# complex values in listbox. Some fixed
# point must be garanteed
value = self.getValue()
if isSimpleType(value) or isinstance(value, DateTime):
return str(value)
elif isinstance(value, (list, tuple)):
value = map(lambda x:str(x), value)
return (' %s ' % self.operator).join(value)
def _getSearchKeyClassByType(self, type, search_key_class = None):
""" Return search key class based on type of value. """
name_search_key_map = {'keyword': KeyWordKey,
'default': DefaultKey,
'fulltext': FullTextKey,
'date': DateTimeKey,
'float': FloatKey,
'int': DefaultKey,}
return name_search_key_map.get(type, search_key_class)
def _getSearchKeyClassByValue(self, value, search_key_class = None):
""" Return search key class based on type of value. """
if isinstance(value, basestring):
if value.find('%')!=-1:
# it's likely a KeyWordKey
search_key_class = KeyWordKey
else:
search_key_class = DefaultKey
elif isinstance(value, DateTime):
search_key_class = DateTimeKey
elif isinstance(value, (int, long,)):
search_key_class = DefaultKey
elif isinstance(value, float):
search_key_class = FloatKey
return search_key_class
def _asSQLExpression(self, search_key_class, key, value, format=None, mode=None, range_value=None, stat__=None):
""" Generate SQL expressions based on respective search_key passed. """
lexer = getSearchKeyInstance(search_key_class)
where_expression, select_expression_list = \
lexer.buildSQLExpression(key, value, format, mode, range_value, stat__)
sql_expressions = {'where_expression': where_expression,
'select_expression_list': select_expression_list,}
return sql_expressions
def asSQLExpression(self, key_alias_dict = None, keyword_search_keys = [],
datetime_search_keys = [], full_text_search_keys = [],
ignore_empty_string = 1, stat__ = 0):
"""
Build the sql expressions string
"""
search_key_class = None
value = self.getValue()
key = self.getKey()
operator = self.getOperator()
type = self.getType()
format = self.getFormat()
search_mode = self.getSearchMode()
range_value = self.getRange()
search_key = self.getSearchKey()
# key can have an alias definition which we should acquire
if key_alias_dict is not None:
key = key_alias_dict.get(key, None)
search_key_class = None
where_expression_list = []
select_expression_list = []
sql_expressions = {'where_expression': '1',
'select_expression_list': []}
# some use cases where we can just return SQL without grammar staff
if key is None or (ignore_empty_string and \
isinstance(value, basestring) and \
value.strip() == ''):
# do not further generate sql expressions because
# we ignore empty strings by default
return sql_expressions
elif ignore_empty_string==0 and isinstance(value, basestring) and value.strip() == '':
# explicitly requested not to ignore empty strings
sql_expressions = {'where_expression': "%s = ''" %key,
'select_expression_list': []}
return sql_expressions
else:
# search for 'NULL' values
if value is None:
sql_expressions = {'where_expression': "%s is NULL" % (key),
'select_expression_list': [],}
return sql_expressions
# get search class based on explicitly passed key type
if search_key_class is None:
search_key_class = self._getSearchKeyClassByType(type)
# we have a list of values and respective operator defined
if isinstance(value, (tuple, list)):
if range_value is None:
# use operators to build sql expressions
if operator in ('IN',):
# values in list are not treated as searchable strings but
# they should be SQL quoted at least
if len(value) > 1:
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
search_key_instance = getSearchKeyInstance(search_key_class)
escaped_value_list = [search_key_instance.quoteSQLString(x, format) for x in value]
escaped_value_string = ', '.join(escaped_value_list)
where_expression_list.append("%s IN (%s)" % (key, escaped_value_string))
elif len(value) == 1:
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
search_key_instance = getSearchKeyInstance(search_key_class)
where_expression_list.append("%s = %s"
%(key, search_key_instance.quoteSQLString(value[0], format)))
else:
# empty list
where_expression_list.append("0")
elif operator in ('OR', 'AND',):
# each of the list elements can be treated as a Key, so
# leave SQL generation to Key itself
if len(value) > 1:
sql_logical_sub_expressions = []
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
for item in value:
list_item_sql_expressions = self._asSQLExpression(search_key_class, key, \
item, format, search_mode, range_value, stat__)
sql_logical_sub_expressions.append('%s' %list_item_sql_expressions['where_expression'])
# join list items (now sql logical expressions) using respective operator
where_expression = (' %s ' %operator).join(sql_logical_sub_expressions)
where_expression_list.append("(%s)" % (where_expression))
elif len(value) == 1:
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
item_sql_expressions = self._asSQLExpression(search_key_class, key, \
value[0], format, search_mode, range_value, stat__)
where_expression_list.append(item_sql_expressions['where_expression'])
# join where expressions list
where_expression = ' '.join(where_expression_list)
sql_expressions = {'where_expression': where_expression,
'select_expression_list': [],}
return sql_expressions
else:
# we can have range specified
if search_key_class is None:
# try to guess by type of first_element in list
search_key_class = self._getSearchKeyClassByValue(value[0])
# try to get search key type by the key definitions passed
if search_key_class is None:
if search_key == EXACT_MATCH_SEARCH_MODE:
search_key_class = RawKey
elif search_key == KEYWORD_SEARCH_MODE or \
(key in keyword_search_keys):
search_key_class = KeyWordKey
elif search_key == DATETIME_SEARCH_MODE or \
(key in datetime_search_keys):
search_key_class = DateTimeKey
elif search_key == FULL_TEXT_SEARCH_MODE or \
(key in full_text_search_keys):
search_key_class = FullTextKey
# get search class based on value of value
if search_key_class is None:
search_key_class = self._getSearchKeyClassByValue(value)
# last fallback case
if search_key_class is None:
search_key_class = DefaultKey
# use respective search key.
sql_expressions = self._asSQLExpression(search_key_class, key,
value, format, search_mode, range_value, stat__)
return sql_expressions
allow_class(SimpleQuery)
from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
product/ZSQLCatalog/SQLCatalog.py
View file @ 3aeb0ab9
...@@ -114,13 +114,6 @@ def manage_addSQLCatalog(self, id, title, ...@@ -114,13 +114,6 @@ def manage_addSQLCatalog(self, id, title,
if REQUEST is not None: if REQUEST is not None:
return self.manage_main(self, REQUEST,update_menu=1) return self.manage_main(self, REQUEST,update_menu=1)
def isSimpleType(value):
return isinstance(value, basestring) or \
isinstance(value, int) or \
isinstance(value, long) or \
isinstance(value, float)
class UidBuffer(TM): class UidBuffer(TM):
"""Uid Buffer class caches a list of reserved uids in a transaction-safe way.""" """Uid Buffer class caches a list of reserved uids in a transaction-safe way."""
...@@ -188,414 +181,6 @@ class UidBuffer(TM): ...@@ -188,414 +181,6 @@ class UidBuffer(TM):
tid = get_ident() tid = get_ident()
self.temporary_buffer.setdefault(tid, []).extend(iterable) self.temporary_buffer.setdefault(tid, []).extend(iterable)
# valid search modes for queries
FULL_TEXT_SEARCH_MODE = 'FullText'
EXACT_MATCH_SEARCH_MODE = 'ExactMatch'
KEYWORD_SEARCH_MODE = 'Keyword'
DATETIME_SEARCH_MODE = 'DateTime'
class QueryMixin:
"""
Mixing class which implements methods which are
common to all kinds of Queries
"""
operator = None
format = None
type = None
def getOperator(self):
return self.operator
def getFormat(self):
return self.format
def getType(self):
return self.type
def getLogicalOperator(self):
return self.logical_operator
def _quoteSQLString(self, value):
"""Return a quoted string of the value.
"""
format = self.getFormat()
type = self.getType()
if format is not None and type is not None:
if type == 'date':
if hasattr(value, 'strftime'):
value = value.strftime(format)
if isinstance(value, basestring):
value = "STR_TO_DATE('%s','%s')" % (value, format)
if type == 'float':
# Make sure there is no space in float values
value = value.replace(' ','')
value = "'%s'" % value
else:
if getattr(value, 'ISO', None) is not None:
value = "'%s'" % value.toZone('UTC').ISO()
else:
value = "'%s'" % sql_quote(str(value))
return value
def _quoteSQLKey(self, key):
"""Return a quoted string of the value.
"""
format = self.getFormat()
type = self.getType()
if format is not None and type is not None:
if type == 'date':
key = "STR_TO_DATE(DATE_FORMAT(%s,'%s'),'%s')" % (key, format, format)
if type == 'float':
float_format = format.replace(' ','')
if float_format.find('.') >= 0:
precision = len(float_format.split('.')[1])
key = "TRUNCATE(%s,%s)" % (key, precision)
return key
def asSQLExpression(self, key_alias_dict=None,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
ignore_empty_string=1, stat__=0):
"""
Return a dictionnary containing the keys and value types:
'where_expression': string
'select_expression_list': string
"""
raise NotImplementedError
def getSQLKeyList(self):
"""
Return a list of keys used by this query and its subqueries.
"""
raise NotImplementedError
def getRelatedTableMapDict(self):
"""
Return for each key used by this query (plus ones used by its
subqueries) the table alias mapping.
"""
raise NotImplementedError
class NegatedQuery(QueryMixin): # XXX Bad name JPS - NotQuery or NegativeQuery is better NegationQuery
"""
Do a boolean negation of given query.
"""
def __init__(self, query):
self._query = query
def asSQLExpression(self, *args, **kw):
sql_expression_dict = self._query.asSQLExpression(*args, **kw)
sql_expression_dict['where_expression'] = '(NOT (%s))' % \
(sql_expression_dict['where_expression'], )
return sql_expression_dict
def getSQLKeyList(self, *args, **kw):
return self._query.getSQLKeyList(*args, **kw)
def getRelatedTableMapDict(self, *args, **kw):
return self._query.getRelatedTableMapDict(*args, **kw)
# asSearchTextExpression is still not implemented
allow_class(NegatedQuery)
class Query(QueryMixin):
"""
This allow to define constraints on a sql column
format - type date : %d/%m/%Y
type float : 1 234.12
"""
def __init__(self, format=None, operator=None, range=None, key=None,
search_mode=None, table_alias_list=None, type=None, **kw):
self.format = format
if operator is None:
operator = 'OR'
self.operator = operator
self.range = range
self.search_mode = search_mode
self.table_alias_list = table_alias_list
key_list = kw.keys()
if len(key_list) != 1:
raise KeyError, 'Query must have only one key'
self.key = key_list[0]
self.value = kw[self.key]
self.type = type
self.search_key = key
def __call__(self, **kw):
return self.asSQLExpression(**kw)
def getRange(self):
return self.range
def getTableAliasList(self):
return self.table_alias_list
def getRelatedTableMapDict(self):
result = {}
table_alias_list = self.getTableAliasList()
if table_alias_list is not None:
result[self.getKey()] = table_alias_list
return result
def getSearchMode(self):
"""Search mode used for Full Text search
"""
return self.search_mode
def asSearchTextExpression(self):
# This will be the standard way to represent
# complex values in listbox. Some fixed
# point must be garanteed
value = self.value
if isSimpleType(value) or isinstance(value, DateTime):
return str(value)
elif isinstance(value, (list, tuple)):
value = map(lambda x:str(x), value)
return (' %s ' % self.operator).join(value)
def asSQLExpression(self, key_alias_dict=None,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
ignore_empty_string=1, stat__=0):
"""
Build the sql string
"""
sql_expression = ''
value = self.getValue()
key = self.getKey()
search_key = self.search_key
ignore_key = 0
if key_alias_dict is not None:
# Try to find the alias
if key not in key_alias_dict:
ignore_key=1
else:
key = key_alias_dict.get(key)
if key is None:
ignore_key=1
where_expression = []
select_expression = []
# Default case: variable equality
range_value = self.getRange()
format = self.getFormat()
if ignore_key:
pass
elif range_value is not None:
if isinstance(value, (list, tuple)):
if format is None:
query_min = min(value)
query_max = max(value)
else:
query_min = value[0]
query_max = value[1]
else:
query_min=query_max=value
query_min = self._quoteSQLString(query_min)
query_max = self._quoteSQLString(query_max)
if range_value == 'min' :
where_expression.append("%s >= %s" % (key, query_min))
elif range_value == 'max' :
where_expression.append("%s < %s" % (key, query_max))
elif range_value == 'minmax' :
where_expression.append("%s >= %s and %s < %s" % (key, query_min, key, query_max))
elif range_value == 'minngt' :
where_expression.append("%s >= %s and %s <= %s" % (key, query_min, key, query_max))
elif range_value == 'ngt' :
where_expression.append("%s <= %s" % (key, query_max))
elif range_value == 'nlt' :
where_expression.append("%s > %s" % (key, query_max))
elif isSimpleType(value) or isinstance(value, DateTime) \
or (isinstance(value, (list, tuple)) and self.operator.upper() != 'IN'):
# Convert into lists any value which contain 'OR'
# Refer to _listGlobalActions DCWorkflow patch for example of use
if isinstance(value, basestring) \
and search_key != EXACT_MATCH_SEARCH_MODE:
value = value.split(' OR ')
value = map(lambda x:x.strip(), value)
value_list = value
if isSimpleType(value) or isinstance(value, DateTime):
value_list = [value]
# For security.
for value in value_list:
comparison_operator = None
if (value != '' or not ignore_empty_string) \
and isinstance(value, basestring):
if '%' in value and search_key != EXACT_MATCH_SEARCH_MODE:
comparison_operator = 'LIKE'
elif search_key == DATETIME_SEARCH_MODE or (
datetime_search_keys is not None and key in datetime_search_keys):
if len(value) >= 1 and value[0:2] in ('<=','!=','>='):
comparison_operator = value[0:2]
value = value[2:]
elif len(value) >= 1 and value[0] in ('=','>','<'):
comparison_operator = value[0]
value = value[1:]
if comparison_operator is None:
comparison_operator = '='
# this seems like a DateTime bug!
# 2002/02/01 ==>(UTC) 2002-01-31 22:00:00
# 2002-02-01 ==>(UTC) 2002-02-01 00:00:00 (!)
value = value.replace('-', '/')
value = DateTime(value).toZone('UTC')
elif len(value) >= 1 and value[0:2] in ('<=','!=','>='):
comparison_operator = value[0:2]
value = value[2:]
elif len(value) >= 1 and value[0] in ('=','>','<'):
comparison_operator = value[0]
value = value[1:]
elif search_key == KEYWORD_SEARCH_MODE or (
key in keyword_search_keys and
search_key != EXACT_MATCH_SEARCH_MODE):
# We must add % in the request to simulate the catalog
comparison_operator = 'LIKE'
value = '%%%s%%' % value
elif search_key == FULL_TEXT_SEARCH_MODE or (
key in full_text_search_keys
and search_key != EXACT_MATCH_SEARCH_MODE):
# We must add % in the request to simulate the catalog
# we first check if there is a special search_mode for this key
# incl. table name, or for all keys of that name,
# or there is a search_mode supplied for all fulltext keys
# or we fall back to natural mode
search_mode=self.getSearchMode()
if search_mode is None:
search_mode = 'natural'
search_mode=search_mode.lower()
mode = full_text_search_modes.get(search_mode,'')
where_expression.append(
"MATCH %s AGAINST ('%s' %s)" % (key, value, mode))
if not stat__:
# we return relevance as Table_Key_relevance
select_expression.append(
"MATCH %s AGAINST ('%s' %s) AS %s_relevance"
% (key, value, mode,key.replace('.','_')))
# and for simplicity as Key_relevance
if '.' in key:
select_expression.append(
"MATCH %s AGAINST ('%s' %s) AS %s_relevance" %
(key, value, mode,key.split('.')[1]))
else:
comparison_operator = '='
elif not isinstance(value, basestring):
comparison_operator = '='
if comparison_operator is not None:
key = self._quoteSQLKey(key)
value = self._quoteSQLString(value)
where_expression.append("%s %s %s" %
(key, comparison_operator, value))
elif value is None:
where_expression.append("%s is NULL" % (key))
elif isinstance(value, (tuple, list)) and self.operator.upper() == 'IN':
if len(value) > 1:
escaped_value_list = [self._quoteSQLString(x) for x in value]
escaped_value_string = ', '.join(escaped_value_list)
where_expression.append("%s IN (%s)" % (key, escaped_value_string))
elif len(value) == 1:
where_expression.append("%s = %s" % (key, self._quoteSQLString(value[0])))
else:
where_expression.append('0') # "foo IN ()" is invalid SQL syntax, so use a "false" value.
else:
where_expression.append("%s = %s" %
(self._quoteSQLKey(key), self._quoteSQLString(value)))
if len(where_expression)>0:
if len(where_expression)==1:
where_expression = where_expression[0]
else:
where_expression = '(%s)' % (' %s ' % self.getOperator()).join(where_expression)
else:
where_expression = '1' # It is better to have a valid default
return {'where_expression':where_expression,
'select_expression_list':select_expression}
def getKey(self):
return self.key
def getValue(self):
return self.value
def getSQLKeyList(self):
"""
Returns the list of keys used by this
instance
"""
return [self.getKey()]
allow_class(Query)
class ComplexQuery(QueryMixin):
"""
Used in order to concatenate many queries
"""
def __init__(self, *args, **kw):
self.query_list = args
self.operator = kw.pop('operator', 'AND')
# XXX: What is that used for ?! It's utterly dangerous.
self.__dict__.update(kw)
def __call__(self, **kw):
return self.asSQLExpression(**kw)
def getQueryList(self):
return self.query_list
def getRelatedTableMapDict(self):
result = {}
for query in self.getQueryList():
if not(isinstance(query, basestring)):
result.update(query.getRelatedTableMapDict())
return result
def asSQLExpression(self, key_alias_dict=None,
ignore_empty_string=1,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
stat__=0):
"""
Build the sql string
"""
sql_expression_list = []
select_expression_list = []
for query in self.getQueryList():
if isinstance(query, basestring):
sql_expression_list.append(query)
else:
query_result = query.asSQLExpression( key_alias_dict=key_alias_dict,
ignore_empty_string=ignore_empty_string,
keyword_search_keys=keyword_search_keys,
full_text_search_keys=full_text_search_keys,
stat__=stat__)
sql_expression_list.append(query_result['where_expression'])
select_expression_list.extend(query_result['select_expression_list'])
operator = self.getOperator()
result = {'where_expression':('(%s)' % \
(' %s ' % operator).join(['(%s)' % x for x in sql_expression_list])),
'select_expression_list':select_expression_list}
return result
def getSQLKeyList(self):
"""
Returns the list of keys used by this
instance
"""
key_list=[]
for query in self.getQueryList():
if not(isinstance(query, basestring)):
key_list.extend(query.getSQLKeyList())
return key_list
allow_class(ComplexQuery)
class Catalog(Folder, class Catalog(Folder,
Persistent, Persistent,
Acquisition.Implicit, Acquisition.Implicit,
...@@ -2191,7 +1776,12 @@ class Catalog(Folder, ...@@ -2191,7 +1776,12 @@ class Catalog(Folder,
for t in self.sql_catalog_scriptable_keys: for t in self.sql_catalog_scriptable_keys:
t = t.split('|') t = t.split('|')
key = t[0].strip() key = t[0].strip()
if len(t)>1:
# method defined that will generate a ComplexQuery
method_id = t[1].strip() method_id = t[1].strip()
else:
# no method define, let ScriptableKey generate a ComplexQuery
method_id = None
scriptable_key_dict[key] = method_id scriptable_key_dict[key] = method_id
# Build the list of Queries and ComplexQueries # Build the list of Queries and ComplexQueries
...@@ -2210,9 +1800,14 @@ class Catalog(Folder, ...@@ -2210,9 +1800,14 @@ class Catalog(Folder,
if isinstance(value, (Query, ComplexQuery)): if isinstance(value, (Query, ComplexQuery)):
current_query = value current_query = value
elif scriptable_key_dict.has_key(key): elif scriptable_key_dict.has_key(key):
if scriptable_key_dict[key] is not None:
# Turn this key into a query by invoking a script # Turn this key into a query by invoking a script
method = getattr(self, scriptable_key_dict[key]) method = getattr(self, scriptable_key_dict[key])
current_query = method(value) # May return None current_query = method(value) # May return None
else:
# let default implementation of ScriptableKey generate ComplexQuery
search_key_instance = getSearchKeyInstance(ScriptableKey)
current_query = search_key_instance.buildQuery('', value)
if hasattr(current_query, 'order_by'): query_group_by_list = current_query.order_by if hasattr(current_query, 'order_by'): query_group_by_list = current_query.order_by
else: else:
if isinstance(value, dict): if isinstance(value, dict):
...@@ -2745,3 +2340,37 @@ class Catalog(Folder, ...@@ -2745,3 +2340,37 @@ class Catalog(Folder,
Globals.default__class_init__(Catalog) Globals.default__class_init__(Catalog)
class CatalogError(Exception): pass class CatalogError(Exception): pass
# hook search keys and Query implementation
def getSearchKeyInstance(search_key_class):
""" Return instance of respective search_key class.
We should have them initialized only once."""
global SEARCH_KEY_INSTANCE_POOL
lexer = SEARCH_KEY_INSTANCE_POOL[search_key_class]
return lexer
from Query.Query import QueryMixin
from Query.SimpleQuery import NegatedQuery, SimpleQuery
from Query.ComplexQuery import ComplexQuery
# for of backwards compatability
QueryMixin = QueryMixin
Query = SimpleQuery
NegatedQuery = NegatedQuery
ComplexQuery = ComplexQuery
from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
# pool of global preinitialized search keys instances
SEARCH_KEY_INSTANCE_POOL = {}
for search_key_class in (DefaultKey, RawKey, KeyWordKey, DateTimeKey,
FullTextKey, FloatKey, ScriptableKey, KeyMappingKey):
search_key_instance = search_key_class()
search_key_instance.build()
SEARCH_KEY_INSTANCE_POOL[search_key_class] = search_key_instance
product/ZSQLCatalog/SearchKey/DateTimeKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from DateTime import DateTime
from Key import BaseKey
from pprint import pprint
class DateTimeKey(BaseKey):
""" DateTimeKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try to match values in DateTime MySQL columns.
It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
addition to main logical operators like ['OR', 'or', 'AND', 'and'].
Note: because all ERP5 datetime values are indexed in MySQL in 'UTC'
the respective passed date will be first converted to 'UTC' before inserted into
respective SQL query!
Examples (GMT+02, Bulgaria/Sofia for 'delivery.start_date'):
* '15/01/2008' --> "delivery.start_date = '2008-01-14 22:00'"
* '>=15/01/2008' --> "delivery.start_date >= '2008-01-14 22:00'"
* '>=15/01/2008 or <=20/01/2008'
--> "delivery.start_date >= '2008-01-14 22:00' or delivery.start_date<='2008-01-19 22:00'"
* '>=15/01/2008 10:00 GMT+02 OR <=20/01/2008 05:12 Universal'
-->
"delivery.start_date >= '2008-01-15 08:00 Universal'
OR
delivery.start_date <= '2008-01-20 05:12 Universal'
"
"""
tokens = ('DATE', 'OR', 'AND', 'NOT', 'EQUAL',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT', 'EQUAL',)
def t_OR(self, t):
r'(\s+OR\s+|\s+or\s+)'
# operator has leading and trailing ONLY one white space character
t.value = 'OR'
return t
def t_AND(self, t):
r'(\s+AND\s+|\s+and\s+)'
# operator has leading and trailing ONLY one white space character
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+NOT\s+|\s+not\s+|!=)'
# operator has leading and trailing ONLY one white space character
t.value = t.value.upper().strip()
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
t_EQUAL = r'='
t_DATE = r'\d{1,4}[(/|\.|\-) /.]\d{1,4}[(/|\.|\-) /.]\d{1,4}((\s.)*\d{0,2}:\d{0,2}(:\d{0,2})?)?(\sUniversal|\sGMT\+\d\d)?|\d\d\d\d%?'
def quoteSQLString(self, value, format):
""" Return a quoted string of the value.
Make sure to convert it to UTC first."""
if getattr(value, 'ISO', None) is not None:
value = "'%s'" % value.toZone('UTC').ISO()
else:
value = "'%s'" %DateTime(value).toZone('UTC').ISO()
return value
def buildQueryForTokenList(self, tokens, key, value, format):
""" Build a ComplexQuery for a token list """
query_list = []
for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
token_values = [x.value for x in group_tokens]
sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
date_value = sub_tokens[0].value
days_offset = 0
# some format require special handling
if format != '%Y':
# full format (Year/Month/Day)
if sub_operator in ('=',):
# 2007/01/01 00:00 <= date < 2007/01/02
days_offset = 1
elif format == '%Y':
# incomplete format only Year because DateTime can not handle
# extend format and value by assumption that start of year is ment
# add days ofset accordingly
format = '%%%s/%%m/%%d' %format
date_value = '%s/01/01' %date_value
days_offset_map = {'=' : 366, '>' : 366,
'>=' : 366, '<': -366, '<=':-366}
days_offset = days_offset_map[sub_operator]
# convert to UTC in given format
is_valid_date = 1
try:
if format != '%m/%d/%Y':
# treat ambigious dates as "days before month before year"
date_value = DateTime(date_value, datefmt="international").toZone('UTC')
else:
# US style "month before day before year"
date_value = DateTime(date_value).toZone('UTC')
except:
is_valid_date = 0
query_kw = None
if is_valid_date:
if sub_operator == '=':
# transform to range 'key >= date AND date < key'
query_kw = {key: (date_value, date_value + days_offset,),
'range': 'minmax'}
else:
query_kw = {key: date_value + days_offset,
'range': sub_operator}
query_kw['type'] = 'date'
else:
# not a valid date, try to get an year range
is_year = 1
date_value = date_value.replace('%', '')
try: date_value = int(date_value)
except: is_year = 0
if is_year:
date_value = '%s/01/01' % date_value
date_value = DateTime(date_value).toZone('UTC')
query_kw = {key: (date_value, date_value + 366,),
'type': 'date',
'range': 'minmax'}
# append only if it was possible to generate query
if query_kw is not None:
query_list.append(Query(**query_kw))
# join query list in one really big ComplexQuery
if len(query_list):
complex_query = ComplexQuery(*query_list,
**{'operator': 'AND'})
return complex_query
## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
## """ Tokenize/analyze passed string value and generate SQL query expressions. """
## where_expression = ''
## key = self.quoteSQLKey(key, format)
## tokens = self.tokenize(value)
## operators_mapping_list = self.groupByOperator(tokens)
## # new one
## for item in operators_mapping_list:
## row_tokens_values = []
## tokens = item['tokens']
## operator = item['operator']
## operator_value = None
## if operator is not None:
## # operator is standalone expression
## operator_value = operator.value
## where_expressions.append('%s' %operator_value)
## if len(tokens):
## # no it's not a stand alone expression,
## # determine it from list of tokens
## operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
## row_tokens_values = [self.quoteSQLString(x.value, format) for x in sub_tokens]
## where_expression = "%s %s %s" %(key, operator_value, ' '.join(row_tokens_values))
## return where_expression, []
product/ZSQLCatalog/SearchKey/DefaultKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Key import BaseKey
from pprint import pprint
class DefaultKey(BaseKey):
""" DefaultKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try to exactly one value.
It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
addition to main logical operators like ['OR', 'or', 'AND', 'and'].
Examples for title column:
* 'foo or bar' --> "title = 'foo' OR title = 'bar'"
* 'foo or =bar' --> "title = 'foo' OR title = 'bar'"
* '%foo% or bar' --> "title = '%foo%' OR title = 'bar'"
* 'Organisation Module' --> "title = 'Organisation Module'"
* '"Organisation Module"' --> "title = 'Organisation Module'"
* '="Organisation Module"' --> "title = 'Organisation Module'"
"""
# default type of sub Queries to be generated out fo a search string
default_key_type = 'default'
tokens = ('OR', 'AND', 'NOT', 'WORDSET', 'WORD',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'(\s+OR\s+|\s+or\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'(\s+AND\s+|\s+and\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+NOT\s+|\s+not\s+|!=)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = '!='
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
def t_WORD(self, t):
r'[\x7F-\xFF\w\d\/~!@#$%^&*()_+\n][\x7F-\xFF\w\d\/~!@#$%^&*()_+\n]*'
#r'[\x7F-\xFF\w\d\/%][\x7F-\xFF\w\d\/%]*'
# WORD may contain arbitrary letters and numbers without white space
# WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
value = t.value.strip()
t.value = "%s" %value
return t
def t_WORDSET(self, t):
r'"[\x7F-\xFF\w\d\s\/~!@#$%^&*()_+][\x7F-\xFF\w\d\s\/~!@#$%^&*()_+]*"'
#r'"[\x7F-\xFF\w\d\s/%][\x7F-\xFF\w\d\s/%]*"'
# WORDSET is a combination of WORDs separated by white space
# and starting/ending with "
value = t.value.replace('"', '').strip()
t.value = "%s" %value
return t
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
if isinstance(value, (int, long,)):
return str(value)
return "'%s'" %value
## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
## """ Tokenize/analyze passed string value and generate SQL query expressions. """
## where_expressions = []
## select_expressions = []
## tokens = self.tokenize(value)
## operators_mapping_list = self.groupByOperator(tokens)
##
## # find if any logical operator exists
## tokens_values = []
## logical_operator_found = 0
## for token in tokens:
## if token.type not in ('WORDSET', 'WORD',):
## logical_operator_found = 1
## break
## tokens_values.append(token.value.replace("'", ""))
##
## # build expressions
## if not logical_operator_found:
## # no logical operator found so we assume that we search for a combination of words
## where_expressions.append("%s = '%s'" %(key, ' '.join(tokens_values)))
## else:
## # in the search string we have explicitly defined an operator
## for item in operators_mapping_list:
## row_tokens_values = []
## tokens = item['tokens']
## operator = item['operator']
## operator_value = None
## if operator is not None:
## # operator is standalone expression
## operator_value = operator.value
## where_expressions.append('%s' %operator_value)
## if len(tokens):
## # no it's not a stand alone expression,
## # determine it from list of tokens
## operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
## row_tokens_values = [x.value for x in sub_tokens]
## where_expressions.append("%s %s '%s'" %(key, operator_value, ' '.join(row_tokens_values)))
## return where_expressions, select_expressions
product/ZSQLCatalog/SearchKey/FloatKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License,] or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Key import BaseKey
class FloatKey(BaseKey):
""" FloatKey key is an ERP5 portal_catalog search key which is used to render
float like SQL expression.
"""
# default type of sub Queries to be generated out fo a search string
default_key_type = 'float'
tokens = ('OR', 'AND', 'NOT', 'FLOAT',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'(\s+OR\s+|\s+or\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'(\s+AND\s+|\s+and\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+NOT\s+|\s+not\s+|!=)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = '!='
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
def t_FLOAT(self, t):
r'[\d.][\d.]*'
# FLOAT is a float number
value = t.value.replace('"', '').strip()
t.value = "%s" %value
return t
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
# Make sure there is no space in float values
return "'%s'" %str(value).replace(' ', '')
def quoteSQLKey(self, key, format):
""" Return a quoted string of the value. """
if format is not None:
float_format = format.replace(' ', '')
if float_format.find('.') >= 0:
precision = len(float_format.split('.')[1])
key = "TRUNCATE(%s,%s)" % (key, precision)
return key
product/ZSQLCatalog/SearchKey/FullTextKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Key import BaseKey
SEARCH_MODE_MAPPING = {'in_boolean_mode': 'IN BOOLEAN MODE',
'with_query_expansion': 'WITH QUERY EXPANSION'}
class FullTextKey(BaseKey):
""" FullTextKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try match all possible values using
MySQL's fulltext search support.
See syntax see MySQL's FullText search reference:
http://dev.mysql.com/doc/refman/5.0/en/fulltext-search.html
"""
tokens = ('PLUS', 'MINUS', 'WORD', 'GREATERTHAN', 'LESSTHAN', 'LEFTPARENTHES',
'RIGHTPARENTHES', 'TILDE', 'ASTERISK', 'DOUBLEQUOTE',)
# SQL expressions patterns
relevance = '%s_relevance'
where_match_against = "MATCH %s AGAINST ('%s' %s)"
select_match_against_as = "MATCH %s AGAINST ('%s' %s) AS %s"
t_PLUS = r'(\+)'
t_MINUS = r'(\-)'
t_GREATERTHAN = r'(\>)'
t_LESSTHAN = r'(\<)'
t_LEFTPARENTHES = r'(\()'
t_RIGHTPARENTHES = r'(\))'
t_TILDE = r'(\~)'
t_ASTERISK = r'(\*)'
t_DOUBLEQUOTE = r'(\")'
def t_WORD(self, t):
r'[\x7F-\xFF\w\d\/!@#$%^&_][\x7F-\xFF\w\d\/!@#$%^&_]*'
#r'[\x7F-\xFF\w\d][\x7F-\xFF\w\d]*'
# WORD may contain arbitrary letters and numbers without white space
word_value = t.value
t.value = "'%s'" %word_value
return t
def buildSQLExpression(self, key, value,
format=None, mode=None, range_value=None, stat__=None):
""" Analize token list and generate SQL expressions."""
tokens = self.tokenize(value)
# based on type tokens we may switch to different search mode
mode = SEARCH_MODE_MAPPING.get(mode, '')
if mode == '':
# determine it based on list of tokens i.e if we have only words
# leave as its but if we have '-' or '+' use boolean mode
for token in tokens:
if token.type != 'WORD':
mode = SEARCH_MODE_MAPPING['in_boolean_mode']
break
# split (if possible) to column.key
if key.find('.') != -1:
table, column = key.split('.')
relevance_key1 = self.relevance %key.replace('.', '_')
relevance_key2 = self.relevance %column
else:
relevance_key1 = self.relevance %key
relevance_key2 = None
select_expression_list = []
where_expression = self.where_match_against %(key, value, mode)
if not stat__:
# stat__ is an internal implementation artifact to prevent adding
# select_expression for countFolder
select_expression_list = [self.select_match_against_as %(key, value, mode, relevance_key1),]
if relevance_key2 is not None:
select_expression_list.append(self.select_match_against_as %(key, value, mode, relevance_key2))
return where_expression, select_expression_list
product/ZSQLCatalog/SearchKey/Key.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License,] or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
import ply.yacc as yacc
import ply.lex as lex
class BaseKey:
""" BaseKey is a base class that implements a parser of
search grammar used in ERP5. It also implements all generic
search key class methods."""
# main logical operators
operators = ('OR', 'AND',)
default_operator = '='
# in ERP5 search grammer white space is extremely important
# so we can not ignore it.
#t_ignore = ' \t'
# no need to rack down line numbers
#def t_newline(self, t):
# r'\n+'
# #t.lexer.lineno += len(t.value)
def t_error(self, t):
#print "Illegal character '%s'" % t.value[0]
t.lexer.skip(1)
def p_error(self, p):
pass
def build(self, **kwargs):
""" This method will initialize respective search key class with
tokens' definitions. """
self.lexer = lex.lex(object = self, **kwargs)
def tokenize(self, data):
""" Return list of tokens according to respective
search key tokens' definitions. """
result = []
self.lexer.input(data)
while 1:
tok = self.lexer.token()
if not tok:
break
result.append(tok)
return result
# Grouping of tokens
def getOperatorForTokenList(self, tokens):
""" Generic implementation that will return respective
operator for a token list. The first found occurence wins."""
token = tokens[0]
token_type = token.type
if token_type in self.sub_operators:
return token.value, tokens[1:]
else:
return self.default_operator, tokens
def groupByLogicalOperator(self, tokens, logical_operator ='OR'):
""" Split tokens list into one or many OR concatanated tokens list
"""
sub_tokens_or_groups = []
tmp_token_list = []
for token in tokens:
if token.type != logical_operator:
tmp_token_list.append(token)
else:
sub_tokens_or_groups.append(tmp_token_list)
tmp_token_list = []
# append remainig last tokens
sub_tokens_or_groups.append(tmp_token_list)
return sub_tokens_or_groups
# SQL quoting (each search key should override them it if needed)
def quoteSQLKey(self, key, format):
""" Return a quoted string of the value. """
return key
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
return "'%s'" %value
# SQL generation
def buildSQLExpression(self, key, value,
format = None, mode = None, range_value = None, stat__=0):
""" Generic implementation. Leave details to respective key. """
if range_value is not None:
# if range_value we handle directly (i.e no parsing of search string)
where_expressions, select_expressions = \
self.buildSQLExpressionFromRange(key, value,
format, mode, range_value, stat__)
else:
# search string parsing is needed
where_expressions, select_expressions = \
self.buildSQLExpressionFromSearchString(key, str(value),
format, mode, range_value, stat__)
return where_expressions, select_expressions
def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
complex_query = self.buildQuery(key, value, format, mode, range_value, stat__)
if complex_query is None:
# Query could not be generated from search string
sql_expression = {'where_expression': '1',
'select_expression_list': []}
else:
sql_expression = complex_query(keyword_search_keys = [],
datetime_search_keys = [],
full_text_search_keys = [])
return sql_expression['where_expression'], sql_expression['select_expression_list']
def buildQuery(self, key, value, format, mode, range_value, stat__):
""" Build Query """
query_list = []
# tokenize searchs string into tokens for Search Key
tokens = self.tokenize(value)
# split tokens list into one or more 'OR' tokens lists
tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
# remove empty tokens lists
tokens_or_groups = filter(lambda x: len(x), tokens_or_groups)
# get a ComplexQuery for a sub token list
for tokens_or_group in tokens_or_groups:
query = self.buildQueryForTokenList(tokens_or_group, key, value, format)
if query is not None:
# query could be generated for token list
query_list.append(query)
if len(query_list):
# join query list in one really big ComplexQuery
return ComplexQuery(*query_list,
**{'operator':'OR'})
def buildQueryForTokenList(self, tokens, key, value, format):
""" Build a ComplexQuery for a token list """
query_list = []
logical_groups = self.groupByLogicalOperator(tokens, 'AND')
for group_tokens in logical_groups:
token_values = [x.value for x in group_tokens]
sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
sub_tokens_values = [x.value for x in sub_tokens]
query_kw = {key: ' '.join(sub_tokens_values),
'type': self.default_key_type,
'format': format,
'range': sub_operator}
query_list.append(Query(**query_kw))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator': 'AND'})
return complex_query
def buildSQLExpressionFromRange(self, key, value, format, mode, range_value, stat__):
""" This method will generate SQL expressions
from explicitly passed list of values and
range_value in ('min', 'max', ..)"""
key = self.quoteSQLKey(key, format)
where_expression = ''
select_expressions = []
if isinstance(value, (list, tuple)):
if len(value) > 1:
# value should contain at least two items
query_min = self.quoteSQLString(value[0], format)
query_max = self.quoteSQLString(value[1], format)
else:
# value contains only one item
query_min = query_max = self.quoteSQLString(value[0], format)
else:
query_min = query_max = self.quoteSQLString(value, format)
if range_value == 'min':
where_expression = "%s >= %s" % (key, query_min)
elif range_value == 'max':
where_expression = "%s < %s" % (key, query_max)
elif range_value == 'minmax' :
where_expression = "%s >= %s AND %s < %s" % (key, query_min, key, query_max)
elif range_value == 'minngt' :
where_expression = "%s >= %s AND %s <= %s" % (key, query_min, key, query_max)
elif range_value == 'ngt':
where_expression = "%s <= %s" % (key, query_max)
elif range_value == 'nlt':
where_expression = "%s > %s" % (key, query_max)
elif range_value == 'like':
where_expression = "%s LIKE %s" % (key, query_max)
elif range_value == 'not_like':
where_expression = "%s NOT LIKE %s" % (key, query_max)
elif range_value in ('=', '>', '<', '>=', '<=','!=',):
where_expression = "%s %s %s" % (key, range_value, query_max)
return where_expression, select_expressions
## def groupByOperator(self, tokens, group_by_operators_list = operators):
## """ Generic implementation of splitting tokens into logical
## groups defided by respective list of logical operator
## defined for respective search key. """
## items = []
## last_operator = None
## operators_mapping_list = []
## last_operator = {'operator': None,
## 'tokens': []}
## for token in tokens:
## token_type = token.type
## token_value = token.value
## if token_type in group_by_operators_list:
## # (re) init it
## last_operator = {'operator': token,
## 'tokens': []}
## operators_mapping_list.append(last_operator)
## else:
## # not an operator just a value token
## last_operator['tokens'].append(token)
## if last_operator not in operators_mapping_list:
## operators_mapping_list.append(last_operator)
## return operators_mapping_list
product/ZSQLCatalog/SearchKey/KeyWordKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from Key import BaseKey
from pprint import pprint
class KeyWordKey(BaseKey):
""" KeyWordKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try to match all possible values in a greedy manner.
It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
addition to main logical operators like ['OR', 'or', 'AND', 'and'].
Examples for title column:
* 'foo or bar' --> "title LIKE '%foo%' OR title LIKE '%bar%'"
* 'foo or =bar' --> "title LIKE '%foo%' OR title = 'bar'"
* 'Organisation Module' --> "title LIKE '%Organisation Module%'"
* '"Organisation Module"' --> "title LIKE '%Organisation Module%'"
* '="Organisation Module"' --> "title = 'Organisation Module'"
"""
tokens = ('OR', 'AND', 'NOT',
'KEYWORD', 'WORDSET', 'WORD', 'EXPLICITEQUALLITYWORD',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
# this is the default operator
default_operator = 'like'
# if token's list starts with left sided operator
# use this map to transfer it to range operator
token_operator_range_map = {'like': 'like',
'!=': 'not_like',
'=': '=',}
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'(\s+OR\s+|\s+or\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'(\s+AND\s+|\s+and\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+NOT\s+|\s+not\s+|!=)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = t.value.upper().strip()
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
def t_EXPLICITEQUALLITYWORD(self, t):
r'=[\x7F-\xFF\w\d\/~!@#$^&*()_+][\x7F-\xFF\w\d\/~!@#$^&*()_+]*'
# EXPLICITEQUALLITYWORD may contain arbitrary letters and numbers without white space
# EXPLICITEQUALLITYWORD must contain '=' at the beginning
value = t.value.strip()
# get rid of leading '='
t.value = value[1:]
return t
def t_KEYWORD(self, t):
r'%?[\x7F-\xFF\w\d/~!@#$%^&*()_+][\x7F-\xFF\w\d/~!@#$%^&*()_+]*%?'
# KEYWORD may starts(1) and may ends (2) with '%' but always must either #1 or #2
# be true. It may contains arbitrary letters, numbers and white space
value = t.value.strip()
if not value.startswith('%') and not value.endswith('%'):
t.type = 'WORD'
t.value = value
return t
def t_WORD(self, t):
r'[\x7F-\xFF\w\d\/~!@#$^&*()_+][\x7F-\xFF\w\d\/~!@#$^&*()_+]*'
# WORD may contain arbitrary letters and numbers without white space
# WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
value = t.value.strip()
t.value = value
return t
def t_WORDSET(self, t):
r'=?"[\x7F-\xFF\w\d\s\/~!@#$%^&*()_+][\x7F-\xFF\w\d\s\/~!@#$%^&*()_+]*"'
# WORDSET is a combination of WORDs separated by white space
# and starting/ending with " (optionally with '=')
value = t.value.replace('"', '')
t.value = "%s" %value
return t
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
return "'%s'" %value
def getOperatorForTokenList(self, tokens):
""" Generic implementation that will return respective
operator for a token list. The first found occurence wins."""
token = tokens[0]
token_type = token.type
if token_type in self.sub_operators:
return token.value, tokens[1:]
elif token.type == 'EXPLICITEQUALLITYWORD':
# even though it's keyword key we can still explicitly define
# that we want equality
return '=', tokens
else:
return self.default_operator, tokens
def buildQueryForTokenList(self, tokens, key, value, format):
""" Build a ComplexQuery for a token list """
query_list = []
for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
token_values = [x.value for x in group_tokens]
sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
first_token = sub_tokens[0]
range = self.token_operator_range_map.get(sub_operator)
sub_tokens_values = [x.value for x in sub_tokens]
right_side_expression = ' '.join(sub_tokens_values)
if first_token.type == 'WORDSET' and first_token.value.startswith('='):
range = '='
right_side_expression = first_token.value[1:]
elif first_token.type in ('WORDSET', 'WORD',) and range == 'like':
# add trailing and leading '%' to get more results
right_side_expression = '%%%s%%' %right_side_expression
query_kw = {key: right_side_expression,
'range': range}
query_list.append(Query(**query_kw))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator': 'AND'})
return complex_query
## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
## """ Tokenize/analyze passed string value and generate SQL query expressions. """
## where_expressions = []
## select_expressions = []
## tokens = self.tokenize(value)
## operators_mapping_list = self.groupByOperator(tokens)
##
## # find if any logical operator exists
## tokens_values = []
## logical_operator_found = 0
## for token in tokens:
## if token.type not in ('WORD',):
## logical_operator_found = 1
## break
## tokens_values.append(token.value.replace("'", ""))
##
## # build expressions
## if not logical_operator_found:
## # no logical operator found so we assume that we search
## # for a combination of words
## where_expressions.append("%s LIKE '%%%s%%'" %(key, ' '.join(tokens_values)))
## else:
## # in the search string we have explicitly defined an operator
## for item in operators_mapping_list:
## row_tokens_values = []
## tokens = item['tokens']
## operator = item['operator']
## operator_value = None
## if operator is not None:
## # operator is standalone expression
## where_expressions.append('%s' %operator.value)
## if len(tokens):
## # no it's not a stand alone expression,
## # determine it from list of tokens
## sub_where_expression = ''
## tokens_number = len(tokens)
## if tokens_number == 1:
## # no left sided operator (<, >, >=, <=) found
## token = tokens[0]
## if token.type == 'WORD':
## sub_where_expression = "LIKE '%%%s%%'" %token.value
## elif token.type == 'KEYWORD':
## sub_where_expression = "LIKE '%s'" %token.value
## elif token.type == 'EXPLICITEQUALLITYWORD':
## sub_where_expression = "= '%s'" %token.value
## elif token.type == 'WORDSET' and token.value.startswith('='):
## # if WORDSET starts with '=' it's an equality
## sub_where_expression = " = '%s'" %token.value[1:]
## else:
## sub_where_expression = "LIKE '%%%s%%'" %token.value
## else:
## # we have two or more tokens, by definition first one should be
## # logical operator like (<, >, >=, <=)
## operator = tokens[0]
## operator_value = operator.value
## if operator.type in ('KEYWORD', 'WORDSET', 'WORD'):
## # no operator for this token list, assume it's 'LIKE'
## sub_where_expression = "LIKE '%s'" %' '.join([x.value for x in tokens])
## else:
## # we have operator and by convention if operator is used it's applyied to one token only
## sub_where_expression = "%s'%s'" %(operator_value, tokens[1].value)
## where_expressions.append('%s %s' %(key, sub_where_expression))
## return where_expressions, select_expressions
product/ZSQLCatalog/SearchKey/RawKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
class RawKey:
""" RawKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will match exactly what's passed to it using equality ."""
def build(self, **kwargs):
# this key doesn't require parsing
# It's required to implement it as it's used ONLY for ExactMath
pass
def buildSQLExpression(self, key, value,
format=None, mode=None, range_value=None, stat__=None):
where_expression = "%s = '%s'" %(key, value)
return where_expression, []
product/ZSQLCatalog/SearchKey/ScriptableKey.py 0 → 100644
View file @ 3aeb0ab9
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from Products.PythonScripts.Utility import allow_class
from Key import BaseKey
from pprint import pprint
# these keys are used to build query in case for ScriptableKey
# when no key was specified in fornt of value
DEFAULT_SEARCH_KEYS = ('SearchableText', 'reference', 'title',)
class KeyMappingKey(BaseKey):
""" Usable lexer class used (internally) by ScriptableKey lexer than can parse following:
VALUE OPERATOR VALUE
Examples:
* "portal_type : Person"
* "creation_date > 2007-01-01"
"""
tokens = ('OPERATOR', 'COLONOPERATOR', 'VALUE',)
t_OPERATOR = r'>=|<=|>|<'
t_VALUE = r'[\x7F-\xFF\w\d\/~!@#$^&*()_+-][\x7F-\xFF\w\d\/~!@#$^&*()_+-]*'
def t_COLONOPERATOR(self, t):
r':'
# ':' is the same as '=' (equality)
t.value = '='
return t
class ScriptableKey(BaseKey):
""" KeyWordKey key is an ERP5 portal_catalog search key which is used to generate a
ComplexQuery instance out of an arbitrary search string.
Examples:
* "John Doe AND portal_type:Person AND creation_date > 2007-01-01"
would be turned into following ComplexQuery:
* ComplexQuery(Query(portal_type='Person'),
Query(creation_date='2007-01-01', operator='>'),
ComplexQuery(Query(searchable_text='John Doe'),
Query(title='John Doe'),
Query(reference='John Doe'),
operator='OR')
operator='AND'))
"""
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL',)
tokens = ('OR', 'AND',
'DATE', 'WORD', 'KEYMAPPING',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'EQUAL')
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
t_EQUAL = r'='
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'(\s+OR\s+|\s+or\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'(\s+AND\s+|\s+and\s+)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_KEYMAPPING(self, t):
r'[\x7F-\xFF\w\d\/~!@#$^&*()_+-][\x7F-\xFF\w\d\/~!@#$^&*()_+-]*\s*(>|<|<=|>=|:)\s*[\x7F-\xFF\w\d\/~!@#$^&*()_+-][\x7F-\xFF\w\d\/~!@#$^&*()_+-]*'
# KEYMAPPING has following format: KEY OPERATOR VALUE
# where OPERATOR in ['<', '>', '<=', '>=', ':']
# example: 'creation_date < 2007-12-12'
value = t.value.strip()
t.value = value
return t
def t_WORD(self, t):
r'[\x7F-\xFF\w\d\/~!@#$^&*()_+][\x7F-\xFF\w\d\/~!@#$^&*()_+]*'
# WORD may contain arbitrary letters and numbers without white space
# WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
value = t.value.strip()
t.value = value
return t
def buildQueryForTokenList(self, tokens):
""" Build a ComplexQuery for a token list """
query_list = []
for group in self.groupByLogicalOperator(tokens, 'AND'):
group_tokens = group
first_group_token = group_tokens[0]
if first_group_token.type == 'KEYMAPPING':
# user specified a full sub query definition following this format:
# 'key operator value'
sub_search_string = group_tokens[0].value
keymapping_lexer = getSearchKeyInstance(KeyMappingKey)
sub_tokens = keymapping_lexer.tokenize(sub_search_string)
sub_tokens_values = [x.value for x in sub_tokens]
search_key, search_operator, search_value = sub_tokens_values
query_kw = {search_key: search_value,
'range' : search_operator,}
query_list.append(Query( **query_kw))
elif first_group_token.type in self.sub_operators:
# user specified a incomplete sub query definition following this format:
# 'operator value'. Assume that he ment to search for 'title' and
# use supplied 'operator'
search_operator = first_group_token.value
simple_query_value = ' '.join([x.value for x in group_tokens[1:]])
query_kw = {'title': simple_query_value,
'range' : search_operator,}
query_list.append(Query( **query_kw))
else:
# user specified a VERY incomplete sub query definition following this format:
# 'value'. Let's search against most common search_keys and assume operator
# is '=' (by default) and try to get as much possible results
simple_query_value = ' '.join([x.value for x in group_tokens])
sub_query_list = []
for default_key in DEFAULT_SEARCH_KEYS:
query_kw = {default_key: simple_query_value}
sub_query_list.append(Query(**query_kw))
query_list.append(ComplexQuery(*sub_query_list,
**{'operator':'OR'}))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator':'AND'})
return complex_query
def buildQuery(self, key, value,
format=None, mode=None, range_value=None, stat__=None):
""" Build ComplexQuery from passed search string value.
When grouping expressions we use the following assumptions
that 'OR' operator has higher priority in a sense:
* "John Doe AND portal_type:Person OR creation_date>=2005/12/12"
is considered as:
* (John Doe AND portal_type:Person) OR (creation_date>=2005/12/12)"
"""
query_list = []
tokens = self.tokenize(value)
# split tokens list into one or many OR concatanated expressions
sub_tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
# get a ComplexQuery for a sub token list
for tokens_or_group in sub_tokens_or_groups:
query_list.append(self.buildQueryForTokenList(tokens_or_group))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator':'OR'})
return complex_query
allow_class(ScriptableKey)
product/ZSQLCatalog/__init__.py
View file @ 3aeb0ab9
...@@ -43,5 +43,4 @@ def initialize(context): ...@@ -43,5 +43,4 @@ def initialize(context):
from AccessControl import ModuleSecurityInfo, ClassSecurityInfo from AccessControl import ModuleSecurityInfo, ClassSecurityInfo
ModuleSecurityInfo('Products.ZSQLCatalog.SQLCatalog').declarePublic( ModuleSecurityInfo('Products.ZSQLCatalog.SQLCatalog').declarePublic(
'ComplexQuery', 'Query', 'NegatedQuery') 'ComplexQuery', 'Query', 'NegatedQuery',)
product/ZSQLCatalog/tests/testZSQLCatalog.py
View file @ 3aeb0ab9
...@@ -102,9 +102,11 @@ class TestQuery(unittest.TestCase): ...@@ -102,9 +102,11 @@ class TestQuery(unittest.TestCase):
def testSimpleQuery(self): def testSimpleQuery(self):
q = Query(title='Foo') q = Query(title='Foo')
self.assertEquals( self.assertEquals(
dict(where_expression="title = 'Foo'", dict(where_expression="((((title = 'Foo'))))",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testQueryMultipleKeys(self): def testQueryMultipleKeys(self):
# using multiple keys is invalid and raises # using multiple keys is invalid and raises
...@@ -116,7 +118,9 @@ class TestQuery(unittest.TestCase): ...@@ -116,7 +118,9 @@ class TestQuery(unittest.TestCase):
self.assertEquals( self.assertEquals(
dict(where_expression="title is NULL", dict(where_expression="title is NULL",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testEmptyQueryNotIgnoreEmptyString(self): def testEmptyQueryNotIgnoreEmptyString(self):
q = Query(title='') q = Query(title='')
...@@ -127,6 +131,7 @@ class TestQuery(unittest.TestCase): ...@@ -127,6 +131,7 @@ class TestQuery(unittest.TestCase):
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(ignore_empty_string=0, q.asSQLExpression(ignore_empty_string=0,
keyword_search_keys=[], keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])) full_text_search_keys=[]))
def testEmptyQuery(self): def testEmptyQuery(self):
...@@ -135,52 +140,67 @@ class TestQuery(unittest.TestCase): ...@@ -135,52 +140,67 @@ class TestQuery(unittest.TestCase):
self.assertEquals( self.assertEquals(
dict(where_expression="1", dict(where_expression="1",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testMultiValuedQuery(self): def testMultiValuedQuery(self):
q = Query(title=['Foo', 'Bar']) q = Query(title=['Foo', 'Bar'])
self.assertEquals( self.assertEquals(
dict(where_expression="(title = 'Foo' OR title = 'Bar')", dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testINQuery(self): def testINQuery(self):
q = Query(title=['Foo', 'Bar'], operator='IN') q = Query(title=['Foo', 'Bar'], operator='IN')
self.assertEquals( self.assertEquals(
dict(where_expression="title IN ('Foo', 'Bar')", dict(where_expression="title IN ('Foo', 'Bar')",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testEmptyINQuery(self): def testEmptyINQuery(self):
q = Query(title=[], operator='IN') q = Query(title=[], operator='IN')
self.assertEquals( self.assertEquals(
dict(where_expression="0", dict(where_expression="0",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testMinQuery(self): def testMinQuery(self):
q = Query(title='Foo', range='min') q = Query(title='Foo', range='min')
self.assertEquals( self.assertEquals(
dict(where_expression="title >= 'Foo'", dict(where_expression="title >= 'Foo'",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testMaxQuery(self): def testMaxQuery(self):
q = Query(title='Foo', range='max') q = Query(title='Foo', range='max')
self.assertEquals( self.assertEquals(
dict(where_expression="title < 'Foo'", dict(where_expression="title < 'Foo'",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
# format # format
def testDateFormat(self): def testDateFormat(self):
q = Query(date=DateTime(2001, 02, 03), format='%Y/%m/%d', type='date') date = DateTime(2001, 02, 03)
q = Query(date=date, format='%Y/%m/%d', type='date')
self.assertEquals( self.assertEquals(
dict(where_expression= dict(where_expression=
"STR_TO_DATE(DATE_FORMAT(date,'%Y/%m/%d'),'%Y/%m/%d')" "((((date >= '%s' AND date < '%s'))))" \
" = STR_TO_DATE('2001/02/03','%Y/%m/%d')", %(date.toZone('UTC').ISO(), (date + 1).toZone('UTC').ISO()),
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], full_text_search_keys=[])) q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
# full text # full text
def testSimpleQueryFullText(self): def testSimpleQueryFullText(self):
...@@ -189,6 +209,7 @@ class TestQuery(unittest.TestCase): ...@@ -189,6 +209,7 @@ class TestQuery(unittest.TestCase):
select_expression_list= select_expression_list=
["MATCH title AGAINST ('Foo' ) AS title_relevance"]), ["MATCH title AGAINST ('Foo' ) AS title_relevance"]),
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=['title'])) full_text_search_keys=['title']))
def testSimpleQueryFullTextSearchMode(self): def testSimpleQueryFullTextSearchMode(self):
...@@ -199,6 +220,7 @@ class TestQuery(unittest.TestCase): ...@@ -199,6 +220,7 @@ class TestQuery(unittest.TestCase):
select_expression_list= select_expression_list=
["MATCH title AGAINST ('Foo' IN BOOLEAN MODE) AS title_relevance"]), ["MATCH title AGAINST ('Foo' IN BOOLEAN MODE) AS title_relevance"]),
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=['title'])) full_text_search_keys=['title']))
def testSimpleQueryFullTextStat__(self): def testSimpleQueryFullTextStat__(self):
...@@ -209,23 +231,26 @@ class TestQuery(unittest.TestCase): ...@@ -209,23 +231,26 @@ class TestQuery(unittest.TestCase):
where_expression="MATCH title AGAINST ('Foo' )", where_expression="MATCH title AGAINST ('Foo' )",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=['title'], full_text_search_keys=['title'],
stat__=1)) stat__=1))
def testSimpleQueryKeywordSearchKey(self): def testSimpleQueryKeywordSearchKey(self):
q = Query(title='Foo') q = Query(title='Foo')
self.assertEquals(dict(where_expression="title LIKE '%Foo%'", self.assertEquals(dict(where_expression="((((title LIKE '%Foo%'))))",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'], q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[])) full_text_search_keys=[]))
def testNegatedQuery(self): def testNegatedQuery(self):
q1 = Query(title='Foo') q1 = Query(title='Foo')
q = NegatedQuery(q1) q = NegatedQuery(q1)
self.assertEquals( self.assertEquals(
dict(where_expression="(NOT (title = 'Foo'))", dict(where_expression="(NOT (((((title = 'Foo'))))))",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])) full_text_search_keys=[]))
# complex queries # complex queries
...@@ -234,9 +259,10 @@ class TestQuery(unittest.TestCase): ...@@ -234,9 +259,10 @@ class TestQuery(unittest.TestCase):
q2 = Query(reference='Bar') q2 = Query(reference='Bar')
q = ComplexQuery(q1, q2) q = ComplexQuery(q1, q2)
self.assertEquals( self.assertEquals(
dict(where_expression="((title = 'Foo') AND (reference = 'Bar'))", dict(where_expression="((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])) full_text_search_keys=[]))
def testNegatedComplexQuery(self): def testNegatedComplexQuery(self):
...@@ -246,35 +272,40 @@ class TestQuery(unittest.TestCase): ...@@ -246,35 +272,40 @@ class TestQuery(unittest.TestCase):
q = NegatedQuery(q3) q = NegatedQuery(q3)
self.assertEquals( self.assertEquals(
# maybe too many parents here # maybe too many parents here
dict(where_expression="(NOT (((title = 'Foo') AND (reference = 'Bar'))))", dict(where_expression="(NOT (((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))))",
select_expression_list=[]), select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])) full_text_search_keys=[]))
# forced keys # forced keys
def testSimpleQueryForcedKeywordSearchKey(self): def testSimpleQueryForcedKeywordSearchKey(self):
q = Query(title='Foo', key='Keyword') q = Query(title='Foo', key='Keyword')
self.assertEquals("title LIKE '%Foo%'", self.assertEquals("((((title LIKE '%Foo%'))))",
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression']) full_text_search_keys=[])['where_expression'])
def testSimpleQueryForcedFullText(self): def testSimpleQueryForcedFullText(self):
q = Query(title='Foo', key='FullText') q = Query(title='Foo', key='FullText')
self.assertEquals("MATCH title AGAINST ('Foo' )", self.assertEquals("MATCH title AGAINST ('Foo' )",
q.asSQLExpression(keyword_search_keys=[], q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression']) full_text_search_keys=[])['where_expression'])
def testSimpleQueryForcedExactMatch(self): def testSimpleQueryForcedExactMatch(self):
q = Query(title='Foo', key='ExactMatch') q = Query(title='Foo', key='ExactMatch')
self.assertEquals("title = 'Foo'", self.assertEquals("title = 'Foo'",
q.asSQLExpression(keyword_search_keys=['title'], q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression']) full_text_search_keys=[])['where_expression'])
def testSimpleQueryForcedExactMatchOR(self): def testSimpleQueryForcedExactMatchOR(self):
q = Query(title='Foo% OR %?ar', key='ExactMatch') q = Query(title='Foo% OR %?ar', key='ExactMatch')
self.assertEquals("title = 'Foo% OR %?ar'", self.assertEquals("title = 'Foo% OR %?ar'",
q.asSQLExpression(keyword_search_keys=['title'], q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression']) full_text_search_keys=[])['where_expression'])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7