Commit 51cad3f4 authored by Vincent Pelletier's avatar Vincent Pelletier

Major SQLCatalog rework on query-generation (not on indexation):

- further extend SearchKey concept
- define, implement and validate Interfaces
- add a column mapper (mapping is decided on a completely-formed query tree instead of being done individualy on each used column)
- state what is present for backward compatibility, warn about deprecated/dangerous uses


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@25706 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 68a0d3c2
No related merge requests found
##############################################################################
#
# Copyright (c) 2008-2009 Nexedi SARL and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import sys
from zLOG import LOG
from Interface.IColumnMap import IColumnMap
from Interface.Verify import verifyClass
from SQLCatalog import profiler_decorator
DEFAULT_GROUP_ID = None
TRACE = False
# TODO: handle left joins
# TODO: handle straight joins
# TODO: make it possible to do: query=ComplexQuery(Query(source_title='foo'), Query(source_title='bar')), sort_on=[('source_title_1', )]
# currently, it's not possible because related_key_dict is indexed by related key name, which makes 'source_title_1' lookup fail. It should be indexed by group (probably).
# TODO: rename all "related_key" references into "virtual_column"
class ColumnMap(object):
__implements__ = IColumnMap
@profiler_decorator
def __init__(self, catalog_table_name=None):
self.catalog_table_name = catalog_table_name
# Key: group
# Value: set of column names
self.registry = {}
# Key: group
# Value: dict
# Key: column name
# Value: set of SimpleQuery
self.simple_query_dict = {}
# Key: (group, column name)
# Value: table name
self.column_map = {}
# Key: (group, table name)
# Value: table alias
self.table_alias_dict = {}
# Key: related key name
# Value: (group, column name)
self.related_key_dict = {}
# Key: related_column
# Value: last used alias order
self.related_key_order_dict = {}
# Key: group
# Value: relate_key
self.related_group_dict = {}
# Key: table alias
# Value: table name
self.table_map = {}
# Key: raw column
# Value: (function, column)
self.raw_column_dict = {}
# Entries: column name
self.column_ignore_set = set()
self.join_table_set = set()
self.straight_join_table_list = []
self.left_join_table_list = []
self.join_query_list = []
@profiler_decorator
def registerColumn(self, raw_column, group=DEFAULT_GROUP_ID, simple_query=None):
assert ' as ' not in raw_column.lower()
# Sanitize input: extract column from raw column (might contain COUNT, ...).
if '(' in raw_column:
function, column = raw_column.split('(')
column = column.strip()
assert column[-1] == ')'
column = column[:-1].strip()
else:
function = None
column = raw_column
# Remove '`' from column.
column = column.replace('`', '')
# Extract table name from column, if any.
if '.' in column:
# Assuming the part before the dot is a real table name, not an alias.
table, column = column.split('.')
LOG('ColumnMap', 100, 'Column %r is pre-mapped. This use is strongly discouraged.' % (raw_column, ))
else:
table = None
self.raw_column_dict[raw_column] = (function, column)
self.registry.setdefault(group, set()).add(column)
self.simple_query_dict.setdefault(group, {}).setdefault(column, set()).add(simple_query)
if table is not None:
# Register table alias and mark column as resolved.
self.registerTable(table, alias=table, group=group)
self.resolveColumn(column, table, group=group)
if group is DEFAULT_GROUP_ID and table != self.catalog_table_name:
# When a column is registered in default group and is explicitely
# mapped to a table, we must mark its table as requiring a join with
# catalog table (unless it's the catalog table, of course).
self._addJoinTable(table, group)
def ignoreColumn(self, column):
self.column_ignore_set.add(column)
@profiler_decorator
def registerRelatedKey(self, related_column, column):
# XXX: should we store the group, or directly the table on which the column is mapped ?
# The former avoids duplicating data, but requires one more lookup (group + column -> table)
# The latter makes it harder (?) to split the mapping in multiple queries (if splitting by groups turns out to be a good idea)
real_related_column = related_column
order = self.related_key_order_dict.get(real_related_column, 0) + 1
related_column = '%s_%s' % (related_column, order)
group = 'related_%s' % (related_column, )
assert group not in self.registry
assert group not in self.related_group_dict
self.related_key_order_dict[real_related_column] = order
self.related_key_dict[real_related_column] = (group, column)
self.registerColumn(column, group=group)
self.related_group_dict[group] = related_column
# XXX: hardcoded translation table column names: they are not present in sql_catalog.getColumnMap(), and this table cannot be joined by uid, forbidding implicit join.
if column in ('translated_message', 'language', 'message_context', 'original_message'):
self.registerTable('translation', alias='translation', group=group)
self.resolveColumn(column, 'translation', group=group)
# Likewise, for measure table. Moreover, there is a related key named the same way as a column of that table (designed to do the join).
elif column in ('metric_type_uid', ):
self.registerTable('measure', group=group)
self.resolveColumn(column, 'measure', group=group)
return group
@profiler_decorator
def registerCatalog(self):
assert self.catalog_table_name is not None
LOG('ColumnMap', 100, 'Registering implicit catalog. This use is strongly discouraged.')
self.registerTable(self.catalog_table_name)
self.resolveTable(self.catalog_table_name, self.catalog_table_name)
@profiler_decorator
def registerRelatedKeyColumn(self, related_column, position, group):
assert group in self.related_group_dict
group = self.getRelatedKeyGroup(position, group)
assert group not in self.related_group_dict
self.related_group_dict[group] = related_column
return group
def getRelatedKeyGroup(self, position, group):
return '%s_column_%s' % (group, position)
@profiler_decorator
def registerTable(self, table_name, alias=None, group=DEFAULT_GROUP_ID):
table_alias_dict = self.table_alias_dict
table_alias_key = (group, table_name)
existing_value = table_alias_dict.get(table_alias_key)
# alias = None, existing = None -> store
# alias = None, existing ! None -> skip
# alias ! None, existing = None -> store & resolve
# alias ! None, existing ! None -> skip if alias = existing, raise otherwise
if existing_value is None:
table_alias_dict[table_alias_key] = alias
if alias is not None:
self.resolveTable(table_name, alias, group=group)
elif alias is not None and alias != existing_value:
raise ValueError, 'Table %r for group %r is aliased as %r, can\'t alias it now as %r' % (table_name, group, existing_value, alias)
@profiler_decorator
def _mapColumns(self, column_table_map, table_usage_dict, column_name_set, group, vote_result_dict):
mapping_dict = {}
catalog_table_name = self.catalog_table_name
# Map all columns to tables decided by vote.
for column_name, candidate_dict in vote_result_dict.iteritems():
# candidate_dict is never empty
max_score = 0
for table_name, score in candidate_dict.iteritems():
if score > max_score:
max_score = score
best_count = 0
best_choice = table_name
elif score == max_score:
best_count += 1
if best_count:
LOG('ColumnMap', 100, 'Mapping vote led to a tie. Mapping to %r' % (best_choice, ))
if TRACE:
LOG('ColumnMap', 0, 'Mapping by vote %r to %r' % (column_name, best_choice))
mapping_dict[column_name] = best_choice
column_name_set.remove(column_name)
for table_name, column_set in table_usage_dict.iteritems():
if table_name != best_choice:
column_set.discard(column_name)
# Map all remaning columns.
def table_weight(a):
"""
Compute each table weight.
"""
if (group, a[0]) in self.table_alias_dict:
result = (2, )
elif a[0] == catalog_table_name:
result = (1, )
else:
result = (0, len(a[1]))
return result
# Sort table name list, first has the most required columns
weighted_table_list = sorted(table_usage_dict.iteritems(), key=table_weight)
while len(weighted_table_list):
table_name, column_set = weighted_table_list.pop()
if len(column_set):
common_column_set = column_name_set.intersection(column_set)
if len(common_column_set):
# Only allow usage of this table if any of those is true:
# - current table is the catalog (if any catalog was provided)
# - there are column used on that table which are already mapped
# (this does not include columns mapped by this code)
# If columns are mapped to this table in current group, then using
# it will not require a new join, so it should be allowed.
# Note: it would be good to take indexes into account when there
# are multiple candidate tables.
# - any of those columns belongs exclusively to this table
# Although the list of tables those columns belong to is known
# earlier (in "build"), mapping them here
# - avoids code duplication (registerTable, resolveColumn,
# _addJoinTable)
# - offers user to vote for an unknown table, overriding this
# forced mapping.
use_allowed = table_name == catalog_table_name or \
len(common_column_set) < len(column_set)
if not use_allowed:
for column_name in column_set:
if len(column_table_map.get(column_name, [])) == 1:
# There is no alternative, mark as required
use_allowed = True
break
if use_allowed:
for column_name in common_column_set:
if TRACE:
LOG('ColumnMap', 0, 'Mapping by default %r to %r' % \
(column_name, table_name))
mapping_dict[column_name] = table_name
# This column must not be resolved any longer
column_name_set.remove(column_name)
# Remove this column from sets containing it. This prevents from
# giving a high score to a table which columns would already have
# been mapped to another table.
for ignored, other_column_set in weighted_table_list:
other_column_set.discard(column_name)
weighted_table_list.sort(key=table_weight)
else:
# All column which are mappable on that table are to-be-mapped
# columns. This means that this table was not explicitely used, and
# as each table contain a different amount of lines, we should not
# join with any non-explicit table. Hence, we skip this mapping.
LOG('ColumnMap', 0, 'Skipping possible map of %r on %r as that table' \
' is not explicitely used.' % (common_column_set, table_name))
# Detect incomplete mappings
if len(column_name_set):
raise ValueError, 'Could not map those columns: %r' % (column_name_set, )
# Do the actual mapping
for column_name, table_name in mapping_dict.iteritems():
# Mark this column as resolved
if TRACE:
LOG('ColumnMap', 0, 'Mapping column %s to table %s' % (column_name, table_name))
self.registerTable(table_name, group=group)
self.resolveColumn(column_name, table_name, group=group)
if table_name != catalog_table_name:
self._addJoinTable(table_name, group)
@profiler_decorator
def build(self, sql_catalog):
catalog_table_name = self.catalog_table_name
if catalog_table_name is None:
return
column_table_map = sql_catalog.getColumnMap()
table_vote_method_list = [getattr(sql_catalog, x) for x in sql_catalog.sql_catalog_table_vote_scripts]
# Generate missing joins from default group (this is required to allow using related keys outside of queries: order_by, sort_on, ...)
column_set = self.registry.get(DEFAULT_GROUP_ID, [])
for column_name in column_set:
if column_name not in column_table_map and column_name not in self.related_key_dict:
related_key_definition = sql_catalog.getRelatedKeyDefinition(column_name)
if related_key_definition is not None:
join_query = sql_catalog.getSearchKey(column_name, 'RelatedKey').buildQuery(sql_catalog=sql_catalog, related_key_definition=related_key_definition)
join_query.registerColumnMap(sql_catalog, self)
self._addJoinQuery(join_query)
# List all possible tables, with all used column for each
for group, column_set in self.registry.iteritems():
# unique needed column name set
column_name_set = set()
# table -> column_set, including alternatives
table_usage_dict = {}
for column_name in column_set:
if column_name == '*' or column_name in self.column_ignore_set:
continue
table_name_list = column_table_map.get(column_name, [])
if len(table_name_list) == 0:
if not(group is DEFAULT_GROUP_ID and column_name in self.related_key_dict):
LOG('ColumnMap', 100, 'Not a known column name: %r' % (column_name, ))
continue
column_map_key = (group, column_name)
if column_map_key in self.column_map:
# Column is already mapped, so we must count this column as being available only on that table. Its mapping will not change, and it will impact table schema choice.
table_name = self.column_map[column_map_key]
assert table_name in table_name_list, '%r not in %r' % (table_name, table_name_list)
table_name_list = [table_name]
else:
# Mark this column as requiring to be mapped.
column_name_set.add(column_name)
for table_name in table_name_list:
table_usage_dict.setdefault(table_name, set()).add(column_name)
# XXX: mutable datatypes are provided to vote method. if it modifies
# them, it can introduce mapping bugs. Copying them might be costly,
# especialy if done before each call, since they also contain mutable
# types.
# XXX: the API of vote methods is not stable yet. Parameters should
# always be passed and expected by name, to make it less painful to
# change API.
# XXX: there is no check that the table voted for contains mapped
# column. It is up to the user not to do stupid things.
vote_result_dict = {}
simple_query_dict = self.simple_query_dict[group]
for table_vote_method in table_vote_method_list:
vote_dict = table_vote_method(column_name_set=column_name_set,
simple_query_dict=simple_query_dict,
table_usage_dict=table_usage_dict,
group=group)
if isinstance(vote_dict, dict):
for column, table in vote_dict.iteritems():
if column in column_name_set:
column_vote_dict = vote_result_dict.setdefault(column, {})
column_vote_dict[table] = column_vote_dict.get(table, 0) + 1
else:
LOG('ColumnMap', 100, 'Vote script %r voted for a ' \
'non-candidate column: %r, candidates are: %r. Ignored.' %
(table_vote_method, column, column_name_set))
else:
LOG('ColumnMap', 100, 'Vote script %r returned invalid data: %r. ' \
'Ignored.' % (table_vote_method, vote_dict))
self._mapColumns(column_table_map, table_usage_dict, column_name_set, group, vote_result_dict)
table_alias_number_dict = {}
for (group, table_name), alias in self.table_alias_dict.iteritems():
if alias is None:
if group in self.related_group_dict:
alias_table_name = 'related_%s_%s' % (self.related_group_dict[group], table_name)
else:
alias_table_name = table_name
table_alias_number = table_alias_number_dict.get(alias_table_name, 0)
while True:
if table_alias_number == 0:
alias = alias_table_name
else:
alias = '%s_%s' % (alias_table_name, table_alias_number)
table_alias_number += 1
if alias not in self.table_map:
break
table_alias_number_dict[alias_table_name] = table_alias_number
self.resolveTable(table_name, alias, group=group)
if TRACE:
# Key: group
# Value: 2-tuple
# dict
# Key: column
# Value: table name
# dict
# Key: table name
# Value: table alias
summary_dict = {}
for (group, column), table_name in self.column_map.iteritems():
column_dict = summary_dict.setdefault(group, ({}, {}))[0]
assert column not in column_dict, '%r in %r' % (column, column_dict)
column_dict[column] = table_name
for (group, table_name), table_alias in self.table_alias_dict.iteritems():
table_dict = summary_dict.setdefault(group, ({}, {}))[1]
assert table_name not in table_dict, '%r in %r' % (table_name, table_dict)
table_dict[table_name] = table_alias
for group, (column_dict, table_dict) in summary_dict.iteritems():
LOG('ColumnMap', 0, 'Group %r:' % (group, ))
LOG('ColumnMap', 0, ' Columns:')
for column, table_name in column_dict.iteritems():
LOG('ColumnMap', 0, ' %r from table %r' % (column, table_name))
LOG('ColumnMap', 0, ' Tables:')
for table_name, table_alias in table_dict.iteritems():
LOG('ColumnMap', 0, ' %r as %r' % (table_name, table_alias))
def asSQLColumn(self, raw_column, group=DEFAULT_GROUP_ID):
if self.catalog_table_name is None or raw_column in self.column_ignore_set or \
'.' in raw_column or '*' in raw_column:
result = raw_column
else:
function, column = self.raw_column_dict.get(raw_column, (None, raw_column))
if group is DEFAULT_GROUP_ID:
group, column = self.related_key_dict.get(column, (group, raw_column))
alias = self.table_alias_dict[(group, self.column_map[(group, column)])]
result = '`%s`.`%s`' % (alias, column)
if function is not None:
result = '%s(%s)' % (function, result)
return result
def getCatalogTableAlias(self, group=DEFAULT_GROUP_ID):
return self.table_alias_dict[(group, self.catalog_table_name)]
def getTableAliasDict(self):
return self.table_map.copy()
@profiler_decorator
def resolveColumn(self, column, table_name, group=DEFAULT_GROUP_ID):
assert group in self.registry
assert column in self.registry[group]
column_map_key = (group, column)
column_map = self.column_map
assert (group, table_name) in self.table_alias_dict
previous_value = column_map.get(column_map_key)
if previous_value is None:
column_map[column_map_key] = table_name
elif previous_value != table_name:
if column == 'uid':
LOG('ColumnMap', 100, 'Attempt to remap uid from %r to %r ignored.' % (previous_value, table_name))
else:
raise ValueError, 'Cannot remap a column to another table. column_map[%r] = %r, new = %r' % (column_map_key, column_map.get(column_map_key), table_name)
@profiler_decorator
def resolveTable(self, table_name, alias, group=DEFAULT_GROUP_ID):
table_alias_key = (group, table_name)
assert table_alias_key in self.table_alias_dict
assert self.table_alias_dict[table_alias_key] in (None, alias)
self.table_alias_dict[table_alias_key] = alias
assert self.table_map.get(alias) in (None, table_name)
self.table_map[alias] = table_name
def getTableAlias(self, table_name, group=DEFAULT_GROUP_ID):
return self.table_alias_dict[(group, table_name)]
def _addJoinQuery(self, query):
self.join_query_list.append(query)
def addJoinQuery(self, query):
LOG('ColumnMap', 0, 'addJoinQuery use is discouraged')
self._addJoinQuery(query)
def iterJoinQueryList(self):
return iter(self.join_query_list)
@profiler_decorator
def _addJoinTable(self, table_name, group=DEFAULT_GROUP_ID):
"""
Declare given table as requiring to be joined with catalog table.
table_name (string)
Table name.
group (string)
Group id of given table.
"""
catalog_table = self.catalog_table_name
if catalog_table is not None:
# Only join tables when there is a catalog table
# Register unconditionaly catalog table
self.registerTable(catalog_table)
if 'uid' not in self.registry.get(DEFAULT_GROUP_ID, ()):
# Register uid column if it is not already
self.registerColumn('uid')
self.resolveColumn('uid', catalog_table)
self.join_table_set.add((group, table_name))
def getJoinTableAliasList(self):
return [self.getTableAlias(table_name, group=group)
for (group, table_name) in self.join_table_set]
def getStraightJoinTableList(self):
return self.straight_join_table_list[:]
def getLeftJoinTableList(self):
return self.left_join_table_list[:]
verifyClass(IColumnMap, ColumnMap)
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class INode(Interface):
"""
Any kind of node in an Abstract Syntax Tree.
"""
def isLeaf():
"""
Returns True if current node is a leaf in node tree.
Returns False otherwise.
"""
def isColumn():
"""
Returns True if current node is a column in node tree.
Returns False otherwise.
"""
class IValueNode(INode):
"""
Value- and comparison-operator-containig node.
They are leaf nodes in the syntax tree.
"""
def getValue():
"""
Returns node's value.
"""
def getComparisonOperator():
"""
Returns node's comparison operator.
"""
class ILogicalNode(INode):
"""
Logical-operator-containing node.
They are internal tree nodes.
"""
def getLogicalOperator():
"""
Returns node's logical operator.
"""
def getNodeList():
"""
Returns the list of subnodes.
"""
class IColumnNode(INode):
"""
Column-name-containing node.
They are internal tree nodes.
Their value applies to any contained ValueNode, except if there is another
ColumnNode between current one and a ValueNode, for which the other
ColumnNode will take precedence.
"""
def getColumnName():
"""
Returns node's column name.
"""
def getSubNode():
"""
Returns node's (only) subnode.
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class IColumnMap(Interface):
"""
The role of the column mapper is to make possible to have a "flat"
overview of all columns used in a query, to choose how they will be
mapped to catalog tables, and how those tables will be aliased in the SQL
rendering of that query.
Typical usage:
- Instanciate (with or without a catalog table)
- Register all columns the query will use
- Build the column map
- Fetch SQL representation of each registered column.
- Fetch table alias mapping
- Fetch the list of table aliases which require a join with catalog
- Fetch the list of queries implied by columns. This happens when there
are virtual columns registered which were not expanded in the query
already. In that case, column map generate queries required to reach
the real column behind the virtual one.
Note that, although it's not enforced, it is meaningless to:
- call build more than once
- register columns after build happened
- fetch SQL representation before build happened (this will most probably
lead to KeyErrors anyway)
Groups.
All references to a given table withing a given group will cause all
columns referencing that table to be resolved to the same table alias
in resulting query.
Ex:
Note: instead of resolving columns by hand, it is also possible to
call "build", but this way it makes .
Registration:
registerColumn('bar')
registerColumn('baz')
resolveColumn('bar', 'foo')
resolveColumn('baz', 'foo')
Result:
asSQLColumn('bar') -> 'foo_alias.bar'
asSQLColumn('baz') -> 'foo_alias.baz'
Complementary, any reference to a given table from one group to
another will cause columns to be resolved to distinct table aliases
in resulting query.
Ex:
Registration:
registerColumn('bar')
registerColumn('baz', group='hoge')
resolveColumn('bar', 'foo')
resolveColumn('baz', 'foo', group='hoge')
Result:
asSQLColumn('bar') -> 'foo_alias.bar'
asSQLColumn('baz', group='hoge') -> 'other_foo_alias.baz'
"""
def __init__(catalog_table_name=None):
"""
Create a column map.
Initialises internal data structures and set the table name to use as
catalog table.
"""
def registerColumn(column, group=None, simple_query=None):
"""
Register a column in given group on this column map.
column (string)
Contains the name of the column to register.
This name might contain a dot, in which case a table name is extracted
from it and that column is resolved to that table within its group.
Note that this practice is discouraged, but not deprecated.
group (string)
Group id of given column.
simple_query (SimpleQuery)
The SimpleQuery doing a comparison on given column. This can be
accessed from vote script at ColumnMap build time.
"""
def ignoreColumn(column):
"""
Act as if given column was valid and its mapping succeeded.
ie: supresses all warnings relative to that column, and all mapping
attemps.
It is supposed to be used when given value is an SQL alias for a
separately registered column.
column (string)
The value to ignore.
"""
def registerRelatedKey(related_column, column):
"""
Register a "virtual column".
This method is to be called when registering any virtual column.
A virtual column makes use internaly of multiple catalog columns.
A related key is an example of a virtual column.
This method internaly generates a new group id that caller must pass to
registerRelatedKeyColumn when registering those internal (and real)
columns. This is because the same virtual column can be used more than
once in a query without interferences from other uses, and hence must be
put in separate groups, without having to know that there are other uses
for the same virtual column.
virtual_column
Name of the virtual column to register.
column
Name of the "last" real column behind the virtual column. Comparisons
done on the virtual column will happen on that real column instead.
For example, it's "title" for "source_title" related key.
"""
def registerCatalog():
"""
Register catalog table as being used in default group.
This is for backward compatibility with text-only related keys, since
they often hardcode catalog table name.
"""
def registerRelatedKeyColumn(related_column, position, group):
"""
Register given column as part of already registered virtual column.
related_column (string)
Name of the virtual column registered column is a part of.
position (int)
Unique id of this column in the list of all columns used by the
virtual column it is a part of (it might use the same column name
while expecting uses to be mapped to different aliases or evne
different tables).
Typically, this is table position in the parameter list the related
key expects.
group (string)
Group id as returned by registerRelatedKey of given virtual column.
"""
def getRelatedKeyGroup(position, group):
"""
For given virtual key position and group, return a group.
This is here so that all group generation code is inside ColumnMap
class.
"""
def registerTable(table_name, alias=None, group=None):
"""
Register given table name as being used in given group.
This method should not be called outside of this class except for
backward compatibility purposes.
It is implicitely done most of the time, and should only be used only
when there is no control over chosen table alias or when table
registration cannot be done by a Query.
There are 2 cases where it cas required:
- pre-mapped columns which won't be actually used, but which force table
alias choice (see EntireQuery).
- related keys, where the Related Query cannot register columns it uses
by itself (since its content is not based on Queries but on raw SQL
text).
"""
def build(sql_catalog):
"""
Resolve all unresolved registered columns (ie, not mapped to tables).
Resolve all used tables to unique aliases.
Chosen table aliases are based on table name and virtual column name the
column comes from, if applicable. This is done to make final SQL more
reader-friendly.
"""
def asSQLColumn(column, group=None):
"""
Return an SQL rendering of given column, with the table alias it has
been mapped to.
column (string)
Column name. Can be the name of a virtual column.
This name might contain a dot, in which case a table name is extracted
from it and that column is resolved to that table within its group.
Note that this practice is discouraged, but not deprecated.
Note also that it does not apply to virtual columns (they cannot
contain a dot).
group (string)
Group id of given column.
"""
def getCatalogTableAlias(group=None):
"""
Return the alias of catalog table name given at instantiation.
"""
def getTableAliasDict():
"""
Return a copy of table mapping.
returned value (dict)
key (string)
Table alias.
value (string)
Table name.
"""
def resolveColumn(column, table_name, group=None):
"""
Map given column to given table within given group.
column (string)
Name of the column to map to a table. This cannot be a virtual column.
table_name (string)
Name of the table column must be mapped to. This is not an alias.
It must be already known to be used within given group.
group (string)
Name of the group column and table are in.
"""
def resolveTable(table_name, alias, group=None):
"""
Resolve given table of given group as given alias.
table_name (string)
Table to alias.
alias (string)
Table alias. It must be unique at query scope to work as intended.
gorup (string)
Group the table belongs to.
"""
def getTableAlias(table_name, group=None):
"""
Get table alias in given group.
table_name (string)
Table name.
group (string)
Group id of given table.
"""
def addJoinQuery(query):
"""
Adds a query to a list of top-level queries.
This method is only made available at interface level for backward
compatibility.
"""
def iterJoinQueryList():
"""
Get an iterator over queries internally generated when resolving column
map. Those queries are generated when a virtual column was registered as
a real one. Queries required to map virtual columns to real ones are
put in that list by build.
"""
def getJoinTableAliasList():
"""
Return a copy of the table alias list for tables requiring a join with
catalog table.
"""
def getStraightJoinTableList():
"""
Returns the list of tables used this search and which
need to be joined with the main table using explicit
indices.
"""
def getLeftJoinTableList():
"""
Returns the list of tables used this search and which
need to be LEFT joined with the main table using explicit
indices.
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class IEntireQuery(Interface):
"""
A EntireQuery represents an entire SQL expression, where a Query
represents only the "WHERE" part of that expression.
A EntireQuery contains:
- a Query instance
- a limit expression
- a group-by expression
- an order-by expression
- a select expression
It internaly uses a ColumnMap instance to resolve tables to use to
generate a "from" expression.
"""
def __init__(query, order_by_list=None, group_by_list=None,
select_dict=None, limit=None, catalog_table_name=None,
extra_column_list=None, from_expression=None,
order_by_override_list=None):
"""
query (Query instance)
The root of the Query tree this query will contain.
order_by_list (list of 1-tuple, 2-tuple or 3-tuple)
The list of columns which will be sorted by SQL.
Tuple values are:
- mandatory: column name
- optionnal: sort order (can be "ASC" or "DESC", "ASC" by default)
- optionnal: type cast (no cast by default, see "CAST" SQL method)
group_by_list (list of string)
The list of columns which will be groupped by value by SQL.
select_dict (dict, key: string, value: string, None)
Given values describe columns to make available in SQL result.
If column is aliased in result set, key is the alias and value is the
column.
Otherwise, key is the column, and value must be None.
limit
See SQLExpression.
catalog_table_name (string)
Name of the table to use as a catalog.
Deprecated parameters.
extra_column_list (list of string)
The list of columns to register to column map. They will not be used
in final rendering, but are hint on which table are supposed to be
used when mapping columns.
from_expression
See SQLExpression.
order_by_override_list (list of string)
If a column is in order_by_list, cannot be mapped to a table column
but is present in this list, it will be passed through to
SQLExpression.
"""
def asSQLExpression(sql_catalog, only_group_columns):
"""
Instantiate a column map, process parameters given at instantiation and
register them to column map.
Register query to column map.
Build column map.
Generate extra SQLExpressions from column map.
Generate SQLExpression instance and return it.
"""
def asSearchTextExpression(sql_catalog):
"""
This is just a passthrough to embeded Query's asSearchTextExpression
method.
This means that only the where expression can be represented as a
SearchText, but not sort, limits, ...
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class IOperator(Interface):
"""
An operator is responsible for rendering a value and a column name as SQL
or Search Text.
This class is designed to be used as a singleton-per-operator.
"""
def __init__(operator):
"""
operator (string)
Operator's text representation. It is used both for SQL and SearchText
rendering.
"""
def asSearchText(value):
"""
Render given value as Search Text
value (see _renderValue)
Value to render as a string for use in a Search Text expression.
"""
def asSQLExpression(column, value_list, only_group_columns):
"""
Construct a SQLExpression instance from given column and value, with
contained glue text.
value_list can be a non-list instance, which must be handled that same
way as a list of one item.
only_group_columns (bool)
If false, the operator can add group columns in the "select_dict" of
returned SLQExpression.
Otherwise, it must not (SQL would be invalid).
"""
def getOperator():
"""
Accessor for operator's text representation.
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class IQuery(Interface):
"""
A Query contains:
- a value
- an operator
- a column
It is the python representation of a predicate, independently of its
rendering (SQL or SearchText).
For SQL rendering to be possible, it is necesary for some data to be
centralized in a data structure known at EntireQuery level (to be able to
generate unique table aliases, for exemple). This is the role of
ColumnMap, and registerColumnMap method on this interface.
This interface also offers various rendering methods, one per rendering
format.
"""
def asSearchTextExpression(sql_catalog, column=None):
"""
Render a query in a user-oriented SearchText.
Returns None if there is this query has no SearchText representation,
but is SearchText-aware.
If column is provided, it must be used instead of local knowledge of
column name. It is used to make queries inside a related key render
correctly.
"""
def asSQLExpression(sql_catalog, column_map, only_group_columns):
"""
Render a query as an SQLExpression instance.
"""
def registerColumnMap(sql_catalog, column_map):
"""
Register a query to given column_map.
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class ISearchKeyCatalog(Interface):
def buildQuery(kw, ignore_empty_string=True, operator='and'):
"""
Build a ComplexQuery from kw values.
kw (dict: string keys, any value)
A query will be emited based on its value. Depending on the type of
the value it is handled differently. Query values will be passed
through to result (key is ignored). For all other types, their key
must be either a known column of a sql_search_tables table, or a
related key name.
- String values will be parsed according to the default SearchKey of
their real column (even for related keys). If parsing was
successful, Queries will be generated from its output.
Otherwise, that value will be taken as such.
- Dictionary values can be composed of the following keys:
'query': Their payload value, considered as empty if not given.
'key': The SearchKey to use for this value, overriding default
column configuration.
(for other possible keys, see SearchKeys)
They will be taken as such.
- All other types will be taken as such, and no "empty" check will be
performed on them.
ignore_empty_string (boolean)
If True, values from kw which are empty will be skipped.
operator (string)
Used to explicit the logical relation between kw entries.
It must be a valid ComplexQuery logical operator ('and', 'or').
"""
def buildSQLQuery(query_table='catalog', REQUEST=None,
ignore_empty_string=1, only_group_columns=False,
limit=None, extra_column_list=None,
**kw):
"""
Construct and return an instance of EntireQuery class from given
parameters by calling buildQuery.
ignore_empty_string (boolean)
See buildQuery.
limit (1-tuple, 2-tuple)
If given, will emit SQL to limit the number of result lines.
group_by_list (list of strings)
If given, will emit SQL to group found lines on given parameter names
(their column if they are column names, corresponding virtual columns
otherwise - as for related keys).
select_dict (dict, key: string, value: string, None)
Given values describe columns to make available in SQL result.
If column is aliased in result set, key is the alias and value is the
column.
Otherwise, key is the column, and value can be None or the same as
key.
select_list (list of strings)
Same as providing select_dict with select_list items as keys, and None
values.
order_by_list (list of 1-, 2-, or 3-tuples of strings)
If given, will emit SQL to sort result lines.
Sort will happen with decreasing precedence in list order.
Given n-tuples can contain those values, always in this order:
- parameter name
- sort order (see SQL documentation of 'ORDER BY')
- type cast (see SQL documentation of 'CAST')
Sort will happen on given parameter name (its column if it's a column
name, corresponding virtual column otherwise - as for related keys).
Extra parameters are passed through to buildQuery.
Backward compatibility parameters:
Those parameters are deprecated and should not be used. They are present
to provide backward compatibility with former ZSQLCatalog version.
REQUEST
Ignored.
extra_column_list (list)
query_table (string, None)
The table to use as catalog table.
If given and None, not catalog table will be used. Use this when you
are using SQLCatalog to generate manualy a part of another query.
That table has a special position in returned query:
- all other tables are joined on this one (when it is required to use
other tables)
- it is expected to have some columns (uid, path)
It is strongly discouraged to use this parameter for any value other
than None.
group_by
Use group_by_list instead.
group_by_expression
Use group_by_list instead.
select_expression
Use select_list or select_dict instead.
sort_on
Use order_by_list instead.
sort_order
Use order_by_list instead.
from_expression
This value will be emited in SQL directly in addition to computed
value.
There is no replacement.
where_expression
This value will be emited in SQL directly in addition to computed
value.
Use Query instances instead.
select_expression_key
This prevents given column from being ignored even if they could not
be mapped.
There is no replacement.
only_group_columns
Replaces former stat__ parameter.
Used to globally disalow use of non-group columns in SQL.
"""
def getSearchKey(column, search_key=None):
"""
Returns the default SearchKey instance for the
requested column. There is one instance per
search_key (incl. virtual keys surch as
source_title) and we try to compute it once
only and cache it.
If search_key is provided, it is used as the
name of the search key class to return.
"""
def getComparisonOperator(operator):
"""
Return a comparison operator matching given string.
String must be a valid SQL comparison operator (=, LIKE, IN, ...).
String case does not matter.
There is one comparison operator instance per possible string value.
"""
# TODO: add support for other operators (logical, ensemblist (?))
def searchResults(REQUEST=None, **kw):
"""
Invokes queryResults with the appropriate
ZSQL Method to return a list of results
"""
def countResults(REQUEST=None, **kw):
"""
Invokes queryResults with the appropriate
ZSQL Method to return a statistics for
a list of results
"""
def queryResults(sql_method, REQUEST=None, src__=0, build_sql_query_method=None, **kw):
"""
Return the result of the given 'sql_method' ZSQL Method after
processing all parameters to build a Query object passed to
that method.
The implementation should do the following.
1- Use **kw parameters to build a Query object
by invoking buildQuery
2- Build a ColumnMap instance by invoking
the buildColumnMap on the Query. Some
optmisation may happen here to try
to build the best possible ColumnMap and
use the best possible indices for joining.
During the ColumnMap build process, the
Search Key associated to each Query node
in the Query tree registers the columns
which are used (ex. to search) or provided
(ex. MATCH value for full text search,
interleave expression or parameter in a
UNION Query)
3- Render the query object as an SQLExpression
instance. This instance contains all necessary
parts to generate:
- where_expression
- sort_expression
- group_by_expression
- select_expression
4- Invoke sql_method
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class ISQLExpression(Interface):
"""
This is a container for chunks of SQL generated by a Query.
Each Query instance generates its own SQLExpression instance.
SQLExpressions can be nested (an SQLExpression can contain 0, 1 or more
SQLExpressions).
Chunks are:
- table_alias_dict
- order_by_list
- group_by_list
- select_dict
- limit
Mutualy exclusive chunks are:
- where expression
- sql_expression_list
Deprecated chunks are:
- from_expression
Providing sql_expression_list with more than one entry makes
where_expression_operator mandatory.
"""
def __init__(query,
table_alias_dict=None,
order_by_list=None,
order_by_dict=None,
group_by_list=None,
where_expression=None,
where_expression_operator=None,
sql_expression_list=None,
select_dict=None,
limit=None,
from_expression=None):
"""
Instantiate an SQLExpression object.
This method does consistency checks on received parameters, so that
failures are detected as early as possible.
Also, it casts most optional parameters into empty lists and empty dicts
to make code using those values simpler.
query (Query)
The Query instance which called this constructor.
table_alias_dict (dict, key: string, value: string)
Table alias dict as returned by ColumnMap.getTableAliasDict() .
order_by_list (list of strings)
List of result ordering, pre-rendered.
order_by_dict (dict, key: string, value: string)
Column rendering replacement specific to order_by.
group_by_list (list of strings)
List of column names on which result line list will be grouped.
where_expression (string)
Text representing a "where" expression of an SQL query.
where_expression_operator ("and", "or", "not", None)
Operator to apply on immediately contained SQLExpressions.
It must be "and" or "or" when there are multiple contained
SQLExpressions, it can be "not" if there is exactly one contained
SQLExpression, and must not be provided if there is no contained
SQLExpression.
sql_expression_list (list of SQLExpression)
List of immediately contained SQLExpressions.
select_dict (dict, key:string, value:string or Null)
Lists all columns to be part of select expression.
Key is column alias.
Value is column name, or Null. If it is Null, the alias will also be
used as column name.
limit (1-tuple, 2-tuple, other)
First item is the number of lines expected, second one if given is the
offset of limited result list within the unlimited result list.
If it is not a tuple it is treated as the value of a 1-tuple
parameter.
from_expression (string)
This parameter is deprecated.
"""
def asSQLExpressionDict():
"""
Returns a dictionnary usable as a **kw for a catalog sql method.
It renders aliases (see getTableAliasDict) as a list of strings and
"from" expression (see getFromExpression) as a list of strings.
See getWhereExpression, getOrderByExpression, getLimitExpression,
getSelectExpression, getGroupByExpression.
"""
##############################################################################
#
# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Interface import Interface
class ISearchKey(Interface):
"""
A SearchKey generates Query instances.
It is responsible for parsing (or not) input value and generating the
appropriate Query structure to match the intended search.
This class is designed as a "singleton per column".
"""
def __init__(column):
"""
column (string)
The column this SearchKey is instanciated for.
"""
def getColumn():
"""
Returns: string
The column this SearchKey is instanciated for.
"""
def buildSQLExpression(operator, value, column_map, only_group_columns, group):
"""
This method passes on SQLExpression creation to operator, giving it the
resolved column of this RelatedKey.
operator (Operator)
A comparison operator. It is used to render value, column and itself
as valid SQL.
value (anything)
column_map (ColumnMap)
The (built) column map to render our column.
group (string)
The group our column was part of.
Returns: SQLExpression
SQLExpression build by operator for current Query.
"""
def buildSearchTextExpression(operator, value, column=None):
"""
Render comparison using operator between value and our column as
SearchText.
operator (Operator)
A comparison operator. It is used to render value as value SearchText.
value (anything)
column (string)
If given, it overrides our column in SearchText rendering. This is
useful when SearchText rendering is done for a virtual column,
because the name of the virtual column must appear when rendered as
SearchText.
"""
def registerColumnMap(column_map, group, simple_query):
"""
Register the column of this SearchKey to given column map.
column_map (ColumnMap)
The ColumnMap instance to register to.
group (string)
The group registration will be part of.
simple_query (SimpleQuery)
The SimpleQuery being registered.
Returns: string
The group assigned to caller. See ColumnMap for methods redefining
caller's group.
"""
def buildQuery(search_value, group=None, logical_operator=None, comparison_operator=None):
"""
Generate queries from given search_value.
logical_operator ('and', 'or', 'not', None)
If no logical operator can be found in search_value, it will use given
default operator.
comparison_operator (string, None)
If given, expresses the comparison between column and value.
"""
def parseSearchText(value):
"""
Parse given value to generate an Abstract Syntax Tree representing its
logical structure, or None if there is no obvious structure in given
value.
See SearchText for parsing code.
value (string)
The string to parse.
Returns: (None, AbstratSyntaxNode)
AbstratSyntaxNode complies with the IAbstractSyntaxNode interface.
"""
class IRelatedKey(ISearchKey):
"""
A RelatedKey is a special variation of a SearchKey.
Only a small set of methods differ. They are defined in this class.
"""
def registerColumnMap(column_map, table_alias_list=None):
"""
This is slightly different for regular registerColumnMap in that it must
register multiple tables (and not columns, since RelatedKeys do not
provide this information).
Also, it must register namely "catalog" table and resolve its alias,
angain since it's hardcoded in RelatedKey ZSQLMethods and not provided
by their definitions.
column_map (ColumnMap)
Deprecated:
table_alias_list (None, list of 2-tuples of strings)
This list must have the exact same length as the list of tables
"""
def buildSQLExpression(sql_catalog, column_map, only_group_columns, group):
"""
operator and value parameters are useless, since this type of SearhKey
does not compare a value to any column, but uses a ZSQLMethod.
To reach that ZSQLMethod, it also required a new sql_catalog parameter.
sql_catalog (SQLCatalog)
Used to retrieve related key's ZSQLMethod.
"""
def buildQuery(sql_catalog, related_key_definition, search_value=None, search_key_name=None, logical_operator=None, comparison_operator=None):
"""
group is useless here, since group is determined by ColumnMap at
registration time. search_value becomes optional.
sql_catalog (SQLCatalog)
Used to retrieve real column's SearchKey. For example, a RelatedKey
used to compare with a "title" column will retrieve title's default
SearchKey (which should be a KeywordKey).
related_key_definition (string)
Describes parameters of a RelatedKey. It is composed of 3 mains parts,
separated by '/':
- a list of table names
Table names are separated by ','
- a column name
- the name of the related key ZSQLMethod
search_value (anything)
If given, a condition on real column will be generated.
Otherwise, only the SQL required to reach that column will be
generated. This is usefull when sorting on a virtual column, for
example.
search_key_name (string, None)
If given, it overrides real column's default SearchKey.
logical_operator (string, None)
If given, expresses the default logical link between operands.
It must be one of None, 'or' and 'and'.
It is overriden by operator present in search_value if it is a dict
and contains an 'operator' key.
'or' is assumed if not given or given with a None value.
comparison_operator (string, None)
If given, expresses the comparison between column and value.
"""
# This imports from Zope's products, which would otherwise be unreachable from parent folder.
from Interface.Verify import verifyClass
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from OperatorBase import OperatorBase
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.Interface.IOperator import IOperator
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
class ComparisonOperatorBase(OperatorBase):
@profiler_decorator
def asSQLExpression(self, column, value_list, only_group_columns):
"""
In a Comparison Operator, rendering order is:
<column> <operator> <value_list>
"""
column, value_list = self.render(column, value_list)
return SQLExpression(self, where_expression='%s %s %s' % (column, self.getOperator().upper(), value_list))
def render(self, column, value_list):
raise NotImplementedError, 'This method must be overloaded by a subclass.'
def renderValue(self, value_list):
raise NotImplementedError, 'This method must be overloaded by a subclass.'
verifyClass(IOperator, ComparisonOperatorBase)
class MonovaluedComparisonOperator(ComparisonOperatorBase):
@profiler_decorator
def renderValue(self, value_list):
"""
value_list must either be a non-list or a single-value list.
"""
if isinstance(value_list, (tuple, list)):
if len(value_list) > 1:
raise ValueError, '%r: value_list must not contain more than one item. Got %r' % (self, value_list)
value_list = value_list[0]
return self._renderValue(value_list)
@profiler_decorator
def render(self, column, value_list):
"""
value_list must either be a non-list or a single-value list.
"""
if isinstance(value_list, (tuple, list)):
if len(value_list) > 1:
raise ValueError, '%r: value_list must not contain more than one item. Got %r' % (self, value_list)
value_list = value_list[0]
return self._render(column, value_list)
verifyClass(IOperator, MonovaluedComparisonOperator)
class MultivaluedComparisonOperator(ComparisonOperatorBase):
@profiler_decorator
def renderValue(self, value_list):
"""
value_list must be a multi-value list (more than one item).
"""
if not isinstance(value_list, (tuple, list)) or len(value_list) < 2:
raise ValueError, '%r: value_list must be a list of more than one item. Got %r' % (self, value_list)
return '(%s)' % (', '.join([self._renderValue(x) for x in value_list]), )
@profiler_decorator
def render(self, column, value_list):
"""
value_list must be a multi-value list (more than one item).
"""
if not isinstance(value_list, (tuple, list)) or len(value_list) < 2:
raise ValueError, '%r: value_list must be a list of more than one item. Got %r' % (self, value_list)
return column, '(%s)' % (', '.join([self._renderValue(x) for x in value_list]), )
verifyClass(IOperator, MultivaluedComparisonOperator)
class MatchComparisonOperator(MonovaluedComparisonOperator):
def __init__(self, operator, mode=''):
MonovaluedComparisonOperator.__init__(self, operator)
self.where_expression_format_string = 'MATCH (%%(column)s) AGAINST (%%(value_list)s%s)' % (mode, )
@profiler_decorator
def asSQLExpression(self, column, value_list, only_group_columns):
"""
This operator can emit a select expression, so it overrides
asSQLExpression inseatd of just defining a render method.
"""
match_string = self.where_expression_format_string % {
'column': column,
'value_list': self.renderValue(value_list),
}
select_dict = {}
if not only_group_columns:
select_dict[column.replace('`', '').split('.')[-1]] = match_string
# Sort on this column uses relevance.
# TODO: Add a way to allow sorting by raw column value.
order_by_dict = {
column: self.where_expression_format_string,
}
return SQLExpression(
self,
select_dict=select_dict,
where_expression=match_string,
order_by_dict=order_by_dict,
)
verifyClass(IOperator, MatchComparisonOperator)
operator_dict = {
'=': MonovaluedComparisonOperator('='),
'!=': MonovaluedComparisonOperator('!='),
'>': MonovaluedComparisonOperator('>'),
'<': MonovaluedComparisonOperator('<'),
'<=': MonovaluedComparisonOperator('<='),
'>=': MonovaluedComparisonOperator('>='),
'like': MonovaluedComparisonOperator('like'),
'match': MatchComparisonOperator('match'),
'match_boolean': MatchComparisonOperator('match_boolean', mode=' IN BOOLEAN MODE'),
'match_expansion': MatchComparisonOperator('match_expansion', mode=' WITH QUERY EXPANSION'),
'in': MultivaluedComparisonOperator('in'),
'is': MonovaluedComparisonOperator('is'),
}
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from zLOG import LOG
from Products.ZSQLCatalog.Interface.IOperator import IOperator
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
@profiler_decorator
def escapeString(value):
# Inspired from ERP5Type/Utils:sqlquote, but this product must not depend on it.
return "'" + value.replace('\\', '\\\\').replace("'", "''") + "'"
@profiler_decorator
def valueFloatRenderer(value):
if isinstance(value, basestring):
value = float(value.replace(' ', ''))
return repr(value)
@profiler_decorator
def valueDateTimeRenderer(value):
return '"%s"' % (value.toZone('UTC').ISO(), )
@profiler_decorator
def valueDefaultRenderer(value):
LOG('OperatorBase', 0, 'Unhandled value class: %s (%r). Converted to string and escaped.' % (value.__class__.__name__, value))
return escapeString(str(value))
@profiler_decorator
def valueNoneRenderer(value):
return 'NULL'
value_renderer = {
'int': str,
'long': str,
'float': valueFloatRenderer,
'DateTime': valueDateTimeRenderer,
'NoneType': valueNoneRenderer,
}
value_search_text_renderer = {
'DateTime': str,
}
@profiler_decorator
def valueDefaultSearchTextRenderer(value):
"""
This is just repr, but always surrounding text strings with doublequotes.
"""
if isinstance(value, basestring):
result = '"%s"' % (value.replace('\\', '\\\\').replace('"', '\\"'), )
else:
result = repr(value)
return result
@profiler_decorator
def columnFloatRenderer(column, format=None):
if format is not None:
if '.' in format:
format = format.replace(' ', '')
column = "TRUNCATE(%s, %s)" % (column, len(format.split('.')[-1]))
return column
@profiler_decorator
def columnDefaultRenderer(column, format=None):
return column
column_renderer = {
'float': columnFloatRenderer
}
class OperatorBase(object):
__implements__ = IOperator
def __init__(self, operator):
self.operator = operator
def getOperator(self):
return self.operator
@profiler_decorator
def _render(self, column, value):
"""
Render given column and value for use in SQL.
Value is rendered to convert it to SQL-friendly value.
Column is rendered to include possible cast code.
column (string)
Column on which the value will be matched
value (see _renderValue)
Value to render.
"""
if isinstance(value, dict):
type = value['type']
column = column_renderer.get(type, columnDefaultRenderer)(column, format=value['format'])
value = value_renderer.get(type, valueDefaultRenderer)(value['query'])
else:
value = self._renderValue(value)
return column, value
@profiler_decorator
def _renderValue(self, value):
"""
Render given value as string.
value (int, float, long, DateTime, string, None)
Value to render as a string for use in SQL (quoted, escaped).
"""
if isinstance(value, basestring):
value = escapeString(value)
else:
value = value_renderer.get(value.__class__.__name__, valueDefaultRenderer)(value)
return value
@profiler_decorator
def asSearchText(self, value):
return value_search_text_renderer.get(value.__class__.__name__, valueDefaultSearchTextRenderer)(value)
def asSQLExpression(self, column, value_list, only_group_columns):
raise NotImplementedError, 'This method must be overloaded by a subclass ' \
'to be able to get an SQL representation of this operator.'
def __repr__(self):
return '<%s(%r) at %s>' % (self.__class__.__name__, self.getOperator(), id(self))
verifyClass(IOperator, OperatorBase)
from ComparisonOperator import operator_dict
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Query import Query
from zLOG import LOG
from Products.ZSQLCatalog.Interface.IQuery import IQuery
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
class AutoQuery(Query):
"""
An AutoQuery is a compatibility layer for former Query class.
It passes parameters given at instantiation time to SQLCatalog's
buildQuery or buildSingleQuery, and wraps resulting Query instance (proxy
behaviour).
This is only here for backward compatibility, and use is strongly
discouraged. Use SQLCatalog API instead.
"""
wrapped_query = None
@profiler_decorator
def __init__(self, *args, **kw):
"""
Note: "operator" might contain a logical or a comparison operator.
"""
if len(args):
LOG('AutoQuery', 100, 'Got extra positional parameters (will be ignored): %r' % (args, ))
self.table_alias_list = kw.pop('table_alias_list', None)
self.kw = kw
operator = kw.pop('operator', None)
if isinstance(operator, basestring):
operator = operator.lower()
self.operator = operator
self.ignore_empty_string = kw.pop('ignore_empty_string', True)
if 'key' in kw and len(kw) > 2:
raise ValueError, '"key" parameter cannot be used when more than one column is given. key=%r' % (self.search_key, )
self.search_key = kw.pop('key', None)
@profiler_decorator
def _createWrappedQuery(self, sql_catalog):
"""
Create wrapped query. This requires being able to reach catalog, since
we use it as a query producer.
"""
kw = self.kw
operator = self.operator
if 'range' in kw:
# If we received a range parameter we are building a single query.
# Recreate value as a dict and pass it to buildSingleQuery.
range = kw.pop('range')
assert len(kw) == 1, repr(kw)
key, value = kw.items()[0]
query = sql_catalog.buildSingleQuery(key, {'query': value,
'range': range})
elif operator == 'in':
# 'in' is a *comparison* operator, not a logical operator.
# Transform kw into the proper form.
assert len(kw) == 1, repr(kw)
key, value = kw.items()[0]
query = sql_catalog.buildSingleQuery(key, {'query': value,
'operator': operator})
elif len(kw) == 1 and isinstance(kw.values()[0], (tuple, list)) and \
operator in ('and', 'or'):
# If there is only one parameter, and operator was given and is a
# known logical operator, then operator will apply to it.
# For example (from testDomainTool):
# kw = {'portal_type': ['!=a', '!=b'], 'operator': 'AND'}
# In such case, expected result is
# "portal_type!='a' AND portal_type!='b'"
key, value = kw.items()[0]
query = sql_catalog.buildSingleQuery(key, value, logical_operator=operator)
else:
# Otherwise, the operator will apply to the relationship between
# parameters.
if operator is None:
operator = 'and'
if self.search_key is not None:
key, value = kw.items()[0]
kw = {key: {'query': value, 'key': self.search_key}}
query = sql_catalog.buildQuery(kw, operator=operator, ignore_empty_string=self.ignore_empty_string)
if self.table_alias_list is not None:
query.setTableAliasList(self.table_alias_list)
self.wrapped_query = query
@profiler_decorator
def asSearchTextExpression(self, sql_catalog, column=None):
if self.wrapped_query is None:
self._createWrappedQuery(sql_catalog)
return self.wrapped_query.asSearchTextExpression(sql_catalog, column=column)
@profiler_decorator
def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
if self.wrapped_query is None:
self._createWrappedQuery(sql_catalog)
return self.wrapped_query.asSQLExpression(sql_catalog, column_map, only_group_columns=only_group_columns)
@profiler_decorator
def registerColumnMap(self, sql_catalog, column_map):
if self.wrapped_query is None:
self._createWrappedQuery(sql_catalog)
return self.wrapped_query.registerColumnMap(sql_catalog, column_map)
def __repr__(self):
if self.wrapped_query is None:
result = '<%s(**%r) at %s>' % (self.__class__.__name__, self.kw, id(self))
else:
result = '<%s %r>' % (self.__class__.__name__, self.wrapped_query)
return result
verifyClass(IQuery, AutoQuery)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -26,68 +28,127 @@
#
##############################################################################
from Products.PythonScripts.Utility import allow_class
from Query import QueryMixin
from Query import Query
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from SQLQuery import SQLQuery
from Products.ZSQLCatalog.Interface.IQuery import IQuery
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
class ComplexQuery(QueryMixin):
class ComplexQuery(Query):
"""
Used in order to concatenate many queries
A ComplexQuery represents logical operations between Query instances.
"""
@profiler_decorator
def __init__(self, *args, **kw):
self.query_list = args
self.operator = kw.pop('operator', 'AND')
# XXX: What is that used for ?! It's utterly dangerous.
#self.__dict__.update(kw)
def getQueryList(self):
return self.query_list
def getRelatedTableMapDict(self):
result = {}
for query in self.getQueryList():
if not(isinstance(query, basestring)):
result.update(query.getRelatedTableMapDict())
return result
def asSQLExpression(self, key_alias_dict=None,
ignore_empty_string=1,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
stat__=0):
"""
Build the sql string
*args (tuple of Query or of list of Query)
list-type entry will extend subquery list, other entries will be
appended.
logical_operator ('and', 'or', 'not')
Logical operator.
Default: 'and'
Deprecated
operator ('and', 'or', 'not')
See logical_operator.
logical_operator takes precedence if given.
unknown_column_dict (dict)
Only one key of this dictionnary is used here:
key: 'from_expression'
value: string
This value will be passed through to SQLExpression. If it is
provided, this ComplexQuery must have no subquery (regular
SQLExpression limitation)
implicit_table_list (list of strings)
Each entry in this list will be registered to column map. This is
used to make column mapper choose tables differently.
"""
sql_expression_list = []
select_expression_list = []
for query in self.getQueryList():
if isinstance(query, basestring):
sql_expression_list.append(query)
self.logical_operator = kw.pop('logical_operator', kw.pop('operator', 'and')).lower()
assert self.logical_operator in ('and', 'or', 'not'), self.logical_operator
unknown_column_dict = kw.pop('unknown_column_dict', {})
self.from_expression = unknown_column_dict.pop('from_expression', None)
self.implicit_table_list = kw.pop('implicit_table_list', [])
query_list = []
append = query_list.append
extend = query_list.extend
# Flaten the first level of list-type arguments
for arg in args:
if isinstance(arg, (list, tuple)):
extend(arg)
else:
append(arg)
new_query_list = []
append = new_query_list.append
# Iterate over the flaten argument list to cast each into a query type.
for query in query_list:
if not isinstance(query, Query):
query = SQLQuery(query)
append(query)
self.query_list = new_query_list
@profiler_decorator
def asSearchTextExpression(self, sql_catalog, column=None):
if column in (None, ''):
query_column = column
else:
query_column = ''
search_text_list = [y for y in [x.asSearchTextExpression(sql_catalog, column=query_column) for x in self.query_list] if y is not None]
if len(search_text_list) == 0:
result = ''
else:
query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
ignore_empty_string=ignore_empty_string,
keyword_search_keys=keyword_search_keys,
datetime_search_keys=datetime_search_keys,
full_text_search_keys=full_text_search_keys,
stat__=stat__)
sql_expression_list.append(query_result['where_expression'])
select_expression_list.extend(query_result['select_expression_list'])
operator = self.getOperator()
result = {'where_expression':('(%s)' % \
(' %s ' % operator).join(['(%s)' % x for x in sql_expression_list])),
'select_expression_list':select_expression_list}
if self.logical_operator in ('and', 'or'):
if len(search_text_list) == 1:
result = search_text_list[0]
else:
logical_operator = ' %s ' % (self.logical_operator.upper(), )
result = '(%s)' % (logical_operator.join(search_text_list), )
elif self.logical_operator == 'not':
assert len(search_text_list) == 1
result = '(NOT %s)' % (search_text_list[0], )
else:
raise ValueError, 'Unknown operator %r' % (self.logical_operator, )
if column not in (None, ''):
result = '%s:%s' % (column, result)
return result
def getSQLKeyList(self):
@profiler_decorator
def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
sql_expression_list = [x.asSQLExpression(sql_catalog, column_map, only_group_columns)
for x in self.query_list]
if len(sql_expression_list) == 0:
sql_expression_list = [SQLExpression(self, where_expression='1')]
return SQLExpression(self,
sql_expression_list=sql_expression_list,
where_expression_operator=self.logical_operator,
from_expression=self.from_expression)
@profiler_decorator
def registerColumnMap(self, sql_catalog, column_map):
for implicit_table_column in self.implicit_table_list:
column_map.registerColumn(implicit_table_column)
for query in self.query_list:
query.registerColumnMap(sql_catalog, column_map)
def __repr__(self):
return '<%s of %r.join(%r)>' % (self.__class__.__name__, self.logical_operator, self.query_list)
@profiler_decorator
def setTableAliasList(self, table_alias_list):
"""
Returns the list of keys used by this
instance
This function is here for backward compatibility.
This can only be used when there is one and only one subquery which
defines a setTableAliasList method.
See RelatedQuery.
"""
key_list=[]
for query in self.getQueryList():
if not(isinstance(query, basestring)):
key_list.extend(query.getSQLKeyList())
return key_list
assert len(self.query_list) == 1
self.query_list[0].setTableAliasList(table_alias_list)
@profiler_decorator
def setGroup(self, group):
for query in self.query_list:
query.setGroup(group)
verifyClass(IQuery, ComplexQuery)
allow_class(ComplexQuery)
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.ColumnMap import ColumnMap
from zLOG import LOG
from Products.ZSQLCatalog.Interface.IEntireQuery import IEntireQuery
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
class EntireQuery(object):
"""
This is not a Query subclass, since it does not define a
registerColumnMap method, and instead does the ColumnMap handling
internaly.
"""
__implements__ = IEntireQuery
column_map = None
@profiler_decorator
def __init__(self, query, order_by_list=None, group_by_list=None,
select_dict=None, limit=None, catalog_table_name=None,
extra_column_list=None, from_expression=None,
order_by_override_list=None):
def default(value):
if value is None:
return []
assert isinstance(value, (tuple, list))
return value
def defaultDict(value):
if value is None:
return {}
assert isinstance(value, dict)
return value
self.query = query
self.order_by_list = default(order_by_list)
self.order_by_override_set = frozenset(default(order_by_override_list))
self.group_by_list = default(group_by_list)
self.select_dict = defaultDict(select_dict)
self.limit = limit
self.catalog_table_name = catalog_table_name
self.extra_column_list = default(extra_column_list)
self.from_expression = from_expression
def asSearchTextExpression(self, sql_catalog):
return query.asSearchTextExpression(sql_catalog)
@profiler_decorator
def asSQLExpression(self, sql_catalog, only_group_columns):
column_map = self.column_map
if column_map is None:
# XXX: should we provide a way to register column map as a separate mathod or do it here ?
# Column Map was not built yet, do it.
self.column_map = column_map = ColumnMap(catalog_table_name=self.catalog_table_name)
for extra_column in self.extra_column_list:
table, column = extra_column.replace('`', '').split('.')
if table != self.catalog_table_name:
raise ValueError, 'Extra columns must be catalog columns. %r does not follow this rule (catalog=%r, extra_column_list=%r)' % (extra_column, self.catalog_table_name, self.extra_column_list)
column_map.registerColumn(extra_column)
for column in self.group_by_list:
column_map.registerColumn(column)
for alias, column in self.select_dict.iteritems():
if column is None:
column = alias
else:
column_map.ignoreColumn(alias)
column_map.registerColumn(column)
for override in self.order_by_override_set:
column_map.ignoreColumn(override)
for order_by in self.order_by_list:
assert isinstance(order_by, (tuple, list))
assert len(order_by)
column_map.registerColumn(order_by[0])
self.query.registerColumnMap(sql_catalog, column_map)
column_map.build(sql_catalog)
# Replace given group_by_list entries by their mapped representations.
new_column_list = []
append = new_column_list.append
for column in self.group_by_list:
try:
append(column_map.asSQLColumn(column))
except KeyError:
LOG('EntireQuery', 100, 'Group-by column %r could not be mapped, but is passed through. This use is strongly discouraged.' % (column, ))
append(column)
self.group_by_list = new_column_list
# Build a dictionnary from select_dict aliasing their mapped representations
self.final_select_dict = select_dict = {}
for alias, raw_column in self.select_dict.iteritems():
if raw_column is None:
column = alias
else:
column = raw_column
try:
rendered = column_map.asSQLColumn(column)
except KeyError:
LOG('EntireQuery', 100, 'Select column %r could not be mapped, but is passed through. This use is strongly discouraged.' % (column, ))
rendered = column
select_dict[alias] = rendered
# Replace given order_by_list entries by their mapped representations.
new_order_by_list = []
append = new_order_by_list.append
for order_by in self.order_by_list:
column = order_by[0]
if column in self.order_by_override_set:
LOG('EntireQuery', 100, 'Order-by column %r is forcibly accepted. This use is strongly discouraged.' % (column, ))
rendered = column
else:
try:
rendered = column_map.asSQLColumn(column)
except KeyError:
LOG('SQLCatalog', 100, 'Order by %r ignored: it could not be mapped to a known column.' % (order_by, ))
rendered = None
if rendered is not None:
if len(order_by) > 1:
if len(order_by) > 2 and order_by[2] not in (None, ''):
rendered = 'CAST(%s AS %s)' % (rendered, order_by[2])
rendered = '%s %s' % (rendered, order_by[1])
append(rendered)
self.order_by_list = new_order_by_list
# generate SQLExpression from query
sql_expression_list = [self.query.asSQLExpression(sql_catalog, column_map, only_group_columns)]
# generate join expression based on column_map.getJoinTableAliasList
append = sql_expression_list.append
for join_query in column_map.iterJoinQueryList():
append(join_query.asSQLExpression(sql_catalog, column_map, only_group_columns))
join_table_list = column_map.getJoinTableAliasList()
if len(join_table_list):
# XXX: Is there any special rule to observe when joining tables ?
# Maybe we could check which column is a primary key instead of
# hardcoding "uid".
where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
(column_map.getCatalogTableAlias(), )
# XXX: It would cleaner from completeness point of view to use column
# mapper to render column, but makes code much more complex to just do
# a simple text rendering. If there is any reason why we should have
# those column in the mapper, then we should use the clean way.
append(SQLExpression(self, where_expression=' AND '.join(
where_pattern % (x, ) for x in join_table_list
)))
self.sql_expression_list = sql_expression_list
return SQLExpression(
self,
table_alias_dict=column_map.getTableAliasDict(),
from_expression=self.from_expression,
order_by_list=self.order_by_list,
group_by_list=self.group_by_list,
select_dict=self.final_select_dict,
limit=self.limit,
where_expression_operator='and',
sql_expression_list=self.sql_expression_list)
verifyClass(IEntireQuery, EntireQuery)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -26,117 +28,35 @@
#
##############################################################################
from DocumentTemplate.DT_Var import sql_quote
from Products.ZSQLCatalog.Interface.IQuery import IQuery
from Interface.Verify import verifyClass
class QueryMixin:
class Query(object):
"""
Mixing class which implements methods which are
common to all kinds of Queries
This is the base class of all kind of queries. Its only purpose is to be
able to distinguish any kind of value from a query.
"""
operator = None
format = None
type = None
def __call__(self, **kw):
return self.asSQLExpression(**kw)
__implements__ = IQuery
__allow_access_to_unprotected_subobjects__ = 1
def getOperator(self):
return self.operator
def getFormat(self):
return self.format
def getType(self):
return self.type
def getRange(self):
return self.range
def getTableAliasList(self):
return self.table_alias_list
def getSearchMode(self):
"""Search mode used for Full Text search
def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
"""
return self.search_mode
def getSearchKey(self):
"""Search mode used for Full Text search
To enable SQL rendering, overload this method in a subclass.
"""
return self.search_key
def getKey(self):
return self.key
def getValue(self):
return self.value
raise TypeError, 'A %s cannot be rendered as an SQL expression.' % (self.__class__.__name__, )
def getOperator(self):
return self.operator.upper().strip()
def asSearchTextExpression(self):
raise NotImplementedError
def asSQLExpression(self, key_alias_dict=None,
keyword_search_keys=None,
datetime_search_keys=None,
full_text_search_keys=None,
ignore_empty_string=1, stat__=0):
def asSearchTextExpression(self, sql_catalog, column=None):
"""
Return a dictionnary containing the keys and value types:
'where_expression': string
'select_expression_list': string
To enable Search Text rendering, overload this method in a subclass.
"""
raise NotImplementedError
raise TypeError, 'A %s cannot be rendered as a SearchText expression.' % (self.__class__.__name__, )
def getSQLKeyList(self):
def registerColumnMap(self, sql_catalog, column_map):
"""
Return a list of keys used by this query and its subqueries.
This method must always be overloaded by subclasses.
"""
raise NotImplementedError
raise NotImplementedError, '%s is incompeltely implemented.' % (self.__class__.__name__, )
def getRelatedTableMapDict(self):
"""
Return for each key used by this query (plus ones used by its
subqueries) the table alias mapping.
"""
raise NotImplementedError
verifyClass(IQuery, Query)
def _quoteSQLString(self, value):
"""Return a quoted string of the value.
XXX: Left for backwards compatability!
"""
format = self.getFormat()
type = self.getType()
if format is not None and type is not None:
if type == 'date':
if hasattr(value, 'strftime'):
value = value.strftime(format)
if isinstance(value, basestring):
value = "STR_TO_DATE('%s','%s')" % (value, format)
if type == 'float':
# Make sure there is no space in float values
value = value.replace(' ','')
value = "'%s'" % value
else:
if getattr(value, 'ISO', None) is not None:
value = "'%s'" % value.toZone('UTC').ISO()
else:
value = "'%s'" % sql_quote(str(value))
return value
def _quoteSQLKey(self, key):
"""Return a quoted string of the value.
XXX: Left for backwards compatability!
"""
format = self.getFormat()
type = self.getType()
if format is not None and type is not None:
if type == 'date':
key = "STR_TO_DATE(DATE_FORMAT(%s,'%s'),'%s')" % (key, format, format)
if type == 'float':
float_format = format.replace(' ','')
if float_format.find('.') >= 0:
precision = len(float_format.split('.')[1])
key = "TRUNCATE(%s,%s)" % (key, precision)
return key
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Query import Query
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.Interface.IQuery import IQuery
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
class RelatedQuery(Query):
"""
A RelatedQuery represents the is a container for a join condition.
"""
@profiler_decorator
def __init__(self, search_key, join_condition=None, table_alias_list=None):
"""
search_key (SearchKey)
join_condition (Query)
If given, it will be registered and rendered by this query.
Deprecated
table_alias_list (list of 2-tuple of strings)
See setTableAliasList.
"""
self.search_key = search_key
self.join_condition = join_condition
self.table_alias_list = table_alias_list
@profiler_decorator
def setTableAliasList(self, table_alias_list):
"""
This function is here for backward compatibility.
table_alias_list (list of 2-tuples of strings)
Each 2-tuple contains the name of a related key parameter and the
table alias it must be mapped on (respectively).
"""
self.table_alias_list = table_alias_list
@profiler_decorator
def asSearchTextExpression(self, sql_catalog, column=None):
assert column is None
join_condition = self.join_condition
if join_condition is None:
result = None
else:
result = join_condition.asSearchTextExpression(sql_catalog, column=self.search_key.getColumn())
return result
@profiler_decorator
def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
sql_expression_list = [self.search_key.buildSQLExpression(sql_catalog, column_map, only_group_columns, self.group)]
join_condition = self.join_condition
if join_condition is not None:
sql_expression_list.append(join_condition.asSQLExpression(sql_catalog, column_map, only_group_columns))
return SQLExpression(self, sql_expression_list=sql_expression_list, where_expression_operator='and')
@profiler_decorator
def registerColumnMap(self, sql_catalog, column_map):
self.group = self.search_key.registerColumnMap(column_map, table_alias_list=self.table_alias_list)
join_condition = self.join_condition
if join_condition is not None:
# Update its group
join_condition.setGroup(self.group)
# Propagate registration to embeded query
join_condition.registerColumnMap(sql_catalog, column_map)
def __repr__(self):
return '<%s on %r with %r>' % (self.__class__.__name__, self.search_key.getColumn(), self.join_condition)
verifyClass(IQuery, RelatedQuery)
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Query import Query
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.Interface.IQuery import IQuery
from Interface.Verify import verifyClass
class SQLQuery(Query):
"""
This Query subclass is used to wrap raw SQL text.
Use of this class is strongly discouraged, and it is only here for
backward compatibility.
"""
def __init__(self, payload):
"""
payload (string)
Raw SQL text.
"""
if not isinstance(payload, basestring):
raise TypeError, 'Payload must be a string, got a %r: %r' % (type(payload), payload)
assert len(payload)
self.payload = '(' + payload + ')'
def asSearchText(self, sql_catalog):
return None
def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
return SQLExpression(self, where_expression=self.payload)
def registerColumnMap(self, sql_catalog, column_map):
"""
There is nothing to register for this type of Query subclass.
"""
pass
def __repr__(self):
return '<%s (%r)>' % (self.__class__.__name__, self.payload)
verifyClass(IQuery, SQLQuery)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -26,282 +28,104 @@
#
##############################################################################
from Products.PythonScripts.Utility import allow_class
from DateTime import DateTime
from Query import QueryMixin
from pprint import pprint
from Query import Query
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.Interface.IQuery import IQuery
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
# valid search modes for queries
FULL_TEXT_SEARCH_MODE = 'FullText'
EXACT_MATCH_SEARCH_MODE = 'ExactMatch'
KEYWORD_SEARCH_MODE = 'Keyword'
DATETIME_SEARCH_MODE = 'DateTime'
def isSimpleType(value):
return isinstance(value, basestring) or \
isinstance(value, int) or \
isinstance(value, long) or \
isinstance(value, float)
# XXX Bad name JPS - NotQuery or NegativeQuery is better NegationQuery
class NegatedQuery(QueryMixin):
class SimpleQuery(Query):
"""
Do a boolean negation of given query.
A SimpleQuery represents a single comparison between a single column and
one or more values.
"""
def __init__(self, query):
self._query = query
def asSQLExpression(self, *args, **kw):
sql_expression_dict = self._query.asSQLExpression(*args, **kw)
sql_expression_dict['where_expression'] = '(NOT (%s))' % \
(sql_expression_dict['where_expression'], )
return sql_expression_dict
def getSQLKeyList(self, *args, **kw):
return self._query.getSQLKeyList(*args, **kw)
def getRelatedTableMapDict(self, *args, **kw):
return self._query.getRelatedTableMapDict(*args, **kw)
allow_class(NegatedQuery)
class SimpleQuery(QueryMixin):
@profiler_decorator
def __init__(self, search_key=None, operator='=', group=None, **kw):
"""
This allow to define constraints on a sql column
format - type date : %d/%m/%Y
type float : 1 234.12
search_key (None, SearchKey instance)
If given, the instance of SearchKey which is responsible for column
map registration and rendering (SQL and SearchText).
operator (string)
The comparison operator which will be applied between column and
values.
See Operator/ComparisonOperator.py for possible values.
group
See ColumnMap.
**kw
Must contain exactly one item.
item key (string)
column name
item value
one or more values
"""
def __init__(self, format=None, operator=None, range=None, key=None,
search_mode=None, table_alias_list=None, type=None, **kw):
self.format = format
if operator is None:
operator = 'OR'
self.search_key = search_key
if len(kw) != 1:
raise ValueError, 'SimpleQuery can support one and one only column. Got %r.' % (kw, )
self.column, value = kw.popitem()
# Backward compatibility code (those changes should not be needed when
# this Query is instanciated by a SearchKey, as operator should be correct
# already).
operator = operator.lower()
if operator == 'in':
if isinstance(value, (list, tuple)):
if len(value) == 0:
raise ValueError, 'Empty lists are not allowed.'
elif len(value) == 1:
value = value[0]
operator = '='
else:
operator = '='
elif operator == '=':
if isinstance(value, (list, tuple)):
if len(value) == 0:
raise ValueError, 'Empty lists are not allowed.'
elif len(value) == 1:
value = value[0]
else:
operator = 'in'
self.value = value
self.operator = operator
self.range = range
self.search_mode = search_mode
self.table_alias_list = table_alias_list
key_list = kw.keys()
if len(key_list) != 1:
raise KeyError, 'Query must have only one key'
self.key = key_list[0]
self.value = kw[self.key]
self.type = type
self.search_key = key
def getRelatedTableMapDict(self):
result = {}
table_alias_list = self.getTableAliasList()
if table_alias_list is not None:
result[self.getKey()] = table_alias_list
return result
def getSQLKeyList(self):
"""
Returns the list of keys used by this
instance
"""
return [self.getKey()]
self.group = group
def asSearchTextExpression(self):
# This will be the standard way to represent
# complex values in listbox. Some fixed
# point must be garanteed
value = self.getValue()
if isSimpleType(value) or isinstance(value, DateTime):
return str(value)
elif isinstance(value, (list, tuple)):
value = map(lambda x:str(x), value)
return (' %s ' % self.operator).join(value)
@profiler_decorator
def asSearchTextExpression(self, sql_catalog, column=None):
return self.getSearchKey(sql_catalog).buildSearchTextExpression(self.getOperator(sql_catalog), self.getValue(), column=column)
def _getSearchKeyClassByType(self, type, search_key_class = None):
""" Return search key class based on type of value. """
name_search_key_map = {'keyword': KeyWordKey,
'default': DefaultKey,
'fulltext': FullTextKey,
'date': DateTimeKey,
'float': FloatKey,
'int': DefaultKey,}
return name_search_key_map.get(type, search_key_class)
def _getSearchKeyClassByValue(self, value, search_key_class = None):
""" Return search key class based on type of value. """
if isinstance(value, basestring):
if value.find('%')!=-1:
# it's likely a KeyWordKey
search_key_class = KeyWordKey
else:
search_key_class = DefaultKey
elif isinstance(value, DateTime):
search_key_class = DateTimeKey
elif isinstance(value, (int, long,)):
search_key_class = DefaultKey
elif isinstance(value, float):
search_key_class = FloatKey
elif value is None:
return RawKey
return search_key_class
@profiler_decorator
def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
return self.getSearchKey(sql_catalog).buildSQLExpression(
self.getOperator(sql_catalog), self.getValue(),
column_map, only_group_columns, group=self.group)
def _asSQLExpression(self, search_key_class, key, value, format=None, mode=None, range_value=None, stat__=None):
""" Generate SQL expressions based on respective search_key passed. """
lexer = getSearchKeyInstance(search_key_class)
where_expression, select_expression_list = \
lexer.buildSQLExpression(key, value, format, mode, range_value, stat__)
sql_expressions = {'where_expression': where_expression,
'select_expression_list': select_expression_list,}
return sql_expressions
@profiler_decorator
def registerColumnMap(self, sql_catalog, column_map):
self.group = self.getSearchKey(sql_catalog).registerColumnMap(column_map, group=self.group, simple_query=self)
def asSQLExpression(self, key_alias_dict=None, keyword_search_keys=None,
datetime_search_keys=None, full_text_search_keys=None,
ignore_empty_string=1, stat__=0):
def getOperator(self, sql_catalog):
"""
Build the sql expressions string
Return an instance of OperatorBase class.
"""
search_key_class = None
value = self.getValue()
key = self.getKey()
operator = self.getOperator()
type = self.getType()
format = self.getFormat()
search_mode = self.getSearchMode()
range_value = self.getRange()
search_key = self.getSearchKey()
if keyword_search_keys is None:
keyword_search_keys = []
if datetime_search_keys is None:
datetime_search_keys = []
if full_text_search_keys is None:
full_text_search_keys = []
# key can have an alias definition which we should acquire
if key_alias_dict is not None:
key = key_alias_dict.get(key, None)
search_key_class = None
where_expression_list = []
select_expression_list = []
sql_expressions = {'where_expression': '1',
'select_expression_list': []}
# try to get search key type by the key definitions passed
if search_key_class is None:
if search_key == EXACT_MATCH_SEARCH_MODE:
search_key_class = RawKey
elif search_key == KEYWORD_SEARCH_MODE or \
(key in keyword_search_keys):
search_key_class = KeyWordKey
elif search_key == DATETIME_SEARCH_MODE or \
(key in datetime_search_keys):
search_key_class = DateTimeKey
elif search_key == FULL_TEXT_SEARCH_MODE or \
(key in full_text_search_keys):
search_key_class = FullTextKey
return sql_catalog.getComparisonOperator(self.operator)
# get search class based on explicitly passed key type
if search_key_class is None:
search_key_class = self._getSearchKeyClassByType(type)
# some use cases where we can just return SQL without grammar staff
if key is None or (ignore_empty_string and \
isinstance(value, basestring) and \
value.strip() == ''):
# do not further generate sql expressions because
# we ignore empty strings by default
return sql_expressions
elif (not ignore_empty_string) \
and isinstance(value, basestring) \
and value.strip() == '':
# explicitly requested not to ignore empty strings
sql_expressions = {'where_expression': "%s = ''" % key,
'select_expression_list': []}
return sql_expressions
else:
# search for 'NULL' values
if value is None:
sql_expressions = {'where_expression': "%s is NULL" % (key),
'select_expression_list': [],}
return sql_expressions
def getSearchKey(self, sql_catalog):
"""
Return an instance of SearchKey class.
"""
if self.search_key is None:
self.search_key = sql_catalog.getSearchKey(self.getColumn())
return self.search_key
# we have a list of values and respective operator defined
if isinstance(value, (tuple, list)):
if range_value is None:
# use operators to build sql expressions
if operator in ('IN',):
# values in list are not treated as searchable strings but
# they should be SQL quoted at least
if len(value) > 1:
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
search_key_instance = getSearchKeyInstance(search_key_class)
escaped_value_list = [search_key_instance.quoteSQLString(x, format) for x in value]
escaped_value_string = ', '.join(escaped_value_list)
where_expression_list.append("%s IN (%s)" % (key, escaped_value_string))
elif len(value) == 1:
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
search_key_instance = getSearchKeyInstance(search_key_class)
where_expression_list.append("%s = %s"
%(key, search_key_instance.quoteSQLString(value[0], format)))
else:
# empty list
where_expression_list.append("0")
elif operator in ('OR', 'AND',):
# each of the list elements can be treated as a Key, so
# leave SQL generation to Key itself
if len(value) > 1:
sql_logical_sub_expressions = []
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
for item in value:
list_item_sql_expressions = self._asSQLExpression(search_key_class, key, \
item, format, search_mode, range_value, stat__)
sql_logical_sub_expressions.append('%s' %list_item_sql_expressions['where_expression'])
# join list items (now sql logical expressions) using respective operator
where_expression = (' %s ' %operator).join(sql_logical_sub_expressions)
where_expression_list.append("(%s)" % (where_expression))
elif len(value) == 1:
if search_key_class is None:
# no explicitly defined, try to find by value
search_key_class = self._getSearchKeyClassByValue(value[0])
item_sql_expressions = self._asSQLExpression(search_key_class, key, \
value[0], format, search_mode, range_value, stat__)
where_expression_list.append(item_sql_expressions['where_expression'])
# join where expressions list
where_expression = ' '.join(where_expression_list)
sql_expressions = {'where_expression': where_expression,
'select_expression_list': [],}
return sql_expressions
else:
# we can have range specified
if search_key_class is None:
# try to guess by type of first_element in list
search_key_class = self._getSearchKeyClassByValue(value[0])
def getColumn(self):
return self.column
# get search class based on value of value
if search_key_class is None:
search_key_class = self._getSearchKeyClassByValue(value)
def getValue(self):
return self.value
# last fallback case
if search_key_class is None:
search_key_class = DefaultKey
def __repr__(self):
return '<%s %r %s %r>' % (self.__class__.__name__, self.getColumn(), self.operator, self.getValue())
# use respective search key.
sql_expressions = self._asSQLExpression(search_key_class, key,
value, format, search_mode, range_value, stat__)
return sql_expressions
def setGroup(self, group):
self.group = group
allow_class(SimpleQuery)
verifyClass(IQuery, SimpleQuery)
from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
##############################################################################
#
# Copyright (c) 2002 Nexedi SARL. All Rights Reserved.
# Copyright (c) 2002-2009 Nexedi SARL. All Rights Reserved.
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
......@@ -45,7 +45,16 @@ from xml.dom.minidom import parse
from xml.sax.saxutils import escape, quoteattr
import os
import md5
from sets import ImmutableSet
from Interface.IQueryCatalog import ISearchKeyCatalog
from Interface.Verify import verifyClass
PROFILING_ENABLED = False
if PROFILING_ENABLED:
from tiny_profiler import profiler_decorator, profiler_report, profiler_reset
else:
def profiler_decorator(func):
return func
try:
from Products.CMFCore.Expression import Expression
......@@ -64,6 +73,7 @@ try:
from Products.ERP5Type.Cache import enableReadOnlyTransactionCache
from Products.ERP5Type.Cache import disableReadOnlyTransactionCache, CachingMethod
except ImportError:
LOG('SQLCatalog', 100, 'Count not import CachingMethod, expect slowness.')
def doNothing(context):
pass
class CachingMethod:
......@@ -77,6 +87,44 @@ except ImportError:
enableReadOnlyTransactionCache = doNothing
disableReadOnlyTransactionCache = doNothing
class caching_class_method_decorator:
def __init__(self, *args, **kw):
self.args = args
self.kw = kw
def __call__(self, method):
caching_method = CachingMethod(method, *self.args, **self.kw)
return lambda *args, **kw: caching_method(*args, **kw)
#def wrapper(wrapped_self):
# LOG('caching_class_method_decorator', 0, 'lookup')
# return caching_method(wrapped_self)
#return wrapper
try:
from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
except ImportError:
LOG('SQLCatalog', 100, 'Count not import getTransactionalVariable, expect slowness.')
def getTransactionalVariable(context):
return {}
class transactional_cache_decorator:
"""
Implements singleton-style caching.
Wrapped method must have no parameters (besides "self").
"""
def __init__(self, cache_id):
self.cache_id = cache_id
def __call__(self, method):
def wrapper(wrapped_self):
transactional_cache = getTransactionalVariable(None)
try:
return transactional_cache[self.cache_id]
except KeyError:
result = transactional_cache[self.cache_id] = method(wrapped_self)
return result
return wrapper
try:
from ZPublisher.HTTPRequest import record
except ImportError:
......@@ -184,6 +232,10 @@ class UidBuffer(TM):
tid = get_ident()
self.temporary_buffer.setdefault(tid, []).extend(iterable)
DEBUG = False
related_key_definition_cache = {}
class Catalog(Folder,
Persistent,
Acquisition.Implicit,
......@@ -219,6 +271,9 @@ class Catalog(Folder,
- optmization: indexing objects should be deferred
until timeout value or end of transaction
"""
__implements__ = ISearchKeyCatalog
meta_type = "SQLCatalog"
icon = 'misc_/ZCatalog/ZCatalog.gif' # FIXME: use a different icon
security = ClassSecurityInfo()
......@@ -463,6 +518,13 @@ class Catalog(Folder,
'a monovalued local role',
'type': 'lines',
'mode': 'w' },
{ 'id': 'sql_catalog_table_vote_scripts',
'title': 'Table vote scripts',
'description': 'Scripts helping column mapping resolution',
'type': 'multiple selection',
'select_variable' : 'getPythonMethodIds',
'mode': 'w' },
)
sql_catalog_produce_reserved = ''
......@@ -500,6 +562,7 @@ class Catalog(Folder,
sql_catalog_scriptable_keys = ()
sql_catalog_role_keys = ()
sql_catalog_local_role_keys = ()
sql_catalog_table_vote_scripts = ()
# These are ZODB variables, so shared by multiple Zope instances.
# This is set to the last logical time when clearReserved is called.
......@@ -887,26 +950,24 @@ class Catalog(Folder,
keys = keys.keys()
keys.sort()
return keys
return CachingMethod(_getColumnIds, id='SQLCatalog.getColumnIds', cache_factory='erp5_content_long')()
return CachingMethod(_getColumnIds, id='SQLCatalog.getColumnIds', cache_factory='erp5_content_long')()[:]
@profiler_decorator
@transactional_cache_decorator('SQLCatalog.getColumnMap')
@profiler_decorator
@caching_class_method_decorator(id='SQLCatalog.getColumnMap', cache_factory='erp5_content_long')
@profiler_decorator
def getColumnMap(self):
"""
Calls the show column method and returns dictionnary of
Field Ids
"""
def _getColumnMap():
keys = {}
result = {}
for table in self.getCatalogSearchTableIds():
field_list = self._getCatalogSchema(table=table)
for field in field_list:
key = field
if not keys.has_key(key): keys[key] = []
keys[key].append(table)
key = '%s.%s' % (table, key)
if not keys.has_key(key): keys[key] = []
keys[key].append(table) # Is this inconsistent ?
return keys
return CachingMethod(_getColumnMap, id='SQLCatalog.getColumnMap', cache_factory='erp5_content_long')()
for field in self._getCatalogSchema(table=table):
result.setdefault(field, []).append(table)
result.setdefault('%s.%s' % (table, field), []).append(table) # Is this inconsistent ?
return result
def getResultColumnIds(self):
"""
......@@ -1332,7 +1393,7 @@ class Catalog(Folder,
if self.isMethodFiltered(method_name):
catalogged_object_list = []
filter = self.filter_dict[method_name]
type_set = ImmutableSet(filter['type']) or None
type_set = frozenset(filter['type']) or None
expression = filter['expression_instance']
expression_cache_key_list = filter.get('expression_cache_key', '').split()
for object in object_list:
......@@ -1633,7 +1694,8 @@ class Catalog(Folder,
""" Accesses a single record for a given path """
return self.getMetadataForPath(path)
def getCatalogMethodIds(self):
def getCatalogMethodIds(self,
valid_method_meta_type_list=valid_method_meta_type_list):
"""Find Z SQL methods in the current folder and above
This function return a list of ids.
"""
......@@ -1658,6 +1720,26 @@ class Catalog(Folder,
ids.sort()
return ids
def getPythonMethodIds(self):
"""
Returns a list of all python scripts available in
current sql catalog.
"""
return self.getCatalogMethodIds(valid_method_meta_type_list=('Script (Python)', ))
@profiler_decorator
@transactional_cache_decorator('SQLCatalog._getSQLCatalogRelatedKeyList')
@profiler_decorator
def _getSQLCatalogRelatedKeySet(self):
column_map = self.getColumnMap()
column_set = set(column_map)
for related_key in self.sql_catalog_related_keys:
related_key_id = related_key.split(' | ')[0].strip()
if related_key_id in column_set:
LOG('SQLCatalog', 100, 'Related key %r has the same name as an existing column on tables %r' % (related_key_id, column_map[related_key_id]))
column_set.add(related_key_id)
return column_set
def getSQLCatalogRelatedKeyList(self, key_list=None):
"""
Return the list of related keys.
......@@ -1666,10 +1748,9 @@ class Catalog(Folder,
"""
if key_list is None:
key_list = []
column_map = self._getSQLCatalogRelatedKeySet()
# Do not generate dynamic related key for acceptable_keys
dynamic_key_list = [k for k in key_list \
if k not in self.getColumnMap().keys()]
dynamic_key_list = [k for k in key_list if k not in column_map]
dynamic_list = self.getDynamicRelatedKeyList(dynamic_key_list)
full_list = list(dynamic_list) + list(self.sql_catalog_related_keys)
return full_list
......@@ -1702,486 +1783,412 @@ class Catalog(Folder,
%(table_index, table))
return table_index
return CachingMethod(_getTableIndex, id='SQLCatalog.getTableIndex', \
cache_factory='erp5_content_long')(table=table)
def getIndex(self, table, column_list, all_column_list):
"""
Return possible index for a column list in a given table
"""
def _getIndex(table, column_list, all_column_list):
index_dict = self.getTableIndex(table)
if isinstance(column_list, str):
column_list = [column_list,]
# Get possible that can be used
possible_index = []
for index in index_dict.keys():
index_columns = index_dict[index]
for column in index_columns:
if column in column_list:
if index not in possible_index:
possible_index.append(index)
if len(possible_index) == 0:
return []
# Get the most suitable index
for index in possible_index:
# Make sure all column in index are used by the query
index_column = index_dict[index]
for column in index_column:
if column in column_list or column in all_column_list:
continue
cache_factory='erp5_content_long')(table=table).copy()
@profiler_decorator
def getRelatedKeyDefinition(self, key):
"""
Returns the definition of given related key name if found, None
otherwise.
"""
try:
result = related_key_definition_cache[key]
except KeyError:
result = None
for entire_definition in self.getSQLCatalogRelatedKeyList([key]):
name, definition = entire_definition.split(' | ')
if name == key:
result = definition
break
if result is not None:
related_key_definition_cache[key] = result
return result
@profiler_decorator
def getColumnSearchKey(self, key, search_key_name=None):
"""
Return a SearchKey instance for given key, using search_key_name
as a SearchKey name if given, otherwise guessing from catalog
configuration. If there is no search_key_name given and no
SearchKey can be found, return None.
Also return a related key definition string with following rules:
- If returned SearchKey is a RelatedKey, value is its definition
- Otherwise, value is None
"""
# Is key a "real" column or some related key ?
related_key_definition = None
if key in self.getColumnMap():
search_key = self.getSearchKey(key, search_key_name)
else:
possible_index.remove(index)
LOG("SQLCatalog.getIndex", INFO, "index = %s for table %s and columns %s" \
%(possible_index, table, column_list))
return possible_index
return CachingMethod(_getIndex, id='SQLCatalog.getIndex', cache_factory='erp5_content_long')\
(table=table, column_list=column_list, all_column_list=all_column_list)
# Maybe a related key...
related_key_definition = self.getRelatedKeyDefinition(key)
if related_key_definition is None:
# Unknown
search_key = None
else:
# It's a related key
search_key = self.getSearchKey(key, 'RelatedKey')
return search_key, related_key_definition
@profiler_decorator
def getColumnDefaultSearchKey(self, key):
"""
Return a SearchKey instance which would ultimately receive the value
associated with given key.
"""
search_key, related_key_definition = self.getColumnSearchKey(key)
if search_key is None:
result = None
else:
if related_key_definition is not None:
search_key = search_key.getSearchKey(self, related_key_definition)
return search_key
def buildSQLQuery(self, query_table='catalog', REQUEST=None,
ignore_empty_string=1, query=None, stat__=0, **kw):
""" Builds a complex SQL query to simulate ZCatalog behaviour """
# Get search arguments:
if REQUEST is None and (kw is None or kw == {}):
# We try to get the REQUEST parameter
# since we have nothing handy
try: REQUEST=self.REQUEST
except AttributeError: pass
#LOG('SQLCatalog.buildSQLQuery, kw',0,kw)
# If kw and query are not set, then use REQUEST instead
if query is None and (kw is None or kw == {}):
kw = REQUEST
acceptable_key_map = self.getColumnMap()
full_text_search_keys = list(self.sql_catalog_full_text_search_keys)
keyword_search_keys = list(self.sql_catalog_keyword_search_keys)
datetime_search_keys = list(self.sql_catalog_datetime_search_keys)
topic_search_keys = self.sql_catalog_topic_search_keys
multivalue_keys = self.sql_catalog_multivalue_keys
# Compute "sort_index", which is a sort index, or none:
if kw.has_key('sort-on'):
sort_index=kw['sort-on']
elif hasattr(self, 'sort-on'):
sort_index=getattr(self, 'sort-on')
elif kw.has_key('sort_on'):
sort_index=kw['sort_on']
else: sort_index=None
# Compute the sort order
if kw.has_key('sort-order'):
so=kw['sort-order']
elif hasattr(self, 'sort-order'):
so=getattr(self, 'sort-order')
elif kw.has_key('sort_order'):
so=kw['sort_order']
else: so=None
# We must now turn sort_index into
# a dict with keys as sort keys and values as sort order
if isinstance(sort_index, basestring):
sort_index = [(sort_index, so)]
elif not isinstance(sort_index, (list, tuple)):
sort_index = None
# Rebuild keywords to behave as new style query (_usage='toto:titi' becomes {'toto':'titi'})
new_kw = {}
usage_len = len('_usage')
for k, v in kw.items():
if k.endswith('_usage'):
new_k = k[0:-usage_len]
if not new_kw.has_key(new_k):
new_kw[new_k] = {}
if not isinstance(new_kw[new_k], dict_type_list):
new_kw[new_k] = {'query': new_kw[new_k]}
split_v = v.split(':')
new_kw[new_k] = {split_v[0]: split_v[1]}
@profiler_decorator
def buildSingleQuery(self, key, value, search_key_name=None, logical_operator=None, comparison_operator=None):
"""
From key and value, determine the SearchKey to use and generate a Query
from it.
"""
search_key, related_key_definition = self.getColumnSearchKey(key, search_key_name)
if search_key is None:
result = None
else:
if not new_kw.has_key(k):
new_kw[k] = v
if related_key_definition is None:
result = search_key.buildQuery(value, logical_operator=logical_operator, comparison_operator=comparison_operator)
else:
new_kw[k]['query'] = v
kw = new_kw
result = search_key.buildQuery(search_value=value, sql_catalog=self, search_key_name=search_key_name, related_key_definition=related_key_definition, logical_operator=logical_operator, comparison_operator=comparison_operator)
return result
@profiler_decorator
def buildQueryFromAbstractSyntaxTreeNode(self, node, key):
"""
Build a query from given Abstract Syntax Tree (AST) node by recursing in
its childs.
This method calls itself recursively when walking the tree.
node
AST node being treated.
key
Default column (used when there is no explicit column in an AST leaf).
# Initialise Scriptable Dict
scriptable_key_dict = {}
for t in self.sql_catalog_scriptable_keys:
t = t.split('|')
key = t[0].strip()
if len(t)>1:
# method defined that will generate a ComplexQuery
method_id = t[1].strip()
Expected node API is described in Interface/IAbstractSyntaxNode.py .
"""
if node.isLeaf():
result = self.buildSingleQuery(key, node.getValue(), comparison_operator=node.getComparisonOperator())
if result is None:
# Unknown, skip loudly
LOG('SQLCatalog', 100, 'Unknown column %r, skipped.' % (key, ))
elif node.isColumn():
result = self.buildQueryFromAbstractSyntaxTreeNode(node.getSubNode(), node.getColumnName())
else:
query_list = []
value_dict = {}
append = query_list.append
for subnode in node.getNodeList():
if subnode.isLeaf():
value_dict.setdefault(subnode.getComparisonOperator(), []).append(subnode.getValue())
else:
# no method define, let ScriptableKey generate a ComplexQuery
method_id = None
scriptable_key_dict[key] = method_id
# Build the list of Queries and ComplexQueries
query_dict = {}
key_list = [] # the list of column keys
key_alias_dict = {}
query_group_by_list = None # Useful to keep a default group_by passed by scriptable keys
query_related_table_map_dict = {}
if query is not None:
kw ['query'] = query
for key in kw.keys():
if key not in RESERVED_KEY_LIST:
value = kw[key]
current_query = None
new_query_dict = {}
if isinstance(value, (Query, ComplexQuery)):
current_query = value
elif scriptable_key_dict.has_key(key):
if scriptable_key_dict[key] is not None:
# Turn this key into a query by invoking a script
method = getattr(self, scriptable_key_dict[key])
current_query = method(value) # May return None
subquery = self.buildQueryFromAbstractSyntaxTreeNode(subnode, key)
if subquery is not None:
append(subquery)
for comparison_operator, value_list in value_dict.iteritems():
subquery = self.buildSingleQuery(key, value_list, comparison_operator=comparison_operator)
if subquery is None:
LOG('SQLCatalog', 100, 'Unknown column %r, skipped.' % (key, ))
else:
# let default implementation of ScriptableKey generate ComplexQuery
search_key_instance = getSearchKeyInstance(ScriptableKey)
current_query = search_key_instance.buildQuery('', value)
if hasattr(current_query, 'order_by'): query_group_by_list = current_query.order_by
append(subquery)
operator = node.getLogicalOperator()
if operator == 'not' or len(query_list) > 1:
result = ComplexQuery(query_list, operator=operator)
elif len(query_list) == 1:
result = query_list[0]
else:
result = None
return result
@profiler_decorator
def buildQuery(self, kw, ignore_empty_string=True, operator='and'):
query_list = []
append = query_list.append
# unknown_column_dict: contains all (key, value) pairs which could not be
# changed into queries. This is here for backward compatibility, because
# scripts can invoke this method and expect extra parameters (such as
# from_expression) to be handled. As they are normaly handled at
# buildSQLQuery level, we must store them into final ComplexQuery, which
# will handle them.
unknown_column_dict = {}
# implicit_table_list: contains all tables explicitely given as par of
# column names with empty values. This is for backward compatibility. See
# comment about empty values.
implicit_table_list = []
for key, value in kw.iteritems():
result = None
if isinstance(value, dict_type_list):
new_query_dict = value.copy()
if 'query' in new_query_dict:
new_query_dict[key] = new_query_dict.pop('query')
# Cast dict-ish types into plain dicts.
value = dict(value)
if ignore_empty_string and (
value == ''
or (isinstance(value, (list, tuple)) and len(value) == 0)
or (isinstance(value, dict) and (
'query' not in value
or value['query'] == ''
or (isinstance(value['query'], (list, tuple))
and len(value['query']) == 0)))):
# We have an empty value:
# - do not create a query from it
# - if key has a dot, add its left part to the list of "hint" tables
# This is for backward compatibility, when giving a mapped column
# with an empty value caused a join with catalog to appear in
# resulting where-expression)
if '.' in key:
implicit_table_list.append(key)
LOG('buildQuery', WARNING, 'Discarding empty value for key %r: %r' % (key, value))
else:
new_query_dict[key] = value
current_query = Query(**new_query_dict)
if current_query is not None:
query_dict[key] = current_query
key_list.extend(current_query.getSQLKeyList())
query_related_table_map_dict.update(current_query.getRelatedTableMapDict())
# if we have a sort index, we must take it into account to get related
# keys.
sort_key_dict = dict()
if sort_index:
for sort_info in sort_index:
sort_key = sort_info[0]
if sort_key not in key_list:
key_list.append(sort_key)
sort_key_dict[sort_key] = 1
related_tuples = self.getSQLCatalogRelatedKeyList(key_list=key_list)
# Define related maps
# each tuple from `related_tuples` has the form (key,
# 'table1,table2,table3/column/where_expression')
related_keys = {}
related_method = {}
related_table_map = {}
related_column = {}
related_table_list = {}
table_rename_index = 0
related_methods = {} # related methods which need to be used
for t in related_tuples:
t_tuple = t.split('|')
key = t_tuple[0].strip()
if key in key_list:
if ignore_empty_string \
and kw.get(key, None) in ('', [], ())\
and key not in sort_key_dict:
# We don't ignore 0 and None, but if the key is used for sorting,
# we should not discard this key
continue
join_tuple = t_tuple[1].strip().split('/')
related_keys[key] = None
method_id = join_tuple[2]
table_list = tuple(join_tuple[0].split(','))
related_method[key] = method_id
related_table_list[key] = table_list
related_column[key] = join_tuple[1]
# Check if some aliases where specified in queries
map_list = query_related_table_map_dict.get(key,None)
# Rename tables to prevent conflicts
if not related_table_map.has_key((table_list,method_id)):
if map_list is None:
map_list = []
for table_id in table_list:
map_list.append((table_id,
"related_%s_%s" % (table_id, table_rename_index))) # We add an index in order to alias tables in the join
table_rename_index += 1 # and prevent name conflicts
related_table_map[(table_list,method_id)] = map_list
# We take additional parameters from the REQUEST
# and give priority to the REQUEST
if REQUEST is not None:
for key in acceptable_key_map.iterkeys():
if REQUEST.has_key(key):
# Only copy a few keys from the REQUEST
if key in self.sql_catalog_request_keys:
kw[key] = REQUEST[key]
def getNewKeyAndUpdateVariables(key):
new_key = None
if query_table:
key_is_acceptable = key in acceptable_key_map # Only calculate once
key_is_related = key in related_keys
if key_is_acceptable or key_is_related:
if key_is_related: # relation system has priority (ex. security_uid)
# We must rename the key
method_id = related_method[key]
table_list = related_table_list[key]
if not related_methods.has_key((table_list,method_id)):
related_methods[(table_list,method_id)] = 1
# Prepend renamed table name
new_key = "%s.%s" % (related_table_map[(table_list,method_id)][-1][-1],
related_column[key])
elif key_is_acceptable:
if key.find('.') < 0:
# if the key is only used by one table, just append its name
if len(acceptable_key_map[key]) == 1 :
new_key = '%s.%s' % (acceptable_key_map[key][0], key)
# query_table specifies what table name should be used by default
elif '%s.%s' % (query_table, key) in acceptable_key_map:
new_key = '%s.%s' % (query_table, key)
elif key == 'uid':
# uid is always ambiguous so we can only change it here
new_key = 'catalog.uid'
if isinstance(value, _Query):
# Query instance: use as such, ignore key.
result = value
elif isinstance(value, basestring):
# String: parse using key's default search key.
search_key = self.getColumnDefaultSearchKey(key)
if search_key is not None:
abstract_syntax_tree = search_key.parseSearchText(value)
if abstract_syntax_tree is None:
# Parsing failed, create a query from the bare string.
result = self.buildSingleQuery(key, value)
else:
LOG('SQLCatalog', WARNING, 'buildSQLQuery this key is too ambiguous : %s' % key)
if DEBUG:
LOG('SQLCatalog', 0, 'Building queries from abstract syntax tree: %r' % (abstract_syntax_tree, ))
result = self.buildQueryFromAbstractSyntaxTreeNode(abstract_syntax_tree, key)
elif isinstance(value, dict):
# Dictionnary: might contain the search key to use.
search_key_name = value.get('key')
# Backward compatibility: former "Keyword" key is now named
# "KeywordKey".
if search_key_name == 'Keyword':
search_key_name = value['key'] = 'KeywordKey'
result = self.buildSingleQuery(key, value, search_key_name)
else:
new_key = key
if new_key is not None:
# Add table to table dict, we use catalog by default
from_table_dict[acceptable_key_map[new_key][0]] = acceptable_key_map[new_key][0]
# Any other type, just create a query. (can be a DateTime, ...)
result = self.buildSingleQuery(key, value)
if result is None:
# No query could be created, emit a log, add to unknown column dict.
unknown_column_dict[key] = value
else:
new_key = key
key_alias_dict[key] = new_key
return new_key
where_expression_list = []
select_expression_list = []
group_by_expression_list = []
where_expression = ''
select_expression = ''
group_by_expression = ''
select_expression_key = ''
from_table_dict = {'catalog' : 'catalog'} # Always include catalog table
if len(kw):
if kw.has_key('select_expression_key'):
select_expression_key = kw['select_expression_key']
if type(select_expression_key) is type('a'):
select_expression_key = [select_expression_key]
if kw.has_key('select_expression'):
select_expression_list.append(kw['select_expression'])
if kw.has_key('group_by_expression'):
group_by_expression_list.append(kw['group_by_expression'])
# Grouping
group_by_list = kw.get('group_by', query_group_by_list)
if type(group_by_list) is type('a'): group_by_list = [group_by_list]
if group_by_list is not None:
try:
for key in group_by_list:
new_key = getNewKeyAndUpdateVariables(key)
group_by_expression_list.append(new_key)
except ConflictError:
raise
except:
LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build the new group by expression', error=sys.exc_info())
group_by_expression = ''
if len(group_by_expression_list)>0:
group_by_expression = ','.join(group_by_expression_list)
group_by_expression = str(group_by_expression)
sort_on = None
sort_key_list = []
if sort_index is not None:
new_sort_index = []
for sort in sort_index:
if len(sort) == 2:
# Try to analyse expressions of the form "title AS unsigned"
sort_key_list = sort[0].split()
if len(sort_key_list) == 3:
sort_key = sort_key_list[0]
sort_type = sort_key_list[2]
elif len(sort_key_list):
sort_key = sort_key_list[0]
sort_type = None
append(result)
if len(unknown_column_dict):
LOG('SQLCatalog', 100, 'Unknown columns %r, skipped.' % (unknown_column_dict.keys(), ))
return ComplexQuery(query_list, operator=operator, unknown_column_dict=unknown_column_dict, implicit_table_list=implicit_table_list)
@profiler_decorator
def buildOrderByList(self, sort_on=None, sort_order=None, order_by_expression=None):
"""
Internal method. Should not be used by code outside buildSQLQuery.
It is in a separate method because this code is here to keep backward
compatibility with an ambiguous API, and as such is ugly. So it's better
to conceal it to its own method.
It does not preserve backward compatibility for:
'sort-on' parameter
'sort-on' property
'sort-order' parameter
'sort-order' property
"""
order_by_list = []
append = order_by_list.append
if sort_on is not None:
if order_by_expression is not None:
LOG('SQLCatalog', 0, 'order_by_expression (%r) and sort_on (%r) were given. Ignoring order_by_expression.' % (order_by_expression, sort_on))
if not isinstance(sort_on, (tuple, list)):
sort_on = [[sort_on]]
for item in sort_on:
if isinstance(item, (tuple, list)):
item = list(item)
else:
sort_key = sort[0]
sort_type = None
new_sort_index.append((sort_key, sort[1], sort_type))
elif len(sort) == 3:
new_sort_index.append(sort)
sort_index = new_sort_index
try:
new_sort_index = []
for (original_key, so, as_type) in sort_index:
key = getNewKeyAndUpdateVariables(original_key)
if key is None:
if original_key in select_expression_key:
key = original_key
if key is not None:
sort_key_list.append(key)
if as_type == 'int':
key = 'CAST(%s AS SIGNED)' % key
elif as_type:
key = 'CAST(%s AS %s)' % (key, as_type) # Different casts are possible
if so in ('descending', 'reverse', 'DESC'):
new_sort_index.append('%s DESC' % key)
item = [item]
if sort_order is not None and len(item) == 1:
item.append(sort_order)
if len(item) > 1:
if item[1] in ('descending', 'reverse', 'DESC'):
item[1] = 'DESC'
else:
new_sort_index.append('%s' % key)
item[1] = 'ASC'
if len(item) > 2:
if item[2] == 'int':
item[2] = 'SIGNED'
append(item)
elif order_by_expression is not None:
if not isinstance(order_by_expression, basestring):
raise TypeError, 'order_by_expression must be a basestring instance. Got %r.' % (order_by_expression, )
order_by_list = [[x.strip()] for x in order_by_expression.split(',')]
return order_by_list
@profiler_decorator
def buildSQLQuery(self, query_table='catalog', REQUEST=None,
ignore_empty_string=1, only_group_columns=False,
limit=None, extra_column_list=None,
**kw):
# from traceback import format_list, extract_stack
# LOG('buildSQLQuery', 0, ''.join(format_list(extract_stack())))
if DEBUG:
LOG('buildSQLQuery', 0, repr(kw))
group_by_list = kw.pop('group_by_list', kw.pop('group_by', kw.pop('group_by_expression', None)))
if isinstance(group_by_list, basestring):
group_by_list = [x.strip() for x in group_by_list.split(',')]
select_dict = kw.pop('select_dict', kw.pop('select_list', kw.pop('select_expression', None)))
if isinstance(select_dict, basestring):
if len(select_dict):
real_select_dict = {}
for column in select_dict.split(','):
index = column.lower().find(' as ')
if index != -1:
real_select_dict[column[index + 4:].strip()] = column[:index].strip()
else:
LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build sort '
'index (%s -> %s)' % (original_key, key))
sort_index = join(new_sort_index,',')
sort_on = str(sort_index)
except ConflictError:
raise
except:
LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build the new sort index', error=sys.exc_info())
sort_on = ''
sort_key_list = []
for key in key_list:
if not key_alias_dict.has_key(key):
getNewKeyAndUpdateVariables(key)
if len(query_dict):
for key, query in query_dict.items():
query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
full_text_search_keys=full_text_search_keys,
keyword_search_keys=keyword_search_keys,
datetime_search_keys=datetime_search_keys,
ignore_empty_string=ignore_empty_string,
stat__=stat__)
if query_result['where_expression'] not in ('',None):
where_expression_list.append(query_result['where_expression'])
select_expression_list.extend(query_result['select_expression_list'])
# Calculate extra where_expression based on required joins
if query_table:
for k, tid in from_table_dict.items():
if k != query_table:
where_expression_list.append('%s.uid = %s.uid' % (query_table, tid))
# Calculate extra where_expressions based on related definition
for (table_list, method_id) in related_methods.keys():
related_method = getattr(self, method_id, None)
if related_method is not None:
table_id = {'src__' : 1} # Return query source, do not evaluate
table_id['query_table'] = query_table
table_index = 0
for t_tuple in related_table_map[(table_list,method_id)]:
table_id['table_%s' % table_index] = t_tuple[1] # table_X is set to mapped id
from_table_dict[t_tuple[1]] = t_tuple[0]
table_index += 1
where_expression_list.append(related_method(**table_id))
# Concatenate expressions
if kw.get('where_expression',None) not in (None,''):
where_expression_list.append(kw['where_expression'])
if len(where_expression_list)>1:
where_expression_list = ['(%s)' % x for x in where_expression_list]
where_expression = join(where_expression_list, ' AND ')
select_expression= join(select_expression_list,',')
limit_expression = kw.get('limit', None)
if isinstance(limit_expression, (list, tuple)):
limit_expression = '%s,%s' % (limit_expression[0], limit_expression[1])
elif limit_expression is not None:
limit_expression = str(limit_expression)
# force index if exists when doing sort as mysql doesn't manage them efficiently
if len(sort_key_list) > 0:
index_from_table = {}
# first group columns from a same table
for key in sort_key_list:
try:
related_table, column = key.split('.')
except ValueError:
# key is not of the form table.column
# so get table from dict
if len(from_table_dict) != 1:
continue
column = key
related_table = from_table_dict.keys()[0]
table = from_table_dict[related_table]
# Check if it's a column for which we want to specify index
index_columns = getattr(self, 'sql_catalog_index_on_order_keys', [])
sort_column = '%s.%s' %(table, column)
if not sort_column in index_columns:
continue
# Group columns
if not index_from_table.has_key(table):
index_from_table[table] = [column,]
real_select_dict[column.strip()] = None
select_dict = real_select_dict
else:
index_from_table[table].append(column)
# second ask index
for table in index_from_table.keys():
available_index_list = self.getIndex(table, index_from_table[table], key_list)
if len(available_index_list) > 0:
# Always give MySQL a chance to use PRIMARY key. It is much faster if
# current table is used in a join on primary key than forcing it to
# use another index.
# Note: due to a bug (?) in MySQL (at least 5.0.45 community), it is
# a syntax error to put "PRIMARY" keyword anywere besides at first
# position. Hence the "insert(0".
if 'PRIMARY' not in available_index_list:
available_index_list.insert(0, 'PRIMARY')
# tell mysql to use these index
table = from_table_dict.pop(related_table)
index_list_string = ', '.join(available_index_list)
table_with_index = "%s use index(%s)" %(related_table, index_list_string)
from_table_dict[table_with_index] = table
from_expression = kw.get('from_expression', None)
if from_expression is not None:
final_from_expression = ', '.join(
[from_expression.get(table, '`%s` AS `%s`' % (table, alias))
for alias, table in from_table_dict.iteritems()])
select_dict = None
elif isinstance(select_dict, (list, tuple)):
select_dict = dict([(x, None) for x in select_dict])
# Handle order_by_list
order_by_list = kw.pop('order_by_list', None)
sort_on = kw.pop('sort_on', None)
sort_order = kw.pop('sort_order', None)
order_by_expression = kw.pop('order_by_expression', None)
if order_by_list is None:
order_by_list = self.buildOrderByList(
sort_on=sort_on,
sort_order=sort_order,
order_by_expression=order_by_expression
)
else:
final_from_expression = None
# Use a dictionary at the moment.
return { 'from_table_list' : from_table_dict.items(),
'from_expression' : final_from_expression,
'order_by_expression' : sort_on,
'where_expression' : where_expression,
'limit_expression' : limit_expression,
'select_expression': select_expression,
'group_by_expression' : group_by_expression}
if sort_on is not None:
LOG('SQLCatalog', 0, 'order_by_list and sort_on were given, ignoring sort_on.')
if sort_order is not None:
LOG('SQLCatalog', 0, 'order_by_list and sort_order were given, ignoring sort_order.')
if order_by_expression is not None:
LOG('SQLCatalog', 0, 'order_by_list and order_by_expression were given, ignoring order_by_expression.')
# Handle from_expression
from_expression = kw.pop('from_expression', None)
# Handle where_expression
where_expression = kw.get('where_expression', None)
if isinstance(where_expression, basestring) and len(where_expression):
LOG('SQLCatalog', 100, 'Giving where_expression a string value is deprecated.')
# Transform given where_expression into a query, and update kw.
kw['where_expression'] = SQLQuery(where_expression)
# Handle select_expression_key
# It is required to support select_expression_key parameter for backward
# compatiblity, but I'm not sure if there can be a serious use for it in
# new API.
order_by_override_list = kw.pop('select_expression_key', None)
query = EntireQuery(
query=self.buildQuery(kw, ignore_empty_string=ignore_empty_string),
order_by_list=order_by_list,
order_by_override_list=order_by_override_list,
group_by_list=group_by_list,
select_dict=select_dict,
limit=limit,
catalog_table_name=query_table,
extra_column_list=extra_column_list,
from_expression=from_expression)
result = query.asSQLExpression(self, only_group_columns).asSQLExpressionDict()
if DEBUG:
LOG('buildSQLQuery', 0, repr(result))
return result
# Compatibililty SQL Sql
buildSqlQuery = buildSQLQuery
def queryResults(self, sql_method, REQUEST=None, used=None, src__=0, build_sql_query_method=None, **kw):
@profiler_decorator
@transactional_cache_decorator('SQLCatalog._getSearchKeyDict')
@profiler_decorator
@caching_class_method_decorator(id='SQLCatalog._getSearchKeyDict', cache_factory='erp5_content_long')
@profiler_decorator
def _getSearchKeyDict(self):
result = {}
search_key_column_dict = {
'KeywordKey': self.sql_catalog_keyword_search_keys,
'FullTextKey': self.sql_catalog_full_text_search_keys,
'DateTimeKey': self.sql_catalog_datetime_search_keys,
}
for key, column_list in search_key_column_dict.iteritems():
for column in column_list:
if column in result:
LOG('SQLCatalog', 100, 'Ambiguous configuration: column %r is set to use %r key, but also to use %r key. Former takes precedence.' % (column, result[column], key))
else:
result[column] = key
return result
@profiler_decorator
def getSearchKey(self, column, search_key=None):
"""
Return an instance of a SearchKey class.
column (string)
The column for which the search key will be returned.
search_key (string)
If given, must be the name of a SearchKey class to be returned.
Returned value will be an instance of that class, even if column has
been configured to use a different one.
"""
if search_key is None:
search_key = self._getSearchKeyDict().get(column, 'DefaultKey')
return getSearchKeyInstance(search_key, column)
def getComparisonOperator(self, operator):
"""
Return an instance of an Operator class.
operator (string)
String defining the expected operator class.
See Operator module to have a list of available operators.
"""
return getComparisonOperatorInstance(operator)
@profiler_decorator
def _queryResults(self, REQUEST=None, build_sql_query_method=None, **kw):
""" Returns a list of brains from a set of constraints on variables """
if build_sql_query_method is None:
build_sql_query_method = self.buildSQLQuery
query = build_sql_query_method(REQUEST=REQUEST, **kw)
# XXX: decide if this should be made normal
ENFORCE_SEPARATION = True
if ENFORCE_SEPARATION:
new_kw = {}
# Some parameters must be propagated:
for parameter_id in ('selection_domain', 'selection_report'):
if parameter_id in kw:
new_kw[parameter_id] = kw[parameter_id]
kw = new_kw
kw['where_expression'] = query['where_expression']
kw['sort_on'] = query['order_by_expression']
kw['from_table_list'] = query['from_table_list']
kw['from_expression'] = query.get('from_expression')
kw['from_expression'] = query['from_expression']
kw['limit_expression'] = query['limit_expression']
kw['select_expression'] = query['select_expression']
kw['group_by_expression'] = query['group_by_expression']
# Return the result
return kw
#LOG('acceptable_keys',0,'acceptable_keys: %s' % str(acceptable_keys))
#LOG('acceptable_key_map',0,'acceptable_key_map: %s' % str(acceptable_key_map))
#LOG('queryResults',0,'kw: %s' % str(kw))
#LOG('queryResults',0,'from_table_list: %s' % str(query['from_table_list']))
return sql_method(src__=src__, **kw)
def queryResults(self, sql_method, REQUEST=None, src__=0, build_sql_query_method=None, **kw):
sql_kw = self._queryResults(REQUEST=REQUEST, build_sql_query_method=build_sql_query_method, **kw)
if DEBUG and not src__:
LOG('queryResults', 0, sql_method(src__=1, **sql_kw))
return sql_method(src__=src__, **sql_kw)
def searchResults(self, REQUEST=None, used=None, **kw):
def searchResults(self, REQUEST=None, **kw):
""" Returns a list of brains from a set of constraints on variables """
# The used argument is deprecated and is ignored
method = getattr(self, self.sql_search_results)
return self.queryResults(method, REQUEST=REQUEST, used=used, **kw)
return self.queryResults(method, REQUEST=REQUEST, extra_column_list=self.getCatalogSearchResultKeys(), **kw)
__call__ = searchResults
def countResults(self, REQUEST=None, used=None, stat__=1, **kw):
def countResults(self, REQUEST=None, **kw):
""" Returns the number of items which satisfy the where_expression """
# Get the search method
method = getattr(self, self.sql_count_results)
return self.queryResults(method, REQUEST=REQUEST, used=used, stat__=stat__, **kw)
return self.queryResults(method, REQUEST=REQUEST, extra_column_list=self.getCatalogSearchResultKeys(), only_group_columns=True, **kw)
def recordObjectList(self, path_list, catalog=1):
"""
......@@ -2394,40 +2401,56 @@ Globals.default__class_init__(Catalog)
class CatalogError(Exception): pass
# pool of global preinitialized search keys instances
SEARCH_KEY_INSTANCE_POOL = threading.local()
# hook search keys and Query implementation
def getSearchKeyInstance(search_key_class):
""" Return instance of respective search_key class.
We should have them initialized only once."""
global SEARCH_KEY_INSTANCE_POOL
if not hasattr(SEARCH_KEY_INSTANCE_POOL, 'pool'):
pool = dict()
for klass in (DefaultKey, RawKey, KeyWordKey, DateTimeKey,
FullTextKey, FloatKey, ScriptableKey, KeyMappingKey):
search_key_instance = klass()
search_key_instance.build()
pool[klass] = search_key_instance
SEARCH_KEY_INSTANCE_POOL.pool = pool
return SEARCH_KEY_INSTANCE_POOL.pool[search_key_class]
from Query.Query import QueryMixin
from Query.SimpleQuery import NegatedQuery, SimpleQuery
from Query.Query import Query as _Query
from Query.SimpleQuery import SimpleQuery
from Query.ComplexQuery import ComplexQuery
from Query.AutoQuery import AutoQuery as Query
def NegatedQuery(query):
return ComplexQuery(query, operator='not')
allow_class(SimpleQuery)
allow_class(ComplexQuery)
import SearchKey
SEARCH_KEY_INSTANCE_POOL = {}
SEARCH_KEY_CLASS_CACHE = {}
@profiler_decorator
def getSearchKeyInstance(search_key_class_name, column):
assert isinstance(search_key_class_name, basestring)
try:
search_key_class = SEARCH_KEY_CLASS_CACHE[search_key_class_name]
except KeyError:
search_key_class = getattr(getattr(SearchKey, search_key_class_name),
search_key_class_name)
SEARCH_KEY_CLASS_CACHE[search_key_class_name] = search_key_class
try:
instance_dict = SEARCH_KEY_INSTANCE_POOL[search_key_class]
except KeyError:
instance_dict = SEARCH_KEY_INSTANCE_POOL[search_key_class] = {}
try:
result = instance_dict[column]
except KeyError:
result = instance_dict[column] = search_key_class(column)
return result
from Operator import operator_dict
def getComparisonOperatorInstance(operator):
return operator_dict[operator]
from Query.EntireQuery import EntireQuery
from Query.SQLQuery import SQLQuery
verifyClass(ISearchKeyCatalog, Catalog)
# for of backwards compatability
QueryMixin = QueryMixin
Query = SimpleQuery
NegatedQuery = NegatedQuery
ComplexQuery = ComplexQuery
if PROFILING_ENABLED:
def Catalog_dumpProfilerData(self):
return profiler_report()
from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
def Catalog_resetProfilerData(self):
profiler_reset()
Catalog.dumpProfilerData = Catalog_dumpProfilerData
Catalog.resetProfilerData = Catalog_resetProfilerData
##############################################################################
#
# Copyright (c) 2008-2009 Nexedi SARL and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from zLOG import LOG
from Interface.ISQLExpression import ISQLExpression
from Interface.Verify import verifyClass
from types import NoneType
from SQLCatalog import profiler_decorator
SQL_LIST_SEPARATOR = ', '
SQL_TABLE_FORMAT = '%s' # XXX: should be changed to '`%s`', but this breaks some ZSQLMethods.
SQL_SELECT_ALIAS_FORMAT = '%s AS `%s`'
"""
TODO:
- change table_alias_dict in internals to represent computed tables:
ie: '(SELECT * FROM `bar` WHERE `baz` = "hoge") AS `foo`'
'`foo` LEFT JOIN `bar` WHERE (`baz` = "hoge")'
"""
# Set to true to keep a reference to the query which created us.
# Set to false to avoid keeping a reference to an object.
DEBUG = True
def defaultDict(value):
if value is None:
return {}
assert isinstance(value, dict)
return value.copy()
class SQLExpression(object):
__implements__ = ISQLExpression
@profiler_decorator
def __init__(self,
query,
table_alias_dict=None,
order_by_list=(),
order_by_dict=None,
group_by_list=(),
where_expression=None,
where_expression_operator=None,
sql_expression_list=(),
select_dict=None,
limit=None,
from_expression=None):
if DEBUG:
self.query = query
self.table_alias_dict = defaultDict(table_alias_dict)
self.order_by_list = list(order_by_list)
self.group_by_list = list(group_by_list)
self.order_by_dict = defaultDict(order_by_dict)
# Only one of (where_expression, where_expression_operator) must be given (never both)
assert None in (where_expression, where_expression_operator)
# Exactly one of (where_expression, where_expression_operator) must be given, except if sql_expression_list is given and contains exactly one entry
assert where_expression is not None or where_expression_operator is not None or (sql_expression_list is not None and len(sql_expression_list) == 1)
# where_expression must be a basestring instance if given
assert isinstance(where_expression, (NoneType, basestring))
# where_expression_operator must be 'and', 'or' or 'not' (if given)
assert where_expression_operator in (None, 'and', 'or', 'not'), where_expression_operator
self.where_expression = where_expression
self.where_expression_operator = where_expression_operator
# Exactly one of (where_expression, sql_expression_list) must be given (XXX: duplicate of previous conditions ?)
assert where_expression is not None or sql_expression_list is not None
if isinstance(sql_expression_list, (list, tuple)):
sql_expression_list = [x for x in sql_expression_list if x is not None]
self.sql_expression_list = list(sql_expression_list)
self.select_dict = defaultDict(select_dict)
if limit is None:
self.limit = ()
elif isinstance(limit, (list, tuple)):
if len(limit) < 3:
self.limit = limit
else:
raise ValueError, 'Unrecognized "limit" value: %r' % (limit, )
else:
self.limit = (limit, )
if from_expression is not None:
LOG('SQLExpression', 0, 'Providing a from_expression is deprecated.')
self.from_expression = from_expression
@profiler_decorator
def getTableAliasDict(self):
"""
Returns a dictionary:
key: table alias (string)
value: table name (string)
If there are nested SQLExpressions, it aggregates their mappings and
checks that they don't alias different table with the same name. If they
do, it raises a ValueError.
"""
result = self.table_alias_dict.copy()
for sql_expression in self.sql_expression_list:
for alias, table_name in sql_expression.getTableAliasDict().iteritems():
existing_value = result.get(alias)
if existing_value not in (None, table_name):
message = '%r is a known alias for table %r, can\'t alias it now to table %r' % (alias, existing_value, table_name)
if DEBUG:
message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
self.query,
sql_expression,
sql_expression.getQuery(),
', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
raise ValueError, message
result[alias] = table_name
return result
@profiler_decorator
def getFromExpression(self):
"""
Returns a string.
If there are nested SQLExpression, it checks that they either don't
define any from_expression or the exact same from_expression. Otherwise,
it raises a ValueError.
"""
result = self.from_expression
for sql_expression in self.sql_expression_list:
from_expression = sql_expression.getFromExpression()
if None not in (result, from_expression):
message = 'I don\'t know how to merge from_expressions'
if DEBUG:
message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
self.query,
sql_expression,
sql_expression.getQuery(),
', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
raise ValueError, message
return result
@profiler_decorator
def getOrderByList(self):
"""
Returns a list of strings.
If there are nested SQLExpression, it checks that they don't define
sorts for columns which are already sorted. If they do, it raises a
ValueError.
"""
result = self.order_by_list[:]
known_column_set = set([x[0] for x in result])
for sql_expression in self.sql_expression_list:
for order_by in sql_expression.getOrderByList():
if order_by[0] in known_column_set:
raise ValueError, 'I don\'t know how to merge order_by yet'
else:
result.append(order_by)
known_column_set.add(order_by[0])
return result
@profiler_decorator
def getOrderByDict(self):
result_dict = self.order_by_dict.copy()
for sql_expression in self.sql_expression_list:
order_by_dict = sql_expression.getOrderByDict()
for key, value in order_by_dict.iteritems():
if key in result_dict and value != result_dict[key]:
message = 'I don\'t know how to merge order_by_dict with ' \
'conflicting entries for key %r: %r vs. %r' % (key, result_dict[key], value)
if DEBUG:
message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
self.query,
sql_expression,
sql_expression.getQuery(),
', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
raise ValueError, message
result_dict.update(order_by_dict)
return result_dict
@profiler_decorator
def getOrderByExpression(self):
"""
Returns a string.
Returns a rendered "order by" expression. See getOrderByList.
"""
order_by_dict = self.getOrderByDict()
get = order_by_dict.get
return SQL_LIST_SEPARATOR.join(get(x, str(x)) \
for x in self.getOrderByList())
@profiler_decorator
def getWhereExpression(self):
"""
Returns a string.
Returns a rendered "where" expression.
"""
if self.where_expression is not None:
result = self.where_expression
else:
if self.where_expression_operator == 'not':
assert len(self.sql_expression_list) == 1
result = '(NOT %s)' % (self.sql_expression_list[0].getWhereExpression())
elif len(self.sql_expression_list) == 1:
result = self.sql_expression_list[0].getWhereExpression()
elif len(self.sql_expression_list) == 0:
result = '(1)'
else:
operator = '\n ' + self.where_expression_operator.upper() + ' '
result = '(%s)' % (operator.join(x.getWhereExpression() for x in self.sql_expression_list), )
return result
@profiler_decorator
def getLimit(self):
"""
Returns a list of 1 or 2 items (int or string).
If there are nested SQLExpression, it checks that they either don't
define any limit or the exact same limit. Otherwise it raises a
ValueError.
"""
result = list(self.limit)
for sql_expression in self.sql_expression_list:
other_limit = sql_expression.getLimit()
if other_limit not in ([], result):
message = 'I don\'t know how to merge limits yet'
if DEBUG:
message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
self.query,
sql_expression,
sql_expression.getQuery(),
', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
raise ValueError, message
return result
@profiler_decorator
def getLimitExpression(self):
"""
Returns a string.
Returns a rendered "limit" expression. See getLimit.
"""
return SQL_LIST_SEPARATOR.join(str(x) for x in self.getLimit())
@profiler_decorator
def getGroupByset(self):
"""
Returns a set of strings.
If there are nested SQLExpression, it merges (union of sets) them with
local value.
"""
result = set(self.group_by_list)
for sql_expression in self.sql_expression_list:
result.update(sql_expression.getGroupByset())
return result
@profiler_decorator
def getGroupByExpression(self):
"""
Returns a string.
Returns a rendered "group by" expression. See getGroupBySet.
"""
return SQL_LIST_SEPARATOR.join(self.getGroupByset())
@profiler_decorator
def getSelectDict(self):
"""
Returns a dict:
key: alias (string)
value: column (string) or None
If there are nested SQLExpression, it aggregates their mappings and
checks that they don't alias different columns with the same name. If
they do, it raises a ValueError.
"""
result = self.select_dict.copy()
for sql_expression in self.sql_expression_list:
for alias, column in sql_expression.getSelectDict().iteritems():
existing_value = result.get(alias)
if existing_value not in (None, column):
message = '%r is a known alias for column %r, can\'t alias it now to column %r' % (alias, existing_value, column)
if DEBUG:
message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
self.query,
sql_expression,
sql_expression.getQuery(),
', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
raise ValueError, message
result[alias] = column
return result
@profiler_decorator
def getSelectExpression(self):
"""
Returns a string.
Returns a rendered "select" expression. See getSelectDict.
"""
return SQL_LIST_SEPARATOR.join(
SQL_SELECT_ALIAS_FORMAT % (column, alias)
for alias, column in self.getSelectDict().iteritems())
@profiler_decorator
def asSQLExpressionDict(self):
table_alias_dict = self.getTableAliasDict()
from_table_list = []
append = from_table_list.append
for alias, table in table_alias_dict.iteritems():
append((SQL_TABLE_FORMAT % (alias, ), SQL_TABLE_FORMAT % (table, )))
from_expression_dict = self.getFromExpression()
if from_expression_dict is not None:
from_expression = SQL_LIST_SEPARATOR.join(
from_expression_dict.get(table, '`%s` AS `%s`' % (table, alias))
for alias, table in table_alias_dict.iteritems())
else:
from_expression = None
return {
'where_expression': self.getWhereExpression(),
'order_by_expression': self.getOrderByExpression(),
'from_table_list': from_table_list,
'from_expression': from_expression,
'limit_expression': self.getLimitExpression(),
'select_expression': self.getSelectExpression(),
'group_by_expression': self.getGroupByExpression()
}
verifyClass(ISQLExpression, SQLExpression)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -26,171 +28,241 @@
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from DateTime import DateTime
import sys
from SearchKey import SearchKey
from pprint import pprint
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
#from Products.ZSQLCatalog.SQLExpression import SQLExpression
from zLOG import LOG
from DateTime.DateTime import DateTime, DateTimeError, _cache
from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
MARKER = []
class DateTimeKey(SearchKey):
""" DateTimeKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try to match values in DateTime MySQL columns.
It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
addition to main logical operators like ['OR', 'or', 'AND', 'and'].
timezone_dict = _cache._zmap
date_completion_format_dict = {
None: ['01/01/%s', '01/%s'],
'international': ['%s/01/01', '%s/01']
}
@profiler_decorator
def _DateTime(*args, **kw):
return DateTime(*args, **kw)
Note: because all ERP5 datetime values are indexed in MySQL in 'UTC'
the respective passed date will be first converted to 'UTC' before inserted into
respective SQL query!
@profiler_decorator
def castDate(value):
date_kw = {'datefmt': 'international'}
if isinstance(value, dict):
# Convert value into a DateTime, and guess desired delta from what user
# input.
assert value['type'] == 'date'
format = value.get('format')
value = value['query']
if format == '%m/%d/%Y':
date_kw.pop('datefmt')
if isinstance(value, DateTime):
pass
elif isinstance(value, basestring):
try:
value = _DateTime(value, **date_kw)
except DateTimeError:
delimiter_count = countDelimiters(value)
if delimiter_count < 3:
split_value = value.split()
if split_value[-1].lower() in timezone_dict:
value = '%s %s' % (date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (' '.join(split_value[:-1]), ), split_value[-1])
else:
value = date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (value, )
value = _DateTime(value, **date_kw)
else:
raise
else:
raise TypeError, 'Unknown date type: %r' % (value)
return value.toZone('UTC')
Examples (GMT+02, Bulgaria/Sofia for 'delivery.start_date'):
# (strongly) inspired from DateTime.DateTime.py
delimiter_list = ' -/.:,+'
* '15/01/2008' --> "delivery.start_date = '2008-01-14 22:00'"
def getMonthLen(datetime):
return datetime._month_len[datetime.isLeapYear()][datetime.month()]
* '>=15/01/2008' --> "delivery.start_date >= '2008-01-14 22:00'"
def getYearLen(datetime):
return 365 + datetime.isLeapYear()
* '>=15/01/2008 or <=20/01/2008'
--> "delivery.start_date >= '2008-01-14 22:00' or delivery.start_date<='2008-01-19 22:00'"
delta_list = [getYearLen, getMonthLen, 1, 1.0 / 24, 1.0 / (24 * 60), 1.0 / (24 * 60 * 60)]
* '>=15/01/2008 10:00 GMT+02 OR <=20/01/2008 05:12 Universal'
-->
"delivery.start_date >= '2008-01-15 08:00 Universal'
OR
delivery.start_date <= '2008-01-20 05:12 Universal'
"
"""
@profiler_decorator
def countDelimiters(value):
assert isinstance(value, basestring)
# Detect if timezone was provided, to avoid counting it as in precision computation.
split_value = value.split()
if split_value[-1].lower() in timezone_dict:
value = ' '.join(split_value[:-1])
# Count delimiters
delimiter_count = 0
for char in value:
if char in delimiter_list:
delimiter_count += 1
return delimiter_count
tokens = ('DATE', 'OR', 'AND', 'NOT', 'EQUAL',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT', 'EQUAL',)
def t_OR(self, t):
r'\s+(OR|or)\s+'
# operator has leading and trailing ONLY one white space character
t.value = 'OR'
return t
def t_AND(self, t):
r'\s+(AND|and)\s+'
# operator has leading and trailing ONLY one white space character
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+(NOT|not)\s+|!=)'
# operator has leading and trailing ONLY one white space character
t.value = t.value.upper().strip()
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
t_EQUAL = r'='
t_DATE = r'\d{1,4}[(/|\.|\-) /.]\d{1,4}[(/|\.|\-) /.]\d{1,4}((\s.)*\d{0,2}:\d{0,2}(:\d{0,2})?)?(\sUniversal|\sGMT\+\d\d)?|\d\d\d\d%?'
def quoteSQLString(self, value, format):
""" Return a quoted string of the value.
Make sure to convert it to UTC first."""
if getattr(value, 'ISO', None) is not None:
value = "'%s'" % value.toZone('UTC').ISO()
@profiler_decorator
def getPeriodBoundaries(value):
first_date = castDate(value)
if isinstance(value, dict):
value = value['query']
# Try to guess how much was given in query.
if isinstance(value, basestring):
delimiter_count = countDelimiters(value)
elif isinstance(value, DateTime):
raise TypeError, 'Impossible to guess a precision from a DateTime type.'
else:
value = "'%s'" %DateTime(value).toZone('UTC').ISO()
return value
raise TypeError, 'Unknown date type: %r' % (value)
delta = delta_list[delimiter_count]
if callable(delta):
delta = delta(first_date)
return first_date, first_date + delta
def buildQueryForTokenList(self, tokens, key, value, format):
""" Build a ComplexQuery for a token list """
@profiler_decorator
def wholePeriod(search_key, group, column, value_list, exclude=False):
if exclude:
first_operator = '<'
second_operator = '>='
logical_operator = 'or'
else:
first_operator = '>='
second_operator = '<'
logical_operator = 'and'
query_list = []
for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
token_values = [x.value for x in group_tokens]
sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
date_value = sub_tokens[0].value
days_offset = 0
# some format require special handling
if format != '%Y':
# full format (Year/Month/Day)
if sub_operator in ('=',):
# 2007/01/01 00:00 <= date < 2007/01/02
days_offset = 1
elif format == '%Y':
# incomplete format only Year because DateTime can not handle
# extend format and value by assumption that start of year is ment
# add days ofset accordingly
format = '%%%s/%%m/%%d' %format
date_value = '%s/01/01' %date_value
days_offset_map = {'=' : 366, '>' : 366,
'>=' : 366, '<': -366, '<=':-366}
days_offset = days_offset_map[sub_operator]
# convert to UTC in given format
is_valid_date = 1
try:
if format != '%m/%d/%Y':
# treat ambigious dates as "days before month before year"
date_value = DateTime(date_value, datefmt="international").toZone('UTC')
append = query_list.append
for value in value_list:
first_date, second_date = getPeriodBoundaries(value)
append(ComplexQuery([SimpleQuery(search_key=search_key, operator=first_operator, group=group, **{column: first_date}),
SimpleQuery(search_key=search_key, operator=second_operator, group=group, **{column: second_date})],
operator=logical_operator))
return query_list
def matchWholePeriod(search_key, group, column, value_list, *ignored):
return wholePeriod(search_key, group, column, value_list)
def matchNotWholePeriod(search_key, group, column, value_list, *ignored):
return wholePeriod(search_key, group, column, value_list, exclude=True)
@profiler_decorator
def matchExact(search_key, group, column, value_list, comparison_operator, logical_operator):
if comparison_operator is None:
comparison_operator = '='
value_list = [castDate(x) for x in value_list]
if logical_operator == 'or' and comparison_operator == '=':
query_list = [SimpleQuery(search_key=search_key, operator='in', group=group, **{column: value_list})]
else:
# US style "month before day before year"
date_value = DateTime(date_value).toZone('UTC')
except:
is_valid_date = 0
query_kw = None
if is_valid_date:
if sub_operator == '=':
# transform to range 'key >= date AND date < key'
query_kw = {key: (date_value, date_value + days_offset,),
'range': 'minmax'}
query_list = [SimpleQuery(search_key=search_key, operator=comparison_operator, group=group, **{column: x}) for x in value_list]
return query_list
def getNextPeriod(value):
return getPeriodBoundaries(value)[1]
@profiler_decorator
def matchBeforeNextPeriod(search_key, group, column, value_list, comparison_operator, logical_operator):
return matchExact(search_key, group, column, [getNextPeriod(x) for x in value_list], '<', logical_operator)
@profiler_decorator
def matchAfterPeriod(search_key, group, column, value_list, comparison_operator, logical_operator):
return matchExact(search_key, group, column, [getNextPeriod(x) for x in value_list], '>=', logical_operator)
operator_matcher_dict = {
None: matchWholePeriod,
'=': matchWholePeriod,
'!=': matchNotWholePeriod,
'<': matchExact,
'>=': matchExact,
'<=': matchBeforeNextPeriod,
'>': matchAfterPeriod,
}
# Behaviour of date time operators
# Objects:
# 2005/03/14 23:59:59
# 2005/03/15 00:00:00
# 2005/03/15 00:00:01
# 2005/03/15 23:59:59
# 2005/03/16 00:00:00
# 2005/03/16 00:00:01
#
# Searches:
# "2005/03/15" (operator = None)
# Implicitely matches the whole period.
# 2005/03/15 00:00:00
# 2005/03/15 00:00:01
# 2005/03/15 23:59:59
#
# "=2005/03/15" (operator = '=')
# Behaves the same way as None operator.
# 2005/03/15 00:00:00
# 2005/03/15 00:00:01
# 2005/03/15 23:59:59
#
# "!=2005/03/15" (operator = '!=')
# Complementary of '=' operator.
# 2005/03/14 23:59:59
# 2005/03/16 00:00:00
# 2005/03/16 00:00:01
#
# "<2005/03/15" (operator = '<')
# Non-ambiguous (no difference wether time is considered as a period or a single point in time).
# 2005/03/14 23:59:59
#
# ">=2005/03/15" (operator = '>=')
# Complementary of '<' operator, and also non-ambiguous.
# 2005/03/15 00:00:00
# 2005/03/15 00:00:01
# 2005/03/15 23:59:59
# 2005/03/16 00:00:00
# 2005/03/16 00:00:01
#
# "<=2005/03/15" (operator = '<=')
# Union of results from '=' and '<' operators.
# 2005/03/14 23:59:59
# 2005/03/15 00:00:00
# 2005/03/15 00:00:01
# 2005/03/15 23:59:59
#
# ">2005/03/15" (operator = '>')
# Complementary of '<=' operator.
# 2005/03/16 00:00:00
# 2005/03/16 00:00:01
class DateTimeKey(SearchKey):
"""
This SearchKey allows generating date ranges from single, user-input dates.
"""
default_comparison_operator = None
get_operator_from_value = True
def _renderValueAsSearchText(self, value, operator):
return '"%s"' % (DateTime(value).ISO(), )
@profiler_decorator
def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
column = self.getColumn()
query_list = []
extend = query_list.extend
for comparison_operator, value_list in operator_value_dict.iteritems():
try:
if parsed:
subquery_list = operator_matcher_dict[comparison_operator](
self, group, column, value_list, comparison_operator,
logical_operator)
else:
query_kw = {key: date_value + days_offset,
'range': sub_operator}
query_kw['type'] = 'date'
subquery_list = matchExact(self, group, column, value_list, comparison_operator, logical_operator)
except DateTimeError:
LOG('DateTimeKey', 100, 'Got an exception while generating a query for %r %r.' % (comparison_operator, value_list), error=sys.exc_info())
else:
# not a valid date, try to get an year range
is_year = 1
date_value = date_value.replace('%', '')
try: date_value = int(date_value)
except: is_year = 0
if is_year:
date_value = '%s/01/01' % date_value
date_value = DateTime(date_value).toZone('UTC')
query_kw = {key: (date_value, date_value + 366,),
'type': 'date',
'range': 'minmax'}
# append only if it was possible to generate query
if query_kw is not None:
query_list.append(Query(**query_kw))
# join query list in one really big ComplexQuery
if len(query_list):
complex_query = ComplexQuery(*query_list,
**{'operator': 'AND'})
return complex_query
## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
## """ Tokenize/analyze passed string value and generate SQL query expressions. """
## where_expression = ''
## key = self.quoteSQLKey(key, format)
## tokens = self.tokenize(value)
## operators_mapping_list = self.groupByOperator(tokens)
## # new one
## for item in operators_mapping_list:
## row_tokens_values = []
## tokens = item['tokens']
## operator = item['operator']
## operator_value = None
## if operator is not None:
## # operator is standalone expression
## operator_value = operator.value
## where_expressions.append('%s' %operator_value)
## if len(tokens):
## # no it's not a stand alone expression,
## # determine it from list of tokens
## operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
## row_tokens_values = [self.quoteSQLString(x.value, format) for x in sub_tokens]
## where_expression = "%s %s %s" %(key, operator_value, ' '.join(row_tokens_values))
## return where_expression, []
extend(subquery_list)
return query_list
verifyClass(ISearchKey, DateTimeKey)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -26,124 +28,36 @@
#
##############################################################################
from DocumentTemplate.DT_Var import sql_quote
from SearchKey import SearchKey
from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
from Interface.Verify import verifyClass
class DefaultKey(SearchKey):
""" DefaultKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try to exactly one value.
It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
addition to main logical operators like ['OR', 'or', 'AND', 'and'].
Examples for title column:
* 'foo or bar' --> "title = 'foo' OR title = 'bar'"
* 'foo or =bar' --> "title = 'foo' OR title = 'bar'"
* '%foo% or bar' --> "title = '%foo%' OR title = 'bar'"
* 'Organisation Module' --> "title = 'Organisation Module'"
* '"Organisation Module"' --> "title = 'Organisation Module'"
* '="Organisation Module"' --> "title = 'Organisation Module'"
"""
# default type of sub Queries to be generated out fo a search string
default_key_type = 'default'
tokens = ('OR', 'AND', 'NOT', 'WORDSET', 'WORD',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'\s+(OR|or)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'\s+(AND|and)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+(NOT|not)\s+|!=)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = '!='
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
def t_WORD(self, t):
r'([^"\s<>!][\S\n]*|!([^=\s][\S\n]*)?)'
# newlines are allowed, because variations are delimited by newlines.
# WORD may contain arbitrary letters and numbers without white space
# WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
value = t.value.strip()
if value[0] == '=':
value = value[1:]
t.value = value
return t
def t_WORDSET(self, t):
r'"[^"]*"'
#r'"[\x7F-\xFF\w\d\s/%][\x7F-\xFF\w\d\s/%]*"'
# WORDSET is a combination of WORDs separated by white space
# and starting/ending with "
value = t.value.replace('"', '').strip()
t.value = value
return t
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
if isinstance(value, (int, long,)):
return str(value)
return "'%s'" % sql_quote(value)
## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
## """ Tokenize/analyze passed string value and generate SQL query expressions. """
## where_expressions = []
## select_expressions = []
## tokens = self.tokenize(value)
## operators_mapping_list = self.groupByOperator(tokens)
##
## # find if any logical operator exists
## tokens_values = []
## logical_operator_found = 0
## for token in tokens:
## if token.type not in ('WORDSET', 'WORD',):
## logical_operator_found = 1
## break
## tokens_values.append(token.value.replace("'", ""))
##
## # build expressions
## if not logical_operator_found:
## # no logical operator found so we assume that we search for a combination of words
## where_expressions.append("%s = '%s'" %(key, ' '.join(tokens_values)))
## else:
## # in the search string we have explicitly defined an operator
## for item in operators_mapping_list:
## row_tokens_values = []
## tokens = item['tokens']
## operator = item['operator']
## operator_value = None
## if operator is not None:
## # operator is standalone expression
## operator_value = operator.value
## where_expressions.append('%s' %operator_value)
## if len(tokens):
## # no it's not a stand alone expression,
## # determine it from list of tokens
## operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
## row_tokens_values = [x.value for x in sub_tokens]
## where_expressions.append("%s %s '%s'" %(key, operator_value, ' '.join(row_tokens_values)))
## return where_expressions, select_expressions
This SearchKey behaves like an ExactMatch SearchKey, except if value is a
string and contains a '%' sign, in which case it behaves like a
KeywordKey.
"""
default_comparison_operator = '='
get_operator_from_value = True
def _guessComparisonOperator(self, value):
if isinstance(value, basestring) and '%' in value:
operator = 'like'
else:
operator = SearchKey._guessComparisonOperator(self, value)
return operator
def buildSearchTextExpression(self, operator, value, column=None):
operator_text = operator.getOperator()
if column is None:
column = self.getColumn()
if operator_text == 'like':
assert isinstance(value, basestring)
assert '%' in value
result = '%s:%s' % (column, value)
else:
result = SearchKey.buildSearchTextExpression(self, operator, value, column=column)
return result
verifyClass(ISearchKey, DefaultKey)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -27,77 +29,19 @@
##############################################################################
from SearchKey import SearchKey
import re
SEARCH_MODE_MAPPING = {'in_boolean_mode': 'IN BOOLEAN MODE',
'with_query_expansion': 'WITH QUERY EXPANSION'}
from Products.ZSQLCatalog.SearchText import parse
from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
from Interface.Verify import verifyClass
class FullTextKey(SearchKey):
""" FullTextKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try match all possible values using
MySQL's fulltext search support.
See syntax see MySQL's FullText search reference:
http://dev.mysql.com/doc/refman/5.0/en/fulltext-search.html
"""
This SearchKey generates SQL fulltext comparisons.
"""
default_comparison_operator = 'match'
get_operator_from_value = False
tokens = ('PLUS', 'MINUS', 'WORD', 'GREATERTHAN', 'LESSTHAN', 'LEFTPARENTHES',
'RIGHTPARENTHES', 'TILDE', 'ASTERISK', 'DOUBLEQUOTE',)
# SQL expressions patterns
relevance = '%s_relevance'
where_match_against = "MATCH %s AGAINST (%s %s)"
select_match_against_as = "MATCH %s AGAINST (%s %s) AS %s"
t_PLUS = r'(\+)'
t_MINUS = r'(\-)'
t_GREATERTHAN = r'(\>)'
t_LESSTHAN = r'(\<)'
t_LEFTPARENTHES = r'(\()'
t_RIGHTPARENTHES = r'(\))'
t_TILDE = r'(\~)'
t_ASTERISK = r'(\*)'
t_DOUBLEQUOTE = r'(\")'
def parseSearchText(self, value):
return parse(value)
def t_WORD(self, t):
r'[^\+\-<>\(\)\~\*\"\s]$|[^\+\-<>\(\)\~\*\"\s]+[^\*\"\s\)]'
#r'[^\+\-<>\(\)\~\*\"\s]\S*'
#r'[\x7F-\xFF\w\d][\x7F-\xFF\w\d]*'
# WORD may contain arbitrary letters and numbers without white space
return t
verifyClass(ISearchKey, FullTextKey)
def buildSQLExpression(self, key, value,
format=None, mode=None, range_value=None, stat__=None):
""" Analize token list and generate SQL expressions."""
tokens = self.tokenize(value)
# based on type tokens we may switch to different search mode
mode = SEARCH_MODE_MAPPING.get(mode, '')
if mode == '':
# determine it based on list of tokens i.e if we have only words
# leave as its but if we have '-' or '+' use boolean mode
for token in tokens:
if token.type != 'WORD':
mode = SEARCH_MODE_MAPPING['in_boolean_mode']
break
if mode == '' and len(tokens) > 1:
value = ' '.join(['+%s' % x.value for x in tokens])
mode = SEARCH_MODE_MAPPING['in_boolean_mode']
# split (if possible) to column.key
if key.find('.') != -1:
table, column = key.split('.')
relevance_key1 = self.relevance %key.replace('.', '_')
relevance_key2 = self.relevance %column
else:
relevance_key1 = self.relevance %key
relevance_key2 = None
select_expression_list = []
where_expression = self.where_match_against % (key,
self.quoteSQLString(value, ''), mode)
if not stat__:
# stat__ is an internal implementation artifact to prevent adding
# select_expression for countFolder
select_expression_list = [self.select_match_against_as % (key,
self.quoteSQLString(value, ''), mode, relevance_key1),]
if relevance_key2 is not None:
select_expression_list.append(self.select_match_against_as % (
key, self.quoteSQLString(value, ''), mode, relevance_key2))
return where_expression, select_expression_list
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from SearchKey import SearchKey
from pprint import pprint
class KeyWordKey(SearchKey):
""" KeyWordKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will try to match all possible values in a greedy manner.
It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
addition to main logical operators like ['OR', 'or', 'AND', 'and'].
Examples for title column:
* 'foo or bar' --> "title LIKE '%foo%' OR title LIKE '%bar%'"
* 'foo or =bar' --> "title LIKE '%foo%' OR title = 'bar'"
* 'Organisation Module' --> "title LIKE '%Organisation Module%'"
* '"Organisation Module"' --> "title LIKE '%Organisation Module%'"
* '="Organisation Module"' --> "title = 'Organisation Module'"
"""
tokens = ('OR', 'AND', 'NOT',
'KEYWORD', 'WORDSET', 'WORD', 'EXPLICITEQUALLITYWORD',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
# this is the default operator
default_operator = 'like'
# if token's list starts with left sided operator
# use this map to transfer it to range operator
token_operator_range_map = {'like': 'like',
'!=': 'not_like',
'=': '=',}
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'\s+(OR|or)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'\s+(AND|and)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_NOT(self, t):
r'(\s+(NOT|not)\s+|!=)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = t.value.upper().strip()
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
def t_EXPLICITEQUALLITYWORD(self, t):
r'=\S*'
# EXPLICITEQUALLITYWORD may contain arbitrary letters and numbers without white space
# EXPLICITEQUALLITYWORD must contain '=' at the beginning
value = t.value.strip()
# get rid of leading '='
t.value = value[1:]
return t
def t_KEYWORD(self, t):
r'(%\S*|[^!<>=\s%]*%)(?!\S)'
# KEYWORD must start and/or end with '%'.
# It may contain arbitrary letters and numbers without white space
value = t.value.strip()
t.value = value
return t
def t_WORD(self, t):
r'([^"\s<>!=%]([^ \t\r\f\v]*[^ \t\r\f\v%])?|!([^= \t\r\f\v%]|[^= \t\r\f\v][\S\n]*[^ \t\r\f\v%])?)'
# WORD may contain arbitrary letters and numbers without white space
# WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
value = t.value.strip()
t.value = value
return t
def t_WORDSET(self, t):
r'=?"[^"]*"'
# WORDSET is a combination of WORDs separated by white space
# and starting/ending with " (optionally with '=')
value = t.value.replace('"', '')
if value[0] == '=':
value = value[1:]
t.value = value
return t
def getOperatorForTokenList(self, tokens):
""" Generic implementation that will return respective
operator for a token list. The first found occurence wins."""
token = tokens[0]
token_type = token.type
if token_type in self.sub_operators:
return token.value, tokens[1:]
elif token.type == 'EXPLICITEQUALLITYWORD':
# even though it's keyword key we can still explicitly define
# that we want equality
return '=', tokens
else:
return self.default_operator, tokens
def buildQueryForTokenList(self, tokens, key, value, format):
""" Build a ComplexQuery for a token list """
query_list = []
for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
token_values = [x.value for x in group_tokens]
sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
first_token = sub_tokens[0]
range = self.token_operator_range_map.get(sub_operator)
sub_tokens_values = [x.value for x in sub_tokens]
right_side_expression = ' '.join(sub_tokens_values)
if first_token.type == 'WORDSET' and first_token.value.startswith('='):
range = '='
right_side_expression = first_token.value[1:]
elif first_token.type in ('WORDSET', 'WORD',) and range == 'like':
if '%' not in right_side_expression:
# If the search string doesn't already contain '%', add trailing and
# leading '%' to get more results
right_side_expression = '%%%s%%' % right_side_expression
query_kw = {key: right_side_expression,
'range': range}
query_list.append(Query(**query_kw))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator': 'AND'})
return complex_query
## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
## """ Tokenize/analyze passed string value and generate SQL query expressions. """
## where_expressions = []
## select_expressions = []
## tokens = self.tokenize(value)
## operators_mapping_list = self.groupByOperator(tokens)
##
## # find if any logical operator exists
## tokens_values = []
## logical_operator_found = 0
## for token in tokens:
## if token.type not in ('WORD',):
## logical_operator_found = 1
## break
## tokens_values.append(token.value.replace("'", ""))
##
## # build expressions
## if not logical_operator_found:
## # no logical operator found so we assume that we search
## # for a combination of words
## where_expressions.append("%s LIKE '%%%s%%'" %(key, ' '.join(tokens_values)))
## else:
## # in the search string we have explicitly defined an operator
## for item in operators_mapping_list:
## row_tokens_values = []
## tokens = item['tokens']
## operator = item['operator']
## operator_value = None
## if operator is not None:
## # operator is standalone expression
## where_expressions.append('%s' %operator.value)
## if len(tokens):
## # no it's not a stand alone expression,
## # determine it from list of tokens
## sub_where_expression = ''
## tokens_number = len(tokens)
## if tokens_number == 1:
## # no left sided operator (<, >, >=, <=) found
## token = tokens[0]
## if token.type == 'WORD':
## sub_where_expression = "LIKE '%%%s%%'" %token.value
## elif token.type == 'KEYWORD':
## sub_where_expression = "LIKE '%s'" %token.value
## elif token.type == 'EXPLICITEQUALLITYWORD':
## sub_where_expression = "= '%s'" %token.value
## elif token.type == 'WORDSET' and token.value.startswith('='):
## # if WORDSET starts with '=' it's an equality
## sub_where_expression = " = '%s'" %token.value[1:]
## else:
## sub_where_expression = "LIKE '%%%s%%'" %token.value
## else:
## # we have two or more tokens, by definition first one should be
## # logical operator like (<, >, >=, <=)
## operator = tokens[0]
## operator_value = operator.value
## if operator.type in ('KEYWORD', 'WORDSET', 'WORD'):
## # no operator for this token list, assume it's 'LIKE'
## sub_where_expression = "LIKE '%s'" %' '.join([x.value for x in tokens])
## else:
## # we have operator and by convention if operator is used it's applyied to one token only
## sub_where_expression = "%s'%s'" %(operator_value, tokens[1].value)
## where_expressions.append('%s %s' %(key, sub_where_expression))
## return where_expressions, select_expressions
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from SearchKey import SearchKey
from Products.ZSQLCatalog.SearchText import parse
from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
from Interface.Verify import verifyClass
class KeywordKey(SearchKey):
"""
This SearchKey generates matching comparison Queries suited for strings
with wilcards.
"""
default_comparison_operator = 'like'
get_operator_from_value = True
def parseSearchText(self, value):
return parse(value)
verifyClass(ISearchKey, KeywordKey)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -27,23 +29,15 @@
##############################################################################
from SearchKey import SearchKey
from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
from Interface.Verify import verifyClass
class RawKey(SearchKey):
""" RawKey key is an ERP5 portal_catalog search key which is used to render
SQL expression that will match exactly what's passed to it using equality ."""
"""
This SearchKey does not do any parsing of given value.
"""
default_comparison_operator = '='
get_operator_from_value = False
def build(self, **kwargs):
# this key doesn't require parsing
# It's required to implement it as it's used ONLY for ExactMath
pass
verifyClass(ISearchKey, RawKey)
def buildSQLExpression(self, key, value,
format=None, mode=None, range_value=None, stat__=None):
if value is not None:
value = self.quoteSQLString(value, format)
key = self.quoteSQLKey(key, format)
where_expression = "%s = %s" % (key, value)
else:
where_expression = "%s is NULL" % (key)
return where_expression, []
##############################################################################
#
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from SearchKey import SearchKey
from Products.ZSQLCatalog.Query.Query import Query
from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
from Products.ZSQLCatalog.Query.SQLQuery import SQLQuery
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from zLOG import LOG
from Products.ZSQLCatalog.Interface.ISearchKey import IRelatedKey
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
MARKER = []
BACKWARD_COMPATIBILITY = True
class RelatedKey(SearchKey):
"""
This SearchKey handles searched on virtual columns of RelatedKey type.
It generates joins required by the virtual column to reach the actual
column to compare, plus a regular query on that column if needed.
"""
__implements__ = IRelatedKey
related_key_definition = None
@profiler_decorator
def _buildRelatedKey(self, related_key_definition):
"""
Extract RelatedKey parameters from its definition, and cache this
result. If related_key_definition changes since last computation, cached
values will be refreshed.
related_key_definition (string)
Describes parameters of a RelatedKey. It is composed of 3 mains parts,
separated by '/':
- a list of table names
Table names are separated by ','
- a column name
- the name of the related key ZSQLMethod
"""
assert related_key_definition is not None
if self.related_key_definition != related_key_definition:
self.related_key_definition = related_key_definition
# Extract related_key_id, column_id and table_list from related_key_definition
table_list, self.real_column, self.related_key_id = related_key_definition.split('/')
self.table_list = table_list.split(',')
@profiler_decorator
def _getSearchKey(self, sql_catalog, search_key_name):
"""
Get search key relevant to the actual column.
sql_catalog (SQLCatalog)
Used to access SearchKey provider.
search_key_name (string, None)
See SQLCatalog.getSearchKey.
"""
return sql_catalog.getSearchKey(self.real_column, search_key_name)
@profiler_decorator
def getSearchKey(self, sql_catalog, related_key_definition, search_key_name=None):
"""
Get search key relevant to the actual column, extracting information
about that column first if needed.
sql_catalog (SQLCatalog)
Used to access SearchKey provider.
related_key_definition (string)
See _buildRelatedKey.
search_key_name (string, None)
See SQLCatalog.getSearchKey.
"""
self._buildRelatedKey(related_key_definition)
return self._getSearchKey(sql_catalog, search_key_name)
@profiler_decorator
def buildQuery(self, sql_catalog, related_key_definition,
search_value=MARKER, search_key_name=None,
logical_operator=None, comparison_operator=None):
self._buildRelatedKey(related_key_definition)
if search_value is MARKER:
join_condition = None
else:
join_condition = self._getSearchKey(sql_catalog, search_key_name).buildQuery(
search_value, group=self.getColumn(),
logical_operator=logical_operator,
comparison_operator=comparison_operator)
return RelatedQuery(search_key=self,
join_condition=join_condition)
@profiler_decorator
def registerColumnMap(self, column_map, table_alias_list=None):
related_column = self.getColumn()
group = column_map.registerRelatedKey(related_column, self.real_column)
# Each table except last one must be registered to their own group, so that
# the same table can be used multiple time (and aliased multiple times)
# in the same related key. The last one must be register to related key
# "main" group (ie, the value of the "group" variable) to be the same as
# the ta ble used in join_condition.
if table_alias_list is not None:
assert len(self.table_list) == len(table_alias_list)
for table_position in xrange(len(self.table_list) - 1):
table_name = self.table_list[table_position]
local_group = column_map.registerRelatedKeyColumn(related_column, table_position, group)
column_map.registerTable(table_name, group=local_group)
if table_alias_list is not None:
# Pre-resolve all tables with given aliases
given_name, given_alias = table_alias_list[table_position]
assert table_name == given_name
column_map.resolveTable(table_name, given_alias, group=local_group)
table_name = self.table_list[-1]
column_map.registerTable(table_name, group=group)
if table_alias_list is not None:
given_name, given_alias = table_alias_list[-1]
assert table_name == given_name
column_map.resolveTable(table_name, given_alias, group=group)
# Resolve (and register) related key column in related key group with its last table.
column_map.registerColumn(self.real_column, group=group)
column_map.resolveColumn(self.real_column, table_name, group=group)
# Always register catalog, since it is always the "base" table of
# RelatedKeys.
column_map.registerCatalog()
return group
@profiler_decorator
def buildSQLExpression(self, sql_catalog, column_map, only_group_columns, group):
"""
Render RelatedKey's ZSQLMethod by providing it table aliases from
ColumnMap.
sql_catalog (SQLCatalog)
column_map (ColumnMap)
group (string)
only_group_columns (bool)
Ignored.
"""
related_key = getattr(sql_catalog, self.related_key_id)
table_alias_dict = dict(
[('table_%s' % (x, ), column_map.getTableAlias(self.table_list[x], group=column_map.getRelatedKeyGroup(x, group)))
for x in xrange(len(self.table_list) - 1)])
x = len(table_alias_dict)
assert x == len(self.table_list) - 1
table_alias_dict['table_%s' % (x, )] = column_map.getTableAlias(self.table_list[x], group=group)
rendered_related_key = related_key(
query_table=column_map.getCatalogTableAlias(),
src__=1,
**table_alias_dict)
# Important:
# Former catalog separated join condition from related query.
# Example:
# ComplexQuery(Query(title="foo"),
# Query(subordination_title="bar")
# , operator='OR')
# Former catalog rendering (truncated where-expression):
# AND ((catalog.title LIKE '%bil%') OR
# (related_catalog_1.title = 'My Organisation'))
# AND (related_catalog_1.uid = related_category_0.category_uid AND
# related_category_0.base_category_uid = 873 AND
# related_category_0.uid = catalog.uid)
# As you can see, the part of the query joining the tables is *out* of the
# OR expression, and therefor applies to the entire query.
# This was done on purpose, because doing otherwise gives very poor
# performances (on a simple data set, similar query can take *minutes* to
# execute - as of MySQL 5.x).
# Doing the same way as the former catalog is required for backward
# compatibility, until a decent alternative is found (like spliting the
# "OR" expression into ensemblist operations at query level).
# Note that doing this has a side effect on result list, as objects
# lacking a relation will never appear in the result.
if BACKWARD_COMPATIBILITY:
column_map.addJoinQuery(SQLQuery(rendered_related_key))
return None
else:
return SQLExpression(self, where_expression=rendered_related_key)
verifyClass(IRelatedKey, RelatedKey)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from Products.PythonScripts.Utility import allow_class
from SearchKey import SearchKey
from pprint import pprint
# these keys are used to build query in case for ScriptableKey
# when no key was specified in fornt of value
DEFAULT_SEARCH_KEYS = ('SearchableText', 'reference', 'title',)
class KeyMappingKey(SearchKey):
""" Usable lexer class used (internally) by ScriptableKey lexer than can parse following:
VALUE OPERATOR VALUE
Examples:
* "portal_type : Person"
* "creation_date > 2007-01-01"
"""
tokens = ('OPERATOR', 'COLONOPERATOR', 'VALUE',)
t_OPERATOR = r'>=|<=|>|<'
t_VALUE = r'[\x7F-\xFF\w\d\/~!@#$^&*()_+-][\x7F-\xFF\w\d\/~!@#$^&*()_+-]*'
def t_COLONOPERATOR(self, t):
r':'
# ':' is the same as '=' (equality)
t.value = '='
return t
class ScriptableKey(SearchKey):
""" KeyWordKey key is an ERP5 portal_catalog search key which is used to generate a
ComplexQuery instance out of an arbitrary search string.
Examples:
* "John Doe AND portal_type:Person AND creation_date > 2007-01-01"
would be turned into following ComplexQuery:
* ComplexQuery(Query(portal_type='Person'),
Query(creation_date='2007-01-01', operator='>'),
ComplexQuery(Query(searchable_text='John Doe'),
Query(title='John Doe'),
Query(reference='John Doe'),
operator='OR')
operator='AND'))
"""
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL',)
tokens = ('OR', 'AND',
'DATE', 'WORD', 'KEYMAPPING',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'EQUAL')
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
t_EQUAL = r'='
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'\s+(OR|or)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
return t
def t_AND(self, t):
r'\s+(AND|and)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
return t
def t_KEYMAPPING(self, t):
r'[^<>=:\s]+\s*(>|<|<=|>=|:)\s*\S+'
# KEYMAPPING has following format: KEY OPERATOR VALUE
# where OPERATOR in ['<', '>', '<=', '>=', ':']
# example: 'creation_date < 2007-12-12'
value = t.value.strip()
t.value = value
return t
def t_WORD(self, t):
r'[^<>=\s:]+'
# WORD may contain arbitrary letters and numbers without white space
# WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
value = t.value.strip()
t.value = value
return t
def buildQueryForTokenList(self, tokens):
""" Build a ComplexQuery for a token list """
query_list = []
for group in self.groupByLogicalOperator(tokens, 'AND'):
group_tokens = group
first_group_token = group_tokens[0]
if first_group_token.type == 'KEYMAPPING':
# user specified a full sub query definition following this format:
# 'key operator value'
sub_search_string = group_tokens[0].value
keymapping_lexer = getSearchKeyInstance(KeyMappingKey)
sub_tokens = keymapping_lexer.tokenize(sub_search_string)
sub_tokens_values = [x.value for x in sub_tokens]
search_key, search_operator, search_value = sub_tokens_values
query_kw = {search_key: search_value,
'range' : search_operator,}
query_list.append(Query( **query_kw))
elif first_group_token.type in self.sub_operators:
# user specified a incomplete sub query definition following this format:
# 'operator value'. Assume that he ment to search for 'title' and
# use supplied 'operator'
search_operator = first_group_token.value
simple_query_value = ' '.join([x.value for x in group_tokens[1:]])
query_kw = {'title': simple_query_value,
'range' : search_operator,}
query_list.append(Query( **query_kw))
else:
# user specified a VERY incomplete sub query definition following this format:
# 'value'. Let's search against most common search_keys and assume operator
# is '=' (by default) and try to get as much possible results
simple_query_value = ' '.join([x.value for x in group_tokens])
sub_query_list = []
for default_key in DEFAULT_SEARCH_KEYS:
query_kw = {default_key: simple_query_value}
sub_query_list.append(Query(**query_kw))
query_list.append(ComplexQuery(*sub_query_list,
**{'operator':'OR'}))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator':'AND'})
return complex_query
def buildQuery(self, key, value,
format=None, mode=None, range_value=None, stat__=None):
""" Build ComplexQuery from passed search string value.
When grouping expressions we use the following assumptions
that 'OR' operator has higher priority in a sense:
* "John Doe AND portal_type:Person OR creation_date>=2005/12/12"
is considered as:
* (John Doe AND portal_type:Person) OR (creation_date>=2005/12/12)"
"""
query_list = []
tokens = self.tokenize(value)
# split tokens list into one or many OR concatanated expressions
sub_tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
# get a ComplexQuery for a sub token list
for tokens_or_group in sub_tokens_or_groups:
query_list.append(self.buildQueryForTokenList(tokens_or_group))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator':'OR'})
return complex_query
allow_class(ScriptableKey)
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jean-Paul Smets-Solanes <jp@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -13,7 +15,7 @@
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License,] or (at your option) any later version.
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
......@@ -26,220 +28,322 @@
#
##############################################################################
from DocumentTemplate.DT_Var import sql_quote
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
from zLOG import LOG
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
import ply.yacc as yacc
import ply.lex as lex
class SearchKey:
""" BaseKey is a base class that implements a parser of
search grammar used in ERP5. It also implements all generic
search key class methods."""
# main logical operators
operators = ('OR', 'AND',)
default_operator = '='
# in ERP5 search grammar white space is extremely important
# so we can not ignore it.
#t_ignore = ' \t'
# no need to rack down line numbers
#def t_newline(self, t):
# r'\n+'
# #t.lexer.lineno += len(t.value)
def t_error(self, t):
#print "Illegal character '%s'" % t.value[0]
t.lexer.skip(1)
def p_error(self, p):
pass
def build(self, **kwargs):
""" This method will initialize respective search key class with
tokens' definitions. """
self.lexer = lex.lex(object = self, **kwargs)
def tokenize(self, data):
""" Return list of tokens according to respective
search key tokens' definitions. """
result = []
self.lexer.input(data)
while 1:
tok = self.lexer.token()
if not tok:
break
result.append(tok)
from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
from Interface.Verify import verifyClass
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
single_operator_dict = {
'min': '>=',
'max': '<',
'ngt': '<=',
'nlt': '>'
}
dual_operator_dict = {
'minmax': ('>=', '<'),
'minngt': ('>=', '<=')
}
# List of operators searched for at value's begnining when it's a basestring.
# Order is important: an operator whose left part would be matching another operator of lower index would never be used.
operator_list = ('>=', '<=', '>', '<', '=', '!=')
def preprocessLikeValue(value):
if '%' not in value:
value = '%%%s%%' % (value, )
return value
operator_value_preprocessor_dict = {
'like': preprocessLikeValue
}
def deprocessLikeValue(value):
assert isinstance(value, basestring)
if len(value) >= 2 and value[0] == '%' and value[-1] == '%':
value = value.strip('%')
return value
operator_value_deprocessor_dict = {
'like': deprocessLikeValue
}
class SearchKey(object):
__implements__ = ISearchKey
# Comparison operator to use when parsing a string value and no operator is
# found.
# Note: for non-string values, "=" is always used by default.
default_comparison_operator = '='
# Wether or not to allow a basestring value to be searched for a comparison
# operator.
get_operator_from_value = True
def __init__(self, column):
self.column = column
def getColumn(self):
return self.column
@profiler_decorator
def buildSQLExpression(self, operator, value, column_map, only_group_columns, group):
column_name = self.getColumn()
rendered_column = column_map.asSQLColumn(column_name, group=group)
return operator.asSQLExpression(rendered_column, value, only_group_columns)
@profiler_decorator
def _renderValueAsSearchText(self, value, operator):
"""
Render a single value as valid SearchText using provided operator.
This is also responsible for undoing any formatting the value received
from the SearchKey.
value (anything)
operator (Operator)
The operator used to render value.
"""
operator_value_deprocessor = operator_value_deprocessor_dict.get(operator.getOperator())
if operator_value_deprocessor is not None:
value = operator_value_deprocessor(value)
return operator.asSearchText(value)
@profiler_decorator
def buildSearchTextExpression(self, operator, value, column=None):
operator_text = operator.getOperator()
if column is None:
column = self.getColumn()
if isinstance(value, (list, tuple)):
assert operator_text == 'in'
assert len(value)
value = [self._renderValueAsSearchText(x, operator) for x in value]
if self.default_comparison_operator != '=':
value = ['=%s' % (x, ) for x in value]
# XXX: operator used to join value elements should be reused from parser data (?)
result = '(%s)' % (' OR '.join(value), )
else:
result = self._renderValueAsSearchText(value, operator)
if operator_text != self.default_comparison_operator:
result = '%s%s' % (operator_text, result)
if len(column):
result = '%s:%s' % (column, result)
return result
# Grouping of tokens
def getOperatorForTokenList(self, tokens):
""" Generic implementation that will return respective
operator for a token list. The first found occurence wins."""
token = tokens[0]
token_type = token.type
if token_type in self.sub_operators:
return token.value, tokens[1:]
@profiler_decorator
def registerColumnMap(self, column_map, group, simple_query):
column_map.registerColumn(self.getColumn(), group=group, simple_query=simple_query)
return group
@profiler_decorator
def _getComparisonOperator(self, value):
"""
From a basestring instance, return a contained operator and value without
that operator.
value (string)
Returns: 2-tuple of strings
First element is the operator. None if there was no operator in value.
Second element is the value without the operator.
"""
startswith = value.startswith
for operator in operator_list:
if startswith(operator):
value = value[len(operator):].lstrip()
break
else:
return self.default_operator, tokens
operator = self._guessComparisonOperator(value)
return operator, value
@profiler_decorator
def _guessComparisonOperator(self, value):
"""
From a basestring instance, return a contained operator.
Value cannot be altered in the process.
value (string)
Returns: 2-tuple of strings
First element is the operator. None if there was no operator in value.
Second element is the value without the operator.
"""
return self.default_comparison_operator
@profiler_decorator
def _preprocessValue(self, value, operator):
operator_value_preprocessor = operator_value_preprocessor_dict.get(operator)
if operator_value_preprocessor is not None:
value = operator_value_preprocessor(value)
return value
def groupByLogicalOperator(self, tokens, logical_operator ='OR'):
""" Split tokens list into one or many OR concatanated tokens list
@profiler_decorator
def _processSearchValue(self, search_value, default_logical_operator, comparison_operator):
"""
sub_tokens_or_groups = []
tmp_token_list = []
for token in tokens:
if token.type != logical_operator:
tmp_token_list.append(token)
Change search_value into a list of values, one or more logical operators,
and a comparison operator. If no default_logical_operator is given,
'or' is used.
search_value
basestring
int
dict
list or tuple
Non-empty
Composed of homogeneous items
Returns: 3-tuple
dict:
key (string)
Comparison operator
value (list of anything)
List of values applying to this operator.
string:
Logical operator applied to all elements of returned dict.
bool:
True if logical operators were searched for in values, False
otherwise. Useful to give different meanings to in-value operators
and others.
"""
if comparison_operator == '':
comparison_operator = None
get_operator_from_value = False
else:
get_operator_from_value = self.get_operator_from_value
logical_operator = None
if default_logical_operator is None:
default_logical_operator = 'or'
parsed = False
if isinstance(search_value, dict):
# comparison_operator parameter collides with dict's 'operator' key.
# Fail loudly.
assert comparison_operator is None
actual_value = search_value['query']
if search_value.get('key') not in (None, self.__class__.__name__):
LOG(self.__class__.__name__, 100, '"key" dict entry does not match current class: %r' % (search_value, ))
if 'type' in search_value:
assert 'operator' not in search_value, search_value
assert 'range' not in search_value, search_value
else:
value_operator = search_value.get('operator')
value_range = search_value.get('range')
if value_range is not None:
if value_operator is not None:
LOG('SearchKey', 100, '"range" and "operator" are mutualy exclusive, ignoring operator: %r' % (search_value, ))
if value_range in single_operator_dict:
comparison_operator = single_operator_dict[value_range]
elif value_range in dual_operator_dict:
if not isinstance(actual_value, (tuple, list)):
raise TypeError, 'Operator %r requires value to be a tuple/list. (%r)' % (value_range, search_value)
if len(actual_value) != 2:
raise TypeError, 'Operator %r requires value to have a length of 2. len(%r) = %i (%r)' % (value_range, actual_value, len(actual_value), search_value)
comparison_operator = dual_operator_dict[value_range]
logical_operator = 'and'
else:
raise ValueError, 'Unknown "range" value in %r' % (search_value, )
if value_operator is not None:
if not isinstance(value_operator, basestring):
raise TypeError, 'Operator must be of a string type. Got a %r' % (type(value_operator), )
value_operator = value_operator.lower()
if not isinstance(actual_value, (tuple, list)):
raise TypeError, 'When specifying an operator, query must be a list.'
if value_operator == 'in':
comparison_operator = '='
logical_operator = 'or'
else:
logical_operator = value_operator
search_value = actual_value
# Cast to list
if isinstance(search_value, (tuple, list)):
# Check list content (not empty, homogenous)
search_value_len = len(search_value)
if search_value_len == 0:
raise ValueError, 'Value cannot be an empty list/tuple: %r' % (search_value, )
reference_class = search_value[0].__class__
for x in search_value[1:]:
if x.__class__ != reference_class:
raise TypeError, 'List elements must be of the same class: %r' % (search_value, )
else:
sub_tokens_or_groups.append(tmp_token_list)
tmp_token_list = []
# append remainig last tokens
sub_tokens_or_groups.append(tmp_token_list)
return sub_tokens_or_groups
# SQL quoting (each search key should override them it if needed)
def quoteSQLKey(self, key, format):
""" Return a quoted string of the value. """
return key
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
return "'%s'" % sql_quote(str(value))
# SQL generation
def buildSQLExpression(self, key, value,
format=None, mode=None, range_value=None, stat__=0):
""" Generic implementation. Leave details to respective key. """
if range_value is not None:
# if range_value we handle directly (i.e no parsing of search string)
where_expressions, select_expressions = \
self.buildSQLExpressionFromRange(key, value,
format, mode, range_value, stat__)
assert logical_operator is None
if isinstance(search_value, dict):
reference_class = search_value['query'].__class__
else:
# search string parsing is needed
where_expressions, select_expressions = \
self.buildSQLExpressionFromSearchString(key, str(value),
format, mode, range_value, stat__)
return where_expressions, select_expressions
def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
complex_query = self.buildQuery(key, value, format, mode, range_value, stat__)
if complex_query is None:
# Query could not be generated from search string
sql_expression = {'where_expression': '1',
'select_expression_list': []}
reference_class = search_value.__class__
search_value = [search_value]
if logical_operator is None:
logical_operator = default_logical_operator
operator_value_dict = {}
if None in search_value:
if comparison_operator not in (None, 'is'):
LOG('KeywordKey', 100, 'None value requires an "is" comparison operator. Fixed.')
operator_value_dict['is'] = search_value
elif comparison_operator is None:
if issubclass(reference_class, basestring):
if get_operator_from_value:
parsed = True
for value in search_value:
if isinstance(value, dict):
operator, value['query'] = self._getComparisonOperator(value['query'])
else:
sql_expression = complex_query(keyword_search_keys = [],
datetime_search_keys = [],
full_text_search_keys = [])
return sql_expression['where_expression'], sql_expression['select_expression_list']
operator, value = self._getComparisonOperator(value)
operator_value_dict.setdefault(operator, []).append(self._preprocessValue(value, operator))
else:
for value in search_value:
if isinstance(value, dict):
operator = self._guessComparisonOperator(value['query'])
else:
operator = self._guessComparisonOperator(value)
operator_value_dict.setdefault(operator, []).append(self._preprocessValue(value, operator))
else:
# XXX: comparison operator is hardcoded for non-strings.
operator_value_dict['='] = search_value
elif isinstance(comparison_operator, (tuple, list)):
assert len(comparison_operator) == len(search_value)
for operator, value in zip(comparison_operator, search_value):
operator_value_dict.setdefault(operator, []).append(value)
else:
operator_value_dict[comparison_operator] = search_value
return operator_value_dict, logical_operator, parsed
def buildQuery(self, key, value, format, mode, range_value, stat__):
""" Build Query """
query_list = []
# tokenize searchs string into tokens for Search Key
tokens = self.tokenize(value)
@profiler_decorator
def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
"""
Create Queries from values, logical and comparison operators.
# split tokens list into one or more 'OR' tokens lists
tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
operator_value_dict (dict)
See _processSearchValue.
logical_operator (string)
See _processSearchValue.
parsed (bool)
See _processSearchValue.
group (string)
The gorup all queries will belong to.
"""
column = self.getColumn()
query_list = []
append = query_list.append
if logical_operator == 'or' and '=' in operator_value_dict:
# Special case for equality with an 'or' logical operator: use SQL 'in'.
append(SimpleQuery(search_key=self, operator='in', group=group, **{column: operator_value_dict.pop('=')}))
for comparison_operator, value_list in operator_value_dict.iteritems():
for value in value_list:
append(SimpleQuery(search_key=self, operator=comparison_operator, group=group, **{column: value}))
return query_list
# remove empty tokens lists
tokens_or_groups = filter(lambda x: len(x), tokens_or_groups)
@profiler_decorator
def buildQuery(self, search_value, group=None, logical_operator=None, comparison_operator=None):
assert logical_operator in (None, 'and', 'or'), repr(logical_operator)
operator_value_dict, logical_operator, parsed = self._processSearchValue(search_value, logical_operator, comparison_operator)
query_list = self._buildQuery(operator_value_dict, logical_operator, parsed, group)
if len(query_list) == 1:
query = query_list[0]
else:
query = ComplexQuery(query_list, operator=logical_operator)
return query
# get a ComplexQuery for a sub token list
for tokens_or_group in tokens_or_groups:
query = self.buildQueryForTokenList(tokens_or_group, key, value, format)
if query is not None:
# query could be generated for token list
query_list.append(query)
def parseSearchText(self, value):
return None
if len(query_list):
# join query list in one really big ComplexQuery
return ComplexQuery(*query_list,
**{'operator':'OR'})
verifyClass(ISearchKey, SearchKey)
def buildQueryForTokenList(self, tokens, key, value, format):
""" Build a ComplexQuery for a token list """
query_list = []
logical_groups = self.groupByLogicalOperator(tokens, 'AND')
for group_tokens in logical_groups:
token_values = [x.value for x in group_tokens]
sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
sub_tokens_values = [x.value for x in sub_tokens]
query_kw = {key: ' '.join(sub_tokens_values),
'type': self.default_key_type,
'format': format,
'range': sub_operator}
query_list.append(Query(**query_kw))
# join query list in one really big ComplexQuery
complex_query = ComplexQuery(*query_list,
**{'operator': 'AND'})
return complex_query
def buildSQLExpressionFromRange(self, key, value, format, mode, range_value, stat__):
""" This method will generate SQL expressions
from explicitly passed list of values and
range_value in ('min', 'max', ..)"""
key = self.quoteSQLKey(key, format)
where_expression = ''
select_expressions = []
if isinstance(value, (list, tuple)):
if len(value) > 1:
# value should contain at least two items
query_min = self.quoteSQLString(value[0], format)
query_max = self.quoteSQLString(value[1], format)
else:
# value contains only one item
query_min = query_max = self.quoteSQLString(value[0], format)
else:
query_min = query_max = self.quoteSQLString(value, format)
if range_value == 'min':
where_expression = "%s >= %s" % (key, query_min)
elif range_value == 'max':
where_expression = "%s < %s" % (key, query_max)
elif range_value == 'minmax' :
where_expression = "%s >= %s AND %s < %s" % (key, query_min, key, query_max)
elif range_value == 'minngt' :
where_expression = "%s >= %s AND %s <= %s" % (key, query_min, key, query_max)
elif range_value == 'ngt':
where_expression = "%s <= %s" % (key, query_max)
elif range_value == 'nlt':
where_expression = "%s > %s" % (key, query_max)
elif range_value == 'like':
where_expression = "%s LIKE %s" % (key, query_max)
elif range_value == 'not_like':
where_expression = "%s NOT LIKE %s" % (key, query_max)
elif range_value in ('=', '>', '<', '>=', '<=','!=',):
where_expression = "%s %s %s" % (key, range_value, query_max)
return where_expression, select_expressions
## def groupByOperator(self, tokens, group_by_operators_list = operators):
## """ Generic implementation of splitting tokens into logical
## groups defided by respective list of logical operator
## defined for respective search key. """
## items = []
## last_operator = None
## operators_mapping_list = []
## last_operator = {'operator': None,
## 'tokens': []}
## for token in tokens:
## token_type = token.type
## token_value = token.value
## if token_type in group_by_operators_list:
## # (re) init it
## last_operator = {'operator': token,
## 'tokens': []}
## operators_mapping_list.append(last_operator)
## else:
## # not an operator just a value token
## last_operator['tokens'].append(token)
## if last_operator not in operators_mapping_list:
## operators_mapping_list.append(last_operator)
## return operators_mapping_list
# Automaticaly import all SearchKeys
import os
module_path = os.path.dirname(os.path.abspath(__file__))
global_dict = globals()
__relative_file__ = os.path.basename(__file__)
for filename in os.listdir(module_path):
if filename.endswith('.py') and filename != __relative_file__:
modulename = filename[:-3]
try:
module = __import__(modulename, global_dict, None, [])
except ImportError:
continue
##############################################################################
#
# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
# Ivan Tyagov <ivan@nexedi.com>
# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
......@@ -13,7 +13,7 @@
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License,] or (at your option) any later version.
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
......@@ -26,67 +26,85 @@
#
##############################################################################
from SearchKey import SearchKey
from lexer import lexer, update_docstrings
class FloatKey(SearchKey):
""" FloatKey key is an ERP5 portal_catalog search key which is used to render
float like SQL expression.
"""
# default type of sub Queries to be generated out fo a search string
default_key_type = 'float'
# TODO: remove the special OPERATOR case: it does not work when there are both a valid and an invalid operator
tokens = ('OR', 'AND', 'NOT', 'FLOAT',
'GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL')
class AdvancedSearchTextDetector(lexer):
sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
def t_OPERATOR(self, t):
r'(>=?|<=?|!?=)[ ]*'
return t
def t_LEFT_PARENTHESE(self, t):
self.found = True
t.type = 'WORD'
return t
def t_STRING(self, t):
self.found = True
t.type = 'WORD'
return t
def t_COLUMN(self, t):
self.found = True
t.type = 'WORD'
return t
# Note: Order of placing rules (t_WORD for example) is very important
def t_OR(self, t):
r'\s+(OR|or)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'OR'
self.found = True
t.type = 'WORD'
return t
def t_AND(self, t):
r'\s+(AND|and)\s+'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = 'AND'
self.found = True
t.type = 'WORD'
return t
def t_NOT(self, t):
r'(\s+(NOT|not)\s+|!=)'
# operator must have leading and trailing ONLY one white space character
# otherwise it's treated as a WORD
t.value = '!='
self.found = True
t.type = 'WORD'
return t
t_GREATERTHANEQUAL = r'>='
t_LESSTHANEQUAL = r'<='
t_GREATERTHAN = r'>'
t_LESSTHAN = r'<'
def p_search_text(self, p):
'''search_text : value
| value search_text'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[1] or p[2]
def t_FLOAT(self, t):
r'[\d.][\d.]*'
# FLOAT is a float number
value = t.value.replace('"', '').strip()
t.value = value
return t
def p_value(self, p):
'''value : WORD
| OPERATOR WORD'''
p[0] = len(p) == 3 and ' ' not in p[1]
tokens = (
'WORD',
'OPERATOR')
def real_token(self):
return lexer.token(self)
def token(self):
return self.token_list.pop(0)
def __call__(self, input):
self.found = False
check_grammar = False
self.token_list = token_list = []
append = token_list.append
self.input(input)
while not self.found:
token = self.real_token()
append(token)
if token is None:
break
if token.type == 'OPERATOR':
check_grammar = True
if not self.found and check_grammar:
self.found = self.parse()
return self.found
update_docstrings(AdvancedSearchTextDetector)
def quoteSQLString(self, value, format):
""" Return a quoted string of the value. """
# Make sure there is no space in float values
return "'%s'" %str(value).replace(' ', '')
def quoteSQLKey(self, key, format):
""" Return a quoted string of the value. """
if format is not None:
float_format = format.replace(' ', '')
if float_format.find('.') >= 0:
precision = len(float_format.split('.')[1])
key = "TRUNCATE(%s,%s)" % (key, precision)
return key
##############################################################################
#
# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from lexer import lexer, update_docstrings
try:
from Products.ZSQLCatalog.Interface.IAbstractSyntaxNode import INode, IValueNode, ILogicalNode, IColumnNode
from Interface.Verify import verifyClass
except ImportError:
INode = None
IValueNode = None
ILogicalNode = None
IColumnNode = None
def verifyClass(*args, **kw):
pass
class Node(object):
__implements__ = INode
def isLeaf(self):
return False
def isColumn(self):
return False
def push(self, logical_operator, node):
return LogicalNode(logical_operator, self, node)
verifyClass(INode, Node)
class ValueNode(Node):
__implements__ = IValueNode
def __init__(self, value, comparison_operator=''):
self.value = value
self.comparison_operator = comparison_operator
def isLeaf(self):
return True
def getComparisonOperator(self):
return self.comparison_operator
def getValue(self):
return self.value
def __repr__(self):
return '<%s %r %r>' % (self.__class__.__name__, self.comparison_operator, self.value)
verifyClass(INode, ValueNode)
verifyClass(IValueNode, ValueNode)
class NotNode(Node):
__implements__ = ILogicalNode
def __init__(self, node):
self.node = node
def getLogicalOperator(self):
return 'not'
def getNodeList(self):
return [self.node]
def __repr__(self):
return '<%s %r>' % (self.__class__.__name__, self.node)
verifyClass(INode, NotNode)
verifyClass(ILogicalNode, NotNode)
class LogicalNode(Node):
__implements__ = ILogicalNode
def __init__(self, logical_operator, node, other):
self.logical_operator = logical_operator
self.node_list = []
self._push(node)
self._push(other)
def getLogicalOperator(self):
return self.logical_operator
def getNodeList(self):
return self.node_list
def _push(self, node):
if isinstance(node, LogicalNode) and node.logical_operator == self.logical_operator:
self.node_list.extend(node.node_list)
else:
self.node_list.append(node)
def __repr__(self):
return '<%s %r %r>' % (self.__class__.__name__, self.logical_operator, self.node_list)
verifyClass(INode, LogicalNode)
verifyClass(ILogicalNode, LogicalNode)
class ColumnNode(Node):
__implements__ = IColumnNode
def __init__(self, column_name, node):
self.column_name = column_name
self.node = node
def isColumn(self):
return True
def getColumnName(self):
return self.column_name
def getSubNode(self):
return self.node
def __repr__(self):
return '<%s %r %r>' % (self.__class__.__name__, self.column_name, self.node)
verifyClass(INode, ColumnNode)
verifyClass(IColumnNode, ColumnNode)
class AdvancedSearchTextParser(lexer):
def p_seach_text(self, p):
'''search_text : and_expression
| and_expression OR search_text
| and_expression search_text'''
if len(p) == 2:
p[0] = p[1]
elif len(p) == 3:
p[0] = p[1].push('or', p[2])
else:
p[0] = p[1].push('or', p[3])
def p_and_expression(self, p):
'''and_expression : boolean_expression
| boolean_expression AND and_expression'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[1].push('and', p[3])
def p_boolean_expression(self, p):
'''boolean_expression : NOT expression
| expression'''
if len(p) == 3:
p[0] = NotNode(p[2])
else:
p[0] = p[1]
def p_expression(self, p):
'''expression : LEFT_PARENTHESE search_text RIGHT_PARENTHESE
| column
| value'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[2]
def p_column(self, p):
'''column : COLUMN value_expression'''
p[0] = ColumnNode(p[1], p[2])
def p_value_expression(self, p):
'''value_expression : LEFT_PARENTHESE value_or_expression RIGHT_PARENTHESE
| value'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[2]
def p_value_or_expression(self, p):
'''value_or_expression : value_and_expression
| value_and_expression value_or_expression
| value_and_expression OR value_or_expression'''
if len(p) == 2:
p[0] = p[1]
elif len(p) == 3:
p[0] = p[1].push('or', p[2])
else:
p[0] = p[1].push('or', p[3])
def p_value_and_expression(self, p):
'''value_and_expression : value_expression
| value_expression AND value_and_expression'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[1].push('and', p[3])
def p_value(self, p):
'''value : OPERATOR STRING
| OPERATOR WORD
| STRING
| WORD'''
if len(p) == 2:
p[0] = ValueNode(p[1])
else:
p[0] = ValueNode(p[2], comparison_operator=p[1])
update_docstrings(AdvancedSearchTextParser)
#!/usr/bin/python
# -*- coding: utf8 -*-
##############################################################################
#
# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import threading
from AdvancedSearchTextDetector import AdvancedSearchTextDetector
from AdvancedSearchTextParser import AdvancedSearchTextParser
from lexer import ParserOrLexerError
try:
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
except ImportError:
def profiler_decorator(funct):
return funct
if __name__ == '__main__':
DEBUG = 1
else:
DEBUG = 0
parser_pool = threading.local()
@profiler_decorator
def getAdvancedSearchTextDetector():
try:
return parser_pool.advanced_search_text_detector
except AttributeError:
advanced_search_text_detector = AdvancedSearchTextDetector()
advanced_search_text_detector.init(debug=DEBUG)
parser_pool.advanced_search_text_detector = advanced_search_text_detector
return advanced_search_text_detector
@profiler_decorator
def getAdvancedSearchTextParser():
try:
return parser_pool.parser
except AttributeError:
parser = AdvancedSearchTextParser()
parser.init(debug=DEBUG)
parser_pool.parser = parser
return parser
@profiler_decorator
def _parse(input, *args, **kw):
if getAdvancedSearchTextDetector()(input):
result = getAdvancedSearchTextParser()(input, *args, **kw)
else:
result = None
return result
@profiler_decorator
def parse(*args, **kw):
try:
result = _parse(*args, **kw)
except (KeyboardInterrupt, SystemExit):
raise
except:
result = None
return result
if __name__ == '__main__':
class Query:
def __init__(self, column, value, comparison_operator='='):
self.column = column
self.comparison_operator = comparison_operator
if isinstance(value, (list, tuple)):
value = ''.join(value)
self.value = value
def asTuple(self):
return (self.column, self.value, self.comparison_operator)
def __repr__(self):
value = self.value
if len(value) == 1:
value = value[0]
return 'Query(%r, %r, %r)' % (self.column, value, self.comparison_operator)
def __eq__(self, other):
if isinstance(other, Query):
return self.asTuple() == other.asTuple()
else:
return False
def __ne__(self, other):
return not (self == other)
class ComplexQuery:
def __init__(self, query_list, operator):
self.operator = operator
self.query_list = query_list
def __repr__(self):
return 'ComplexQuery(%r, operator=%r)' % (self.query_list, self.operator)
def __eq__(self, other):
if isinstance(other, ComplexQuery):
if self.operator != other.operator:
return False
other_query_list = other.query_list[:]
for my_query in self.query_list:
for other_index in xrange(len(other_query_list)):
other_query = other_query_list[other_index]
if my_query == other_query:
other_query_list.pop(other_index)
break
else:
return False
return len(other_query_list) == 0
else:
return False
def __ne__(self, other):
return not (self == other)
check_list = [
('foo', None),
('foo bar', None),
('foo bar', None),
('foo%', None),
('%foo', None),
('%foo%', None),
('foo%bar', None),
('foo% bar', None),
('foo %bar', None),
('foo and bar', None),
('foo or bar', None),
('foo - bar', None),
('foo- bar', None),
('!-1', None),
('->1', None),
('+=1', None),
('jean-paul', None),
('JeanAndPaul', None),
('totoORtata', None),
('NORD', None),
('OR ARGENT', None),
('CUIVRE OR ARGENT', None), # XXX
('title :foo', None),
('-foo', None),
('foo -bar', None),
('+foo -bar', None),
('+1', None),
('-1', None),
('foo OR "-" OR bar OR -baz', ComplexQuery([Query(None, 'foo'), Query(None, '-'), Query(None, 'bar'), Query(None, '-baz')], operator='or')),
('foo "-" bar -baz', ComplexQuery([Query(None, 'foo'), Query(None, '-'), Query(None, 'bar'), Query(None, '-baz')], operator='or')),
('title:foo', Query('title', 'foo')),
('title: foo', Query('title', 'foo')),
('title:foo bar', ComplexQuery([Query('title', 'foo'), Query(None, 'bar')], operator='or')),
('title:"foo bar"', Query('title', 'foo bar')),
('"title:foo bar"', Query(None, 'title:foo bar')),
('"foo bar"', Query(None, 'foo bar')),
('"foo bar"', Query(None, 'foo bar')),
('foo AND bar', ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='and')),
('foo OR bar', ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='or')),
('"foo AND bar"', Query(None, 'foo AND bar')),
('"foo and bar"', Query(None, 'foo and bar')),
('"foo OR bar"', Query(None, 'foo OR bar')),
('"foo or bar"', Query(None, 'foo or bar')),
('"foo% bar"', Query(None, 'foo% bar')),
('"foo %bar"', Query(None, 'foo %bar')),
('>1', Query(None, '1', '>')),
('">1"', Query(None, '>1')),
('>a', Query(None, 'a', '>')),
('">a"', Query(None, '>a')),
('>1 0', ComplexQuery([Query(None, '1', '>'), Query(None, '0')], operator='or')),
('>=1', Query(None, '1', '>=')),
('>"=1"', Query(None, '=1', '>')),
('-"1"', ComplexQuery([Query(None, '-'), Query(None, '1')], operator='or')),
('"!-1"', Query(None, '!-1')),
# (r"a:'tu:\'tu\''", ['a', "tu:'tu'"]),
(r'''b:"tu:\'tu\'"''', Query('b', "tu:\\'tu\\'")),
(r'''c:"tu:'tu'"''', Query('c', "tu:'tu'")),
(r'd:"tu:\"tu\""', Query('d', 'tu:"tu"')),
('toto: tutu tutu', ComplexQuery([Query('toto', 'tutu'), Query(None, 'tutu')], operator='or')),
('(tutu) (toto:tata)', ComplexQuery([Query(None, 'tutu'), Query('toto', 'tata')], operator='or')),
('(tutu) (toto:"tata")', ComplexQuery([Query(None, 'tutu'), Query('toto', 'tata')], operator='or')),
# ('toto:', ['toto', '']),
('toto:""', Query('toto', '')),
# ("''", ''),
('""', Query(None, '')),
(r'"\""', Query(None, '"')),
(r'"\n"', Query(None, '\\n')),
#こんにちは
(u'ん', None),
(u'(toto:ん) OR (titi:ん)', ComplexQuery([Query('toto', u'ん'), Query('titi', u'ん')], operator='or')),
('ん', None),
('(toto:ん) OR (titi:ん)', ComplexQuery([Query('toto', 'ん'), Query('titi', 'ん')], operator='or')),
('(foo)', Query(None, 'foo')),
('toto:(foo)', Query('toto', 'foo')),
('(foo OR bar)', ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='or')),
('(a AND b) OR (c AND (d OR e))',
ComplexQuery([ComplexQuery([Query(None, 'a'), Query(None, 'b')], operator='and'), ComplexQuery([Query(None, 'c'), ComplexQuery([Query(None, 'd'), Query(None, 'e')], operator='or')], operator='and')], operator='or')),
('(foo:"") (bar:baz)', ComplexQuery([Query('foo', ''), Query('bar', 'baz')], operator='or')),
('(foo:"") (OR:bar)', ComplexQuery([Query('foo', ''), Query('OR', 'bar')], operator='or')),
# ('foo: OR', ['foo', 'or']),
# ('foo: OR ', ['foo', 'or']),
# ('(foo:)', ['foo', '']),
('(foo: bar)', Query('foo', 'bar')),
('(a:b) AND (c:d)', ComplexQuery([Query('a', 'b'), Query('c', 'd')], operator='and')),
('a:(b c)', ComplexQuery([Query('a', 'b'), Query('a', 'c')], operator='or')),
('a:(b OR c)', ComplexQuery([Query('a', 'b'), Query('a', 'c')], operator='or')),
('a:(b c d)', ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
('a:(b (c d))', ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
('a:(b OR (c d))', ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
('"JeanANDPaul"', Query(None, 'JeanANDPaul')),
('"Jean" AND "Paul"', ComplexQuery([Query(None, 'Jean'), Query(None, 'Paul')], operator='and')),
('"jean paul" OR "thierry"', ComplexQuery([Query(None, 'jean paul'), Query(None, 'thierry')], operator='or')),
('title:Paul Jean Lili', ComplexQuery([Query('title', 'Paul'), Query(None, 'Jean'), Query(None, 'Lili')], operator='or')),
('toto AND titi OR tutu AND tata OR toto',
ComplexQuery([ComplexQuery([Query(None, 'toto'), Query(None, 'titi')], operator='and'), ComplexQuery([Query(None, 'tutu'), Query(None, 'tata')], operator='and'), Query(None, 'toto')], operator='or')),
('toto AND (titi OR tutu) AND tata OR toto',
ComplexQuery([ComplexQuery([Query(None, 'toto'), ComplexQuery([Query(None, 'titi'), Query(None, 'tutu')], operator='or'), Query(None, 'tata')], operator='and'), Query(None, 'toto')], operator='or')),
('"OR ARGENT"', Query(None, 'OR ARGENT')),
('1 AND 2 OR 3', ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3')], operator='or')),
('1 OR 2 AND 3', ComplexQuery([Query(None, '1'), ComplexQuery([Query(None, '2'), Query(None, '3')], operator='and')], operator='or')),
('1 AND 2 3', ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3')], operator='or')),
('1 2 AND 3', ComplexQuery([Query(None, '1'), ComplexQuery([Query(None, '2'), Query(None, '3')], operator='and')], operator='or')),
('10 11 OR 12 13', ComplexQuery([Query(None, '10'), Query(None, '11'), Query(None, '12'), Query(None, '13')], operator='or')),
('((1 AND 2 OR 3) OR (4 AND 5 6) OR (7 8 AND 9) OR (10 11 OR 12 13))',
ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3'), ComplexQuery([Query(None, '4'), Query(None, '5')], operator='and'), Query(None, '6'), Query(None, '7'), ComplexQuery([Query(None, '8'), Query(None, '9')], operator='and'), Query(None, '10'), Query(None, '11'), Query(None, '12'), Query(None, '13')], operator='or')),
('((titi:foo) AND (toto:bar)) OR ((titi:bar) AND (toto:foo))',
ComplexQuery([ComplexQuery([Query('titi', 'foo'), Query('toto', 'bar')], operator='and'), ComplexQuery([Query('titi', 'bar'), Query('toto', 'foo')], operator='and')], operator='or')),
('title:(Paul Jean OR Lili)', ComplexQuery([Query('title', 'Paul'), Query('title', 'Jean'), Query('title', 'Lili')], operator='or')),
('title:Paul Jean OR Lili', ComplexQuery([Query('title', 'Paul'), Query(None, 'Jean'), Query(None, 'Lili')], operator='or')),
]
def walk(node, key=None):
"""
Recusrively walk given AST and build ComplexQuery & Query instances for each node.
"""
if node.isLeaf():
comparison_operator = node.getComparisonOperator()
if comparison_operator == '':
comparison_operator = '='
result = Query(key, node.getValue(), comparison_operator=comparison_operator)
elif node.isColumn():
result = walk(node.getSubNode(), node.getColumnName())
else:
query_list = [walk(x, key) for x in node.getNodeList()]
operator = node.getLogicalOperator()
if operator == 'not' or len(query_list) > 1:
result = ComplexQuery(query_list, operator=operator)
elif len(query_list) == 1:
result = query_list[0]
else:
result = None
return result
original_parse = _parse
def parse(input, *args, **kw):
"""
Parse input and walk generated AST.
"""
result = original_parse(input, *args, **kw)
if result is not None:
#print repr(result)
result = walk(result)
return result
success_count = 0
for input, expected in check_list:
try:
result = parse(input)
except ParserOrLexerError, message:
print "ERROR when checking %r" % (input, )
print " crashed with: %s" % (message, )
print " instead of producing %r" % (expected, )
else:
if result != expected:
print "ERROR when checking %r:" % (input, )
print " produced %r" % (result, )
print " instead of %r" % (expected, )
else:
success_count += 1
print '%i/%i checks succeeded.' % (success_count, len(check_list))
while 1:
try:
input = raw_input('catalog> ')
except (EOFError, KeyboardInterrupt):
break
print repr(input)
try:
try:
detector_result = getAdvancedSearchTextDetector()(input)
except ParserOrLexerError, message:
print ' Detector raise: %r' % (message, )
detector_result = False
else:
print ' Detector: %r' % (detector_result, )
if detector_result:
print ' LEX:'
lexer = getAdvancedSearchTextParser().lexer
lexer.input(input)
while 1:
tok = lexer.token()
if not tok: break # No more input
print ' %s' % (tok, )
print ' YACC:'
print ' %r' % (parse(input, debug=2), )
else:
print ' %r' % (input, )
except ParserOrLexerError, message:
print message
print
from SearchTextParser import parse
##############################################################################
#
# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from ply import lex, yacc
import os
import sys
from cStringIO import StringIO
try:
from zLOG import LOG
except ImportError:
def LOG(channel, level, message):
print >>sys.stderr, message
module_path = os.path.dirname(os.path.abspath(__file__))
class ParserOrLexerError(Exception):
pass
class LexerError(ParserOrLexerError):
pass
class ParserError(ParserOrLexerError):
pass
class lexer(object):
def init(self, **kw):
debug = kw.pop('debug', False)
# Catch all logs with a cStringIO
output = sys.stdout = sys.stderr = StringIO()
self.lexer = lex.lex(object=self, **kw)
self.parser = yacc.yacc(module=self, debug=debug,
debugfile="%s.out" % (self.__class__.__name__, ),
tabmodule="%s_parsetab" % (self.__class__.__name__, ),
outputdir=module_path)
sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__
# Emit all logs with regular Zope logging
for line in output.getvalue().split('\n'):
if len(line):
LOG('lexer', 0, line)
def t_error(self, t):
raise LexerError, 'ERROR: Illegal character %r' % (t.value[0], )
def p_error(self, p):
raise ParserError, 'Syntax error in input: %r' % (p, )
def input(self, string):
self.lexer.input(string)
def token(self):
return self.lexer.token()
tokens = (
'OR',
'AND',
'NOT',
'COLUMN',
'STRING',
'WORD',
'OPERATOR',
'LEFT_PARENTHESE',
'RIGHT_PARENTHESE')
t_ignore = ' '
def t_LEFT_PARENTHESE(self, t):
r'\('
return t
def t_RIGHT_PARENTHESE(self, t):
r'\)'
return t
def t_OPERATOR(self, t):
r'(>=?|<=?|!?=)'
return t
def t_STRING(self, t):
r'"(\\.|[^\\"])*"'
# Unescape value and strip surrounding quotes
value_list = []
append = value_list.append
escaped = False
for char in t.value[1:-1]:
if escaped:
escaped = False
if char != '"':
append('\\')
else:
if char == '\\':
escaped = True
continue
append(char)
assert not escaped
t.value = ''.join(value_list)
return t
def t_COLUMN(self, t):
r'[^><= :\(\)"][^ :\(\)"]*:'
t.value = t.value[:-1]
return t
def t_OR(self, t):
r'OR'
return t
def t_AND(self, t):
r'AND'
return t
def t_NOT(self, t):
r'NOT'
return t
def t_WORD(self, t):
r'[^><= :\(\)"][^ :\(\)"]*'
return t
def parse(self, *args, **kw):
kw['lexer'] = self
return self.parser.parse(*args, **kw)
__call__ = parse
def update_docstrings(klass):
for property in dir(klass):
if property.startswith('t_'):
source = getattr(lexer, property, None)
if callable(source):
destination = getattr(klass, property)
assert callable(destination)
if destination.__doc__ is None:
destination.im_func.__doc__ = source.__doc__
......@@ -790,8 +790,9 @@ class ZCatalog(Folder, Persistent, Implicit):
except ConflictError:
raise
except:
raise
LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
failed_object_list.append(obj)
failed_object_list.append(obj) # XXX Strange JPS - why LOG and keep on ?? wrap_obj not defined
# run activity or execute for each archive depending on priority
if len(catalog_dict):
......@@ -1043,7 +1044,7 @@ class ZCatalog(Folder, Persistent, Implicit):
security.declarePublic('buildSqlQuery')
buildSqlQuery = buildSQLQuery
def searchResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
def searchResults(self, REQUEST=None, sql_catalog_id=None, **kw):
"""
Search the catalog according to the ZTables search interface.
Search terms can be passed in the REQUEST or as keyword
......@@ -1051,18 +1052,18 @@ class ZCatalog(Folder, Persistent, Implicit):
"""
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return apply(catalog.searchResults, (REQUEST,used), kw)
return apply(catalog.searchResults, (REQUEST, ), kw)
return []
__call__=searchResults
def countResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
def countResults(self, REQUEST=None, sql_catalog_id=None, **kw):
"""
Counts the number of items which satisfy the query defined in kw.
"""
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return apply(catalog.countResults, (REQUEST,used), kw)
return apply(catalog.countResults, (REQUEST, ), kw)
return []
## this stuff is so the find machinery works
......
......@@ -44,3 +44,6 @@ def initialize(context):
from AccessControl import ModuleSecurityInfo, ClassSecurityInfo
ModuleSecurityInfo('Products.ZSQLCatalog.SQLCatalog').declarePublic(
'ComplexQuery', 'Query', 'NegatedQuery',)
from Query import Query, SimpleQuery
from SearchKey import SearchKey
\ No newline at end of file
##############################################################################
#
# Copyright (c) 2006-2009 Nexedi SA and Contributors. All Rights Reserved.
# Jerome Perrin <jerome@nexedi.com>
# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
import sys
from DateTime import DateTime
from Products.ZSQLMethods.SQL import SQL as ZSQLMethod
from Products.CMFCore.Expression import Expression
from Products.ZSQLCatalog.SQLCatalog import Catalog as SQLCatalog
from Products.ZSQLCatalog.ZSQLCatalog import ZCatalog as ZSQLCatalog
from Products.ZSQLCatalog.SQLCatalog import Query
from Products.ZSQLCatalog.SQLCatalog import ComplexQuery
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery
from Products.ZSQLCatalog.SQLCatalog import NegatedQuery
from Products.ZSQLCatalog.Query.Query import Query as _Query
from Products.ZSQLCatalog.Query.EntireQuery import EntireQuery
from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
from DateTime import DateTime
class ReferenceQuery:
operator = None
column = None
value = None
def __init__(self, *args, **kw):
self.operator = kw.pop('operator', None)
assert len(args) == 0 or len(kw) == 0
self.args = []
for arg in args:
if isinstance(arg, (tuple, list)):
self.args.extend(arg)
else:
self.args.append(arg)
if len(kw) == 1:
self.column, self.value = kw.items()[0]
elif len(kw) > 1:
raise ValueError, 'kw must not have more than one item: %r' % (kw, )
def __eq__(self, other):
if isinstance(other, SimpleQuery):
return self.column is not None and \
other.getColumn() == self.column and \
other.getValue() == self.value and \
other.operator == self.operator
elif isinstance(other, ComplexQuery):
if not (len(other.query_list) == len(self.args) and \
other.logical_operator == self.operator):
return False
other_query_list = other.query_list[:]
for subquery in self.args:
for other_query_id in xrange(len(other_query_list)):
other_query = other_query_list[other_query_id]
if subquery == other_query:
other_query_list.pop(other_query_id)
break
else:
return False
return len(other_query_list) == 0
elif isinstance(other, EntireQuery):
return len(self.args) == 1 and \
self.args[0] == other.query
elif isinstance(other, RelatedQuery):
return self == other.join_condition
elif isinstance(other, Query):
return self == other.wrapped_query
else:
raise TypeError, 'Compared value is not a (known) Query instance: (%s) %r' % (other.__class__.__name__, other)
def __repr__(self):
return '<%s column=%r operator=%r value=%r args=%r>' % \
(self.__class__.__name__, self.column, self.operator, self.value, self.args)
class RelatedReferenceQuery:
def __init__(self, reference_subquery):
self.subquery = reference_subquery
def __eq__(self, other):
return isinstance(other, RelatedQuery) and \
self.subquery == other.join_condition
def __repr__(self):
return '<%s %r>' % (self.__class__.__name__, self.subquery)
class DummyCatalog(SQLCatalog):
"""
Mimic a table stucture.
"""
sql_catalog_keyword_search_keys = ('keyword', )
sql_catalog_datetime_search_keys = ('date', )
def getColumnMap(self):
return {
'uid': ['foo', 'bar'],
'default': ['foo', ],
'keyword': ['foo', ],
'date': ['foo', ],
'other_uid': ['bar', ]
}
def getSQLCatalogRelatedKeyList(self, key_list):
return [
'related_default | bar,foo/default/z_related_table',
'related_keyword | bar,foo/keyword/z_related_table',
'related_date | bar,foo/date/z_related_table'
]
def z_related_table(self, *args, **kw):
assert kw.get('src__', False)
assert 'query_table' in kw
assert 'table_0' in kw
assert 'table_1' in kw
assert len(kw) == 4
return '%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s' % kw
class TestSQLCatalog(unittest.TestCase):
def setUp(self):
self._catalog = DummyCatalog('dummy_catalog')
def assertCatalogRaises(self, exception, kw):
self.assertRaises(exception, self._catalog, src__=1, query_table='foo', **kw)
def catalog(self, reference_tree, kw, check_search_text=True):
reference_param_dict = self._catalog._queryResults(query_table='foo', **kw)
query = self._catalog.buildQuery(kw)
self.assertEqual(reference_tree, query)
search_text = query.asSearchTextExpression(self._catalog)
if check_search_text:
# XXX: sould "keyword" be always used for search text searches ?
search_text_param_dict = self._catalog._queryResults(query_table='foo', keyword=search_text)
self.assertEqual(reference_param_dict, search_text_param_dict,
'Query: %r\nSearchText: %r\nReference: %r\nSecond rendering: %r' % \
(query, search_text, reference_param_dict, search_text_param_dict))
def test_001_DefaultKey(self):
for column in ('default', 'related_default'):
self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='a'), operator='and'),
{column: 'a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', default='%a'), operator='and'),
{column: '%a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='<', default='a'), operator='and'),
{column: '<a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='<', default='path'), operator='and'),
{column: {'query': 'path', 'range': 'max'}})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', default=['a', 'b']), operator='and'),
{column: ['a', 'b']})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', default=['a', 'b']), operator='and'),
{column: ['=a', '=b']})
def test_002_keyOverride(self):
self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='%a'), operator='and'),
{'default': {'query': '%a', 'key': 'ExactMatch'}},
check_search_text=False)
self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='<a'), operator='and'),
{'default': {'query': '<a', 'key': 'ExactMatch'}},
check_search_text=False)
def test_003_DateTimeKey(self):
for column in ('date', 'related_date'):
self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21')), operator='and'),
{column: {'query': '>2008/10/01 12:10:20', 'format': '%y/%m/%d'}})
self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21 CEST')), operator='and'),
{column: {'query': '>2008/10/01 12:10:20 CEST', 'format': '%y/%m/%d'}})
self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21 CET')), operator='and'),
{column: {'query': '>2008/10/01 12:10:20 CET', 'format': '%y/%m/%d'}})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
, operator='and'), operator='and'),
{column: '2008/10/01 UTC'})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='>=', date=DateTime('2008/01/01 UTC')),
ReferenceQuery(operator='<', date=DateTime('2009/01/01 UTC'))
, operator='and'), operator='and'),
{column: '2008 UTC'})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='>=', date=DateTime('2008/01/01 UTC')),
ReferenceQuery(operator='<', date=DateTime('2008/02/01 UTC'))
, operator='and'), operator='and'),
{column: '2008/01 UTC'})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
, operator='and'), operator='and'),
{column: {'type': 'date', 'query': '10/01/2008 UTC', 'format': '%m/%d/%Y'}})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
, operator='and'), operator='and'),
{column: {'type': 'date', 'query': '01/10/2008 UTC', 'format': '%d/%m/%Y'}})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', date=[DateTime('2008/01/10 UTC'), DateTime('2008/01/09 UTC')]), operator='and'),
{column: {'query': ['2008/01/10 UTC', '2008/01/09 UTC'], 'operator': 'in'}},
check_search_text=False)
self.catalog(ReferenceQuery(ReferenceQuery(operator='>', date=DateTime('2008/01/10 UTC')), operator='and'),
{column: {'query': '2008/01/10 UTC', 'range': 'nlt'}},
check_search_text=False)
def test_004_KeywordKey(self):
for column in ('keyword', 'related_keyword'):
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a%'), operator='and'),
{column: 'a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a'), operator='and'),
{column: '%a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='a'), operator='and'),
{column: '<a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='path'), operator='and'),
{column: {'query': 'path', 'range': 'max'}})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='like', keyword='%a%'),
ReferenceQuery(operator='like', keyword='%b%')
, operator='or'), operator='and'),
{column: ['a', 'b']})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
{column: ['=a', '=b']})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='like', keyword='%a%'),
ReferenceQuery(operator='<', keyword='b')
, operator='or'), operator='and'),
{column: ['a', '<b']})
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='like', keyword='%a%'),
ReferenceQuery(operator='like', keyword='%b')
, operator='or'), operator='and'),
{column: ['a', '%b']})
def test_005_SearchText(self):
self.catalog(ReferenceQuery(ReferenceQuery(ReferenceQuery(operator='like', keyword='%=a%'), ReferenceQuery(operator='like', keyword='%=b%'), operator='or'), operator='and'),
{'keyword': '"=a" OR "=b"'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
{'keyword': '="a" OR ="b"'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
{'keyword': '=a OR =b'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b', 'c']), operator='and'),
{'keyword': '=a OR =b OR =c'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a%'), operator='and'),
{'keyword': 'keyword:a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='a'), operator='and'),
{'keyword': 'default:a'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a b%'), operator='and'),
{'keyword': 'a b'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%=a OR =b%'), operator='and'),
{'keyword': '"=a OR =b"'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='=', keyword='=a OR =b'), operator='and'),
{'keyword': '="=a OR =b"'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='=a OR =b'), operator='and'),
{'keyword': '<"=a OR =b"'})
self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%"a" OR "b"%'), operator='and'),
{'keyword': '"\\"a\\" OR \\"b\\""'})
def test_006_testRelatedKey_with_multiple_join(self):
# The name of catalog parameter does not matter at all
# ComplexQuery(ComplexQuery(AutoQuery(RelatedQuery(SimpleQuery())), AutoQuery(RelatedQuery(SimpleQuery()))))
# 'AutoQuery' doesn't need any ReferenceQuery equivalent.
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(RelatedReferenceQuery(ReferenceQuery(operator='=', default='a')), operator='and'),
ReferenceQuery(RelatedReferenceQuery(ReferenceQuery(operator='=', default='b')), operator='and')
, operator='and'), operator='and'),
{'query': ComplexQuery(Query(related_default='a'), Query(related_default='b'))})
##return catalog(title=Query(title='a', operator='not'))
#return catalog(title={'query': 'a', 'operator': 'not'})
#return catalog(title={'query': ['a', 'b'], 'operator': 'not'})
#return context.portal_catalog(source_title="toto", source_description="tutu", src__=1)
#print catalog(query=ComplexQuery(Query(title='1'), ComplexQuery(Query(portal_type='Foo') ,Query(portal_type='Bar'), operator='or'), operator='and'))
#print catalog(title={'query': ('path', 2), 'operator': 'and'}, exception=TypeError)
#print catalog(sort_on=[('source_title', )], check_search_text=False)
#print catalog(query=ComplexQuery(Query(source_title='foo'), Query(source_title='bar')), sort_on=[('source_title', ), ('source_title_1', )], check_search_text=False)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestSQLCatalog))
return suite
......@@ -28,6 +28,7 @@
import unittest
from DateTime import DateTime
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
......@@ -37,7 +38,7 @@ from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
class TestSearchKeyLexer(unittest.TestCase):
class TestSearchKeyLexer(ERP5TypeTestCase):
"""Test search keys
"""
run_all_test = 1
......@@ -48,7 +49,8 @@ class TestSearchKeyLexer(unittest.TestCase):
key = getSearchKeyInstance(search_key_class)
tokens = key.tokenize(search_value)
token_types = [x.type for x in tokens]
self.assertEqual(token_types, list(expected_token_types))
self.assertSameSet(token_types, expected_token_types)
def test_01ProperPoolInitialization(self, quiet=quiet, run=run_all_test):
""" Check that search key pool is properly initialized """
......@@ -82,7 +84,6 @@ size/Child/34"""
self.compare(DefaultKey,
'S\xc3\xa9bastien or !="Doe John1" and Doe',
('WORD', 'OR', 'NOT', 'WORDSET', 'AND', 'WORD',))
self.compare(DefaultKey, '.', ('WORD',))
def test_03KeyWordKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for KeyWordKey."""
......@@ -102,9 +103,8 @@ size/Child/34"""
('LESSTHANEQUAL', 'KEYWORD', 'AND',
'GREATERTHANEQUAL', 'WORDSET',))
self.compare(KeyWordKey, '=John% and >="JOHN John"',
('EXPLICITEQUALLITYWORD', 'AND',
('EXPLICITEQUALLITYWORD', 'KEYWORD', 'AND',
'GREATERTHANEQUAL', 'WORDSET',))
self.compare(KeyWordKey, '.', ('WORD',))
def test_04DateTimeKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for DateTimeKey."""
......@@ -121,17 +121,10 @@ size/Child/34"""
def test_05FullTextKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for FullTextKey."""
if not run: return
self.compare(FullTextKey, 'John', ('WORD',))
self.compare(FullTextKey, 'John Doe', ('WORD', 'WORD',))
self.compare(FullTextKey, 'John Doe',
('WORD', 'WORD',))
self.compare(FullTextKey, '+John -Doe',
('PLUS', 'WORD', 'MINUS', 'WORD',))
self.compare(FullTextKey, 'John*', ('WORD', 'ASTERISK'))
self.compare(FullTextKey, '+John*', ('PLUS', 'WORD', 'ASTERISK'))
self.compare(FullTextKey, '.', ('WORD',))
self.compare(FullTextKey, '"John Doe"', ('DOUBLEQUOTE', 'WORD', 'WORD', 'DOUBLEQUOTE'))
self.compare(FullTextKey, '+apple +(>turnover <strudel)',
('PLUS', 'WORD', 'PLUS', 'LEFTPARENTHES', 'GREATERTHAN', 'WORD',
'LESSTHAN', 'WORD', 'RIGHTPARENTHES',))
def test_06ScriptableKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for ScriptableKey."""
......@@ -143,7 +136,7 @@ size/Child/34"""
'John Doe OR creation_date>=2005/12/12',
('WORD', 'WORD', 'OR', 'KEYMAPPING',))
class TestSearchKeyQuery(unittest.TestCase):
class TestSearchKeyQuery(ERP5TypeTestCase):
"""Test search keys query generation
"""
run_all_test = 1
......@@ -206,12 +199,6 @@ class TestSearchKeyQuery(unittest.TestCase):
'%John and !=Doe%',
"((((title = '%John') AND (title != 'Doe%'))))",
[])
# special chars
self.compare(DefaultKey,
'title',
'.',
"((((title = '.'))))",
[])
def test_02KeyWordKey(self, quiet=quiet, run=run_all_test):
""" Check DefaultKey query generation"""
......@@ -241,12 +228,6 @@ class TestSearchKeyQuery(unittest.TestCase):
'%John Doe% or =Doe John',
"((((title LIKE '%John Doe%'))) OR (((title = 'Doe John'))))",
[])
# special chars
self.compare(KeyWordKey,
'title',
'.',
"((((title LIKE '%.%'))))",
[])
def test_03DateTimeKey(self, quiet=quiet, run=run_all_test):
""" Check DefaultKey query generation"""
......@@ -368,18 +349,7 @@ class TestSearchKeyQuery(unittest.TestCase):
'1ab521ty',
"delivery.stock = '1ab521ty'",
[])
# special chars
self.compare(RawKey,
'delivery.stock',
'.',
"delivery.stock = '.'",
[])
#None values
self.compare(RawKey,
'title',
None,
"title is NULL",
[])
def test_06FullTextKey(self, quiet=quiet, run=run_all_test):
""" Check FullTextKey query generation"""
if not run: return
......@@ -390,28 +360,6 @@ class TestSearchKeyQuery(unittest.TestCase):
["MATCH full_text.SearchableText AGAINST ('john' ) AS full_text_SearchableText_relevance",
"MATCH full_text.SearchableText AGAINST ('john' ) AS SearchableText_relevance"])
# special chars
self.compare(FullTextKey,
'full_text.SearchableText',
'.',
"MATCH full_text.SearchableText AGAINST ('.' )",
["MATCH full_text.SearchableText AGAINST ('.' ) AS full_text_SearchableText_relevance",
"MATCH full_text.SearchableText AGAINST ('.' ) AS SearchableText_relevance"])
#Boolean Mode
self.compare(FullTextKey,
'full_text.SearchableText',
'john stuart mill',
"MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE)",
["MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE) AS full_text_SearchableText_relevance",
"MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE) AS SearchableText_relevance"])
self.compare(FullTextKey,
'full_text.SearchableText',
'John*',
"MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE)",
["MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE) AS full_text_SearchableText_relevance",
"MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE) AS SearchableText_relevance"])
def test_07ScriptableKey(self, quiet=quiet, run=run_all_test):
""" Check ScriptableKey query generation"""
if not run: return
......
......@@ -95,369 +95,9 @@ class TestSQLCatalog(unittest.TestCase):
self.assertFalse(
self._catalog.isPortalTypeSelected('not_exists', 'Selected'))
class TestQuery(unittest.TestCase):
"""Test SQL bits generated from Queries
"""
def testSimpleQuery(self):
q = Query(title='Foo')
self.assertEquals(
dict(where_expression="((((title = 'Foo'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testQueryMultipleKeys(self):
# using multiple keys is invalid and raises
# KeyError: 'Query must have only one key'
self.assertRaises(KeyError, Query, title='Foo', reference='bar')
def testNoneQuery(self):
q = Query(title=None)
self.assertEquals(
dict(where_expression="title is NULL",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testEmptyQueryNotIgnoreEmptyString(self):
q = Query(title='')
# if you want to search with an empty string, pass ignore_empty_string=0 to
# asSQLExpression. XXX not to __init__ ?
self.assertEquals(
dict(where_expression="title = ''",
select_expression_list=[]),
q.asSQLExpression(ignore_empty_string=0,
keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testEmptyQuery(self):
q = Query(title='')
# query are true by default
self.assertEquals(
dict(where_expression="1",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testMultiValuedQuery(self):
q = Query(title=['Foo', 'Bar'])
self.assertEquals(
dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testINQuery(self):
q = Query(title=['Foo', 'Bar'], operator='IN')
self.assertEquals(
dict(where_expression="title IN ('Foo', 'Bar')",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testEmptyINQuery(self):
q = Query(title=[], operator='IN')
self.assertEquals(
dict(where_expression="0",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testMinQuery(self):
q = Query(title='Foo', range='min')
self.assertEquals(
dict(where_expression="title >= 'Foo'",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testMaxQuery(self):
q = Query(title='Foo', range='max')
self.assertEquals(
dict(where_expression="title < 'Foo'",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
# format
def testDateFormat(self):
date = DateTime(2001, 02, 03)
q = Query(date=date, format='%Y/%m/%d', type='date')
self.assertEquals(
dict(where_expression=
"((((date >= '%s' AND date < '%s'))))" \
%(date.toZone('UTC').ISO(), (date + 1).toZone('UTC').ISO()),
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
# full text
def testSimpleQueryFullText(self):
q = Query(title='Foo')
self.assertEquals(dict(where_expression="MATCH title AGAINST ('Foo' )",
select_expression_list=
["MATCH title AGAINST ('Foo' ) AS title_relevance"]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=['title']))
def testSimpleQueryFullTextSearchMode(self):
q = Query(title='Foo',
search_mode='in_boolean_mode')
self.assertEquals(dict(
where_expression="MATCH title AGAINST ('Foo' IN BOOLEAN MODE)",
select_expression_list=
["MATCH title AGAINST ('Foo' IN BOOLEAN MODE) AS title_relevance"]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=['title']))
def testSimpleQueryFullAutomaticTextSearchMode(self):
q = Query(title='Foo*',)
self.assertEquals(dict(
where_expression="MATCH title AGAINST ('Foo*' IN BOOLEAN MODE)",
select_expression_list=
["MATCH title AGAINST ('Foo*' IN BOOLEAN MODE) AS title_relevance"]),
q.asSQLExpression(full_text_search_keys=['title']))
def testSimpleQueryFullTextStat__(self):
# stat__ is an internal implementation artifact to prevent adding
# select_expression for countFolder
q = Query(title='Foo')
self.assertEquals(dict(
where_expression="MATCH title AGAINST ('Foo' )",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=['title'],
stat__=1))
def testSimpleQueryKeywordSearchKey(self):
q = Query(title='Foo')
self.assertEquals(dict(where_expression="((((title LIKE '%Foo%'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[]))
def testQueryKeywordSearchKeyWithPercent(self):
q = Query(title='Fo%oo')
self.assertEquals(dict(where_expression="((((title LIKE 'Fo%oo'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],))
def testQueryKeywordSearchKeyWithPercentAndOnlyOneLetter(self):
q = Query(title='F%o')
self.assertEquals(dict(where_expression="((((title LIKE 'F%o'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title']))
def testQueryKeywordSearchKeyWithPercentOnly(self):
q = Query(title='%')
self.assertEquals(dict(where_expression="((((title LIKE '%'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],))
def testQueryKeywordSearchKeyWithMinus(self):
q = Query(title='F-o')
self.assertEquals(dict(where_expression="((((title LIKE '%F-o%'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[]))
def testQueryKeywordSearchKeyWithSpace(self):
q = Query(title='F o')
self.assertEquals(dict(where_expression="((((title LIKE '%F o%'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[]))
def testQueryKeywordSearchKeyWithPercentAtTheEnd(self):
q = Query(title='F%')
self.assertEquals(dict(where_expression="((((title LIKE 'F%'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],))
q = Query(title='Fo%')
self.assertEquals(dict(where_expression="((((title LIKE 'Fo%'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],))
def testQueryKeywordSearchKeyWithPercentAtTheBeginning(self):
q = Query(title='%o')
self.assertEquals(dict(where_expression="((((title LIKE '%o'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],))
q = Query(title='%oo')
self.assertEquals(dict(where_expression="((((title LIKE '%oo'))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=['title'],))
def testNegatedQuery(self):
q1 = Query(title='Foo')
q = NegatedQuery(q1)
self.assertEquals(
dict(where_expression="(NOT (((((title = 'Foo'))))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
# complex queries
def testSimpleComplexQuery(self):
q1 = Query(title='Foo')
q2 = Query(reference='Bar')
q = ComplexQuery(q1, q2)
self.assertEquals(
dict(where_expression="((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
def testNegatedComplexQuery(self):
q1 = Query(title='Foo')
q2 = Query(reference='Bar')
q3 = ComplexQuery(q1, q2)
q = NegatedQuery(q3)
self.assertEquals(
# maybe too many parents here
dict(where_expression="(NOT (((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))))",
select_expression_list=[]),
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[]))
# forced keys
def testSimpleQueryForcedKeywordSearchKey(self):
q = Query(title='Foo', key='Keyword')
self.assertEquals("((((title LIKE '%Foo%'))))",
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression'])
def testSimpleQueryForcedFullText(self):
q = Query(title='Foo', key='FullText')
self.assertEquals("MATCH title AGAINST ('Foo' )",
q.asSQLExpression(keyword_search_keys=[],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression'])
def testSimpleQueryForcedExactMatch(self):
q = Query(title='Foo', key='ExactMatch')
self.assertEquals("title = 'Foo'",
q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression'])
def testSimpleQueryForcedExactMatchOR(self):
q = Query(title='Foo% OR %?ar', key='ExactMatch')
self.assertEquals("title = 'Foo% OR %?ar'",
q.asSQLExpression(keyword_search_keys=['title'],
datetime_search_keys = [],
full_text_search_keys=[])['where_expression'])
def testQuotedStringDefaultKey(self):
q = Query(title='Foo d\'Ba')
self.assertEquals(
dict(where_expression="((((title = 'Foo d''Ba'))))",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringKeywordKey(self):
q = Query(title='Foo d\'Ba', key='Keyword')
self.assertEquals(
dict(where_expression="((((title LIKE '%Foo d''Ba%'))))",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringExactMatch(self):
q = Query(title='Foo d\'Ba', key='ExactMatch')
self.assertEquals(
dict(where_expression="title = 'Foo d''Ba'",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringFullTextKey(self):
q = Query(title='Foo d\'Ba', type='fulltext')
self.assertEquals(
dict(where_expression="MATCH title AGAINST ('+Foo +d''Ba' IN BOOLEAN MODE)",
select_expression_list=["MATCH title AGAINST ('+Foo +d''Ba' IN BOOLEAN MODE)"
" AS title_relevance"]),
q.asSQLExpression())
def testQuotedStringListKeywordKey(self):
q = Query(title=('Foo d\'Ba',), key='Keyword')
self.assertEquals(
dict(where_expression="((((title LIKE '%Foo d''Ba%'))))",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringListExactMatch(self):
q = Query(title=('Foo d\'Ba',), key='ExactMatch')
self.assertEquals(
dict(where_expression="title = 'Foo d''Ba'",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringDateKey(self):
q = Query(title='Foo d\'Ba', type='date')
self.assertEquals(
# I don't know exactly what we should expect here.
dict(where_expression="1",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringFloatKey(self):
q = Query(title='Foo d\'Ba', type='float')
self.assertEquals(
# I don't know exactly what we should expect here.
# At least it's safe.
dict(where_expression="1",
select_expression_list=[]),
q.asSQLExpression())
def testQuotedStringIntKey(self):
q = Query(title='Foo d\'Ba', type='int')
self.assertEquals(
dict(where_expression="((((title = 'Foo d''Ba'))))",
select_expression_list=[]),
q.asSQLExpression())
def testListValuesInQuery(self):
q = Query(title=('Foo', 'Bar'))
self.assertEquals(
dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
select_expression_list=[]),
q.asSQLExpression())
def testListValuesInQueryWithKey(self):
q = Query(title=('Foo', 'Bar'), key='Keyword')
self.assertEquals(
dict(where_expression=
"(((((title LIKE '%Foo%')))) OR ((((title LIKE '%Bar%')))))",
select_expression_list=[]),
q.asSQLExpression())
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestSQLCatalog))
suite.addTest(unittest.makeSuite(TestZSQLCatalog))
suite.addTest(unittest.makeSuite(TestQuery))
return suite
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment