Major SQLCatalog rework on query-generation (not on indexation):

- further extend SearchKey concept
- define, implement and validate Interfaces
- add a column mapper (mapping is decided on a completely-formed query tree instead of being done individualy on each used column)
- state what is present for backward compatibility, warn about deprecated/dangerous uses


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@25706 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ZSQLCatalog/ColumnMap.py

+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import sys
+from zLOG import LOG
+from Interface.IColumnMap import IColumnMap
+from Interface.Verify import verifyClass
+from SQLCatalog import profiler_decorator
+
+DEFAULT_GROUP_ID = None
+
+TRACE = False
+
+# TODO: handle left joins
+# TODO: handle straight joins
+# TODO: make it possible to do: query=ComplexQuery(Query(source_title='foo'), Query(source_title='bar')), sort_on=[('source_title_1', )]
+#       currently, it's not possible because related_key_dict is indexed by related key name, which makes 'source_title_1' lookup fail. It should be indexed by group (probably).
+# TODO: rename all "related_key" references into "virtual_column"
+
+class ColumnMap(object):
+
+  __implements__ = IColumnMap
+
+  @profiler_decorator
+  def __init__(self, catalog_table_name=None):
+    self.catalog_table_name = catalog_table_name
+    # Key: group
+    # Value: set of column names
+    self.registry = {}
+    # Key: group
+    # Value: dict
+    #  Key: column name
+    #  Value: set of SimpleQuery
+    self.simple_query_dict = {}
+    # Key: (group, column name)
+    # Value: table name
+    self.column_map = {}
+    # Key: (group, table name)
+    # Value: table alias
+    self.table_alias_dict = {}
+    # Key: related key name
+    # Value: (group, column name)
+    self.related_key_dict = {}
+    # Key: related_column
+    # Value: last used alias order
+    self.related_key_order_dict = {}
+    # Key: group
+    # Value: relate_key
+    self.related_group_dict = {}
+    # Key: table alias
+    # Value: table name
+    self.table_map = {}
+    # Key: raw column
+    # Value: (function, column)
+    self.raw_column_dict = {}
+    # Entries: column name
+    self.column_ignore_set = set()
+    self.join_table_set = set()
+    self.straight_join_table_list = []
+    self.left_join_table_list = []
+    self.join_query_list = []
+
+  @profiler_decorator
+  def registerColumn(self, raw_column, group=DEFAULT_GROUP_ID, simple_query=None):
+    assert ' as ' not in raw_column.lower()
+    # Sanitize input: extract column from raw column (might contain COUNT, ...).
+    if '(' in raw_column:
+      function, column = raw_column.split('(')
+      column = column.strip()
+      assert column[-1] == ')'
+      column = column[:-1].strip()
+    else:
+      function = None
+      column = raw_column
+    # Remove '`' from column.
+    column = column.replace('`', '')
+    # Extract table name from column, if any.
+    if '.' in column:
+      # Assuming the part before the dot is a real table name, not an alias.
+      table, column = column.split('.')
+      LOG('ColumnMap', 100, 'Column %r is pre-mapped. This use is strongly discouraged.' % (raw_column, ))
+    else:
+      table = None
+
+    self.raw_column_dict[raw_column] = (function, column)
+    self.registry.setdefault(group, set()).add(column)
+    self.simple_query_dict.setdefault(group, {}).setdefault(column, set()).add(simple_query)
+    if table is not None:
+      # Register table alias and mark column as resolved.
+      self.registerTable(table, alias=table, group=group)
+      self.resolveColumn(column, table, group=group)
+      if group is DEFAULT_GROUP_ID and table != self.catalog_table_name:
+        # When a column is registered  in default group and is explicitely
+        # mapped to a table, we must mark its table as requiring a join with
+        # catalog table (unless it's the catalog table, of course).
+        self._addJoinTable(table, group)
+
+  def ignoreColumn(self, column):
+    self.column_ignore_set.add(column)
+
+  @profiler_decorator
+  def registerRelatedKey(self, related_column, column):
+    # XXX: should we store the group, or directly the table on which the column is mapped ?
+    # The former avoids duplicating data, but requires one more lookup (group + column -> table)
+    # The latter makes it harder (?) to split the mapping in multiple queries (if splitting by groups turns out to be a good idea)
+    real_related_column = related_column
+    order = self.related_key_order_dict.get(real_related_column, 0) + 1
+    related_column = '%s_%s' % (related_column, order)
+    group = 'related_%s' % (related_column, )
+    assert group not in self.registry
+    assert group not in self.related_group_dict
+    self.related_key_order_dict[real_related_column] = order
+    self.related_key_dict[real_related_column] = (group, column)
+    self.registerColumn(column, group=group)
+    self.related_group_dict[group] = related_column
+    # XXX: hardcoded translation table column names: they are not present in sql_catalog.getColumnMap(), and this table cannot be joined by uid, forbidding implicit join.
+    if column in ('translated_message', 'language', 'message_context', 'original_message'):
+      self.registerTable('translation', alias='translation', group=group)
+      self.resolveColumn(column, 'translation', group=group)
+    # Likewise, for measure table. Moreover, there is a related key named the same way as a column of that table (designed to do the join).
+    elif column in ('metric_type_uid', ):
+      self.registerTable('measure', group=group)
+      self.resolveColumn(column, 'measure', group=group)
+    return group
+
+  @profiler_decorator
+  def registerCatalog(self):
+    assert self.catalog_table_name is not None
+    LOG('ColumnMap', 100, 'Registering implicit catalog. This use is strongly discouraged.')
+    self.registerTable(self.catalog_table_name)
+    self.resolveTable(self.catalog_table_name, self.catalog_table_name)
+
+  @profiler_decorator
+  def registerRelatedKeyColumn(self, related_column, position, group):
+    assert group in self.related_group_dict
+    group = self.getRelatedKeyGroup(position, group)
+    assert group not in self.related_group_dict
+    self.related_group_dict[group] = related_column
+    return group
+
+  def getRelatedKeyGroup(self, position, group):
+    return '%s_column_%s' % (group, position)
+
+  @profiler_decorator
+  def registerTable(self, table_name, alias=None, group=DEFAULT_GROUP_ID):
+    table_alias_dict = self.table_alias_dict
+    table_alias_key = (group, table_name)
+    existing_value = table_alias_dict.get(table_alias_key)
+    # alias = None, existing = None -> store
+    # alias = None, existing ! None -> skip
+    # alias ! None, existing = None -> store & resolve
+    # alias ! None, existing ! None -> skip if alias = existing, raise otherwise
+    if existing_value is None:
+      table_alias_dict[table_alias_key] = alias
+      if alias is not None:
+        self.resolveTable(table_name, alias, group=group)
+    elif alias is not None and alias != existing_value:
+      raise ValueError, 'Table %r for group %r is aliased as %r, can\'t alias it now as %r' % (table_name, group, existing_value, alias)
+
+  @profiler_decorator
+  def _mapColumns(self, column_table_map, table_usage_dict, column_name_set, group, vote_result_dict):
+    mapping_dict = {}
+    catalog_table_name = self.catalog_table_name
+
+    # Map all columns to tables decided by vote.
+    for column_name, candidate_dict in vote_result_dict.iteritems():
+      # candidate_dict is never empty
+      max_score = 0
+      for table_name, score in candidate_dict.iteritems():
+        if score > max_score:
+          max_score = score
+          best_count = 0
+          best_choice = table_name
+        elif score == max_score:
+          best_count += 1
+      if best_count:
+        LOG('ColumnMap', 100, 'Mapping vote led to a tie. Mapping to %r' % (best_choice, ))
+      if TRACE:
+        LOG('ColumnMap', 0, 'Mapping by vote %r to %r' % (column_name, best_choice))
+      mapping_dict[column_name] = best_choice
+      column_name_set.remove(column_name)
+      for table_name, column_set in table_usage_dict.iteritems():
+        if table_name != best_choice:
+          column_set.discard(column_name)
+
+    # Map all remaning columns.
+    def table_weight(a):
+      """
+        Compute each table weight.
+      """
+      if (group, a[0]) in self.table_alias_dict:
+        result = (2, )
+      elif a[0] == catalog_table_name:
+        result = (1, )
+      else:
+        result = (0, len(a[1]))
+      return result
+    # Sort table name list, first has the most required columns
+    weighted_table_list = sorted(table_usage_dict.iteritems(), key=table_weight)
+    while len(weighted_table_list):
+      table_name, column_set = weighted_table_list.pop()
+      if len(column_set):
+        common_column_set = column_name_set.intersection(column_set)
+        if len(common_column_set):
+          # Only allow usage of this table if any of those is true:
+          # - current table is the catalog  (if any catalog was provided)
+          # - there are column used on that table which are already mapped
+          #   (this does not include columns mapped by this code)
+          #   If columns are mapped to this table in current group, then using
+          #   it will not require a new join, so it should be allowed.
+          #   Note: it would be good to take indexes into account when there
+          #   are multiple candidate tables.
+          # - any of those columns belongs exclusively to this table
+          #   Although the list of tables those columns belong to is known
+          #   earlier (in "build"), mapping them here
+          #   - avoids code duplication (registerTable, resolveColumn,
+          #     _addJoinTable)
+          #   - offers user to vote for an unknown table, overriding this
+          #     forced mapping.
+          use_allowed = table_name == catalog_table_name or \
+                        len(common_column_set) < len(column_set)
+          if not use_allowed:
+            for column_name in column_set:
+              if len(column_table_map.get(column_name, [])) == 1:
+                # There is no alternative, mark as required
+                use_allowed = True
+                break
+          if use_allowed:
+            for column_name in common_column_set:
+              if TRACE:
+                LOG('ColumnMap', 0, 'Mapping by default %r to %r' % \
+                    (column_name, table_name))
+              mapping_dict[column_name] = table_name
+              # This column must not be resolved any longer
+              column_name_set.remove(column_name)
+              # Remove this column from sets containing it. This prevents from
+              # giving a high score to a table which columns would already have
+              # been mapped to another table.
+              for ignored, other_column_set in weighted_table_list:
+                other_column_set.discard(column_name)
+            weighted_table_list.sort(key=table_weight)
+          else:
+            # All column which are mappable on that table are to-be-mapped
+            # columns. This means that this table was not explicitely used, and
+            # as each table contain a different amount of lines, we should not
+            # join with any non-explicit table. Hence, we skip this mapping.
+            LOG('ColumnMap', 0, 'Skipping possible map of %r on %r as that table' \
+                ' is not explicitely used.' % (common_column_set, table_name))
+
+    # Detect incomplete mappings
+    if len(column_name_set):
+      raise ValueError, 'Could not map those columns: %r' % (column_name_set, )
+
+    # Do the actual mapping
+    for column_name, table_name in mapping_dict.iteritems():
+      # Mark this column as resolved
+      if TRACE:
+        LOG('ColumnMap', 0, 'Mapping column %s to table %s' % (column_name, table_name))
+      self.registerTable(table_name, group=group)
+      self.resolveColumn(column_name, table_name, group=group)
+      if table_name != catalog_table_name:
+        self._addJoinTable(table_name, group)
+
+  @profiler_decorator
+  def build(self, sql_catalog):
+    catalog_table_name = self.catalog_table_name
+    if catalog_table_name is None:
+      return
+
+    column_table_map = sql_catalog.getColumnMap()
+    table_vote_method_list = [getattr(sql_catalog, x) for x in sql_catalog.sql_catalog_table_vote_scripts]
+
+    # Generate missing joins from default group (this is required to allow using related keys outside of queries: order_by, sort_on, ...)
+    column_set = self.registry.get(DEFAULT_GROUP_ID, [])
+    for column_name in column_set:
+      if column_name not in column_table_map and column_name not in self.related_key_dict:
+        related_key_definition = sql_catalog.getRelatedKeyDefinition(column_name)
+        if related_key_definition is not None:
+          join_query = sql_catalog.getSearchKey(column_name, 'RelatedKey').buildQuery(sql_catalog=sql_catalog, related_key_definition=related_key_definition)
+          join_query.registerColumnMap(sql_catalog, self)
+          self._addJoinQuery(join_query)
+
+    # List all possible tables, with all used column for each
+    for group, column_set in self.registry.iteritems():
+      # unique needed column name set
+      column_name_set = set()
+      # table -> column_set, including alternatives
+      table_usage_dict = {}
+
+      for column_name in column_set:
+        if column_name == '*' or column_name in self.column_ignore_set:
+          continue
+        table_name_list = column_table_map.get(column_name, [])
+        if len(table_name_list) == 0:
+          if not(group is DEFAULT_GROUP_ID and column_name in self.related_key_dict):
+            LOG('ColumnMap', 100, 'Not a known column name: %r' % (column_name, ))
+          continue
+        column_map_key = (group, column_name)
+        if column_map_key in self.column_map:
+          # Column is already mapped, so we must count this column as being available only on that table. Its mapping will not change, and it will impact table schema choice.
+          table_name = self.column_map[column_map_key]
+          assert table_name in table_name_list, '%r not in %r' % (table_name, table_name_list)
+          table_name_list = [table_name]
+        else:
+          # Mark this column as requiring to be mapped.
+          column_name_set.add(column_name)
+        for table_name in table_name_list:
+          table_usage_dict.setdefault(table_name, set()).add(column_name)
+      # XXX: mutable datatypes are provided to vote method. if it modifies
+      # them, it can introduce mapping bugs. Copying them might be costly,
+      # especialy if done before each call, since they also contain mutable
+      # types.
+      # XXX: the API of vote methods is not stable yet. Parameters should
+      # always be passed and expected by name, to make it less painful to
+      # change API.
+      # XXX: there is no check that the table voted for contains mapped
+      # column. It is up to the user not to do stupid things.
+      vote_result_dict = {}
+      simple_query_dict = self.simple_query_dict[group]
+      for table_vote_method in table_vote_method_list:
+        vote_dict = table_vote_method(column_name_set=column_name_set,
+                                      simple_query_dict=simple_query_dict,
+                                      table_usage_dict=table_usage_dict,
+                                      group=group)
+        if isinstance(vote_dict, dict):
+          for column, table in vote_dict.iteritems():
+            if column in column_name_set:
+              column_vote_dict = vote_result_dict.setdefault(column, {})
+              column_vote_dict[table] = column_vote_dict.get(table, 0) + 1
+            else:
+              LOG('ColumnMap', 100, 'Vote script %r voted for a ' \
+                  'non-candidate column: %r, candidates are: %r. Ignored.' %
+                  (table_vote_method, column, column_name_set))
+        else:
+          LOG('ColumnMap', 100, 'Vote script %r returned invalid data: %r. ' \
+              'Ignored.' % (table_vote_method, vote_dict))
+      self._mapColumns(column_table_map, table_usage_dict, column_name_set, group, vote_result_dict)
+
+    table_alias_number_dict = {}
+
+    for (group, table_name), alias in self.table_alias_dict.iteritems():
+      if alias is None:
+        if group in self.related_group_dict:
+          alias_table_name = 'related_%s_%s' % (self.related_group_dict[group], table_name)
+        else:
+          alias_table_name = table_name
+        table_alias_number = table_alias_number_dict.get(alias_table_name, 0)
+        while True:
+          if table_alias_number == 0:
+            alias = alias_table_name
+          else:
+            alias = '%s_%s' % (alias_table_name, table_alias_number)
+          table_alias_number += 1
+          if alias not in self.table_map:
+            break
+        table_alias_number_dict[alias_table_name] = table_alias_number
+      self.resolveTable(table_name, alias, group=group)
+
+    if TRACE:
+      # Key: group
+      # Value: 2-tuple
+      #  dict
+      #   Key: column
+      #   Value: table name
+      #  dict
+      #   Key: table name
+      #   Value: table alias
+      summary_dict = {}
+      for (group, column), table_name in self.column_map.iteritems():
+        column_dict = summary_dict.setdefault(group, ({}, {}))[0]
+        assert column not in column_dict, '%r in %r' % (column, column_dict)
+        column_dict[column] = table_name
+      for (group, table_name), table_alias in self.table_alias_dict.iteritems():
+        table_dict = summary_dict.setdefault(group, ({}, {}))[1]
+        assert table_name not in table_dict, '%r in %r' % (table_name, table_dict)
+        table_dict[table_name] = table_alias
+      for group, (column_dict, table_dict) in summary_dict.iteritems():
+        LOG('ColumnMap', 0, 'Group %r:' % (group, ))
+        LOG('ColumnMap', 0, ' Columns:')
+        for column, table_name in column_dict.iteritems():
+          LOG('ColumnMap', 0, '  %r from table %r' % (column, table_name))
+        LOG('ColumnMap', 0, ' Tables:')
+        for table_name, table_alias in table_dict.iteritems():
+          LOG('ColumnMap', 0, '  %r as %r' % (table_name, table_alias))
+
+  def asSQLColumn(self, raw_column, group=DEFAULT_GROUP_ID):
+    if self.catalog_table_name is None or raw_column in self.column_ignore_set or \
+       '.' in raw_column or '*' in raw_column:
+      result = raw_column
+    else:
+      function, column = self.raw_column_dict.get(raw_column, (None, raw_column))
+      if group is DEFAULT_GROUP_ID:
+        group, column = self.related_key_dict.get(column, (group, raw_column))
+      alias = self.table_alias_dict[(group, self.column_map[(group, column)])]
+      result = '`%s`.`%s`' % (alias, column)
+      if function is not None:
+        result = '%s(%s)' % (function, result)
+    return result
+
+  def getCatalogTableAlias(self, group=DEFAULT_GROUP_ID):
+    return self.table_alias_dict[(group, self.catalog_table_name)]
+
+  def getTableAliasDict(self):
+    return self.table_map.copy()
+
+  @profiler_decorator
+  def resolveColumn(self, column, table_name, group=DEFAULT_GROUP_ID):
+    assert group in self.registry
+    assert column in self.registry[group]
+    column_map_key = (group, column)
+    column_map = self.column_map
+    assert (group, table_name) in self.table_alias_dict
+    previous_value = column_map.get(column_map_key)
+    if previous_value is None:
+      column_map[column_map_key] = table_name
+    elif previous_value != table_name:
+      if column == 'uid':
+        LOG('ColumnMap', 100, 'Attempt to remap uid from %r to %r ignored.' % (previous_value, table_name))
+      else:
+        raise ValueError, 'Cannot remap a column to another table. column_map[%r] = %r, new = %r' % (column_map_key, column_map.get(column_map_key), table_name)
+
+  @profiler_decorator
+  def resolveTable(self, table_name, alias, group=DEFAULT_GROUP_ID):
+    table_alias_key = (group, table_name)
+    assert table_alias_key in self.table_alias_dict
+    assert self.table_alias_dict[table_alias_key] in (None, alias)
+    self.table_alias_dict[table_alias_key] = alias
+    assert self.table_map.get(alias) in (None, table_name)
+    self.table_map[alias] = table_name
+
+  def getTableAlias(self, table_name, group=DEFAULT_GROUP_ID):
+    return self.table_alias_dict[(group, table_name)]
+
+  def _addJoinQuery(self, query):
+    self.join_query_list.append(query)
+
+  def addJoinQuery(self, query):
+    LOG('ColumnMap', 0, 'addJoinQuery use is discouraged')
+    self._addJoinQuery(query)
+
+  def iterJoinQueryList(self):
+    return iter(self.join_query_list)
+
+  @profiler_decorator
+  def _addJoinTable(self, table_name, group=DEFAULT_GROUP_ID):
+    """
+      Declare given table as requiring to be joined with catalog table.
+
+      table_name (string)
+        Table name.
+      group (string)
+        Group id of given table.
+    """
+    catalog_table = self.catalog_table_name
+    if catalog_table is not None:
+      # Only join tables when there is a catalog table
+      # Register unconditionaly catalog table
+      self.registerTable(catalog_table)
+      if 'uid' not in self.registry.get(DEFAULT_GROUP_ID, ()):
+        # Register uid column if it is not already
+        self.registerColumn('uid')
+        self.resolveColumn('uid', catalog_table)
+      self.join_table_set.add((group, table_name))
+
+  def getJoinTableAliasList(self):
+    return [self.getTableAlias(table_name, group=group)
+            for (group, table_name) in self.join_table_set]
+
+  def getStraightJoinTableList(self):
+    return self.straight_join_table_list[:]
+
+  def getLeftJoinTableList(self):
+    return self.left_join_table_list[:]
+
+verifyClass(IColumnMap, ColumnMap)
+

product/ZSQLCatalog/Interface/IAbstractSyntaxNode.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class INode(Interface):
+  """
+    Any kind of node in an Abstract Syntax Tree.
+  """
+
+  def isLeaf():
+    """
+      Returns True if current node is a leaf in node tree.
+      Returns False otherwise.
+    """
+
+  def isColumn():
+    """
+      Returns True if current node is a column in node tree.
+      Returns False otherwise.
+    """
+
+class IValueNode(INode):
+  """
+    Value- and comparison-operator-containig node.
+    They are leaf nodes in the syntax tree.
+  """
+
+  def getValue():
+    """
+      Returns node's value.
+    """
+
+  def getComparisonOperator():
+    """
+      Returns node's comparison operator.
+    """
+
+class ILogicalNode(INode):
+  """
+    Logical-operator-containing node.
+    They are internal tree nodes.
+  """
+
+  def getLogicalOperator():
+    """
+      Returns node's logical operator.
+    """
+
+  def getNodeList():
+    """
+      Returns the list of subnodes.
+    """
+
+class IColumnNode(INode):
+  """
+    Column-name-containing node.
+    They are internal tree nodes.
+    Their value applies to any contained ValueNode, except if there is another
+    ColumnNode between current one and a ValueNode, for which the other
+    ColumnNode will take precedence.
+  """
+
+  def getColumnName():
+    """
+      Returns node's column name.
+    """
+
+  def getSubNode():
+    """
+      Returns node's (only) subnode.
+    """
+

product/ZSQLCatalog/Interface/IColumnMap.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IColumnMap(Interface):
+  """
+    The role of the column mapper is to make possible to have a "flat"
+    overview of all columns used in a query, to choose how they will be
+    mapped to catalog tables, and how those tables will be aliased in the SQL
+    rendering of that query.
+
+    Typical usage:
+    - Instanciate (with or without a catalog table)
+    - Register all columns the query will use
+    - Build the column map
+    - Fetch SQL representation of each registered column.
+    - Fetch table alias mapping
+    - Fetch the list of table aliases which require a join with catalog
+    - Fetch the list of queries implied by columns. This happens when there
+      are virtual columns registered which were not expanded in the query
+      already. In that case, column map generate queries required to reach
+      the real column behind the virtual one.
+
+    Note that, although it's not enforced, it is meaningless to:
+    - call build more than once
+    - register columns after build happened
+    - fetch SQL representation before build happened (this will most probably
+      lead to KeyErrors anyway)
+
+    Groups.
+      All references to a given table withing a given group will cause all
+      columns referencing that table to be resolved to the same table alias
+      in resulting query.
+      Ex:
+        Note: instead of resolving columns by hand, it is also possible to
+        call "build", but this way it makes .
+        Registration:
+          registerColumn('bar')
+          registerColumn('baz')
+          resolveColumn('bar', 'foo')
+          resolveColumn('baz', 'foo')
+        Result:
+          asSQLColumn('bar') -> 'foo_alias.bar'
+          asSQLColumn('baz') -> 'foo_alias.baz'
+      Complementary, any reference to a given table from one group to
+      another will cause columns to be resolved to distinct table aliases
+      in resulting query.
+      Ex:
+        Registration:
+          registerColumn('bar')
+          registerColumn('baz', group='hoge')
+          resolveColumn('bar', 'foo')
+          resolveColumn('baz', 'foo', group='hoge')
+        Result:
+          asSQLColumn('bar')               -> 'foo_alias.bar'
+          asSQLColumn('baz', group='hoge') -> 'other_foo_alias.baz'
+  """
+
+  def __init__(catalog_table_name=None):
+    """
+      Create a column map.
+
+      Initialises internal data structures and set the table name to use as
+      catalog table.
+    """
+
+  def registerColumn(column, group=None, simple_query=None):
+    """
+      Register a column in given group on this column map.
+
+      column (string)
+        Contains the name of the column to register.
+
+        This name might contain a dot, in which case a table name is extracted
+        from it and that column is resolved to that table within its group.
+        Note that this practice is discouraged, but not deprecated.
+      group (string)
+        Group id of given column.
+      simple_query (SimpleQuery)
+        The SimpleQuery doing a comparison on given column. This can be
+        accessed from vote script at ColumnMap build time.
+    """
+
+  def ignoreColumn(column):
+    """
+      Act as if given column was valid and its mapping succeeded.
+      ie: supresses all warnings relative to that column, and all mapping
+      attemps.
+      It is supposed to be used when given value is an SQL alias for a
+      separately registered column.
+
+      column (string)
+        The value to ignore.
+    """
+
+  def registerRelatedKey(related_column, column):
+    """
+      Register a "virtual column".
+      This method is to be called when registering any virtual column.
+      A virtual column makes use internaly of multiple catalog columns.
+      A related key is an example of a virtual column.
+
+      This method internaly generates a new group id that caller must pass to
+      registerRelatedKeyColumn when registering those internal (and real)
+      columns. This is because the same virtual column can be used more than
+      once in a query without interferences from other uses, and hence must be
+      put in separate groups, without having to know that there are other uses
+      for the same virtual column.
+
+      virtual_column
+        Name of the virtual column to register.
+      column
+        Name of the "last" real column behind the virtual column. Comparisons
+        done on the virtual column will happen on that real column instead.
+        For example, it's "title" for "source_title" related key.
+    """
+
+  def registerCatalog():
+    """
+      Register catalog table as being used in default group.
+
+      This is for backward compatibility with text-only related keys, since
+      they often hardcode catalog table name.
+    """
+
+  def registerRelatedKeyColumn(related_column, position, group):
+    """
+      Register given column as part of already registered virtual column.
+
+      related_column (string)
+        Name of the virtual column registered column is a part of.
+      position (int)
+        Unique id of this column in the list of all columns used by the
+        virtual column it is a part of (it might use the same column name
+        while expecting uses to be mapped to different aliases or evne
+        different tables).
+        Typically, this is table position in the parameter list the related
+        key expects.
+      group (string)
+        Group id as returned by registerRelatedKey of given virtual column.
+    """
+
+  def getRelatedKeyGroup(position, group):
+    """
+      For given virtual key position and group, return a group.
+
+      This is here so that all group generation code is inside ColumnMap
+      class.
+    """
+
+  def registerTable(table_name, alias=None, group=None):
+    """
+      Register given table name as being used in given group.
+
+      This method should not be called outside of this class except for
+      backward compatibility purposes.
+
+      It is implicitely done most of the time, and should only be used only
+      when there is no control over chosen table alias or when table
+      registration cannot be done by a Query.
+      There are 2 cases where it cas required:
+      - pre-mapped columns which won't be actually used, but which force table
+        alias choice (see EntireQuery).
+      - related keys, where the Related Query cannot register columns it uses
+        by itself (since its content is not based on Queries but on raw SQL
+        text).
+    """
+
+  def build(sql_catalog):
+    """
+      Resolve all unresolved registered columns (ie, not mapped to tables).
+      Resolve all used tables to unique aliases.
+
+      Chosen table aliases are based on table name and virtual column name the
+      column comes from, if applicable. This is done to make final SQL more
+      reader-friendly.
+    """
+
+  def asSQLColumn(column, group=None):
+    """
+      Return an SQL rendering of given column, with the table alias it has
+      been mapped to.
+
+      column (string)
+        Column name. Can be the name of a virtual column.
+
+        This name might contain a dot, in which case a table name is extracted
+        from it and that column is resolved to that table within its group.
+        Note that this practice is discouraged, but not deprecated.
+        Note also that it does not apply to virtual columns (they cannot
+        contain a dot).
+      group (string)
+        Group id of given column.
+    """
+
+  def getCatalogTableAlias(group=None):
+    """
+      Return the alias of catalog table name given at instantiation.
+    """
+
+  def getTableAliasDict():
+    """
+      Return a copy of table mapping.
+
+      returned value (dict)
+        key (string)
+          Table alias.
+        value (string)
+          Table name.
+    """
+
+  def resolveColumn(column, table_name, group=None):
+    """
+      Map given column to given table within given group.
+
+      column (string)
+        Name of the column to map to a table. This cannot be a virtual column.
+      table_name (string)
+        Name of the table column must be mapped to. This is not an alias.
+        It must be already known to be used within given group.
+      group (string)
+        Name of the group column and table are in.
+    """
+
+  def resolveTable(table_name, alias, group=None):
+    """
+      Resolve given table of given group as given alias.
+
+      table_name (string)
+        Table to alias.
+      alias (string)
+        Table alias. It must be unique at query scope to work as intended.
+      gorup (string)
+        Group the table belongs to.
+    """
+
+  def getTableAlias(table_name, group=None):
+    """
+      Get table alias in given group.
+
+      table_name (string)
+        Table name.
+      group (string)
+        Group id of given table.
+    """
+
+  def addJoinQuery(query):
+    """
+      Adds a query to a list of top-level queries.
+      This method is only made available at interface level for backward
+      compatibility.
+    """
+
+  def iterJoinQueryList():
+    """
+      Get an iterator over queries internally generated when resolving column
+      map. Those queries are generated when a virtual column was registered as
+      a real one. Queries required to map virtual columns to real ones are
+      put in that list by build.
+    """
+
+  def getJoinTableAliasList():
+    """
+      Return a copy of the table alias list for tables requiring a join with
+      catalog table.
+    """
+
+  def getStraightJoinTableList():
+    """
+      Returns the list of tables used this search and which
+      need to be joined with the main table using explicit
+      indices.
+    """
+
+  def getLeftJoinTableList():
+    """
+      Returns the list of tables used this search and which
+      need to be LEFT joined with the main table using explicit
+      indices.
+    """
+

product/ZSQLCatalog/Interface/IEntireQuery.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IEntireQuery(Interface):
+  """
+    A EntireQuery represents an entire SQL expression, where a Query
+    represents only the "WHERE" part of that expression.
+    A EntireQuery contains:
+     - a Query instance
+     - a limit expression
+     - a group-by expression
+     - an order-by expression
+     - a select expression
+    It internaly uses a ColumnMap instance to resolve tables to use to
+    generate a "from" expression.
+  """
+
+  def __init__(query, order_by_list=None, group_by_list=None,
+    select_dict=None, limit=None, catalog_table_name=None,
+    extra_column_list=None, from_expression=None,
+    order_by_override_list=None):
+    """
+      query (Query instance)
+        The root of the Query tree this query will contain.
+      order_by_list (list of 1-tuple, 2-tuple or 3-tuple)
+        The list of columns which will be sorted by SQL.
+        Tuple values are:
+        - mandatory: column name
+        - optionnal: sort order (can be "ASC" or "DESC", "ASC" by default)
+        - optionnal: type cast (no cast by default, see "CAST" SQL method)
+      group_by_list (list of string)
+        The list of columns which will be groupped by value by SQL.
+      select_dict (dict, key: string, value: string, None)
+        Given values describe columns to make available in SQL result.
+        If column is aliased in result set, key is the alias and value is the
+        column.
+        Otherwise, key is the column, and value must be None.
+      limit
+        See SQLExpression.
+      catalog_table_name (string)
+        Name of the table to use as a catalog.
+
+      Deprecated parameters.
+      extra_column_list (list of string)
+        The list of columns to register to column map. They will not be used
+        in final rendering, but are hint on which table are supposed to be
+        used when mapping columns.
+      from_expression
+        See SQLExpression.
+      order_by_override_list (list of string)
+        If a column is in order_by_list, cannot be mapped to a table column
+        but is present in this list, it will be passed through to
+        SQLExpression.
+    """
+
+  def asSQLExpression(sql_catalog, only_group_columns):
+    """
+      Instantiate a column map, process parameters given at instantiation and
+      register them to column map.
+      Register query to column map.
+      Build column map.
+      Generate extra SQLExpressions from column map.
+      Generate SQLExpression instance and return it.
+    """
+
+  def asSearchTextExpression(sql_catalog):
+    """
+      This is just a passthrough to embeded Query's asSearchTextExpression
+      method.
+      This means that only the where expression can be represented as a
+      SearchText, but not sort, limits, ...
+    """
+

product/ZSQLCatalog/Interface/IOperator.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IOperator(Interface):
+  """
+    An operator is responsible for rendering a value and a column name as SQL
+    or Search Text.
+
+    This class is designed to be used as a singleton-per-operator.
+  """
+
+  def __init__(operator):
+    """
+      operator (string)
+        Operator's text representation. It is used both for SQL and SearchText
+        rendering.
+    """
+
+  def asSearchText(value):
+    """
+      Render given value as Search Text
+
+      value (see _renderValue)
+        Value to render as a string for use in a Search Text expression.
+    """
+
+  def asSQLExpression(column, value_list, only_group_columns):
+    """
+      Construct a SQLExpression instance from given column and value, with
+      contained glue text.
+      value_list can be a non-list instance, which must be handled that same
+      way as a list of one item.
+      only_group_columns (bool)
+        If false, the operator can add group columns in the "select_dict" of
+        returned SLQExpression.
+        Otherwise, it must not (SQL would be invalid).
+    """
+
+  def getOperator():
+    """
+      Accessor for operator's text representation.
+    """
+

product/ZSQLCatalog/Interface/IQuery.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IQuery(Interface):
+  """
+    A Query contains:
+    - a value
+    - an operator
+    - a column
+
+    It is the python representation of a predicate, independently of its
+    rendering (SQL or SearchText).
+    For SQL rendering to be possible, it is necesary for some data to be
+    centralized in a data structure known at EntireQuery level (to be able to
+    generate unique table aliases, for exemple). This is the role of
+    ColumnMap, and registerColumnMap method on this interface.
+
+    This interface also offers various rendering methods, one per rendering
+    format.
+  """
+
+  def asSearchTextExpression(sql_catalog, column=None):
+    """
+      Render a query in a user-oriented SearchText.
+      Returns None if there is this query has no SearchText representation,
+      but is SearchText-aware.
+      If column is provided, it must be used instead of local knowledge of
+      column name. It is used to make queries inside a related key render
+      correctly.
+    """
+
+  def asSQLExpression(sql_catalog, column_map, only_group_columns):
+    """
+      Render a query as an SQLExpression instance.
+    """
+
+  def registerColumnMap(sql_catalog, column_map):
+    """
+      Register a query to given column_map.
+    """
+

product/ZSQLCatalog/Interface/IQueryCatalog.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class ISearchKeyCatalog(Interface):
+
+  def buildQuery(kw, ignore_empty_string=True, operator='and'):
+    """
+      Build a ComplexQuery from kw values.
+
+      kw (dict: string keys, any value)
+        A query will be emited based on its value. Depending on the type of
+        the value it is handled differently. Query values will be passed
+        through to result (key is ignored). For all other types, their key
+        must be either a known column of a sql_search_tables table, or a
+        related key name.
+        - String values will be parsed according to the default SearchKey of
+          their real column (even for related keys). If parsing was
+          successful, Queries will be generated from its output.
+          Otherwise, that value will be taken as such.
+        - Dictionary values can be composed of the following keys:
+          'query': Their payload value, considered as empty if not given.
+          'key': The SearchKey to use for this value, overriding default
+                 column configuration.
+          (for other possible keys, see SearchKeys)
+          They will be taken as such.
+        - All other types will be taken as such, and no "empty" check will be
+          performed on them.
+      ignore_empty_string (boolean)
+        If True, values from kw which are empty will be skipped.
+      operator (string)
+        Used to explicit the logical relation between kw entries.
+        It must be a valid ComplexQuery logical operator ('and', 'or').
+    """
+
+  def buildSQLQuery(query_table='catalog', REQUEST=None,
+                    ignore_empty_string=1, only_group_columns=False,
+                    limit=None, extra_column_list=None,
+                    **kw):
+    """
+      Construct and return an instance of EntireQuery class from given
+      parameters by calling buildQuery.
+
+      ignore_empty_string (boolean)
+        See buildQuery.
+      limit (1-tuple, 2-tuple)
+        If given, will emit SQL to limit the number of result lines.
+      group_by_list (list of strings)
+        If given, will emit SQL to group found lines on given parameter names
+        (their column if they are column names, corresponding virtual columns
+        otherwise - as for related keys).
+      select_dict (dict, key: string, value: string, None)
+        Given values describe columns to make available in SQL result.
+        If column is aliased in result set, key is the alias and value is the
+        column.
+        Otherwise, key is the column, and value can be None or the same as
+        key.
+      select_list (list of strings)
+        Same as providing select_dict with select_list items as keys, and None
+        values.
+      order_by_list (list of 1-, 2-, or 3-tuples of strings)
+        If given, will emit SQL to sort result lines.
+        Sort will happen with decreasing precedence in list order.
+        Given n-tuples can contain those values, always in this order:
+         - parameter name
+         - sort order (see SQL documentation of 'ORDER BY')
+         - type cast (see SQL documentation of 'CAST')
+        Sort will happen on given parameter name (its column if it's a column
+        name, corresponding virtual column otherwise - as for related keys).
+      Extra parameters are passed through to buildQuery.
+
+      Backward compatibility parameters:
+      Those parameters are deprecated and should not be used. They are present
+      to provide backward compatibility with former ZSQLCatalog version.
+      REQUEST
+        Ignored.
+      extra_column_list (list)
+      query_table (string, None)
+        The table to use as catalog table.
+        If given and None, not catalog table will be used. Use this when you
+        are using SQLCatalog to generate manualy a part of another query.
+        That table has a special position in returned query:
+         - all other tables are joined on this one (when it is required to use
+           other tables)
+         - it is expected to have some columns (uid, path)
+        It is strongly discouraged to use this parameter for any value other
+        than None.
+      group_by
+        Use group_by_list instead.
+      group_by_expression
+        Use group_by_list instead.
+      select_expression
+        Use select_list or select_dict instead.
+      sort_on
+        Use order_by_list instead.
+      sort_order
+        Use order_by_list instead.
+      from_expression
+        This value will be emited in SQL directly in addition to computed
+        value.
+        There is no replacement.
+      where_expression
+        This value will be emited in SQL directly in addition to computed
+        value.
+        Use Query instances instead.
+      select_expression_key
+        This prevents given column from being ignored even if they could not
+        be mapped.
+        There is no replacement.
+      only_group_columns
+        Replaces former stat__ parameter.
+        Used to globally disalow use of non-group columns in SQL.
+    """
+
+  def getSearchKey(column, search_key=None):
+    """
+      Returns the default SearchKey instance for the
+      requested column. There is one instance per
+      search_key (incl. virtual keys surch as
+      source_title) and we try to compute it once
+      only and cache it.
+      If search_key is provided, it is used as the
+      name of the search key class to return.
+    """
+
+  def getComparisonOperator(operator):
+    """
+      Return a comparison operator matching given string.
+      String must be a valid SQL comparison operator (=, LIKE, IN, ...).
+      String case does not matter.
+      There is one comparison operator instance per possible string value.
+    """
+
+  # TODO: add support for other operators (logical, ensemblist (?))
+
+  def searchResults(REQUEST=None, **kw):
+    """
+      Invokes queryResults with the appropriate
+      ZSQL Method to return a list of results
+    """
+
+  def countResults(REQUEST=None, **kw):
+    """
+      Invokes queryResults with the appropriate
+      ZSQL Method to return a statistics for
+      a list of results
+    """
+
+  def queryResults(sql_method, REQUEST=None, src__=0, build_sql_query_method=None, **kw):
+    """
+      Return the result of the given 'sql_method' ZSQL Method after
+      processing all parameters to build a Query object passed to
+      that method.
+
+      The implementation should do the following.
+
+      1- Use **kw parameters to build a Query object
+         by invoking buildQuery
+
+      2- Build a ColumnMap instance by invoking
+         the buildColumnMap on the Query. Some
+         optmisation may happen here to try
+         to build the best possible ColumnMap and
+         use the best possible indices for joining.
+         During the ColumnMap build process, the
+         Search Key associated to each Query node
+         in the Query tree registers the columns
+         which are used (ex. to search) or provided
+         (ex. MATCH value for full text search, 
+         interleave expression or parameter in a
+         UNION Query)
+
+      3- Render the query object as an SQLExpression
+         instance. This instance contains all necessary
+         parts to generate:
+           - where_expression
+           - sort_expression
+           - group_by_expression
+           - select_expression
+
+      4- Invoke sql_method
+    """

product/ZSQLCatalog/Interface/ISQLExpression.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class ISQLExpression(Interface):
+  """
+    This is a container for chunks of SQL generated by a Query.
+    Each Query instance generates its own SQLExpression instance.
+    SQLExpressions can be nested (an SQLExpression can contain 0, 1 or more
+    SQLExpressions).
+
+    Chunks are:
+    - table_alias_dict
+    - order_by_list
+    - group_by_list
+    - select_dict
+    - limit
+    Mutualy exclusive chunks are:
+    - where expression
+    - sql_expression_list
+    Deprecated chunks are:
+    - from_expression
+
+    Providing sql_expression_list with more than one entry makes
+    where_expression_operator mandatory.
+  """
+
+  def __init__(query,
+               table_alias_dict=None,
+               order_by_list=None,
+               order_by_dict=None,
+               group_by_list=None,
+               where_expression=None,
+               where_expression_operator=None,
+               sql_expression_list=None,
+               select_dict=None,
+               limit=None,
+               from_expression=None):
+    """
+      Instantiate an SQLExpression object.
+
+      This method does consistency checks on received parameters, so that
+      failures are detected as early as possible.
+
+      Also, it casts most optional parameters into empty lists and empty dicts
+      to make code using those values simpler.
+
+      query (Query)
+        The Query instance which called this constructor.
+      table_alias_dict (dict, key: string, value: string)
+        Table alias dict as returned by ColumnMap.getTableAliasDict() .
+      order_by_list (list of strings)
+        List of result ordering, pre-rendered.
+      order_by_dict (dict, key: string, value: string)
+        Column rendering replacement specific to order_by.
+      group_by_list (list of strings)
+        List of column names on which result line list will be grouped.
+      where_expression (string)
+        Text representing a "where" expression of an SQL query.
+      where_expression_operator ("and", "or", "not", None)
+        Operator to apply on immediately contained SQLExpressions.
+        It must be "and" or "or" when there are multiple contained
+        SQLExpressions, it can be "not" if there is exactly one contained
+        SQLExpression, and must not be provided if there is no contained
+        SQLExpression.
+      sql_expression_list (list of SQLExpression)
+        List of immediately contained SQLExpressions.
+      select_dict (dict, key:string, value:string or Null)
+        Lists all columns to be part of select expression.
+        Key is column alias.
+        Value is column name, or Null. If it is Null, the alias will also be
+        used as column name.
+      limit (1-tuple, 2-tuple, other)
+        First item is the number of lines expected, second one if given is the
+        offset of limited result list within the unlimited result list.
+        If it is not a tuple it is treated as the value of a 1-tuple
+        parameter.
+      from_expression (string)
+        This parameter is deprecated.
+    """
+
+  def asSQLExpressionDict():
+    """
+      Returns a dictionnary usable as a **kw for a catalog sql method.
+
+      It renders aliases (see getTableAliasDict) as a list of strings and
+      "from" expression (see getFromExpression) as a list of strings.
+
+      See getWhereExpression, getOrderByExpression, getLimitExpression,
+      getSelectExpression, getGroupByExpression.
+    """
+

product/ZSQLCatalog/Interface/ISearchKey.py

+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class ISearchKey(Interface):
+  """
+    A SearchKey generates Query instances.
+    It is responsible for parsing (or not) input value and generating the
+    appropriate Query structure to match the intended search.
+
+    This class is designed as a "singleton per column".
+  """
+
+  def __init__(column):
+    """
+      column (string)
+        The column this SearchKey is instanciated for.
+    """
+
+  def getColumn():
+    """
+      Returns: string
+        The column this SearchKey is instanciated for.
+    """
+
+  def buildSQLExpression(operator, value, column_map, only_group_columns, group):
+    """
+      This method passes on SQLExpression creation to operator, giving it the
+      resolved column of this RelatedKey.
+
+      operator (Operator)
+        A comparison operator. It is used to render value, column and itself
+        as valid SQL.
+      value (anything)
+      column_map (ColumnMap)
+        The (built) column map to render our column.
+      group (string)
+        The group our column was part of.
+
+      Returns: SQLExpression
+        SQLExpression build by operator for current Query.
+    """
+
+  def buildSearchTextExpression(operator, value, column=None):
+    """
+      Render comparison using operator between value and our column as
+      SearchText.
+
+      operator (Operator)
+        A comparison operator. It is used to render value as value SearchText.
+      value (anything)
+      column (string)
+        If given, it overrides our column in SearchText rendering. This is
+        useful when SearchText rendering is done for a virtual column,
+        because the name of the virtual column must appear when rendered as
+        SearchText.
+    """
+
+  def registerColumnMap(column_map, group, simple_query):
+    """
+      Register the column of this SearchKey to given column map.
+
+      column_map (ColumnMap)
+        The ColumnMap instance to register to.
+      group (string)
+        The group registration will be part of.
+      simple_query (SimpleQuery)
+        The SimpleQuery being registered.
+
+      Returns: string
+        The group assigned to caller. See ColumnMap for methods redefining
+        caller's group.
+    """
+
+  def buildQuery(search_value, group=None, logical_operator=None, comparison_operator=None):
+    """
+      Generate queries from given search_value.
+      logical_operator ('and', 'or', 'not', None)
+        If no logical operator can be found in search_value, it will use given
+        default operator.
+      comparison_operator (string, None)
+        If given, expresses the comparison between column and value.
+    """
+
+  def parseSearchText(value):
+    """
+      Parse given value to generate an Abstract Syntax Tree representing its
+      logical structure, or None if there is no obvious structure in given
+      value.
+      See SearchText for parsing code.
+
+      value (string)
+        The string to parse.
+
+      Returns: (None, AbstratSyntaxNode)
+        AbstratSyntaxNode complies with the IAbstractSyntaxNode interface.
+    """
+
+class IRelatedKey(ISearchKey):
+  """
+    A RelatedKey is a special variation of a SearchKey.
+    Only a small set of methods differ. They are defined in this class.
+  """
+
+  def registerColumnMap(column_map, table_alias_list=None):
+    """
+      This is slightly different for regular registerColumnMap in that it must
+      register multiple tables (and not columns, since RelatedKeys do not
+      provide this information).
+      Also, it must register namely "catalog" table and resolve its alias,
+      angain since it's hardcoded in RelatedKey ZSQLMethods and not provided
+      by their definitions.
+
+      column_map (ColumnMap)
+
+      Deprecated:
+        table_alias_list (None, list of 2-tuples of strings)
+          This list must have the exact same length as the list of tables 
+    """
+
+  def buildSQLExpression(sql_catalog, column_map, only_group_columns, group):
+    """
+      operator and value parameters are useless, since this type of SearhKey
+      does not compare a value to any column, but uses a ZSQLMethod.
+      To reach that ZSQLMethod, it also required a new sql_catalog parameter.
+
+      sql_catalog (SQLCatalog)
+        Used to retrieve related key's ZSQLMethod.
+    """
+
+  def buildQuery(sql_catalog, related_key_definition, search_value=None, search_key_name=None, logical_operator=None, comparison_operator=None):
+    """
+      group is useless here, since group is determined by ColumnMap at
+      registration time. search_value becomes optional.
+
+      sql_catalog (SQLCatalog)
+        Used to retrieve real column's SearchKey. For example, a RelatedKey
+        used to compare with a "title" column will retrieve title's default
+        SearchKey (which should be a KeywordKey).
+      related_key_definition (string)
+        Describes parameters of a RelatedKey. It is composed of 3 mains parts,
+        separated by '/':
+        - a list of table names
+          Table names are separated by ','
+        - a column name
+        - the name of the related key ZSQLMethod
+      search_value (anything)
+        If given, a condition on real column will be generated.
+        Otherwise, only the SQL required to reach that column will be
+        generated. This is usefull when sorting on a virtual column, for
+        example.
+      search_key_name (string, None)
+        If given, it overrides real column's default SearchKey.
+      logical_operator (string, None)
+        If given, expresses the default logical link between operands.
+        It must be one of None, 'or' and 'and'.
+        It is overriden by operator present in search_value if it is a dict
+        and contains an 'operator' key.
+        'or' is assumed if not given or given with a None value.
+      comparison_operator (string, None)
+        If given, expresses the comparison between column and value.
+    """
+

product/ZSQLCatalog/Interface/Verify.py

+# This imports from Zope's products, which would otherwise be unreachable from parent folder.
+from Interface.Verify import verifyClass
+

product/ZSQLCatalog/Interface/__init__.py

+

product/ZSQLCatalog/Operator/ComparisonOperator.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from OperatorBase import OperatorBase
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IOperator import IOperator
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class ComparisonOperatorBase(OperatorBase):
+  @profiler_decorator
+  def asSQLExpression(self, column, value_list, only_group_columns):
+    """
+      In a Comparison Operator, rendering order is:
+        <column> <operator> <value_list>
+    """
+    column, value_list = self.render(column, value_list)
+    return SQLExpression(self, where_expression='%s %s %s' % (column, self.getOperator().upper(), value_list))
+
+  def render(self, column, value_list):
+    raise NotImplementedError, 'This method must be overloaded by a subclass.'
+
+  def renderValue(self, value_list):
+    raise NotImplementedError, 'This method must be overloaded by a subclass.'
+
+verifyClass(IOperator, ComparisonOperatorBase)
+
+class MonovaluedComparisonOperator(ComparisonOperatorBase):
+  @profiler_decorator
+  def renderValue(self, value_list):
+    """
+      value_list must either be a non-list or a single-value list.
+    """
+    if isinstance(value_list, (tuple, list)):
+      if len(value_list) > 1:
+        raise ValueError, '%r: value_list must not contain more than one item. Got %r' % (self, value_list)
+      value_list = value_list[0]
+    return self._renderValue(value_list)
+
+  @profiler_decorator
+  def render(self, column, value_list):
+    """
+      value_list must either be a non-list or a single-value list.
+    """
+    if isinstance(value_list, (tuple, list)):
+      if len(value_list) > 1:
+        raise ValueError, '%r: value_list must not contain more than one item. Got %r' % (self, value_list)
+      value_list = value_list[0]
+    return self._render(column, value_list)
+
+verifyClass(IOperator, MonovaluedComparisonOperator)
+
+class MultivaluedComparisonOperator(ComparisonOperatorBase):
+  @profiler_decorator
+  def renderValue(self, value_list):
+    """
+      value_list must be a multi-value list (more than one item).
+    """
+    if not isinstance(value_list, (tuple, list)) or len(value_list) < 2:
+      raise ValueError, '%r: value_list must be a list of more than one item. Got %r' % (self, value_list)
+    return '(%s)' % (', '.join([self._renderValue(x) for x in value_list]), )
+
+  @profiler_decorator
+  def render(self, column, value_list):
+    """
+      value_list must be a multi-value list (more than one item).
+    """
+    if not isinstance(value_list, (tuple, list)) or len(value_list) < 2:
+      raise ValueError, '%r: value_list must be a list of more than one item. Got %r' % (self, value_list)
+    return column, '(%s)' % (', '.join([self._renderValue(x) for x in value_list]), )
+
+verifyClass(IOperator, MultivaluedComparisonOperator)
+
+class MatchComparisonOperator(MonovaluedComparisonOperator):
+  def __init__(self, operator, mode=''):
+    MonovaluedComparisonOperator.__init__(self, operator)
+    self.where_expression_format_string = 'MATCH (%%(column)s) AGAINST (%%(value_list)s%s)' % (mode, )
+
+  @profiler_decorator
+  def asSQLExpression(self, column, value_list, only_group_columns):
+    """
+      This operator can emit a select expression, so it overrides
+      asSQLExpression inseatd of just defining a render method.
+    """
+    match_string = self.where_expression_format_string % {
+      'column': column,
+      'value_list': self.renderValue(value_list),
+    }
+    select_dict = {}
+    if not only_group_columns:
+      select_dict[column.replace('`', '').split('.')[-1]] = match_string
+    # Sort on this column uses relevance.
+    # TODO: Add a way to allow sorting by raw column value.
+    order_by_dict = {
+      column: self.where_expression_format_string,
+    }
+    return SQLExpression(
+      self,
+      select_dict=select_dict,
+      where_expression=match_string,
+      order_by_dict=order_by_dict,
+    )
+
+verifyClass(IOperator, MatchComparisonOperator)
+
+operator_dict = {
+  '=': MonovaluedComparisonOperator('='),
+  '!=': MonovaluedComparisonOperator('!='),
+  '>': MonovaluedComparisonOperator('>'),
+  '<': MonovaluedComparisonOperator('<'),
+  '<=': MonovaluedComparisonOperator('<='),
+  '>=': MonovaluedComparisonOperator('>='),
+  'like': MonovaluedComparisonOperator('like'),
+  'match': MatchComparisonOperator('match'),
+  'match_boolean': MatchComparisonOperator('match_boolean', mode=' IN BOOLEAN MODE'),
+  'match_expansion': MatchComparisonOperator('match_expansion', mode=' WITH QUERY EXPANSION'),
+  'in': MultivaluedComparisonOperator('in'),
+  'is': MonovaluedComparisonOperator('is'),
+}
+

product/ZSQLCatalog/Operator/OperatorBase.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.IOperator import IOperator
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+@profiler_decorator
+def escapeString(value):
+  # Inspired from ERP5Type/Utils:sqlquote, but this product must not depend on it.
+  return "'" + value.replace('\\', '\\\\').replace("'", "''") + "'"
+
+@profiler_decorator
+def valueFloatRenderer(value):
+  if isinstance(value, basestring):
+    value = float(value.replace(' ', ''))
+  return repr(value)
+
+@profiler_decorator
+def valueDateTimeRenderer(value):
+  return '"%s"' % (value.toZone('UTC').ISO(), )
+
+@profiler_decorator
+def valueDefaultRenderer(value):
+  LOG('OperatorBase', 0, 'Unhandled value class: %s (%r). Converted to string and escaped.' % (value.__class__.__name__, value))
+  return escapeString(str(value))
+
+@profiler_decorator
+def valueNoneRenderer(value):
+  return 'NULL'
+
+value_renderer = {
+  'int': str,
+  'long': str,
+  'float': valueFloatRenderer,
+  'DateTime': valueDateTimeRenderer,
+  'NoneType': valueNoneRenderer,
+}
+
+value_search_text_renderer = {
+  'DateTime': str,
+}
+
+@profiler_decorator
+def valueDefaultSearchTextRenderer(value):
+  """
+    This is just repr, but always surrounding text strings with doublequotes.
+  """
+  if isinstance(value, basestring):
+    result = '"%s"' % (value.replace('\\', '\\\\').replace('"', '\\"'), )
+  else:
+    result = repr(value)
+  return result
+
+@profiler_decorator
+def columnFloatRenderer(column, format=None):
+  if format is not None:
+    if '.' in format:
+      format = format.replace(' ', '')
+      column = "TRUNCATE(%s, %s)" % (column, len(format.split('.')[-1]))
+  return column
+
+@profiler_decorator
+def columnDefaultRenderer(column, format=None):
+  return column
+
+column_renderer = {
+  'float': columnFloatRenderer
+}
+
+class OperatorBase(object):
+
+  __implements__ = IOperator
+
+  def __init__(self, operator):
+    self.operator = operator
+
+  def getOperator(self):
+    return self.operator
+
+  @profiler_decorator
+  def _render(self, column, value):
+    """
+      Render given column and value for use in SQL.
+      Value is rendered to convert it to SQL-friendly value.
+      Column is rendered to include possible cast code.
+
+      column (string)
+        Column on which the value will be matched
+      value (see _renderValue)
+        Value to render.
+    """
+    if isinstance(value, dict):
+      type = value['type']
+      column = column_renderer.get(type, columnDefaultRenderer)(column, format=value['format'])
+      value = value_renderer.get(type, valueDefaultRenderer)(value['query'])
+    else:
+      value = self._renderValue(value)
+    return column, value
+
+  @profiler_decorator
+  def _renderValue(self, value):
+    """
+      Render given value as string.
+
+      value (int, float, long, DateTime, string, None)
+        Value to render as a string for use in SQL (quoted, escaped).
+    """
+    if isinstance(value, basestring):
+      value = escapeString(value)
+    else:
+      value = value_renderer.get(value.__class__.__name__, valueDefaultRenderer)(value)
+    return value
+
+  @profiler_decorator
+  def asSearchText(self, value):
+    return value_search_text_renderer.get(value.__class__.__name__, valueDefaultSearchTextRenderer)(value)
+
+  def asSQLExpression(self, column, value_list, only_group_columns):
+    raise NotImplementedError, 'This method must be overloaded by a subclass ' \
+      'to be able to get an SQL representation of this operator.'
+
+  def __repr__(self):
+    return '<%s(%r) at %s>' % (self.__class__.__name__, self.getOperator(), id(self))
+
+verifyClass(IOperator, OperatorBase)
+

product/ZSQLCatalog/Operator/__init__.py

+from ComparisonOperator import operator_dict
+

product/ZSQLCatalog/Query/AutoQuery.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Query import Query
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class AutoQuery(Query):
+  """
+    An AutoQuery is a compatibility layer for former Query class.
+    It passes parameters given at instantiation time to SQLCatalog's
+    buildQuery or buildSingleQuery, and wraps resulting Query instance (proxy
+    behaviour).
+
+    This is only here for backward compatibility, and use is strongly
+    discouraged. Use SQLCatalog API instead.
+  """
+  wrapped_query = None
+
+  @profiler_decorator
+  def __init__(self, *args, **kw):
+    """
+      Note: "operator" might contain a logical or a comparison operator.
+    """
+    if len(args):
+      LOG('AutoQuery', 100, 'Got extra positional parameters (will be ignored): %r' % (args, ))
+    self.table_alias_list = kw.pop('table_alias_list', None)
+    self.kw = kw
+    operator = kw.pop('operator', None)
+    if isinstance(operator, basestring):
+      operator = operator.lower()
+    self.operator = operator
+    self.ignore_empty_string = kw.pop('ignore_empty_string', True)
+    if 'key' in kw and len(kw) > 2:
+      raise ValueError, '"key" parameter cannot be used when more than one column is given. key=%r' % (self.search_key, )
+    self.search_key = kw.pop('key', None)
+
+  @profiler_decorator
+  def _createWrappedQuery(self, sql_catalog):
+    """
+      Create wrapped query. This requires being able to reach catalog, since
+      we use it as a query producer.
+    """
+    kw = self.kw
+    operator = self.operator
+    if 'range' in kw:
+      # If we received a range parameter we are building a single query.
+      # Recreate value as a dict and pass it to buildSingleQuery.
+      range = kw.pop('range')
+      assert len(kw) == 1, repr(kw)
+      key, value = kw.items()[0]
+      query = sql_catalog.buildSingleQuery(key, {'query': value,
+                                                 'range': range})
+    elif operator == 'in':
+      # 'in' is a *comparison* operator, not a logical operator.
+      # Transform kw into the proper form.
+      assert len(kw) == 1, repr(kw)
+      key, value = kw.items()[0]
+      query = sql_catalog.buildSingleQuery(key, {'query': value,
+                                                 'operator': operator})
+    elif len(kw) == 1 and isinstance(kw.values()[0], (tuple, list)) and \
+       operator in ('and', 'or'):
+      # If there is only one parameter, and operator was given and is a
+      # known logical operator, then operator will apply to it.
+      # For example (from testDomainTool):
+      #  kw = {'portal_type': ['!=a', '!=b'], 'operator': 'AND'}
+      #  In such case, expected result is
+      #  "portal_type!='a' AND portal_type!='b'"
+      key, value = kw.items()[0]
+      query = sql_catalog.buildSingleQuery(key, value, logical_operator=operator)
+    else:
+      # Otherwise, the operator will apply to the relationship between
+      # parameters.
+      if operator is None:
+        operator = 'and'
+      if self.search_key is not None:
+        key, value = kw.items()[0]
+        kw = {key: {'query': value, 'key': self.search_key}}
+      query = sql_catalog.buildQuery(kw, operator=operator, ignore_empty_string=self.ignore_empty_string)
+    if self.table_alias_list is not None:
+      query.setTableAliasList(self.table_alias_list)
+    self.wrapped_query = query
+
+  @profiler_decorator
+  def asSearchTextExpression(self, sql_catalog, column=None):
+    if self.wrapped_query is None:
+      self._createWrappedQuery(sql_catalog)
+    return self.wrapped_query.asSearchTextExpression(sql_catalog, column=column)
+
+  @profiler_decorator
+  def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+    if self.wrapped_query is None:
+      self._createWrappedQuery(sql_catalog)
+    return self.wrapped_query.asSQLExpression(sql_catalog, column_map, only_group_columns=only_group_columns)
+
+  @profiler_decorator
+  def registerColumnMap(self, sql_catalog, column_map):
+    if self.wrapped_query is None:
+      self._createWrappedQuery(sql_catalog)
+    return self.wrapped_query.registerColumnMap(sql_catalog, column_map)
+
+  def __repr__(self):
+    if self.wrapped_query is None:
+      result = '<%s(**%r) at %s>' % (self.__class__.__name__, self.kw, id(self))
+    else:
+      result = '<%s %r>' % (self.__class__.__name__, self.wrapped_query)
+    return result
+
+verifyClass(IQuery, AutoQuery)
+

product/ZSQLCatalog/Query/ComplexQuery.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -26,68 +28,127 @@
 #
 ##############################################################################
 
-from Products.PythonScripts.Utility import allow_class
-from Query import QueryMixin
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from SQLQuery import SQLQuery
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
 
-class ComplexQuery(QueryMixin):
+class ComplexQuery(Query):
   """
-  Used in order to concatenate many queries
+    A ComplexQuery represents logical operations between Query instances.
   """
-
+  @profiler_decorator
   def __init__(self, *args, **kw):
-    self.query_list = args
-    self.operator = kw.pop('operator', 'AND')
-    # XXX: What is that used for ?! It's utterly dangerous.
-    #self.__dict__.update(kw)
-
-  def getQueryList(self):
-    return self.query_list
-
-  def getRelatedTableMapDict(self):
-    result = {}
-    for query in self.getQueryList():
-      if not(isinstance(query, basestring)):
-        result.update(query.getRelatedTableMapDict())
-    return result
-
-  def asSQLExpression(self, key_alias_dict=None,
-                            ignore_empty_string=1,
-                            keyword_search_keys=None,
-                            datetime_search_keys=None,
-                            full_text_search_keys=None,
-                            stat__=0):
     """
-    Build the sql string
+      *args (tuple of Query or of list of Query)
+        list-type entry will extend subquery list, other entries will be
+        appended.
+      logical_operator ('and', 'or', 'not')
+        Logical operator.
+        Default: 'and'
+
+      Deprecated 
+        operator ('and', 'or', 'not')
+          See logical_operator.
+          logical_operator takes precedence if given.
+        unknown_column_dict (dict)
+          Only one key of this dictionnary is used here:
+            key: 'from_expression'
+            value: string
+            This value will be passed through to SQLExpression. If it is
+            provided, this ComplexQuery must have no subquery (regular
+            SQLExpression limitation)
+        implicit_table_list (list of strings)
+          Each entry in this list will be registered to column map. This is
+          used to make column mapper choose tables differently.
     """
-    sql_expression_list = []
-    select_expression_list = []
-    for query in self.getQueryList():
-      if isinstance(query, basestring):
-        sql_expression_list.append(query)
+    self.logical_operator = kw.pop('logical_operator', kw.pop('operator', 'and')).lower()
+    assert self.logical_operator in ('and', 'or', 'not'), self.logical_operator
+    unknown_column_dict = kw.pop('unknown_column_dict', {})
+    self.from_expression = unknown_column_dict.pop('from_expression', None)
+    self.implicit_table_list = kw.pop('implicit_table_list', [])
+    query_list = []
+    append = query_list.append
+    extend = query_list.extend
+    # Flaten the first level of list-type arguments
+    for arg in args:
+      if isinstance(arg, (list, tuple)):
+        extend(arg)
+      else:
+        append(arg)
+    new_query_list = []
+    append = new_query_list.append
+    # Iterate over the flaten argument list to cast each into a query type.
+    for query in query_list:
+      if not isinstance(query, Query):
+        query = SQLQuery(query)
+      append(query)
+    self.query_list = new_query_list
+
+  @profiler_decorator
+  def asSearchTextExpression(self, sql_catalog, column=None):
+    if column in (None, ''):
+      query_column = column
+    else:
+      query_column = ''
+    search_text_list = [y for y in [x.asSearchTextExpression(sql_catalog, column=query_column) for x in self.query_list] if y is not None]
+    if len(search_text_list) == 0:
+      result = ''
     else:
-        query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
-                               ignore_empty_string=ignore_empty_string,
-                               keyword_search_keys=keyword_search_keys,
-                               datetime_search_keys=datetime_search_keys,
-                               full_text_search_keys=full_text_search_keys,
-                               stat__=stat__)
-        sql_expression_list.append(query_result['where_expression'])
-        select_expression_list.extend(query_result['select_expression_list'])
-    operator = self.getOperator()
-    result = {'where_expression':('(%s)' %  \
-                         (' %s ' % operator).join(['(%s)' % x for x in sql_expression_list])),
-              'select_expression_list':select_expression_list}
+      if self.logical_operator in ('and', 'or'):
+        if len(search_text_list) == 1:
+          result = search_text_list[0]
+        else:
+          logical_operator = ' %s ' % (self.logical_operator.upper(), )
+          result = '(%s)' % (logical_operator.join(search_text_list), )
+      elif self.logical_operator == 'not':
+        assert len(search_text_list) == 1
+        result = '(NOT %s)' % (search_text_list[0], )
+      else:
+        raise ValueError, 'Unknown operator %r' % (self.logical_operator, )
+      if column not in (None, ''):
+        result = '%s:%s' % (column, result)
     return result
 
-  def getSQLKeyList(self):
+  @profiler_decorator
+  def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+    sql_expression_list = [x.asSQLExpression(sql_catalog, column_map, only_group_columns)
+                           for x in self.query_list]
+    if len(sql_expression_list) == 0:
+      sql_expression_list = [SQLExpression(self, where_expression='1')]
+    return SQLExpression(self,
+      sql_expression_list=sql_expression_list,
+      where_expression_operator=self.logical_operator,
+      from_expression=self.from_expression)
+
+  @profiler_decorator
+  def registerColumnMap(self, sql_catalog, column_map):
+    for implicit_table_column in self.implicit_table_list:
+      column_map.registerColumn(implicit_table_column)
+    for query in self.query_list:
+      query.registerColumnMap(sql_catalog, column_map)
+
+  def __repr__(self):
+    return '<%s of %r.join(%r)>' % (self.__class__.__name__, self.logical_operator, self.query_list)
+
+  @profiler_decorator
+  def setTableAliasList(self, table_alias_list):
     """
-    Returns the list of keys used by this
-    instance
+      This function is here for backward compatibility.
+      This can only be used when there is one and only one subquery which
+      defines a setTableAliasList method.
+      
+      See RelatedQuery.
     """
-    key_list=[]
-    for query in self.getQueryList():
-      if not(isinstance(query, basestring)):
-        key_list.extend(query.getSQLKeyList())
-    return key_list
+    assert len(self.query_list) == 1
+    self.query_list[0].setTableAliasList(table_alias_list)
+
+  @profiler_decorator
+  def setGroup(self, group):
+    for query in self.query_list:
+      query.setGroup(group)
+
+verifyClass(IQuery, ComplexQuery)
 
-allow_class(ComplexQuery)

product/ZSQLCatalog/Query/EntireQuery.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.ColumnMap import ColumnMap
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.IEntireQuery import IEntireQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class EntireQuery(object):
+  """
+    This is not a Query subclass, since it does not define a
+    registerColumnMap method, and instead does the ColumnMap handling
+    internaly.
+  """
+
+  __implements__ = IEntireQuery
+
+  column_map = None
+
+  @profiler_decorator
+  def __init__(self, query, order_by_list=None, group_by_list=None,
+               select_dict=None, limit=None, catalog_table_name=None,
+               extra_column_list=None, from_expression=None,
+               order_by_override_list=None):
+    def default(value):
+      if value is None:
+        return []
+      assert isinstance(value, (tuple, list))
+      return value
+    def defaultDict(value):
+      if value is None:
+        return {}
+      assert isinstance(value, dict)
+      return value
+    self.query = query
+    self.order_by_list = default(order_by_list)
+    self.order_by_override_set = frozenset(default(order_by_override_list))
+    self.group_by_list = default(group_by_list)
+    self.select_dict = defaultDict(select_dict)
+    self.limit = limit
+    self.catalog_table_name = catalog_table_name
+    self.extra_column_list = default(extra_column_list)
+    self.from_expression = from_expression
+
+  def asSearchTextExpression(self, sql_catalog):
+    return query.asSearchTextExpression(sql_catalog)
+
+  @profiler_decorator
+  def asSQLExpression(self, sql_catalog, only_group_columns):
+    column_map = self.column_map
+    if column_map is None:
+      # XXX: should we provide a way to register column map as a separate mathod or do it here ?
+      # Column Map was not built yet, do it.
+      self.column_map = column_map = ColumnMap(catalog_table_name=self.catalog_table_name)
+      for extra_column in self.extra_column_list:
+        table, column = extra_column.replace('`', '').split('.')
+        if table != self.catalog_table_name:
+          raise ValueError, 'Extra columns must be catalog columns. %r does not follow this rule (catalog=%r, extra_column_list=%r)' % (extra_column, self.catalog_table_name, self.extra_column_list)
+        column_map.registerColumn(extra_column)
+      for column in self.group_by_list:
+        column_map.registerColumn(column)
+      for alias, column in self.select_dict.iteritems():
+        if column is None:
+          column = alias
+        else:
+          column_map.ignoreColumn(alias)
+        column_map.registerColumn(column)
+      for override in self.order_by_override_set:
+        column_map.ignoreColumn(override)
+      for order_by in self.order_by_list:
+        assert isinstance(order_by, (tuple, list))
+        assert len(order_by)
+        column_map.registerColumn(order_by[0])
+      self.query.registerColumnMap(sql_catalog, column_map)
+      column_map.build(sql_catalog)
+      # Replace given group_by_list entries by their mapped representations.
+      new_column_list = []
+      append = new_column_list.append
+      for column in self.group_by_list:
+        try:
+          append(column_map.asSQLColumn(column))
+        except KeyError:
+          LOG('EntireQuery', 100, 'Group-by column %r could not be mapped, but is passed through. This use is strongly discouraged.' % (column, ))
+          append(column)
+      self.group_by_list = new_column_list
+      # Build a dictionnary from select_dict aliasing their mapped representations
+      self.final_select_dict = select_dict = {}
+      for alias, raw_column in self.select_dict.iteritems():
+        if raw_column is None:
+          column = alias
+        else:
+          column = raw_column
+        try:
+          rendered = column_map.asSQLColumn(column)
+        except KeyError:
+          LOG('EntireQuery', 100, 'Select column %r could not be mapped, but is passed through. This use is strongly discouraged.' % (column, ))
+          rendered = column
+        select_dict[alias] = rendered
+      # Replace given order_by_list entries by their mapped representations.
+      new_order_by_list = []
+      append = new_order_by_list.append
+      for order_by in self.order_by_list:
+        column = order_by[0]
+        if column in self.order_by_override_set:
+          LOG('EntireQuery', 100, 'Order-by column %r is forcibly accepted. This use is strongly discouraged.' % (column, ))
+          rendered = column
+        else:
+          try:
+            rendered = column_map.asSQLColumn(column)
+          except KeyError:
+            LOG('SQLCatalog', 100, 'Order by %r ignored: it could not be mapped to a known column.' % (order_by, ))
+            rendered = None
+        if rendered is not None:
+          if len(order_by) > 1:
+            if len(order_by) > 2 and order_by[2] not in (None, ''):
+              rendered = 'CAST(%s AS %s)' % (rendered, order_by[2])
+            rendered = '%s %s' % (rendered, order_by[1])
+          append(rendered)
+      self.order_by_list = new_order_by_list
+      # generate SQLExpression from query
+      sql_expression_list = [self.query.asSQLExpression(sql_catalog, column_map, only_group_columns)]
+      # generate join expression based on column_map.getJoinTableAliasList
+      append = sql_expression_list.append
+      for join_query in column_map.iterJoinQueryList():
+        append(join_query.asSQLExpression(sql_catalog, column_map, only_group_columns))
+      join_table_list = column_map.getJoinTableAliasList()
+      if len(join_table_list):
+        # XXX: Is there any special rule to observe when joining tables ?
+        # Maybe we could check which column is a primary key instead of
+        # hardcoding "uid".
+        where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
+          (column_map.getCatalogTableAlias(), )
+        # XXX: It would cleaner from completeness point of view to use column
+        # mapper to render column, but makes code much more complex to just do
+        # a simple text rendering. If there is any reason why we should have
+        # those column in the mapper, then we should use the clean way.
+        append(SQLExpression(self, where_expression=' AND '.join(
+          where_pattern % (x, ) for x in join_table_list
+        )))
+      self.sql_expression_list = sql_expression_list
+    return SQLExpression(
+      self,
+      table_alias_dict=column_map.getTableAliasDict(),
+      from_expression=self.from_expression,
+      order_by_list=self.order_by_list,
+      group_by_list=self.group_by_list,
+      select_dict=self.final_select_dict,
+      limit=self.limit,
+      where_expression_operator='and',
+      sql_expression_list=self.sql_expression_list)
+
+verifyClass(IEntireQuery, EntireQuery)
+

product/ZSQLCatalog/Query/Query.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -26,117 +28,35 @@
 #
 ##############################################################################
 
-from DocumentTemplate.DT_Var import sql_quote
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
 
-class QueryMixin:
+class Query(object):
   """
-    Mixing class which implements methods which are
-    common to all kinds of Queries
+    This is the base class of all kind of queries. Its only purpose is to be
+    able to distinguish any kind of value from a query.
   """
-  operator = None
-  format = None
-  type = None
 
-  def __call__(self, **kw):
-    return self.asSQLExpression(**kw)  
+  __implements__ = IQuery
+  __allow_access_to_unprotected_subobjects__ = 1
 
-  def getOperator(self):
-    return self.operator
-
-  def getFormat(self):
-    return self.format
-
-  def getType(self):
-    return self.type
-
-  def getRange(self):
-    return self.range
-
-  def getTableAliasList(self):
-    return self.table_alias_list
-
-  def getSearchMode(self):
-    """Search mode used for Full Text search
+  def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
     """
-    return self.search_mode
-    
-  def getSearchKey(self):
-    """Search mode used for Full Text search
+      To enable SQL rendering, overload this method in a subclass.
     """
-    return self.search_key
-  
-  def getKey(self):
-    return self.key
-
-  def getValue(self):
-    return self.value 
+    raise TypeError, 'A %s cannot be rendered as an SQL expression.' % (self.__class__.__name__, )
 
-  def getOperator(self):
-    return self.operator.upper().strip()
-
-  def asSearchTextExpression(self):
-    raise NotImplementedError
-    
-  def asSQLExpression(self, key_alias_dict=None,
-                      keyword_search_keys=None,
-                      datetime_search_keys=None,
-                      full_text_search_keys=None,
-                      ignore_empty_string=1, stat__=0):
+  def asSearchTextExpression(self, sql_catalog, column=None):
     """
-      Return a dictionnary containing the keys and value types:
-        'where_expression': string
-        'select_expression_list': string
+      To enable Search Text rendering, overload this method in a subclass.
     """
-    raise NotImplementedError
+    raise TypeError, 'A %s cannot be rendered as a SearchText expression.' % (self.__class__.__name__, )
 
-  def getSQLKeyList(self):
+  def registerColumnMap(self, sql_catalog, column_map):
     """
-      Return a list of keys used by this query and its subqueries.
+      This method must always be overloaded by subclasses.
     """
-    raise NotImplementedError
+    raise NotImplementedError, '%s is incompeltely implemented.' % (self.__class__.__name__, )
 
-  def getRelatedTableMapDict(self):
-    """
-      Return for each key used by this query (plus ones used by its
-      subqueries) the table alias mapping.
-    """
-    raise NotImplementedError
+verifyClass(IQuery, Query)
 
-  def _quoteSQLString(self, value):
-    """Return a quoted string of the value.
-       XXX: Left for backwards compatability!
-    """
-    format = self.getFormat()
-    type = self.getType()
-    if format is not None and type is not None:
-      if type == 'date':
-        if hasattr(value, 'strftime'):
-          value = value.strftime(format)
-        if isinstance(value, basestring):
-          value = "STR_TO_DATE('%s','%s')" % (value, format)
-      if type == 'float':
-        # Make sure there is no space in float values
-        value = value.replace(' ','')
-        value = "'%s'" % value
-    else:
-      if getattr(value, 'ISO', None) is not None:
-        value = "'%s'" % value.toZone('UTC').ISO()
-      else:
-        value = "'%s'" % sql_quote(str(value))
-    return value
-
-  def _quoteSQLKey(self, key):
-    """Return a quoted string of the value.
-       XXX: Left for backwards compatability!
-    """
-    format = self.getFormat()
-    type = self.getType()
-    if format is not None and type is not None:
-      if type == 'date':
-        key = "STR_TO_DATE(DATE_FORMAT(%s,'%s'),'%s')" % (key, format, format)
-      if type == 'float':
-        float_format = format.replace(' ','')
-        if float_format.find('.') >= 0:
-          precision = len(float_format.split('.')[1])
-          key = "TRUNCATE(%s,%s)" % (key, precision)
-    return key    

product/ZSQLCatalog/Query/RelatedQuery.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class RelatedQuery(Query):
+  """
+    A RelatedQuery represents the is a container for a join condition.
+  """
+  @profiler_decorator
+  def __init__(self, search_key, join_condition=None, table_alias_list=None):
+    """
+      search_key (SearchKey)
+      join_condition (Query)
+        If given, it will be registered and rendered by this query.
+
+      Deprecated
+        table_alias_list (list of 2-tuple of strings)
+          See setTableAliasList.
+    """
+    self.search_key = search_key
+    self.join_condition = join_condition
+    self.table_alias_list = table_alias_list
+
+  @profiler_decorator
+  def setTableAliasList(self, table_alias_list):
+    """
+      This function is here for backward compatibility.
+
+      table_alias_list (list of 2-tuples of strings)
+        Each 2-tuple contains the name of a related key parameter and the
+        table alias it must be mapped on (respectively).
+    """
+    self.table_alias_list = table_alias_list
+
+  @profiler_decorator
+  def asSearchTextExpression(self, sql_catalog, column=None):
+    assert column is None
+    join_condition = self.join_condition
+    if join_condition is None:
+      result = None
+    else:
+      result = join_condition.asSearchTextExpression(sql_catalog, column=self.search_key.getColumn())
+    return result
+
+  @profiler_decorator
+  def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+    sql_expression_list = [self.search_key.buildSQLExpression(sql_catalog, column_map, only_group_columns, self.group)]
+    join_condition = self.join_condition
+    if join_condition is not None:
+      sql_expression_list.append(join_condition.asSQLExpression(sql_catalog, column_map, only_group_columns))
+    return SQLExpression(self, sql_expression_list=sql_expression_list, where_expression_operator='and')
+
+  @profiler_decorator
+  def registerColumnMap(self, sql_catalog, column_map):
+    self.group = self.search_key.registerColumnMap(column_map, table_alias_list=self.table_alias_list)
+    join_condition = self.join_condition
+    if join_condition is not None:
+      # Update its group
+      join_condition.setGroup(self.group)
+      # Propagate registration to embeded query
+      join_condition.registerColumnMap(sql_catalog, column_map)
+
+  def __repr__(self):
+    return '<%s on %r with %r>' % (self.__class__.__name__, self.search_key.getColumn(), self.join_condition)
+
+verifyClass(IQuery, RelatedQuery)
+

product/ZSQLCatalog/Query/SQLQuery.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+
+class SQLQuery(Query):
+  """
+    This Query subclass is used to wrap raw SQL text.
+    Use of this class is strongly discouraged, and it is only here for
+    backward compatibility.
+  """
+  def __init__(self, payload):
+    """
+      payload (string)
+        Raw SQL text.
+    """
+    if not isinstance(payload, basestring):
+      raise TypeError, 'Payload must be a string, got a %r: %r' % (type(payload), payload)
+    assert len(payload)
+    self.payload = '(' + payload + ')'
+
+  def asSearchText(self, sql_catalog):
+    return None
+
+  def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+    return SQLExpression(self, where_expression=self.payload)
+
+  def registerColumnMap(self, sql_catalog, column_map):
+    """
+      There is nothing to register for this type of Query subclass.
+    """
+    pass
+
+  def __repr__(self):
+    return '<%s (%r)>' % (self.__class__.__name__, self.payload)
+
+verifyClass(IQuery, SQLQuery)
+

product/ZSQLCatalog/Query/SimpleQuery.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -26,282 +28,104 @@
 #
 ##############################################################################
 
-from Products.PythonScripts.Utility import allow_class
-from DateTime import DateTime
-from Query import QueryMixin
-from pprint import pprint
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
 
-# valid search modes for queries
-FULL_TEXT_SEARCH_MODE = 'FullText'
-EXACT_MATCH_SEARCH_MODE = 'ExactMatch'
-KEYWORD_SEARCH_MODE = 'Keyword'
-DATETIME_SEARCH_MODE = 'DateTime'
-
-def isSimpleType(value):
-  return isinstance(value, basestring) or \
-         isinstance(value, int) or \
-         isinstance(value, long) or \
-         isinstance(value, float)
-
-# XXX Bad name JPS - NotQuery or NegativeQuery is better NegationQuery
-class NegatedQuery(QueryMixin):
+class SimpleQuery(Query):
   """
-    Do a boolean negation of given query.
+    A SimpleQuery represents a single comparison between a single column and
+    one or more values.
   """
-
-  def __init__(self, query):
-    self._query = query
-
-  def asSQLExpression(self, *args, **kw):
-    sql_expression_dict = self._query.asSQLExpression(*args, **kw)
-    sql_expression_dict['where_expression'] = '(NOT (%s))' % \
-      (sql_expression_dict['where_expression'], )
-    return sql_expression_dict
-
-  def getSQLKeyList(self, *args, **kw):
-    return self._query.getSQLKeyList(*args, **kw)
-
-  def getRelatedTableMapDict(self, *args, **kw):
-    return self._query.getRelatedTableMapDict(*args, **kw)
-
-allow_class(NegatedQuery)
-
-class SimpleQuery(QueryMixin):
+  @profiler_decorator
+  def __init__(self, search_key=None, operator='=', group=None, **kw):
     """
-  This allow to define constraints on a sql column
-
-  format - type date : %d/%m/%Y
-           type float : 1 234.12
+      search_key (None, SearchKey instance)
+        If given, the instance of SearchKey which is responsible for column
+        map registration and rendering (SQL and SearchText).
+      operator (string)
+        The comparison operator which will be applied between column and
+        values.
+        See Operator/ComparisonOperator.py for possible values.
+      group
+        See ColumnMap.
+      **kw
+        Must contain exactly one item.
+        item key (string)
+          column name
+        item value
+          one or more values
     """
-  
-  def __init__(self, format=None, operator=None, range=None, key=None,
-                     search_mode=None, table_alias_list=None, type=None, **kw):
-    self.format = format
-    if operator is None:
-      operator = 'OR'
+    self.search_key = search_key
+    if len(kw) != 1:
+      raise ValueError, 'SimpleQuery can support one and one only column. Got %r.' % (kw, )
+    self.column, value = kw.popitem()
+    # Backward compatibility code (those changes should not be needed when
+    # this Query is instanciated by a SearchKey, as operator should be correct
+    # already).
+    operator = operator.lower()
+    if operator == 'in':
+      if isinstance(value, (list, tuple)):
+        if len(value) == 0:
+          raise ValueError, 'Empty lists are not allowed.'
+        elif len(value) == 1:
+          value = value[0]
+          operator = '='
+      else:
+        operator = '='
+    elif operator == '=':
+      if isinstance(value, (list, tuple)):
+        if len(value) == 0:
+          raise ValueError, 'Empty lists are not allowed.'
+        elif len(value) == 1:
+          value = value[0]
+        else:
+          operator = 'in'
+    self.value = value
     self.operator = operator
-    self.range = range
-    self.search_mode = search_mode
-    self.table_alias_list = table_alias_list
-    key_list = kw.keys()
-    if len(key_list) != 1:
-      raise KeyError, 'Query must have only one key'
-    self.key = key_list[0]
-    self.value = kw[self.key]
-    self.type = type
-    self.search_key = key
-
-  def getRelatedTableMapDict(self):
-    result = {}
-    table_alias_list = self.getTableAliasList()
-    if table_alias_list is not None:
-      result[self.getKey()] = table_alias_list
-    return result
-    
-  def getSQLKeyList(self):
-    """
-    Returns the list of keys used by this
-    instance
-    """
-    return [self.getKey()]
+    self.group = group
 
-  def asSearchTextExpression(self):
-    # This will be the standard way to represent
-    # complex values in listbox. Some fixed
-    # point must be garanteed
-    value = self.getValue()
-    if isSimpleType(value) or isinstance(value, DateTime):
-      return str(value)
-    elif isinstance(value, (list, tuple)):
-      value = map(lambda x:str(x), value)
-      return (' %s ' % self.operator).join(value)
+  @profiler_decorator
+  def asSearchTextExpression(self, sql_catalog, column=None):
+    return self.getSearchKey(sql_catalog).buildSearchTextExpression(self.getOperator(sql_catalog), self.getValue(), column=column)
 
-  def _getSearchKeyClassByType(self, type, search_key_class = None):
-    """ Return search key class based on type of value. """
-    name_search_key_map = {'keyword': KeyWordKey,
-                           'default': DefaultKey,
-                           'fulltext': FullTextKey,
-                           'date': DateTimeKey,
-                           'float': FloatKey,
-                           'int': DefaultKey,}
-    return name_search_key_map.get(type, search_key_class)
-    
-  def _getSearchKeyClassByValue(self, value, search_key_class = None):
-    """ Return search key class based on type of value. """
-    if isinstance(value, basestring):
-      if value.find('%')!=-1:
-        # it's likely a KeyWordKey
-        search_key_class = KeyWordKey
-      else:
-        search_key_class = DefaultKey
-    elif isinstance(value, DateTime):
-      search_key_class = DateTimeKey
-    elif isinstance(value, (int, long,)):
-      search_key_class = DefaultKey
-    elif isinstance(value, float):
-      search_key_class = FloatKey
-    elif value is None:
-      return RawKey
-    return search_key_class
+  @profiler_decorator
+  def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+    return self.getSearchKey(sql_catalog).buildSQLExpression(
+      self.getOperator(sql_catalog), self.getValue(),
+      column_map, only_group_columns, group=self.group)
 
-  def _asSQLExpression(self, search_key_class, key, value, format=None, mode=None, range_value=None, stat__=None):
-    """ Generate SQL expressions based on respective search_key passed. """
-    lexer = getSearchKeyInstance(search_key_class)
-    where_expression, select_expression_list = \
-             lexer.buildSQLExpression(key, value, format, mode, range_value, stat__)
-    sql_expressions = {'where_expression': where_expression,
-                       'select_expression_list': select_expression_list,}
-    return sql_expressions
+  @profiler_decorator
+  def registerColumnMap(self, sql_catalog, column_map):
+    self.group = self.getSearchKey(sql_catalog).registerColumnMap(column_map, group=self.group, simple_query=self)
 
-  def asSQLExpression(self, key_alias_dict=None, keyword_search_keys=None,
-                      datetime_search_keys=None, full_text_search_keys=None,
-                      ignore_empty_string=1, stat__=0):
+  def getOperator(self, sql_catalog):
     """
-    Build the sql expressions string
+      Return an instance of OperatorBase class.
     """
-    search_key_class = None
-    value = self.getValue()
-    key = self.getKey()
-    operator = self.getOperator()
-    type = self.getType()
-    format = self.getFormat()
-    search_mode = self.getSearchMode()
-    range_value = self.getRange()
-    search_key = self.getSearchKey()
-
-    if keyword_search_keys is None:
-      keyword_search_keys = []
-    if datetime_search_keys is None:
-      datetime_search_keys = []
-    if full_text_search_keys is None:
-      full_text_search_keys = []
-
-    # key can have an alias definition which we should acquire
-    if key_alias_dict is not None:
-      key = key_alias_dict.get(key, None)
-
-    search_key_class = None
-    where_expression_list = []
-    select_expression_list = []
-    sql_expressions = {'where_expression': '1',
-                       'select_expression_list': []}
-    
-    # try to get search key type by the key definitions passed
-    if search_key_class is None:
-      if search_key == EXACT_MATCH_SEARCH_MODE:
-        search_key_class =  RawKey
-      elif search_key == KEYWORD_SEARCH_MODE or \
-          (key in keyword_search_keys):
-        search_key_class =  KeyWordKey
-      elif search_key == DATETIME_SEARCH_MODE or \
-        (key in datetime_search_keys):
-        search_key_class =  DateTimeKey
-      elif search_key == FULL_TEXT_SEARCH_MODE or \
-        (key in full_text_search_keys):
-        search_key_class =  FullTextKey
+    return sql_catalog.getComparisonOperator(self.operator)
 
-    # get search class based on explicitly passed key type
-    if search_key_class is None:
-      search_key_class = self._getSearchKeyClassByType(type)
-    
-    # some use cases where we can just return SQL without grammar staff
-    if key is None or (ignore_empty_string and \
-                       isinstance(value, basestring) and \
-                       value.strip() == ''):
-      # do not further generate sql expressions because
-      # we ignore empty strings by default
-      return sql_expressions
-    elif (not ignore_empty_string) \
-        and isinstance(value, basestring) \
-        and value.strip() == '':
-      # explicitly requested not to ignore empty strings
-      sql_expressions = {'where_expression': "%s = ''"  % key,
-                         'select_expression_list': []}
-      return sql_expressions
-    else:
-      # search for 'NULL' values 
-      if value is None:
-        sql_expressions = {'where_expression':  "%s is NULL" % (key),
-                           'select_expression_list': [],}
-        return sql_expressions
+  def getSearchKey(self, sql_catalog):
+    """
+      Return an instance of SearchKey class.
+    """
+    if self.search_key is None:
+      self.search_key = sql_catalog.getSearchKey(self.getColumn())
+    return self.search_key
 
-      # we have a list of values and respective operator defined
-      if isinstance(value, (tuple, list)):
-        if range_value is None:
-          # use operators to build sql expressions
-          if operator in ('IN',):
-            # values in list are not treated as searchable strings but 
-            # they should be SQL quoted at least
-            if len(value) > 1:
-              if search_key_class is None:
-                # no explicitly defined, try to find by value
-                search_key_class = self._getSearchKeyClassByValue(value[0])
-              search_key_instance = getSearchKeyInstance(search_key_class)
-              escaped_value_list = [search_key_instance.quoteSQLString(x, format) for x in value]
-              escaped_value_string = ', '.join(escaped_value_list)
-              where_expression_list.append("%s IN (%s)" % (key, escaped_value_string))
-            elif len(value) == 1:
-              if search_key_class is None:
-                # no explicitly defined, try to find by value            
-                search_key_class = self._getSearchKeyClassByValue(value[0])
-              search_key_instance = getSearchKeyInstance(search_key_class)
-              where_expression_list.append("%s = %s" 
-                                           %(key, search_key_instance.quoteSQLString(value[0], format)))
-            else:
-              # empty list
-              where_expression_list.append("0")
-          elif operator in ('OR', 'AND',):
-            # each of the list elements can be treated as a Key, so 
-            # leave SQL generation to Key itself
-            if len(value) > 1:
-              sql_logical_sub_expressions = []
-              if search_key_class is None:
-                # no explicitly defined, try to find by value
-                search_key_class = self._getSearchKeyClassByValue(value[0])
-              for item in value:
-                list_item_sql_expressions = self._asSQLExpression(search_key_class, key, \
-                                                                   item, format, search_mode, range_value, stat__)
-                sql_logical_sub_expressions.append('%s' %list_item_sql_expressions['where_expression'])
-              # join list items (now sql logical expressions) using respective operator
-              where_expression = (' %s ' %operator).join(sql_logical_sub_expressions)
-              where_expression_list.append("(%s)" % (where_expression))
-            elif len(value) == 1:
-              if search_key_class is None:
-                # no explicitly defined, try to find by value
-                search_key_class = self._getSearchKeyClassByValue(value[0])
-              item_sql_expressions = self._asSQLExpression(search_key_class, key, \
-                                                            value[0], format, search_mode, range_value, stat__)
-              where_expression_list.append(item_sql_expressions['where_expression'])
-          # join where expressions list
-          where_expression = ' '.join(where_expression_list)
-          sql_expressions = {'where_expression': where_expression,
-                             'select_expression_list': [],}
-          return sql_expressions
-        else:
-          # we can have range specified
-          if search_key_class is None:
-            # try to guess by type of first_element in list
-            search_key_class = self._getSearchKeyClassByValue(value[0])
+  def getColumn(self):
+    return self.column
 
-      # get search class based on value of value
-      if search_key_class is None:
-        search_key_class = self._getSearchKeyClassByValue(value)
+  def getValue(self):
+    return self.value
 
-      # last fallback case
-      if search_key_class is None:
-        search_key_class = DefaultKey
+  def __repr__(self):
+    return '<%s %r %s %r>' % (self.__class__.__name__, self.getColumn(), self.operator, self.getValue())
 
-      # use respective search key.
-      sql_expressions = self._asSQLExpression(search_key_class, key,
-                                                value, format, search_mode, range_value, stat__)
-      return sql_expressions
+  def setGroup(self, group):
+    self.group = group
 
-allow_class(SimpleQuery)
+verifyClass(IQuery, SimpleQuery)
 
-from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
-from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
-from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
-from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
-from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
-from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance

product/ZSQLCatalog/SQLCatalog.py

 ##############################################################################
 #
-# Copyright (c) 2002 Nexedi SARL. All Rights Reserved.
+# Copyright (c) 2002-2009 Nexedi SARL. All Rights Reserved.
 # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -45,7 +45,16 @@ from xml.dom.minidom import parse
 from xml.sax.saxutils import escape, quoteattr
 import os
 import md5
-from sets import ImmutableSet
+
+from Interface.IQueryCatalog import ISearchKeyCatalog
+from Interface.Verify import verifyClass
+
+PROFILING_ENABLED = False
+if PROFILING_ENABLED:
+  from tiny_profiler import profiler_decorator, profiler_report, profiler_reset
+else:
+  def profiler_decorator(func):
+    return func
 
 try:
   from Products.CMFCore.Expression import Expression
@@ -64,6 +73,7 @@ try:
   from Products.ERP5Type.Cache import enableReadOnlyTransactionCache
   from Products.ERP5Type.Cache import disableReadOnlyTransactionCache, CachingMethod
 except ImportError:
+  LOG('SQLCatalog', 100, 'Count not import CachingMethod, expect slowness.')
   def doNothing(context):
     pass
   class CachingMethod:
@@ -77,6 +87,44 @@ except ImportError:
   enableReadOnlyTransactionCache = doNothing
   disableReadOnlyTransactionCache = doNothing
 
+class caching_class_method_decorator:
+  def __init__(self, *args, **kw):
+    self.args = args
+    self.kw = kw
+
+  def __call__(self, method):
+    caching_method = CachingMethod(method, *self.args, **self.kw)
+    return lambda *args, **kw: caching_method(*args, **kw)
+    #def wrapper(wrapped_self):
+    #  LOG('caching_class_method_decorator', 0, 'lookup')
+    #  return caching_method(wrapped_self)
+    #return wrapper
+
+try:
+  from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
+except ImportError:
+  LOG('SQLCatalog', 100, 'Count not import getTransactionalVariable, expect slowness.')
+  def getTransactionalVariable(context):
+    return {}
+
+class transactional_cache_decorator:
+  """
+    Implements singleton-style caching.
+    Wrapped method must have no parameters (besides "self").
+  """
+  def __init__(self, cache_id):
+    self.cache_id = cache_id
+
+  def __call__(self, method):
+    def wrapper(wrapped_self):
+      transactional_cache = getTransactionalVariable(None)
+      try:
+        return transactional_cache[self.cache_id]
+      except KeyError:
+        result = transactional_cache[self.cache_id] = method(wrapped_self)
+        return result
+    return wrapper
+
 try:
   from ZPublisher.HTTPRequest import record
 except ImportError:
@@ -184,6 +232,10 @@ class UidBuffer(TM):
     tid = get_ident()
     self.temporary_buffer.setdefault(tid, []).extend(iterable)
 
+DEBUG = False
+
+related_key_definition_cache = {}
+
 class Catalog(Folder,
               Persistent,
               Acquisition.Implicit,
@@ -219,6 +271,9 @@ class Catalog(Folder,
     - optmization: indexing objects should be deferred
       until timeout value or end of transaction
   """
+
+  __implements__ = ISearchKeyCatalog
+
   meta_type = "SQLCatalog"
   icon = 'misc_/ZCatalog/ZCatalog.gif' # FIXME: use a different icon
   security = ClassSecurityInfo()
@@ -463,6 +518,13 @@ class Catalog(Folder,
                       'a monovalued local role',
       'type': 'lines',
       'mode': 'w' },
+    { 'id': 'sql_catalog_table_vote_scripts',
+      'title': 'Table vote scripts',
+      'description': 'Scripts helping column mapping resolution',
+      'type': 'multiple selection',
+      'select_variable' : 'getPythonMethodIds',
+      'mode': 'w' },
+
   )
 
   sql_catalog_produce_reserved = ''
@@ -500,6 +562,7 @@ class Catalog(Folder,
   sql_catalog_scriptable_keys = ()
   sql_catalog_role_keys = ()
   sql_catalog_local_role_keys = ()
+  sql_catalog_table_vote_scripts = ()
 
   # These are ZODB variables, so shared by multiple Zope instances.
   # This is set to the last logical time when clearReserved is called.
@@ -887,26 +950,24 @@ class Catalog(Folder,
       keys = keys.keys()
       keys.sort()
       return keys
-    return CachingMethod(_getColumnIds, id='SQLCatalog.getColumnIds', cache_factory='erp5_content_long')()
+    return CachingMethod(_getColumnIds, id='SQLCatalog.getColumnIds', cache_factory='erp5_content_long')()[:]
 
+  @profiler_decorator
+  @transactional_cache_decorator('SQLCatalog.getColumnMap')
+  @profiler_decorator
+  @caching_class_method_decorator(id='SQLCatalog.getColumnMap', cache_factory='erp5_content_long')
+  @profiler_decorator
   def getColumnMap(self):
     """
     Calls the show column method and returns dictionnary of
     Field Ids
     """
-    def _getColumnMap():
-      keys = {}
+    result = {}
     for table in self.getCatalogSearchTableIds():
-        field_list = self._getCatalogSchema(table=table)
-        for field in field_list:
-          key = field
-          if not keys.has_key(key): keys[key] = []
-          keys[key].append(table)
-          key = '%s.%s' % (table, key)
-          if not keys.has_key(key): keys[key] = []
-          keys[key].append(table) # Is this inconsistent ?
-      return keys
-    return CachingMethod(_getColumnMap, id='SQLCatalog.getColumnMap', cache_factory='erp5_content_long')()
+      for field in self._getCatalogSchema(table=table):
+        result.setdefault(field, []).append(table)
+        result.setdefault('%s.%s' % (table, field), []).append(table) # Is this inconsistent ?
+    return result
 
   def getResultColumnIds(self):
     """
@@ -1332,7 +1393,7 @@ class Catalog(Folder,
         if self.isMethodFiltered(method_name):
           catalogged_object_list = []
           filter = self.filter_dict[method_name]
-          type_set = ImmutableSet(filter['type']) or None
+          type_set = frozenset(filter['type']) or None
           expression = filter['expression_instance']
           expression_cache_key_list = filter.get('expression_cache_key', '').split()
           for object in object_list:
@@ -1633,7 +1694,8 @@ class Catalog(Folder,
     """ Accesses a single record for a given path """
     return self.getMetadataForPath(path)
 
-  def getCatalogMethodIds(self):
+  def getCatalogMethodIds(self,
+      valid_method_meta_type_list=valid_method_meta_type_list):
     """Find Z SQL methods in the current folder and above
     This function return a list of ids.
     """
@@ -1658,6 +1720,26 @@ class Catalog(Folder,
     ids.sort()
     return ids
 
+  def getPythonMethodIds(self):
+    """
+      Returns a list of all python scripts available in
+      current sql catalog.
+    """
+    return self.getCatalogMethodIds(valid_method_meta_type_list=('Script (Python)', ))
+
+  @profiler_decorator
+  @transactional_cache_decorator('SQLCatalog._getSQLCatalogRelatedKeyList')
+  @profiler_decorator
+  def _getSQLCatalogRelatedKeySet(self):
+    column_map = self.getColumnMap()
+    column_set = set(column_map)
+    for related_key in self.sql_catalog_related_keys:
+      related_key_id = related_key.split(' | ')[0].strip()
+      if related_key_id in column_set:
+        LOG('SQLCatalog', 100, 'Related key %r has the same name as an existing column on tables %r' % (related_key_id, column_map[related_key_id]))
+      column_set.add(related_key_id)
+    return column_set
+
   def getSQLCatalogRelatedKeyList(self, key_list=None):
     """
     Return the list of related keys.
@@ -1666,10 +1748,9 @@ class Catalog(Folder,
     """
     if key_list is None:
       key_list = []
+    column_map = self._getSQLCatalogRelatedKeySet()
     # Do not generate dynamic related key for acceptable_keys
-    dynamic_key_list = [k for k in key_list \
-        if k not in self.getColumnMap().keys()]
-
+    dynamic_key_list = [k for k in key_list if k not in column_map]
     dynamic_list = self.getDynamicRelatedKeyList(dynamic_key_list)
     full_list = list(dynamic_list) + list(self.sql_catalog_related_keys)
     return full_list
@@ -1702,486 +1783,412 @@ class Catalog(Folder,
           %(table_index, table))
       return table_index
     return CachingMethod(_getTableIndex, id='SQLCatalog.getTableIndex', \
-                         cache_factory='erp5_content_long')(table=table)
-
-
-  def getIndex(self, table, column_list, all_column_list):
-    """
-    Return possible index for a column list in a given table
-    """
-    def _getIndex(table, column_list, all_column_list):
-      index_dict = self.getTableIndex(table)
-      if isinstance(column_list, str):
-        column_list = [column_list,]
-      # Get possible that can be used
-      possible_index = []
-      for index in index_dict.keys():
-        index_columns = index_dict[index]
-        for column in index_columns:
-          if column in column_list:
-            if index not in possible_index:
-              possible_index.append(index)
-      if len(possible_index) == 0:
-        return []
-      # Get the most suitable index
-      for index in possible_index:
-        # Make sure all column in index are used by the query
-        index_column = index_dict[index]
-        for column in index_column:
-          if column in column_list or column in all_column_list:
-            continue
+                         cache_factory='erp5_content_long')(table=table).copy()
+
+  @profiler_decorator
+  def getRelatedKeyDefinition(self, key):
+    """
+      Returns the definition of given related key name if found, None
+      otherwise.
+    """
+    try:
+      result = related_key_definition_cache[key]
+    except KeyError:
+      result = None
+      for entire_definition in self.getSQLCatalogRelatedKeyList([key]):
+        name, definition = entire_definition.split(' | ')
+        if name == key:
+          result = definition
+          break
+      if result is not None:
+        related_key_definition_cache[key] = result
+    return result
+
+  @profiler_decorator
+  def getColumnSearchKey(self, key, search_key_name=None):
+    """
+      Return a SearchKey instance for given key, using search_key_name
+      as a SearchKey name if given, otherwise guessing from catalog
+      configuration. If there is no search_key_name given and no
+      SearchKey can be found, return None.
+
+      Also return a related key definition string with following rules:
+       - If returned SearchKey is a RelatedKey, value is its definition
+       - Otherwise, value is None
+    """
+    # Is key a "real" column or some related key ?
+    related_key_definition = None
+    if key in self.getColumnMap():
+      search_key = self.getSearchKey(key, search_key_name)
     else:
-            possible_index.remove(index)
-      LOG("SQLCatalog.getIndex", INFO, "index = %s for table %s and columns %s" \
-          %(possible_index, table, column_list))
-      return possible_index
-    return CachingMethod(_getIndex, id='SQLCatalog.getIndex', cache_factory='erp5_content_long')\
-          (table=table, column_list=column_list, all_column_list=all_column_list)
+      # Maybe a related key...
+      related_key_definition = self.getRelatedKeyDefinition(key)
+      if related_key_definition is None:
+        # Unknown
+        search_key = None
+      else:
+        # It's a related key
+        search_key = self.getSearchKey(key, 'RelatedKey')
+    return search_key, related_key_definition
 
+  @profiler_decorator
+  def getColumnDefaultSearchKey(self, key):
+    """
+      Return a SearchKey instance which would ultimately receive the value
+      associated with given key.
+    """
+    search_key, related_key_definition = self.getColumnSearchKey(key)
+    if search_key is None:
+      result = None
+    else:
+      if related_key_definition is not None:
+        search_key = search_key.getSearchKey(self, related_key_definition)
+    return search_key
 
-  def buildSQLQuery(self, query_table='catalog', REQUEST=None,
-                          ignore_empty_string=1, query=None, stat__=0, **kw):
-    """ Builds a complex SQL query to simulate ZCatalog behaviour """
-    # Get search arguments:
-    if REQUEST is None and (kw is None or kw == {}):
-      # We try to get the REQUEST parameter
-      # since we have nothing handy
-      try: REQUEST=self.REQUEST
-      except AttributeError: pass
-
-    #LOG('SQLCatalog.buildSQLQuery, kw',0,kw)
-    # If kw and query are not set, then use REQUEST instead
-    if query is None and (kw is None or kw == {}):
-      kw = REQUEST
-
-    acceptable_key_map = self.getColumnMap()
-    full_text_search_keys = list(self.sql_catalog_full_text_search_keys)
-    keyword_search_keys = list(self.sql_catalog_keyword_search_keys)
-    datetime_search_keys = list(self.sql_catalog_datetime_search_keys)
-    topic_search_keys = self.sql_catalog_topic_search_keys
-    multivalue_keys = self.sql_catalog_multivalue_keys
-
-
-    # Compute "sort_index", which is a sort index, or none:
-    if kw.has_key('sort-on'):
-      sort_index=kw['sort-on']
-    elif hasattr(self, 'sort-on'):
-      sort_index=getattr(self, 'sort-on')
-    elif kw.has_key('sort_on'):
-      sort_index=kw['sort_on']
-    else: sort_index=None
-
-    # Compute the sort order
-    if kw.has_key('sort-order'):
-      so=kw['sort-order']
-    elif hasattr(self, 'sort-order'):
-      so=getattr(self, 'sort-order')
-    elif kw.has_key('sort_order'):
-      so=kw['sort_order']
-    else: so=None
-
-    # We must now turn sort_index into
-    # a dict with keys as sort keys and values as sort order
-    if isinstance(sort_index, basestring):
-      sort_index = [(sort_index, so)]
-    elif not isinstance(sort_index, (list, tuple)):
-      sort_index = None
-
-    # Rebuild keywords to behave as new style query (_usage='toto:titi' becomes {'toto':'titi'})
-    new_kw = {}
-    usage_len = len('_usage')
-    for k, v in kw.items():
-      if k.endswith('_usage'):
-        new_k = k[0:-usage_len]
-        if not new_kw.has_key(new_k):
-          new_kw[new_k] = {}
-        if not isinstance(new_kw[new_k], dict_type_list):
-          new_kw[new_k] = {'query': new_kw[new_k]}
-        split_v = v.split(':')
-        new_kw[new_k] = {split_v[0]: split_v[1]}
+  @profiler_decorator
+  def buildSingleQuery(self, key, value, search_key_name=None, logical_operator=None, comparison_operator=None):
+    """
+      From key and value, determine the SearchKey to use and generate a Query
+      from it.
+    """
+    search_key, related_key_definition = self.getColumnSearchKey(key, search_key_name)
+    if search_key is None:
+      result = None
     else:
-        if not new_kw.has_key(k):
-          new_kw[k] = v
+      if related_key_definition is None:
+        result = search_key.buildQuery(value, logical_operator=logical_operator, comparison_operator=comparison_operator)
       else:
-          new_kw[k]['query'] = v
-    kw = new_kw
+        result = search_key.buildQuery(search_value=value, sql_catalog=self, search_key_name=search_key_name, related_key_definition=related_key_definition, logical_operator=logical_operator, comparison_operator=comparison_operator)
+    return result
+
+  @profiler_decorator
+  def buildQueryFromAbstractSyntaxTreeNode(self, node, key):
+    """
+      Build a query from given Abstract Syntax Tree (AST) node by recursing in
+      its childs.
+      This method calls itself recursively when walking the tree.
+
+      node
+        AST node being treated.
+      key
+        Default column (used when there is no explicit column in an AST leaf).
 
-    # Initialise Scriptable Dict
-    scriptable_key_dict = {}
-    for t in self.sql_catalog_scriptable_keys:
-      t = t.split('|')
-      key = t[0].strip()
-      if len(t)>1:
-        # method defined that will generate a ComplexQuery
-        method_id = t[1].strip()
+      Expected node API is described in Interface/IAbstractSyntaxNode.py .
+    """
+    if node.isLeaf():
+      result = self.buildSingleQuery(key, node.getValue(), comparison_operator=node.getComparisonOperator())
+      if result is None:
+        # Unknown, skip loudly
+        LOG('SQLCatalog', 100, 'Unknown column %r, skipped.' % (key, ))
+    elif node.isColumn():
+      result = self.buildQueryFromAbstractSyntaxTreeNode(node.getSubNode(), node.getColumnName())
+    else:
+      query_list = []
+      value_dict = {}
+      append = query_list.append
+      for subnode in node.getNodeList():
+        if subnode.isLeaf():
+          value_dict.setdefault(subnode.getComparisonOperator(), []).append(subnode.getValue())
         else:
-        # no method define, let ScriptableKey generate a ComplexQuery
-        method_id = None
-      scriptable_key_dict[key] = method_id
-
-    # Build the list of Queries and ComplexQueries
-    query_dict = {}
-    key_list = [] # the list of column keys
-    key_alias_dict = {}
-    query_group_by_list = None # Useful to keep a default group_by passed by scriptable keys
-    query_related_table_map_dict = {}
-    if query is not None:
-      kw ['query'] = query
-    for key in kw.keys():
-      if key not in RESERVED_KEY_LIST:
-        value = kw[key]
-        current_query = None
-        new_query_dict = {}
-        if isinstance(value, (Query, ComplexQuery)):
-          current_query = value
-        elif scriptable_key_dict.has_key(key):
-          if scriptable_key_dict[key] is not None:
-            # Turn this key into a query by invoking a script          
-            method = getattr(self, scriptable_key_dict[key])
-            current_query = method(value) # May return None            
+          subquery = self.buildQueryFromAbstractSyntaxTreeNode(subnode, key)
+          if subquery is not None:
+            append(subquery)
+      for comparison_operator, value_list in value_dict.iteritems():
+        subquery = self.buildSingleQuery(key, value_list, comparison_operator=comparison_operator)
+        if subquery is None:
+          LOG('SQLCatalog', 100, 'Unknown column %r, skipped.' % (key, ))
         else:
-            # let default implementation of ScriptableKey generate ComplexQuery
-            search_key_instance = getSearchKeyInstance(ScriptableKey)
-            current_query = search_key_instance.buildQuery('', value)
-          if hasattr(current_query, 'order_by'): query_group_by_list = current_query.order_by
+          append(subquery)
+      operator = node.getLogicalOperator()
+      if operator == 'not' or len(query_list) > 1:
+        result = ComplexQuery(query_list, operator=operator)
+      elif len(query_list) == 1:
+        result = query_list[0]
       else:
+        result = None
+    return result
+
+  @profiler_decorator
+  def buildQuery(self, kw, ignore_empty_string=True, operator='and'):
+    query_list = []
+    append = query_list.append
+    # unknown_column_dict: contains all (key, value) pairs which could not be
+    # changed into queries. This is here for backward compatibility, because
+    # scripts can invoke this method and expect extra parameters (such as
+    # from_expression) to be handled. As they are normaly handled at
+    # buildSQLQuery level, we must store them into final ComplexQuery, which
+    # will handle them.
+    unknown_column_dict = {}
+    # implicit_table_list: contains all tables explicitely given as par of
+    # column names with empty values. This is for backward compatibility. See
+    # comment about empty values.
+    implicit_table_list = []
+    for key, value in kw.iteritems():
+      result = None
       if isinstance(value, dict_type_list):
-            new_query_dict = value.copy()
-            if 'query' in new_query_dict:
-              new_query_dict[key] = new_query_dict.pop('query')
+        # Cast dict-ish types into plain dicts.
+        value = dict(value)
+      if ignore_empty_string and (
+          value == ''
+          or (isinstance(value, (list, tuple)) and len(value) == 0)
+          or (isinstance(value, dict) and (
+            'query' not in value
+            or value['query'] == ''
+            or (isinstance(value['query'], (list, tuple))
+              and len(value['query']) == 0)))):
+        # We have an empty value:
+        # - do not create a query from it
+        # - if key has a dot, add its left part to the list of "hint" tables
+        #   This is for backward compatibility, when giving a mapped column
+        #   with an empty value caused a join with catalog to appear in
+        #   resulting where-expression)
+        if '.' in key:
+          implicit_table_list.append(key)
+        LOG('buildQuery', WARNING, 'Discarding empty value for key %r: %r' % (key, value))
       else:
-            new_query_dict[key] = value
-          current_query = Query(**new_query_dict)
-        if current_query is not None:
-          query_dict[key] = current_query
-          key_list.extend(current_query.getSQLKeyList())
-          query_related_table_map_dict.update(current_query.getRelatedTableMapDict())
-
-    # if we have a sort index, we must take it into account to get related
-    # keys.
-    sort_key_dict = dict()
-    if sort_index:
-      for sort_info in sort_index:
-        sort_key = sort_info[0]
-        if sort_key not in key_list:
-          key_list.append(sort_key)
-        sort_key_dict[sort_key] = 1
-
-    related_tuples = self.getSQLCatalogRelatedKeyList(key_list=key_list)
-    
-    # Define related maps
-    # each tuple from `related_tuples` has the form (key,
-    # 'table1,table2,table3/column/where_expression')
-    related_keys = {}
-    related_method = {}
-    related_table_map = {}
-    related_column = {}
-    related_table_list = {}
-    table_rename_index = 0
-    related_methods = {} # related methods which need to be used
-    for t in related_tuples:
-      t_tuple = t.split('|')
-      key = t_tuple[0].strip()
-      if key in key_list:
-        if ignore_empty_string \
-            and kw.get(key, None) in ('', [], ())\
-            and key not in sort_key_dict:
-              # We don't ignore 0 and None, but if the key is used for sorting,
-              # we should not discard this key
-          continue
-        join_tuple = t_tuple[1].strip().split('/')
-        related_keys[key] = None
-        method_id = join_tuple[2]
-        table_list = tuple(join_tuple[0].split(','))
-        related_method[key] = method_id
-        related_table_list[key] = table_list
-        related_column[key] = join_tuple[1]
-        # Check if some aliases where specified in queries
-        map_list = query_related_table_map_dict.get(key,None)
-        # Rename tables to prevent conflicts
-        if not related_table_map.has_key((table_list,method_id)):
-          if map_list is None:
-            map_list = []
-            for table_id in table_list:
-              map_list.append((table_id,
-                 "related_%s_%s" % (table_id, table_rename_index))) # We add an index in order to alias tables in the join
-              table_rename_index += 1 # and prevent name conflicts
-          related_table_map[(table_list,method_id)] = map_list
-
-    # We take additional parameters from the REQUEST
-    # and give priority to the REQUEST
-    if REQUEST is not None:
-      for key in acceptable_key_map.iterkeys():
-        if REQUEST.has_key(key):
-          # Only copy a few keys from the REQUEST
-          if key in self.sql_catalog_request_keys:
-            kw[key] = REQUEST[key]
-
-    def getNewKeyAndUpdateVariables(key):
-      new_key = None
-      if query_table:
-        key_is_acceptable = key in acceptable_key_map # Only calculate once
-        key_is_related = key in related_keys
-        if key_is_acceptable or key_is_related:
-          if key_is_related: # relation system has priority (ex. security_uid)
-            # We must rename the key
-            method_id = related_method[key]
-            table_list = related_table_list[key]
-            if not related_methods.has_key((table_list,method_id)):
-              related_methods[(table_list,method_id)] = 1
-            # Prepend renamed table name
-            new_key = "%s.%s" % (related_table_map[(table_list,method_id)][-1][-1],
-                                 related_column[key])
-          elif key_is_acceptable:
-            if key.find('.') < 0:
-              # if the key is only used by one table, just append its name
-              if len(acceptable_key_map[key]) == 1 :
-                new_key = '%s.%s' % (acceptable_key_map[key][0], key)
-              # query_table specifies what table name should be used by default
-              elif '%s.%s' % (query_table, key) in acceptable_key_map:
-                new_key = '%s.%s' % (query_table, key)
-              elif key == 'uid':
-                # uid is always ambiguous so we can only change it here
-                new_key = 'catalog.uid'
+        if isinstance(value, _Query):
+          # Query instance: use as such, ignore key.
+          result = value
+        elif isinstance(value, basestring):
+          # String: parse using key's default search key.
+          search_key = self.getColumnDefaultSearchKey(key)
+          if search_key is not None:
+            abstract_syntax_tree = search_key.parseSearchText(value)
+            if abstract_syntax_tree is None:
+              # Parsing failed, create a query from the bare string.
+              result = self.buildSingleQuery(key, value)
             else:
-                LOG('SQLCatalog', WARNING, 'buildSQLQuery this key is too ambiguous : %s' % key)
+              if DEBUG:
+                LOG('SQLCatalog', 0, 'Building queries from abstract syntax tree: %r' % (abstract_syntax_tree, ))
+              result = self.buildQueryFromAbstractSyntaxTreeNode(abstract_syntax_tree, key)
+        elif isinstance(value, dict):
+          # Dictionnary: might contain the search key to use.
+          search_key_name = value.get('key')
+          # Backward compatibility: former "Keyword" key is now named
+          # "KeywordKey".
+          if search_key_name == 'Keyword':
+            search_key_name = value['key'] = 'KeywordKey'
+          result = self.buildSingleQuery(key, value, search_key_name)
         else:
-              new_key = key
-            if new_key is not None:
-              # Add table to table dict, we use catalog by default
-              from_table_dict[acceptable_key_map[new_key][0]] = acceptable_key_map[new_key][0]
+          # Any other type, just create a query. (can be a DateTime, ...)
+          result = self.buildSingleQuery(key, value)
+        if result is None:
+          # No query could be created, emit a log, add to unknown column dict.
+          unknown_column_dict[key] = value
         else:
-        new_key = key
-      key_alias_dict[key] = new_key
-      return new_key
-
-    where_expression_list = []
-    select_expression_list = []
-    group_by_expression_list = []
-    where_expression = ''
-    select_expression = ''
-    group_by_expression = ''
-    select_expression_key = ''
-    
-    from_table_dict = {'catalog' : 'catalog'} # Always include catalog table
-    if len(kw):
-      if kw.has_key('select_expression_key'):
-        select_expression_key = kw['select_expression_key']
-        if type(select_expression_key) is type('a'):
-          select_expression_key = [select_expression_key]
-      if kw.has_key('select_expression'):
-        select_expression_list.append(kw['select_expression'])
-      if kw.has_key('group_by_expression'):
-        group_by_expression_list.append(kw['group_by_expression'])
-      # Grouping
-      group_by_list = kw.get('group_by', query_group_by_list)
-      if type(group_by_list) is type('a'): group_by_list = [group_by_list]
-      if group_by_list is not None:
-        try:
-          for key in group_by_list:
-            new_key = getNewKeyAndUpdateVariables(key)
-            group_by_expression_list.append(new_key)
-        except ConflictError:
-          raise
-        except:
-          LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build the new group by expression', error=sys.exc_info())
-          group_by_expression = ''
-      if len(group_by_expression_list)>0:
-        group_by_expression = ','.join(group_by_expression_list)
-        group_by_expression = str(group_by_expression)
-
-    sort_on = None
-    sort_key_list = []
-    if sort_index is not None:
-      new_sort_index = []
-      for sort in sort_index:
-        if len(sort) == 2:
-          # Try to analyse expressions of the form "title AS unsigned"
-          sort_key_list = sort[0].split()
-          if len(sort_key_list) == 3:
-            sort_key = sort_key_list[0]
-            sort_type = sort_key_list[2]
-          elif len(sort_key_list):
-            sort_key = sort_key_list[0]
-            sort_type = None
+          append(result)
+    if len(unknown_column_dict):
+      LOG('SQLCatalog', 100, 'Unknown columns %r, skipped.' % (unknown_column_dict.keys(), ))
+    return ComplexQuery(query_list, operator=operator, unknown_column_dict=unknown_column_dict, implicit_table_list=implicit_table_list)
+
+  @profiler_decorator
+  def buildOrderByList(self, sort_on=None, sort_order=None, order_by_expression=None):
+    """
+      Internal method. Should not be used by code outside buildSQLQuery.
+
+      It is in a separate method because this code is here to keep backward
+      compatibility with an ambiguous API, and as such is ugly. So it's better
+      to conceal it to its own method.
+
+      It does not preserve backward compatibility for:
+        'sort-on' parameter
+        'sort-on' property
+        'sort-order' parameter
+        'sort-order' property
+    """
+    order_by_list = []
+    append = order_by_list.append
+    if sort_on is not None:
+      if order_by_expression is not None:
+        LOG('SQLCatalog', 0, 'order_by_expression (%r) and sort_on (%r) were given. Ignoring order_by_expression.' % (order_by_expression, sort_on))
+      if not isinstance(sort_on, (tuple, list)):
+        sort_on = [[sort_on]]
+      for item in sort_on:
+        if isinstance(item, (tuple, list)):
+          item = list(item)
         else:
-            sort_key = sort[0]
-            sort_type = None
-          new_sort_index.append((sort_key, sort[1], sort_type))
-        elif len(sort) == 3:
-          new_sort_index.append(sort)
-      sort_index = new_sort_index
-      try:
-        new_sort_index = []
-        for (original_key, so, as_type) in sort_index:
-          key = getNewKeyAndUpdateVariables(original_key)
-          if key is None:
-            if original_key in select_expression_key:
-              key = original_key
-          if key is not None:
-            sort_key_list.append(key)
-            if as_type == 'int':
-              key = 'CAST(%s AS SIGNED)' % key
-            elif as_type:
-              key = 'CAST(%s AS %s)' % (key, as_type) # Different casts are possible
-            if so in ('descending', 'reverse', 'DESC'):
-              new_sort_index.append('%s DESC' % key)
+          item = [item]
+        if sort_order is not None and len(item) == 1:
+          item.append(sort_order)
+        if len(item) > 1:
+          if item[1] in ('descending', 'reverse', 'DESC'):
+            item[1] = 'DESC'
           else:
-              new_sort_index.append('%s' % key)
+            item[1] = 'ASC'
+          if len(item) > 2:
+            if item[2] == 'int':
+              item[2] = 'SIGNED'
+        append(item)
+    elif order_by_expression is not None:
+      if not isinstance(order_by_expression, basestring):
+        raise TypeError, 'order_by_expression must be a basestring instance. Got %r.' % (order_by_expression, )
+      order_by_list = [[x.strip()] for x in order_by_expression.split(',')]
+    return order_by_list
+
+  @profiler_decorator
+  def buildSQLQuery(self, query_table='catalog', REQUEST=None,
+                          ignore_empty_string=1, only_group_columns=False,
+                          limit=None, extra_column_list=None,
+                          **kw):
+#    from traceback import format_list, extract_stack
+#    LOG('buildSQLQuery', 0, ''.join(format_list(extract_stack())))
+    if DEBUG:
+      LOG('buildSQLQuery', 0, repr(kw))
+    group_by_list = kw.pop('group_by_list', kw.pop('group_by', kw.pop('group_by_expression', None)))
+    if isinstance(group_by_list, basestring):
+      group_by_list = [x.strip() for x in group_by_list.split(',')]
+    select_dict = kw.pop('select_dict', kw.pop('select_list', kw.pop('select_expression', None)))
+    if isinstance(select_dict, basestring):
+      if len(select_dict):
+        real_select_dict = {}
+        for column in select_dict.split(','):
+          index = column.lower().find(' as ')
+          if index != -1:
+            real_select_dict[column[index + 4:].strip()] = column[:index].strip()
           else:
-            LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build sort '
-                'index (%s -> %s)' % (original_key, key))
-        sort_index = join(new_sort_index,',')
-        sort_on = str(sort_index)
-      except ConflictError:
-        raise
-      except:
-        LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build the new sort index', error=sys.exc_info())
-        sort_on = ''
-        sort_key_list = []
-
-    for key in key_list:
-      if not key_alias_dict.has_key(key):
-        getNewKeyAndUpdateVariables(key)
-    if len(query_dict):
-      for key, query in query_dict.items():
-        query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
-                                    full_text_search_keys=full_text_search_keys,
-                                    keyword_search_keys=keyword_search_keys,
-                                    datetime_search_keys=datetime_search_keys,
-                                    ignore_empty_string=ignore_empty_string,
-                                    stat__=stat__)
-        if query_result['where_expression'] not in ('',None):
-          where_expression_list.append(query_result['where_expression'])
-        select_expression_list.extend(query_result['select_expression_list'])
-
-    # Calculate extra where_expression based on required joins
-    if query_table:
-      for k, tid in from_table_dict.items():
-        if k != query_table:
-          where_expression_list.append('%s.uid = %s.uid' % (query_table, tid))
-    # Calculate extra where_expressions based on related definition
-    for (table_list, method_id) in related_methods.keys():
-      related_method = getattr(self, method_id, None)
-      if related_method is not None:
-        table_id = {'src__' : 1} # Return query source, do not evaluate
-        table_id['query_table'] = query_table
-        table_index = 0
-        for t_tuple in related_table_map[(table_list,method_id)]:
-          table_id['table_%s' % table_index] = t_tuple[1] # table_X is set to mapped id
-          from_table_dict[t_tuple[1]] = t_tuple[0]
-          table_index += 1
-        where_expression_list.append(related_method(**table_id))
-    # Concatenate expressions
-    if kw.get('where_expression',None) not in (None,''):
-      where_expression_list.append(kw['where_expression'])
-    if len(where_expression_list)>1:
-      where_expression_list = ['(%s)' % x for x in where_expression_list]
-    where_expression = join(where_expression_list, ' AND ')
-    select_expression= join(select_expression_list,',')
-
-    limit_expression = kw.get('limit', None)
-    if isinstance(limit_expression, (list, tuple)):
-      limit_expression = '%s,%s' % (limit_expression[0], limit_expression[1])
-    elif limit_expression is not None:
-      limit_expression = str(limit_expression)
-
-    # force index if exists when doing sort as mysql doesn't manage them efficiently
-    if len(sort_key_list) > 0:
-      index_from_table = {}
-      # first group columns from a same table
-      for key in sort_key_list:
-        try:
-          related_table, column = key.split('.')
-        except ValueError:
-          # key is not of the form table.column
-          # so get table from dict
-          if len(from_table_dict) != 1:
-            continue
-          column = key
-          related_table = from_table_dict.keys()[0]
-
-        table = from_table_dict[related_table]
-        # Check if it's a column for which we want to specify index
-        index_columns = getattr(self, 'sql_catalog_index_on_order_keys', [])
-        sort_column = '%s.%s' %(table, column)
-        if not sort_column in index_columns:
-          continue
-        # Group columns
-        if not index_from_table.has_key(table):
-          index_from_table[table] = [column,]
+            real_select_dict[column.strip()] = None
+        select_dict = real_select_dict
       else:
-          index_from_table[table].append(column)
-      # second ask index
-      for table in index_from_table.keys():
-        available_index_list = self.getIndex(table, index_from_table[table], key_list)
-        if len(available_index_list) > 0:
-          # Always give MySQL a chance to use PRIMARY key. It is much faster if
-          # current table is used in a join on primary key than forcing it to
-          # use another index.
-          # Note: due to a bug (?) in MySQL (at least 5.0.45 community), it is
-          # a syntax error to put "PRIMARY" keyword anywere besides at first
-          # position. Hence the "insert(0".
-          if 'PRIMARY' not in available_index_list:
-            available_index_list.insert(0, 'PRIMARY')
-          # tell mysql to use these index
-          table = from_table_dict.pop(related_table)
-          index_list_string = ', '.join(available_index_list)
-          table_with_index =  "%s use index(%s)"  %(related_table, index_list_string)
-          from_table_dict[table_with_index] = table
-
-    from_expression = kw.get('from_expression', None)
-    if from_expression is not None:
-      final_from_expression = ', '.join(
-        [from_expression.get(table, '`%s` AS `%s`' % (table, alias))
-         for alias, table in from_table_dict.iteritems()])
+        select_dict = None
+    elif isinstance(select_dict, (list, tuple)):
+      select_dict = dict([(x, None) for x in select_dict])
+    # Handle order_by_list
+    order_by_list = kw.pop('order_by_list', None)
+    sort_on = kw.pop('sort_on', None)
+    sort_order = kw.pop('sort_order', None)
+    order_by_expression = kw.pop('order_by_expression', None)
+    if order_by_list is None:
+      order_by_list = self.buildOrderByList(
+        sort_on=sort_on,
+        sort_order=sort_order,
+        order_by_expression=order_by_expression
+      )
     else:
-      final_from_expression = None
-    # Use a dictionary at the moment.
-    return { 'from_table_list' : from_table_dict.items(),
-             'from_expression' : final_from_expression,
-             'order_by_expression' : sort_on,
-             'where_expression' : where_expression,
-             'limit_expression' : limit_expression,
-             'select_expression': select_expression,
-             'group_by_expression' : group_by_expression}
+      if sort_on is not None:
+        LOG('SQLCatalog', 0, 'order_by_list and sort_on were given, ignoring sort_on.')
+      if sort_order is not None:
+        LOG('SQLCatalog', 0, 'order_by_list and sort_order were given, ignoring sort_order.')
+      if order_by_expression is not None:
+        LOG('SQLCatalog', 0, 'order_by_list and order_by_expression were given, ignoring order_by_expression.')
+    # Handle from_expression
+    from_expression = kw.pop('from_expression', None)
+    # Handle where_expression
+    where_expression = kw.get('where_expression', None)
+    if isinstance(where_expression, basestring) and len(where_expression):
+      LOG('SQLCatalog', 100, 'Giving where_expression a string value is deprecated.')
+      # Transform given where_expression into a query, and update kw.
+      kw['where_expression'] = SQLQuery(where_expression)
+    # Handle select_expression_key
+    # It is required to support select_expression_key parameter for backward
+    # compatiblity, but I'm not sure if there can be a serious use for it in
+    # new API.
+    order_by_override_list = kw.pop('select_expression_key', None)
+    query = EntireQuery(
+      query=self.buildQuery(kw, ignore_empty_string=ignore_empty_string),
+      order_by_list=order_by_list,
+      order_by_override_list=order_by_override_list,
+      group_by_list=group_by_list,
+      select_dict=select_dict,
+      limit=limit,
+      catalog_table_name=query_table,
+      extra_column_list=extra_column_list,
+      from_expression=from_expression)
+    result = query.asSQLExpression(self, only_group_columns).asSQLExpressionDict()
+    if DEBUG:
+      LOG('buildSQLQuery', 0, repr(result))
+    return result
 
   # Compatibililty SQL Sql
   buildSqlQuery = buildSQLQuery
 
-  def queryResults(self, sql_method, REQUEST=None, used=None, src__=0, build_sql_query_method=None, **kw):
+  @profiler_decorator
+  @transactional_cache_decorator('SQLCatalog._getSearchKeyDict')
+  @profiler_decorator
+  @caching_class_method_decorator(id='SQLCatalog._getSearchKeyDict', cache_factory='erp5_content_long')
+  @profiler_decorator
+  def _getSearchKeyDict(self):
+    result = {}
+    search_key_column_dict = {
+      'KeywordKey': self.sql_catalog_keyword_search_keys,
+      'FullTextKey': self.sql_catalog_full_text_search_keys,
+      'DateTimeKey': self.sql_catalog_datetime_search_keys,
+    }
+    for key, column_list in search_key_column_dict.iteritems():
+      for column in column_list:
+        if column in result:
+          LOG('SQLCatalog', 100, 'Ambiguous configuration: column %r is set to use %r key, but also to use %r key. Former takes precedence.' % (column, result[column], key))
+        else:
+          result[column] = key
+    return result
+
+  @profiler_decorator
+  def getSearchKey(self, column, search_key=None):
+    """
+      Return an instance of a SearchKey class.
+
+      column (string)
+        The column for which the search key will be returned.
+      search_key (string)
+        If given, must be the name of a SearchKey class to be returned.
+        Returned value will be an instance of that class, even if column has
+        been configured to use a different one.
+    """
+    if search_key is None:
+      search_key = self._getSearchKeyDict().get(column, 'DefaultKey')
+    return getSearchKeyInstance(search_key, column)
+
+  def getComparisonOperator(self, operator):
+    """
+      Return an instance of an Operator class.
+
+      operator (string)
+        String defining the expected operator class.
+        See Operator module to have a list of available operators.
+    """
+    return getComparisonOperatorInstance(operator)
+
+  @profiler_decorator
+  def _queryResults(self, REQUEST=None, build_sql_query_method=None, **kw):
     """ Returns a list of brains from a set of constraints on variables """
     if build_sql_query_method is None:
       build_sql_query_method = self.buildSQLQuery
     query = build_sql_query_method(REQUEST=REQUEST, **kw)
+    # XXX: decide if this should be made normal
+    ENFORCE_SEPARATION = True
+    if ENFORCE_SEPARATION:
+      new_kw = {}
+      # Some parameters must be propagated:
+      for parameter_id in ('selection_domain', 'selection_report'):
+        if parameter_id in kw:
+          new_kw[parameter_id] = kw[parameter_id]
+      kw = new_kw
     kw['where_expression'] = query['where_expression']
     kw['sort_on'] = query['order_by_expression']
     kw['from_table_list'] = query['from_table_list']
-    kw['from_expression'] = query.get('from_expression')
+    kw['from_expression'] = query['from_expression']
     kw['limit_expression'] = query['limit_expression']
     kw['select_expression'] = query['select_expression']
     kw['group_by_expression'] = query['group_by_expression']
-    # Return the result
+    return kw
 
-    #LOG('acceptable_keys',0,'acceptable_keys: %s' % str(acceptable_keys))
-    #LOG('acceptable_key_map',0,'acceptable_key_map: %s' % str(acceptable_key_map))
-    #LOG('queryResults',0,'kw: %s' % str(kw))
-    #LOG('queryResults',0,'from_table_list: %s' % str(query['from_table_list']))
-    return sql_method(src__=src__, **kw)
+  def queryResults(self, sql_method, REQUEST=None, src__=0, build_sql_query_method=None, **kw):
+    sql_kw = self._queryResults(REQUEST=REQUEST, build_sql_query_method=build_sql_query_method, **kw)
+    if DEBUG and not src__:
+      LOG('queryResults', 0, sql_method(src__=1, **sql_kw))
+    return sql_method(src__=src__, **sql_kw)
       
-  def searchResults(self, REQUEST=None, used=None, **kw):
+  def searchResults(self, REQUEST=None, **kw):
     """ Returns a list of brains from a set of constraints on variables """
-    # The used argument is deprecated and is ignored
     method = getattr(self, self.sql_search_results)
-    return self.queryResults(method, REQUEST=REQUEST, used=used, **kw)
+    return self.queryResults(method, REQUEST=REQUEST, extra_column_list=self.getCatalogSearchResultKeys(), **kw)
 
   __call__ = searchResults
 
-  def countResults(self, REQUEST=None, used=None, stat__=1, **kw):
+  def countResults(self, REQUEST=None, **kw):
     """ Returns the number of items which satisfy the where_expression """
     # Get the search method
     method = getattr(self, self.sql_count_results)
-    return self.queryResults(method, REQUEST=REQUEST, used=used, stat__=stat__, **kw)
+    return self.queryResults(method, REQUEST=REQUEST, extra_column_list=self.getCatalogSearchResultKeys(), only_group_columns=True, **kw)
 
   def recordObjectList(self, path_list, catalog=1):
     """
@@ -2394,40 +2401,56 @@ Globals.default__class_init__(Catalog)
 
 class CatalogError(Exception): pass
 
-# pool of global preinitialized search keys instances
-SEARCH_KEY_INSTANCE_POOL = threading.local()
-# hook search keys and Query implementation 
-def getSearchKeyInstance(search_key_class):
-  """ Return instance of respective search_key class.
-      We should have them initialized only once."""
-  global SEARCH_KEY_INSTANCE_POOL
-  if not hasattr(SEARCH_KEY_INSTANCE_POOL, 'pool'):
-    pool = dict()
-    for klass in (DefaultKey, RawKey, KeyWordKey, DateTimeKey,
-                             FullTextKey, FloatKey, ScriptableKey, KeyMappingKey):
-      search_key_instance = klass()
-      search_key_instance.build()
-      pool[klass] = search_key_instance
-    SEARCH_KEY_INSTANCE_POOL.pool = pool
-
-  return SEARCH_KEY_INSTANCE_POOL.pool[search_key_class]
-  
-from Query.Query import QueryMixin
-from Query.SimpleQuery import NegatedQuery, SimpleQuery
+from Query.Query import Query as _Query
+from Query.SimpleQuery import SimpleQuery
 from Query.ComplexQuery import ComplexQuery
+from Query.AutoQuery import AutoQuery as Query
+
+def NegatedQuery(query):
+  return ComplexQuery(query, operator='not')
+
+allow_class(SimpleQuery)
+allow_class(ComplexQuery)
+
+import SearchKey
+SEARCH_KEY_INSTANCE_POOL = {}
+SEARCH_KEY_CLASS_CACHE = {}
+
+@profiler_decorator
+def getSearchKeyInstance(search_key_class_name, column):
+  assert isinstance(search_key_class_name, basestring)
+  try:
+    search_key_class = SEARCH_KEY_CLASS_CACHE[search_key_class_name]
+  except KeyError:
+    search_key_class = getattr(getattr(SearchKey, search_key_class_name),
+                               search_key_class_name)
+    SEARCH_KEY_CLASS_CACHE[search_key_class_name] = search_key_class
+  try:
+    instance_dict = SEARCH_KEY_INSTANCE_POOL[search_key_class]
+  except KeyError:
+    instance_dict = SEARCH_KEY_INSTANCE_POOL[search_key_class] = {}
+  try:
+    result = instance_dict[column]
+  except KeyError:
+    result = instance_dict[column] = search_key_class(column)
+  return result
+
+from Operator import operator_dict
+def getComparisonOperatorInstance(operator):
+  return operator_dict[operator]
+
+from Query.EntireQuery import EntireQuery
+from Query.SQLQuery import SQLQuery
+
+verifyClass(ISearchKeyCatalog, Catalog)
 
-# for of backwards compatability  
-QueryMixin = QueryMixin
-Query = SimpleQuery
-NegatedQuery = NegatedQuery
-ComplexQuery = ComplexQuery
+if PROFILING_ENABLED:
+  def Catalog_dumpProfilerData(self):
+    return profiler_report()
 
-from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
-from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
-from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
-from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
-from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
-from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
-from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
+  def Catalog_resetProfilerData(self):
+    profiler_reset()
 
+  Catalog.dumpProfilerData = Catalog_dumpProfilerData
+  Catalog.resetProfilerData = Catalog_resetProfilerData
 

product/ZSQLCatalog/SQLExpression.py

+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from zLOG import LOG
+from Interface.ISQLExpression import ISQLExpression
+from Interface.Verify import verifyClass
+from types import NoneType
+from SQLCatalog import profiler_decorator
+
+SQL_LIST_SEPARATOR = ', '
+SQL_TABLE_FORMAT = '%s' # XXX: should be changed to '`%s`', but this breaks some ZSQLMethods.
+SQL_SELECT_ALIAS_FORMAT = '%s AS `%s`'
+
+"""
+  TODO:
+    - change table_alias_dict in internals to represent computed tables:
+       ie: '(SELECT * FROM `bar` WHERE `baz` = "hoge") AS `foo`'
+           '`foo` LEFT JOIN `bar` WHERE (`baz` = "hoge")'
+"""
+
+# Set to true to keep a reference to the query which created us.
+# Set to false to avoid keeping a reference to an object.
+DEBUG = True
+
+def defaultDict(value):
+  if value is None:
+    return {}
+  assert isinstance(value, dict)
+  return value.copy()
+
+class SQLExpression(object):
+
+  __implements__ = ISQLExpression
+
+  @profiler_decorator
+  def __init__(self,
+               query,
+               table_alias_dict=None,
+               order_by_list=(),
+               order_by_dict=None,
+               group_by_list=(),
+               where_expression=None,
+               where_expression_operator=None,
+               sql_expression_list=(),
+               select_dict=None,
+               limit=None,
+               from_expression=None):
+    if DEBUG:
+      self.query = query
+    self.table_alias_dict = defaultDict(table_alias_dict)
+    self.order_by_list = list(order_by_list)
+    self.group_by_list = list(group_by_list)
+    self.order_by_dict = defaultDict(order_by_dict)
+    # Only one of (where_expression, where_expression_operator) must be given (never both)
+    assert None in (where_expression, where_expression_operator)
+    # Exactly one of (where_expression, where_expression_operator) must be given, except if sql_expression_list is given and contains exactly one entry
+    assert where_expression is not None or where_expression_operator is not None or (sql_expression_list is not None and len(sql_expression_list) == 1)
+    # where_expression must be a basestring instance if given
+    assert isinstance(where_expression, (NoneType, basestring))
+    # where_expression_operator must be 'and', 'or' or 'not' (if given)
+    assert where_expression_operator in (None, 'and', 'or', 'not'), where_expression_operator
+    self.where_expression = where_expression
+    self.where_expression_operator = where_expression_operator
+    # Exactly one of (where_expression, sql_expression_list) must be given (XXX: duplicate of previous conditions ?)
+    assert where_expression is not None or sql_expression_list is not None
+    if isinstance(sql_expression_list, (list, tuple)):
+      sql_expression_list = [x for x in sql_expression_list if x is not None]
+    self.sql_expression_list = list(sql_expression_list)
+    self.select_dict = defaultDict(select_dict)
+    if limit is None:
+      self.limit = ()
+    elif isinstance(limit, (list, tuple)):
+      if len(limit) < 3:
+        self.limit = limit
+      else:
+        raise ValueError, 'Unrecognized "limit" value: %r' % (limit, )
+    else:
+      self.limit = (limit, )
+    if from_expression is not None:
+      LOG('SQLExpression', 0, 'Providing a from_expression is deprecated.')
+    self.from_expression = from_expression
+
+  @profiler_decorator
+  def getTableAliasDict(self):
+    """
+      Returns a dictionary:
+        key: table alias (string)
+        value: table name (string)
+
+      If there are nested SQLExpressions, it aggregates their mappings and
+      checks that they don't alias different table with the same name. If they
+      do, it raises a ValueError.
+    """
+    result = self.table_alias_dict.copy()
+    for sql_expression in self.sql_expression_list:
+      for alias, table_name in sql_expression.getTableAliasDict().iteritems():
+        existing_value = result.get(alias)
+        if existing_value not in (None, table_name):
+          message = '%r is a known alias for table %r, can\'t alias it now to table %r' % (alias, existing_value, table_name)
+          if DEBUG:
+            message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+              self.query,
+              sql_expression,
+              sql_expression.getQuery(),
+              ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+          raise ValueError, message
+        result[alias] = table_name
+    return result
+
+  @profiler_decorator
+  def getFromExpression(self):
+    """
+      Returns a string.
+
+      If there are nested SQLExpression, it checks that they either don't
+      define any from_expression or the exact same from_expression. Otherwise,
+      it raises a ValueError.
+    """
+    result = self.from_expression
+    for sql_expression in self.sql_expression_list:
+      from_expression = sql_expression.getFromExpression()
+      if None not in (result, from_expression):
+        message = 'I don\'t know how to merge from_expressions'
+        if DEBUG:
+          message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+            self.query,
+            sql_expression,
+            sql_expression.getQuery(),
+            ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+        raise ValueError, message
+    return result
+
+  @profiler_decorator
+  def getOrderByList(self):
+    """
+      Returns a list of strings.
+
+      If there are nested SQLExpression, it checks that they don't define
+      sorts for columns which are already sorted. If they do, it raises a
+      ValueError.
+    """
+    result = self.order_by_list[:]
+    known_column_set = set([x[0] for x in result])
+    for sql_expression in self.sql_expression_list:
+      for order_by in sql_expression.getOrderByList():
+        if order_by[0] in known_column_set:
+          raise ValueError, 'I don\'t know how to merge order_by yet'
+        else:
+          result.append(order_by)
+          known_column_set.add(order_by[0])
+    return result
+
+  @profiler_decorator
+  def getOrderByDict(self):
+    result_dict = self.order_by_dict.copy()
+    for sql_expression in self.sql_expression_list:
+      order_by_dict = sql_expression.getOrderByDict()
+      for key, value in order_by_dict.iteritems():
+        if key in result_dict and value != result_dict[key]:
+          message = 'I don\'t know how to merge order_by_dict with ' \
+                    'conflicting entries for key %r: %r vs. %r' % (key, result_dict[key], value)
+          if DEBUG:
+            message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+              self.query,
+              sql_expression,
+              sql_expression.getQuery(),
+              ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+          raise ValueError, message
+      result_dict.update(order_by_dict)
+    return result_dict
+
+  @profiler_decorator
+  def getOrderByExpression(self):
+    """
+      Returns a string.
+
+      Returns a rendered "order by" expression. See getOrderByList.
+    """
+    order_by_dict = self.getOrderByDict()
+    get = order_by_dict.get
+    return SQL_LIST_SEPARATOR.join(get(x, str(x)) \
+                                   for x in self.getOrderByList())
+
+  @profiler_decorator
+  def getWhereExpression(self):
+    """
+      Returns a string.
+
+      Returns a rendered "where" expression.
+    """
+    if self.where_expression is not None:
+      result = self.where_expression
+    else:
+      if self.where_expression_operator == 'not':
+        assert len(self.sql_expression_list) == 1
+        result = '(NOT %s)' % (self.sql_expression_list[0].getWhereExpression())
+      elif len(self.sql_expression_list) == 1:
+        result = self.sql_expression_list[0].getWhereExpression()
+      elif len(self.sql_expression_list) == 0:
+        result = '(1)'
+      else:
+        operator = '\n  ' + self.where_expression_operator.upper() + ' '
+        result = '(%s)' % (operator.join(x.getWhereExpression() for x in self.sql_expression_list), )
+    return result
+
+  @profiler_decorator
+  def getLimit(self):
+    """
+      Returns a list of 1 or 2 items (int or string).
+
+      If there are nested SQLExpression, it checks that they either don't
+      define any limit or the exact same limit. Otherwise it raises a
+      ValueError.
+    """
+    result = list(self.limit)
+    for sql_expression in self.sql_expression_list:
+      other_limit = sql_expression.getLimit()
+      if other_limit not in ([], result):
+        message = 'I don\'t know how to merge limits yet'
+        if DEBUG:
+          message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+            self.query,
+            sql_expression,
+            sql_expression.getQuery(),
+            ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+        raise ValueError, message
+    return result
+
+  @profiler_decorator
+  def getLimitExpression(self):
+    """
+      Returns a string.
+      
+      Returns a rendered "limit" expression. See getLimit.
+    """
+    return SQL_LIST_SEPARATOR.join(str(x) for x in self.getLimit())
+
+  @profiler_decorator
+  def getGroupByset(self):
+    """
+      Returns a set of strings.
+
+      If there are nested SQLExpression, it merges (union of sets) them with
+      local value.
+    """
+    result = set(self.group_by_list)
+    for sql_expression in self.sql_expression_list:
+      result.update(sql_expression.getGroupByset())
+    return result
+
+  @profiler_decorator
+  def getGroupByExpression(self):
+    """
+      Returns a string.
+
+      Returns a rendered "group by" expression. See getGroupBySet.
+    """
+    return SQL_LIST_SEPARATOR.join(self.getGroupByset())
+
+  @profiler_decorator
+  def getSelectDict(self):
+    """
+      Returns a dict:
+        key: alias (string)
+        value: column (string) or None
+
+      If there are nested SQLExpression, it aggregates their mappings and
+      checks that they don't alias different columns with the same name. If
+      they do, it raises a ValueError.
+    """
+    result = self.select_dict.copy()
+    for sql_expression in self.sql_expression_list:
+      for alias, column in sql_expression.getSelectDict().iteritems():
+        existing_value = result.get(alias)
+        if existing_value not in (None, column):
+          message = '%r is a known alias for column %r, can\'t alias it now to column %r' % (alias, existing_value, column)
+          if DEBUG:
+            message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+              self.query,
+              sql_expression,
+              sql_expression.getQuery(),
+              ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+          raise ValueError, message
+        result[alias] = column
+    return result
+
+  @profiler_decorator
+  def getSelectExpression(self):
+    """
+      Returns a string.
+
+      Returns a rendered "select" expression. See getSelectDict.
+    """
+    return SQL_LIST_SEPARATOR.join(
+      SQL_SELECT_ALIAS_FORMAT % (column, alias)
+      for alias, column in self.getSelectDict().iteritems())
+
+  @profiler_decorator
+  def asSQLExpressionDict(self):
+    table_alias_dict = self.getTableAliasDict()
+    from_table_list = []
+    append = from_table_list.append
+    for alias, table in table_alias_dict.iteritems():
+      append((SQL_TABLE_FORMAT % (alias, ), SQL_TABLE_FORMAT % (table, )))
+    from_expression_dict = self.getFromExpression()
+    if from_expression_dict is not None:
+      from_expression = SQL_LIST_SEPARATOR.join(
+        from_expression_dict.get(table, '`%s` AS `%s`' % (table, alias))
+        for alias, table in table_alias_dict.iteritems())
+    else:
+      from_expression = None
+    return {
+      'where_expression': self.getWhereExpression(),
+      'order_by_expression': self.getOrderByExpression(),
+      'from_table_list': from_table_list,
+      'from_expression': from_expression,
+      'limit_expression': self.getLimitExpression(),
+      'select_expression': self.getSelectExpression(),
+      'group_by_expression': self.getGroupByExpression()
+    }
+
+verifyClass(ISQLExpression, SQLExpression)
+

product/ZSQLCatalog/SearchKey/DateTimeKey.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -26,171 +28,241 @@
 #
 ##############################################################################
 
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
-from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-from DateTime import DateTime
+import sys
 from SearchKey import SearchKey
-from pprint import pprint
+from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
+from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
+#from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from zLOG import LOG
+from DateTime.DateTime import DateTime, DateTimeError, _cache
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
 
+MARKER = []
 
-class DateTimeKey(SearchKey):
-  """ DateTimeKey key is an ERP5 portal_catalog search key which is used to render
-      SQL expression that will try to match values in DateTime MySQL columns.
-      It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
-      addition to main logical operators like ['OR', 'or', 'AND', 'and'].
+timezone_dict = _cache._zmap
+
+date_completion_format_dict = {
+  None: ['01/01/%s', '01/%s'],
+  'international': ['%s/01/01', '%s/01']
+}
+
+@profiler_decorator
+def _DateTime(*args, **kw):
+  return DateTime(*args, **kw)
 
-      Note: because all ERP5 datetime values are indexed in MySQL in 'UTC'
-      the respective passed date will be first converted to 'UTC' before inserted into
-      respective SQL query!
+@profiler_decorator
+def castDate(value):
+  date_kw = {'datefmt': 'international'}
+  if isinstance(value, dict):
+    # Convert value into a DateTime, and guess desired delta from what user
+    # input.
+    assert value['type'] == 'date'
+    format = value.get('format')
+    value = value['query']
+    if format == '%m/%d/%Y':
+      date_kw.pop('datefmt')
+  if isinstance(value, DateTime):
+    pass
+  elif isinstance(value, basestring):
+    try:
+      value = _DateTime(value, **date_kw)
+    except DateTimeError:
+      delimiter_count = countDelimiters(value)
+      if delimiter_count < 3:
+        split_value = value.split()
+        if split_value[-1].lower() in timezone_dict:
+          value = '%s %s' % (date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (' '.join(split_value[:-1]), ), split_value[-1])
+        else:
+          value = date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (value, )
+        value = _DateTime(value, **date_kw)
+      else:
+        raise
+  else:
+    raise TypeError, 'Unknown date type: %r' % (value)
+  return value.toZone('UTC')
 
-      Examples (GMT+02, Bulgaria/Sofia for 'delivery.start_date'):
+# (strongly) inspired from DateTime.DateTime.py
+delimiter_list = ' -/.:,+'
 
-        * '15/01/2008' --> "delivery.start_date = '2008-01-14 22:00'"
+def getMonthLen(datetime):
+  return datetime._month_len[datetime.isLeapYear()][datetime.month()]
 
-        * '>=15/01/2008' --> "delivery.start_date >= '2008-01-14 22:00'"     
+def getYearLen(datetime):
+  return 365 + datetime.isLeapYear()
 
-        * '>=15/01/2008 or <=20/01/2008'
-          --> "delivery.start_date >= '2008-01-14 22:00' or delivery.start_date<='2008-01-19 22:00'"
+delta_list = [getYearLen, getMonthLen, 1, 1.0 / 24, 1.0 / (24 * 60), 1.0 / (24 * 60 * 60)]
 
-        * '>=15/01/2008 10:00 GMT+02 OR <=20/01/2008 05:12 Universal'
-          -->
-          "delivery.start_date >= '2008-01-15 08:00 Universal'
-            OR
-          delivery.start_date <= '2008-01-20 05:12 Universal'
-          "
-  """
+@profiler_decorator
+def countDelimiters(value):
+  assert isinstance(value, basestring)
+  # Detect if timezone was provided, to avoid counting it as in precision computation.
+  split_value = value.split()
+  if split_value[-1].lower() in timezone_dict:
+    value = ' '.join(split_value[:-1])
+  # Count delimiters
+  delimiter_count = 0
+  for char in value:
+    if char in delimiter_list:
+      delimiter_count += 1
+  return delimiter_count
 
-  tokens =  ('DATE', 'OR', 'AND', 'NOT', 'EQUAL',
-             'GREATERTHAN', 'GREATERTHANEQUAL',
-             'LESSTHAN', 'LESSTHANEQUAL')
-
-  sub_operators =  ('GREATERTHAN', 'GREATERTHANEQUAL',
-                    'LESSTHAN', 'LESSTHANEQUAL', 'NOT', 'EQUAL',)
-
-  def t_OR(self, t):
-    r'\s+(OR|or)\s+'
-    # operator has leading and trailing ONLY one white space character
-    t.value = 'OR'
-    return t
-
-  def t_AND(self, t):
-    r'\s+(AND|and)\s+'
-    # operator has leading and trailing ONLY one white space character
-    t.value = 'AND'
-    return t
-
-  def t_NOT(self, t):
-    r'(\s+(NOT|not)\s+|!=)'
-    # operator has leading and trailing ONLY one white space character
-    t.value = t.value.upper().strip()
-    return t
-
-  t_GREATERTHANEQUAL = r'>=' 
-  t_LESSTHANEQUAL = r'<=' 
-  t_GREATERTHAN = r'>'
-  t_LESSTHAN = r'<'
-  t_EQUAL = r'='
-  t_DATE = r'\d{1,4}[(/|\.|\-) /.]\d{1,4}[(/|\.|\-) /.]\d{1,4}((\s.)*\d{0,2}:\d{0,2}(:\d{0,2})?)?(\sUniversal|\sGMT\+\d\d)?|\d\d\d\d%?'
-
-  def quoteSQLString(self, value, format):
-    """ Return a quoted string of the value.
-        Make sure to convert it to UTC first."""
-    if getattr(value, 'ISO', None) is not None:
-      value = "'%s'" % value.toZone('UTC').ISO()
+@profiler_decorator
+def getPeriodBoundaries(value):
+  first_date = castDate(value)
+  if isinstance(value, dict):
+    value = value['query']
+  # Try to guess how much was given in query.
+  if isinstance(value, basestring):
+    delimiter_count = countDelimiters(value)
+  elif isinstance(value, DateTime):
+    raise TypeError, 'Impossible to guess a precision from a DateTime type.'
   else:
-      value = "'%s'" %DateTime(value).toZone('UTC').ISO()
-    return value
+    raise TypeError, 'Unknown date type: %r' % (value)
+  delta = delta_list[delimiter_count]
+  if callable(delta):
+    delta = delta(first_date)
+  return first_date, first_date + delta
 
-  def buildQueryForTokenList(self, tokens, key, value, format):
-    """ Build a ComplexQuery for a token list """
+@profiler_decorator
+def wholePeriod(search_key, group, column, value_list, exclude=False):
+  if exclude:
+    first_operator = '<'
+    second_operator = '>='
+    logical_operator = 'or'
+  else:
+    first_operator = '>='
+    second_operator = '<'
+    logical_operator = 'and'
   query_list = []
-    for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
-      token_values = [x.value for x in group_tokens]
-      sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
-      date_value = sub_tokens[0].value
-      days_offset = 0
-      # some format require special handling
-      if format != '%Y':
-        # full format (Year/Month/Day)
-        if sub_operator in ('=',):
-          # 2007/01/01 00:00 <= date < 2007/01/02
-          days_offset = 1
-      elif format == '%Y':
-        # incomplete format only Year because DateTime can not handle
-        # extend format and value by assumption that start of year is ment
-        # add days ofset accordingly
-        format = '%%%s/%%m/%%d' %format
-        date_value = '%s/01/01' %date_value
-        days_offset_map = {'=' : 366, '>' : 366,
-                           '>=' : 366, '<': -366, '<=':-366}
-        days_offset = days_offset_map[sub_operator]
-
-      # convert to UTC in given format
-      is_valid_date = 1
-      try:
-        if format != '%m/%d/%Y':
-          # treat ambigious dates as "days before month before year"
-          date_value = DateTime(date_value, datefmt="international").toZone('UTC')
+  append = query_list.append
+  for value in value_list:
+    first_date, second_date = getPeriodBoundaries(value)
+    append(ComplexQuery([SimpleQuery(search_key=search_key, operator=first_operator, group=group, **{column: first_date}),
+                         SimpleQuery(search_key=search_key, operator=second_operator, group=group, **{column: second_date})],
+                        operator=logical_operator))
+  return query_list
+
+def matchWholePeriod(search_key, group, column, value_list, *ignored):
+  return wholePeriod(search_key, group, column, value_list)
+
+def matchNotWholePeriod(search_key, group, column, value_list, *ignored):
+  return wholePeriod(search_key, group, column, value_list, exclude=True)
+
+@profiler_decorator
+def matchExact(search_key, group, column, value_list, comparison_operator, logical_operator):
+  if comparison_operator is None:
+    comparison_operator = '='
+  value_list = [castDate(x) for x in value_list]
+  if logical_operator == 'or' and comparison_operator == '=':
+    query_list = [SimpleQuery(search_key=search_key, operator='in', group=group, **{column: value_list})]
   else:
-          # US style "month before day before year"
-          date_value = DateTime(date_value).toZone('UTC')
-      except:
-        is_valid_date = 0
-
-      query_kw = None
-      if is_valid_date:
-        if sub_operator == '=':
-          # transform to range 'key >= date  AND  date < key'
-          query_kw = {key: (date_value, date_value + days_offset,),
-                      'range': 'minmax'}
+    query_list = [SimpleQuery(search_key=search_key, operator=comparison_operator, group=group, **{column: x}) for x in value_list]
+  return query_list
+
+def getNextPeriod(value):
+  return getPeriodBoundaries(value)[1]
+
+@profiler_decorator
+def matchBeforeNextPeriod(search_key, group, column, value_list, comparison_operator, logical_operator):
+  return matchExact(search_key, group, column, [getNextPeriod(x) for x in value_list], '<', logical_operator)
+
+@profiler_decorator
+def matchAfterPeriod(search_key, group, column, value_list, comparison_operator, logical_operator):
+  return matchExact(search_key, group, column, [getNextPeriod(x) for x in value_list], '>=', logical_operator)
+
+operator_matcher_dict = {
+  None: matchWholePeriod,
+  '=': matchWholePeriod,
+  '!=': matchNotWholePeriod,
+  '<': matchExact,
+  '>=': matchExact,
+  '<=': matchBeforeNextPeriod,
+  '>': matchAfterPeriod,
+}
+
+# Behaviour of date time operators
+# Objects:
+#   2005/03/14 23:59:59
+#   2005/03/15 00:00:00
+#   2005/03/15 00:00:01
+#   2005/03/15 23:59:59
+#   2005/03/16 00:00:00
+#   2005/03/16 00:00:01
+#
+# Searches:
+#   "2005/03/15" (operator = None)
+#     Implicitely matches the whole period.
+#     2005/03/15 00:00:00
+#     2005/03/15 00:00:01
+#     2005/03/15 23:59:59
+#
+#   "=2005/03/15" (operator = '=')
+#     Behaves the same way as None operator.
+#     2005/03/15 00:00:00
+#     2005/03/15 00:00:01
+#     2005/03/15 23:59:59
+#
+#   "!=2005/03/15" (operator = '!=')
+#     Complementary of '=' operator.
+#     2005/03/14 23:59:59
+#     2005/03/16 00:00:00
+#     2005/03/16 00:00:01
+#
+#   "<2005/03/15" (operator = '<')
+#     Non-ambiguous (no difference wether time is considered as a period or a single point in time).
+#     2005/03/14 23:59:59
+#
+#   ">=2005/03/15" (operator = '>=')
+#     Complementary of '<' operator, and also non-ambiguous.
+#     2005/03/15 00:00:00
+#     2005/03/15 00:00:01
+#     2005/03/15 23:59:59
+#     2005/03/16 00:00:00
+#     2005/03/16 00:00:01
+#
+#   "<=2005/03/15" (operator = '<=')
+#     Union of results from '=' and '<' operators.
+#     2005/03/14 23:59:59
+#     2005/03/15 00:00:00
+#     2005/03/15 00:00:01
+#     2005/03/15 23:59:59
+#
+#   ">2005/03/15" (operator = '>')
+#     Complementary of '<=' operator.
+#     2005/03/16 00:00:00
+#     2005/03/16 00:00:01
+
+class DateTimeKey(SearchKey):
+  """
+    This SearchKey allows generating date ranges from single, user-input dates.
+  """
+
+  default_comparison_operator = None
+  get_operator_from_value = True
+
+  def _renderValueAsSearchText(self, value, operator):
+    return '"%s"' % (DateTime(value).ISO(), )
+
+  @profiler_decorator
+  def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
+    column = self.getColumn()
+    query_list = []
+    extend = query_list.extend
+    for comparison_operator, value_list in operator_value_dict.iteritems():
+      try:
+        if parsed:
+          subquery_list = operator_matcher_dict[comparison_operator](
+                   self, group, column, value_list, comparison_operator,
+                   logical_operator)
         else:
-          query_kw = {key: date_value + days_offset,
-                      'range': sub_operator}  
-        query_kw['type'] = 'date'
+          subquery_list = matchExact(self, group, column, value_list, comparison_operator, logical_operator)
+      except DateTimeError:
+        LOG('DateTimeKey', 100, 'Got an exception while generating a query for %r %r.' % (comparison_operator, value_list), error=sys.exc_info())
       else:
-        # not a valid date, try to get an year range
-        is_year = 1
-        date_value = date_value.replace('%', '')
-        try: date_value = int(date_value)
-        except: is_year = 0
-        if is_year:
-          date_value = '%s/01/01' % date_value
-          date_value = DateTime(date_value).toZone('UTC')
-          query_kw = {key: (date_value, date_value + 366,),
-                      'type': 'date',
-                      'range': 'minmax'}
-
-      # append only if it was possible to generate query
-      if query_kw is not None:
-        query_list.append(Query(**query_kw))
-
-    # join query list in one really big ComplexQuery
-    if len(query_list):
-      complex_query = ComplexQuery(*query_list,
-                                   **{'operator': 'AND'})
-      return complex_query
-
-##  def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-##    """ Tokenize/analyze passed string value and generate SQL query expressions. """
-##    where_expression = ''
-##    key = self.quoteSQLKey(key, format)
-##    tokens = self.tokenize(value)
-##    operators_mapping_list = self.groupByOperator(tokens)
-##    # new one
-##    for item in operators_mapping_list:
-##      row_tokens_values = []
-##      tokens = item['tokens']
-##      operator = item['operator']
-##      operator_value = None
-##      if operator is not None:
-##        # operator is standalone expression
-##        operator_value = operator.value
-##        where_expressions.append('%s' %operator_value)
-##      if len(tokens):
-##        # no it's not a stand alone expression,
-##        # determine it from list of tokens
-##        operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
-##        row_tokens_values = [self.quoteSQLString(x.value, format) for x in sub_tokens]
-##        where_expression = "%s %s %s" %(key, operator_value, ' '.join(row_tokens_values))
-##    return where_expression, []
+        extend(subquery_list)
+    return query_list
+
+verifyClass(ISearchKey, DateTimeKey)
+

product/ZSQLCatalog/SearchKey/DefaultKey.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -26,124 +28,36 @@
 #
 ##############################################################################
 
-from DocumentTemplate.DT_Var import sql_quote
 from SearchKey import SearchKey
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
 
 class DefaultKey(SearchKey):
-  """ DefaultKey key is an ERP5 portal_catalog search key which is used to render
-      SQL expression that will try to exactly one value.
-      It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
-      addition to main logical operators like ['OR', 'or', 'AND', 'and'].
-
-      Examples for title column:
-        * 'foo or bar'  --> "title = 'foo' OR title = 'bar'"
-        * 'foo or =bar'  --> "title = 'foo' OR title = 'bar'"
-        * '%foo% or bar' --> "title = '%foo%' OR title = 'bar'"
-        * 'Organisation Module' -->  "title = 'Organisation Module'"
-        * '"Organisation Module"' --> "title = 'Organisation Module'"
-        * '="Organisation Module"' --> "title = 'Organisation Module'"
   """
-
-  # default type of sub Queries to be generated out fo a search string
-  default_key_type = 'default'
-
-  tokens =  ('OR', 'AND', 'NOT', 'WORDSET', 'WORD',
-             'GREATERTHAN', 'GREATERTHANEQUAL',
-             'LESSTHAN', 'LESSTHANEQUAL')
-
-  sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
-                    'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
-
-
-  # Note: Order of placing rules (t_WORD for example) is very important
-  def t_OR(self, t):
-    r'\s+(OR|or)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'OR'
-    return t
-
-  def t_AND(self, t):
-    r'\s+(AND|and)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'AND'
-    return t
-
-  def t_NOT(self, t):
-    r'(\s+(NOT|not)\s+|!=)'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = '!='
-    return t
-
-  t_GREATERTHANEQUAL = r'>='
-  t_LESSTHANEQUAL = r'<='
-  t_GREATERTHAN = r'>'
-  t_LESSTHAN = r'<'
-
-  def t_WORD(self, t):
-    r'([^"\s<>!][\S\n]*|!([^=\s][\S\n]*)?)'
-    # newlines are allowed, because variations are delimited by newlines.
-    # WORD may contain arbitrary letters and numbers without white space
-    # WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
-    value = t.value.strip()
-    if value[0] == '=':
-      value = value[1:]
-    t.value = value
-    return t
-
-  def t_WORDSET(self, t):
-    r'"[^"]*"'
-    #r'"[\x7F-\xFF\w\d\s/%][\x7F-\xFF\w\d\s/%]*"'
-    # WORDSET is a combination of WORDs separated by white space
-    # and starting/ending with "
-    value = t.value.replace('"', '').strip()
-    t.value = value
-    return t
-
-  def quoteSQLString(self, value, format):
-    """ Return a quoted string of the value. """
-    if isinstance(value, (int, long,)):
-      return str(value)
-    return "'%s'" % sql_quote(value)
-
-
-##  def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-##    """ Tokenize/analyze passed string value and generate SQL query expressions. """
-##    where_expressions = []
-##    select_expressions = []
-##    tokens = self.tokenize(value)
-##    operators_mapping_list = self.groupByOperator(tokens)
-##
-##    # find if any logical operator exists
-##    tokens_values = []
-##    logical_operator_found = 0
-##    for token in tokens:
-##      if token.type not in ('WORDSET', 'WORD',):
-##        logical_operator_found = 1
-##        break
-##      tokens_values.append(token.value.replace("'", ""))
-##
-##    # build expressions
-##    if not logical_operator_found:
-##      # no logical operator found so we assume that we search for a combination of words
-##      where_expressions.append("%s = '%s'" %(key, ' '.join(tokens_values)))
-##    else:
-##      # in the search string we have explicitly defined an operator
-##      for item in operators_mapping_list:
-##        row_tokens_values = []
-##        tokens = item['tokens']
-##        operator = item['operator']
-##        operator_value = None
-##        if operator is not None:
-##          # operator is standalone expression
-##          operator_value = operator.value
-##          where_expressions.append('%s' %operator_value)
-##        if len(tokens):
-##          # no it's not a stand alone expression,
-##          # determine it from list of tokens
-##          operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
-##          row_tokens_values = [x.value for x in sub_tokens]
-##          where_expressions.append("%s %s '%s'" %(key, operator_value, ' '.join(row_tokens_values)))
-##    return where_expressions, select_expressions   
+    This SearchKey behaves like an ExactMatch SearchKey, except if value is a
+    string and contains a '%' sign, in which case it behaves like a
+    KeywordKey.
+  """
+  default_comparison_operator = '='
+  get_operator_from_value = True
+
+  def _guessComparisonOperator(self, value):
+    if isinstance(value, basestring) and '%' in value:
+      operator = 'like'
+    else:
+      operator = SearchKey._guessComparisonOperator(self, value)
+    return operator
+
+  def buildSearchTextExpression(self, operator, value, column=None):
+    operator_text = operator.getOperator()
+    if column is None:
+      column = self.getColumn()
+    if operator_text == 'like':
+      assert isinstance(value, basestring)
+      assert '%' in value
+      result = '%s:%s' % (column, value)
+    else:
+      result = SearchKey.buildSearchTextExpression(self, operator, value, column=column)
+    return result
+
+verifyClass(ISearchKey, DefaultKey)

product/ZSQLCatalog/SearchKey/FullTextKey.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -27,77 +29,19 @@
 ##############################################################################
 
 from SearchKey import SearchKey
-import re
-
-SEARCH_MODE_MAPPING = {'in_boolean_mode': 'IN BOOLEAN MODE',
-                       'with_query_expansion': 'WITH QUERY EXPANSION'}
+from Products.ZSQLCatalog.SearchText import parse
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
 
 class FullTextKey(SearchKey):
-  """ FullTextKey key is an ERP5 portal_catalog search key which is used to render
-      SQL expression that will try match all possible values using
-      MySQL's fulltext search support.
-      See syntax see MySQL's FullText search reference:
-      http://dev.mysql.com/doc/refman/5.0/en/fulltext-search.html
   """
+    This SearchKey generates SQL fulltext comparisons.
+  """
+  default_comparison_operator = 'match'
+  get_operator_from_value = False
 
-  tokens =  ('PLUS', 'MINUS', 'WORD', 'GREATERTHAN', 'LESSTHAN', 'LEFTPARENTHES',
-             'RIGHTPARENTHES', 'TILDE', 'ASTERISK', 'DOUBLEQUOTE',)
-
-  # SQL expressions patterns
-  relevance = '%s_relevance'
-  where_match_against = "MATCH %s AGAINST (%s %s)"
-  select_match_against_as = "MATCH %s AGAINST (%s %s) AS %s"
-
-  t_PLUS = r'(\+)'
-  t_MINUS = r'(\-)'
-  t_GREATERTHAN = r'(\>)'
-  t_LESSTHAN = r'(\<)'
-  t_LEFTPARENTHES = r'(\()'
-  t_RIGHTPARENTHES = r'(\))'
-  t_TILDE = r'(\~)'
-  t_ASTERISK = r'(\*)'
-  t_DOUBLEQUOTE = r'(\")'
+  def parseSearchText(self, value):
+    return parse(value)
 
-  def t_WORD(self, t):
-    r'[^\+\-<>\(\)\~\*\"\s]$|[^\+\-<>\(\)\~\*\"\s]+[^\*\"\s\)]'
-    #r'[^\+\-<>\(\)\~\*\"\s]\S*'
-    #r'[\x7F-\xFF\w\d][\x7F-\xFF\w\d]*'
-    # WORD may contain arbitrary letters and numbers without white space
-    return t
+verifyClass(ISearchKey, FullTextKey)
 
-  def buildSQLExpression(self, key, value,
-                         format=None, mode=None, range_value=None, stat__=None):
-    """ Analize token list and generate SQL expressions."""
-    tokens = self.tokenize(value)
-    # based on type tokens we may switch to different search mode
-    mode = SEARCH_MODE_MAPPING.get(mode, '')
-    if mode == '':
-      # determine it based on list of tokens  i.e if we have only words
-      # leave as its but if we have '-' or '+' use boolean mode
-      for token in tokens:
-        if token.type != 'WORD':
-          mode = SEARCH_MODE_MAPPING['in_boolean_mode']
-          break
-      if mode == '' and len(tokens) > 1:
-        value = ' '.join(['+%s' %  x.value for x in tokens])
-        mode = SEARCH_MODE_MAPPING['in_boolean_mode']
-    # split (if possible) to column.key
-    if key.find('.') != -1:
-      table, column = key.split('.')
-      relevance_key1 = self.relevance %key.replace('.', '_')
-      relevance_key2 = self.relevance %column
-    else:
-      relevance_key1 = self.relevance %key
-      relevance_key2 = None
-    select_expression_list = []
-    where_expression = self.where_match_against % (key,
-                            self.quoteSQLString(value, ''), mode)
-    if not stat__:
-      # stat__ is an internal implementation artifact to prevent adding
-      # select_expression for countFolder
-      select_expression_list = [self.select_match_against_as % (key,
-                    self.quoteSQLString(value, ''), mode, relevance_key1),]
-      if relevance_key2 is not None:
-        select_expression_list.append(self.select_match_against_as % (
-          key, self.quoteSQLString(value, ''), mode, relevance_key2))
-    return where_expression, select_expression_list

product/ZSQLCatalog/SearchKey/KeyWordKey.py

-##############################################################################
-#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsability of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# garantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
-from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-from SearchKey import SearchKey
-from pprint import pprint
-
-class KeyWordKey(SearchKey):
-  """ KeyWordKey key is an ERP5 portal_catalog search key which is used to render
-      SQL expression that will try to match all possible values in a greedy manner.
-      It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
-      addition to main logical operators like ['OR', 'or', 'AND', 'and'].
-
-      Examples for title column: 
-        * 'foo or bar'  --> "title LIKE '%foo%' OR title LIKE '%bar%'"
-        * 'foo or =bar'  --> "title LIKE '%foo%' OR title = 'bar'"
-        * 'Organisation Module' -->  "title LIKE '%Organisation Module%'"
-        * '"Organisation Module"' --> "title LIKE '%Organisation Module%'"
-        * '="Organisation Module"' --> "title  = 'Organisation Module'"
-
-  """
-
-  tokens =  ('OR', 'AND', 'NOT', 
-             'KEYWORD', 'WORDSET', 'WORD', 'EXPLICITEQUALLITYWORD',
-             'GREATERTHAN', 'GREATERTHANEQUAL', 
-             'LESSTHAN', 'LESSTHANEQUAL')
-
-  sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL', 
-                    'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
-
-  # this is the default operator
-  default_operator = 'like'
-  
-  # if token's list starts with left sided operator
-  # use this map to transfer it to range operator
-  token_operator_range_map = {'like': 'like',
-                              '!=': 'not_like',
-                              '=': '=',}
-                    
-  # Note: Order of placing rules (t_WORD for example) is very important
-  def t_OR(self, t):
-    r'\s+(OR|or)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'OR'
-    return t
-
-  def t_AND(self, t):
-    r'\s+(AND|and)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'AND'
-    return t  
-
-  def t_NOT(self, t):
-    r'(\s+(NOT|not)\s+|!=)'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = t.value.upper().strip()
-    return t
-
-  t_GREATERTHANEQUAL = r'>='
-  t_LESSTHANEQUAL = r'<='
-  t_GREATERTHAN = r'>'
-  t_LESSTHAN = r'<'
-
-  def t_EXPLICITEQUALLITYWORD(self, t):
-    r'=\S*'
-    # EXPLICITEQUALLITYWORD may contain arbitrary letters and numbers without white space
-    # EXPLICITEQUALLITYWORD must contain '=' at the beginning
-    value = t.value.strip()
-    # get rid of leading '='
-    t.value = value[1:]
-    return t
-
-  def t_KEYWORD(self, t):
-    r'(%\S*|[^!<>=\s%]*%)(?!\S)'
-    # KEYWORD must start and/or end with '%'.
-    # It may contain arbitrary letters and numbers without white space
-    value = t.value.strip()
-    t.value = value
-    return t    
-    
-  def t_WORD(self, t):
-    r'([^"\s<>!=%]([^ \t\r\f\v]*[^ \t\r\f\v%])?|!([^= \t\r\f\v%]|[^= \t\r\f\v][\S\n]*[^ \t\r\f\v%])?)'
-    # WORD may contain arbitrary letters and numbers without white space
-    # WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
-    value = t.value.strip()
-    t.value = value
-    return t
-  
-  def t_WORDSET(self, t):
-    r'=?"[^"]*"'
-    # WORDSET is a combination of WORDs separated by white space
-    # and starting/ending with " (optionally with '=')
-    value = t.value.replace('"', '')
-    if value[0] == '=':
-      value = value[1:]
-    t.value = value
-    return t
-  
-  def getOperatorForTokenList(self, tokens):
-    """ Generic implementation that will return respective 
-        operator for a token list. The first found occurence wins."""
-    token = tokens[0]        
-    token_type = token.type
-    if token_type in self.sub_operators:
-      return token.value, tokens[1:]
-    elif token.type == 'EXPLICITEQUALLITYWORD':
-      # even though it's keyword key we can still explicitly define 
-      # that we want equality
-      return '=', tokens
-    else:
-      return self.default_operator, tokens  
-  
-  def buildQueryForTokenList(self, tokens, key, value, format):
-    """ Build a ComplexQuery for a token list """
-    query_list = []
-    for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
-      token_values = [x.value for x in group_tokens]
-      sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
-      first_token = sub_tokens[0]
-      range = self.token_operator_range_map.get(sub_operator)
-      
-      sub_tokens_values = [x.value for x in sub_tokens]
-      right_side_expression = ' '.join(sub_tokens_values)
-      if first_token.type == 'WORDSET' and first_token.value.startswith('='):
-        range = '='
-        right_side_expression = first_token.value[1:]
-      elif first_token.type in ('WORDSET', 'WORD',) and range == 'like':
-        if '%' not in right_side_expression:
-          # If the search string doesn't already contain '%', add trailing and
-          # leading '%' to get more results
-          right_side_expression = '%%%s%%' % right_side_expression
-      query_kw = {key: right_side_expression,
-                  'range': range}
-      query_list.append(Query(**query_kw))
-
-    # join query list in one really big ComplexQuery
-    complex_query = ComplexQuery(*query_list, 
-                                 **{'operator': 'AND'})
-    return complex_query
-
-    
-##  def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-##    """ Tokenize/analyze passed string value and generate SQL query expressions. """
-##    where_expressions = []
-##    select_expressions = []
-##    tokens = self.tokenize(value)
-##    operators_mapping_list = self.groupByOperator(tokens)
-##    
-##    # find if any logical operator exists
-##    tokens_values = []
-##    logical_operator_found = 0
-##    for token in tokens:
-##      if token.type not in ('WORD',):
-##        logical_operator_found = 1
-##        break
-##      tokens_values.append(token.value.replace("'", ""))
-##      
-##    # build expressions
-##    if not logical_operator_found:
-##      # no logical operator found so we assume that we search 
-##      # for a combination of words
-##      where_expressions.append("%s LIKE '%%%s%%'" %(key, ' '.join(tokens_values)))
-##    else:
-##      # in the search string we have explicitly defined an operator
-##      for item in operators_mapping_list:
-##        row_tokens_values = []
-##        tokens = item['tokens']
-##        operator = item['operator']
-##        operator_value = None
-##        if operator is not None:
-##          # operator is standalone expression
-##          where_expressions.append('%s' %operator.value)
-##        if len(tokens):
-##          # no it's not a stand alone expression, 
-##          # determine it from list of tokens
-##          sub_where_expression = ''
-##          tokens_number = len(tokens)
-##          if tokens_number == 1:
-##            # no left sided operator (<, >, >=, <=) found
-##            token = tokens[0]
-##            if token.type == 'WORD':
-##              sub_where_expression = "LIKE '%%%s%%'" %token.value
-##            elif token.type == 'KEYWORD':
-##              sub_where_expression = "LIKE '%s'" %token.value
-##            elif token.type == 'EXPLICITEQUALLITYWORD':
-##             sub_where_expression = "= '%s'" %token.value
-##            elif token.type == 'WORDSET' and token.value.startswith('='):
-##              # if WORDSET starts with '=' it's an equality
-##              sub_where_expression = " = '%s'" %token.value[1:]
-##            else:
-##              sub_where_expression = "LIKE '%%%s%%'" %token.value
-##          else:
-##            # we have two or more tokens, by definition first one should be 
-##            # logical operator like (<, >, >=, <=)
-##            operator = tokens[0]
-##            operator_value = operator.value
-##            if operator.type in ('KEYWORD', 'WORDSET', 'WORD'):
-##              # no operator for this token list, assume it's 'LIKE'
-##              sub_where_expression = "LIKE '%s'" %' '.join([x.value for x in tokens])
-##            else:
-##              # we have operator and by convention if operator is used it's applyied to one token only
-##              sub_where_expression = "%s'%s'" %(operator_value, tokens[1].value)
-##          where_expressions.append('%s %s' %(key, sub_where_expression))
-##    return where_expressions, select_expressions

product/ZSQLCatalog/SearchKey/KeywordKey.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from SearchKey import SearchKey
+from Products.ZSQLCatalog.SearchText import parse
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
+
+class KeywordKey(SearchKey):
+  """
+    This SearchKey generates matching comparison Queries suited for strings
+    with wilcards.
+  """
+  default_comparison_operator = 'like'
+  get_operator_from_value = True
+ 
+  def parseSearchText(self, value):
+    return parse(value)
+
+verifyClass(ISearchKey, KeywordKey)
+

product/ZSQLCatalog/SearchKey/RawKey.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -27,23 +29,15 @@
 ##############################################################################
 
 from SearchKey import SearchKey
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
 
 class RawKey(SearchKey):
-  """ RawKey key is an ERP5 portal_catalog search key which is used to render
-      SQL expression that will match exactly what's passed to it using equality ."""
+  """
+    This SearchKey does not do any parsing of given value.
+  """
+  default_comparison_operator = '='
+  get_operator_from_value = False
 
-  def build(self, **kwargs):
-    # this key doesn't require parsing
-    # It's required to implement it as it's used ONLY for ExactMath
-    pass
+verifyClass(ISearchKey, RawKey)
 
-  def buildSQLExpression(self, key, value, 
-                         format=None, mode=None, range_value=None, stat__=None):
-
-    if value is not None:
-      value = self.quoteSQLString(value, format)
-      key = self.quoteSQLKey(key, format)
-      where_expression = "%s = %s" % (key, value)
-    else:
-      where_expression = "%s is NULL" % (key)
-    return where_expression, []

product/ZSQLCatalog/SearchKey/RelatedKey.py

+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from SearchKey import SearchKey
+from Products.ZSQLCatalog.Query.Query import Query
+from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
+from Products.ZSQLCatalog.Query.SQLQuery import SQLQuery
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.ISearchKey import IRelatedKey
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+MARKER = []
+BACKWARD_COMPATIBILITY = True
+
+class RelatedKey(SearchKey):
+  """
+    This SearchKey handles searched on virtual columns of RelatedKey type.
+    It generates joins required by the virtual column to reach the actual
+    column to compare, plus a regular query on that column if needed.
+  """
+
+  __implements__ = IRelatedKey
+
+  related_key_definition = None
+
+  @profiler_decorator
+  def _buildRelatedKey(self, related_key_definition):
+    """
+      Extract RelatedKey parameters from its definition, and cache this
+      result. If related_key_definition changes since last computation, cached
+      values will be refreshed.
+
+      related_key_definition (string)
+        Describes parameters of a RelatedKey. It is composed of 3 mains parts,
+        separated by '/':
+        - a list of table names
+          Table names are separated by ','
+        - a column name
+        - the name of the related key ZSQLMethod
+    """
+    assert related_key_definition is not None
+    if self.related_key_definition != related_key_definition:
+      self.related_key_definition = related_key_definition
+      # Extract related_key_id, column_id and table_list from related_key_definition
+      table_list, self.real_column, self.related_key_id = related_key_definition.split('/')
+      self.table_list = table_list.split(',')
+
+  @profiler_decorator
+  def _getSearchKey(self, sql_catalog, search_key_name):
+    """
+      Get search key relevant to the actual column.
+
+      sql_catalog (SQLCatalog)
+        Used to access SearchKey provider.
+      search_key_name (string, None)
+        See SQLCatalog.getSearchKey.
+    """
+    return sql_catalog.getSearchKey(self.real_column, search_key_name)
+
+  @profiler_decorator
+  def getSearchKey(self, sql_catalog, related_key_definition, search_key_name=None):
+    """
+      Get search key relevant to the actual column, extracting information
+      about that column first if needed.
+
+      sql_catalog (SQLCatalog)
+        Used to access SearchKey provider.
+      related_key_definition (string)
+        See _buildRelatedKey.
+      search_key_name (string, None)
+        See SQLCatalog.getSearchKey.
+    """
+    self._buildRelatedKey(related_key_definition)
+    return self._getSearchKey(sql_catalog, search_key_name)
+
+  @profiler_decorator
+  def buildQuery(self, sql_catalog, related_key_definition,
+                 search_value=MARKER, search_key_name=None,
+                 logical_operator=None, comparison_operator=None):
+    self._buildRelatedKey(related_key_definition)
+    if search_value is MARKER:
+      join_condition = None
+    else:
+      join_condition = self._getSearchKey(sql_catalog, search_key_name).buildQuery(
+        search_value, group=self.getColumn(),
+        logical_operator=logical_operator,
+        comparison_operator=comparison_operator)
+    return RelatedQuery(search_key=self,
+                        join_condition=join_condition)
+
+  @profiler_decorator
+  def registerColumnMap(self, column_map, table_alias_list=None):
+    related_column = self.getColumn()
+    group = column_map.registerRelatedKey(related_column, self.real_column)
+    # Each table except last one must be registered to their own group, so that
+    # the same table can be used multiple time (and aliased multiple times)
+    # in the same related key. The last one must be register to related key
+    # "main" group (ie, the value of the "group" variable) to be the same as
+    # the ta ble used in join_condition.
+    if table_alias_list is not None:
+      assert len(self.table_list) == len(table_alias_list)
+    for table_position in xrange(len(self.table_list) - 1):
+      table_name = self.table_list[table_position]
+      local_group = column_map.registerRelatedKeyColumn(related_column, table_position, group)
+      column_map.registerTable(table_name, group=local_group)
+      if table_alias_list is not None:
+        # Pre-resolve all tables with given aliases
+        given_name, given_alias = table_alias_list[table_position]
+        assert table_name == given_name
+        column_map.resolveTable(table_name, given_alias, group=local_group)
+    table_name = self.table_list[-1]
+    column_map.registerTable(table_name, group=group)
+    if table_alias_list is not None:
+      given_name, given_alias = table_alias_list[-1]
+      assert table_name == given_name
+      column_map.resolveTable(table_name, given_alias, group=group)
+    # Resolve (and register) related key column in related key group with its last table.
+    column_map.registerColumn(self.real_column, group=group)
+    column_map.resolveColumn(self.real_column, table_name, group=group)
+    # Always register catalog, since it is always the "base" table of
+    # RelatedKeys.
+    column_map.registerCatalog()
+    return group
+
+  @profiler_decorator
+  def buildSQLExpression(self, sql_catalog, column_map, only_group_columns, group):
+    """
+      Render RelatedKey's ZSQLMethod by providing it table aliases from
+      ColumnMap.
+
+      sql_catalog (SQLCatalog)
+      column_map (ColumnMap)
+      group (string)
+      only_group_columns (bool)
+        Ignored.
+    """
+    related_key = getattr(sql_catalog, self.related_key_id)
+    table_alias_dict = dict(
+      [('table_%s' % (x, ), column_map.getTableAlias(self.table_list[x], group=column_map.getRelatedKeyGroup(x, group)))
+       for x in xrange(len(self.table_list) - 1)])
+    x = len(table_alias_dict)
+    assert x == len(self.table_list) - 1
+    table_alias_dict['table_%s' % (x, )] = column_map.getTableAlias(self.table_list[x], group=group)
+    rendered_related_key = related_key(
+      query_table=column_map.getCatalogTableAlias(),
+      src__=1,
+      **table_alias_dict)
+    # Important:
+    # Former catalog separated join condition from related query.
+    # Example:
+    #   ComplexQuery(Query(title="foo"),
+    #                Query(subordination_title="bar")
+    #                , operator='OR')
+    # Former catalog rendering (truncated where-expression):
+    #   AND ((catalog.title LIKE '%bil%') OR
+    #        (related_catalog_1.title = 'My Organisation'))
+    #   AND (related_catalog_1.uid = related_category_0.category_uid AND
+    #        related_category_0.base_category_uid = 873 AND
+    #        related_category_0.uid = catalog.uid)
+    # As you can see, the part of the query joining the tables is *out* of the
+    # OR expression, and therefor applies to the entire query.
+    # This was done on purpose, because doing otherwise gives very poor
+    # performances (on a simple data set, similar query can take *minutes* to
+    # execute - as of MySQL 5.x).
+    # Doing the same way as the former catalog is required for backward
+    # compatibility, until a decent alternative is found (like spliting the
+    # "OR" expression into ensemblist operations at query level).
+    # Note that doing this has a side effect on result list, as objects
+    # lacking a relation will never appear in the result.
+    if BACKWARD_COMPATIBILITY:
+      column_map.addJoinQuery(SQLQuery(rendered_related_key))
+      return None
+    else:
+      return SQLExpression(self, where_expression=rendered_related_key)
+
+verifyClass(IRelatedKey, RelatedKey)
+

product/ZSQLCatalog/SearchKey/ScriptableKey.py

-##############################################################################
-#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsability of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# garantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
-from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-from Products.PythonScripts.Utility import allow_class
-
-from SearchKey import SearchKey
-from pprint import pprint
-
-# these keys are used to build query in case for ScriptableKey 
-# when no key was specified in fornt of value
-DEFAULT_SEARCH_KEYS = ('SearchableText', 'reference', 'title',)
-
-class KeyMappingKey(SearchKey):
-  """ Usable lexer class used (internally) by ScriptableKey lexer than can parse following:
-      VALUE OPERATOR VALUE
-
-      Examples:
-        * "portal_type : Person"
-        * "creation_date > 2007-01-01"
-  """
-
-  tokens =  ('OPERATOR', 'COLONOPERATOR', 'VALUE',)
-
-  t_OPERATOR = r'>=|<=|>|<'
-  t_VALUE = r'[\x7F-\xFF\w\d\/~!@#$^&*()_+-][\x7F-\xFF\w\d\/~!@#$^&*()_+-]*'
-
-  def t_COLONOPERATOR(self, t):
-    r':'
-    # ':' is the same as '=' (equality)
-    t.value = '='
-    return t
-
-class ScriptableKey(SearchKey):
-  """ KeyWordKey key is an ERP5 portal_catalog search key which is used to generate a 
-      ComplexQuery instance out of an arbitrary search string.
-
-      Examples: 
-        * "John Doe AND portal_type:Person AND creation_date > 2007-01-01" 
-
-        would be turned into following ComplexQuery:
-
-        *  ComplexQuery(Query(portal_type='Person'),
-                       Query(creation_date='2007-01-01', operator='>'),
-                       ComplexQuery(Query(searchable_text='John Doe'),
-                                    Query(title='John Doe'),
-                                    Query(reference='John Doe'),
-                                    operator='OR')
-                       operator='AND'))
-  """
-  sub_operators =  ('GREATERTHAN', 'GREATERTHANEQUAL', 
-                    'LESSTHAN', 'LESSTHANEQUAL',)
-
-  tokens =  ('OR', 'AND',  
-             'DATE', 'WORD', 'KEYMAPPING',
-             'GREATERTHAN', 'GREATERTHANEQUAL', 
-             'LESSTHAN', 'LESSTHANEQUAL', 'EQUAL')
-
-  t_GREATERTHANEQUAL = r'>='  
-  t_LESSTHANEQUAL = r'<='  
-  t_GREATERTHAN = r'>'
-  t_LESSTHAN = r'<'
-  t_EQUAL = r'='
-
-  # Note: Order of placing rules (t_WORD for example) is very important
-  def t_OR(self, t):
-    r'\s+(OR|or)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'OR'
-    return t
-
-  def t_AND(self, t):
-    r'\s+(AND|and)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'AND'
-    return t  
-
-  def t_KEYMAPPING(self, t):
-    r'[^<>=:\s]+\s*(>|<|<=|>=|:)\s*\S+'
-    # KEYMAPPING has following format: KEY OPERATOR VALUE 
-    # where OPERATOR in ['<', '>', '<=', '>=', ':']
-    # example: 'creation_date < 2007-12-12'
-    value = t.value.strip()
-    t.value = value
-    return t 
-
-  def t_WORD(self, t):
-    r'[^<>=\s:]+'
-    # WORD may contain arbitrary letters and numbers without white space
-    # WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
-    value = t.value.strip()
-    t.value = value
-    return t
-
-  def buildQueryForTokenList(self, tokens):
-    """ Build a ComplexQuery for a token list """
-    query_list = []
-    for group in self.groupByLogicalOperator(tokens, 'AND'):
-      group_tokens = group
-      first_group_token = group_tokens[0]
-      if first_group_token.type == 'KEYMAPPING':
-        # user specified a full sub query definition following this format:
-        # 'key operator value'
-        sub_search_string = group_tokens[0].value
-        keymapping_lexer = getSearchKeyInstance(KeyMappingKey)
-        sub_tokens = keymapping_lexer.tokenize(sub_search_string)
-        sub_tokens_values = [x.value for x in sub_tokens]
-        search_key, search_operator, search_value = sub_tokens_values
-        query_kw = {search_key: search_value,
-                    'range' : search_operator,}
-        query_list.append(Query( **query_kw))
-      elif first_group_token.type in self.sub_operators:
-        # user specified a incomplete sub query definition following this format:
-        # 'operator value'. Assume that he ment to search for 'title' and
-        # use supplied 'operator'
-        search_operator = first_group_token.value
-        simple_query_value = ' '.join([x.value for x in group_tokens[1:]])
-        query_kw = {'title': simple_query_value,
-                    'range' : search_operator,}
-        query_list.append(Query( **query_kw))
-      else:
-        # user specified a VERY incomplete sub query definition following this format:
-        # 'value'. Let's search against most common search_keys and assume operator
-        # is '=' (by default) and try to get as much possible results
-        simple_query_value = ' '.join([x.value for x in group_tokens])
-        sub_query_list = []
-        for default_key in DEFAULT_SEARCH_KEYS:
-          query_kw = {default_key: simple_query_value}
-          sub_query_list.append(Query(**query_kw))
-        query_list.append(ComplexQuery(*sub_query_list, 
-                                       **{'operator':'OR'}))
-    # join query list in one really big ComplexQuery
-    complex_query = ComplexQuery(*query_list, 
-                                 **{'operator':'AND'})
-    return complex_query
-
-  def buildQuery(self, key, value, 
-                format=None, mode=None, range_value=None, stat__=None):
-    """ Build ComplexQuery from passed search string value.
-        When grouping expressions we use the following assumptions 
-        that 'OR' operator has higher priority in a sense:
-
-         * "John Doe AND portal_type:Person OR creation_date>=2005/12/12"
-
-         is considered as:
-
-         * (John Doe AND portal_type:Person) OR (creation_date>=2005/12/12)"
-    """
-    query_list = []
-    tokens = self.tokenize(value)
-
-    # split tokens list into one or many OR concatanated expressions
-    sub_tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
-
-    # get a ComplexQuery for a sub token list    
-    for tokens_or_group in sub_tokens_or_groups:
-      query_list.append(self.buildQueryForTokenList(tokens_or_group))
-
-    # join query list in one really big ComplexQuery
-    complex_query = ComplexQuery(*query_list, 
-                                 **{'operator':'OR'})      
-    return complex_query
-
-allow_class(ScriptableKey)

product/ZSQLCatalog/SearchKey/SearchKey.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Jean-Paul Smets-Solanes <jp@nexedi.com>
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -13,7 +15,7 @@
 # This program is Free Software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
-# of the License,] or (at your option) any later version.
+# of the License, or (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -26,220 +28,322 @@
 #
 ##############################################################################
 
-from DocumentTemplate.DT_Var import sql_quote
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
+from zLOG import LOG
+from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
 from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-
-import ply.yacc as yacc
-import ply.lex as lex
-
-class SearchKey:
-  """ BaseKey is a base class that implements a parser of 
-      search grammar used in ERP5. It also implements all generic 
-      search key class methods."""
-
-  # main logical operators
-  operators = ('OR', 'AND',)
-  default_operator = '='
-
-  # in ERP5 search grammar white space is extremely important
-  # so we can not ignore it.
-  #t_ignore  = ' \t' 
-
-  # no need to rack down line numbers
-  #def t_newline(self, t):
-  #  r'\n+'
-  #  #t.lexer.lineno += len(t.value)
-
-  def t_error(self, t):
-    #print "Illegal character '%s'" % t.value[0]
-    t.lexer.skip(1)
-
-  def p_error(self, p):
-    pass
-
-  def build(self, **kwargs):
-    """ This method will initialize respective search key class with 
-        tokens' definitions. """
-    self.lexer = lex.lex(object = self, **kwargs)
-
-  def tokenize(self, data):
-    """ Return list of tokens according to respective 
-        search key tokens' definitions. """
-    result = []
-    self.lexer.input(data)
-    while 1:
-      tok = self.lexer.token()
-      if not tok: 
-        break
-      result.append(tok) 
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+single_operator_dict = {
+  'min': '>=',
+  'max': '<',
+  'ngt': '<=',
+  'nlt': '>'
+}
+
+dual_operator_dict = {
+  'minmax': ('>=', '<'),
+  'minngt': ('>=', '<=')
+}
+
+# List of operators searched for at value's begnining when it's a basestring.
+# Order is important: an operator whose left part would be matching another operator of lower index would never be used.
+operator_list = ('>=', '<=', '>', '<', '=', '!=')
+
+def preprocessLikeValue(value):
+  if '%' not in value:
+    value = '%%%s%%' % (value, )
+  return value
+
+operator_value_preprocessor_dict = {
+  'like': preprocessLikeValue
+}
+
+def deprocessLikeValue(value):
+  assert isinstance(value, basestring)
+  if len(value) >= 2 and value[0] == '%' and value[-1] == '%':
+    value = value.strip('%')
+  return value
+
+operator_value_deprocessor_dict = {
+  'like': deprocessLikeValue
+}
+
+class SearchKey(object):
+
+  __implements__ = ISearchKey
+
+  # Comparison operator to use when parsing a string value and no operator is
+  # found.
+  # Note: for non-string values, "=" is always used by default.
+  default_comparison_operator = '='
+  # Wether or not to allow a basestring value to be searched for a comparison
+  # operator.
+  get_operator_from_value = True
+
+  def __init__(self, column):
+    self.column = column
+
+  def getColumn(self):
+    return self.column
+
+  @profiler_decorator
+  def buildSQLExpression(self, operator, value, column_map, only_group_columns, group):
+    column_name = self.getColumn()
+    rendered_column = column_map.asSQLColumn(column_name, group=group)
+    return operator.asSQLExpression(rendered_column, value, only_group_columns)
+
+  @profiler_decorator
+  def _renderValueAsSearchText(self, value, operator):
+    """
+      Render a single value as valid SearchText using provided operator.
+      This is also responsible for undoing any formatting the value received
+      from the SearchKey.
+
+      value (anything)
+      operator (Operator)
+        The operator used to render value.
+    """
+    operator_value_deprocessor = operator_value_deprocessor_dict.get(operator.getOperator())
+    if operator_value_deprocessor is not None:
+      value = operator_value_deprocessor(value)
+    return operator.asSearchText(value)
+
+  @profiler_decorator
+  def buildSearchTextExpression(self, operator, value, column=None):
+    operator_text = operator.getOperator()
+    if column is None:
+      column = self.getColumn()
+    if isinstance(value, (list, tuple)):
+      assert operator_text == 'in'
+      assert len(value)
+      value = [self._renderValueAsSearchText(x, operator) for x in value]
+      if self.default_comparison_operator != '=':
+        value = ['=%s' % (x, ) for x in value]
+      # XXX: operator used to join value elements should be reused from parser data (?)
+      result = '(%s)' % (' OR '.join(value), )
+    else:
+      result = self._renderValueAsSearchText(value, operator)
+      if operator_text != self.default_comparison_operator:
+        result = '%s%s' % (operator_text, result)
+    if len(column):
+      result = '%s:%s' % (column, result)
     return result
 
-  # Grouping of tokens
-  def getOperatorForTokenList(self, tokens):
-    """ Generic implementation that will return respective 
-        operator for a token list. The first found occurence wins."""
-    token = tokens[0]
-    token_type = token.type
-    if token_type in self.sub_operators:
-      return token.value, tokens[1:]
+  @profiler_decorator
+  def registerColumnMap(self, column_map, group, simple_query):
+    column_map.registerColumn(self.getColumn(), group=group, simple_query=simple_query)
+    return group
+
+  @profiler_decorator
+  def _getComparisonOperator(self, value):
+    """
+      From a basestring instance, return a contained operator and value without
+      that operator.
+
+      value (string)
+      
+      Returns: 2-tuple of strings
+        First element is the operator. None if there was no operator in value.
+        Second element is the value without the operator.
+    """
+    startswith = value.startswith
+    for operator in operator_list:
+      if startswith(operator):
+        value = value[len(operator):].lstrip()
+        break
     else:
-      return self.default_operator, tokens    
+      operator = self._guessComparisonOperator(value)
+    return operator, value
+
+  @profiler_decorator
+  def _guessComparisonOperator(self, value):
+    """
+      From a basestring instance, return a contained operator.
+      Value cannot be altered in the process.
+
+      value (string)
+      
+      Returns: 2-tuple of strings
+        First element is the operator. None if there was no operator in value.
+        Second element is the value without the operator.
+    """
+    return self.default_comparison_operator
+
+  @profiler_decorator
+  def _preprocessValue(self, value, operator):
+    operator_value_preprocessor = operator_value_preprocessor_dict.get(operator)
+    if operator_value_preprocessor is not None:
+      value = operator_value_preprocessor(value)
+    return value
 
-  def groupByLogicalOperator(self, tokens, logical_operator ='OR'):
-    """ Split tokens list into one or many OR concatanated tokens list 
+  @profiler_decorator
+  def _processSearchValue(self, search_value, default_logical_operator, comparison_operator):
     """
-    sub_tokens_or_groups = []
-    tmp_token_list = []
-    for token in tokens:
-      if token.type != logical_operator:
-        tmp_token_list.append(token)
+      Change search_value into a list of values, one or more logical operators,
+      and a comparison operator. If no default_logical_operator is given, 
+      'or' is used.
+
+      search_value
+        basestring
+        int
+        dict
+        list or tuple
+          Non-empty
+          Composed of homogeneous items
+      Returns: 3-tuple
+        dict:
+          key (string)
+            Comparison operator
+          value (list of anything)
+            List of values applying to this operator.
+        string:
+          Logical operator applied to all elements of returned dict.
+        bool:
+          True if logical operators were searched for in values, False
+          otherwise. Useful to give different meanings to in-value operators
+          and others.
+    """
+    if comparison_operator == '':
+      comparison_operator = None
+      get_operator_from_value = False
+    else:
+      get_operator_from_value = self.get_operator_from_value
+    logical_operator = None
+    if default_logical_operator is None:
+      default_logical_operator = 'or'
+    parsed = False
+    if isinstance(search_value, dict):
+      # comparison_operator parameter collides with dict's 'operator' key.
+      # Fail loudly.
+      assert comparison_operator is None
+      actual_value = search_value['query']
+      if search_value.get('key') not in (None, self.__class__.__name__):
+        LOG(self.__class__.__name__, 100, '"key" dict entry does not match current class: %r' % (search_value, ))
+      if 'type' in search_value:
+        assert 'operator' not in search_value, search_value
+        assert 'range' not in search_value, search_value
+      else:
+        value_operator = search_value.get('operator')
+        value_range = search_value.get('range')
+        if value_range is not None:
+          if value_operator is not None:
+            LOG('SearchKey', 100, '"range" and "operator" are mutualy exclusive, ignoring operator: %r' % (search_value, ))
+          if value_range in single_operator_dict:
+            comparison_operator = single_operator_dict[value_range]
+          elif value_range in dual_operator_dict:
+            if not isinstance(actual_value, (tuple, list)):
+              raise TypeError, 'Operator %r requires value to be a tuple/list. (%r)' % (value_range, search_value)
+            if len(actual_value) != 2:
+              raise TypeError, 'Operator %r requires value to have a length of 2. len(%r) = %i (%r)' % (value_range, actual_value, len(actual_value), search_value)
+            comparison_operator = dual_operator_dict[value_range]
+            logical_operator = 'and'
+          else:
+            raise ValueError, 'Unknown "range" value in %r' % (search_value, )
+        if value_operator is not None:
+          if not isinstance(value_operator, basestring):
+            raise TypeError, 'Operator must be of a string type. Got a %r' % (type(value_operator), )
+          value_operator = value_operator.lower()
+          if not isinstance(actual_value, (tuple, list)):
+            raise TypeError, 'When specifying an operator, query must be a list.'
+          if value_operator == 'in':
+            comparison_operator = '='
+            logical_operator = 'or'
+          else:
+            logical_operator = value_operator
+        search_value = actual_value
+    # Cast to list
+    if isinstance(search_value, (tuple, list)):
+      # Check list content (not empty, homogenous)
+      search_value_len = len(search_value)
+      if search_value_len == 0:
+        raise ValueError, 'Value cannot be an empty list/tuple: %r' % (search_value, )
+      reference_class = search_value[0].__class__
+      for x in search_value[1:]:
+        if x.__class__ != reference_class:
+          raise TypeError, 'List elements must be of the same class: %r' % (search_value, )
     else:
-        sub_tokens_or_groups.append(tmp_token_list)
-        tmp_token_list = []
-    # append remainig last tokens
-    sub_tokens_or_groups.append(tmp_token_list)
-    return sub_tokens_or_groups    
-
-  # SQL quoting (each search key should override them it if needed)
-  def quoteSQLKey(self, key, format):
-    """ Return a quoted string of the value. """
-    return key
-
-  def quoteSQLString(self, value, format):
-    """ Return a quoted string of the value. """
-    return "'%s'" % sql_quote(str(value))
-
-  # SQL generation
-  def buildSQLExpression(self, key, value, 
-                               format=None, mode=None, range_value=None, stat__=0):
-    """ Generic implementation. Leave details to respective key. """
-    if range_value is not None:
-      # if range_value we handle directly (i.e no parsing of search string)
-      where_expressions, select_expressions = \
-         self.buildSQLExpressionFromRange(key, value, 
-                                          format, mode, range_value, stat__) 
+      assert logical_operator is None
+      if isinstance(search_value, dict):
+        reference_class = search_value['query'].__class__
       else:
-      # search string parsing is needed
-      where_expressions, select_expressions = \
-        self.buildSQLExpressionFromSearchString(key, str(value), 
-                                                format, mode, range_value, stat__) 
-    return where_expressions, select_expressions
-
-  def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-    complex_query = self.buildQuery(key, value, format, mode, range_value, stat__)
-    if complex_query is None:
-      # Query could not be generated from search string
-      sql_expression = {'where_expression': '1', 
-                        'select_expression_list': []}
+        reference_class = search_value.__class__
+      search_value = [search_value]
+    if logical_operator is None:
+      logical_operator = default_logical_operator
+    operator_value_dict = {}
+    if None in search_value:
+      if comparison_operator not in (None, 'is'):
+        LOG('KeywordKey', 100, 'None value requires an "is" comparison operator. Fixed.')
+      operator_value_dict['is'] = search_value
+    elif comparison_operator is None:
+      if issubclass(reference_class, basestring):
+        if get_operator_from_value:
+          parsed = True
+          for value in search_value:
+            if isinstance(value, dict):
+              operator, value['query'] = self._getComparisonOperator(value['query'])
             else:
-      sql_expression = complex_query(keyword_search_keys = [],
-                                     datetime_search_keys = [],
-                                     full_text_search_keys = [])
-    return sql_expression['where_expression'], sql_expression['select_expression_list']
+              operator, value = self._getComparisonOperator(value)
+            operator_value_dict.setdefault(operator, []).append(self._preprocessValue(value, operator))
+        else:
+          for value in search_value:
+            if isinstance(value, dict):
+              operator = self._guessComparisonOperator(value['query'])
+            else:
+              operator = self._guessComparisonOperator(value)
+            operator_value_dict.setdefault(operator, []).append(self._preprocessValue(value, operator))
+      else:
+        # XXX: comparison operator is hardcoded for non-strings.
+        operator_value_dict['='] = search_value
+    elif isinstance(comparison_operator, (tuple, list)):
+      assert len(comparison_operator) == len(search_value)
+      for operator, value in zip(comparison_operator, search_value):
+        operator_value_dict.setdefault(operator, []).append(value)
+    else:
+      operator_value_dict[comparison_operator] = search_value
+    return operator_value_dict, logical_operator, parsed
 
-  def buildQuery(self, key, value, format, mode, range_value, stat__):
-    """ Build Query """
-    query_list = []
-    # tokenize searchs string into tokens for Search Key
-    tokens = self.tokenize(value)
+  @profiler_decorator
+  def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
+    """
+      Create Queries from values, logical and comparison operators.
 
-    # split tokens list into one or more 'OR' tokens lists
-    tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
+      operator_value_dict (dict)
+        See _processSearchValue.
+      logical_operator (string)
+        See _processSearchValue.
+      parsed (bool)
+        See _processSearchValue.
+      group (string)
+        The gorup all queries will belong to.
+    """
+    column = self.getColumn()
+    query_list = []
+    append = query_list.append
+    if logical_operator == 'or' and '=' in operator_value_dict:
+      # Special case for equality with an 'or' logical operator: use SQL 'in'.
+      append(SimpleQuery(search_key=self, operator='in', group=group, **{column: operator_value_dict.pop('=')}))
+    for comparison_operator, value_list in operator_value_dict.iteritems():
+      for value in value_list:
+        append(SimpleQuery(search_key=self, operator=comparison_operator, group=group, **{column: value}))
+    return query_list
 
-    # remove empty tokens lists
-    tokens_or_groups = filter(lambda x: len(x), tokens_or_groups)
+  @profiler_decorator
+  def buildQuery(self, search_value, group=None, logical_operator=None, comparison_operator=None):
+    assert logical_operator in (None, 'and', 'or'), repr(logical_operator)
+    operator_value_dict, logical_operator, parsed = self._processSearchValue(search_value, logical_operator, comparison_operator)
+    query_list = self._buildQuery(operator_value_dict, logical_operator, parsed, group)
+    if len(query_list) == 1:
+      query = query_list[0]
+    else:
+      query = ComplexQuery(query_list, operator=logical_operator)
+    return query
 
-    # get a ComplexQuery for a sub token list
-    for tokens_or_group in tokens_or_groups:
-      query = self.buildQueryForTokenList(tokens_or_group, key, value, format)
-      if query is not None:
-        # query could be generated for token list
-        query_list.append(query)
+  def parseSearchText(self, value):
+    return None
 
-    if len(query_list):
-      # join query list in one really big ComplexQuery
-      return ComplexQuery(*query_list,
-                          **{'operator':'OR'})
+verifyClass(ISearchKey, SearchKey)
 
-  def buildQueryForTokenList(self, tokens, key, value, format):
-    """ Build a ComplexQuery for a token list """
-    query_list = []
-    logical_groups = self.groupByLogicalOperator(tokens, 'AND')
-    for group_tokens in logical_groups:
-      token_values = [x.value for x in group_tokens]
-      sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
-      sub_tokens_values = [x.value for x in sub_tokens]
-      query_kw = {key: ' '.join(sub_tokens_values),
-                  'type': self.default_key_type,
-                  'format': format,
-                  'range': sub_operator}
-      query_list.append(Query(**query_kw))
-
-    # join query list in one really big ComplexQuery
-    complex_query = ComplexQuery(*query_list, 
-                                 **{'operator': 'AND'})
-    return complex_query
-
-  def buildSQLExpressionFromRange(self, key, value, format, mode, range_value, stat__):
-    """ This method will generate SQL expressions
-       from explicitly passed list of values and 
-       range_value in ('min', 'max', ..)"""
-    key = self.quoteSQLKey(key, format)  
-    where_expression = ''
-    select_expressions = []
-    if isinstance(value, (list, tuple)):
-      if len(value) > 1:
-        # value should contain at least two items
-        query_min = self.quoteSQLString(value[0], format)
-        query_max = self.quoteSQLString(value[1], format)
-      else:
-        # value contains only one item
-        query_min = query_max = self.quoteSQLString(value[0], format)          
-    else:
-      query_min = query_max = self.quoteSQLString(value, format)
-    if range_value == 'min':
-      where_expression = "%s >= %s" % (key, query_min)
-    elif range_value == 'max':
-      where_expression = "%s < %s" % (key, query_max)
-    elif range_value == 'minmax' :
-      where_expression = "%s >= %s AND %s < %s" % (key, query_min, key, query_max)
-    elif range_value == 'minngt' :
-      where_expression = "%s >= %s AND %s <= %s" % (key, query_min, key, query_max)
-    elif range_value == 'ngt':
-      where_expression =  "%s <= %s" % (key, query_max)
-    elif range_value == 'nlt':
-      where_expression = "%s > %s" % (key, query_max)
-    elif range_value == 'like':
-      where_expression = "%s LIKE %s" % (key, query_max)
-    elif range_value == 'not_like':
-      where_expression = "%s NOT LIKE %s" % (key, query_max)
-    elif range_value in ('=', '>', '<', '>=', '<=','!=',):
-      where_expression = "%s %s %s" % (key, range_value, query_max)    
-    return where_expression, select_expressions
-
-
-##  def groupByOperator(self, tokens, group_by_operators_list = operators):
-##    """ Generic implementation of splitting tokens into logical
-##        groups defided by respective list of logical operator
-##        defined for respective search key.  """
-##    items = []
-##    last_operator = None
-##    operators_mapping_list = []
-##    last_operator = {'operator': None,
-##                     'tokens': []}
-##    for token in tokens:
-##      token_type = token.type
-##      token_value = token.value
-##      if token_type in group_by_operators_list:
-##        # (re) init it
-##        last_operator = {'operator': token,
-##                         'tokens': []}
-##        operators_mapping_list.append(last_operator)
-##      else:
-##        # not an operator just a value token
-##        last_operator['tokens'].append(token)
-##        if last_operator not in operators_mapping_list:
-##          operators_mapping_list.append(last_operator)
-##    return operators_mapping_list      

product/ZSQLCatalog/SearchKey/__init__.py

+# Automaticaly import all SearchKeys
+import os
+module_path = os.path.dirname(os.path.abspath(__file__))
+global_dict = globals()
+__relative_file__ = os.path.basename(__file__)
+for filename in os.listdir(module_path):
+  if filename.endswith('.py') and filename != __relative_file__:
+    modulename = filename[:-3]
+    try:
+      module = __import__(modulename, global_dict, None, [])
+    except ImportError:
+      continue
 

product/ZSQLCatalog/SearchKey/FloatKey.py → product/ZSQLCatalog/SearchText/AdvancedSearchTextDetector.py

 ##############################################################################
 #
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-#                     Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsability of assessing all potential
@@ -13,7 +13,7 @@
 # This program is Free Software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
-# of the License,] or (at your option) any later version.
+# of the License, or (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -26,67 +26,85 @@
 #
 ##############################################################################
 
-from SearchKey import SearchKey
+from lexer import lexer, update_docstrings
 
-class FloatKey(SearchKey):
-  """ FloatKey key is an ERP5 portal_catalog search key which is used to render
-      float like SQL expression.
-  """
-  # default type of sub Queries to be generated out fo a search string
-  default_key_type = 'float'
+# TODO: remove the special OPERATOR case: it does not work when there are both a valid and an invalid operator
 
-  tokens =  ('OR', 'AND', 'NOT', 'FLOAT',
-             'GREATERTHAN', 'GREATERTHANEQUAL', 
-             'LESSTHAN', 'LESSTHANEQUAL')
+class AdvancedSearchTextDetector(lexer):
 
-  sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL', 
-                    'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
+  def t_OPERATOR(self, t):
+    r'(>=?|<=?|!?=)[ ]*'
+    return t
+
+  def t_LEFT_PARENTHESE(self, t):
+    self.found = True
+    t.type = 'WORD'
+    return t
 
+  def t_STRING(self, t):
+    self.found = True
+    t.type = 'WORD'
+    return t
+
+  def t_COLUMN(self, t):
+    self.found = True
+    t.type = 'WORD'
+    return t
 
-  # Note: Order of placing rules (t_WORD for example) is very important
   def t_OR(self, t):
-    r'\s+(OR|or)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'OR'
+    self.found = True
+    t.type = 'WORD'
     return t
 
   def t_AND(self, t):
-    r'\s+(AND|and)\s+'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = 'AND'
+    self.found = True
+    t.type = 'WORD'
     return t
 
   def t_NOT(self, t):
-    r'(\s+(NOT|not)\s+|!=)'
-    # operator must have leading and trailing ONLY one white space character
-    # otherwise it's treated as a WORD
-    t.value = '!=' 
+    self.found = True
+    t.type = 'WORD'
     return t
 
-  t_GREATERTHANEQUAL = r'>='  
-  t_LESSTHANEQUAL = r'<='  
-  t_GREATERTHAN = r'>'
-  t_LESSTHAN = r'<'  
+  def p_search_text(self, p):
+    '''search_text : value
+                   | value search_text'''
+    if len(p) == 2:
+      p[0] = p[1]
+    else:
+      p[0] = p[1] or p[2]
 
-  def t_FLOAT(self, t):
-    r'[\d.][\d.]*'
-    # FLOAT is a float number
-    value = t.value.replace('"', '').strip()
-    t.value = value
-    return t
+  def p_value(self, p):
+    '''value : WORD
+             | OPERATOR WORD'''
+    p[0] = len(p) == 3 and ' ' not in p[1]
+
+  tokens = (
+    'WORD',
+    'OPERATOR')
+
+  def real_token(self):
+    return lexer.token(self)
+
+  def token(self):
+    return self.token_list.pop(0)
+
+  def __call__(self, input):
+    self.found = False
+    check_grammar = False
+    self.token_list = token_list = []
+    append = token_list.append
+    self.input(input)
+    while not self.found:
+      token = self.real_token()
+      append(token)
+      if token is None:
+        break
+      if token.type == 'OPERATOR':
+        check_grammar = True
+    if not self.found and check_grammar:
+      self.found = self.parse()
+    return self.found
+
+update_docstrings(AdvancedSearchTextDetector)
 
-  def quoteSQLString(self, value, format):
-    """ Return a quoted string of the value. """
-    # Make sure there is no space in float values
-    return "'%s'" %str(value).replace(' ', '')
-
-  def quoteSQLKey(self, key, format):
-    """ Return a quoted string of the value. """
-    if format is not None:
-      float_format = format.replace(' ', '')
-      if float_format.find('.') >= 0:
-        precision = len(float_format.split('.')[1])
-        key = "TRUNCATE(%s,%s)" % (key, precision)    
-    return key    

product/ZSQLCatalog/SearchText/AdvancedSearchTextParser.py

+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from lexer import lexer, update_docstrings
+try:
+  from Products.ZSQLCatalog.Interface.IAbstractSyntaxNode import INode, IValueNode, ILogicalNode, IColumnNode
+  from Interface.Verify import verifyClass
+except ImportError:
+  INode = None
+  IValueNode = None
+  ILogicalNode = None
+  IColumnNode = None
+  def verifyClass(*args, **kw):
+    pass
+
+class Node(object):
+
+  __implements__ = INode
+
+  def isLeaf(self):
+    return False
+
+  def isColumn(self):
+    return False
+
+  def push(self, logical_operator, node):
+    return LogicalNode(logical_operator, self, node)
+
+verifyClass(INode, Node)
+
+class ValueNode(Node):
+
+  __implements__ = IValueNode
+
+  def __init__(self, value, comparison_operator=''):
+    self.value = value
+    self.comparison_operator = comparison_operator
+
+  def isLeaf(self):
+    return True
+
+  def getComparisonOperator(self):
+    return self.comparison_operator
+
+  def getValue(self):
+    return self.value
+
+  def __repr__(self):
+    return '<%s %r %r>' % (self.__class__.__name__, self.comparison_operator, self.value)
+
+verifyClass(INode, ValueNode)
+verifyClass(IValueNode, ValueNode)
+
+class NotNode(Node):
+
+  __implements__ = ILogicalNode
+
+  def __init__(self, node):
+    self.node = node
+
+  def getLogicalOperator(self):
+    return 'not'
+
+  def getNodeList(self):
+    return [self.node]
+
+  def __repr__(self):
+    return '<%s %r>' % (self.__class__.__name__, self.node)
+
+verifyClass(INode, NotNode)
+verifyClass(ILogicalNode, NotNode)
+
+class LogicalNode(Node):
+
+  __implements__ = ILogicalNode
+
+  def __init__(self, logical_operator, node, other):
+    self.logical_operator = logical_operator
+    self.node_list = []
+    self._push(node)
+    self._push(other)
+
+  def getLogicalOperator(self):
+    return self.logical_operator
+
+  def getNodeList(self):
+    return self.node_list
+
+  def _push(self, node):
+    if isinstance(node, LogicalNode) and node.logical_operator == self.logical_operator:
+      self.node_list.extend(node.node_list)
+    else:
+      self.node_list.append(node)
+
+  def __repr__(self):
+    return '<%s %r %r>' % (self.__class__.__name__, self.logical_operator, self.node_list)
+
+verifyClass(INode, LogicalNode)
+verifyClass(ILogicalNode, LogicalNode)
+
+class ColumnNode(Node):
+
+  __implements__ = IColumnNode
+
+  def __init__(self, column_name, node):
+    self.column_name = column_name
+    self.node = node
+
+  def isColumn(self):
+    return True
+
+  def getColumnName(self):
+    return self.column_name
+
+  def getSubNode(self):
+    return self.node
+
+  def __repr__(self):
+    return '<%s %r %r>' % (self.__class__.__name__, self.column_name, self.node)
+
+verifyClass(INode, ColumnNode)
+verifyClass(IColumnNode, ColumnNode)
+
+class AdvancedSearchTextParser(lexer):
+
+  def p_seach_text(self, p):
+    '''search_text : and_expression
+                   | and_expression OR search_text
+                   | and_expression search_text'''
+    if len(p) == 2:
+      p[0] = p[1]
+    elif len(p) == 3:
+      p[0] = p[1].push('or', p[2])
+    else:
+      p[0] = p[1].push('or', p[3])
+
+  def p_and_expression(self, p):
+    '''and_expression : boolean_expression
+                      | boolean_expression AND and_expression'''
+    if len(p) == 2:
+      p[0] = p[1]
+    else:
+      p[0] = p[1].push('and', p[3])
+
+  def p_boolean_expression(self, p):
+    '''boolean_expression : NOT expression
+                          | expression'''
+    if len(p) == 3:
+      p[0] = NotNode(p[2])
+    else:
+      p[0] = p[1]
+
+  def p_expression(self, p):
+    '''expression : LEFT_PARENTHESE search_text RIGHT_PARENTHESE
+                  | column
+                  | value'''
+    if len(p) == 2:
+      p[0] = p[1]
+    else:
+      p[0] = p[2]
+
+  def p_column(self, p):
+    '''column : COLUMN value_expression'''
+    p[0] = ColumnNode(p[1], p[2])
+
+  def p_value_expression(self, p):
+    '''value_expression : LEFT_PARENTHESE value_or_expression RIGHT_PARENTHESE
+                        | value'''
+    if len(p) == 2:
+      p[0] = p[1]
+    else:
+      p[0] = p[2]
+
+  def p_value_or_expression(self, p):
+    '''value_or_expression : value_and_expression
+                           | value_and_expression value_or_expression
+                           | value_and_expression OR value_or_expression'''
+    if len(p) == 2:
+      p[0] = p[1]
+    elif len(p) == 3:
+      p[0] = p[1].push('or', p[2])
+    else:
+      p[0] = p[1].push('or', p[3])
+
+  def p_value_and_expression(self, p):
+    '''value_and_expression : value_expression
+                            | value_expression AND value_and_expression'''
+    if len(p) == 2:
+      p[0] = p[1]
+    else:
+      p[0] = p[1].push('and', p[3])
+
+  def p_value(self, p):
+    '''value : OPERATOR STRING
+             | OPERATOR WORD
+             | STRING
+             | WORD'''
+    if len(p) == 2:
+      p[0] = ValueNode(p[1])
+    else:
+      p[0] = ValueNode(p[2], comparison_operator=p[1])
+
+update_docstrings(AdvancedSearchTextParser)
+

product/ZSQLCatalog/SearchText/SearchTextParser.py

+#!/usr/bin/python
+# -*- coding: utf8 -*-
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import threading
+from AdvancedSearchTextDetector import AdvancedSearchTextDetector
+from AdvancedSearchTextParser import AdvancedSearchTextParser
+from lexer import ParserOrLexerError
+try:
+  from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+except ImportError:
+  def profiler_decorator(funct):
+    return funct
+
+if __name__ == '__main__':
+  DEBUG = 1
+else:
+  DEBUG = 0
+
+parser_pool = threading.local()
+
+@profiler_decorator
+def getAdvancedSearchTextDetector():
+  try:
+    return parser_pool.advanced_search_text_detector
+  except AttributeError:
+    advanced_search_text_detector = AdvancedSearchTextDetector()
+    advanced_search_text_detector.init(debug=DEBUG)
+    parser_pool.advanced_search_text_detector = advanced_search_text_detector
+    return advanced_search_text_detector
+
+@profiler_decorator
+def getAdvancedSearchTextParser():
+  try:
+    return parser_pool.parser
+  except AttributeError:
+    parser = AdvancedSearchTextParser()
+    parser.init(debug=DEBUG)
+    parser_pool.parser = parser
+    return parser
+
+@profiler_decorator
+def _parse(input, *args, **kw):
+  if getAdvancedSearchTextDetector()(input):
+    result = getAdvancedSearchTextParser()(input, *args, **kw)
+  else:
+    result = None
+  return result
+
+@profiler_decorator
+def parse(*args, **kw):
+  try:
+    result = _parse(*args, **kw)
+  except (KeyboardInterrupt, SystemExit):
+    raise
+  except:
+    result = None
+  return result
+
+if __name__ == '__main__':
+  class Query:
+    def __init__(self, column, value, comparison_operator='='):
+      self.column = column
+      self.comparison_operator = comparison_operator
+      if isinstance(value, (list, tuple)):
+        value = ''.join(value)
+      self.value = value
+
+    def asTuple(self):
+      return (self.column, self.value, self.comparison_operator)
+
+    def __repr__(self):
+      value = self.value
+      if len(value) == 1:
+        value = value[0]
+      return 'Query(%r, %r, %r)' % (self.column, value, self.comparison_operator)
+
+    def __eq__(self, other):
+      if isinstance(other, Query):
+        return self.asTuple() == other.asTuple()
+      else:
+        return False
+
+    def __ne__(self, other):
+      return not (self == other)
+
+  class ComplexQuery:
+    def __init__(self, query_list, operator):
+      self.operator = operator
+      self.query_list = query_list
+
+    def __repr__(self):
+      return 'ComplexQuery(%r, operator=%r)' % (self.query_list, self.operator)
+
+    def __eq__(self, other):
+      if isinstance(other, ComplexQuery):
+        if self.operator != other.operator:
+          return False
+        other_query_list = other.query_list[:]
+        for my_query in self.query_list:
+          for other_index in xrange(len(other_query_list)):
+            other_query = other_query_list[other_index]
+            if my_query == other_query:
+              other_query_list.pop(other_index)
+              break
+          else:
+            return False
+        return len(other_query_list) == 0
+      else:
+        return False
+
+    def __ne__(self, other):
+      return not (self == other)
+
+  check_list = [
+    ('foo', None),
+    ('foo bar', None),
+    ('foo   bar', None),
+    ('foo%', None),
+    ('%foo', None),
+    ('%foo%', None),
+    ('foo%bar', None),
+    ('foo% bar', None),
+    ('foo %bar', None),
+    ('foo and bar', None),
+    ('foo or bar', None),
+    ('foo - bar', None),
+    ('foo- bar', None),
+    ('!-1', None),
+    ('->1', None),
+    ('+=1', None),
+    ('jean-paul', None),
+    ('JeanAndPaul', None),
+    ('totoORtata', None),
+    ('NORD', None),
+    ('OR ARGENT', None),
+    ('CUIVRE OR ARGENT', None), # XXX
+    ('title :foo', None),
+    ('-foo', None),
+    ('foo -bar', None),
+    ('+foo -bar', None),
+    ('+1', None),
+    ('-1', None),
+
+    ('foo OR "-" OR bar OR -baz', ComplexQuery([Query(None, 'foo'), Query(None, '-'), Query(None, 'bar'), Query(None, '-baz')], operator='or')),
+    ('foo "-" bar -baz',          ComplexQuery([Query(None, 'foo'), Query(None, '-'), Query(None, 'bar'), Query(None, '-baz')], operator='or')),
+    ('title:foo',                 Query('title', 'foo')),
+    ('title: foo',                Query('title', 'foo')),
+    ('title:foo bar',             ComplexQuery([Query('title', 'foo'), Query(None, 'bar')], operator='or')),
+    ('title:"foo bar"',           Query('title', 'foo bar')),
+    ('"title:foo bar"',           Query(None, 'title:foo bar')),
+    ('"foo bar"',                 Query(None, 'foo bar')),
+    ('"foo   bar"',               Query(None, 'foo   bar')),
+    ('foo AND bar',               ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='and')),
+    ('foo OR bar',                ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='or')),
+    ('"foo AND bar"',             Query(None, 'foo AND bar')),
+    ('"foo and bar"',             Query(None, 'foo and bar')),
+    ('"foo OR bar"',              Query(None, 'foo OR bar')),
+    ('"foo or bar"',              Query(None, 'foo or bar')),
+    ('"foo% bar"',                Query(None, 'foo% bar')),
+    ('"foo %bar"',                Query(None, 'foo %bar')),
+    ('>1',                        Query(None, '1', '>')),
+    ('">1"',                      Query(None, '>1')),
+    ('>a',                        Query(None, 'a', '>')),
+    ('">a"',                      Query(None, '>a')),
+    ('>1 0',                      ComplexQuery([Query(None, '1', '>'), Query(None, '0')], operator='or')),
+    ('>=1',                       Query(None, '1', '>=')),
+    ('>"=1"',                     Query(None, '=1', '>')),
+    ('-"1"',                      ComplexQuery([Query(None, '-'), Query(None, '1')], operator='or')),
+    ('"!-1"',                     Query(None, '!-1')),
+#    (r"a:'tu:\'tu\''",            ['a', "tu:'tu'"]),
+    (r'''b:"tu:\'tu\'"''',        Query('b', "tu:\\'tu\\'")),
+    (r'''c:"tu:'tu'"''',          Query('c', "tu:'tu'")),
+    (r'd:"tu:\"tu\""',            Query('d', 'tu:"tu"')),
+    ('toto: tutu tutu',           ComplexQuery([Query('toto', 'tutu'), Query(None, 'tutu')], operator='or')),
+    ('(tutu) (toto:tata)',        ComplexQuery([Query(None, 'tutu'), Query('toto', 'tata')], operator='or')),
+    ('(tutu) (toto:"tata")',      ComplexQuery([Query(None, 'tutu'), Query('toto', 'tata')], operator='or')),
+#    ('toto:',                     ['toto', '']),
+    ('toto:""',                   Query('toto', '')),
+#    ("''",                        ''),
+    ('""',                        Query(None, '')),
+    (r'"\""',                     Query(None, '"')),
+    (r'"\n"',                     Query(None, '\\n')),
+#こんにちは
+    (u'ん',                       None),
+    (u'(toto:ん) OR (titi:ん)',   ComplexQuery([Query('toto', u'ん'), Query('titi', u'ん')], operator='or')),
+    ('ん',                        None),
+    ('(toto:ん) OR (titi:ん)',    ComplexQuery([Query('toto', 'ん'), Query('titi', 'ん')], operator='or')),
+    ('(foo)',                     Query(None, 'foo')),
+    ('toto:(foo)',                Query('toto', 'foo')),
+    ('(foo OR bar)',              ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='or')),
+    ('(a AND b) OR (c AND (d OR e))',
+                                  ComplexQuery([ComplexQuery([Query(None, 'a'), Query(None, 'b')], operator='and'), ComplexQuery([Query(None, 'c'), ComplexQuery([Query(None, 'd'), Query(None, 'e')], operator='or')], operator='and')], operator='or')),
+    ('(foo:"") (bar:baz)',        ComplexQuery([Query('foo', ''), Query('bar', 'baz')], operator='or')),
+    ('(foo:"") (OR:bar)',         ComplexQuery([Query('foo', ''), Query('OR', 'bar')], operator='or')),
+#    ('foo: OR',                   ['foo', 'or']),
+#    ('foo: OR ',                  ['foo', 'or']),
+#    ('(foo:)',                    ['foo', '']),
+    ('(foo: bar)',                Query('foo', 'bar')),
+    ('(a:b) AND (c:d)',           ComplexQuery([Query('a', 'b'), Query('c', 'd')], operator='and')),
+    ('a:(b c)',                   ComplexQuery([Query('a', 'b'), Query('a', 'c')], operator='or')),
+    ('a:(b OR c)',                ComplexQuery([Query('a', 'b'), Query('a', 'c')], operator='or')),
+    ('a:(b c d)',                 ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
+    ('a:(b (c d))',               ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
+    ('a:(b OR (c d))',            ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
+    ('"JeanANDPaul"',             Query(None, 'JeanANDPaul')),
+    ('"Jean" AND "Paul"',         ComplexQuery([Query(None, 'Jean'), Query(None, 'Paul')], operator='and')),
+    ('"jean paul" OR "thierry"',  ComplexQuery([Query(None, 'jean paul'), Query(None, 'thierry')], operator='or')),
+    ('title:Paul Jean Lili',      ComplexQuery([Query('title', 'Paul'), Query(None, 'Jean'), Query(None, 'Lili')], operator='or')),
+    ('toto AND titi OR tutu AND tata OR toto',
+                                  ComplexQuery([ComplexQuery([Query(None, 'toto'), Query(None, 'titi')], operator='and'), ComplexQuery([Query(None, 'tutu'), Query(None, 'tata')], operator='and'), Query(None, 'toto')], operator='or')),
+    ('toto AND (titi OR tutu) AND tata OR toto',
+                                  ComplexQuery([ComplexQuery([Query(None, 'toto'), ComplexQuery([Query(None, 'titi'), Query(None, 'tutu')], operator='or'), Query(None, 'tata')], operator='and'), Query(None, 'toto')], operator='or')),
+    ('"OR ARGENT"',               Query(None, 'OR ARGENT')),
+    ('1 AND 2 OR 3',              ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3')], operator='or')),
+    ('1 OR 2 AND 3',              ComplexQuery([Query(None, '1'), ComplexQuery([Query(None, '2'), Query(None, '3')], operator='and')], operator='or')),
+    ('1 AND 2 3',                 ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3')], operator='or')),
+    ('1 2 AND 3',                 ComplexQuery([Query(None, '1'), ComplexQuery([Query(None, '2'), Query(None, '3')], operator='and')], operator='or')),
+    ('10 11 OR 12 13',            ComplexQuery([Query(None, '10'), Query(None, '11'), Query(None, '12'), Query(None, '13')], operator='or')),
+    ('((1 AND 2 OR 3) OR (4 AND 5 6) OR (7 8 AND 9) OR (10 11 OR 12 13))',
+                                  ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3'), ComplexQuery([Query(None, '4'), Query(None, '5')], operator='and'), Query(None, '6'), Query(None, '7'), ComplexQuery([Query(None, '8'), Query(None, '9')], operator='and'), Query(None, '10'), Query(None, '11'), Query(None, '12'), Query(None, '13')], operator='or')),
+    ('((titi:foo) AND (toto:bar)) OR ((titi:bar) AND (toto:foo))',
+                                  ComplexQuery([ComplexQuery([Query('titi', 'foo'), Query('toto', 'bar')], operator='and'), ComplexQuery([Query('titi', 'bar'), Query('toto', 'foo')], operator='and')], operator='or')),
+    ('title:(Paul Jean OR Lili)', ComplexQuery([Query('title', 'Paul'), Query('title', 'Jean'), Query('title', 'Lili')], operator='or')),
+    ('title:Paul Jean OR Lili',   ComplexQuery([Query('title', 'Paul'), Query(None, 'Jean'), Query(None, 'Lili')], operator='or')),
+  ]
+
+  def walk(node, key=None):
+    """
+      Recusrively walk given AST and build ComplexQuery & Query instances for each node.
+    """
+    if node.isLeaf():
+      comparison_operator = node.getComparisonOperator()
+      if comparison_operator == '':
+        comparison_operator = '='
+      result = Query(key, node.getValue(), comparison_operator=comparison_operator)
+    elif node.isColumn():
+      result = walk(node.getSubNode(), node.getColumnName())
+    else:
+      query_list = [walk(x, key) for x in node.getNodeList()]
+      operator = node.getLogicalOperator()
+      if operator == 'not' or len(query_list) > 1:
+        result = ComplexQuery(query_list, operator=operator)
+      elif len(query_list) == 1:
+        result = query_list[0]
+      else:
+        result = None
+    return result
+
+  original_parse = _parse
+
+  def parse(input, *args, **kw):
+    """
+      Parse input and walk generated AST.
+    """
+    result = original_parse(input, *args, **kw)
+    if result is not None:
+      #print repr(result)
+      result = walk(result)
+    return result
+
+  success_count = 0
+  for input, expected in check_list:
+    try:
+      result = parse(input)
+    except ParserOrLexerError, message:
+      print "ERROR when checking %r" % (input, )
+      print " crashed with: %s" % (message, )
+      print " instead of producing %r" % (expected, )
+    else:
+      if result != expected:
+        print "ERROR when checking %r:" % (input, )
+        print " produced   %r" % (result, )
+        print " instead of %r" % (expected, )
+      else:
+        success_count += 1
+  print '%i/%i checks succeeded.' % (success_count, len(check_list))
+  while 1:
+    try:
+      input = raw_input('catalog> ')
+    except (EOFError, KeyboardInterrupt):
+      break
+    print repr(input)
+    try:
+      try:
+        detector_result = getAdvancedSearchTextDetector()(input)
+      except ParserOrLexerError, message:
+        print '  Detector raise: %r' % (message, )
+        detector_result = False
+      else:
+        print '  Detector: %r' % (detector_result, )
+      if detector_result:
+        print '  LEX:'
+        lexer = getAdvancedSearchTextParser().lexer
+        lexer.input(input)
+        while 1:
+          tok = lexer.token()
+          if not tok: break      # No more input
+          print '    %s' % (tok, )
+        print '  YACC:'
+        print '    %r' % (parse(input, debug=2), )
+      else:
+        print '    %r' % (input, )
+    except ParserOrLexerError, message:
+      print message
+  print
+

product/ZSQLCatalog/SearchText/__init__.py

+from SearchTextParser import parse
+

product/ZSQLCatalog/SearchText/lexer.py

+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from ply import lex, yacc
+import os
+import sys
+from cStringIO import StringIO
+
+try:
+  from zLOG import LOG
+except ImportError:
+  def LOG(channel, level, message):
+    print >>sys.stderr, message
+
+module_path = os.path.dirname(os.path.abspath(__file__))
+
+class ParserOrLexerError(Exception):
+  pass
+
+class LexerError(ParserOrLexerError):
+  pass
+
+class ParserError(ParserOrLexerError):
+  pass
+
+class lexer(object):
+  def init(self, **kw):
+    debug = kw.pop('debug', False)
+    # Catch all logs with a cStringIO
+    output = sys.stdout = sys.stderr = StringIO()
+    self.lexer = lex.lex(object=self, **kw)
+    self.parser = yacc.yacc(module=self, debug=debug,
+                            debugfile="%s.out" % (self.__class__.__name__, ),
+                            tabmodule="%s_parsetab" % (self.__class__.__name__, ),
+                            outputdir=module_path)
+    sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__
+    # Emit all logs with regular Zope logging
+    for line in output.getvalue().split('\n'):
+      if len(line):
+        LOG('lexer', 0, line)
+
+  def t_error(self, t):
+    raise LexerError, 'ERROR: Illegal character %r' % (t.value[0], )
+
+  def p_error(self, p):
+    raise ParserError, 'Syntax error in input: %r' % (p, )
+
+  def input(self, string):
+    self.lexer.input(string)
+
+  def token(self):
+    return self.lexer.token()
+
+  tokens = (
+    'OR',
+    'AND',
+    'NOT',
+    'COLUMN',
+    'STRING',
+    'WORD',
+    'OPERATOR',
+    'LEFT_PARENTHESE',
+    'RIGHT_PARENTHESE')
+
+  t_ignore = ' '
+
+  def t_LEFT_PARENTHESE(self, t):
+    r'\('
+    return t
+
+  def t_RIGHT_PARENTHESE(self, t):
+    r'\)'
+    return t
+
+  def t_OPERATOR(self, t):
+    r'(>=?|<=?|!?=)'
+    return t
+
+  def t_STRING(self, t):
+    r'"(\\.|[^\\"])*"'
+    # Unescape value and strip surrounding quotes
+    value_list = []
+    append = value_list.append
+    escaped = False
+    for char in t.value[1:-1]:
+      if escaped:
+        escaped = False
+        if char != '"':
+          append('\\')
+      else:
+        if char == '\\':
+          escaped = True
+          continue
+      append(char)
+    assert not escaped
+    t.value = ''.join(value_list)
+    return t
+
+  def t_COLUMN(self, t):
+    r'[^><= :\(\)"][^ :\(\)"]*:'
+    t.value = t.value[:-1]
+    return t
+
+  def t_OR(self, t):
+    r'OR'
+    return t
+
+  def t_AND(self, t):
+    r'AND'
+    return t
+
+  def t_NOT(self, t):
+    r'NOT'
+    return t
+
+  def t_WORD(self, t):
+    r'[^><= :\(\)"][^ :\(\)"]*'
+    return t
+
+  def parse(self, *args, **kw):
+    kw['lexer'] = self
+    return self.parser.parse(*args, **kw)
+
+  __call__ = parse
+
+def update_docstrings(klass):
+  for property in dir(klass):
+    if property.startswith('t_'):
+      source = getattr(lexer, property, None)
+      if callable(source):
+        destination = getattr(klass, property)
+        assert callable(destination)
+        if destination.__doc__ is None:
+          destination.im_func.__doc__ = source.__doc__
+

product/ZSQLCatalog/ZSQLCatalog.py

@@ -790,8 +790,9 @@ class ZCatalog(Folder, Persistent, Implicit):
         except ConflictError:
           raise
         except:
+          raise
           LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
-          failed_object_list.append(obj)
+          failed_object_list.append(obj) # XXX Strange JPS - why LOG and keep on ?? wrap_obj not defined
 
     # run activity or execute for each archive depending on priority
     if len(catalog_dict):
@@ -1043,7 +1044,7 @@ class ZCatalog(Folder, Persistent, Implicit):
   security.declarePublic('buildSqlQuery')
   buildSqlQuery = buildSQLQuery
 
-  def searchResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
+  def searchResults(self, REQUEST=None, sql_catalog_id=None, **kw):
     """
     Search the catalog according to the ZTables search interface.
     Search terms can be passed in the REQUEST or as keyword
@@ -1051,18 +1052,18 @@ class ZCatalog(Folder, Persistent, Implicit):
     """
     catalog = self.getSQLCatalog(sql_catalog_id)
     if catalog is not None:
-      return apply(catalog.searchResults, (REQUEST,used), kw)
+      return apply(catalog.searchResults, (REQUEST, ), kw)
     return []
 
   __call__=searchResults
 
-  def countResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
+  def countResults(self, REQUEST=None, sql_catalog_id=None, **kw):
     """
     Counts the number of items which satisfy the query defined in kw.
     """
     catalog = self.getSQLCatalog(sql_catalog_id)
     if catalog is not None:
-      return apply(catalog.countResults, (REQUEST,used), kw)
+      return apply(catalog.countResults, (REQUEST, ), kw)
     return []
 
 ## this stuff is so the find machinery works

product/ZSQLCatalog/__init__.py

@@ -44,3 +44,6 @@ def initialize(context):
 from AccessControl import ModuleSecurityInfo, ClassSecurityInfo
 ModuleSecurityInfo('Products.ZSQLCatalog.SQLCatalog').declarePublic(
             'ComplexQuery', 'Query', 'NegatedQuery',)
+
+from Query import Query, SimpleQuery
+from SearchKey import SearchKey
\ No newline at end of file

product/ZSQLCatalog/tests/testSQLCatalog.py

+##############################################################################
+#
+# Copyright (c) 2006-2009 Nexedi SA and Contributors. All Rights Reserved.
+#          Jerome Perrin <jerome@nexedi.com>
+#          Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import sys
+
+from DateTime import DateTime
+from Products.ZSQLMethods.SQL import SQL as ZSQLMethod
+from Products.CMFCore.Expression import Expression
+
+from Products.ZSQLCatalog.SQLCatalog import Catalog as SQLCatalog
+from Products.ZSQLCatalog.ZSQLCatalog import ZCatalog as ZSQLCatalog
+from Products.ZSQLCatalog.SQLCatalog import Query
+from Products.ZSQLCatalog.SQLCatalog import ComplexQuery
+from Products.ZSQLCatalog.SQLCatalog import SimpleQuery
+from Products.ZSQLCatalog.SQLCatalog import NegatedQuery
+from Products.ZSQLCatalog.Query.Query import Query as _Query
+from Products.ZSQLCatalog.Query.EntireQuery import EntireQuery
+from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
+from DateTime import DateTime
+
+class ReferenceQuery:
+  operator = None
+  column = None
+  value = None
+
+  def __init__(self, *args, **kw):
+    self.operator = kw.pop('operator', None)
+    assert len(args) == 0 or len(kw) == 0
+    self.args = []
+    for arg in args:
+      if isinstance(arg, (tuple, list)):
+        self.args.extend(arg)
+      else:
+        self.args.append(arg)
+    if len(kw) == 1:
+      self.column, self.value = kw.items()[0]
+    elif len(kw) > 1:
+      raise ValueError, 'kw must not have more than one item: %r' % (kw, )
+
+  def __eq__(self, other):
+    if isinstance(other, SimpleQuery):
+      return self.column is not None and \
+             other.getColumn() == self.column and \
+             other.getValue() == self.value and \
+             other.operator == self.operator
+    elif isinstance(other, ComplexQuery):
+      if not (len(other.query_list) == len(self.args) and \
+              other.logical_operator == self.operator):
+        return False
+      other_query_list = other.query_list[:]
+      for subquery in self.args:
+        for other_query_id in xrange(len(other_query_list)):
+          other_query = other_query_list[other_query_id]
+          if subquery == other_query:
+            other_query_list.pop(other_query_id)
+            break
+        else:
+          return False
+      return len(other_query_list) == 0
+    elif isinstance(other, EntireQuery):
+      return len(self.args) == 1 and \
+             self.args[0] == other.query
+    elif isinstance(other, RelatedQuery):
+      return self == other.join_condition
+    elif isinstance(other, Query):
+      return self == other.wrapped_query
+    else:
+      raise TypeError, 'Compared value is not a (known) Query instance: (%s) %r' % (other.__class__.__name__, other)
+
+  def __repr__(self):
+    return '<%s column=%r operator=%r value=%r args=%r>' % \
+      (self.__class__.__name__, self.column, self.operator, self.value, self.args)
+
+class RelatedReferenceQuery:
+  def __init__(self, reference_subquery):
+    self.subquery = reference_subquery
+
+  def __eq__(self, other):
+    return isinstance(other, RelatedQuery) and \
+           self.subquery == other.join_condition
+
+  def __repr__(self):
+    return '<%s %r>' % (self.__class__.__name__, self.subquery)
+
+class DummyCatalog(SQLCatalog):
+  """
+    Mimic a table stucture.
+  """
+
+  sql_catalog_keyword_search_keys = ('keyword', )
+  sql_catalog_datetime_search_keys = ('date', )
+
+  def getColumnMap(self):
+    return {
+      'uid': ['foo', 'bar'],
+      'default': ['foo', ],
+      'keyword': ['foo', ],
+      'date': ['foo', ],
+      'other_uid': ['bar', ]
+    }
+
+  def getSQLCatalogRelatedKeyList(self, key_list):
+    return [
+      'related_default | bar,foo/default/z_related_table',
+      'related_keyword | bar,foo/keyword/z_related_table',
+      'related_date | bar,foo/date/z_related_table'
+    ]
+
+  def z_related_table(self, *args, **kw):
+    assert kw.get('src__', False)
+    assert 'query_table' in kw
+    assert 'table_0' in kw
+    assert 'table_1' in kw
+    assert len(kw) == 4
+    return '%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s' % kw
+
+class TestSQLCatalog(unittest.TestCase):
+  def setUp(self):
+    self._catalog = DummyCatalog('dummy_catalog')
+
+  def assertCatalogRaises(self, exception, kw):
+    self.assertRaises(exception, self._catalog, src__=1, query_table='foo', **kw)
+
+  def catalog(self, reference_tree, kw, check_search_text=True):
+    reference_param_dict = self._catalog._queryResults(query_table='foo', **kw)
+    query = self._catalog.buildQuery(kw)
+    self.assertEqual(reference_tree, query)
+    search_text = query.asSearchTextExpression(self._catalog)
+    if check_search_text:
+      # XXX: sould "keyword" be always used for search text searches ?
+      search_text_param_dict = self._catalog._queryResults(query_table='foo', keyword=search_text)
+      self.assertEqual(reference_param_dict, search_text_param_dict,
+          'Query: %r\nSearchText: %r\nReference: %r\nSecond rendering: %r' % \
+                       (query, search_text, reference_param_dict, search_text_param_dict))
+
+  def test_001_DefaultKey(self):
+    for column in ('default', 'related_default'):
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='a'), operator='and'),
+                   {column: 'a'})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='like', default='%a'), operator='and'),
+                   {column: '%a'})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='<', default='a'), operator='and'),
+                   {column: '<a'})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='<', default='path'), operator='and'),
+                   {column: {'query': 'path', 'range': 'max'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='in', default=['a', 'b']), operator='and'),
+                   {column: ['a', 'b']})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='in', default=['a', 'b']), operator='and'),
+                   {column: ['=a', '=b']})
+
+  def test_002_keyOverride(self):
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='%a'), operator='and'),
+                 {'default': {'query': '%a', 'key': 'ExactMatch'}},
+                 check_search_text=False)
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='<a'), operator='and'),
+                 {'default': {'query': '<a', 'key': 'ExactMatch'}},
+                 check_search_text=False)
+
+  def test_003_DateTimeKey(self):
+    for column in ('date', 'related_date'):
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21')), operator='and'),
+                   {column: {'query': '>2008/10/01 12:10:20', 'format': '%y/%m/%d'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21 CEST')), operator='and'),
+                   {column: {'query': '>2008/10/01 12:10:20 CEST', 'format': '%y/%m/%d'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21 CET')), operator='and'),
+                   {column: {'query': '>2008/10/01 12:10:20 CET', 'format': '%y/%m/%d'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                     ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
+                     ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
+                   , operator='and'), operator='and'),
+                   {column: '2008/10/01 UTC'})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                     ReferenceQuery(operator='>=', date=DateTime('2008/01/01 UTC')),
+                     ReferenceQuery(operator='<', date=DateTime('2009/01/01 UTC'))
+                   , operator='and'), operator='and'),
+                   {column: '2008 UTC'})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                     ReferenceQuery(operator='>=', date=DateTime('2008/01/01 UTC')),
+                     ReferenceQuery(operator='<', date=DateTime('2008/02/01 UTC'))
+                   , operator='and'), operator='and'),
+                   {column: '2008/01 UTC'})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                     ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
+                     ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
+                   , operator='and'), operator='and'),
+                   {column: {'type': 'date', 'query': '10/01/2008 UTC', 'format': '%m/%d/%Y'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                     ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
+                     ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
+                   , operator='and'), operator='and'),
+                   {column: {'type': 'date', 'query': '01/10/2008 UTC', 'format': '%d/%m/%Y'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='in', date=[DateTime('2008/01/10 UTC'), DateTime('2008/01/09 UTC')]), operator='and'),
+                   {column: {'query': ['2008/01/10 UTC', '2008/01/09 UTC'], 'operator': 'in'}},
+                   check_search_text=False)
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='>', date=DateTime('2008/01/10 UTC')), operator='and'),
+                   {column: {'query': '2008/01/10 UTC', 'range': 'nlt'}},
+                   check_search_text=False)
+
+  def test_004_KeywordKey(self):
+    for column in ('keyword', 'related_keyword'):
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a%'), operator='and'),
+                   {column: 'a'})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a'), operator='and'),
+                   {column: '%a'})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='a'), operator='and'),
+                   {column: '<a'})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='path'), operator='and'),
+                   {column: {'query': 'path', 'range': 'max'}})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                       ReferenceQuery(operator='like', keyword='%a%'),
+                       ReferenceQuery(operator='like', keyword='%b%')
+                     , operator='or'), operator='and'),
+                   {column: ['a', 'b']})
+      self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
+                   {column: ['=a', '=b']})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                       ReferenceQuery(operator='like', keyword='%a%'),
+                       ReferenceQuery(operator='<', keyword='b')
+                     , operator='or'), operator='and'),
+                   {column: ['a', '<b']})
+      self.catalog(ReferenceQuery(ReferenceQuery(
+                       ReferenceQuery(operator='like', keyword='%a%'),
+                       ReferenceQuery(operator='like', keyword='%b')
+                     , operator='or'), operator='and'),
+                   {column: ['a', '%b']})
+
+  def test_005_SearchText(self):
+    self.catalog(ReferenceQuery(ReferenceQuery(ReferenceQuery(operator='like', keyword='%=a%'), ReferenceQuery(operator='like', keyword='%=b%'), operator='or'), operator='and'),
+                 {'keyword': '"=a" OR "=b"'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
+                 {'keyword': '="a" OR ="b"'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
+                 {'keyword': '=a OR =b'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b', 'c']), operator='and'),
+                 {'keyword': '=a OR =b OR =c'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a%'), operator='and'),
+                 {'keyword': 'keyword:a'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='a'), operator='and'),
+                 {'keyword': 'default:a'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a b%'), operator='and'),
+                 {'keyword': 'a b'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%=a OR =b%'), operator='and'),
+                 {'keyword': '"=a OR =b"'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='=', keyword='=a OR =b'), operator='and'),
+                 {'keyword': '="=a OR =b"'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='=a OR =b'), operator='and'),
+                 {'keyword': '<"=a OR =b"'})
+    self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%"a" OR "b"%'), operator='and'),
+                 {'keyword': '"\\"a\\" OR \\"b\\""'})
+
+  def test_006_testRelatedKey_with_multiple_join(self):
+    # The name of catalog parameter does not matter at all
+    # ComplexQuery(ComplexQuery(AutoQuery(RelatedQuery(SimpleQuery())), AutoQuery(RelatedQuery(SimpleQuery()))))
+    # 'AutoQuery' doesn't need any ReferenceQuery equivalent.
+    self.catalog(ReferenceQuery(ReferenceQuery(
+                     ReferenceQuery(RelatedReferenceQuery(ReferenceQuery(operator='=', default='a')), operator='and'),
+                     ReferenceQuery(RelatedReferenceQuery(ReferenceQuery(operator='=', default='b')), operator='and')
+                   , operator='and'), operator='and'),
+                 {'query': ComplexQuery(Query(related_default='a'), Query(related_default='b'))})
+
+##return catalog(title=Query(title='a', operator='not'))
+#return catalog(title={'query': 'a', 'operator': 'not'})
+#return catalog(title={'query': ['a', 'b'], 'operator': 'not'})
+#return context.portal_catalog(source_title="toto", source_description="tutu", src__=1)
+#print catalog(query=ComplexQuery(Query(title='1'), ComplexQuery(Query(portal_type='Foo') ,Query(portal_type='Bar'), operator='or'), operator='and'))
+#print catalog(title={'query': ('path', 2), 'operator': 'and'}, exception=TypeError)
+#print catalog(sort_on=[('source_title', )], check_search_text=False)
+#print catalog(query=ComplexQuery(Query(source_title='foo'), Query(source_title='bar')), sort_on=[('source_title', ), ('source_title_1', )], check_search_text=False)
+
+def test_suite():
+  suite = unittest.TestSuite()
+  suite.addTest(unittest.makeSuite(TestSQLCatalog))
+  return suite
+

product/ZSQLCatalog/tests/testSearchKeys.py

@@ -28,6 +28,7 @@
 
 import unittest
 from DateTime import DateTime
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
 from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
 from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
@@ -37,7 +38,7 @@ from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
 from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
 from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
 
-class TestSearchKeyLexer(unittest.TestCase):
+class TestSearchKeyLexer(ERP5TypeTestCase):
   """Test search keys
   """
   run_all_test = 1
@@ -48,7 +49,8 @@ class TestSearchKeyLexer(unittest.TestCase):
     key = getSearchKeyInstance(search_key_class)
     tokens = key.tokenize(search_value)  
     token_types = [x.type for x in tokens]
-    self.assertEqual(token_types, list(expected_token_types))
+    self.assertSameSet(token_types, expected_token_types)  
+
   
   def test_01ProperPoolInitialization(self, quiet=quiet, run=run_all_test):
     """ Check that search key pool is properly initialized """
@@ -82,7 +84,6 @@ size/Child/34"""
     self.compare(DefaultKey, 
                 'S\xc3\xa9bastien or !="Doe John1" and Doe', 
                 ('WORD', 'OR', 'NOT', 'WORDSET', 'AND', 'WORD',))  
-    self.compare(DefaultKey, '.', ('WORD',))
                              
   def test_03KeyWordKey(self, quiet=quiet, run=run_all_test):
     """ Check lexer for KeyWordKey."""
@@ -102,9 +103,8 @@ size/Child/34"""
                             ('LESSTHANEQUAL', 'KEYWORD', 'AND', 
                              'GREATERTHANEQUAL', 'WORDSET',))                             
     self.compare(KeyWordKey, '=John% and >="JOHN John"', 
-                            ('EXPLICITEQUALLITYWORD', 'AND',
+                            ('EXPLICITEQUALLITYWORD', 'KEYWORD', 'AND', 
                              'GREATERTHANEQUAL', 'WORDSET',))
-    self.compare(KeyWordKey, '.', ('WORD',))
                              
   def test_04DateTimeKey(self, quiet=quiet, run=run_all_test):
     """ Check lexer for DateTimeKey."""
@@ -121,17 +121,10 @@ size/Child/34"""
   def test_05FullTextKey(self, quiet=quiet, run=run_all_test):
     """ Check lexer for FullTextKey."""  
     if not run: return
-    self.compare(FullTextKey, 'John', ('WORD',))
-    self.compare(FullTextKey, 'John Doe', ('WORD', 'WORD',))
+    self.compare(FullTextKey, 'John Doe', 
+                 ('WORD', 'WORD',)) 
     self.compare(FullTextKey, '+John -Doe', 
                  ('PLUS', 'WORD', 'MINUS', 'WORD',)) 
-    self.compare(FullTextKey, 'John*', ('WORD', 'ASTERISK'))
-    self.compare(FullTextKey, '+John*', ('PLUS', 'WORD', 'ASTERISK'))
-    self.compare(FullTextKey, '.', ('WORD',))
-    self.compare(FullTextKey, '"John Doe"', ('DOUBLEQUOTE', 'WORD', 'WORD', 'DOUBLEQUOTE'))
-    self.compare(FullTextKey, '+apple +(>turnover <strudel)',
-                 ('PLUS', 'WORD', 'PLUS', 'LEFTPARENTHES', 'GREATERTHAN', 'WORD',
-                  'LESSTHAN', 'WORD', 'RIGHTPARENTHES',))
 
   def test_06ScriptableKey(self, quiet=quiet, run=run_all_test):
     """ Check lexer for ScriptableKey."""  
@@ -143,7 +136,7 @@ size/Child/34"""
                 'John Doe OR creation_date>=2005/12/12',
                 ('WORD', 'WORD', 'OR', 'KEYMAPPING',))                
                    
-class TestSearchKeyQuery(unittest.TestCase):
+class TestSearchKeyQuery(ERP5TypeTestCase):
   """Test search keys query generation
   """
   run_all_test = 1
@@ -206,12 +199,6 @@ class TestSearchKeyQuery(unittest.TestCase):
                 '%John and !=Doe%', 
                 "((((title = '%John') AND (title != 'Doe%'))))",
                 [])
-    # special chars
-    self.compare(DefaultKey,
-                 'title',
-                 '.',
-                 "((((title = '.'))))",
-                 [])
                 
   def test_02KeyWordKey(self, quiet=quiet, run=run_all_test):
     """ Check DefaultKey query generation"""
@@ -241,12 +228,6 @@ class TestSearchKeyQuery(unittest.TestCase):
                 '%John Doe% or =Doe John', 
                 "((((title LIKE '%John Doe%'))) OR (((title = 'Doe John'))))",
                 [])       
-    # special chars
-    self.compare(KeyWordKey,
-                 'title',
-                 '.',
-                 "((((title LIKE '%.%'))))",
-                 [])
                 
   def test_03DateTimeKey(self, quiet=quiet, run=run_all_test):
     """ Check DefaultKey query generation"""
@@ -368,18 +349,7 @@ class TestSearchKeyQuery(unittest.TestCase):
                 '1ab521ty', 
                 "delivery.stock = '1ab521ty'",
                 [])    
-    # special chars
-    self.compare(RawKey,
-                 'delivery.stock',
-                 '.',
-                 "delivery.stock = '.'",
-                 [])
-    #None values
-    self.compare(RawKey,
-                 'title',
-                 None,
-                 "title is NULL",
-                 [])
+                
   def test_06FullTextKey(self, quiet=quiet, run=run_all_test):
     """ Check FullTextKey query generation"""
     if not run: return
@@ -390,28 +360,6 @@ class TestSearchKeyQuery(unittest.TestCase):
                 ["MATCH full_text.SearchableText AGAINST ('john' ) AS full_text_SearchableText_relevance", 
                 "MATCH full_text.SearchableText AGAINST ('john' ) AS SearchableText_relevance"])    
     
-    # special chars
-    self.compare(FullTextKey,
-                'full_text.SearchableText',
-                '.',
-                "MATCH full_text.SearchableText AGAINST ('.' )",
-                ["MATCH full_text.SearchableText AGAINST ('.' ) AS full_text_SearchableText_relevance",
-                "MATCH full_text.SearchableText AGAINST ('.' ) AS SearchableText_relevance"])
-    #Boolean Mode
-    self.compare(FullTextKey,
-                'full_text.SearchableText',
-                'john stuart mill',
-                "MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE)",
-                ["MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE) AS full_text_SearchableText_relevance",
-                "MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE) AS SearchableText_relevance"])
-
-    self.compare(FullTextKey,
-                'full_text.SearchableText',
-                'John*',
-                "MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE)",
-                ["MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE) AS full_text_SearchableText_relevance",
-                "MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE) AS SearchableText_relevance"])
-    
   def test_07ScriptableKey(self, quiet=quiet, run=run_all_test):
     """ Check ScriptableKey query generation"""
     if not run: return    

product/ZSQLCatalog/tests/testZSQLCatalog.py

@@ -95,369 +95,9 @@ class TestSQLCatalog(unittest.TestCase):
     self.assertFalse(
         self._catalog.isPortalTypeSelected('not_exists', 'Selected'))
 
-
-class TestQuery(unittest.TestCase):
-  """Test SQL bits generated from Queries
-  """
-  def testSimpleQuery(self):
-    q = Query(title='Foo')
-    self.assertEquals(
-          dict(where_expression="((((title = 'Foo'))))",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testQueryMultipleKeys(self):
-    # using multiple keys is invalid and raises
-    # KeyError: 'Query must have only one key'
-    self.assertRaises(KeyError, Query, title='Foo', reference='bar')
-
-  def testNoneQuery(self):
-    q = Query(title=None)
-    self.assertEquals(
-          dict(where_expression="title is NULL",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testEmptyQueryNotIgnoreEmptyString(self):
-    q = Query(title='')
-    # if you want to search with an empty string, pass ignore_empty_string=0 to
-    # asSQLExpression. XXX not to __init__ ?
-    self.assertEquals(
-          dict(where_expression="title = ''",
-               select_expression_list=[]),
-          q.asSQLExpression(ignore_empty_string=0,
-                            keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testEmptyQuery(self):
-    q = Query(title='')
-    # query are true by default
-    self.assertEquals(
-          dict(where_expression="1",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-    
-  def testMultiValuedQuery(self):
-    q = Query(title=['Foo', 'Bar'])
-    self.assertEquals(
-          dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testINQuery(self):
-    q = Query(title=['Foo', 'Bar'], operator='IN')
-    self.assertEquals(
-          dict(where_expression="title IN ('Foo', 'Bar')",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testEmptyINQuery(self):
-    q = Query(title=[], operator='IN')
-    self.assertEquals(
-          dict(where_expression="0",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testMinQuery(self):
-    q = Query(title='Foo', range='min')
-    self.assertEquals(
-          dict(where_expression="title >= 'Foo'",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-    
-  def testMaxQuery(self):
-    q = Query(title='Foo', range='max')
-    self.assertEquals(
-          dict(where_expression="title < 'Foo'",
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  # format
-  def testDateFormat(self):
-    date = DateTime(2001, 02, 03)
-    q = Query(date=date, format='%Y/%m/%d', type='date')
-    self.assertEquals(
-          dict(where_expression=
-            "((((date >= '%s' AND date < '%s'))))" \
-                 %(date.toZone('UTC').ISO(), (date + 1).toZone('UTC').ISO()),
-               select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[], 
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-  
-  # full text
-  def testSimpleQueryFullText(self):
-    q = Query(title='Foo')
-    self.assertEquals(dict(where_expression="MATCH title AGAINST ('Foo' )",
-                           select_expression_list=
-                        ["MATCH title AGAINST ('Foo' ) AS title_relevance"]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=['title']))
-
-  def testSimpleQueryFullTextSearchMode(self):
-    q = Query(title='Foo',
-              search_mode='in_boolean_mode')
-    self.assertEquals(dict(
-      where_expression="MATCH title AGAINST ('Foo' IN BOOLEAN MODE)",
-      select_expression_list=
-        ["MATCH title AGAINST ('Foo' IN BOOLEAN MODE) AS title_relevance"]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=['title']))
-  
-  def testSimpleQueryFullAutomaticTextSearchMode(self):
-    q = Query(title='Foo*',)
-    self.assertEquals(dict(
-      where_expression="MATCH title AGAINST ('Foo*' IN BOOLEAN MODE)",
-      select_expression_list=
-        ["MATCH title AGAINST ('Foo*' IN BOOLEAN MODE) AS title_relevance"]),
-          q.asSQLExpression(full_text_search_keys=['title']))
-
-  def testSimpleQueryFullTextStat__(self):
-    # stat__ is an internal implementation artifact to prevent adding
-    # select_expression for countFolder
-    q = Query(title='Foo')
-    self.assertEquals(dict(
-                    where_expression="MATCH title AGAINST ('Foo' )",
-                    select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=['title'],
-                            stat__=1))
-
-  def testSimpleQueryKeywordSearchKey(self):
-    q = Query(title='Foo')
-    self.assertEquals(dict(where_expression="((((title LIKE '%Foo%'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testQueryKeywordSearchKeyWithPercent(self):
-    q = Query(title='Fo%oo')
-    self.assertEquals(dict(where_expression="((((title LIKE 'Fo%oo'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],))
-
-  def testQueryKeywordSearchKeyWithPercentAndOnlyOneLetter(self):
-    q = Query(title='F%o')
-    self.assertEquals(dict(where_expression="((((title LIKE 'F%o'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title']))
-
-  def testQueryKeywordSearchKeyWithPercentOnly(self):
-    q = Query(title='%')
-    self.assertEquals(dict(where_expression="((((title LIKE '%'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],))
-
-  def testQueryKeywordSearchKeyWithMinus(self):
-    q = Query(title='F-o')
-    self.assertEquals(dict(where_expression="((((title LIKE '%F-o%'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testQueryKeywordSearchKeyWithSpace(self):
-    q = Query(title='F o')
-    self.assertEquals(dict(where_expression="((((title LIKE '%F o%'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testQueryKeywordSearchKeyWithPercentAtTheEnd(self):
-    q = Query(title='F%')
-    self.assertEquals(dict(where_expression="((((title LIKE 'F%'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],))
-    q = Query(title='Fo%')
-    self.assertEquals(dict(where_expression="((((title LIKE 'Fo%'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],))
-
-  def testQueryKeywordSearchKeyWithPercentAtTheBeginning(self):
-    q = Query(title='%o')
-    self.assertEquals(dict(where_expression="((((title LIKE '%o'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],))
-    q = Query(title='%oo')
-    self.assertEquals(dict(where_expression="((((title LIKE '%oo'))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=['title'],))
-
-  def testNegatedQuery(self):
-    q1 = Query(title='Foo')
-    q = NegatedQuery(q1)
-    self.assertEquals(
-        dict(where_expression="(NOT (((((title = 'Foo'))))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  # complex queries
-  def testSimpleComplexQuery(self):
-    q1 = Query(title='Foo')
-    q2 = Query(reference='Bar')
-    q = ComplexQuery(q1, q2)
-    self.assertEquals(
-        dict(where_expression="((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))",
-                           select_expression_list=[]),
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[]))
-
-  def testNegatedComplexQuery(self):
-    q1 = Query(title='Foo')
-    q2 = Query(reference='Bar')
-    q3 = ComplexQuery(q1, q2)
-    q = NegatedQuery(q3)
-    self.assertEquals(
-      # maybe too many parents here
-     dict(where_expression="(NOT (((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))))",
-          select_expression_list=[]),
-     q.asSQLExpression(keyword_search_keys=[],
-                       datetime_search_keys = [],
-                       full_text_search_keys=[]))
-
-  
-  # forced keys
-  def testSimpleQueryForcedKeywordSearchKey(self):
-    q = Query(title='Foo', key='Keyword')
-    self.assertEquals("((((title LIKE '%Foo%'))))",
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[])['where_expression'])
-
-  def testSimpleQueryForcedFullText(self):
-    q = Query(title='Foo', key='FullText')
-    self.assertEquals("MATCH title AGAINST ('Foo' )",
-          q.asSQLExpression(keyword_search_keys=[],
-                            datetime_search_keys = [],                            
-                            full_text_search_keys=[])['where_expression'])
-
-  def testSimpleQueryForcedExactMatch(self):
-    q = Query(title='Foo', key='ExactMatch')
-    self.assertEquals("title = 'Foo'",
-          q.asSQLExpression(keyword_search_keys=['title'],
-                            datetime_search_keys = [],  
-                            full_text_search_keys=[])['where_expression'])
-
-  def testSimpleQueryForcedExactMatchOR(self):
-    q = Query(title='Foo% OR %?ar', key='ExactMatch')
-    self.assertEquals("title = 'Foo% OR %?ar'",
-          q.asSQLExpression(keyword_search_keys=['title'],
-                            datetime_search_keys = [],
-                            full_text_search_keys=[])['where_expression'])
-
-  def testQuotedStringDefaultKey(self):
-    q = Query(title='Foo d\'Ba')
-    self.assertEquals(
-              dict(where_expression="((((title = 'Foo d''Ba'))))",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testQuotedStringKeywordKey(self):
-    q = Query(title='Foo d\'Ba', key='Keyword')
-    self.assertEquals(
-              dict(where_expression="((((title LIKE '%Foo d''Ba%'))))",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testQuotedStringExactMatch(self):
-    q = Query(title='Foo d\'Ba', key='ExactMatch')
-    self.assertEquals(
-              dict(where_expression="title = 'Foo d''Ba'",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testQuotedStringFullTextKey(self):
-    q = Query(title='Foo d\'Ba', type='fulltext')
-    self.assertEquals(
-        dict(where_expression="MATCH title AGAINST ('+Foo +d''Ba' IN BOOLEAN MODE)",
-             select_expression_list=["MATCH title AGAINST ('+Foo +d''Ba' IN BOOLEAN MODE)"
-                                     " AS title_relevance"]),
-          q.asSQLExpression())
-
-  def testQuotedStringListKeywordKey(self):
-    q = Query(title=('Foo d\'Ba',), key='Keyword')
-    self.assertEquals(
-              dict(where_expression="((((title LIKE '%Foo d''Ba%'))))",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testQuotedStringListExactMatch(self):
-    q = Query(title=('Foo d\'Ba',), key='ExactMatch')
-    self.assertEquals(
-              dict(where_expression="title = 'Foo d''Ba'",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-
-  def testQuotedStringDateKey(self):
-    q = Query(title='Foo d\'Ba', type='date')
-    self.assertEquals(
-        # I don't know exactly what we should expect here.
-              dict(where_expression="1",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testQuotedStringFloatKey(self):
-    q = Query(title='Foo d\'Ba', type='float')
-    self.assertEquals(
-        # I don't know exactly what we should expect here.
-        # At least it's safe.
-              dict(where_expression="1",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testQuotedStringIntKey(self):
-    q = Query(title='Foo d\'Ba', type='int')
-    self.assertEquals(
-              dict(where_expression="((((title = 'Foo d''Ba'))))",
-                   select_expression_list=[]),
-                q.asSQLExpression())
-
-  def testListValuesInQuery(self):
-    q = Query(title=('Foo', 'Bar'))
-    self.assertEquals(
-      dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
-           select_expression_list=[]),
-        q.asSQLExpression())
-
-  def testListValuesInQueryWithKey(self):
-    q = Query(title=('Foo', 'Bar'), key='Keyword')
-    self.assertEquals(
-      dict(where_expression=
-             "(((((title LIKE '%Foo%')))) OR ((((title LIKE '%Bar%')))))",
-           select_expression_list=[]),
-        q.asSQLExpression())
-
-
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestSQLCatalog))
   suite.addTest(unittest.makeSuite(TestZSQLCatalog))
-  suite.addTest(unittest.makeSuite(TestQuery))
   return suite