Merge branch 'catalog_join'

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_related_grand_parent.xml

@@ -14,8 +14,10 @@
         </item>
         <item>
             <key> <string>arguments_src</string> </key>
-            <value> <string>table_0\n
-table_1</string> </value>
+            <value> <string>table_0\r\n
+table_1\r\n
+RELATED_QUERY_SEPARATOR=" AND "\r\n
+query_table="catalog"</string> </value>
         </item>
         <item>
             <key> <string>cache_time_</string> </key>
@@ -55,10 +57,9 @@ table_1</string> </value>
             <key> <string>src</string> </key>
             <value> <string encoding="cdata"><![CDATA[
 
-<dtml-var table_0>.uid = catalog.parent_uid\n
-AND <dtml-var table_1>.uid = <dtml-var table_0>.parent_uid\n
-\n
-
+<dtml-var table_1>.uid = <dtml-var table_0>.parent_uid\n
+<dtml-var RELATED_QUERY_SEPARATOR>\n
+<dtml-var table_0>.uid = <dtml-var query_table>.parent_uid
 
 ]]></string> </value>
         </item>

product/ERP5Type/Core/Predicate.py

@@ -329,7 +329,10 @@ class Predicate(XMLObject):
       catalog_kw['where_expression'] = SQLQuery(sql_text)
     else:
       catalog_kw['where_expression'] = ''
+    # force implicit join
+    catalog_kw['implicit_join'] = True
     sql_query = portal_catalog.buildSQLQuery(**catalog_kw)
+    # XXX from_table_list is None most of the time after the explicit_join work
     for alias, table in sql_query['from_table_list']:
       if from_table_dict.has_key(alias):
         raise KeyError, "The same table is used twice for an identity criterion and for a membership criterion"

product/ZSQLCatalog/Query/EntireQuery.py

@@ -28,6 +28,7 @@
 #
 ##############################################################################
 
+import warnings
 from Products.ZSQLCatalog.SQLExpression import SQLExpression
 from Products.ZSQLCatalog.ColumnMap import ColumnMap
 from zLOG import LOG
@@ -35,6 +36,7 @@ from Products.ZSQLCatalog.interfaces.entire_query import IEntireQuery
 from zope.interface.verify import verifyClass
 from zope.interface import implements
 from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+from Products.ZSQLCatalog.TableDefinition import LegacyTableDefinition
 
 def defaultDict(value):
   if value is None:
@@ -54,19 +56,28 @@ class EntireQuery(object):
   column_map = None
 
   @profiler_decorator
-  def __init__(self, query, order_by_list=(), group_by_list=(),
-               select_dict=None, limit=None, catalog_table_name=None,
-               extra_column_list=(), from_expression=None,
-               order_by_override_list=None):
+  def __init__(self, query,
+               order_by_list=(),
+               group_by_list=(),
+               select_dict=None,
+               left_join_list=(),
+               limit=None,
+               catalog_table_name=None,
+               extra_column_list=(),
+               from_expression=None,
+               order_by_override_list=None,
+               implicit_join=False):
     self.query = query
     self.order_by_list = list(order_by_list)
     self.order_by_override_set = frozenset(order_by_override_list)
     self.group_by_list = list(group_by_list)
     self.select_dict = defaultDict(select_dict)
+    self.left_join_list = left_join_list
     self.limit = limit
     self.catalog_table_name = catalog_table_name
     self.extra_column_list = list(extra_column_list)
     self.from_expression = from_expression
+    self.implicit_join = implicit_join
 
   def asSearchTextExpression(self, sql_catalog):
     return self.query.asSearchTextExpression(sql_catalog)
@@ -78,7 +89,12 @@ class EntireQuery(object):
       # XXX: should we provide a way to register column map as a separate 
       # method or do it here ?
       # Column Map was not built yet, do it.
-      self.column_map = column_map = ColumnMap(catalog_table_name=self.catalog_table_name)
+      column_map = ColumnMap(catalog_table_name=self.catalog_table_name,
+                             table_override_map=self.from_expression,
+                             left_join_list=self.left_join_list,
+                             implicit_join=self.implicit_join,
+                            )
+      self.column_map = column_map
       for extra_column in self.extra_column_list:
         table, column = extra_column.replace('`', '').split('.')
         if table != self.catalog_table_name:
@@ -145,30 +161,55 @@ class EntireQuery(object):
             None, ) * (3 - len(order_by)))
       self.order_by_list = new_order_by_list
       # generate SQLExpression from query
-      sql_expression_list = [self.query.asSQLExpression(sql_catalog, column_map, only_group_columns)]
-      # generate join expression based on column_map.getJoinTableAliasList
+      sql_expression_list = [self.query.asSQLExpression(sql_catalog,
+                                                        column_map,
+                                                        only_group_columns)]
       append = sql_expression_list.append
       for join_query in column_map.iterJoinQueryList():
-        append(join_query.asSQLExpression(sql_catalog, column_map, only_group_columns))
-      join_table_list = column_map.getJoinTableAliasList()
-      if len(join_table_list):
-        # XXX: Is there any special rule to observe when joining tables ?
-        # Maybe we could check which column is a primary key instead of
-        # hardcoding "uid".
-        where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
-          (column_map.getCatalogTableAlias(), )
-        # XXX: It would cleaner from completeness point of view to use column
-        # mapper to render column, but makes code much more complex to just do
-        # a simple text rendering. If there is any reason why we should have
-        # those column in the mapper, then we should use the clean way.
-        append(SQLExpression(self, where_expression=' AND '.join(
-          where_pattern % (x, ) for x in join_table_list
-        )))
+        append(join_query.asSQLExpression(sql_catalog,
+                                          column_map, 
+                                          only_group_columns))
+      # generate join expression based on column_map.getJoinTableAliasList
+      # XXX: This is now done by ColumnMap to its table_definition,
+      # during build()
+      #
+      # join_table_list = column_map.getJoinTableAliasList()
+      # if len(join_table_list):
+      #   # XXX: Is there any special rule to observe when joining tables ?
+      #   # Maybe we could check which column is a primary key instead of
+      #   # hardcoding "uid".
+      #   where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
+      #     (column_map.getCatalogTableAlias(), )
+      #   # XXX: It would cleaner from completeness point of view to use column
+      #   # mapper to render column, but makes code much more complex to just do
+      #   # a simple text rendering. If there is any reason why we should have
+      #   # those column in the mapper, then we should use the clean way.
+      #   append(SQLExpression(self, where_expression=' AND '.join(
+      #     where_pattern % (x, ) for x in join_table_list
+      #   )))
+
+      # BBB self.from_expression forces use of implicit inner join
+      table_alias_dict = column_map.getTableAliasDict()
+      if self.from_expression:
+        warnings.warn("Providing a 'from_expression' is deprecated.",
+                      DeprecationWarning)
+        # XXX: perhaps move this code to ColumnMap?
+        legacy_from_expression = self.from_expression
+        from_expression = LegacyTableDefinition(legacy_from_expression,
+                                                table_alias_dict)
+        table_alias_dict = None
+      else:
+        from_expression = column_map.getTableDefinition()
+        assert ((from_expression is None) !=
+                (table_alias_dict is None)), ("Got both a from_expression "
+                                              "and a table_alias_dict")
       self.sql_expression_list = sql_expression_list
+      # TODO: wrap the table_alias_dict above into a TableDefinition as well,
+      # even without a legacy_table_definition.
     return SQLExpression(
       self,
-      table_alias_dict=column_map.getTableAliasDict(),
-      from_expression=self.from_expression,
+      table_alias_dict=table_alias_dict,
+      from_expression=from_expression,
       order_by_list=self.order_by_list,
       group_by_list=self.group_by_list,
       select_dict=self.final_select_dict,

product/ZSQLCatalog/SQLCatalog.py

@@ -2291,6 +2291,13 @@ class Catalog(Folder,
         select_dict = None
     elif isinstance(select_dict, (list, tuple)):
       select_dict = dict([(x, None) for x in select_dict])
+    # Handle left_join_list
+    left_join_list = kw.pop('left_join_list', ())
+    # Handle implicit_join. It's True by default, as there's a lot of code
+    # in BT5s and elsewhere that calls buildSQLQuery() expecting implicit
+    # join. self._queryResults() defaults it to False for those using
+    # catalog.searchResults(...) or catalog(...) directly.
+    implicit_join = kw.pop('implicit_join', True)
     # Handle order_by_list
     order_by_list = kw.pop('order_by_list', None)
     sort_on = kw.pop('sort_on', None)
@@ -2328,6 +2335,8 @@ class Catalog(Folder,
       order_by_override_list=order_by_override_list,
       group_by_list=group_by_list,
       select_dict=select_dict,
+      left_join_list=left_join_list,
+      implicit_join=implicit_join,
       limit=limit,
       catalog_table_name=query_table,
       extra_column_list=extra_column_list,
@@ -2413,6 +2422,7 @@ class Catalog(Folder,
     """ Returns a list of brains from a set of constraints on variables """
     if build_sql_query_method is None:
       build_sql_query_method = self.buildSQLQuery
+    kw.setdefault('implicit_join', False)
     query = build_sql_query_method(REQUEST=REQUEST, **kw)
     # XXX: decide if this should be made normal
     ENFORCE_SEPARATION = True

product/ZSQLCatalog/SQLExpression.py

@@ -166,7 +166,7 @@ class SQLExpression(object):
   @profiler_decorator
   def getFromExpression(self):
     """
-      Returns a string.
+      Returns a TableDefinition stored in one of the from_expressions or None
 
       If there are nested SQLExpression, it checks that they either don't
       define any from_expression or the exact same from_expression. Otherwise,
@@ -175,7 +175,7 @@ class SQLExpression(object):
     result = self.from_expression
     for sql_expression in self.sql_expression_list:
       from_expression = sql_expression.getFromExpression()
-      if None not in (result, from_expression):
+      if from_expression not in (result, None):
         message = 'I don\'t know how to merge from_expressions'
         if DEBUG:
           message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
@@ -385,20 +385,25 @@ class SQLExpression(object):
       SQL_SELECT_ALIAS_FORMAT % (column, alias)
       for alias, column in self.getSelectDict().iteritems())
 
-  @profiler_decorator
-  def asSQLExpressionDict(self):
+  def getFromTableList(self):
     table_alias_dict = self.getTableAliasDict()
+    if not table_alias_dict:
+      return None
     from_table_list = []
     append = from_table_list.append
     for alias, table in table_alias_dict.iteritems():
       append((SQL_TABLE_FORMAT % (alias, ), SQL_TABLE_FORMAT % (table, )))
-    from_expression_dict = self.getFromExpression()
-    if from_expression_dict is not None:
-      from_expression = SQL_LIST_SEPARATOR.join(
-        from_expression_dict.get(alias, '`%s` AS `%s`' % (table, alias))
-        for alias, table in table_alias_dict.iteritems())
-    else:
-      from_expression = None
+    return from_table_list
+
+  @profiler_decorator
+  def asSQLExpressionDict(self):
+    from_expression = self.getFromExpression()
+    from_table_list = self.getFromTableList()
+    assert None in (from_expression,
+                    from_table_list), ("Cannot return both a from_expression "
+                                       "and a from_table_list")
+    if from_expression is not None:
+      from_expression = from_expression.render()
     return {
       'where_expression': self.getWhereExpression(),
       'order_by_expression': self.getOrderByExpression(),

product/ZSQLCatalog/SearchKey/RelatedKey.py

@@ -37,12 +37,29 @@ from Products.ZSQLCatalog.interfaces.search_key import IRelatedKey
 from zope.interface.verify import verifyClass
 from zope.interface import implements
 from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+from Products.ZSQLCatalog.TableDefinition import TableAlias, InnerJoin, LeftJoin
+
+from logging import getLogger
+log = getLogger(__name__)
 
 BACKWARD_COMPATIBILITY = True
 
+RELATED_QUERY_SEPARATOR = "\nAND -- related query separator\n"
+
+RELATED_KEY_MISMATCH_MESSAGE = "\
+A rendered related key must contain the same number of querying \
+conditions as the tables it relates, properly separated by \
+RELATED_QUERY_SEPARATOR. \n\
+Offending related key: %r, for column %r, table_alias_list: %r, \
+rendered_related_key: \n%s"
+
+RELATED_KEY_ALIASED_MESSAGE = "\
+Support for explicit joins of aliased related keys is not yet implemented. \
+Offending related key: %r, for column %r, table_alias_list: %r"
+
 class RelatedKey(SearchKey):
   """
-    This SearchKey handles searched on virtual columns of RelatedKey type.
+    This SearchKey handles searches on virtual columns of RelatedKey type.
     It generates joins required by the virtual column to reach the actual
     column to compare, plus a regular query on that column if needed.
   """
@@ -115,13 +132,23 @@ class RelatedKey(SearchKey):
   def registerColumnMap(self, column_map, table_alias_list=None):
     related_column = self.getColumn()
     group = column_map.registerRelatedKey(related_column, self.real_column)
-    # Each table except last one must be registered to their own group, so that
-    # the same table can be used multiple time (and aliased multiple times)
-    # in the same related key. The last one must be register to related key
-    # "main" group (ie, the value of the "group" variable) to be the same as
-    # the ta ble used in join_condition.
+    # Each table except last one must be registered to their own
+    # group, so that the same table can be used multiple times (and
+    # aliased multiple times) in the same related key. The last one
+    # must be registered to the related key "main" group (ie, the
+    # value of the "group" variable) to be the same as the table used
+    # in join_condition.
     if table_alias_list is not None:
       assert len(self.table_list) == len(table_alias_list)
+      # XXX-Leo: remove the rest of this 'if' branch after making sure
+      # that ColumnMap.addRelatedKeyJoin() can handle collapsing
+      # chains of inner-joins that are subsets of one another based on
+      # having the same aliases:
+      msg = RELATED_KEY_ALIASED_MESSAGE % (self.related_key_id,
+                                           self.column,
+                                           table_alias_list,)
+      log.warning(msg + "\n\nForcing implicit join...")
+      column_map.implicit_join = True
     for table_position in xrange(len(self.table_list) - 1):
       table_name = self.table_list[table_position]
       local_group = column_map.registerRelatedKeyColumn(related_column, table_position, group)
@@ -145,6 +172,23 @@ class RelatedKey(SearchKey):
     column_map.registerCatalog()
     return group
 
+  def stitchJoinDefinition(self, table_alias_list, join_query_list, column_map):
+    alias, table = table_alias_list[-1]
+    right = column_map.makeTableAliasDefinition(table, alias)
+    if not join_query_list:
+      # nothing to do, just return the table alias
+      assert len(table_alias_list) == 1
+      return right
+    else:
+      # create an InnerJoin of the last element of the alias list with
+      # a chain of InnerJoins of the rest of the list conditioned on
+      # the the last element of the join_query_list
+      left = self.stitchJoinDefinition(table_alias_list[:-1],
+                                       join_query_list[:-1],
+                                       column_map)
+      condition = join_query_list[-1]
+      return InnerJoin(left, right, condition)
+
   @profiler_decorator
   def buildSQLExpression(self, sql_catalog, column_map, only_group_columns, group):
     """
@@ -170,20 +214,26 @@ class RelatedKey(SearchKey):
     table_alias_list = [(getTableAlias(related_table, group=getRelatedKeyGroup(index, group)), related_table)
                         for (index, related_table) in enumerate(related_table_list)]
     # table alias for destination table
-    table_alias_list.append((getTableAlias(destination_table, group=group), destination_table))
+    table_alias_list.append((getTableAlias(destination_table, group=group),
+                             destination_table))
 
     # map aliases to use in ZSQLMethod.
-    table_alias_dict = dict(('table_%s' % (index, ), table_alias[0])
-                            for (index, table_alias) in enumerate(table_alias_list))
+    table_alias_dict = dict(('table_%s' % (index, ), table_alias)
+                            for (index, (table_alias, table_name))
+                            in enumerate(table_alias_list))
 
     assert len(table_alias_list) == len(table_alias_dict)
 
+    query_table=column_map.getCatalogTableAlias()
     rendered_related_key = related_key(
-      query_table=column_map.getCatalogTableAlias(),
+      query_table=query_table,
+      RELATED_QUERY_SEPARATOR=RELATED_QUERY_SEPARATOR,
       src__=1,
       **table_alias_dict)
+    join_condition_list = rendered_related_key.split(RELATED_QUERY_SEPARATOR)
+
     # Important:
-    # Former catalog separated join condition from related query.
+    # Previously the catalog separated join condition from the related query.
     # Example:
     #   ComplexQuery(Query(title="foo"),
     #                Query(subordination_title="bar")
@@ -199,19 +249,53 @@ class RelatedKey(SearchKey):
     # This was done on purpose, because doing otherwise gives very poor
     # performances (on a simple data set, similar query can take *minutes* to
     # execute - as of MySQL 5.x).
-    # Doing the same way as the former catalog is required for backward
-    # compatibility, until a decent alternative is found (like spliting the
-    # "OR" expression into ensemblist operations at query level).
-    # Note that doing this has a side effect on result list, as objects
-    # lacking a relation will never appear in the result.
+    #
+    # Because of this, we never return an SQLExpression here, as it
+    # would mix join definition with column condition in the body of
+    # the WHERE clause. Instead we explicitly define a Join to the
+    # catalog. The ColumnMap defines whether this is an Inner Join or
+    # a Left Outer Join. Notice that if an Inner Join is decided,
+    # objects lacking a relationship will never appear in the result.
+
+    if len(join_condition_list) == len(table_alias_list):
+      # Good! we got a compatible method that splits the join
+      # conditions according to the related tables.
+      #
+      # Add a join on this related key, based on the chain of
+      # inner-joins of the related key tables.
+      query_table_join_condition = join_condition_list.pop()
+      right_side = self.stitchJoinDefinition(table_alias_list,
+                                             join_condition_list,
+                                             column_map)
+      column_map.addRelatedKeyJoin(self.column,
+                                   right_side=right_side,
+                                   condition=query_table_join_condition)
+    else:
+      # Method did not render the related key condition with the
+      # appropriate separators so we could split it
+
+      # XXX: Can we try to parse rendered_related_key to select which
+      # conditions go with each table? Maybe we could still use
+      # explicit joins this way...
+
+      msg = RELATED_KEY_MISMATCH_MESSAGE % (self.related_key_id,
+                                            self.column,
+                                            table_alias_list,
+                                            rendered_related_key)
       if BACKWARD_COMPATIBILITY:
-      # XXX: Calling a private-ish method on column_map.
-      # This should never happen. It should be removed as soon as an
-      # alternative exists.
-      column_map._addJoinQuery(SQLQuery(rendered_related_key))
-      return None
+        # BBB: remove this branch of the condition, and the above
+        # constant, when all zsql_methods have been adapted to return
+        # the join queries properly separated by the
+        # RELATED_QUERY_SEPARATOR.
+
+        # The rendered related key doesn't have the separators for each
+        # joined table, so we revert to doing implicit inner joins:
+        log.warning(msg + "\n\nAdding an Implicit Join Condition...")
+        column_map._addJoinQueryForColumn(self.column,
+                                          SQLQuery(rendered_related_key))
       else:
-      return SQLExpression(self, where_expression=rendered_related_key)
+        raise RuntimeError(msg)
+    return None
 
 verifyClass(IRelatedKey, RelatedKey)
 

product/ZSQLCatalog/interfaces/column_map.py

@@ -284,18 +284,3 @@ class IColumnMap(Interface):
       Return a copy of the table alias list for tables requiring a join with
       catalog table.
     """
-
-  def getStraightJoinTableList():
-    """
-      Returns the list of tables used this search and which
-      need to be joined with the main table using explicit
-      indices.
-    """
-
-  def getLeftJoinTableList():
-    """
-      Returns the list of tables used this search and which
-      need to be LEFT joined with the main table using explicit
-      indices.
-    """
-

product/ZSQLCatalog/tests/testSQLCatalog.py

@@ -202,6 +202,7 @@ class DummyCatalog(SQLCatalog):
     assert 'query_table' in kw
     assert 'table_0' in kw
     assert 'table_1' in kw
+    assert 'AND' in kw.pop('RELATED_QUERY_SEPARATOR')
     assert len(kw) == 4
     return '%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s' % kw
 
@@ -629,7 +630,7 @@ class TestSQLCatalog(unittest.TestCase):
     select_dict = sql_expression.getSelectDict()
     self.assertTrue('ambiguous_mapping' in select_dict, select_dict)
     self.assertTrue('bar' in select_dict['ambiguous_mapping'], select_dict['ambiguous_mapping'])
-    # Doted alias: table name must get stripped. This is required to have an
+    # Dotted alias: table name must get stripped. This is required to have an
     # upgrade path from old ZSQLCatalog versions where pre-mapped columns were
     # used in their select_expression. This must only happen in the
     # "{column: None}" form, as otherwise it's the user explicitely asking for