Commit 9cdf7a81 authored by Leonardo Rochael Almeida's avatar Leonardo Rochael Almeida

Merge branch 'catalog_join'

parents 07cbd374 d2a9088a
...@@ -14,8 +14,10 @@ ...@@ -14,8 +14,10 @@
</item> </item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>table_0\n <value> <string>table_0\r\n
table_1</string> </value> table_1\r\n
RELATED_QUERY_SEPARATOR=" AND "\r\n
query_table="catalog"</string> </value>
</item> </item>
<item> <item>
<key> <string>cache_time_</string> </key> <key> <string>cache_time_</string> </key>
...@@ -55,10 +57,9 @@ table_1</string> </value> ...@@ -55,10 +57,9 @@ table_1</string> </value>
<key> <string>src</string> </key> <key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
<dtml-var table_0>.uid = catalog.parent_uid\n <dtml-var table_1>.uid = <dtml-var table_0>.parent_uid\n
AND <dtml-var table_1>.uid = <dtml-var table_0>.parent_uid\n <dtml-var RELATED_QUERY_SEPARATOR>\n
\n <dtml-var table_0>.uid = <dtml-var query_table>.parent_uid
]]></string> </value> ]]></string> </value>
</item> </item>
......
...@@ -104,6 +104,12 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -104,6 +104,12 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
new_erp5_deferred_sql_connection = 'erp5_sql_deferred_connection2' new_erp5_deferred_sql_connection = 'erp5_sql_deferred_connection2'
new_catalog_id = 'erp5_mysql_innodb2' new_catalog_id = 'erp5_mysql_innodb2'
__cleanups = ()
def _addCleanup(self, callable):
self.__cleanups += (callable,)
return callable
def afterSetUp(self): def afterSetUp(self):
uf = self.getPortal().acl_users uf = self.getPortal().acl_users
uf._doAddUser(self.username, '', ['Manager'], []) uf._doAddUser(self.username, '', ['Manager'], [])
...@@ -128,6 +134,8 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -128,6 +134,8 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
self.portal.manage_delObjects([self.new_erp5_deferred_sql_connection]) self.portal.manage_delObjects([self.new_erp5_deferred_sql_connection])
if self.new_catalog_id in self.portal.portal_catalog.objectIds(): if self.new_catalog_id in self.portal.portal_catalog.objectIds():
self.portal.portal_catalog.manage_delObjects([self.new_catalog_id]) self.portal.portal_catalog.manage_delObjects([self.new_catalog_id])
for cleanup in self.__cleanups:
cleanup(self)
transaction.commit() transaction.commit()
self.tic() self.tic()
...@@ -707,7 +715,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -707,7 +715,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
default_parametrs['where_expression'] = "" default_parametrs['where_expression'] = ""
default_parametrs['order_by_expression'] = None default_parametrs['order_by_expression'] = None
#import pdb; pdb.set_trace()
# check that we retrieve our 2 organisations by default. # check that we retrieve our 2 organisations by default.
kw = default_parametrs.copy() kw = default_parametrs.copy()
kw.update( portal_catalog.buildSQLQuery( kw.update( portal_catalog.buildSQLQuery(
...@@ -955,14 +962,13 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -955,14 +962,13 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
('catalog.title DESC', '`catalog`.`title` DESC')) ('catalog.title DESC', '`catalog`.`title` DESC'))
def test_26_SortOnUnknownKeys(self, quiet=quiet, run=run_all_test): def test_26_SortOnUnknownKeys(self, quiet=quiet, run=run_all_test):
if not run: return
if not run: return if not run: return
if not quiet: if not quiet:
message = 'Test Sort On Unknow Keys' message = 'Test Sort On Unknow Keys'
ZopeTestCase._print('\n%s ' % message) ZopeTestCase._print('\n%s ' % message)
LOG('Testing... ',0,message) LOG('Testing... ',0,message)
self.assertEquals('', self.assertEquals('',
self.getCatalogTool().buildSQLQuery( self.getCatalogTool().buildSQLQuery(select_list=('uid', 'path'),
sort_on=(('ignored', 'ascending'),))['order_by_expression']) sort_on=(('ignored', 'ascending'),))['order_by_expression'])
def test_27_SortOnAmbigousKeys(self, quiet=quiet, run=run_all_test): def test_27_SortOnAmbigousKeys(self, quiet=quiet, run=run_all_test):
...@@ -1028,6 +1034,7 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1028,6 +1034,7 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
'sort_on parameter must be taken into account even if related key ' 'sort_on parameter must be taken into account even if related key '
'is not a parameter of the current query') 'is not a parameter of the current query')
def _makeOrganisation(self, **kw): def _makeOrganisation(self, **kw):
"""Creates an Organisation in it's default module and reindex it. """Creates an Organisation in it's default module and reindex it.
By default, it creates a group/nexedi category, and make the organisation a By default, it creates a group/nexedi category, and make the organisation a
...@@ -1332,11 +1339,13 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor): ...@@ -1332,11 +1339,13 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
# catalog those objects # catalog those objects
transaction.commit() transaction.commit()
self.tic() self.tic()
catalog_tool = self.getCatalogTool()
self.assertEquals([ob], self.assertEquals([ob],
[x.getObject() for x in self.getCatalogTool()( [x.getObject() for x in catalog_tool(portal_type='Organisation',
portal_type='Organisation', SearchableText='title')]) SearchableText='title')])
self.assertEquals(1, self.getCatalogTool().countResults( self.assertEquals(1,
portal_type='Organisation', SearchableText='title')[0][0]) catalog_tool.countResults(portal_type='Organisation',
SearchableText='title')[0][0])
# 'different' is found in more than 50% of records # 'different' is found in more than 50% of records
# MySQL ignores such a word, but Tritonn does not ignore. # MySQL ignores such a word, but Tritonn does not ignore.
...@@ -4279,11 +4288,13 @@ VALUES ...@@ -4279,11 +4288,13 @@ VALUES
self.assertEqual(catalog.countResults(parent_uid=module_uid)[0][0], self.assertEqual(catalog.countResults(parent_uid=module_uid)[0][0],
module_len) module_len)
self.assertEquals(catalog.countResults(from_expression={ from_expression = {
'catalog': '(SELECT sub_catalog.* FROM catalog AS sub_catalog' \ 'catalog': '(SELECT sub_catalog.* FROM catalog AS sub_catalog'
' WHERE sub_catalog.parent_uid=%i)' \ ' WHERE sub_catalog.parent_uid=%i)'
' AS catalog' % (module_uid, ), ' AS catalog' % (module_uid, ),
})[0][0], module_len) }
count = catalog.countResults(from_expression=from_expression)[0][0]
self.assertEqual(count, module_len)
def test_getParentUid(self, quiet=quiet): def test_getParentUid(self, quiet=quiet):
from Products.ERP5.Document.Assignment import Assignment from Products.ERP5.Document.Assignment import Assignment
...@@ -4328,6 +4339,176 @@ VALUES ...@@ -4328,6 +4339,176 @@ VALUES
result = connector.manage_test('select 1 as foo;') result = connector.manage_test('select 1 as foo;')
self.assertEquals(1, result[0].foo) self.assertEquals(1, result[0].foo)
def _createSomeGroupCategories(self):
portal_category = self.getCategoryTool()
group_category = portal_category.group
group_data_map = dict(nexedi=('Nexedi', 'Nexedi Group'),
tiolive=('TIOLive', 'TioLive Group'),)
existing_group_id_list = group_category.objectIds()
for group_id, (title, description) in group_data_map.items():
if group_id in existing_group_id_list:
group = group_category[group_id]
else:
group = group_category.newContent(id=group_id)
group.edit(title=title, description=description)
def test_SelectDictWithDynamicRelatedKey(self, quiet=quiet, run=run_all_test):
if not run: return
if not quiet:
message = 'Select Dict With Dynamic Related Key'
ZopeTestCase._print('\n%s ' % message)
LOG('Testing... ', 0, message)
self._createSomeGroupCategories()
# Create some orgs associated with varying association with the
# groups created above.
module = self.portal.getDefaultModule('Organisation')
# org1 has no groups
org1 = module.newContent(portal_type='Organisation', title='org1')
# org2 has group nexedi
org2 = module.newContent(portal_type='Organisation', title='org2')
org2.setGroupList(['nexedi'])
# org3 has group tiolive
org3 = module.newContent(portal_type='Organisation', title='org3')
org3.setGroupList(['tiolive'])
# org4 has both groups
org4 = module.newContent(portal_type='Organisation', title='org4')
org4.setGroupList(['nexedi', 'tiolive'])
# check associations are correct
actual_group_title_map = dict((org.getTitle(),
sorted(org.getGroupTitleList()))
for org in (org1, org2, org3, org4))
expected_group_title_map = dict(org1=[],
org2=['Nexedi'],
org3=['TIOLive'],
org4=['Nexedi', 'TIOLive'])
self.assertEquals(actual_group_title_map, expected_group_title_map)
# Flush message queue
transaction.commit()
self.tic()
# we will restrict our search to orgs with these ids to be resilient
# to preexisting orgs:
org_id_list = sorted(org.getId() for org in (org1, org2, org3, org4))
# and we'll sort on title to make the output predictable
search_kw = dict(id=org_id_list,
sort_on='title')
# Try to get the organisations with the group title Nexedi to make sure
# searching works correctly
organisation_list = [x.getObject() for x in
module.searchFolder(strict_group_title='Nexedi',
**search_kw)]
self.assertEquals(organisation_list, [org2, org4])
# Now lets fetch the titles of groups of the above orgs using select_dict.
search_kw.update(select_dict=dict(strict_group_title=None))
records = module.searchFolder(**search_kw)
# by default the catalog only returns items containing the
# relationship we asked for (group). Besides, some entries will
# appear many times, according to the number of relationships each
# catalog entry has in that related key.
results = [(rec.title, rec.strict_group_title or '-None-')
for rec in records]
self.assertEquals(sorted(results),
[('org2', 'Nexedi'),
('org3', 'TIOLive'),
('org4', 'Nexedi'),
('org4', 'TIOLive')])
# But if we demand a left-join on that column, then we'll have all
# orgs we created. They'll still be repeated according to their
# relationships, though.
search_kw.update(left_join_list=('strict_group_title',))
records = module.searchFolder(**search_kw)
results = [(rec.title, rec.strict_group_title or '-None-')
for rec in records]
self.assertEquals(sorted(results),
[('org1', '-None-'),
('org2', 'Nexedi'),
('org3', 'TIOLive'),
('org4', 'Nexedi'),
('org4', 'TIOLive')])
# To get only one of each org, we need to group by one of the
# catalog keys.
# Note that this relies on a non-standard behaviour
# of MySQL: If a selected column is not present in the GROUP BY
# clause, only the first ocurrence is taken. Other databases,
# like Oracle, assume that selected columns are either GROUPed BY
# or are inside an aggregation function (COUNT, SUM, GROUP_CONCAT,
# ...), and consider the query to be in error otherwise.
search_kw.update(group_by_list=('uid',))
organisation_list = [x.getObject() for x in
module.searchFolder(**search_kw)]
self.assertEquals(organisation_list, [org1, org2, org3, org4])
def test_BackwardCompatibilityWithOldMethods(self, quiet=quiet,
run=run_all_test):
if not run: return
if not quiet:
message = 'Dealing with RelatedKey methods missing the proper separator'
ZopeTestCase._print('\n%s ' % message)
LOG('Testing... ', 0, message)
module = self.getOrganisationModule()
org_a = self._makeOrganisation(title='abc',default_address_city='abc')
org_a.setReference(org_a.getId())
# sometimes the module itself is not indexed yet...
module.reindexObject()
# Flush message queue
transaction.commit()
self.tic()
# make a query to fetch the address of the organisation above by
# querying, among other things, the grand_parent
query = dict(grand_parent_portal_type="Organisation Module",
parent_reference=org_a.getReference())
catalog = self.getCatalogTool()
# check the query works normally
self.assertEqual([x.getObject() for x in catalog.searchResults(**query)],
[org_a.default_address])
# even if we do a left_join
query_lj = query.copy()
query_lj.update(left_join_list=('grand_parent_portal_type',))
self.assertEqual([x.getObject() for x in catalog.searchResults(**query_lj)],
[org_a.default_address])
# now turn the z_related_grand_parent into an old-style method, without
# RELATED_QUERY_SEPARATOR
method = catalog.getSQLCatalog().z_related_grand_parent
old_src = method.src
@self._addCleanup
def cleanGrandParentMethod(self):
method.manage_edit(method.title, method.connection_id,
method.arguments_src, old_src)
src = old_src.replace('<dtml-var RELATED_QUERY_SEPARATOR>', ' AND ')
method.manage_edit(method.title, method.connection_id, method.arguments_src,
src)
# check that it still works
self.assertEqual([x.getObject() for x in catalog.searchResults(**query)],
[org_a.default_address])
# now try to do a left-join on grand_parent_portal_type which
# shouldn't work
self.assertRaises(RuntimeError, lambda: catalog.searchResults(**query_lj))
# Neither should it work if a left-join is attempted in a column
# that has proper related-key rendering, but is present in the
# same query as a column that hasn't, as the whole query is
# converted into implicit inner joins.
self.tic()
query_lj.update(left_join_list=('strict_group_title',),
select_dict=('strict_group_title',))
self.assertRaises(RuntimeError, lambda: catalog.searchResults(**query_lj))
# though it should work on queries that don't use the broken related-key
del query_lj['grand_parent_portal_type']
self.assertEqual([x.getObject() for x in catalog.searchResults(**query_lj)],
[org_a.default_address])
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestERP5Catalog)) suite.addTest(unittest.makeSuite(TestERP5Catalog))
......
...@@ -329,7 +329,10 @@ class Predicate(XMLObject): ...@@ -329,7 +329,10 @@ class Predicate(XMLObject):
catalog_kw['where_expression'] = SQLQuery(sql_text) catalog_kw['where_expression'] = SQLQuery(sql_text)
else: else:
catalog_kw['where_expression'] = '' catalog_kw['where_expression'] = ''
# force implicit join
catalog_kw['implicit_join'] = True
sql_query = portal_catalog.buildSQLQuery(**catalog_kw) sql_query = portal_catalog.buildSQLQuery(**catalog_kw)
# XXX from_table_list is None most of the time after the explicit_join work
for alias, table in sql_query['from_table_list']: for alias, table in sql_query['from_table_list']:
if from_table_dict.has_key(alias): if from_table_dict.has_key(alias):
raise KeyError, "The same table is used twice for an identity criterion and for a membership criterion" raise KeyError, "The same table is used twice for an identity criterion and for a membership criterion"
......
...@@ -27,11 +27,18 @@ ...@@ -27,11 +27,18 @@
# #
############################################################################## ##############################################################################
import re
import itertools
from zLOG import LOG, WARNING, INFO from zLOG import LOG, WARNING, INFO
from interfaces.column_map import IColumnMap from interfaces.column_map import IColumnMap
from zope.interface.verify import verifyClass from zope.interface.verify import verifyClass
from zope.interface import implements from zope.interface import implements
from SQLCatalog import profiler_decorator from Products.ZSQLCatalog.interfaces.column_map import IColumnMap
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
from Products.ZSQLCatalog.TableDefinition import (PlaceHolderTableDefinition,
TableAlias,
InnerJoin,
LeftJoin)
DEFAULT_GROUP_ID = None DEFAULT_GROUP_ID = None
...@@ -43,12 +50,18 @@ MAPPING_TRACE = False ...@@ -43,12 +50,18 @@ MAPPING_TRACE = False
# currently, it's not possible because related_key_dict is indexed by related key name, which makes 'source_title_1' lookup fail. It should be indexed by group (probably). # currently, it's not possible because related_key_dict is indexed by related key name, which makes 'source_title_1' lookup fail. It should be indexed by group (probably).
# TODO: rename all "related_key" references into "virtual_column" # TODO: rename all "related_key" references into "virtual_column"
re_sql_as = re.compile("\s+AS\s[^)]+$", re.IGNORECASE | re.MULTILINE)
class ColumnMap(object): class ColumnMap(object):
implements(IColumnMap) implements(IColumnMap)
@profiler_decorator @profiler_decorator
def __init__(self, catalog_table_name=None): def __init__(self,
catalog_table_name=None,
table_override_map=None,
left_join_list=None,
implicit_join=False):
self.catalog_table_name = catalog_table_name self.catalog_table_name = catalog_table_name
# Key: group # Key: group
# Value: set of column names # Value: set of column names
...@@ -81,15 +94,28 @@ class ColumnMap(object): ...@@ -81,15 +94,28 @@ class ColumnMap(object):
self.raw_column_dict = {} self.raw_column_dict = {}
# Entries: column name # Entries: column name
self.column_ignore_set = set() self.column_ignore_set = set()
self.join_table_set = set() self.join_table_map = dict()
self.straight_join_table_list = [] # BBB: Remove join_query_list and its uses when all RelatedKey
self.left_join_table_list = [] # methods have been converted to properly return each Join
# condition separately, and all uses of catalog's from_expression
# have been removed.
self.join_query_list = [] self.join_query_list = []
self.table_override_map = table_override_map or {}
self.table_definition = PlaceHolderTableDefinition()
# We need to keep track of the original definition to do inner joins on it
self._inner_table_definition = self.table_definition
self.left_join_list = left_join_list
self.implicit_join = implicit_join
assert not (self.implicit_join and self.left_join_list), (
"Cannot do left_joins while forcing implicit join"
)
@profiler_decorator @profiler_decorator
def registerColumn(self, raw_column, group=DEFAULT_GROUP_ID, simple_query=None): def registerColumn(self, raw_column, group=DEFAULT_GROUP_ID, simple_query=None):
assert ' as ' not in raw_column.lower() assert ' as ' not in raw_column.lower()
# Sanitize input: extract column from raw column (might contain COUNT, ...). # Sanitize input: extract column from raw column (might contain COUNT, ...).
# XXX This is not enough to parse something like:
# GROUP_CONCAT(DISTINCT foo ORDER BY bar)
if '(' in raw_column: if '(' in raw_column:
function, column = raw_column.split('(') function, column = raw_column.split('(')
column = column.strip() column = column.strip()
...@@ -120,7 +146,7 @@ class ColumnMap(object): ...@@ -120,7 +146,7 @@ class ColumnMap(object):
# When a column is registered in default group and is explicitely # When a column is registered in default group and is explicitely
# mapped to a table, we must mark its table as requiring a join with # mapped to a table, we must mark its table as requiring a join with
# catalog table (unless it's the catalog table, of course). # catalog table (unless it's the catalog table, of course).
self._addJoinTable(table, group) self._addJoinTableForColumn(table, table + "." + column, group)
def ignoreColumn(self, column): def ignoreColumn(self, column):
self.column_ignore_set.add(column) self.column_ignore_set.add(column)
...@@ -255,7 +281,7 @@ class ColumnMap(object): ...@@ -255,7 +281,7 @@ class ColumnMap(object):
# Although the list of tables those columns belong to is known # Although the list of tables those columns belong to is known
# earlier (in "build"), mapping them here # earlier (in "build"), mapping them here
# - avoids code duplication (registerTable, resolveColumn, # - avoids code duplication (registerTable, resolveColumn,
# _addJoinTable) # _addJoinTableForColumn)
# - offers user to vote for an unknown table, overriding this # - offers user to vote for an unknown table, overriding this
# forced mapping. # forced mapping.
use_allowed = table_name == catalog_table_name or \ use_allowed = table_name == catalog_table_name or \
...@@ -300,10 +326,11 @@ class ColumnMap(object): ...@@ -300,10 +326,11 @@ class ColumnMap(object):
self.registerTable(table_name, group=group) self.registerTable(table_name, group=group)
self.resolveColumn(column_name, table_name, group=group) self.resolveColumn(column_name, table_name, group=group)
if table_name != catalog_table_name: if table_name != catalog_table_name:
self._addJoinTable(table_name, group) self._addJoinTableForColumn(table_name, column_name, group)
@profiler_decorator @profiler_decorator
def build(self, sql_catalog): def build(self, sql_catalog):
join_query_to_build_list = []
catalog_table_name = self.catalog_table_name catalog_table_name = self.catalog_table_name
if catalog_table_name is None: if catalog_table_name is None:
return return
...@@ -319,7 +346,7 @@ class ColumnMap(object): ...@@ -319,7 +346,7 @@ class ColumnMap(object):
if related_key_definition is not None: if related_key_definition is not None:
join_query = sql_catalog.getSearchKey(column_name, 'RelatedKey').buildQuery(sql_catalog=sql_catalog, related_key_definition=related_key_definition) join_query = sql_catalog.getSearchKey(column_name, 'RelatedKey').buildQuery(sql_catalog=sql_catalog, related_key_definition=related_key_definition)
join_query.registerColumnMap(sql_catalog, self) join_query.registerColumnMap(sql_catalog, self)
self._addJoinQuery(join_query) join_query_to_build_list.append(join_query)
# List all possible tables, with all used column for each # List all possible tables, with all used column for each
for group, column_set in self.registry.iteritems(): for group, column_set in self.registry.iteritems():
...@@ -397,6 +424,20 @@ class ColumnMap(object): ...@@ -397,6 +424,20 @@ class ColumnMap(object):
table_alias_number_dict[alias_table_name] = table_alias_number table_alias_number_dict[alias_table_name] = table_alias_number
self.resolveTable(table_name, alias, group=group) self.resolveTable(table_name, alias, group=group)
# now that we have all aliases, calculate missing joins comming from
# non-RelatedKey relationships (like full_text).
self.registerCatalog()
self._calculateMissingJoins()
# and all left joins that did not come from explicit queries
# (i.e. joins comming from 'sort_on', 'select_dict', etc.)
for join_query in join_query_to_build_list:
# XXX ugly use of inner attribute of join_query. Please Refactor:
# search_keys don't actually return SQLExpressions, but they add
# join definitions in the column_map
join_query.search_key.buildSQLExpression(sql_catalog=sql_catalog,
column_map=self,
only_group_columns=False,
group=join_query.group,)
if MAPPING_TRACE: if MAPPING_TRACE:
# Key: group # Key: group
# Value: 2-tuple # Value: 2-tuple
...@@ -441,8 +482,28 @@ class ColumnMap(object): ...@@ -441,8 +482,28 @@ class ColumnMap(object):
def getCatalogTableAlias(self, group=DEFAULT_GROUP_ID): def getCatalogTableAlias(self, group=DEFAULT_GROUP_ID):
return self.table_alias_dict[(group, self.catalog_table_name)] return self.table_alias_dict[(group, self.catalog_table_name)]
def _isBackwardCompatibilityRequired(self):
return bool(
# if they explicitly ask for implicit
self.implicit_join or
# if they don't pass a catalog alias, we cannot do explicit joins
not self._setMinimalTableDefinition() or
# If one or more RelatedKey methods weren't converted, we'll get
# queries for an implicit inner join, so we have to do all joins
# as implicit.
self.join_query_list or
# for now, work in BW compat mode if a table_override
# is passed. It only works for simple subselect
# definitions anyway, and it's being used primarily
# for writing left-joins manually.
self.table_override_map)
def getTableAliasDict(self): def getTableAliasDict(self):
if self._isBackwardCompatibilityRequired():
# BBB: Using implicit joins or explicit from_expression
return self.table_map.copy() return self.table_map.copy()
else:
return None
@profiler_decorator @profiler_decorator
def resolveColumn(self, column, table_name, group=DEFAULT_GROUP_ID): def resolveColumn(self, column, table_name, group=DEFAULT_GROUP_ID):
...@@ -472,16 +533,30 @@ class ColumnMap(object): ...@@ -472,16 +533,30 @@ class ColumnMap(object):
def getTableAlias(self, table_name, group=DEFAULT_GROUP_ID): def getTableAlias(self, table_name, group=DEFAULT_GROUP_ID):
return self.table_alias_dict[(group, table_name)] return self.table_alias_dict[(group, table_name)]
def _addJoinQuery(self, query): def _addJoinQueryForColumn(self, column, query):
# BBB: This is a backward compatibility method that will be
# removed in the future, when all related key methods have been adapted
# to provide all Join conditions separately
if column in self.left_join_list:
raise RuntimeError('Left Join requested for column: %r, but rendered '
'join query is not compatible and would result in an '
'Implicit Inner Join:\n%s' %
(column, query,))
self.join_query_list.append(query) self.join_query_list.append(query)
def iterJoinQueryList(self): def iterJoinQueryList(self):
return iter(self.join_query_list) if self._isBackwardCompatibilityRequired():
# Return all join queries for implicit join, and all the other
# queries we were using to build explicit joins, but won't be able to.
return itertools.chain(self.join_query_list,
self.table_definition.getJoinConditionQueryList())
return []
@profiler_decorator @profiler_decorator
def _addJoinTable(self, table_name, group=DEFAULT_GROUP_ID): def _addJoinTableForColumn(self, table_name, column, group=DEFAULT_GROUP_ID):
""" """
Declare given table as requiring to be joined with catalog table. Declare given table as requiring to be joined with catalog table on uid.
table_name (string) table_name (string)
Table name. Table name.
...@@ -497,17 +572,137 @@ class ColumnMap(object): ...@@ -497,17 +572,137 @@ class ColumnMap(object):
# Register uid column if it is not already # Register uid column if it is not already
self.registerColumn('uid') self.registerColumn('uid')
self.resolveColumn('uid', catalog_table) self.resolveColumn('uid', catalog_table)
self.join_table_set.add((group, table_name)) self.join_table_map.setdefault((group, table_name), set()).add(column)
def getJoinTableAliasList(self): def getJoinTableAliasList(self):
return [self.getTableAlias(table_name, group=group) return [self.getTableAlias(table_name, group=group)
for (group, table_name) in self.join_table_set] for (group, table_name) in self.join_table_map.keys()]
def getStraightJoinTableList(self): def _getTableOverride(self, table_name):
return self.straight_join_table_list[:] # self.table_override_map is a dictionary mapping table names to
# strings containing aliases of arbitrary table definitions
def getLeftJoinTableList(self): # (including subselects). So we split the alias and discard it
return self.left_join_table_list[:] # since we do our own aliasing.
table_override_w_alias = self.table_override_map.get(table_name)
if table_override_w_alias is None:
return table_name
# XXX move the cleanup of table alias overrides to EntireQuery
# class or ZSQLCatalog, so we don't need SQL syntax knowledge in
# ColumnMap.
#
# Normalise the AS sql keyword to remove the last
# aliasing in the string if present. E.g.:
#
# '(SELECT sub_catalog.*
# FROM catalog AS sub_catalog
# WHERE sub_catalog.parent_uid=183) AS catalog'
#
# becomes:
#
# '(SELECT sub_catalog.*
# FROM catalog AS sub_catalog
# WHERE sub_catalog.parent_uid=183)'
table_override, removed = re_sql_as.subn('', table_override_w_alias)
assert removed < 2, ('More than one table aliasing was removed from %r' %
table_override_w_alias)
if removed:
LOG('ColumnMap', WARNING,
'Table overrides should not contain aliasing: %r' % table_override)
return table_override
def makeTableAliasDefinition(self, table_name, table_alias):
"""Make a table alias, giving a change to ColumnMap to override
the original table definition with another expression"""
table_name = self._getTableOverride(table_name)
assert table_name and table_alias, ("table_name (%r) and table_alias (%r) "
"must both be defined" %
(table_name, table_alias))
return TableAlias(table_name, table_alias)
def _setMinimalTableDefinition(self):
""" Set a minimal table definition: the main catalog alias
We don't do this at __init__ because we have neither the catalog
table name nor its intended alias at that point.
"""
inner_def = self._inner_table_definition
if inner_def.table_definition is None:
try:
catalog_table_alias = self.getCatalogTableAlias()
except KeyError:
LOG('ColumnMap', WARNING,
'_setMinimalTableDefinition called but the main catalog has not '
'yet received an alias!')
return False
inner_def.replace(self.makeTableAliasDefinition(self.catalog_table_name,
catalog_table_alias))
return True
def getTableDefinition(self):
if self._isBackwardCompatibilityRequired():
# BBB: One of the RelatedKeys registered an implicit join, do
# not return a table definition, self.getTableAliasDict() should
# be used instead
return None
self.table_definition.checkTableAliases()
return self.table_definition
def addRelatedKeyJoin(self, column, right_side, condition):
""" Wraps the current table_definition in the left-side of a new
join. Use an InnerJoin or a LeftJoin depending on whether the
column is in the left_join_list or not.
"""
# XXX: to fix TestERP5Catalog.test_52_QueryAndTableAlias, create
# here a list of joins and try to merge each new entry into one of
# the pre-existing entries by comparing their right-sides.
#
# XXX 2: This is the place were we could do ordering of inner and left
# joins so as to get better performance. For instance, a quick win is to
# add all inner-joins first, and all left-joins later. We could also decide
# on the order of left-joins based on the order of self.left_join_list or
# even a catalog property/configuration/script.
#
# XXX 3: This is also the place where we could check if explicit
# table aliases should cause some of these table definitions to be
# collapsed into others.
assert self._setMinimalTableDefinition()
Join = (column in self.left_join_list) and LeftJoin or InnerJoin
join_definition = Join(self.table_definition, right_side,
condition=condition)
self.table_definition = join_definition
# def getFinalTableDefinition(self):
# self._calculateMissingJoins()
# return self.getTableDefinition()
def _calculateMissingJoins(self):
left_join_set = set(self.left_join_list)
self._setMinimalTableDefinition()
catalog_table_alias = self.getCatalogTableAlias()
for (group, table_name), column_set in self.join_table_map.items():
# if any of the columns for this implicit join was requested as a
# left-join, then all columns will be subject to a left-join.
# XXX What if one of the columns was an actual query, as opposed to a
# sort column or select_dict? This would cause results in the main
# catalog that don't match the query to be present as well. We expect
# the user which passes a left_join_list to know what he is doing.
if column_set.intersection(left_join_set):
Join = LeftJoin
else:
Join = InnerJoin
table_alias = self.getTableAlias(table_name, group=group)
table_alias_def = self.makeTableAliasDefinition(table_name, table_alias)
# XXX: perhaps refactor some of the code below to do:
# self._inner_table_definition.addInnerJoin(TableAlias(...),
# condition=(...))
self._inner_table_definition.replace(
Join(self._inner_table_definition.table_definition,
table_alias_def,
# XXX ColumnMap shouldn't have SQL knowledge
condition=('`%s`.`uid` = `%s`.`uid`' %
(table_alias, catalog_table_alias)),
)
)
verifyClass(IColumnMap, ColumnMap) verifyClass(IColumnMap, ColumnMap)
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
# #
############################################################################## ##############################################################################
import warnings
from Products.ZSQLCatalog.SQLExpression import SQLExpression from Products.ZSQLCatalog.SQLExpression import SQLExpression
from Products.ZSQLCatalog.ColumnMap import ColumnMap from Products.ZSQLCatalog.ColumnMap import ColumnMap
from zLOG import LOG from zLOG import LOG
...@@ -35,6 +36,7 @@ from Products.ZSQLCatalog.interfaces.entire_query import IEntireQuery ...@@ -35,6 +36,7 @@ from Products.ZSQLCatalog.interfaces.entire_query import IEntireQuery
from zope.interface.verify import verifyClass from zope.interface.verify import verifyClass
from zope.interface import implements from zope.interface import implements
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
from Products.ZSQLCatalog.TableDefinition import LegacyTableDefinition
def defaultDict(value): def defaultDict(value):
if value is None: if value is None:
...@@ -54,19 +56,28 @@ class EntireQuery(object): ...@@ -54,19 +56,28 @@ class EntireQuery(object):
column_map = None column_map = None
@profiler_decorator @profiler_decorator
def __init__(self, query, order_by_list=(), group_by_list=(), def __init__(self, query,
select_dict=None, limit=None, catalog_table_name=None, order_by_list=(),
extra_column_list=(), from_expression=None, group_by_list=(),
order_by_override_list=None): select_dict=None,
left_join_list=(),
limit=None,
catalog_table_name=None,
extra_column_list=(),
from_expression=None,
order_by_override_list=None,
implicit_join=False):
self.query = query self.query = query
self.order_by_list = list(order_by_list) self.order_by_list = list(order_by_list)
self.order_by_override_set = frozenset(order_by_override_list) self.order_by_override_set = frozenset(order_by_override_list)
self.group_by_list = list(group_by_list) self.group_by_list = list(group_by_list)
self.select_dict = defaultDict(select_dict) self.select_dict = defaultDict(select_dict)
self.left_join_list = left_join_list
self.limit = limit self.limit = limit
self.catalog_table_name = catalog_table_name self.catalog_table_name = catalog_table_name
self.extra_column_list = list(extra_column_list) self.extra_column_list = list(extra_column_list)
self.from_expression = from_expression self.from_expression = from_expression
self.implicit_join = implicit_join
def asSearchTextExpression(self, sql_catalog): def asSearchTextExpression(self, sql_catalog):
return self.query.asSearchTextExpression(sql_catalog) return self.query.asSearchTextExpression(sql_catalog)
...@@ -78,7 +89,12 @@ class EntireQuery(object): ...@@ -78,7 +89,12 @@ class EntireQuery(object):
# XXX: should we provide a way to register column map as a separate # XXX: should we provide a way to register column map as a separate
# method or do it here ? # method or do it here ?
# Column Map was not built yet, do it. # Column Map was not built yet, do it.
self.column_map = column_map = ColumnMap(catalog_table_name=self.catalog_table_name) column_map = ColumnMap(catalog_table_name=self.catalog_table_name,
table_override_map=self.from_expression,
left_join_list=self.left_join_list,
implicit_join=self.implicit_join,
)
self.column_map = column_map
for extra_column in self.extra_column_list: for extra_column in self.extra_column_list:
table, column = extra_column.replace('`', '').split('.') table, column = extra_column.replace('`', '').split('.')
if table != self.catalog_table_name: if table != self.catalog_table_name:
...@@ -145,30 +161,55 @@ class EntireQuery(object): ...@@ -145,30 +161,55 @@ class EntireQuery(object):
None, ) * (3 - len(order_by))) None, ) * (3 - len(order_by)))
self.order_by_list = new_order_by_list self.order_by_list = new_order_by_list
# generate SQLExpression from query # generate SQLExpression from query
sql_expression_list = [self.query.asSQLExpression(sql_catalog, column_map, only_group_columns)] sql_expression_list = [self.query.asSQLExpression(sql_catalog,
# generate join expression based on column_map.getJoinTableAliasList column_map,
only_group_columns)]
append = sql_expression_list.append append = sql_expression_list.append
for join_query in column_map.iterJoinQueryList(): for join_query in column_map.iterJoinQueryList():
append(join_query.asSQLExpression(sql_catalog, column_map, only_group_columns)) append(join_query.asSQLExpression(sql_catalog,
join_table_list = column_map.getJoinTableAliasList() column_map,
if len(join_table_list): only_group_columns))
# XXX: Is there any special rule to observe when joining tables ? # generate join expression based on column_map.getJoinTableAliasList
# Maybe we could check which column is a primary key instead of # XXX: This is now done by ColumnMap to its table_definition,
# hardcoding "uid". # during build()
where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \ #
(column_map.getCatalogTableAlias(), ) # join_table_list = column_map.getJoinTableAliasList()
# XXX: It would cleaner from completeness point of view to use column # if len(join_table_list):
# mapper to render column, but makes code much more complex to just do # # XXX: Is there any special rule to observe when joining tables ?
# a simple text rendering. If there is any reason why we should have # # Maybe we could check which column is a primary key instead of
# those column in the mapper, then we should use the clean way. # # hardcoding "uid".
append(SQLExpression(self, where_expression=' AND '.join( # where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
where_pattern % (x, ) for x in join_table_list # (column_map.getCatalogTableAlias(), )
))) # # XXX: It would cleaner from completeness point of view to use column
# # mapper to render column, but makes code much more complex to just do
# # a simple text rendering. If there is any reason why we should have
# # those column in the mapper, then we should use the clean way.
# append(SQLExpression(self, where_expression=' AND '.join(
# where_pattern % (x, ) for x in join_table_list
# )))
# BBB self.from_expression forces use of implicit inner join
table_alias_dict = column_map.getTableAliasDict()
if self.from_expression:
warnings.warn("Providing a 'from_expression' is deprecated.",
DeprecationWarning)
# XXX: perhaps move this code to ColumnMap?
legacy_from_expression = self.from_expression
from_expression = LegacyTableDefinition(legacy_from_expression,
table_alias_dict)
table_alias_dict = None
else:
from_expression = column_map.getTableDefinition()
assert ((from_expression is None) !=
(table_alias_dict is None)), ("Got both a from_expression "
"and a table_alias_dict")
self.sql_expression_list = sql_expression_list self.sql_expression_list = sql_expression_list
# TODO: wrap the table_alias_dict above into a TableDefinition as well,
# even without a legacy_table_definition.
return SQLExpression( return SQLExpression(
self, self,
table_alias_dict=column_map.getTableAliasDict(), table_alias_dict=table_alias_dict,
from_expression=self.from_expression, from_expression=from_expression,
order_by_list=self.order_by_list, order_by_list=self.order_by_list,
group_by_list=self.group_by_list, group_by_list=self.group_by_list,
select_dict=self.final_select_dict, select_dict=self.final_select_dict,
......
...@@ -2291,6 +2291,13 @@ class Catalog(Folder, ...@@ -2291,6 +2291,13 @@ class Catalog(Folder,
select_dict = None select_dict = None
elif isinstance(select_dict, (list, tuple)): elif isinstance(select_dict, (list, tuple)):
select_dict = dict([(x, None) for x in select_dict]) select_dict = dict([(x, None) for x in select_dict])
# Handle left_join_list
left_join_list = kw.pop('left_join_list', ())
# Handle implicit_join. It's True by default, as there's a lot of code
# in BT5s and elsewhere that calls buildSQLQuery() expecting implicit
# join. self._queryResults() defaults it to False for those using
# catalog.searchResults(...) or catalog(...) directly.
implicit_join = kw.pop('implicit_join', True)
# Handle order_by_list # Handle order_by_list
order_by_list = kw.pop('order_by_list', None) order_by_list = kw.pop('order_by_list', None)
sort_on = kw.pop('sort_on', None) sort_on = kw.pop('sort_on', None)
...@@ -2328,6 +2335,8 @@ class Catalog(Folder, ...@@ -2328,6 +2335,8 @@ class Catalog(Folder,
order_by_override_list=order_by_override_list, order_by_override_list=order_by_override_list,
group_by_list=group_by_list, group_by_list=group_by_list,
select_dict=select_dict, select_dict=select_dict,
left_join_list=left_join_list,
implicit_join=implicit_join,
limit=limit, limit=limit,
catalog_table_name=query_table, catalog_table_name=query_table,
extra_column_list=extra_column_list, extra_column_list=extra_column_list,
...@@ -2413,6 +2422,7 @@ class Catalog(Folder, ...@@ -2413,6 +2422,7 @@ class Catalog(Folder,
""" Returns a list of brains from a set of constraints on variables """ """ Returns a list of brains from a set of constraints on variables """
if build_sql_query_method is None: if build_sql_query_method is None:
build_sql_query_method = self.buildSQLQuery build_sql_query_method = self.buildSQLQuery
kw.setdefault('implicit_join', False)
query = build_sql_query_method(REQUEST=REQUEST, **kw) query = build_sql_query_method(REQUEST=REQUEST, **kw)
# XXX: decide if this should be made normal # XXX: decide if this should be made normal
ENFORCE_SEPARATION = True ENFORCE_SEPARATION = True
......
...@@ -166,7 +166,7 @@ class SQLExpression(object): ...@@ -166,7 +166,7 @@ class SQLExpression(object):
@profiler_decorator @profiler_decorator
def getFromExpression(self): def getFromExpression(self):
""" """
Returns a string. Returns a TableDefinition stored in one of the from_expressions or None
If there are nested SQLExpression, it checks that they either don't If there are nested SQLExpression, it checks that they either don't
define any from_expression or the exact same from_expression. Otherwise, define any from_expression or the exact same from_expression. Otherwise,
...@@ -175,7 +175,7 @@ class SQLExpression(object): ...@@ -175,7 +175,7 @@ class SQLExpression(object):
result = self.from_expression result = self.from_expression
for sql_expression in self.sql_expression_list: for sql_expression in self.sql_expression_list:
from_expression = sql_expression.getFromExpression() from_expression = sql_expression.getFromExpression()
if None not in (result, from_expression): if from_expression not in (result, None):
message = 'I don\'t know how to merge from_expressions' message = 'I don\'t know how to merge from_expressions'
if DEBUG: if DEBUG:
message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % ( message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
...@@ -385,20 +385,25 @@ class SQLExpression(object): ...@@ -385,20 +385,25 @@ class SQLExpression(object):
SQL_SELECT_ALIAS_FORMAT % (column, alias) SQL_SELECT_ALIAS_FORMAT % (column, alias)
for alias, column in self.getSelectDict().iteritems()) for alias, column in self.getSelectDict().iteritems())
@profiler_decorator def getFromTableList(self):
def asSQLExpressionDict(self):
table_alias_dict = self.getTableAliasDict() table_alias_dict = self.getTableAliasDict()
if not table_alias_dict:
return None
from_table_list = [] from_table_list = []
append = from_table_list.append append = from_table_list.append
for alias, table in table_alias_dict.iteritems(): for alias, table in table_alias_dict.iteritems():
append((SQL_TABLE_FORMAT % (alias, ), SQL_TABLE_FORMAT % (table, ))) append((SQL_TABLE_FORMAT % (alias, ), SQL_TABLE_FORMAT % (table, )))
from_expression_dict = self.getFromExpression() return from_table_list
if from_expression_dict is not None:
from_expression = SQL_LIST_SEPARATOR.join( @profiler_decorator
from_expression_dict.get(alias, '`%s` AS `%s`' % (table, alias)) def asSQLExpressionDict(self):
for alias, table in table_alias_dict.iteritems()) from_expression = self.getFromExpression()
else: from_table_list = self.getFromTableList()
from_expression = None assert None in (from_expression,
from_table_list), ("Cannot return both a from_expression "
"and a from_table_list")
if from_expression is not None:
from_expression = from_expression.render()
return { return {
'where_expression': self.getWhereExpression(), 'where_expression': self.getWhereExpression(),
'order_by_expression': self.getOrderByExpression(), 'order_by_expression': self.getOrderByExpression(),
......
...@@ -37,12 +37,29 @@ from Products.ZSQLCatalog.interfaces.search_key import IRelatedKey ...@@ -37,12 +37,29 @@ from Products.ZSQLCatalog.interfaces.search_key import IRelatedKey
from zope.interface.verify import verifyClass from zope.interface.verify import verifyClass
from zope.interface import implements from zope.interface import implements
from Products.ZSQLCatalog.SQLCatalog import profiler_decorator from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
from Products.ZSQLCatalog.TableDefinition import TableAlias, InnerJoin, LeftJoin
from logging import getLogger
log = getLogger(__name__)
BACKWARD_COMPATIBILITY = True BACKWARD_COMPATIBILITY = True
RELATED_QUERY_SEPARATOR = "\nAND -- related query separator\n"
RELATED_KEY_MISMATCH_MESSAGE = "\
A rendered related key must contain the same number of querying \
conditions as the tables it relates, properly separated by \
RELATED_QUERY_SEPARATOR. \n\
Offending related key: %r, for column %r, table_alias_list: %r, \
rendered_related_key: \n%s"
RELATED_KEY_ALIASED_MESSAGE = "\
Support for explicit joins of aliased related keys is not yet implemented. \
Offending related key: %r, for column %r, table_alias_list: %r"
class RelatedKey(SearchKey): class RelatedKey(SearchKey):
""" """
This SearchKey handles searched on virtual columns of RelatedKey type. This SearchKey handles searches on virtual columns of RelatedKey type.
It generates joins required by the virtual column to reach the actual It generates joins required by the virtual column to reach the actual
column to compare, plus a regular query on that column if needed. column to compare, plus a regular query on that column if needed.
""" """
...@@ -115,13 +132,23 @@ class RelatedKey(SearchKey): ...@@ -115,13 +132,23 @@ class RelatedKey(SearchKey):
def registerColumnMap(self, column_map, table_alias_list=None): def registerColumnMap(self, column_map, table_alias_list=None):
related_column = self.getColumn() related_column = self.getColumn()
group = column_map.registerRelatedKey(related_column, self.real_column) group = column_map.registerRelatedKey(related_column, self.real_column)
# Each table except last one must be registered to their own group, so that # Each table except last one must be registered to their own
# the same table can be used multiple time (and aliased multiple times) # group, so that the same table can be used multiple times (and
# in the same related key. The last one must be register to related key # aliased multiple times) in the same related key. The last one
# "main" group (ie, the value of the "group" variable) to be the same as # must be registered to the related key "main" group (ie, the
# the ta ble used in join_condition. # value of the "group" variable) to be the same as the table used
# in join_condition.
if table_alias_list is not None: if table_alias_list is not None:
assert len(self.table_list) == len(table_alias_list) assert len(self.table_list) == len(table_alias_list)
# XXX-Leo: remove the rest of this 'if' branch after making sure
# that ColumnMap.addRelatedKeyJoin() can handle collapsing
# chains of inner-joins that are subsets of one another based on
# having the same aliases:
msg = RELATED_KEY_ALIASED_MESSAGE % (self.related_key_id,
self.column,
table_alias_list,)
log.warning(msg + "\n\nForcing implicit join...")
column_map.implicit_join = True
for table_position in xrange(len(self.table_list) - 1): for table_position in xrange(len(self.table_list) - 1):
table_name = self.table_list[table_position] table_name = self.table_list[table_position]
local_group = column_map.registerRelatedKeyColumn(related_column, table_position, group) local_group = column_map.registerRelatedKeyColumn(related_column, table_position, group)
...@@ -145,6 +172,23 @@ class RelatedKey(SearchKey): ...@@ -145,6 +172,23 @@ class RelatedKey(SearchKey):
column_map.registerCatalog() column_map.registerCatalog()
return group return group
def stitchJoinDefinition(self, table_alias_list, join_query_list, column_map):
alias, table = table_alias_list[-1]
right = column_map.makeTableAliasDefinition(table, alias)
if not join_query_list:
# nothing to do, just return the table alias
assert len(table_alias_list) == 1
return right
else:
# create an InnerJoin of the last element of the alias list with
# a chain of InnerJoins of the rest of the list conditioned on
# the the last element of the join_query_list
left = self.stitchJoinDefinition(table_alias_list[:-1],
join_query_list[:-1],
column_map)
condition = join_query_list[-1]
return InnerJoin(left, right, condition)
@profiler_decorator @profiler_decorator
def buildSQLExpression(self, sql_catalog, column_map, only_group_columns, group): def buildSQLExpression(self, sql_catalog, column_map, only_group_columns, group):
""" """
...@@ -170,20 +214,26 @@ class RelatedKey(SearchKey): ...@@ -170,20 +214,26 @@ class RelatedKey(SearchKey):
table_alias_list = [(getTableAlias(related_table, group=getRelatedKeyGroup(index, group)), related_table) table_alias_list = [(getTableAlias(related_table, group=getRelatedKeyGroup(index, group)), related_table)
for (index, related_table) in enumerate(related_table_list)] for (index, related_table) in enumerate(related_table_list)]
# table alias for destination table # table alias for destination table
table_alias_list.append((getTableAlias(destination_table, group=group), destination_table)) table_alias_list.append((getTableAlias(destination_table, group=group),
destination_table))
# map aliases to use in ZSQLMethod. # map aliases to use in ZSQLMethod.
table_alias_dict = dict(('table_%s' % (index, ), table_alias[0]) table_alias_dict = dict(('table_%s' % (index, ), table_alias)
for (index, table_alias) in enumerate(table_alias_list)) for (index, (table_alias, table_name))
in enumerate(table_alias_list))
assert len(table_alias_list) == len(table_alias_dict) assert len(table_alias_list) == len(table_alias_dict)
query_table=column_map.getCatalogTableAlias()
rendered_related_key = related_key( rendered_related_key = related_key(
query_table=column_map.getCatalogTableAlias(), query_table=query_table,
RELATED_QUERY_SEPARATOR=RELATED_QUERY_SEPARATOR,
src__=1, src__=1,
**table_alias_dict) **table_alias_dict)
join_condition_list = rendered_related_key.split(RELATED_QUERY_SEPARATOR)
# Important: # Important:
# Former catalog separated join condition from related query. # Previously the catalog separated join condition from the related query.
# Example: # Example:
# ComplexQuery(Query(title="foo"), # ComplexQuery(Query(title="foo"),
# Query(subordination_title="bar") # Query(subordination_title="bar")
...@@ -199,19 +249,53 @@ class RelatedKey(SearchKey): ...@@ -199,19 +249,53 @@ class RelatedKey(SearchKey):
# This was done on purpose, because doing otherwise gives very poor # This was done on purpose, because doing otherwise gives very poor
# performances (on a simple data set, similar query can take *minutes* to # performances (on a simple data set, similar query can take *minutes* to
# execute - as of MySQL 5.x). # execute - as of MySQL 5.x).
# Doing the same way as the former catalog is required for backward #
# compatibility, until a decent alternative is found (like spliting the # Because of this, we never return an SQLExpression here, as it
# "OR" expression into ensemblist operations at query level). # would mix join definition with column condition in the body of
# Note that doing this has a side effect on result list, as objects # the WHERE clause. Instead we explicitly define a Join to the
# lacking a relation will never appear in the result. # catalog. The ColumnMap defines whether this is an Inner Join or
# a Left Outer Join. Notice that if an Inner Join is decided,
# objects lacking a relationship will never appear in the result.
if len(join_condition_list) == len(table_alias_list):
# Good! we got a compatible method that splits the join
# conditions according to the related tables.
#
# Add a join on this related key, based on the chain of
# inner-joins of the related key tables.
query_table_join_condition = join_condition_list.pop()
right_side = self.stitchJoinDefinition(table_alias_list,
join_condition_list,
column_map)
column_map.addRelatedKeyJoin(self.column,
right_side=right_side,
condition=query_table_join_condition)
else:
# Method did not render the related key condition with the
# appropriate separators so we could split it
# XXX: Can we try to parse rendered_related_key to select which
# conditions go with each table? Maybe we could still use
# explicit joins this way...
msg = RELATED_KEY_MISMATCH_MESSAGE % (self.related_key_id,
self.column,
table_alias_list,
rendered_related_key)
if BACKWARD_COMPATIBILITY: if BACKWARD_COMPATIBILITY:
# XXX: Calling a private-ish method on column_map. # BBB: remove this branch of the condition, and the above
# This should never happen. It should be removed as soon as an # constant, when all zsql_methods have been adapted to return
# alternative exists. # the join queries properly separated by the
column_map._addJoinQuery(SQLQuery(rendered_related_key)) # RELATED_QUERY_SEPARATOR.
return None
# The rendered related key doesn't have the separators for each
# joined table, so we revert to doing implicit inner joins:
log.warning(msg + "\n\nAdding an Implicit Join Condition...")
column_map._addJoinQueryForColumn(self.column,
SQLQuery(rendered_related_key))
else: else:
return SQLExpression(self, where_expression=rendered_related_key) raise RuntimeError(msg)
return None
verifyClass(IRelatedKey, RelatedKey) verifyClass(IRelatedKey, RelatedKey)
##############################################################################
#
# Copyright (c) 2010 Nexedi SA and Contributors. All Rights Reserved.
# Leonardo Rochael Almeida <leonardo@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
# TODO:
# * collapse of parentheses around chains of inner-joins
# * indentation on rendering
SQL_LIST_SEPARATOR = ', '
SQL_SELECT_ALIAS_FORMAT = '%s AS `%s`'
from Products.ZSQLCatalog.Query.SQLQuery import SQLQuery
def escapeTable(table):
return "`%s`" % table.replace('`', r'\`')
class TableDefinition(object):
"""Base class for all TableDefinition objects. Used for
typechecking (which should become interface checking later) and
for dumping common code (if there is any).
TableDefinition objects describe the table aliasing and joining on
the "FROM" expression of an SQL query. It is supposed to be
decoded by an SQLExpressionObject into a string.
"""
def checkTableAliases(self, current_aliases=None):
"""Check that aliases defined in this table definition don't try
to alias different tables to the same name.
Add all aliases defined to the current_aliases mapping if it is passed in.
"""
if current_aliases is None:
current_aliases = {}
self._checkTableAliases(current_aliases)
def _checkTableAliases(self, current_aliases):
raise NotImplementedError('should be implemented by subclasses')
def render(self):
raise NotImplementedError('should be implemented by subclasses')
def getJoinConditionQueryList(self):
"""Return a list of SQLQuery objects containing all conditions
used in this table definition.
This is a deprecated method that is here only to accomodate the
fact that not all RelatedKey methods have been migrated.
"""
query_list = []
self._extendJoinConditionQueryList(query_list)
return query_list
def _extendJoinConditionQueryList(self, query_list):
raise NotImplementedError('should be implemented by subclasses')
def getSuperSet(self, other):
"""Checks if this TableDefinition is a subset of the other table
definition or vice-versa. Return which one is the superset.
Returns whichever is the superset of the other or None
"""
raise NotImplementedError('should be implemented by subclasses')
class PlaceHolderTableDefinition(TableDefinition):
"""Table Definition that simply holds an inner table definition and
delegates to it the rendering.
This object can be used when you need to change a table definition
in the future, (like replacing a simple table aliasing with an inner
join) but don't know who is going to be holding a reference to the
original table definition to replace it.
"""
def __init__(self, table_definition=None):
self.table_definition = table_definition
replace = __init__
def __repr__(self):
return '<%s for %r>' % (self.__class__.__name__, self.table_definition)
def _checkTableAliases(self, current_aliases):
assert self.table_definition is not None, "table definition wasn't set"
return self.table_definition._checkTableAliases(current_aliases)
def render(self):
assert self.table_definition is not None, "table definition wasn't set"
return self.table_definition.render()
def _extendJoinConditionQueryList(self, query_list):
# XXX _extendJoinConditionQueryList
#assert self.table_definition is not None, "table definition wasn't set"
if self.table_definition is not None:
return self.table_definition._extendJoinConditionQueryList(query_list)
def getSuperSet(self, other):
assert self.table_definition is not None, "table definition wasn't set"
return self.table_definition.getSuperSet(other)
class TableAlias(TableDefinition):
"""Definition of a table alias as a FROM expression"""
def __init__(self, table, alias=None):
self.table = table
self.alias = alias or table
def _checkTableAliases(self, current_aliases):
#table_name = current_aliases.setdefault(self.alias, self.table)
table_name = current_aliases.get(self.alias)
if table_name is None:
current_aliases[self.alias] = self.table
return
if table_name != self.table:
message = ("Attempted to alias both %r and %r to %r" %
(table_name, self.table, self.alias,))
else:
message = ("Attempted to alias %r to %r more than once" %
(self.table, self.alias,))
raise ValueError(message)
def render(self):
"""Render this table definition into an actual FROM expression"""
return SQL_SELECT_ALIAS_FORMAT % (self.table, self.alias)
def __repr__(self):
return '<%s %r AS %r>' % (self.__class__.__name__, self.table, self.alias)
def _extendJoinConditionQueryList(self, query_list):
pass
def __eq__(self, other):
return (isinstance(other, TableAlias) and
self.table == other.table and
self.alias == other.alias)
def getSuperSet(self, other):
"""A TableAlias is a subset of another table Alias if either:
- the other is an equivalent TableAlias
- the other is an InnerJoin where the left-side is an equivalent TableAlias
"""
if isinstance(other, TableAlias) and self == other:
# we're just like the other guy, we could return self or other
return self
# delegate the rest of the job to InnerJoin
return other.getSuperSet(self)
JOIN_FORMAT = """
(
%(left)s
%(join)s
%(right)s
ON
%(condition)s
)
""".strip()
class Join(TableDefinition):
JOIN_TYPE = None
def __init__(self, left_tabledef, right_tabledef, condition):
assert self.JOIN_TYPE, ('Join must be subclassed and self.JOIN_TYPE '
'must be defined.')
assert isinstance(left_tabledef, (TableDefinition, None.__class__))
assert isinstance(right_tabledef, (TableDefinition, None.__class__))
self.left_tabledef = left_tabledef
self.right_tabledef = right_tabledef
# perhaps assert condition is an SQLExpression?
self.condition = condition
def render(self):
"""Render the join as an actual FROM expression, delegating
the rendering of each table to its own object.
"""
assert None not in (self.left_tabledef, self.right_tabledef, self.condition)
return JOIN_FORMAT % dict(left=self.left_tabledef.render(),
right=self.right_tabledef.render(),
join=self.JOIN_TYPE,
condition=self.condition)
def _checkTableAliases(self, current_aliases):
self.left_tabledef._checkTableAliases(current_aliases)
self.right_tabledef._checkTableAliases(current_aliases)
def __repr__(self):
return '<%s of %r and %r on %r>' % (self.__class__.__name__,
self.left_tabledef,
self.right_tabledef,
self.condition)
def _extendJoinConditionQueryList(self, query_list):
self.left_tabledef._extendJoinConditionQueryList(query_list)
self.right_tabledef._extendJoinConditionQueryList(query_list)
query_list.append(SQLQuery(self.condition))
def getSuperSet(self, other):
return None
class InnerJoin(Join):
"""Definition of an inner-join as a FROM expression"""
JOIN_TYPE = "INNER JOIN"
def getSuperSet(self, other):
"""This InnerJoin is a superset of another TableDefinition if either:
- other is a TableAlias (or None) equal to our
left_side. I.e. "other" is at the end of it's inner-join chain.
- other is an InnerJoin, and it's left-side is equal to our
left-side (both TableAliases or None), and our right-side is a
super-set of it's right-side.
"""
if self.left_tabledef == other:
# other and left-side are both None or matching TableAliases
return self
if (isinstance(other, InnerJoin) and
self.left_tabledef == other.left_tabledef):
# our left-sides match. If one of our right sides is a superset of the
# other right side, then we found the superset
sub_superset = self.right_tabledef.getSuperSet(other.right_tabledef)
if sub_superset is self.right_tabledef:
return self
elif sub_superset is other.right_tabledef:
return other
return None
return None
class LeftJoin(InnerJoin):
"""Definition of a left-join as a FROM expression"""
JOIN_TYPE = "LEFT JOIN"
def _extendJoinConditionQueryList(self, query_list):
""" The condition from a left-join cannot be meaningfully
extracted to be used in an implicit Inner Join, as is done when a
query contains a related key that is not formatted to separate the
join conditions for each related table."""
raise RuntimeError("Attempted to collapse table definition for implicit "
"inner join, but this table definition contains a Left "
"Join: %r" % self)
class LegacyTableDefinition(TableDefinition):
"""Table Definition used when a from_expression is passed explicitly.
Mostly used for manual left-join definitions. Deprecated
"""
def __init__(self, from_expression, table_alias_map):
self.from_expression = from_expression
self.table_alias_map = table_alias_map
def render(self):
from_expression_dict = self.from_expression
table_alias_map = self.table_alias_map
from_expression = SQL_LIST_SEPARATOR.join(
from_expression_dict.get(alias, '`%s` AS `%s`' % (table, alias))
for alias, table in table_alias_map.iteritems())
return from_expression
...@@ -284,18 +284,3 @@ class IColumnMap(Interface): ...@@ -284,18 +284,3 @@ class IColumnMap(Interface):
Return a copy of the table alias list for tables requiring a join with Return a copy of the table alias list for tables requiring a join with
catalog table. catalog table.
""" """
def getStraightJoinTableList():
"""
Returns the list of tables used this search and which
need to be joined with the main table using explicit
indices.
"""
def getLeftJoinTableList():
"""
Returns the list of tables used this search and which
need to be LEFT joined with the main table using explicit
indices.
"""
...@@ -202,6 +202,7 @@ class DummyCatalog(SQLCatalog): ...@@ -202,6 +202,7 @@ class DummyCatalog(SQLCatalog):
assert 'query_table' in kw assert 'query_table' in kw
assert 'table_0' in kw assert 'table_0' in kw
assert 'table_1' in kw assert 'table_1' in kw
assert 'AND' in kw.pop('RELATED_QUERY_SEPARATOR')
assert len(kw) == 4 assert len(kw) == 4
return '%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s' % kw return '%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s' % kw
...@@ -629,7 +630,7 @@ class TestSQLCatalog(unittest.TestCase): ...@@ -629,7 +630,7 @@ class TestSQLCatalog(unittest.TestCase):
select_dict = sql_expression.getSelectDict() select_dict = sql_expression.getSelectDict()
self.assertTrue('ambiguous_mapping' in select_dict, select_dict) self.assertTrue('ambiguous_mapping' in select_dict, select_dict)
self.assertTrue('bar' in select_dict['ambiguous_mapping'], select_dict['ambiguous_mapping']) self.assertTrue('bar' in select_dict['ambiguous_mapping'], select_dict['ambiguous_mapping'])
# Doted alias: table name must get stripped. This is required to have an # Dotted alias: table name must get stripped. This is required to have an
# upgrade path from old ZSQLCatalog versions where pre-mapped columns were # upgrade path from old ZSQLCatalog versions where pre-mapped columns were
# used in their select_expression. This must only happen in the # used in their select_expression. This must only happen in the
# "{column: None}" form, as otherwise it's the user explicitely asking for # "{column: None}" form, as otherwise it's the user explicitely asking for
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment