MDEV-5123 Remove duplicated conditions pushed both to join_tab->select_cond...

MDEV-5123 Remove duplicated conditions pushed both to join_tab->select_cond and join_tab->cache_select->cond for blocked joins.
    
BNL and BNLH joins pre-filter the records from a joined table via JOIN_TAB::cache_select->cond.
There is no need to re-evaluate the same conditions via JOIN_TAB::select_cond. This patch removes
the duplicated conditions from the top-level conjuncts of each pushed condition.

The added "Using where" in few EXPLAINs is due to taking into account tab->cache_select->cond
in addition to tab->select_cond in JOIN::save_explain_data_intern.

mysql-test/r/join_cache.result

@@ -5058,7 +5058,7 @@ EXPLAIN
 SELECT * FROM t1,t2 WHERE t1.a < 3 and t2.a IS NULL;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	3	Using where; Using index
-1	SIMPLE	t2	hash_range	idx	#hash#idx:idx	5:5	const	4	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNLH join)
+1	SIMPLE	t2	hash_range	idx	#hash#idx:idx	5:5	const	4	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNLH join)
 SELECT * FROM t1,t2 WHERE t1.a < 3 and t2.a IS NULL;
 a	a	b
 1	NULL	10
@@ -5637,4 +5637,51 @@ c
 set join_buffer_size=default;
 set optimizer_switch=@tmp_optimizer_switch;
 DROP table t1,t2,t3;
+#
+# MDEV-5123 Remove duplicated conditions pushed both to join_tab->select_cond and join_tab->cache_select->cond for blocked joins.
+#
+set join_cache_level=default;
+set expensive_subquery_limit=0;
+create table t1 (c1 int);
+create table t2 (c2 int);
+create table t3 (c3 int);
+insert into t1 values (1), (2);
+insert into t2 values (1), (2);
+insert into t3 values (2);
+explain
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+2	UNCACHEABLE SUBQUERY	t3	system	NULL	NULL	NULL	NULL	1	
+set @counter=0;
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1);
+count(*)
+2
+select @counter;
+@counter
+2
+explain
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and
+c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1) and
+c2 / 2 = 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+2	UNCACHEABLE SUBQUERY	t3	system	NULL	NULL	NULL	NULL	1	
+set @counter=0;
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and
+c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1) and
+c2 / 2 = 1;
+count(*)
+1
+select @counter;
+@counter
+2
+drop table t1,t2,t3;
+set expensive_subquery_limit=default;
 set @@optimizer_switch=@save_optimizer_switch;

mysql-test/r/limit_rows_examined.result

@@ -747,27 +747,27 @@ ERROR HY000: Sort aborted:
 SHOW STATUS LIKE 'Handler_read%';
 Variable_name	Value
 Handler_read_first	0
-Handler_read_key	4
+Handler_read_key	5
 Handler_read_last	0
 Handler_read_next	0
 Handler_read_prev	0
 Handler_read_rnd	0
 Handler_read_rnd_deleted	0
-Handler_read_rnd_next	52
+Handler_read_rnd_next	46
 SHOW STATUS LIKE 'Handler_tmp%';
 Variable_name	Value
 Handler_tmp_update	0
-Handler_tmp_write	66
+Handler_tmp_write	70
 FLUSH STATUS;
 SELECT a AS field1, alias2.d AS field2, alias2.f AS field3, alias2.e AS field4, b AS field5
 FROM t1, t2 AS alias2, t2 AS alias3 
 WHERE alias3.c IN ( SELECT 1 UNION SELECT 6 ) 
 GROUP BY field1, field2, field3, field4, field5
-LIMIT ROWS EXAMINED 250;
+LIMIT ROWS EXAMINED 124;
 field1	field2	field3	field4	field5
 00:21:38	06:07:10	a	2007-06-08 04:35:26	2007-05-28 00:00:00
 Warnings:
-Warning	1931	Query execution was interrupted. The query examined at least 251 rows, which exceeds LIMIT ROWS EXAMINED (250). The query result may be incomplete.
+Warning	1931	Query execution was interrupted. The query examined at least 125 rows, which exceeds LIMIT ROWS EXAMINED (124). The query result may be incomplete.
 SHOW STATUS LIKE 'Handler_read%';
 Variable_name	Value
 Handler_read_first	0
@@ -777,11 +777,11 @@ Handler_read_next	0
 Handler_read_prev	0
 Handler_read_rnd	2
 Handler_read_rnd_deleted	1
-Handler_read_rnd_next	110
+Handler_read_rnd_next	47
 SHOW STATUS LIKE 'Handler_tmp%';
 Variable_name	Value
 Handler_tmp_update	0
-Handler_tmp_write	133
+Handler_tmp_write	70
 drop table t1, t2;
 
 MDEV-161 LIMIT_ROWS EXAMINED: query with the limit and NOT EXISTS, without GROUP BY or aggregate,

mysql-test/r/range.result

@@ -221,27 +221,27 @@ update t1 set y=x;
 explain select * from t1, t1 t2 where t1.y = 8 and t2.x between 7 and t1.y+0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 8 and t2.x >= 7 and t2.x <= t1.y+0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x between t1.y-1 and t1.y+1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Using where; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x >= t1.y-1 and t2.x <= t1.y+1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Using where; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x between 0 and t1.y;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x >= 0 and t2.x <= t1.y;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Using join buffer (flat, BNL join)
 explain select count(*) from t1 where x in (1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	x	x	5	const	1	Using index

mysql-test/r/range_mrr_icp.result

@@ -223,27 +223,27 @@ update t1 set y=x;
 explain select * from t1, t1 t2 where t1.y = 8 and t2.x between 7 and t1.y+0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 8 and t2.x >= 7 and t2.x <= t1.y+0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x between t1.y-1 and t1.y+1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x >= t1.y-1 and t2.x <= t1.y+1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	3	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x between 0 and t1.y;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNL join)
 explain select * from t1, t1 t2 where t1.y = 2 and t2.x >= 0 and t2.x <= t1.y;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	y	y	5	const	1	
-1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Rowid-ordered scan; Using join buffer (flat, BNL join)
+1	SIMPLE	t2	range	x	x	5	NULL	2	Using index condition; Using where; Rowid-ordered scan; Using join buffer (flat, BNL join)
 explain select count(*) from t1 where x in (1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	x	x	5	const	1	Using index

mysql-test/r/subselect_mat.result

@@ -1474,7 +1474,7 @@ EXPLAIN SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	
 1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	4	func	1	
-2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using index condition; Rowid-ordered scan
+2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using index condition; Using where; Rowid-ordered scan
 SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 pk
 2
@@ -2158,7 +2158,7 @@ EXPLAIN SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	
 1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	4	func	1	
-2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using index condition
+2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using index condition; Using where
 SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 pk
 2

mysql-test/r/subselect_sj.result

@@ -803,7 +803,7 @@ EXPLAIN EXTENDED SELECT pk FROM t1 WHERE (a, b) IN (SELECT a, b FROM t2 WHERE pk
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	
 1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	11	func,func	1	100.00	
-2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	100.00	Using index condition; Rowid-ordered scan
+2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	100.00	Using index condition; Using where; Rowid-ordered scan
 Warnings:
 Note	1003	select `test`.`t1`.`pk` AS `pk` from `test`.`t1` semi join (`test`.`t2`) where ((`test`.`t2`.`pk` > 0))
 SELECT pk FROM t1 WHERE (a, b) IN (SELECT a, b FROM t2 WHERE pk > 0);

mysql-test/r/subselect_sj_jcl6.result

@@ -816,7 +816,7 @@ EXPLAIN EXTENDED SELECT pk FROM t1 WHERE (a, b) IN (SELECT a, b FROM t2 WHERE pk
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	
 1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	11	func,func	1	100.00	
-2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	100.00	Using index condition; Rowid-ordered scan
+2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	100.00	Using index condition; Using where; Rowid-ordered scan
 Warnings:
 Note	1003	select `test`.`t1`.`pk` AS `pk` from `test`.`t1` semi join (`test`.`t2`) where ((`test`.`t2`.`pk` > 0))
 SELECT pk FROM t1 WHERE (a, b) IN (SELECT a, b FROM t2 WHERE pk > 0);

mysql-test/r/subselect_sj_mat.result

@@ -1511,7 +1511,7 @@ EXPLAIN SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	
 1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	4	func	1	
-2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using index condition; Rowid-ordered scan
+2	MATERIALIZED	t2	range	PRIMARY	PRIMARY	4	NULL	2	Using index condition; Using where; Rowid-ordered scan
 SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 pk
 2

mysql-test/t/join_cache.test

@@ -3636,5 +3636,49 @@ set optimizer_switch=@tmp_optimizer_switch;
 
 DROP table t1,t2,t3;
 
+--echo #
+--echo # MDEV-5123 Remove duplicated conditions pushed both to join_tab->select_cond and join_tab->cache_select->cond for blocked joins.
+--echo #
+
+set join_cache_level=default;
+set expensive_subquery_limit=0;
+
+create table t1 (c1 int);
+create table t2 (c2 int);
+create table t3 (c3 int);
+
+insert into t1 values (1), (2);
+insert into t2 values (1), (2);
+insert into t3 values (2);
+
+explain
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1);
+
+set @counter=0;
+
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1);
+
+select @counter;
+
+explain
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and
+      c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1) and
+      c2 / 2 = 1;
+
+set @counter=0;
+
+select count(*) from t1 straight_join t2
+where c1 = c2-0 and
+      c2 <= (select max(c3) from t3 where c3 = 2 and @counter:=@counter+1) and
+      c2 / 2 = 1;
+
+select @counter;
+
+drop table t1,t2,t3;
+set expensive_subquery_limit=default;
+
 # this must be the last command in the file
 set @@optimizer_switch=@save_optimizer_switch;

mysql-test/t/limit_rows_examined.test

@@ -502,7 +502,7 @@ SELECT a AS field1, alias2.d AS field2, alias2.f AS field3, alias2.e AS field4,
 FROM t1, t2 AS alias2, t2 AS alias3 
 WHERE alias3.c IN ( SELECT 1 UNION SELECT 6 ) 
 GROUP BY field1, field2, field3, field4, field5
-LIMIT ROWS EXAMINED 250;
+LIMIT ROWS EXAMINED 124;
 SHOW STATUS LIKE 'Handler_read%';
 SHOW STATUS LIKE 'Handler_tmp%';
 

sql/sql_select.cc

@@ -10535,6 +10535,87 @@ restart:
   }
 }
 
+/**
+  Remove pushdown conditions that are already checked by the scan phase
+  of BNL/BNLH joins.
+
+  @note
+  If the single-table condition for this table will be used by a
+  blocked join to pre-filter this table's rows, there is no need
+  to re-check the same single-table condition for each joined record.
+
+  This method removes from JOIN_TAB::select_cond and JOIN_TAB::select::cond
+  all top-level conjuncts that also appear in in JOIN_TAB::cache_select::cond.
+*/
+
+void JOIN_TAB::remove_redundant_bnl_scan_conds()
+{
+  if (!(select_cond && cache_select && cache &&
+        (cache->get_join_alg() == JOIN_CACHE::BNL_JOIN_ALG ||
+         cache->get_join_alg() == JOIN_CACHE::BNLH_JOIN_ALG)))
+    return;
+
+
+  /*
+    select->cond is not processed separately. This method assumes it is always
+    the same as select_cond.
+  */
+  DBUG_ASSERT(!select || !select->cond ||
+              (select->cond == select_cond));
+
+  if (is_cond_and(select_cond))
+  {
+    List_iterator<Item> pushed_cond_li(*((Item_cond*) select_cond)->argument_list());
+    Item *pushed_item;
+    Item_cond_and *reduced_select_cond= new Item_cond_and;
+
+    if (is_cond_and(cache_select->cond))
+    {
+      List_iterator<Item> scan_cond_li(*((Item_cond*) cache_select->cond)->argument_list());
+      Item *scan_item;
+      while ((pushed_item= pushed_cond_li++))
+      {
+        bool found= false;
+        scan_cond_li.rewind();
+        while ((scan_item= scan_cond_li++))
+        {
+          if (pushed_item->eq(scan_item, 0))
+          {
+            found= true;
+            break;
+          }
+        }
+        if (!found)
+          reduced_select_cond->add(pushed_item);
+      }
+    }
+    else
+    {
+      while ((pushed_item= pushed_cond_li++))
+      {
+        if (!pushed_item->eq(cache_select->cond, 0))
+          reduced_select_cond->add(pushed_item);
+      }
+    }
+
+    /*
+      JOIN_CACHE::check_match uses JOIN_TAB::select->cond instead of
+      JOIN_TAB::select_cond. set_cond() sets both pointers.
+    */
+    if (reduced_select_cond->argument_list()->is_empty())
+      set_cond(NULL);
+    else if (reduced_select_cond->argument_list()->elements == 1)
+      set_cond(reduced_select_cond->argument_list()->head());
+    else
+    {
+      reduced_select_cond->quick_fix_field();
+      set_cond(reduced_select_cond);
+    }
+  }
+  else if (select_cond->eq(cache_select->cond, 0))
+    set_cond(NULL);
+}
+
 
 /*
   Plan refinement stage: do various setup things for the executor
@@ -10786,6 +10867,15 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
       abort();
       /* purecov: end */
     }
+
+    tab->remove_redundant_bnl_scan_conds();
+    DBUG_EXECUTE("where",
+                 char buff[256];
+                 String str(buff,sizeof(buff),system_charset_info);
+                 str.length(0);
+                 str.append(tab->table? tab->table->alias.c_ptr() :"<no_table_name>");
+                 str.append(" final_pushdown_cond");
+                 print_where(tab->select_cond, str.c_ptr_safe(), QT_ORDINARY););
   }
   uint n_top_tables= join->join_tab_ranges.head()->end -  
                      join->join_tab_ranges.head()->start;
@@ -23201,7 +23291,8 @@ int JOIN::save_explain_data_intern(Explain_query *output, bool need_tmp_table,
             eta->push_extra(ET_RANGE_CHECKED_FOR_EACH_RECORD);
             eta->range_checked_map= tab->keys;
 	  }
-	  else if (tab->select->cond)
+	  else if (tab->select->cond ||
+                   (tab->cache_select && tab->cache_select->cond))
           {
             const COND *pushed_cond= tab->table->file->pushed_cond;
 

sql/sql_select.h

@@ -544,6 +544,7 @@ typedef struct st_join_table {
             !(used_sjm_lookup_tables & ~emb_sj_nest->sj_inner_tables));
   }
 
+  void remove_redundant_bnl_scan_conds();
 } JOIN_TAB;