BUG#13126: When choosing join order for join with nested joins, don't produce join

orders that cannot be handled by the executioner.

mysql-test/r/bigint.result

-drop table if exists t1;
+drop table if exists t1, t2;
 select 0,256,00000000000000065536,2147483647,-2147483648,2147483648,+4294967296;
 0	256	00000000000000065536	2147483647	-2147483648	2147483648	4294967296
 0	256	65536	2147483647	-2147483648	2147483648	4294967296

mysql-test/r/join_nested.result

@@ -1403,3 +1403,67 @@ SELECT v2.x FROM v2 JOIN t2 ON e=b JOIN t3 ON e=c JOIN t4 USING(d);
 ERROR 42S22: Unknown column 'v2.x' in 'field list'
 DROP VIEW v1, v2;
 DROP TABLE t1, t2, t3, t4, t5, t6;
+create table t1 (id1 int(11) not null);
+insert into t1 values (1),(2);
+create table t2 (id2 int(11) not null);
+insert into t2 values (1),(2),(3),(4);
+create table t3 (id3 char(16) not null);
+insert into t3 values ('100');
+create table t4 (id2 int(11) not null, id3 char(16));
+create table t5 (id1 int(11) not null, key (id1));
+insert into t5 values (1),(2),(1);
+create view v1 as
+select t4.id3 from t4 join t2 on t4.id2 = t2.id2;
+select t1.id1 from t1 inner join (t3 left join v1 on t3.id3 = v1.id3);
+id1
+1
+2
+drop view v1;
+drop table t1, t2, t3, t4, t5;
+create table t0 (a int);
+insert into t0 values (0),(1),(2),(3);
+create table t1(a int);
+insert into t1 select A.a + 10*(B.a) from t0 A, t0 B;
+create table t2 (a int, b int);
+insert into t2 values (1,1), (2,2), (3,3);
+create table t3(a int, b int, filler char(200), key(a));
+insert into t3 select a,a,'filler' from t1;
+insert into t3 select a,a,'filler' from t1;
+create table t4 like t3;
+insert into t4 select * from t3;
+insert into t4 select * from t3;
+create table t5 like t4;
+insert into t5 select * from t4;
+insert into t5 select * from t4;
+create table t6 like t5;
+insert into t6 select * from t5;
+insert into t6 select * from t5;
+create table t7 like t6;
+insert into t7 select * from t6;
+insert into t7 select * from t6;
+explain select * from t4 join 
+t2 left join (t3 join t5 on t5.a=t3.b) on t3.a=t2.b where t4.a<=>t3.b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	X	
+1	SIMPLE	t3	ref	a	a	5	test.t2.b	X	
+1	SIMPLE	t5	ref	a	a	5	test.t3.b	X	
+1	SIMPLE	t4	ref	a	a	5	test.t3.b	X	Using where
+explain select * from (t4 join t6 on t6.a=t4.b) right join t3 on t4.a=t3.b
+join t2 left join (t5 join t7 on t7.a=t5.b) on t5.a=t2.b where t3.a<=>t2.b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	X	
+1	SIMPLE	t3	ref	a	a	5	test.t2.b	X	Using where
+1	SIMPLE	t4	ref	a	a	5	test.t3.b	X	
+1	SIMPLE	t6	ref	a	a	5	test.t4.b	X	
+1	SIMPLE	t5	ref	a	a	5	test.t2.b	X	
+1	SIMPLE	t7	ref	a	a	5	test.t5.b	X	
+explain select * from t2 left join
+(t3 left join (t4 join t6 on t6.a=t4.b) on t4.a=t3.b 
+join t5 on t5.a=t3.b) on t3.a=t2.b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	X	
+1	SIMPLE	t3	ref	a	a	5	test.t2.b	X	
+1	SIMPLE	t4	ref	a	a	5	test.t3.b	X	
+1	SIMPLE	t6	ref	a	a	5	test.t4.b	X	
+1	SIMPLE	t5	ref	a	a	5	test.t3.b	X	
+drop table t0, t1, t2, t4, t5, t6;

mysql-test/t/bigint.test

@@ -2,7 +2,7 @@
 # Initialize
 
 --disable_warnings
-drop table if exists t1;
+drop table if exists t1, t2;
 --enable_warnings
 
 #

mysql-test/t/join_nested.test

@@ -832,3 +832,71 @@ SELECT v2.x FROM v2 JOIN t2 ON e=b JOIN t3 ON e=c JOIN t4 USING(d);
 
 DROP VIEW v1, v2;
 DROP TABLE t1, t2, t3, t4, t5, t6;
+
+#
+# BUG#13126 -test case from bug report
+#
+create table t1 (id1 int(11) not null); 
+insert into t1 values (1),(2);
+
+create table t2 (id2 int(11) not null);
+insert into t2 values (1),(2),(3),(4);
+
+create table t3 (id3 char(16) not null);
+insert into t3 values ('100');
+
+create table t4 (id2 int(11) not null, id3 char(16));
+
+create table t5 (id1 int(11) not null, key (id1));
+insert into t5 values (1),(2),(1);
+
+create view v1 as
+  select t4.id3 from t4 join t2 on t4.id2 = t2.id2;
+
+select t1.id1 from t1 inner join (t3 left join v1 on t3.id3 = v1.id3);
+
+drop view v1;
+drop table t1, t2, t3, t4, t5;
+
+create table t0 (a int);
+insert into t0 values (0),(1),(2),(3);
+create table t1(a int);
+insert into t1 select A.a + 10*(B.a) from t0 A, t0 B;
+
+create table t2 (a int, b int);
+insert into t2 values (1,1), (2,2), (3,3);
+
+create table t3(a int, b int, filler char(200), key(a));
+insert into t3 select a,a,'filler' from t1;
+insert into t3 select a,a,'filler' from t1;
+
+create table t4 like t3;
+insert into t4 select * from t3;
+insert into t4 select * from t3;
+
+create table t5 like t4;
+insert into t5 select * from t4;
+insert into t5 select * from t4;
+
+create table t6 like t5;
+insert into t6 select * from t5;
+insert into t6 select * from t5;
+
+create table t7 like t6;
+insert into t7 select * from t6;
+insert into t7 select * from t6;
+
+--replace_column 9 X
+explain select * from t4 join 
+  t2 left join (t3 join t5 on t5.a=t3.b) on t3.a=t2.b where t4.a<=>t3.b;
+
+--replace_column 9 X
+explain select * from (t4 join t6 on t6.a=t4.b) right join t3 on t4.a=t3.b
+  join t2 left join (t5 join t7 on t7.a=t5.b) on t5.a=t2.b where t3.a<=>t2.b;
+
+--replace_column 9 X
+explain select * from t2 left join
+  (t3 left join (t4 join t6 on t6.a=t4.b) on t4.a=t3.b 
+   join t5 on t5.a=t3.b) on t3.a=t2.b;
+
+drop table t0, t1, t2, t4, t5, t6;

sql/mysql_priv.h

@@ -44,6 +44,11 @@
 typedef ulonglong table_map;          /* Used for table bits in join */
 typedef Bitmap<64> key_map;           /* Used for finding keys */
 typedef ulong key_part_map;           /* Used for finding key parts */
+/*
+  Used for nested join bits within a scope of a join (applicable to non-unary
+  nested joins that have not been simplified away)
+*/
+typedef ulonglong nested_join_map;
 
 /* query_id */
 typedef ulonglong query_id_t;

sql/sql_prepare.cc

@@ -2105,8 +2105,6 @@ void reinit_stmt_before_use(THD *thd, LEX *lex)
       were closed in the end of previous prepare or execute call.
     */
     tables->table= 0;
-    if (tables->nested_join)
-      tables->nested_join->counter= 0;
 
     if (tables->prep_on_expr)
     {

sql/sql_select.h

@@ -136,6 +136,8 @@ typedef struct st_join_table {
   TABLE_REF	ref;
   JOIN_CACHE	cache;
   JOIN		*join;
+  /* Bitmap of nested joins this table is part of */
+  nested_join_map embedding_map;
   
   void cleanup();
 } JOIN_TAB;
@@ -193,6 +195,13 @@ class JOIN :public Sql_alloc
   */
   ha_rows  fetch_limit;
   POSITION positions[MAX_TABLES+1],best_positions[MAX_TABLES+1];
+  
+  /* 
+    Bitmap of nested joins embedding the position at the end of the current 
+    partial join (valid only during join optimizer run).
+  */
+  nested_join_map cur_embedding_map;
+
   double   best_read;
   List<Item> *fields;
   List<Cached_item> group_fields, group_fields_cache;

sql/table.h

@@ -757,7 +757,15 @@ typedef struct st_nested_join
   table_map         used_tables;     /* bitmap of tables in the nested join */
   table_map         not_null_tables; /* tables that rejects nulls           */
   struct st_join_table *first_nested;/* the first nested table in the plan  */
-  uint              counter;         /* to count tables in the nested join  */
+  /* 
+    Used to count tables in the nested join in 2 isolated places:
+    1. In make_outerjoin_info(). 
+    2. check_interleaving_with_nj/restore_prev_nj_state (these are called
+       by the join optimizer. 
+    Before each use the counters are zeroed by reset_nj_counters.
+  */
+  uint              counter;
+  nested_join_map   nj_map;          /* Bit used to identify this nested join*/
 } NESTED_JOIN;