BUG#836507: Crash in setup_sj_materialization_part1() with semijoin+materialization

- setup_sj_materialization() code failed to take into account that it can be that 
  the first [in join order ordering] table inside semi-join-materialization nest 
  is also an inner table wrt an outer join (that is embedded in the semi-join).  
  This can happen when all of the tables that are inside the semi-join but not inside
  the outer join are constant.
- Made a trivial to not assume that table's embedding join nest is the semi-join 
  nest: instead, walk up the outer join nests until we reach the semi-join nest.

mysql-test/r/subselect_mat.result

@@ -1601,6 +1601,32 @@ FROM t4
 );
 a
 DROP TABLE t1, t2, t3, t4;
+#
+# BUG#836507: Crash in setup_sj_materialization_part1() with semijoin+materialization
+#
+CREATE TABLE t1 (a int) ;
+INSERT IGNORE INTO t1 VALUES (1),(1);
+CREATE TABLE t2 (a int);
+INSERT INTO t2 VALUES (1);
+CREATE TABLE t3 (a int);
+CREATE TABLE t4 (a int);
+INSERT INTO t4 VALUES (2),(2);
+CREATE TABLE t5 (a int);
+INSERT INTO t5 VALUES (1);
+SELECT * FROM t1
+WHERE (a) IN (
+SELECT t5.a
+FROM (
+t2
+LEFT JOIN ( t3 , t4 )
+ON 1 = 1
+)
+JOIN t5
+);
+a
+1
+1
+DROP TABLE t1,t2,t3,t4,t5;
 set optimizer_switch=@subselect_sj_mat_tmp;
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';

mysql-test/r/subselect_sj_mat.result

@@ -1639,4 +1639,30 @@ FROM t4
 );
 a
 DROP TABLE t1, t2, t3, t4;
+#
+# BUG#836507: Crash in setup_sj_materialization_part1() with semijoin+materialization
+#
+CREATE TABLE t1 (a int) ;
+INSERT IGNORE INTO t1 VALUES (1),(1);
+CREATE TABLE t2 (a int);
+INSERT INTO t2 VALUES (1);
+CREATE TABLE t3 (a int);
+CREATE TABLE t4 (a int);
+INSERT INTO t4 VALUES (2),(2);
+CREATE TABLE t5 (a int);
+INSERT INTO t5 VALUES (1);
+SELECT * FROM t1
+WHERE (a) IN (
+SELECT t5.a
+FROM (
+t2
+LEFT JOIN ( t3 , t4 )
+ON 1 = 1
+)
+JOIN t5
+);
+a
+1
+1
+DROP TABLE t1,t2,t3,t4,t5;
 set optimizer_switch=@subselect_sj_mat_tmp;

mysql-test/t/subselect_sj_mat.test

@@ -1289,6 +1289,35 @@ HAVING a IN (
 );
 DROP TABLE t1, t2, t3, t4;
 
+--echo #
+--echo # BUG#836507: Crash in setup_sj_materialization_part1() with semijoin+materialization
+--echo #
+CREATE TABLE t1 (a int) ;
+INSERT IGNORE INTO t1 VALUES (1),(1);
+
+CREATE TABLE t2 (a int);
+INSERT INTO t2 VALUES (1);
+
+CREATE TABLE t3 (a int);
+
+CREATE TABLE t4 (a int);
+INSERT INTO t4 VALUES (2),(2);
+
+CREATE TABLE t5 (a int);
+INSERT INTO t5 VALUES (1);
+
+SELECT * FROM t1
+WHERE (a) IN (
+        SELECT t5.a
+        FROM (
+                t2
+                LEFT JOIN ( t3 , t4 )
+                ON 1 = 1
+        )
+        JOIN t5
+);
+
+DROP TABLE t1,t2,t3,t4,t5;
 
 set optimizer_switch=@subselect_sj_mat_tmp;
 

sql/opt_subselect.cc

@@ -2935,6 +2935,11 @@ bool setup_sj_materialization_part1(JOIN_TAB *sjm_tab)
   DBUG_ENTER("setup_sj_materialization");
   JOIN_TAB *tab= sjm_tab->bush_children->start;
   TABLE_LIST *emb_sj_nest= tab->table->pos_in_table_list->embedding;
+  
+  /* Walk out of outer join nests until we reach the semi-join nest we're in */
+  while (!emb_sj_nest->sj_mat_info)
+    emb_sj_nest= emb_sj_nest->embedding;
+
   SJ_MATERIALIZATION_INFO *sjm= emb_sj_nest->sj_mat_info;
   THD *thd= tab->join->thd;
   /* First the calls come to the materialization function */
@@ -2983,6 +2988,9 @@ bool setup_sj_materialization_part2(JOIN_TAB *sjm_tab)
   DBUG_ENTER("setup_sj_materialization_part2");
   JOIN_TAB *tab= sjm_tab->bush_children->start;
   TABLE_LIST *emb_sj_nest= tab->table->pos_in_table_list->embedding;
+  /* Walk out of outer join nests until we reach the semi-join nest we're in */
+  while (!emb_sj_nest->sj_mat_info)
+    emb_sj_nest= emb_sj_nest->embedding;
   SJ_MATERIALIZATION_INFO *sjm= emb_sj_nest->sj_mat_info;
   THD *thd= tab->join->thd;
   uint i;