Fix for LP BUG#859375 and LP BUG#887458.

Stop attempts to apply IN/ALL/ANY optimizations to so called "fake_select"
(used for ordering and filtering results of union) in union subquery execution.

mysql-test/r/subselect.result

@@ -5681,4 +5681,51 @@ select a from t1 where a in (select a from t1 where a in (select a from t1 where
 a
 1
 drop table t1;
+#
+# LP bug #859375: Assertion `0' failed in st_select_lex_unit::optimize
+# with view , UNION and prepared statement (rewriting fake_select
+# condition).
+#
+CREATE TABLE t1 ( f1 int NOT NULL, f4 varchar(1) NOT NULL) ;
+INSERT INTO t1 VALUES (6,'d'),(7,'y');
+CREATE TABLE t2 ( f1 int NOT NULL, f2 int NOT NULL) ;
+INSERT INTO t2 VALUES (10,7);
+CREATE VIEW v2 AS SELECT * FROM t2;
+PREPARE st1 FROM "
+        SELECT *
+        FROM t1
+        LEFT JOIN v2 ON ( v2.f2 = t1.f1 )
+        WHERE v2.f1 NOT IN (
+                SELECT 1 UNION
+                SELECT 247
+        )
+";
+EXECUTE st1;
+f1	f4	f1	f2
+7	y	10	7
+deallocate prepare st1;
+DROP VIEW v2;
+DROP TABLE t1,t2;
+#
+# LP bug #887458 Crash in subselect_union_engine::no_rows with
+# double UNION and join_cache_level=3,8
+# (IN/ALL/ANY optimizations should not be applied to fake_select)
+CREATE TABLE t2 ( a int, b varchar(1)) ;
+INSERT IGNORE INTO t2 VALUES (8,'y'),(8,'y');
+CREATE TABLE t1 ( b varchar(1)) ;
+INSERT IGNORE INTO t1 VALUES (NULL),(NULL);
+set @save_join_cache_level=@@join_cache_level;
+SET SESSION join_cache_level=3;
+SELECT *
+FROM t1, t2
+WHERE t2.b IN (
+SELECT 'm' UNION
+SELECT 'm'
+) OR t1.b <> SOME (
+SELECT 'v' UNION
+SELECT 't'
+);
+b	a	b
+set @@join_cache_level= @save_join_cache_level;
+drop table t1,t2;
 set optimizer_switch=@subselect_tmp;

mysql-test/r/subselect_no_mat.result

@@ -5680,6 +5680,53 @@ select a from t1 where a in (select a from t1 where a in (select a from t1 where
 a
 1
 drop table t1;
+#
+# LP bug #859375: Assertion `0' failed in st_select_lex_unit::optimize
+# with view , UNION and prepared statement (rewriting fake_select
+# condition).
+#
+CREATE TABLE t1 ( f1 int NOT NULL, f4 varchar(1) NOT NULL) ;
+INSERT INTO t1 VALUES (6,'d'),(7,'y');
+CREATE TABLE t2 ( f1 int NOT NULL, f2 int NOT NULL) ;
+INSERT INTO t2 VALUES (10,7);
+CREATE VIEW v2 AS SELECT * FROM t2;
+PREPARE st1 FROM "
+        SELECT *
+        FROM t1
+        LEFT JOIN v2 ON ( v2.f2 = t1.f1 )
+        WHERE v2.f1 NOT IN (
+                SELECT 1 UNION
+                SELECT 247
+        )
+";
+EXECUTE st1;
+f1	f4	f1	f2
+7	y	10	7
+deallocate prepare st1;
+DROP VIEW v2;
+DROP TABLE t1,t2;
+#
+# LP bug #887458 Crash in subselect_union_engine::no_rows with
+# double UNION and join_cache_level=3,8
+# (IN/ALL/ANY optimizations should not be applied to fake_select)
+CREATE TABLE t2 ( a int, b varchar(1)) ;
+INSERT IGNORE INTO t2 VALUES (8,'y'),(8,'y');
+CREATE TABLE t1 ( b varchar(1)) ;
+INSERT IGNORE INTO t1 VALUES (NULL),(NULL);
+set @save_join_cache_level=@@join_cache_level;
+SET SESSION join_cache_level=3;
+SELECT *
+FROM t1, t2
+WHERE t2.b IN (
+SELECT 'm' UNION
+SELECT 'm'
+) OR t1.b <> SOME (
+SELECT 'v' UNION
+SELECT 't'
+);
+b	a	b
+set @@join_cache_level= @save_join_cache_level;
+drop table t1,t2;
 set optimizer_switch=@subselect_tmp;
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';

mysql-test/r/subselect_no_opts.result

@@ -5676,5 +5676,52 @@ select a from t1 where a in (select a from t1 where a in (select a from t1 where
 a
 1
 drop table t1;
+#
+# LP bug #859375: Assertion `0' failed in st_select_lex_unit::optimize
+# with view , UNION and prepared statement (rewriting fake_select
+# condition).
+#
+CREATE TABLE t1 ( f1 int NOT NULL, f4 varchar(1) NOT NULL) ;
+INSERT INTO t1 VALUES (6,'d'),(7,'y');
+CREATE TABLE t2 ( f1 int NOT NULL, f2 int NOT NULL) ;
+INSERT INTO t2 VALUES (10,7);
+CREATE VIEW v2 AS SELECT * FROM t2;
+PREPARE st1 FROM "
+        SELECT *
+        FROM t1
+        LEFT JOIN v2 ON ( v2.f2 = t1.f1 )
+        WHERE v2.f1 NOT IN (
+                SELECT 1 UNION
+                SELECT 247
+        )
+";
+EXECUTE st1;
+f1	f4	f1	f2
+7	y	10	7
+deallocate prepare st1;
+DROP VIEW v2;
+DROP TABLE t1,t2;
+#
+# LP bug #887458 Crash in subselect_union_engine::no_rows with
+# double UNION and join_cache_level=3,8
+# (IN/ALL/ANY optimizations should not be applied to fake_select)
+CREATE TABLE t2 ( a int, b varchar(1)) ;
+INSERT IGNORE INTO t2 VALUES (8,'y'),(8,'y');
+CREATE TABLE t1 ( b varchar(1)) ;
+INSERT IGNORE INTO t1 VALUES (NULL),(NULL);
+set @save_join_cache_level=@@join_cache_level;
+SET SESSION join_cache_level=3;
+SELECT *
+FROM t1, t2
+WHERE t2.b IN (
+SELECT 'm' UNION
+SELECT 'm'
+) OR t1.b <> SOME (
+SELECT 'v' UNION
+SELECT 't'
+);
+b	a	b
+set @@join_cache_level= @save_join_cache_level;
+drop table t1,t2;
 set optimizer_switch=@subselect_tmp;
 set @optimizer_switch_for_subselect_test=null;

mysql-test/r/subselect_no_scache.result

@@ -5685,6 +5685,53 @@ select a from t1 where a in (select a from t1 where a in (select a from t1 where
 a
 1
 drop table t1;
+#
+# LP bug #859375: Assertion `0' failed in st_select_lex_unit::optimize
+# with view , UNION and prepared statement (rewriting fake_select
+# condition).
+#
+CREATE TABLE t1 ( f1 int NOT NULL, f4 varchar(1) NOT NULL) ;
+INSERT INTO t1 VALUES (6,'d'),(7,'y');
+CREATE TABLE t2 ( f1 int NOT NULL, f2 int NOT NULL) ;
+INSERT INTO t2 VALUES (10,7);
+CREATE VIEW v2 AS SELECT * FROM t2;
+PREPARE st1 FROM "
+        SELECT *
+        FROM t1
+        LEFT JOIN v2 ON ( v2.f2 = t1.f1 )
+        WHERE v2.f1 NOT IN (
+                SELECT 1 UNION
+                SELECT 247
+        )
+";
+EXECUTE st1;
+f1	f4	f1	f2
+7	y	10	7
+deallocate prepare st1;
+DROP VIEW v2;
+DROP TABLE t1,t2;
+#
+# LP bug #887458 Crash in subselect_union_engine::no_rows with
+# double UNION and join_cache_level=3,8
+# (IN/ALL/ANY optimizations should not be applied to fake_select)
+CREATE TABLE t2 ( a int, b varchar(1)) ;
+INSERT IGNORE INTO t2 VALUES (8,'y'),(8,'y');
+CREATE TABLE t1 ( b varchar(1)) ;
+INSERT IGNORE INTO t1 VALUES (NULL),(NULL);
+set @save_join_cache_level=@@join_cache_level;
+SET SESSION join_cache_level=3;
+SELECT *
+FROM t1, t2
+WHERE t2.b IN (
+SELECT 'm' UNION
+SELECT 'm'
+) OR t1.b <> SOME (
+SELECT 'v' UNION
+SELECT 't'
+);
+b	a	b
+set @@join_cache_level= @save_join_cache_level;
+drop table t1,t2;
 set optimizer_switch=@subselect_tmp;
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';

mysql-test/r/subselect_no_semijoin.result

@@ -5676,5 +5676,52 @@ select a from t1 where a in (select a from t1 where a in (select a from t1 where
 a
 1
 drop table t1;
+#
+# LP bug #859375: Assertion `0' failed in st_select_lex_unit::optimize
+# with view , UNION and prepared statement (rewriting fake_select
+# condition).
+#
+CREATE TABLE t1 ( f1 int NOT NULL, f4 varchar(1) NOT NULL) ;
+INSERT INTO t1 VALUES (6,'d'),(7,'y');
+CREATE TABLE t2 ( f1 int NOT NULL, f2 int NOT NULL) ;
+INSERT INTO t2 VALUES (10,7);
+CREATE VIEW v2 AS SELECT * FROM t2;
+PREPARE st1 FROM "
+        SELECT *
+        FROM t1
+        LEFT JOIN v2 ON ( v2.f2 = t1.f1 )
+        WHERE v2.f1 NOT IN (
+                SELECT 1 UNION
+                SELECT 247
+        )
+";
+EXECUTE st1;
+f1	f4	f1	f2
+7	y	10	7
+deallocate prepare st1;
+DROP VIEW v2;
+DROP TABLE t1,t2;
+#
+# LP bug #887458 Crash in subselect_union_engine::no_rows with
+# double UNION and join_cache_level=3,8
+# (IN/ALL/ANY optimizations should not be applied to fake_select)
+CREATE TABLE t2 ( a int, b varchar(1)) ;
+INSERT IGNORE INTO t2 VALUES (8,'y'),(8,'y');
+CREATE TABLE t1 ( b varchar(1)) ;
+INSERT IGNORE INTO t1 VALUES (NULL),(NULL);
+set @save_join_cache_level=@@join_cache_level;
+SET SESSION join_cache_level=3;
+SELECT *
+FROM t1, t2
+WHERE t2.b IN (
+SELECT 'm' UNION
+SELECT 'm'
+) OR t1.b <> SOME (
+SELECT 'v' UNION
+SELECT 't'
+);
+b	a	b
+set @@join_cache_level= @save_join_cache_level;
+drop table t1,t2;
 set optimizer_switch=@subselect_tmp;
 set @optimizer_switch_for_subselect_test=null;

mysql-test/t/subselect.test

@@ -4778,4 +4778,60 @@ set @@optimizer_switch='in_to_exists=on,semijoin=off,materialization=off,subquer
 select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1 where a in (select a from t1)))))))))))))))))))))))))))));
 drop table t1;
 
+--echo #
+--echo # LP bug #859375: Assertion `0' failed in st_select_lex_unit::optimize
+--echo # with view , UNION and prepared statement (rewriting fake_select
+--echo # condition).
+--echo #
+
+CREATE TABLE t1 ( f1 int NOT NULL, f4 varchar(1) NOT NULL) ;
+INSERT INTO t1 VALUES (6,'d'),(7,'y');
+
+CREATE TABLE t2 ( f1 int NOT NULL, f2 int NOT NULL) ;
+INSERT INTO t2 VALUES (10,7);
+
+CREATE VIEW v2 AS SELECT * FROM t2;
+
+PREPARE st1 FROM "
+        SELECT *
+        FROM t1
+        LEFT JOIN v2 ON ( v2.f2 = t1.f1 )
+        WHERE v2.f1 NOT IN (
+                SELECT 1 UNION
+                SELECT 247
+        )
+";
+
+EXECUTE st1;
+deallocate prepare st1;
+
+DROP VIEW v2;
+DROP TABLE t1,t2;
+
+--echo #
+--echo # LP bug #887458 Crash in subselect_union_engine::no_rows with
+--echo # double UNION and join_cache_level=3,8
+--echo # (IN/ALL/ANY optimizations should not be applied to fake_select)
+
+CREATE TABLE t2 ( a int, b varchar(1)) ;
+INSERT IGNORE INTO t2 VALUES (8,'y'),(8,'y');
+
+CREATE TABLE t1 ( b varchar(1)) ;
+INSERT IGNORE INTO t1 VALUES (NULL),(NULL);
+
+set @save_join_cache_level=@@join_cache_level;
+SET SESSION join_cache_level=3;
+
+SELECT *
+FROM t1, t2
+WHERE t2.b IN (
+        SELECT 'm' UNION
+        SELECT 'm'
+) OR t1.b <> SOME (
+        SELECT 'v' UNION
+        SELECT 't'
+);
+
+set @@join_cache_level= @save_join_cache_level;
+drop table t1,t2;
 set optimizer_switch=@subselect_tmp;

sql/opt_subselect.cc

@@ -304,6 +304,14 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
   st_select_lex *select_lex= join->select_lex;
   st_select_lex_unit* parent_unit= select_lex->master_unit();
   DBUG_ENTER("check_and_do_in_subquery_rewrites");
+
+  /*
+    IN/ALL/ANY rewrites are not applicable for so called fake select
+    (this select exists only to filter results of union if it is needed).
+  */
+  if (select_lex == select_lex->master_unit()->fake_select_lex)
+    DBUG_RETURN(0);
+
   /*
     If 
       1) this join is inside a subquery (of any type except FROM-clause 
@@ -376,14 +384,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
       if (failure)
         DBUG_RETURN(-1); /* purecov: deadcode */
     }
-    if (select_lex == parent_unit->fake_select_lex)
-    {
-      /*
-        The join and its select_lex object represent the 'fake' select used
-        to compute the result of a UNION.
-      */
-      DBUG_RETURN(0);
-    }
 
     DBUG_PRINT("info", ("Checking if subq can be converted to semi-join"));
     /*
@@ -4503,6 +4503,13 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
   enum_reopt_result reopt_result= REOPT_NONE;
   Item_in_subselect *in_subs;
 
+  /*
+    IN/ALL/ANY optimizations are not applicable for so called fake select
+    (this select exists only to filter results of union if it is needed).
+  */
+  if (select_lex == select_lex->master_unit()->fake_select_lex)
+    return 0;
+
   if (is_in_subquery())
   {
     in_subs= (Item_in_subselect*) unit->item;