5.3 merge

mysql-test/r/ps_max_subselect-5113.result

+CREATE TABLE t1 (b INT NOT NULL);
+INSERT INTO t1 VALUES (0),(8);
+PREPARE stmt FROM '
+ SELECT 1 FROM t1 AS o 
+   WHERE o.b >= ALL ( 
+     SELECT a2.b FROM t1 AS a1 LEFT JOIN t1 AS a2 ON ( a2.b = a1.b ) 
+     WHERE a1.b <= a2.b 
+  )
+';
+EXECUTE stmt;
+1
+1
+EXECUTE stmt;
+1
+1
+DROP TABLE t1;

mysql-test/r/subselect_debug.result

@@ -11,3 +11,10 @@ REVERSE(EXISTS(SELECT RAND() FROM t1))
 0
 SET GLOBAL debug_dbug=@orig_debug;
 DROP TABLE t1;
+create table t1 (i int);
+insert into t1 values (1),(2);
+select * from t1 where (i < 200 or i = 300) and i in (select i from t1);
+i
+1
+2
+drop table t1;

mysql-test/t/ps_max_subselect-5113.test

+#
+# MDEV-5113 Wrong result (extra row) and valgrind warnings in Item_maxmin_subselect::any_value on 2nd execution of PS with SELECT subquery
+#
+
+CREATE TABLE t1 (b INT NOT NULL);
+INSERT INTO t1 VALUES (0),(8);
+
+PREPARE stmt FROM '
+ SELECT 1 FROM t1 AS o 
+   WHERE o.b >= ALL ( 
+     SELECT a2.b FROM t1 AS a1 LEFT JOIN t1 AS a2 ON ( a2.b = a1.b ) 
+     WHERE a1.b <= a2.b 
+  )
+';
+
+EXECUTE stmt;
+EXECUTE stmt;
+
+DROP TABLE t1;
+

mysql-test/t/subselect_debug.test

@@ -15,3 +15,13 @@ SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
 SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
 SET GLOBAL debug_dbug=@orig_debug;
 DROP TABLE t1;
+
+#
+# MDEV-5284 Assertion `!(*expr)->fixed' fails in replace_where_subcondition with IN suquery
+#
+
+create table t1 (i int);
+insert into t1 values (1),(2);
+select * from t1 where (i < 200 or i = 300) and i in (select i from t1);
+drop table t1;
+

sql/item_cmpfunc.h

@@ -513,8 +513,8 @@ public:
   bool fix_fields(THD *thd, Item **ref)
     {return Item_func::fix_fields(thd, ref);}
   virtual void print(String *str, enum_query_type query_type);
-  void set_sum_test(Item_sum_hybrid *item) { test_sum_item= item; };
-  void set_sub_test(Item_maxmin_subselect *item) { test_sub_item= item; };
+  void set_sum_test(Item_sum_hybrid *item) { test_sum_item= item; test_sub_item= 0; };
+  void set_sub_test(Item_maxmin_subselect *item) { test_sub_item= item; test_sum_item= 0;};
   bool empty_underlying_subquery();
   Item *neg_transformer(THD *thd);
 };

sql/opt_subselect.cc

@@ -437,9 +437,7 @@ Currently, solution #2 is implemented.
 
 static
 bool subquery_types_allow_materialization(Item_in_subselect *in_subs);
-static bool replace_where_subcondition(JOIN *join, Item **expr, 
-                                       Item *old_cond, Item *new_cond,
-                                       bool do_fix_fields);
+static bool replace_where_subcondition(JOIN *, Item **, Item *, Item *, bool);
 static int subq_sj_candidate_cmp(Item_in_subselect* el1, Item_in_subselect* el2,
                                  void *arg);
 static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred);
@@ -1309,7 +1307,7 @@ static bool replace_where_subcondition(JOIN *join, Item **expr,
       }
       else if (item->type() == Item::COND_ITEM)
       {
-        DBUG_ASSERT(!(*expr)->fixed);
+        DBUG_ASSERT(!do_fix_fields || !(*expr)->fixed);
         replace_where_subcondition(join, li.ref(),
                                    old_cond, new_cond,
                                    do_fix_fields);