Fix bug lp:782305

Analysis:
Both the wrong result and the valgrind warning were a result
of incomplete cleanup of the MIN/MAX subquery rewrite. At the
first execution of the query, the non-aggregate subquery is
transformed into an aggregate MIN/MAX subquery. During the
fix_fields phase of the MIN/MAX function, it sets the property
st_select_lex::with_sum_func to true.

The second execution of the query finds this flag to be ON.
When optimization reaches the same MIN/MAX subquery
transformation, it tests if the subquery is an aggregate or not.
Since select_lex->with_sum_func == true from the previous
execution, the transformation executes the second branch that
handles aggregate subqueries. This substitutes the subquery
Item into a Item_maxmin_subselect. At the same time elsewhere
it is assumed that the subquery Item is of type
Item_allany_subselect. Ultimately this results in casting the
actual object to the wrong class, and calling the wrong
any_value() method from empty_underlying_subquery().

Solution:
Cleanup the st_select_lex::with_sum_func property in the case
when the MIN/MAX transformation was performed for a non-aggregate
subquery, so that the transformation can be repeated.

mysql-test/r/subselect4.result

@@ -2060,4 +2060,34 @@ WHERE t1.f4 IN ( SELECT f4 FROM t2 ) ;
 f4
 set @@optimizer_switch = @old_optimizer_switch;
 drop table t1, t2, t3;
+#
+# LP BUG#782305: Wrong result/valgrind warning in Item_sum_hybrid::any_value()
+#
+CREATE TABLE t1 ( f1 int) ;
+INSERT INTO t1 VALUES (2),(3);
+CREATE TABLE t2 (f2 int) ;
+INSERT INTO t2 VALUES (2),(3);
+PREPARE st1 FROM '
+SELECT * FROM t2
+WHERE f2 <= SOME ( SELECT f1 FROM t1 );
+';
+EXECUTE st1;
+f2
+2
+3
+EXECUTE st1;
+f2
+2
+3
+PREPARE st2 FROM '
+SELECT * FROM t2
+WHERE f2 <= SOME (SELECT f1-2 FROM t1 UNION SELECT f1-1 FROM t1);
+';
+EXECUTE st2;
+f2
+2
+EXECUTE st2;
+f2
+2
+drop table t1, t2;
 set optimizer_switch=@subselect4_tmp;

mysql-test/t/subselect4.test

@@ -1700,4 +1700,31 @@ set @@optimizer_switch = @old_optimizer_switch;
 
 drop table t1, t2, t3;
 
+
+--echo #
+--echo # LP BUG#782305: Wrong result/valgrind warning in Item_sum_hybrid::any_value()
+--echo #
+
+CREATE TABLE t1 ( f1 int) ;
+INSERT INTO t1 VALUES (2),(3);
+CREATE TABLE t2 (f2 int) ;
+INSERT INTO t2 VALUES (2),(3);
+
+PREPARE st1 FROM '
+SELECT * FROM t2
+WHERE f2 <= SOME ( SELECT f1 FROM t1 );
+';
+EXECUTE st1;
+EXECUTE st1;
+
+PREPARE st2 FROM '
+SELECT * FROM t2
+WHERE f2 <= SOME (SELECT f1-2 FROM t1 UNION SELECT f1-1 FROM t1);
+';
+EXECUTE st2;
+EXECUTE st2;
+
+drop table t1, t2;
+
+
 set optimizer_switch=@subselect4_tmp;

sql/item_subselect.cc

@@ -167,6 +167,23 @@ void Item_in_subselect::cleanup()
   DBUG_VOID_RETURN;
 }
 
+
+void Item_allany_subselect::cleanup()
+{
+  /*
+    The MAX/MIN transformation through injection is reverted through the
+    change_item_tree() mechanism. Revert the select_lex object of the
+    query to its initial state.
+  */
+  for (SELECT_LEX *sl= unit->first_select();
+       sl; sl= sl->next_select())
+    if (in_strategy & SUBS_MAXMIN_INJECTED)
+      sl->with_sum_func= false;
+  Item_in_subselect::cleanup();
+
+}
+
+
 Item_subselect::~Item_subselect()
 {
   DBUG_ENTER("Item_subselect::~Item_subselect");
@@ -1566,7 +1583,7 @@ Item_in_subselect::single_value_transformer(JOIN *join)
 bool Item_allany_subselect::transform_into_max_min(JOIN *join)
 {
   DBUG_ENTER("Item_allany_subselect::transform_into_max_min");
-  if (!(in_strategy & SUBS_MAXMIN))
+  if (!(in_strategy & (SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE)))
     DBUG_RETURN(false);
   Item **place= optimizer->arguments() + 1;
   THD *thd= join->thd;
@@ -1626,6 +1643,12 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
     if (join->prepare_stage2())
       DBUG_RETURN(true);
     subs= new Item_singlerow_subselect(select_lex);
+
+    /*
+      Remove other strategies if any (we already changed the query and
+      can't apply other strategy).
+    */
+    in_strategy= SUBS_MAXMIN_INJECTED;
   }
   else
   {
@@ -1633,6 +1656,11 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
     subs= item= new Item_maxmin_subselect(thd, this, select_lex, func->l_op());
     if (upper_item)
       upper_item->set_sub_test(item);
+    /*
+      Remove other strategies if any (we already changed the query and
+      can't apply other strategy).
+    */
+    in_strategy= SUBS_MAXMIN_ENGINE;
   }
   /* fix fields is already called for  left expression */
   subs= func->create(left_expr, subs);
@@ -1643,11 +1671,6 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
 
   select_lex->master_unit()->uncacheable&= ~UNCACHEABLE_DEPENDENT_INJECTED;
   select_lex->uncacheable&= ~UNCACHEABLE_DEPENDENT_INJECTED;
-  /*
-    Remove other strategies if there was (we already changed the query and
-    can't apply other strategy).
-  */
-  in_strategy= SUBS_MAXMIN;
 
   DBUG_RETURN(false);
 }
@@ -2540,7 +2563,8 @@ bool
 Item_allany_subselect::select_transformer(JOIN *join)
 {
   DBUG_ENTER("Item_allany_subselect::select_transformer");
-  DBUG_ASSERT((in_strategy & ~(SUBS_MAXMIN | SUBS_IN_TO_EXISTS)) == 0);
+  DBUG_ASSERT((in_strategy & ~(SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE |
+                               SUBS_IN_TO_EXISTS)) == 0);
   in_strategy|= SUBS_IN_TO_EXISTS;
   if (upper_item)
     upper_item->show= 1;

sql/item_subselect.h

@@ -366,7 +366,10 @@ TABLE_LIST * const NO_JOIN_NEST=(TABLE_LIST*)0x1;
 #define SUBS_PARTIAL_MATCH_ROWID_MERGE 8
 #define SUBS_PARTIAL_MATCH_TABLE_SCAN 16
 /* ALL/ANY will be transformed with max/min optimization */
-#define SUBS_MAXMIN 32
+/*   The subquery has not aggregates, transform it into a MAX/MIN query. */
+#define SUBS_MAXMIN_INJECTED 32
+/*   The subquery has aggregates, use a special max/min subselect engine. */
+#define SUBS_MAXMIN_ENGINE 64
 
 
 /**
@@ -555,6 +558,7 @@ public:
   Item_allany_subselect(Item * left_expr, chooser_compare_func_creator fc,
                         st_select_lex *select_lex, bool all);
 
+  void cleanup();
   // only ALL subquery has upper not
   subs_type substype() { return all?ALL_SUBS:ANY_SUBS; }
   bool select_transformer(JOIN *join);

sql/opt_subselect.cc

@@ -466,7 +466,7 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
       /* Check if max/min optimization applicable */
       if (allany_subs)
         allany_subs->in_strategy|= (allany_subs->is_maxmin_applicable(join) ?
-                                    SUBS_MAXMIN :
+                                    (SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE) :
                                     SUBS_IN_TO_EXISTS);
 
       /*