MDEV-7445: Server crash with Signal 6

Problem was in rewriting left expression which had 2 references on it. Solved with making subselect reference main.

Item_in_optimized can have not Item_in_subselect reference in left part so type casting with no check is dangerous.

Item::cols() should be checked after Item::fix_fields().

mysql-test/r/subselect.result

@@ -7027,3 +7027,23 @@ From
 Group By  TestCase.Revenue, TestCase.TemplateID;
 ControlRev
 NULL
+#
+# MDEV-7445:Server crash with Signal 6
+#
+CREATE PROCEDURE procedure2()
+BEGIN
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+  From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+END |
+call procedure2();
+ControlRev
+NULL
+call procedure2();
+ControlRev
+NULL
+drop procedure procedure2;

mysql-test/r/subselect_no_mat.result

@@ -7024,6 +7024,26 @@ From
 Group By  TestCase.Revenue, TestCase.TemplateID;
 ControlRev
 NULL
+#
+# MDEV-7445:Server crash with Signal 6
+#
+CREATE PROCEDURE procedure2()
+BEGIN
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+  From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+END |
+call procedure2();
+ControlRev
+NULL
+call procedure2();
+ControlRev
+NULL
+drop procedure procedure2;
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
 @@optimizer_switch like '%materialization=on%'

mysql-test/r/subselect_no_opts.result

@@ -7022,4 +7022,24 @@ From
 Group By  TestCase.Revenue, TestCase.TemplateID;
 ControlRev
 NULL
+#
+# MDEV-7445:Server crash with Signal 6
+#
+CREATE PROCEDURE procedure2()
+BEGIN
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+  From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+END |
+call procedure2();
+ControlRev
+NULL
+call procedure2();
+ControlRev
+NULL
+drop procedure procedure2;
 set @optimizer_switch_for_subselect_test=null;

mysql-test/r/subselect_no_scache.result

@@ -7033,6 +7033,26 @@ From
 Group By  TestCase.Revenue, TestCase.TemplateID;
 ControlRev
 NULL
+#
+# MDEV-7445:Server crash with Signal 6
+#
+CREATE PROCEDURE procedure2()
+BEGIN
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+  From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+END |
+call procedure2();
+ControlRev
+NULL
+call procedure2();
+ControlRev
+NULL
+drop procedure procedure2;
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
 @@optimizer_switch like '%subquery_cache=on%'

mysql-test/r/subselect_no_semijoin.result

@@ -7022,5 +7022,25 @@ From
 Group By  TestCase.Revenue, TestCase.TemplateID;
 ControlRev
 NULL
+#
+# MDEV-7445:Server crash with Signal 6
+#
+CREATE PROCEDURE procedure2()
+BEGIN
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+  From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+END |
+call procedure2();
+ControlRev
+NULL
+call procedure2();
+ControlRev
+NULL
+drop procedure procedure2;
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;

mysql-test/t/subselect.test

@@ -5902,3 +5902,24 @@ From
 (Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
 Group By  TestCase.Revenue, TestCase.TemplateID;
 
+--echo #
+--echo # MDEV-7445:Server crash with Signal 6
+--echo #
+
+--delimiter |
+CREATE PROCEDURE procedure2()
+BEGIN
+  Select 
+    (Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+      Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+     ) As `ControlRev`
+  From 
+  (Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+  Group By  TestCase.Revenue, TestCase.TemplateID;
+
+END |
+--delimiter ;
+call procedure2();
+call procedure2();
+
+drop procedure procedure2;

sql/item_cmpfunc.cc

@@ -1442,9 +1442,25 @@ bool Item_in_optimizer::eval_not_null_tables(uchar *opt_arg)
 bool Item_in_optimizer::fix_left(THD *thd, Item **ref)
 {
   DBUG_ENTER("Item_in_optimizer::fix_left");
-  if ((!args[0]->fixed && args[0]->fix_fields(thd, args)) ||
-      (!cache && !(cache= Item_cache::get_cache(args[0]))))
+  Item **ref0= args;
+  if (args[1]->type() == Item::SUBSELECT_ITEM &&
+      ((Item_subselect *)args[1])->is_in_predicate())
+  {
+    /*
+       left_expr->fix_fields() may cause left_expr to be substituted for
+       another item. (e.g. an Item_field may be changed into Item_ref). This
+       transformation is undone at the end of statement execution (e.g. the
+       Item_ref is deleted). However, Item_in_optimizer::args[0] may keep
+       the pointer to the post-transformation item. Because of that, on the
+       next execution we need to copy args[1]->left_expr again.
+    */
+    ref0= &(((Item_in_subselect *)args[1])->left_expr);
+    args[0]= ref0[0];
+  }
+  if ((!args[0]->fixed && args[0]->fix_fields(thd, ref0)) ||
+      (!cache && !(cache= Item_cache::get_cache(ref0[0]))))
     DBUG_RETURN(1);
+  args[0]= ref0[0];
   DBUG_PRINT("info", ("actual fix fields"));
 
   cache->setup(args[0]);
@@ -1500,6 +1516,16 @@ bool Item_in_optimizer::fix_left(THD *thd, Item **ref)
 bool Item_in_optimizer::fix_fields(THD *thd, Item **ref)
 {
   DBUG_ASSERT(fixed == 0);
+  Item_subselect *sub= 0;
+  uint col;
+
+  /*
+     MAX/MIN optimization can convert the subquery into
+     expr + Item_singlerow_subselect
+   */
+  if (args[1]->type() == Item::SUBSELECT_ITEM)
+    sub= (Item_subselect *)args[1];
+
   if (fix_left(thd, ref))
     return TRUE;
   if (args[0]->maybe_null)
@@ -1507,10 +1533,10 @@ bool Item_in_optimizer::fix_fields(THD *thd, Item **ref)
 
   if (!args[1]->fixed && args[1]->fix_fields(thd, args+1))
     return TRUE;
-  Item_in_subselect * sub= (Item_in_subselect *)args[1];
-  if (args[0]->cols() != sub->engine->cols())
+  if ((sub && ((col= args[0]->cols()) != sub->engine->cols())) ||
+      (!sub && (args[1]->cols() != (col= 1))))
   {
-    my_error(ER_OPERAND_COLUMNS, MYF(0), args[0]->cols());
+    my_error(ER_OPERAND_COLUMNS, MYF(0), col);
     return TRUE;
   }
   if (args[1]->maybe_null)

sql/opt_subselect.cc

@@ -620,6 +620,18 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         thd->stmt_arena->state != Query_arena::PREPARED)
       */
     {
+      SELECT_LEX *current= thd->lex->current_select;
+      thd->lex->current_select= current->return_after_parsing();
+      char const *save_where= thd->where;
+      thd->where= "IN/ALL/ANY subquery";
+
+      bool failure= !in_subs->left_expr->fixed &&
+                     in_subs->left_expr->fix_fields(thd, &in_subs->left_expr);
+      thd->lex->current_select= current;
+      thd->where= save_where;
+      if (failure)
+        DBUG_RETURN(-1); /* purecov: deadcode */
+
       /*
         Check if the left and right expressions have the same # of
         columns, i.e. we don't have a case like 
@@ -633,18 +645,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         my_error(ER_OPERAND_COLUMNS, MYF(0), in_subs->left_expr->cols());
         DBUG_RETURN(-1);
       }
-
-      SELECT_LEX *current= thd->lex->current_select;
-      thd->lex->current_select= current->return_after_parsing();
-      char const *save_where= thd->where;
-      thd->where= "IN/ALL/ANY subquery";
-        
-      bool failure= !in_subs->left_expr->fixed &&
-                     in_subs->left_expr->fix_fields(thd, &in_subs->left_expr);
-      thd->lex->current_select= current;
-      thd->where= save_where;
-      if (failure)
-        DBUG_RETURN(-1); /* purecov: deadcode */
     }
 
     DBUG_PRINT("info", ("Checking if subq can be converted to semi-join"));