Fix for bug #10966: Variance functions return wrong data type.

mysql-test/r/func_group.result

@@ -1089,3 +1089,17 @@ SUM(a)
 6
 DROP TABLE t1;
 set div_precision_increment= @sav_dpi;
+create table t1 select variance(0);
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `variance(0)` double(8,4) default NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+drop table t1;
+create table t1 select stddev(0);
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `stddev(0)` double(8,4) default NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+drop table t1;

mysql-test/t/func_group.test

@@ -703,3 +703,14 @@ SELECT SUM(a) FROM t1 GROUP BY b/c;
 DROP TABLE t1;
 set div_precision_increment= @sav_dpi;
 
+#
+# Bug #10966: Variance functions return wrong data type
+#
+
+create table t1 select variance(0);                                               
+show create table t1;                                                           
+drop table t1;                                                                  
+create table t1 select stddev(0);
+show create table t1;
+drop table t1;
+ 
\ No newline at end of file

sql/item_sum.cc

@@ -1030,9 +1030,6 @@ Field *Item_sum_variance::create_tmp_field(bool group, TABLE *table,
                              sizeof(double)*2) + sizeof(longlong),
                             0, name, table, &my_charset_bin);
   }
-  if (hybrid_type == DECIMAL_RESULT)
-    return new Field_new_decimal(max_length, maybe_null, name, table,
-                                 decimals, unsigned_flag);
   return new Field_double(max_length, maybe_null,name,table,decimals);
 }
 

sql/item_sum.h

@@ -481,7 +481,7 @@ public:
     { return sample ? "var_samp(" : "variance("; }
   Item *copy_or_same(THD* thd);
   Field *create_tmp_field(bool group, TABLE *table, uint convert_blob_length);
-  enum Item_result result_type () const { return hybrid_type; }
+  enum Item_result result_type () const { return REAL_RESULT; }
 };
 
 class Item_sum_std;