Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions

- Honor unsigned_flag in the corresponding functions
- Use compare_int_signed_unsigned()/compare_int_unsigned_signed() instead of explicit comparison in GREATEST() and LEAST()

mysql-test/r/case.result

@@ -177,3 +177,9 @@ from t1 where b=3 group by b;
 min(a)	min(case when 1=1 then a else NULL end)	min(case when 1!=1 then NULL else a end)
 2	2	2
 drop table t1;
+SELECT CASE 1 WHEN 1 THEN 18446744073709551615 ELSE 1 END;
+CASE 1 WHEN 1 THEN 18446744073709551615 ELSE 1 END
+18446744073709551615
+SELECT COALESCE(18446744073709551615);
+COALESCE(18446744073709551615)
+18446744073709551615

mysql-test/r/func_if.result

@@ -99,3 +99,9 @@ a	NULLIF(a,'')
 NULL	NULL
 	NULL
 DROP TABLE t1;
+SELECT IF(1 != 0, 18446744073709551615, 1);
+IF(1 != 0, 18446744073709551615, 1)
+18446744073709551615
+SELECT IFNULL(NULL, 18446744073709551615);
+IFNULL(NULL, 18446744073709551615)
+18446744073709551615

mysql-test/r/func_test.result

@@ -183,3 +183,9 @@ select 5.1 mod 3, 5.1 mod -3, -5.1 mod 3, -5.1 mod -3;
 select 5 mod 3, 5 mod -3, -5 mod 3, -5 mod -3;
 5 mod 3	5 mod -3	-5 mod 3	-5 mod -3
 2	2	-2	-2
+SELECT GREATEST(1, 18446744073709551615);
+GREATEST(1, 18446744073709551615)
+18446744073709551615
+SELECT LEAST(1, 18446744073709551615);
+LEAST(1, 18446744073709551615)
+1

mysql-test/r/user_var.result

@@ -203,3 +203,7 @@ select @@global.version;
 select @@session.VERSION;
 @@session.VERSION
 #
+set @a=18446744073709551615;
+select @a;
+@a
+18446744073709551615

mysql-test/t/case.test

@@ -130,4 +130,10 @@ select min(a), min(case when 1=1 then a else NULL end),
 from t1 where b=3 group by b;
 drop table t1;
 
+#
+# Bug #20924: UNSIGNED values in CASE and COALESCE are treated as SIGNED
+#
+SELECT CASE 1 WHEN 1 THEN 18446744073709551615 ELSE 1 END;
+SELECT COALESCE(18446744073709551615);
+
 # End of 4.1 tests

mysql-test/t/func_if.test

@@ -73,4 +73,14 @@ SELECT a, NULLIF(a,'') FROM t1 WHERE NULLIF(a,'') IS NULL;
 
 DROP TABLE t1;
 
+#
+# Bug #20924: UNSIGNED values in IF() are treated as SIGNED
+#
+SELECT IF(1 != 0, 18446744073709551615, 1);
+
+#
+# Bug #20924: UNSIGNED values in IFNULL() are treated as SIGNED
+#
+SELECT IFNULL(NULL, 18446744073709551615);
+
 # End of 4.1 tests

mysql-test/t/func_test.test

@@ -108,4 +108,10 @@ select 5.1 mod 3, 5.1 mod -3, -5.1 mod 3, -5.1 mod -3;
 
 select 5 mod 3, 5 mod -3, -5 mod 3, -5 mod -3;
 
+#
+# Bug #20924: UNSIGNED values in GREATEST() and LEAST() are treated as SIGNED
+#
+SELECT GREATEST(1, 18446744073709551615);
+SELECT LEAST(1, 18446744073709551615);
+
 # End of 4.1 tests

mysql-test/t/user_var.test

@@ -141,4 +141,10 @@ select @@global.version;
 --replace_column 1 #
 select @@session.VERSION;
 
+#
+# Bug #20924 SET on a user variable saves UNSIGNED as SIGNED
+#
+set @a=18446744073709551615;
+select @a;
+
 # End of 4.1 tests

sql/item_cmpfunc.cc

@@ -619,11 +619,7 @@ int Arg_comparator::compare_int_signed_unsigned()
     if (!(*b)->null_value)
     {
       owner->null_value= 0;
-      if (sval1 < 0 || (ulonglong)sval1 < uval2)
-        return -1;
-      if ((ulonglong)sval1 == uval2)
-        return 0;
-      return 1;
+      return ::compare_int_signed_unsigned(sval1, uval2);
     }
   }
   owner->null_value= 1;
@@ -644,13 +640,7 @@ int Arg_comparator::compare_int_unsigned_signed()
     if (!(*b)->null_value)
     {
       owner->null_value= 0;
-      if (sval2 < 0)
-        return 1;
-      if (uval1 < (ulonglong)sval2)
-        return -1;
-      if (uval1 == (ulonglong)sval2)
-        return 0;
-      return 1;
+      return ::compare_int_unsigned_signed(uval1, sval2);
     }
   }
   owner->null_value= 1;
@@ -1162,11 +1152,13 @@ Item_func_ifnull::val_int()
   if (!args[0]->null_value)
   {
     null_value=0;
+    unsigned_flag= args[0]->unsigned_flag;
     return value;
   }
   value=args[1]->val_int();
   if ((null_value=args[1]->null_value))
     return 0;
+  unsigned_flag= args[1]->unsigned_flag;
   return value;
 }
 
@@ -1286,6 +1278,7 @@ Item_func_if::val_int()
   Item *arg= args[0]->val_int() ? args[1] : args[2];
   longlong value=arg->val_int();
   null_value=arg->null_value;
+  unsigned_flag= arg->unsigned_flag;
   return value;
 }
 
@@ -1492,6 +1485,7 @@ longlong Item_func_case::val_int()
   }
   res=item->val_int();
   null_value=item->null_value;
+  unsigned_flag= item->unsigned_flag;
   return res;
 }
 
@@ -1623,7 +1617,10 @@ longlong Item_func_coalesce::val_int()
   {
     longlong res=args[i]->val_int();
     if (!args[i]->null_value)
+    {
+      unsigned_flag= args[i]->unsigned_flag;  
       return res;
+    }
   }
   null_value=1;
   return 0;

sql/item_cmpfunc.h

@@ -1077,3 +1077,17 @@ inline Item *and_conds(Item *a, Item *b)
 }
 
 Item *and_expressions(Item *a, Item *b, Item **org_item);
+
+inline int compare_int_signed_unsigned(longlong sval, ulonglong uval)
+{
+  if (sval < 0 || (ulonglong)sval < uval)
+    return -1;
+  if ((ulonglong)sval == uval)
+    return 0;
+  return 1;
+}
+
+inline int compare_int_unsigned_signed(ulonglong uval, longlong sval)
+{
+  return -compare_int_signed_unsigned(sval, uval);
+}

sql/item_func.cc

@@ -1235,19 +1235,35 @@ longlong Item_func_min_max::val_int()
 {
   DBUG_ASSERT(fixed == 1);
   longlong value=0;
+  my_bool arg_unsigned_flag;
+  my_bool cmp;
   null_value=1;
   for (uint i=0; i < arg_count ; i++)
   {
+    longlong tmp= args[i]->val_int();
+    arg_unsigned_flag= args[i]->unsigned_flag;
     if (null_value)
     {
-      value=args[i]->val_int();
+      value= tmp;
       null_value=args[i]->null_value;
+      unsigned_flag= arg_unsigned_flag;
     }
     else
     {
-      longlong tmp=args[i]->val_int();
-      if (!args[i]->null_value && (tmp < value ? cmp_sign : -cmp_sign) > 0)
-	value=tmp;
+      if (args[i]->null_value)
+        continue;
+      if (unsigned_flag && arg_unsigned_flag ||
+          (!unsigned_flag && !arg_unsigned_flag))
+        cmp= tmp < value;
+      else if (unsigned_flag)
+        cmp= compare_int_signed_unsigned(tmp, value) < 0;
+      else
+        cmp= compare_int_unsigned_signed(tmp, value) < 0;
+      if ((cmp ? cmp_sign : -cmp_sign) > 0)
+      {
+        value= tmp;
+        unsigned_flag= arg_unsigned_flag;
+      }
     }
   }
   return value;
@@ -2313,6 +2329,7 @@ static user_var_entry *get_variable(HASH *hash, LEX_STRING &name,
     entry->length=0;
     entry->update_query_id=0;
     entry->collation.set(NULL, DERIVATION_IMPLICIT);
+    entry->unsigned_flag= 0;
     /*
       If we are here, we were called from a SET or a query which sets a
       variable. Imagine it is this:
@@ -2390,7 +2407,7 @@ Item_func_set_user_var::fix_length_and_dec()
 bool Item_func_set_user_var::update_hash(void *ptr, uint length,
 					 Item_result type,
 					 CHARSET_INFO *cs,
-					 Derivation dv)
+					 Derivation dv, bool unsigned_arg)
 {
   if ((null_value=args[0]->null_value))
   {
@@ -2437,6 +2454,7 @@ bool Item_func_set_user_var::update_hash(void *ptr, uint length,
     entry->length= length;
     entry->type=type;
     entry->collation.set(cs, dv);
+    entry->unsigned_flag= unsigned_arg;
   }
   return 0;
 
@@ -2507,7 +2525,10 @@ String *user_var_entry::val_str(my_bool *null_value, String *str,
     str->set(*(double*) value, decimals, &my_charset_bin);
     break;
   case INT_RESULT:
-    str->set(*(longlong*) value, &my_charset_bin);
+      if (!unsigned_flag)
+        str->set(*(longlong*) value, &my_charset_bin);
+      else
+        str->set(*(ulonglong*) value, &my_charset_bin);
     break;
   case STRING_RESULT:
     if (str->copy(value, length, collation.collation))
@@ -2548,6 +2569,7 @@ Item_func_set_user_var::check()
   case INT_RESULT:
   {
     save_result.vint= args[0]->val_int();
+    unsigned_flag= args[0]->unsigned_flag;
     break;
   }
   case STRING_RESULT:
@@ -2598,7 +2620,8 @@ Item_func_set_user_var::update()
   case INT_RESULT:
   {
     res= update_hash((void*) &save_result.vint, sizeof(save_result.vint),
-		     INT_RESULT, &my_charset_bin, DERIVATION_IMPLICIT);
+         INT_RESULT, &my_charset_bin, DERIVATION_IMPLICIT,
+         unsigned_flag);
     break;
   }
   case STRING_RESULT:

sql/item_func.h

@@ -962,7 +962,7 @@ public:
   longlong val_int();
   String *val_str(String *str);
   bool update_hash(void *ptr, uint length, enum Item_result type, 
-  		   CHARSET_INFO *cs, Derivation dv);
+  		   CHARSET_INFO *cs, Derivation dv, bool unsigned_arg= 0);
   bool check();
   bool update();
   enum Item_result result_type () const { return cached_result_type; }

sql/sql_class.h

@@ -1460,6 +1460,7 @@ class user_var_entry
   char *value;
   ulong length, update_query_id, used_query_id;
   Item_result type;
+  bool unsigned_flag;
 
   double val(my_bool *null_value);
   longlong val_int(my_bool *null_value);