Bug #8663 (cant use bigint as input to CAST)

decimal->ulong conversion fixed to assign max possible ULONG if decimal
is bigger
Item_func_unsigned now handles DECIMAL parameter separately as we can't
rely on decimal::val_int result here.

mysql-test/r/type_newdecimal.result

@@ -1412,3 +1412,8 @@ i2	count(distinct j)
 1.0	2
 2.0	2
 drop table t1;
+select cast(19999999999999999999 as unsigned);
+cast(19999999999999999999 as unsigned)
+18446744073709551615
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''

mysql-test/t/type_newdecimal.test

@@ -1108,3 +1108,9 @@ insert into t1 values (1,1), (1,2), (2,3), (2,4);
 select i, count(distinct j) from t1 group by i;
 select i+0.0 as i2, count(distinct j) from t1 group by i2;
 drop table t1;
+
+#
+# Bug #8663 (cant use bigint as input to CAST)
+#
+select cast(19999999999999999999 as unsigned);
+

sql/item_func.cc

@@ -964,7 +964,14 @@ longlong Item_func_unsigned::val_int()
   longlong value;
   int error;
 
-  if (args[0]->cast_to_int_type() != STRING_RESULT)
+  if (args[0]->cast_to_int_type() == DECIMAL_RESULT)
+  {
+    my_decimal tmp, *dec= args[0]->val_decimal(&tmp);
+    if (!(null_value= args[0]->null_value))
+      my_decimal2int(E_DEC_FATAL_ERROR, dec, 1, &value);
+    return value;
+  }
+  else if (args[0]->cast_to_int_type() != STRING_RESULT)
   {
     value= args[0]->val_int();
     null_value= args[0]->null_value; 

strings/decimal.c

@@ -1036,7 +1036,7 @@ int decimal2ulonglong(decimal_t *from, ulonglong *to)
     x=x*DIG_BASE + *buf++;
     if (unlikely(y > ((ulonglong) ULONGLONG_MAX/DIG_BASE) || x < y))
     {
-      *to=y;
+      *to=ULONGLONG_MAX;
       return E_DEC_OVERFLOW;
     }
   }