bug #8663 cant use bigint unsigned as input to cast

in the case of the overflow in the decimal->integer conversion
we didn't return the proper boundary value, but just the result
of the conversion we calculated on the moment of the error

mysql-test/r/bigint.result

@@ -352,3 +352,13 @@ select c1 mod 50 as result from t1;
 result
 6
 drop table t1;
+select cast(19999999999999999999 as signed);
+cast(19999999999999999999 as signed)
+9223372036854775807
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+select cast(-19999999999999999999 as signed);
+cast(-19999999999999999999 as signed)
+-9223372036854775808
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''

mysql-test/t/bigint.test

@@ -288,3 +288,9 @@ insert into t1 values (10000002383263201056);
 select c1 mod 50 as result from t1;
 drop table t1;
 
+#
+# Bug #8663 cant use bgint unsigned as input to cast
+#
+
+select cast(19999999999999999999 as signed);
+select cast(-19999999999999999999 as signed);

strings/decimal.c

@@ -1083,7 +1083,11 @@ int decimal2longlong(decimal_t *from, longlong *to)
     x=x*DIG_BASE - *buf++;
     if (unlikely(y < (LONGLONG_MIN/DIG_BASE) || x > y))
     {
-      *to= from->sign ? y : -y;
+      /*
+        the decimal is bigger than any possible integer
+        return border integer depending on the sign
+      */
+      *to= from->sign ? LONGLONG_MIN : LONGLONG_MAX;
       return E_DEC_OVERFLOW;
     }
   }