Bug#57477 SIGFPE when dividing a huge number a negative number

The problem is dividing by const value when
the result is out of supported range.
The fix:
-return LONGLONG_MIN if the result is out of supported range for DIV operator.
-return 0 if divisor is -1 for MOD operator.

mysql-test/r/func_math.result

@@ -482,4 +482,20 @@ RAND(i)
 0.155220427694936
 DROP TABLE t1;
 #
+# Bug#57477 SIGFPE when dividing a huge number a negative number
+#
+SELECT -9999999999999999991 DIV -1;
+-9999999999999999991 DIV -1
+-9223372036854775808
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT -9223372036854775808 DIV -1;
+-9223372036854775808 DIV -1
+-9223372036854775808
+SELECT -9223372036854775808 MOD -1;
+-9223372036854775808 MOD -1
+0
+SELECT -9223372036854775808999 MOD -1;
+-9223372036854775808999 MOD -1
+0
 End of 5.1 tests

mysql-test/t/func_math.test

@@ -308,5 +308,11 @@ SELECT RAND(i) FROM t1;
 DROP TABLE t1;
 
 --echo #
+--echo # Bug#57477 SIGFPE when dividing a huge number a negative number
+--echo #
+SELECT -9999999999999999991 DIV -1;
+SELECT -9223372036854775808 DIV -1;
+SELECT -9223372036854775808 MOD -1;
+SELECT -9223372036854775808999 MOD -1;
 
 --echo End of 5.1 tests

sql/item_func.cc

@@ -1356,9 +1356,13 @@ longlong Item_func_int_div::val_int()
     signal_divide_by_null();
     return 0;
   }
-  return (unsigned_flag ?
-	  (ulonglong) value / (ulonglong) val2 :
-	  value / val2);
+
+  if (unsigned_flag)
+    return  ((ulonglong) value / (ulonglong) val2);
+  else if (value == LONGLONG_MIN && val2 == -1)
+    return LONGLONG_MIN;
+  else
+    return value / val2;
 }
 
 
@@ -1392,9 +1396,9 @@ longlong Item_func_mod::int_op()
   if (args[0]->unsigned_flag)
     result= args[1]->unsigned_flag ? 
       ((ulonglong) value) % ((ulonglong) val2) : ((ulonglong) value) % val2;
-  else
-    result= args[1]->unsigned_flag ?
-      value % ((ulonglong) val2) : value % val2;
+  else result= args[1]->unsigned_flag ?
+         value % ((ulonglong) val2) :
+         (val2 == -1) ? 0 : value % val2;
 
   return result;
 }