Bug#36023: Incorrect handling of zero length caused an assertion to fail.

When a zero length is provided to the my_decimal_length_to_precision
function along with unsigned_flag set to false it returns a negative value.
For queries that employs temporary tables may cause failed assertion or
excessive memory consumption while temporary table creation.

Now the my_decimal_length_to_precision and the my_decimal_precision_to_length
functions take unsigned_flag into account only if the length/precision
argument is non-zero.


mysql-test/t/type_decimal.test:
  Added a test case for the bug#36023: Incorrect handling of zero length caused
   an assertion to fail.
mysql-test/r/type_decimal.result:
  Added a test case for the bug#36023: Incorrect handling of zero length caused
   an assertion to fail.
sql/my_decimal.h:
  Bug#36023: Incorrect handling of zero length caused an assertion to fail.
  Now the my_decimal_length_to_precision and the my_decimal_precision_to_length
  functions take unsigned_flag into account only if the length/precision
  argument is non-zero.

mysql-test/r/type_decimal.result

@@ -946,4 +946,11 @@ SELECT ROUND(20061108085411.000002);
 ROUND(20061108085411.000002)
 20061108085411
 DROP TABLE t1, t2, t3, t4, t5, t6;
+create table t1(`c` decimal(9,2));
+insert into t1 values (300),(201.11);
+select max(case 1 when 1 then c else null end) from t1 group by c;
+max(case 1 when 1 then c else null end)
+201.11
+300.00
+drop table t1;
 End of 5.0 tests

mysql-test/t/type_decimal.test

@@ -521,4 +521,12 @@ SELECT ROUND(20061108085411.000002);
 
 DROP TABLE t1, t2, t3, t4, t5, t6;
 
+#
+# Bug#36023: Incorrect handling of zero length caused an assertion to fail.
+#
+create table t1(`c` decimal(9,2));
+insert into t1 values (300),(201.11);
+select max(case 1 when 1 then c else null end) from t1 group by c;
+drop table t1;
+
 --echo End of 5.0 tests

sql/my_decimal.h

@@ -164,14 +164,23 @@ inline int check_result_and_overflow(uint mask, int result, my_decimal *val)
 inline uint my_decimal_length_to_precision(uint length, uint scale,
                                            bool unsigned_flag)
 {
-  return (uint) (length - (scale>0 ? 1:0) - (unsigned_flag ? 0:1));
+  /* Precision can't be negative thus ignore unsigned_flag when length is 0. */
+  DBUG_ASSERT(length || !scale);
+  return (uint) (length - (scale>0 ? 1:0) -
+                 (unsigned_flag || !length ? 0:1));
 }
 
 inline uint32 my_decimal_precision_to_length(uint precision, uint8 scale,
                                              bool unsigned_flag)
 {
+  /*
+    When precision is 0 it means that original length was also 0. Thus
+    unsigned_flag is ignored in this case.
+  */
+  DBUG_ASSERT(precision || !scale);
   set_if_smaller(precision, DECIMAL_MAX_PRECISION);
-  return (uint32)(precision + (scale>0 ? 1:0) + (unsigned_flag ? 0:1));
+  return (uint32)(precision + (scale>0 ? 1:0) +
+                  (unsigned_flag || !precision ? 0:1));
 }
 
 inline