Bug#36270: incorrect calculation result - works in 4.1 but not in 5.0 or 5.1

When the fractional part in a multiplication of DECIMALs
overflowed, we truncated the first operand rather than the
longest. Now truncating least significant places instead
for more precise multiplications.

(Queuing at demand of Trudy/Davi.)

mysql-test/r/type_newdecimal.result

@@ -1519,4 +1519,9 @@ SELECT f1 FROM t1;
 f1
 99999999999999999999999999999.999999999999999999999999999999
 DROP TABLE t1;
+select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *
+1.01500000 * 1.01500000 * 0.99500000);
+(1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *
+1.01500000 * 1.01500000 * 0.99500000)
+0.812988073953673124592306939480
 End of 5.0 tests

mysql-test/t/type_newdecimal.test

@@ -1216,4 +1216,13 @@ DESC t1;
 SELECT f1 FROM t1;
 DROP TABLE t1;
 
+#
+# Bug #36270: incorrect calculation result - works in 4.1 but not in 5.0 or 5.1
+#
+
+# show that if we need to truncate the scale of an operand, we pick the
+# right one (that is, we discard the least significant decimal places)
+select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *
+        1.01500000 * 1.01500000 * 0.99500000);
+
 --echo End of 5.0 tests

strings/decimal.c

@@ -1999,18 +1999,18 @@ int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to)
 
   sanity(to);
 
-  i=intg0;
+  i=intg0;                                       /* save 'ideal' values */
   j=frac0;
-  FIX_INTG_FRAC_ERROR(to->len, intg0, frac0, error);
+  FIX_INTG_FRAC_ERROR(to->len, intg0, frac0, error);  /* bound size */
   to->sign=from1->sign != from2->sign;
-  to->frac=from1->frac+from2->frac;
+  to->frac=from1->frac+from2->frac;              /* store size in digits */
   to->intg=intg0*DIG_PER_DEC1;
 
   if (unlikely(error))
   {
     set_if_smaller(to->frac, frac0*DIG_PER_DEC1);
     set_if_smaller(to->intg, intg0*DIG_PER_DEC1);
-    if (unlikely(i > intg0))
+    if (unlikely(i > intg0))                     /* bounded integer-part */
     {
       i-=intg0;
       j=i >> 1;
@@ -2018,12 +2018,20 @@ int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to)
       intg2-=i-j;
       frac1=frac2=0; /* frac0 is already 0 here */
     }
-    else
+    else                                         /* bounded fract part */
     {
       j-=frac0;
       i=j >> 1;
-      frac1-= i;
-      frac2-=j-i;
+      if (frac1 <= frac2)
+      {
+        frac1-= i;
+        frac2-=j-i;
+      }
+      else
+      {
+        frac2-= i;
+        frac1-=j-i;
+      }
     }
   }
   start0=to->buf+intg0+frac0-1;