Commit aef0b4a8 authored by Sergey Petrunya's avatar Sergey Petrunya

MDEV-5962: EITS: value "position" calculated incorrectly for CHAR(n) columns

- Dont substract unsigned numbers, use correct calculations.
- (there is no testcase because effort is required to come up with it)
parent 79a8a613
...@@ -1175,6 +1175,15 @@ inline ulonglong char_prefix_to_ulonglong(uchar *src) ...@@ -1175,6 +1175,15 @@ inline ulonglong char_prefix_to_ulonglong(uchar *src)
return uint8korr(src); return uint8korr(src);
} }
/*
Compute res = a - b, without losing precision and taking care that these are
unsigned numbers.
*/
static inline double safe_substract(ulonglong a, ulonglong b)
{
return (a > b)? double(a - b) : -double(b - a);
}
/** /**
@brief @brief
...@@ -1227,10 +1236,10 @@ double Field::pos_in_interval_val_str(Field *min, Field *max, uint data_offset) ...@@ -1227,10 +1236,10 @@ double Field::pos_in_interval_val_str(Field *min, Field *max, uint data_offset)
minp= char_prefix_to_ulonglong(minp_prefix); minp= char_prefix_to_ulonglong(minp_prefix);
maxp= char_prefix_to_ulonglong(maxp_prefix); maxp= char_prefix_to_ulonglong(maxp_prefix);
double n, d; double n, d;
n= mp - minp; n= safe_substract(mp, minp);
if (n < 0) if (n < 0)
return 0.0; return 0.0;
d= maxp - minp; d= safe_substract(maxp, minp);
if (d <= 0) if (d <= 0)
return 1.0; return 1.0;
return MY_MIN(n/d, 1.0); return MY_MIN(n/d, 1.0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment