Commit aef0b4a8 authored by Sergey Petrunya's avatar Sergey Petrunya

MDEV-5962: EITS: value "position" calculated incorrectly for CHAR(n) columns

- Dont substract unsigned numbers, use correct calculations.
- (there is no testcase because effort is required to come up with it)
parent 79a8a613
......@@ -1175,6 +1175,15 @@ inline ulonglong char_prefix_to_ulonglong(uchar *src)
return uint8korr(src);
}
/*
Compute res = a - b, without losing precision and taking care that these are
unsigned numbers.
*/
static inline double safe_substract(ulonglong a, ulonglong b)
{
return (a > b)? double(a - b) : -double(b - a);
}
/**
@brief
......@@ -1227,10 +1236,10 @@ double Field::pos_in_interval_val_str(Field *min, Field *max, uint data_offset)
minp= char_prefix_to_ulonglong(minp_prefix);
maxp= char_prefix_to_ulonglong(maxp_prefix);
double n, d;
n= mp - minp;
n= safe_substract(mp, minp);
if (n < 0)
return 0.0;
d= maxp - minp;
d= safe_substract(maxp, minp);
if (d <= 0)
return 1.0;
return MY_MIN(n/d, 1.0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment