MDEV-27230: Estimation for filtered rows less precise ...

Fix the code in Histogram_json_hb::range_selectivity that handles
special cases: a non-inclusive endpoint hitting a bucket boundary...

mysql-test/main/statistics_json.result

@@ -8194,3 +8194,20 @@ analyze select * from t2 where a =100;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
 1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	1011	1011.00	0.10	0.10	Using where
 drop table t0,t1,t2;
+#
+# MDEV-27230: Estimation for filtered rows less precise ...
+#
+create table t1 (a char(1));
+insert into t1 select chr(seq%26+97) from seq_1_to_50;
+insert into t1 select ':' from t1;
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+analyze select COUNT(*) FROM t1 WHERE a <> 'a';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	100.00	99.00	99.00	Using where
+analyze select COUNT(*) FROM t1 WHERE a < 'a';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	100	100.00	50.00	50.00	Using where
+drop table t1;

mysql-test/main/statistics_json.test

@@ -380,3 +380,13 @@ analyze select * from t2 where a =100;
 
 drop table t0,t1,t2;
 
+--echo #
+--echo # MDEV-27230: Estimation for filtered rows less precise ...
+--echo #
+create table t1 (a char(1));
+insert into t1 select chr(seq%26+97) from seq_1_to_50;
+insert into t1 select ':' from t1;
+analyze table t1 persistent for all;
+analyze select COUNT(*) FROM t1 WHERE a <> 'a';
+analyze select COUNT(*) FROM t1 WHERE a < 'a';
+drop table t1;

sql/opt_histogram_json.cc

@@ -1022,6 +1022,7 @@ double Histogram_json_hb::range_selectivity(Field *field, key_range *min_endp,
         $CONST. Move to the previous bucket.
       */
       idx--;
+      equal= false;
     }
     double left_fract= get_left_fract(idx);