MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results

mysql-test/include/ctype_like_range_mdev14350.inc

+--echo #
+--echo # MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+--echo #
+
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+DROP TABLE t1;

mysql-test/r/ctype_latin1.result

@@ -8804,6 +8804,31 @@ DROP TABLE t1;
 # End of ctype_pad.inc
 #
 SET STORAGE_ENGINE=Default;
+SET NAMES latin1;
+#
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(50) DEFAULT NULL,
+  `b` varchar(50) DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+a	b
+111	111
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+a	b
+111	111
+DROP TABLE t1;
 #
 # End of 10.2 tests
 #

mysql-test/r/ctype_like_range.result

@@ -4430,5 +4430,50 @@ a_	6100	61FF
 a%	61000000000000000000	61FFFFFFFFFFFFFFFFFF
 DROP TABLE t1;
 #
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	313131
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_general_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	313131
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	313131
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	313131
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ucs2 COLLATE ucs2_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	003100310031
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf16 COLLATE utf16_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	003100310031
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf32 COLLATE utf32_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+a	HEX(LIKE_RANGE_MIN(a,200))
+111%	000000310000003100000031
+DROP TABLE t1;
+#
 # End of 10.2 tests
 #

mysql-test/r/ctype_ucs2_uca.result

@@ -559,6 +559,32 @@ DROP TABLE t1;
 # End of ctype_pad.inc
 #
 SET STORAGE_ENGINE=Default;
+SET NAMES utf8, collation_connection=ucs2_unicode_520_nopad_ci;
+#
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(50) CHARACTER SET ucs2 COLLATE ucs2_unicode_520_nopad_ci DEFAULT NULL,
+  `b` varchar(50) CHARACTER SET ucs2 COLLATE ucs2_unicode_520_nopad_ci DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+a	b
+111	111
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+a	b
+111	111
+DROP TABLE t1;
+SET NAMES utf8;
 #
 # End of 10.2 tests
 #

mysql-test/r/ctype_utf16_uca.result

@@ -7866,6 +7866,32 @@ DROP TABLE t1;
 # End of ctype_pad.inc
 #
 SET STORAGE_ENGINE=Default;
+SET NAMES utf8, collation_connection=utf16_unicode_520_nopad_ci;
+#
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(50) CHARACTER SET utf16 COLLATE utf16_unicode_520_nopad_ci DEFAULT NULL,
+  `b` varchar(50) CHARACTER SET utf16 COLLATE utf16_unicode_520_nopad_ci DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+a	b
+111	111
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+a	b
+111	111
+DROP TABLE t1;
+SET NAMES utf8;
 #
 # End of 10.2 tests
 #

mysql-test/r/ctype_utf32_uca.result

@@ -7886,6 +7886,32 @@ DROP TABLE t1;
 # End of ctype_pad.inc
 #
 SET STORAGE_ENGINE=Default;
+SET NAMES utf8, collation_connection=utf32_unicode_520_nopad_ci;
+#
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(50) CHARACTER SET utf32 COLLATE utf32_unicode_520_nopad_ci DEFAULT NULL,
+  `b` varchar(50) CHARACTER SET utf32 COLLATE utf32_unicode_520_nopad_ci DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+a	b
+111	111
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+a	b
+111	111
+DROP TABLE t1;
+SET NAMES utf8;
 #
 # End of 10.2 tests
 #

mysql-test/r/ctype_utf8_uca.result

@@ -559,6 +559,31 @@ DROP TABLE t1;
 # End of ctype_pad.inc
 #
 SET STORAGE_ENGINE=Default;
+SET NAMES utf8 COLLATE utf8_unicode_nopad_ci;
+#
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(50) CHARACTER SET utf8 COLLATE utf8_unicode_nopad_ci DEFAULT NULL,
+  `b` varchar(50) CHARACTER SET utf8 COLLATE utf8_unicode_nopad_ci DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+a	b
+111	111
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+a	b
+111	111
+DROP TABLE t1;
 #
 # End of 10.2 tests
 #

mysql-test/r/ctype_utf8mb4_uca.result

@@ -6576,6 +6576,32 @@ DROP TABLE t1;
 # End of ctype_pad.inc
 #
 SET STORAGE_ENGINE=Default;
+SET NAMES utf8mb4 COLLATE utf8mb4_unicode_520_nopad_ci;
+#
+# MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+#
+CREATE OR REPLACE TABLE t1 AS SELECT SPACE(50) AS a, SPACE (50) AS b;
+ALTER TABLE t1 ADD KEY(a), ADD KEY(b);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_nopad_ci DEFAULT NULL,
+  `b` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_nopad_ci DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+INSERT INTO t1 VALUES ('111', '111');
+INSERT INTO t1 VALUES ('222', '222');
+INSERT INTO t1 VALUES ('333', '333');
+INSERT INTO t1 VALUES ('444', '444');
+SELECT * FROM t1 WHERE a LIKE '111%';
+a	b
+111	111
+SELECT * FROM t1 IGNORE INDEX (a) WHERE a LIKE '111%';
+a	b
+111	111
+DROP TABLE t1;
+SET NAMES utf8mb4;
 #
 # End of 10.2 tests
 #

mysql-test/t/ctype_latin1.test

@@ -411,6 +411,9 @@ let $coll='latin1_nopad_bin';
 let $coll_pad='latin1_bin';
 --source include/ctype_pad_all_engines.inc
 
+SET NAMES latin1;
+--source include/ctype_like_range_mdev14350.inc
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/t/ctype_like_range.test

@@ -154,6 +154,46 @@ INSERT INTO t1 (a) VALUES ('a'),('a_'),('a%');
 SELECT a, HEX(mn), HEX(mx) FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-14350 Index use with collation utf8mb4_unicode_nopad_ci on LIKE pattern with wrong results
+--echo #
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_swedish_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_general_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8 COLLATE utf8_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ucs2 COLLATE ucs2_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf16 COLLATE utf16_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf32 COLLATE utf32_unicode_nopad_ci);
+INSERT INTO t1 VALUES ('111%');
+SELECT a, HEX(LIKE_RANGE_MIN(a,200)) FROM t1;
+DROP TABLE t1;
+
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/t/ctype_ucs2_uca.test

@@ -16,6 +16,10 @@ let $coll='ucs2_unicode_520_nopad_ci';
 let $coll_pad='ucs2_unicode_520_ci';
 --source include/ctype_pad_all_engines.inc
 
+SET NAMES utf8, collation_connection=ucs2_unicode_520_nopad_ci;
+--source include/ctype_like_range_mdev14350.inc
+SET NAMES utf8;
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/t/ctype_utf16_uca.test

@@ -238,6 +238,11 @@ let $coll='utf16_unicode_520_nopad_ci';
 let $coll_pad='utf16_unicode_520_ci';
 --source include/ctype_pad_all_engines.inc
 
+SET NAMES utf8, collation_connection=utf16_unicode_520_nopad_ci;
+--source include/ctype_like_range_mdev14350.inc
+SET NAMES utf8;
+
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/t/ctype_utf32_uca.test

@@ -260,6 +260,11 @@ let $coll='utf32_unicode_520_nopad_ci';
 let $coll_pad='utf32_unicode_520_ci';
 --source include/ctype_pad_all_engines.inc
 
+SET NAMES utf8, collation_connection=utf32_unicode_520_nopad_ci;
+--source include/ctype_like_range_mdev14350.inc
+SET NAMES utf8;
+
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

mysql-test/t/ctype_utf8_uca.test

@@ -14,6 +14,9 @@ let $coll='utf8_unicode_520_nopad_ci';
 let $coll_pad='utf8_unicode_520_ci';
 --source include/ctype_pad_all_engines.inc
 
+SET NAMES utf8 COLLATE utf8_unicode_nopad_ci;
+--source include/ctype_like_range_mdev14350.inc
+
 
 --echo #
 --echo # End of 10.2 tests

mysql-test/t/ctype_utf8mb4_uca.test

@@ -100,6 +100,11 @@ let $coll='utf8mb4_unicode_520_nopad_ci';
 let $coll_pad='utf8mb4_unicode_520_ci';
 --source include/ctype_pad_all_engines.inc
 
+SET NAMES utf8mb4 COLLATE utf8mb4_unicode_520_nopad_ci;
+--source include/ctype_like_range_mdev14350.inc
+SET NAMES utf8mb4;
+
+
 --echo #
 --echo # End of 10.2 tests
 --echo #

strings/ctype-mb.c

@@ -799,9 +799,9 @@ my_bool my_like_range_mb(CHARSET_INFO *cs,
         'a\0\0... is the smallest possible string when we have space expand
         a\ff\ff... is the biggest possible string
       */
-      *min_length= ((cs->state & MY_CS_BINSORT) ? (size_t) (min_str - min_org) :
-                    res_length);
-      *max_length= res_length;
+      *min_length= (cs->state & (MY_CS_BINSORT | MY_CS_NOPAD)) ?
+                    (size_t) (min_str - min_org) :
+                    res_length;
       /* Create min key  */
       do
       {
@@ -1002,9 +1002,9 @@ my_like_range_generic(CHARSET_INFO *cs,
         a\min\min... is the smallest possible string
         a\max\max... is the biggest possible string
       */
-      *min_length= ((cs->state & MY_CS_BINSORT) ?
+      *min_length= (cs->state & (MY_CS_BINSORT | MY_CS_NOPAD)) ?
                     (size_t) (min_str - min_org) :
-                    res_length);
+                    res_length;
       *max_length= res_length;
       goto pad_min_max;
     }

strings/ctype-simple.c

@@ -1059,9 +1059,9 @@ my_bool my_like_range_simple(CHARSET_INFO *cs,
     if (*ptr == w_many)				/* '%' in SQL */
     {
       /* Calculate length of keys */
-      *min_length= ((cs->state & MY_CS_BINSORT) ?
+      *min_length= (cs->state & (MY_CS_BINSORT | MY_CS_NOPAD)) ?
                     (size_t) (min_str - min_org) :
-                    res_length);
+                    res_length;
       *max_length= res_length;
       do
       {