Fixed bug #16674.

The length of the prefix of the pattern string in the LIKE predicate that 
determined the index range to be scanned was calculated incorrectly for
multi-byte character sets. 
As a result of this in 4. 1 the the scanned range was wider then necessary
if the prefix contained not only one-byte characters.  
In 5.0 additionally it caused missing some rows from the result set.

mysql-test/r/ctype_utf8.result

@@ -1124,3 +1124,70 @@ check table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 drop table t1;
+SET NAMES utf8;
+CREATE TABLE t1 (
+a CHAR(13) DEFAULT '',
+INDEX(a)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+INSERT INTO t1 VALUES 
+('Käli Käli 2-4'), ('Käli Käli 2-4'),
+('Käli Käli 2+4'), ('Käli Käli 2+4'),
+('Käli Käli 2-6'), ('Käli Käli 2-6');
+CREATE TABLE t2 (
+a CHAR(13) DEFAULT '',
+INDEX(a)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci;
+INSERT INTO t2 VALUES
+('Kali Kali 2-4'), ('Kali Kali 2-4'),
+('Kali Kali 2+4'), ('Kali Kali 2+4'),
+('Kali Kali 2-6'), ('Kali Kali 2-6');
+SELECT a FROM t1 WHERE a LIKE 'Käli Käli 2+4';
+a
+Käli Käli 2+4
+Käli Käli 2+4
+SELECT a FROM t2 WHERE a LIKE 'Kali Kali 2+4';
+a
+Kali Kali 2+4
+Kali Kali 2+4
+EXPLAIN SELECT a FROM t1 WHERE a LIKE 'Käli Käli 2+4';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	a	a	40	NULL	2	Using where; Using index
+EXPLAIN SELECT a FROM t1 WHERE a = 'Käli Käli 2+4';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	a	a	40	const	2	Using where; Using index
+EXPLAIN SELECT a FROM t2 WHERE a LIKE 'Kali Kali 2+4';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	range	a	a	14	NULL	2	Using where; Using index
+EXPLAIN SELECT a FROM t2 WHERE a = 'Kali Kali 2+4';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	ref	a	a	14	const	2	Using where; Using index
+DROP TABLE t1,t2;
+CREATE TABLE t1 (
+a char(255) DEFAULT '', 
+KEY(a(10))
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+a
+Käli Käli 2-4
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+a
+Käli Käli 2-4
+Käli Käli 2-4
+DROP TABLE t1;
+CREATE TABLE t1 (
+a char(255) DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+a
+Käli Käli 2-4
+Käli Käli 2-4
+ALTER TABLE t1 ADD KEY (a(10));
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+a
+Käli Käli 2-4
+Käli Käli 2-4
+DROP TABLE t1;

mysql-test/t/ctype_utf8.test

@@ -926,4 +926,59 @@ INSERT INTO t1 VALUES('uUABCDEFGHIGKLMNOPRSTUVWXYZ̈bbbbbbbbbbbbbbbbbbbbbbbbbbbb
 check table t1;
 drop table t1;
 
+#
+# Bug#16674: LIKE predicate for a utf8 character set column
+#
+
+SET NAMES utf8;
+
+CREATE TABLE t1 (
+  a CHAR(13) DEFAULT '',
+  INDEX(a)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+INSERT INTO t1 VALUES 
+ ('Käli Käli 2-4'), ('Käli Käli 2-4'),
+ ('Käli Käli 2+4'), ('Käli Käli 2+4'),
+ ('Käli Käli 2-6'), ('Käli Käli 2-6');
+
+CREATE TABLE t2 (
+  a CHAR(13) DEFAULT '',
+  INDEX(a)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci;
+
+INSERT INTO t2 VALUES
+ ('Kali Kali 2-4'), ('Kali Kali 2-4'),
+ ('Kali Kali 2+4'), ('Kali Kali 2+4'),
+ ('Kali Kali 2-6'), ('Kali Kali 2-6');
+
+SELECT a FROM t1 WHERE a LIKE 'Käli Käli 2+4';
+SELECT a FROM t2 WHERE a LIKE 'Kali Kali 2+4';
+
+EXPLAIN SELECT a FROM t1 WHERE a LIKE 'Käli Käli 2+4';
+EXPLAIN SELECT a FROM t1 WHERE a = 'Käli Käli 2+4';
+EXPLAIN SELECT a FROM t2 WHERE a LIKE 'Kali Kali 2+4';
+EXPLAIN SELECT a FROM t2 WHERE a = 'Kali Kali 2+4';
+
+DROP TABLE t1,t2;
+
+CREATE TABLE t1 (
+  a char(255) DEFAULT '', 
+  KEY(a(10))
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+DROP TABLE t1;
+
+CREATE TABLE t1 (
+  a char(255) DEFAULT ''
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+INSERT INTO t1 VALUES ('Käli Käli 2-4');
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+ALTER TABLE t1 ADD KEY (a(10));
+SELECT * FROM t1 WHERE a LIKE 'Käli Käli 2%';
+DROP TABLE t1;
+
 # End of 4.1 tests

strings/ctype-mb.c

@@ -502,21 +502,19 @@ my_bool my_like_range_mb(CHARSET_INFO *cs,
 			 char *min_str,char *max_str,
 			 uint *min_length,uint *max_length)
 {
+  uint mblen;
   const char *end= ptr + ptr_length;
   char *min_org= min_str;
   char *min_end= min_str + res_length;
   char *max_end= max_str + res_length;
-  uint charlen= res_length / cs->mbmaxlen;
+  uint maxcharlen= res_length / cs->mbmaxlen;
 
-  for (; ptr != end && min_str != min_end && charlen > 0 ; ptr++, charlen--)
+  for (; ptr != end && min_str != min_end && maxcharlen ; maxcharlen--)
   {
+    /* We assume here that escape, w_any, w_namy are one-byte characters */
     if (*ptr == escape && ptr+1 != end)
-    {
-      ptr++;					/* Skip escape */
-      *min_str++= *max_str++ = *ptr;
-      continue;
-    }
-    if (*ptr == w_one || *ptr == w_many)	/* '_' and '%' in SQL */
+      ptr++;                                    /* Skip escape */
+    else if (*ptr == w_one || *ptr == w_many)   /* '_' and '%' in SQL */
     {
       /* Write min key  */
       *min_length= (uint) (min_str - min_org);
@@ -534,7 +532,16 @@ my_bool my_like_range_mb(CHARSET_INFO *cs,
       pad_max_char(cs, max_str, max_end);
       return 0;
     }
-    *min_str++= *max_str++ = *ptr;
+    if ((mblen= my_ismbchar(cs, ptr, end)) > 1)
+    {
+      if (ptr+mblen > end || min_str+mblen > min_end)
+        break;
+      while (mblen--)
+       *min_str++= *max_str++= *ptr++;
+    }
+    else
+       *min_str++= *max_str++= *ptr++;    
+
   }
   *min_length= *max_length = (uint) (min_str - min_org);