ctype_ucs.result, ctype_ucs.test, ctype_utf8.result, ctype_utf8.test:

  Fixing tests accordingly.
ctype-ucs2.c:
  The same fix for UCS2.
ctype-utf8.c:
  Bug #9557
  MyISAM utf8 table crash
  The problem was that my_strnncollsp_xxx could
  return big value in the range 0..0xffff.
  for some constant pairs it could return 32738,
  which is defined as MI_FOUND_WRONG_KEY in
  myisamdef.h. As a result, table considered to
  be crashed. 
  Fix to return -1,0 or 1.

mysql-test/r/ctype_ucs.result

@@ -630,3 +630,14 @@ Warnings:
 Warning	1265	Data truncated for column 'Field1' at row 1
 DROP TABLE t1;
 SET NAMES latin1;
+CREATE TABLE t1 (
+a varchar(255) NOT NULL default '',
+KEY a (a)
+) ENGINE=MyISAM DEFAULT CHARSET=ucs2 COLLATE ucs2_general_ci;
+insert into t1 values (0x803d);
+insert into t1 values (0x005b);
+select hex(a) from t1;
+hex(a)
+005B
+803D
+drop table t1;

mysql-test/r/ctype_utf8.result

@@ -939,3 +939,14 @@ content	msisdn
 ERR Имри.Афимим.Аеимимримдмримрмрирор имримримримр имридм ирбднримрфмририримрфмфмим.Ад.Д имдимримрад.Адимримримрмдиримримримр м.Дадимфшьмримд им.Адимимрн имадми	1234567890
 11 g	1234567890
 DROP TABLE t1,t2;
+CREATE TABLE t1 (
+a varchar(255) NOT NULL default '',
+KEY a (a)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE utf8_general_ci;
+insert into t1 values (_utf8 0xe880bd);
+insert into t1 values (_utf8 0x5b);
+select hex(a) from t1;
+hex(a)
+5B
+E880BD
+drop table t1;

mysql-test/t/ctype_ucs.test

@@ -404,3 +404,15 @@ CREATE TABLE t1 (Field1 int(10) unsigned default '0');
 INSERT INTO t1 VALUES ('-1');
 DROP TABLE t1;
 SET NAMES latin1;
+
+#
+# Bug#9557 MyISAM utf8 table crash
+#
+CREATE TABLE t1 (
+  a varchar(255) NOT NULL default '',
+  KEY a (a)
+) ENGINE=MyISAM DEFAULT CHARSET=ucs2 COLLATE ucs2_general_ci;
+insert into t1 values (0x803d);
+insert into t1 values (0x005b);
+select hex(a) from t1;
+drop table t1;

mysql-test/t/ctype_utf8.test

@@ -788,3 +788,15 @@ INSERT INTO t2 VALUES ('1234567890',2,'2005-05-24 13:53:25');
 SELECT content, t2.msisdn FROM t1, t2 WHERE t1.msisdn = '1234567890';
 
 DROP TABLE t1,t2;
+
+#
+# Bug#9557 MyISAM utf8 table crash
+#
+CREATE TABLE t1 (
+  a varchar(255) NOT NULL default '',
+  KEY a (a)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE utf8_general_ci;
+insert into t1 values (_utf8 0xe880bd);
+insert into t1 values (_utf8 0x5b);
+select hex(a) from t1;
+drop table t1;

strings/ctype-ucs2.c

@@ -209,7 +209,7 @@ static int my_strnncoll_ucs2(CHARSET_INFO *cs,
     t_wc = uni_plane[plane] ? uni_plane[plane][t_wc & 0xFF].sort : t_wc;
     if ( s_wc != t_wc )
     {
-      return  ((int) s_wc) - ((int) t_wc);
+      return  s_wc > t_wc ? 1 : -1;
     }
     
     s+=s_res;
@@ -267,7 +267,7 @@ static int my_strnncollsp_ucs2(CHARSET_INFO *cs __attribute__((unused)),
     int t_wc = uni_plane[t[0]] ? (int) uni_plane[t[0]][t[1]].sort : 
                                  (((int) t[0]) << 8) + (int) t[1];
     if ( s_wc != t_wc )
-      return  s_wc - t_wc;
+      return  s_wc > t_wc ? 1 : -1;
 
     s+= 2;
     t+= 2;
@@ -1343,7 +1343,7 @@ int my_strnncoll_ucs2_bin(CHARSET_INFO *cs,
     }
     if ( s_wc != t_wc )
     {
-      return  ((int) s_wc) - ((int) t_wc);
+      return  s_wc > t_wc ? 1 : -1;
     }
     
     s+=s_res;

strings/ctype-utf8.c

@@ -2000,7 +2000,7 @@ static int my_strnncoll_utf8(CHARSET_INFO *cs,
     t_wc = uni_plane[plane] ? uni_plane[plane][t_wc & 0xFF].sort : t_wc;
     if ( s_wc != t_wc )
     {
-      return  ((int) s_wc) - ((int) t_wc);
+      return  s_wc > t_wc ? 1 : -1;
     }
 
     s+=s_res;
@@ -2065,7 +2065,7 @@ static int my_strnncollsp_utf8(CHARSET_INFO *cs,
     t_wc = uni_plane[plane] ? uni_plane[plane][t_wc & 0xFF].sort : t_wc;
     if ( s_wc != t_wc )
     {
-      return  ((int) s_wc) - ((int) t_wc);
+      return  s_wc > t_wc ? 1 : -1;
     }
 
     s+=s_res;