MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)

Also, fixing compilation warnings in ctype-mb.ic (Windows).

mysql-test/r/ctype_ucs.result

@@ -5640,5 +5640,14 @@ Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c`,`test`.`t1`.`d` AS `d` from `test`.`t1` where ((coalesce(`test`.`t1`.`c`,0) = '3 ') and (coalesce(`test`.`t1`.`d`,0) = '3 '))
 DROP TABLE t1;
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED);
+CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1IJ3'
+#
 # End of 10.1 tests
 #

mysql-test/r/ctype_utf16.result

@@ -2190,5 +2190,14 @@ COUNT(DISTINCT a)
 7
 DROP TABLE t1;
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING utf16) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING utf16) AS SIGNED);
+CAST(CONVERT('1IJ3' USING utf16) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1IJ3'
+#
 # End of 10.1 tests
 #

mysql-test/r/ctype_utf16le.result

@@ -2382,5 +2382,14 @@ COUNT(DISTINCT a)
 7
 DROP TABLE t1;
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING utf16le) AS SIGNED);
+CAST(CONVERT('1IJ3' USING utf16le) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1IJ3'
+#
 # End of 10.1 tests
 #

mysql-test/r/ctype_utf32.result

@@ -2226,5 +2226,14 @@ SELECT _utf32 0x10001=_utf32 0x10002;
 _utf32 0x10001=_utf32 0x10002
 1
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING utf32) AS SIGNED);
+CAST(CONVERT('1IJ3' USING utf32) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1IJ3'
+#
 # End of 10.1 tests
 #

mysql-test/t/ctype_ucs.test

@@ -948,6 +948,11 @@ EXPLAIN EXTENDED
 SELECT * FROM t1 WHERE COALESCE(c,0)='3 ' AND COALESCE(d,0)=COALESCE(c,0);
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED);
 
 --echo #
 --echo # End of 10.1 tests

mysql-test/t/ctype_utf16.test

@@ -885,6 +885,13 @@ SELECT id,HEX(a) FROM t1 ORDER BY a DESC,id DESC;
 SELECT COUNT(DISTINCT a) FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING utf16) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING utf16) AS SIGNED);
+
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

mysql-test/t/ctype_utf16le.test

@@ -769,6 +769,12 @@ SELECT id,HEX(a) FROM t1 ORDER BY a DESC,id DESC;
 SELECT COUNT(DISTINCT a) FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING utf16le) AS SIGNED);
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

mysql-test/t/ctype_utf32.test

@@ -970,6 +970,12 @@ SELECT COUNT(DISTINCT a) FROM t1;
 DROP TABLE t1;
 SELECT _utf32 0x10001=_utf32 0x10002;
 
+--echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1IJ3' USING ucs2) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1IJ3' USING utf32) AS SIGNED);
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

strings/ctype-mb.ic

@@ -269,7 +269,7 @@ my_native_to_mb_fixed2(my_wc_t wc, uchar *s, uchar *e)
 {
   /* The caller must insure there is a space for at least one byte */
   DBUG_ASSERT(s < e);
-  s[0]= wc >> 8;
+  s[0]= (uchar) (wc >> 8);
   if (s + 2 > e)
     return MY_CS_TOOSMALL2;
   s[1]= wc & 0xFF;
@@ -286,7 +286,7 @@ my_native_to_mb_fixed3(my_wc_t wc, uchar *s, uchar *e)
 {
   /* The caller must insure there is a space for at least one byte */
   DBUG_ASSERT(s < e);
-  s[0]= wc >> 16;
+  s[0]= (uchar) (wc >> 16);
   if (s + 2 > e)
     return MY_CS_TOOSMALL2;
   s[1]= (wc >> 8) & 0xFF;

strings/ctype-ucs2.c

@@ -818,6 +818,21 @@ cnv:
 
 
 #ifdef HAVE_CHARSET_mb2
+/**
+  Convert a Unicode code point to a digit.
+  @param      wc  - the input Unicode code point
+  @param[OUT] c   - the output character representing the digit value 0..9
+
+  @return   0     - if wc is a good digit
+  @return   1     - if wc is not a digit
+*/
+static inline my_bool
+wc2digit_uchar(uchar *c, my_wc_t wc)
+{
+  return wc > '9' || (c[0]= (uchar) (wc - '0')) > 9;
+}
+
+
 static longlong
 my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
                  const char *nptr, char **endptr, int *error)
@@ -921,7 +936,7 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
   {
     if ((res= mb_wc(cs, &wc, s, n_end)) <= 0)
       break;
-    if ((c= (wc - '0')) > 9)
+    if (wc2digit_uchar(&c, wc))
       goto end_i;
     i= i*10+c;
   }
@@ -938,7 +953,7 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
   {
     if ((res= mb_wc(cs, &wc, s, end)) <= 0)
       goto no_conv;
-    if ((c= (wc - '0')) > 9)
+    if (wc2digit_uchar(&c, wc))
       goto end_i_and_j;
     s+= res;
     j= j * 10 + c;
@@ -961,7 +976,7 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
     goto end4;
   if ((res= mb_wc(cs, &wc, s, end)) <= 0)
     goto no_conv;
-  if ((c= (wc - '0')) > 9)
+  if (wc2digit_uchar(&c, wc))
     goto end4;
   s+= res;
   k= k*10+c;