Add 5.0 part of fix for bug 17047.

mysql-test/r/func_str.result:
  Add result for bug#17047: CHAR() and IN() can return NULL without
  signaling NULL result.
mysql-test/t/func_str.test:
  Add test case for bug#17047: CHAR() and IN() can return NULL without
  signaling NULL result.
sql/item_strfunc.cc:
  Add Item_str_func::fix_fields() implementation, and set maybe_null to
  TRUE if we are in the SQL mode that requires some functions to return
  null even if they normally do not.
sql/item_strfunc.h:
  Add declaration of Item_str_func::fix_fields().
  Do not reset maybe_null in Item_func_char::fix_length_and_dec().

mysql-test/r/func_str.result

@@ -1148,4 +1148,17 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 Warnings:
 Note	1003	select `test`.`t1`.`code` AS `code`,`test`.`t2`.`id` AS `id` from `test`.`t1` join `test`.`t2` where ((`test`.`t1`.`code` = _latin1'a12') and (length(`test`.`t1`.`code`) = 5))
 DROP TABLE t1,t2;
+SET @orig_sql_mode = @@SQL_MODE;
+SET SQL_MODE=traditional;
+SELECT CHAR(0xff,0x8f USING utf8);
+CHAR(0xff,0x8f USING utf8)
+NULL
+Warnings:
+Error	1300	Invalid utf8 character string: 'FF8F'
+SELECT CHAR(0xff,0x8f USING utf8) IS NULL;
+CHAR(0xff,0x8f USING utf8) IS NULL
+1
+Warnings:
+Error	1300	Invalid utf8 character string: 'FF8F'
+SET SQL_MODE=@orig_sql_mode;
 End of 5.0 tests

mysql-test/t/func_str.test

@@ -780,4 +780,18 @@ SELECT * FROM t1 INNER JOIN t2 ON code=id
 
 DROP TABLE t1,t2;
 
+
+#
+# BUG#17047: CHAR() and IN() can return NULL without signaling NULL
+# result
+#
+SET @orig_sql_mode = @@SQL_MODE;
+SET SQL_MODE=traditional;
+
+SELECT CHAR(0xff,0x8f USING utf8);
+SELECT CHAR(0xff,0x8f USING utf8) IS NULL;
+
+SET SQL_MODE=@orig_sql_mode;
+
+
 --echo End of 5.0 tests

sql/item_strfunc.cc

@@ -80,6 +80,20 @@ String *Item_str_func::check_well_formed_result(String *str)
 }
 
 
+bool Item_str_func::fix_fields(THD *thd, Item **ref)
+{
+  bool res= Item_func::fix_fields(thd, ref);
+  /*
+    In Item_str_func::check_well_formed_result() we may set null_value
+    flag on the same condition as in test() below.
+  */
+  maybe_null= (maybe_null ||
+               test(thd->variables.sql_mode &
+                    (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES)));
+  return res;
+}
+
+
 my_decimal *Item_str_func::val_decimal(my_decimal *decimal_value)
 {
   DBUG_ASSERT(fixed == 1);

sql/item_strfunc.h

@@ -37,6 +37,7 @@ public:
   enum Item_result result_type () const { return STRING_RESULT; }
   void left_right_max_length();
   String *check_well_formed_result(String *str);
+  bool fix_fields(THD *thd, Item **ref);
 };
 
 class Item_func_md5 :public Item_str_func
@@ -525,9 +526,8 @@ public:
   { collation.set(cs); }  
   String *val_str(String *);
   void fix_length_and_dec() 
-  { 
-    maybe_null=0;
-    max_length=arg_count * collation.collation->mbmaxlen;
+  {
+    max_length= arg_count * collation.collation->mbmaxlen;
   }
   const char *func_name() const { return "char"; }
 };