Bug#15126 character_set_database is not replicated (LOAD DATA INFILE need it)

This patch fixes problem that LOAD DATA could use different
character sets when loading files on master and on slave sides:
- Adding replication of thd->variables.collation_database
- Adding optional character set clause into LOAD DATA

Note, the second way, with explicit CHARACTER SET clause
should be the recommended way to load data using an alternative
character set.
The old way, using "SET @@character_set_database=xxx" should be
gradually depricated.


mysql-test/r/mysqlbinlog.result:
  Adding test case
mysql-test/t/mysqlbinlog.test:
  Adding test case
sql/log_event.cc:
  Adding logging of thd->variables.collation_database
sql/log_event.h:
  Adding declarations
sql/sql_class.cc:
  Exchange character set is null by default
sql/sql_class.h:
  Adding character set into sql_exchange
sql/sql_load.cc:
  - Using exchange character set (if it was specified in LOAD DATA syntax)
  - Using thd->variables.collation_database by default
sql/sql_yacc.yy:
  Adding optional character set clause into LOAD DATA syntax
mysql-test/r/rpl_loaddata2.result:
  New BitKeeper file ``mysql-test/r/rpl_loaddata2.result''
mysql-test/std_data/loaddata6.dat:
  New BitKeeper file ``mysql-test/std_data/loaddata6.dat''
mysql-test/t/rpl_loaddata2.test:
  New BitKeeper file ``mysql-test/t/rpl_loaddata2.test''

mysql-test/r/mysqlbinlog.result

@@ -270,3 +270,61 @@ call p1();
 1
 drop procedure p1;
 drop table t1, t2, t03, t04, t3, t4, t5;
+flush logs;
+create table t1 (a varchar(64) character set utf8);
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=latin1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=latin1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1 character set koi8r;
+select hex(a) from t1;
+hex(a)
+C3BF
+D0AA
+C3BF
+C3BF
+D0AA
+C3BF
+D0AA
+drop table t1;
+flush logs;
+/*!40019 SET @@session.max_insert_delayed_threads=0*/;
+/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/;
+DELIMITER /*!*/;
+use test/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=1, @@session.unique_checks=1/*!*/;
+SET @@session.sql_mode=0/*!*/;
+/*!\C latin1 *//*!*/;
+SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/;
+create table t1 (a varchar(64) character set utf8)/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-6-0' INTO table t1/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+SET @@session.collation_database=7/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-7-0' INTO table t1/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+SET @@session.collation_database=DEFAULT/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-8-0' INTO table t1/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-9-0' INTO table t1/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+SET @@session.collation_database=7/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-a-0' INTO table t1/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+SET @@session.collation_database=DEFAULT/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-b-0' INTO table t1/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+load data LOCAL INFILE '/home/bar/mysql-5.0.b15126/mysql-test/var/tmp/SQL_LOAD_MB-c-0' INTO table t1 character set koi8r/*!*/;
+SET TIMESTAMP=1000000000/*!*/;
+drop table t1/*!*/;
+DELIMITER ;
+# End of log file
+ROLLBACK /* added by mysqlbinlog */;
+/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;

mysql-test/r/rpl_loaddata2.result

+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+create table t1 (a varchar(10) character set utf8);
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=DEFAULT;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=DEFAULT;
+load data infile '../std_data_ln/loaddata6.dat' into table t1 character set koi8r;
+select hex(a) from t1;
+hex(a)
+C3BF
+D0AA
+C3BF
+C3BF
+C3BF
+D0AA
+D0AA
+select hex(a) from t1;
+hex(a)
+C3BF
+D0AA
+C3BF
+C3BF
+C3BF
+D0AA
+D0AA
+drop table t1;

mysql-test/std_data/loaddata6.dat

+

mysql-test/t/mysqlbinlog.test

@@ -179,4 +179,28 @@ drop procedure p1;
 # clean up
 drop table t1, t2, t03, t04, t3, t4, t5; 
 
+#
+# Bug#15126 character_set_database is not replicated
+# (LOAD DATA INFILE need it)
+#
+
+flush logs;
+create table t1 (a varchar(64) character set utf8);
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=latin1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set character_set_database=latin1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1 character set koi8r;
+select hex(a) from t1;
+drop table t1;
+flush logs;
+--exec $MYSQL_BINLOG --short-form $MYSQLTEST_VARDIR/log/master-bin.000011
+
 # End of 5.0 tests
+

mysql-test/t/rpl_loaddata2.test

+#
+# Check LOAD DATA + character sets + replication
+#
+source include/master-slave.inc;
+
+#
+# Bug#15126 character_set_database is not replicated
+# (LOAD DATA INFILE need it)
+#
+connection master;
+create table t1 (a varchar(10) character set utf8);
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=DEFAULT;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=koi8r;
+load data infile '../std_data_ln/loaddata6.dat' into table t1;
+set @@character_set_database=DEFAULT;
+load data infile '../std_data_ln/loaddata6.dat' into table t1 character set koi8r;
+
+select hex(a) from t1;
+
+save_master_pos;
+connection slave;
+sync_with_master;
+
+select hex(a) from t1;
+connection master;
+drop table t1;
+sync_slave_with_master;

sql/log_event.cc

@@ -1088,7 +1088,8 @@ bool Query_log_event::write(IO_CACHE* file)
             1+4+           // code of autoinc and the 2 autoinc variables
             1+6+           // code of charset and charset
             1+1+MAX_TIME_ZONE_NAME_LENGTH+ // code of tz and tz length and tz name
-            1+2            // code of lc_time_names and lc_time_names_number
+            1+2+           // code of lc_time_names and lc_time_names_number
+            1+2            // code of charset_database and charset_database_number
             ], *start, *start_of_status;
   ulong event_length;
 
@@ -1207,6 +1208,13 @@ bool Query_log_event::write(IO_CACHE* file)
     int2store(start, lc_time_names_number);
     start+= 2;
   }
+  if (charset_database_number)
+  {
+    DBUG_ASSERT(charset_database_number <= 0xFFFF);
+    *start++= Q_CHARSET_DATABASE_CODE;
+    int2store(start, charset_database_number);
+    start+= 2;
+  }
   /*
     Here there could be code like
     if (command-line-option-which-says-"log_this_variable" && inited)
@@ -1272,7 +1280,8 @@ Query_log_event::Query_log_event(THD* thd_arg, const char* query_arg,
    sql_mode(thd_arg->variables.sql_mode),
    auto_increment_increment(thd_arg->variables.auto_increment_increment),
    auto_increment_offset(thd_arg->variables.auto_increment_offset),
-   lc_time_names_number(thd_arg->variables.lc_time_names->number)
+   lc_time_names_number(thd_arg->variables.lc_time_names->number),
+   charset_database_number(0)
 {
   time_t end_time;
   time(&end_time);
@@ -1280,6 +1289,9 @@ Query_log_event::Query_log_event(THD* thd_arg, const char* query_arg,
   catalog_len = (catalog) ? (uint32) strlen(catalog) : 0;
   /* status_vars_len is set just before writing the event */
   db_len = (db) ? (uint32) strlen(db) : 0;
+  if (thd_arg->variables.collation_database != thd_arg->db_charset)
+    charset_database_number= thd_arg->variables.collation_database->number;
+  
   /*
     If we don't use flags2 for anything else than options contained in
     thd->options, it would be more efficient to flags2=thd_arg->options
@@ -1350,7 +1362,7 @@ Query_log_event::Query_log_event(const char* buf, uint event_len,
    db(NullS), catalog_len(0), status_vars_len(0),
    flags2_inited(0), sql_mode_inited(0), charset_inited(0),
    auto_increment_increment(1), auto_increment_offset(1),
-   time_zone_len(0), lc_time_names_number(0)
+   time_zone_len(0), lc_time_names_number(0), charset_database_number(0)
 {
   ulong data_len;
   uint32 tmp;
@@ -1455,6 +1467,10 @@ Query_log_event::Query_log_event(const char* buf, uint event_len,
       lc_time_names_number= uint2korr(pos);
       pos+= 2;
       break;
+    case Q_CHARSET_DATABASE_CODE:
+      charset_database_number= uint2korr(pos);
+      pos+= 2;
+      break;
     default:
       /* That's why you must write status vars in growing order of code */
       DBUG_PRINT("info",("Query_log_event has unknown status vars (first has\
@@ -1652,6 +1668,16 @@ void Query_log_event::print_query_header(FILE* file,
             lc_time_names_number, print_event_info->delimiter);
     print_event_info->lc_time_names_number= lc_time_names_number;
   }
+  if (charset_database_number != print_event_info->charset_database_number)
+  {
+    if (charset_database_number)
+      fprintf(file, "SET @@session.collation_database=%d%s\n",
+              charset_database_number, print_event_info->delimiter);
+    else
+      fprintf(file, "SET @@session.collation_database=DEFAULT%s\n",
+              print_event_info->delimiter);
+    print_event_info->charset_database_number= charset_database_number;
+  }
 }
 
 
@@ -1817,7 +1843,21 @@ int Query_log_event::exec_event(struct st_relay_log_info* rli,
       }
       else
         thd->variables.lc_time_names= &my_locale_en_US;
-
+      if (charset_database_number)
+      {
+        CHARSET_INFO *cs;
+        if (!(cs= get_charset(charset_database_number, MYF(0))))
+        {
+          char buf[20];
+          int10_to_str((int) charset_database_number, buf, -10);
+          my_error(ER_UNKNOWN_COLLATION, MYF(0), buf);
+          goto compare_errors;
+        }
+        thd->variables.collation_database= cs;
+      }
+      else
+        thd->variables.collation_database= thd->db_charset;
+      
       /* Execute the query (note that we bypass dispatch_command()) */
       mysql_parse(thd, thd->query, thd->query_length);
 

sql/log_event.h

@@ -272,6 +272,7 @@ struct sql_ex_info
 
 #define Q_LC_TIME_NAMES_CODE    7
 
+#define Q_CHARSET_DATABASE_CODE 8
 /* Intvar event post-header */
 
 #define I_TYPE_OFFSET        0
@@ -509,10 +510,11 @@ typedef struct st_print_event_info
   char charset[6]; // 3 variables, each of them storable in 2 bytes
   char time_zone_str[MAX_TIME_ZONE_NAME_LENGTH];
   uint lc_time_names_number;
+  uint charset_database_number;
   st_print_event_info()
     :flags2_inited(0), sql_mode_inited(0),
      auto_increment_increment(1),auto_increment_offset(1), charset_inited(0),
-     lc_time_names_number(0)
+     lc_time_names_number(0), charset_database_number(0)
     {
       /*
         Currently we only use static PRINT_EVENT_INFO objects, so zeroed at
@@ -797,6 +799,7 @@ public:
   uint time_zone_len; /* 0 means uninited */
   const char *time_zone_str;
   uint lc_time_names_number; /* 0 means en_US */
+  uint charset_database_number;
 
 #ifndef MYSQL_CLIENT
 

sql/sql_class.cc

@@ -902,6 +902,7 @@ sql_exchange::sql_exchange(char *name,bool flag)
   enclosed=   line_start= &my_empty_string;
   line_term=  &default_line_term;
   escaped=    &default_escaped;
+  cs= NULL;
 }
 
 bool select_send::send_fields(List<Item> &list, uint flags)

sql/sql_class.h

@@ -1687,6 +1687,7 @@ public:
   bool opt_enclosed;
   bool dumpfile;
   ulong skip_lines;
+  CHARSET_INFO *cs;
   sql_exchange(char *name,bool dumpfile_flag);
 };
 

sql/sql_load.cc

@@ -313,7 +313,8 @@ bool mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list,
   info.handle_duplicates=handle_duplicates;
   info.escape_char=escaped->length() ? (*escaped)[0] : INT_MAX;
 
-  READ_INFO read_info(file,tot_length,thd->variables.collation_database,
+  READ_INFO read_info(file,tot_length,
+                      ex->cs ? ex->cs : thd->variables.collation_database,
 		      *field_term,*ex->line_start, *ex->line_term, *enclosed,
 		      info.escape_char, read_file_from_client, is_fifo);
   if (read_info.error)

sql/sql_yacc.yy

@@ -992,6 +992,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 	old_or_new_charset_name_or_default
 	collation_name
 	collation_name_or_default
+	opt_load_data_charset
 
 %type <variable> internal_variable_name
 
@@ -3262,6 +3263,10 @@ charset_name_or_default:
 	charset_name { $$=$1;   }
 	| DEFAULT    { $$=NULL; } ;
 
+opt_load_data_charset:
+	/* Empty */ { $$= NULL; }
+	| charset charset_name_or_default { $$= $2; }
+	;
 
 old_or_new_charset_name:
 	ident_or_text
@@ -7214,6 +7219,8 @@ load_data:
           lex->update_list.empty();
           lex->value_list.empty();
         }
+        opt_load_data_charset
+	{ Lex->exchange->cs= $12; }
         opt_field_term opt_line_term opt_ignore_lines opt_field_or_var_spec
         opt_load_data_set_spec
         {}