Fixed bug #31663: if the FIELDS TERMINATED BY string

in the SELECT INTO OUTFILE clause starts with a special
character (one of n, t, r, b, 0, Z or N) and ENCLOSED BY
is empty, every occurrence of this character within a
field value is duplicated.

Duplication has been avoided.
New warning message has been added: "First character of
the FIELDS TERMINATED string is ambiguous; please use
non-optional and non-empty FIELDS ENCLOSED BY".


mysql-test/r/outfile_loaddata.result:
  BitKeeper file /home/uchum/work/bk/5.0-opt-31663/mysql-test/r/outfile_loaddata.result
  Added test case for bug #31663.
mysql-test/t/outfile_loaddata.test:
  BitKeeper file /home/uchum/work/bk/5.0-opt-31663/mysql-test/t/outfile_loaddata.test
  Added test case for bug #31663.
sql/sql_class.h:
  Fixed bug #31663.
  The select_export::is_ambiguous_field_term field has been added.
  This field is true if select_export::field_sep_char contains
  the first char of the FIELDS TERMINATED BY (ENCLOSED BY is empty),
  and items can contain this character.
  The select_export::field_term_char field has been added (first
  char of the FIELDS TERMINATED BY string or INT_MAX).
sql/sql_class.cc:
  Fixed bug #31663.
  The select_export::prepare method has been modified to calculate
  a value of the select_export::is_ambiguous_field_term field and
  to warn if this value is true.
  The select_export::send_data method has been modified to
  avoid escaping or duplication of the field_set_char if
  is_ambiguous_field_term is true.
sql/share/errmsg.txt:
  Fixed bug #31663.
  The ER_AMBIGUOUS_FIELD_TERM warning has been added.

mysql-test/r/outfile_loaddata.result

+DROP TABLE IF EXISTS t1, t2;
+#
+# Bug#31663 FIELDS TERMINATED BY special character
+#
+CREATE TABLE t1 (i1 int, i2 int, c1 VARCHAR(256), c2 VARCHAR(256));
+INSERT INTO t1 VALUES (101, 202, '-r-', '=raker=');
+# FIELDS TERMINATED BY 'raker', warning:
+SELECT * INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' FIELDS TERMINATED BY 'raker' FROM t1;
+Warnings:
+Warning	1475	First character of the FIELDS TERMINATED string is ambiguous; please use non-optional and non-empty FIELDS ENCLOSED BY
+SELECT LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt');
+LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt')
+101raker202raker-r-raker=raker=
+
+CREATE TABLE t2 SELECT * FROM t1;
+LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 FIELDS TERMINATED BY 'raker';
+Warnings:
+Warning	1262	Row 1 was truncated; it contained more data than there were input columns
+SELECT * FROM t2;
+i1	i2	c1	c2
+101	202	-r-	=raker=
+101	202	-r-	=
+DROP TABLE t2;
+# Only numeric fields, FIELDS TERMINATED BY 'r', no warnings:
+SELECT i1, i2 INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' FIELDS TERMINATED BY 'r' FROM t1;
+SELECT LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt');
+LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt')
+101r202
+
+CREATE TABLE t2 SELECT i1, i2 FROM t1;
+LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 FIELDS TERMINATED BY 'r';
+SELECT i1, i2 FROM t2;
+i1	i2
+101	202
+101	202
+DROP TABLE t2;
+# FIELDS TERMINATED BY '0', warning:
+SELECT * INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' FIELDS TERMINATED BY '0' FROM t1;
+Warnings:
+Warning	1475	First character of the FIELDS TERMINATED string is ambiguous; please use non-optional and non-empty FIELDS ENCLOSED BY
+SELECT LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt');
+LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt')
+10102020-r-0=raker=
+
+CREATE TABLE t2 SELECT * FROM t1;
+LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 FIELDS TERMINATED BY '0';
+Warnings:
+Warning	1262	Row 1 was truncated; it contained more data than there were input columns
+SELECT * FROM t2;
+i1	i2	c1	c2
+101	202	-r-	=raker=
+1	1	2	2
+DROP TABLE t2;
+# FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0', warning:
+SELECT * INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0' FROM t1;
+Warnings:
+Warning	1475	First character of the FIELDS TERMINATED string is ambiguous; please use non-optional and non-empty FIELDS ENCLOSED BY
+SELECT LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt');
+LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt')
+10102020"-r-"0"=raker="
+
+CREATE TABLE t2 SELECT * FROM t1;
+LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0';
+Warnings:
+Warning	1262	Row 1 was truncated; it contained more data than there were input columns
+SELECT * FROM t2;
+i1	i2	c1	c2
+101	202	-r-	=raker=
+1	1	2	2
+DROP TABLE t2;
+# Only string fields, FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0', no warnings:
+SELECT c1, c2 INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0' FROM t1;
+SELECT LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt');
+LOAD_FILE('MYSQLTEST_VARDIR/tmp/bug31663.txt')
+"-r-"0"=raker="
+
+CREATE TABLE t2 SELECT c1, c2 FROM t1;
+LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0';
+SELECT c1, c2 FROM t2;
+c1	c2
+-r-	=raker=
+-r-	=raker=
+DROP TABLE t2;
+DROP TABLE t1;
+# End of 5.0 tests.

mysql-test/t/outfile_loaddata.test

+--disable_warnings
+DROP TABLE IF EXISTS t1, t2;
+--enable_warnings
+
+--echo #
+--echo # Bug#31663 FIELDS TERMINATED BY special character
+--echo #
+
+CREATE TABLE t1 (i1 int, i2 int, c1 VARCHAR(256), c2 VARCHAR(256));
+INSERT INTO t1 VALUES (101, 202, '-r-', '=raker=');
+
+--let $fields=*
+--let $clauses=FIELDS TERMINATED BY 'raker'
+--echo # $clauses, warning:
+
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT $fields INTO OUTFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' $clauses FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT LOAD_FILE('$MYSQLTEST_VARDIR/tmp/bug31663.txt')
+--eval CREATE TABLE t2 SELECT $fields FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 $clauses
+--eval SELECT $fields FROM t2
+--remove_file $MYSQLTEST_VARDIR/tmp/bug31663.txt
+DROP TABLE t2;
+
+--let $fields=i1, i2
+--let $clauses=FIELDS TERMINATED BY 'r'
+--echo # Only numeric fields, $clauses, no warnings:
+
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT $fields INTO OUTFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' $clauses FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT LOAD_FILE('$MYSQLTEST_VARDIR/tmp/bug31663.txt')
+--eval CREATE TABLE t2 SELECT $fields FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 $clauses
+--eval SELECT $fields FROM t2
+--remove_file $MYSQLTEST_VARDIR/tmp/bug31663.txt
+DROP TABLE t2;
+
+--let $fields=*
+--let $clauses=FIELDS TERMINATED BY '0'
+--echo # $clauses, warning:
+
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT $fields INTO OUTFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' $clauses FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT LOAD_FILE('$MYSQLTEST_VARDIR/tmp/bug31663.txt')
+--eval CREATE TABLE t2 SELECT $fields FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 $clauses
+--eval SELECT $fields FROM t2
+--remove_file $MYSQLTEST_VARDIR/tmp/bug31663.txt
+DROP TABLE t2;
+
+--let $fields=*
+--let $clauses=FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0'
+--echo # $clauses, warning:
+
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT $fields INTO OUTFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' $clauses FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT LOAD_FILE('$MYSQLTEST_VARDIR/tmp/bug31663.txt')
+--eval CREATE TABLE t2 SELECT $fields FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 $clauses
+--eval SELECT $fields FROM t2
+--remove_file $MYSQLTEST_VARDIR/tmp/bug31663.txt
+DROP TABLE t2;
+
+--let $fields=c1, c2
+--let $clauses=FIELDS OPTIONALLY ENCLOSED BY '"' TERMINATED BY '0'
+--echo # Only string fields, $clauses, no warnings:
+
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT $fields INTO OUTFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' $clauses FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval SELECT LOAD_FILE('$MYSQLTEST_VARDIR/tmp/bug31663.txt')
+--eval CREATE TABLE t2 SELECT $fields FROM t1
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/bug31663.txt' INTO TABLE t2 $clauses
+--eval SELECT $fields FROM t2
+--remove_file $MYSQLTEST_VARDIR/tmp/bug31663.txt
+DROP TABLE t2;
+
+DROP TABLE t1;
+
+--echo # End of 5.0 tests.

sql/share/errmsg.txt

@@ -5639,3 +5639,5 @@ ER_TOO_HIGH_LEVEL_OF_NESTING_FOR_SELECT
 	eng "Too high level of nesting for select"
 ER_NAME_BECOMES_EMPTY
         eng "Name '%-.64s' has become ''"
+ER_AMBIGUOUS_FIELD_TERM
+	eng "First character of the FIELDS TERMINATED string is ambiguous; please use non-optional and non-empty FIELDS ENCLOSED BY"

sql/sql_class.cc

@@ -1194,6 +1194,7 @@ int
 select_export::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
 {
   bool blob_flag=0;
+  bool string_results= FALSE, non_string_results= FALSE;
   unit= u;
   if ((uint) strlen(exchange->file_name) + NAME_LEN >= FN_REFLEN)
     strmake(path,exchange->file_name,FN_REFLEN-1);
@@ -1211,13 +1212,18 @@ select_export::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
 	blob_flag=1;
 	break;
       }
+      if (item->result_type() == STRING_RESULT)
+        string_results= TRUE;
+      else
+        non_string_results= TRUE;
     }
   }
   field_term_length=exchange->field_term->length();
+  field_term_char= field_term_length ? (*exchange->field_term)[0] : INT_MAX;
   if (!exchange->line_term->length())
     exchange->line_term=exchange->field_term;	// Use this if it exists
   field_sep_char= (exchange->enclosed->length() ? (*exchange->enclosed)[0] :
-		   field_term_length ? (*exchange->field_term)[0] : INT_MAX);
+                   field_term_char);
   escape_char=	(exchange->escaped->length() ? (*exchange->escaped)[0] : -1);
   is_ambiguous_field_sep= test(strchr(ESCAPE_CHARS, field_sep_char));
   is_unsafe_field_sep= test(strchr(NUMERIC_CHARS, field_sep_char));
@@ -1229,12 +1235,25 @@ select_export::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
     exchange->opt_enclosed=1;			// A little quicker loop
   fixed_row_size= (!field_term_length && !exchange->enclosed->length() &&
 		   !blob_flag);
+  if ((is_ambiguous_field_sep && exchange->enclosed->is_empty() &&
+       (string_results || is_unsafe_field_sep)) ||
+      (exchange->opt_enclosed && non_string_results &&
+       field_term_length && strchr(NUMERIC_CHARS, field_term_char)))
+  {
+    push_warning(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
+                 ER_AMBIGUOUS_FIELD_TERM, ER(ER_AMBIGUOUS_FIELD_TERM));
+    is_ambiguous_field_term= TRUE;
+  }
+  else
+    is_ambiguous_field_term= FALSE;
+
   return 0;
 }
 
 
 #define NEED_ESCAPING(x) ((int) (uchar) (x) == escape_char    || \
-                          (int) (uchar) (x) == field_sep_char || \
+                          (enclosed ? (int) (uchar) (x) == field_sep_char      \
+                                    : (int) (uchar) (x) == field_term_char) || \
                           (int) (uchar) (x) == line_sep_char  || \
                           !(x))
 
@@ -1263,8 +1282,10 @@ bool select_export::send_data(List<Item> &items)
   while ((item=li++))
   {
     Item_result result_type=item->result_type();
+    bool enclosed = (exchange->enclosed->length() &&
+                     (!exchange->opt_enclosed || result_type == STRING_RESULT));
     res=item->str_result(&tmp);
-    if (res && (!exchange->opt_enclosed || result_type == STRING_RESULT))
+    if (res && enclosed)
     {
       if (my_b_write(&cache,(byte*) exchange->enclosed->ptr(),
 		     exchange->enclosed->length()))
@@ -1355,11 +1376,16 @@ bool select_export::send_data(List<Item> &items)
             DBUG_ASSERT before the loop makes that sure.
           */
 
-          if (NEED_ESCAPING(*pos) ||
-              (check_second_byte &&
-               my_mbcharlen(character_set_client, (uchar) *pos) == 2 &&
-               pos + 1 < end &&
-               NEED_ESCAPING(pos[1])))
+          if ((NEED_ESCAPING(*pos) ||
+               (check_second_byte &&
+                my_mbcharlen(character_set_client, (uchar) *pos) == 2 &&
+                pos + 1 < end &&
+                NEED_ESCAPING(pos[1]))) &&
+              /*
+               Don't escape field_term_char by doubling - doubling is only
+               valid for ENCLOSED BY characters:
+              */
+              (enclosed || !is_ambiguous_field_term || *pos != field_term_char))
           {
 	    char tmp_buff[2];
             tmp_buff[0]= ((int) *pos == field_sep_char &&
@@ -1398,7 +1424,7 @@ bool select_export::send_data(List<Item> &items)
 	  goto err;
       }
     }
-    if (res && (!exchange->opt_enclosed || result_type == STRING_RESULT))
+    if (res && enclosed)
     {
       if (my_b_write(&cache, (byte*) exchange->enclosed->ptr(),
                      exchange->enclosed->length()))

sql/sql_class.h

@@ -1992,12 +1992,19 @@ public:
 class select_export :public select_to_file {
   uint field_term_length;
   int field_sep_char,escape_char,line_sep_char;
+  int field_term_char; // first char of FIELDS TERMINATED BY or MAX_INT
   /*
     The is_ambiguous_field_sep field is true if a value of the field_sep_char
     field is one of the 'n', 't', 'r' etc characters
     (see the READ_INFO::unescape method and the ESCAPE_CHARS constant value).
   */
   bool is_ambiguous_field_sep;
+  /*
+     The is_ambiguous_field_term is true if field_sep_char contains the first
+     char of the FIELDS TERMINATED BY (ENCLOSED BY is empty), and items can
+     contain this character.
+  */
+  bool is_ambiguous_field_term;
   /*
     The is_unsafe_field_sep field is true if a value of the field_sep_char
     field is one of the '0'..'9', '+', '-', '.' and 'e' characters