Fix for bug #27643 "query failed : 1114 (The table '' is full)

Problem:

HASH indexes on VARCHAR columns with binary collations did not ignore trailing spaces from strings before comparisons. This could result in duplicate records being successfully inserted into a MEMORY table with unique key constraints.

As a direct consequence of the above, internal MEMORY tables used for GROUP BY calculation in testcases for bug #27643 contained duplicate rows which resulted in duplicate key errors when converting those temporary tables to MyISAM. Additionally, that error was incorrectly converted to the 'table is full' error.

Solution:

- ignore trailing spaces in VARCHAR fields with binary collations when calculating hashes.
- return a proper error from create_myisam_from_heap() when conversion fails.

mysql-test/r/ctype_ucs2_def.result

@@ -7,3 +7,11 @@ character_set_server	ucs2
 DROP TABLE IF EXISTS t1;
 create table t1 (a int);
 drop table t1;
+CREATE TABLE t1(col1 VARCHAR(32) CHARACTER SET ucs2 COLLATE ucs2_bin NOT NULL, 
+col2 VARCHAR(32) CHARACTER SET ucs2 COLLATE ucs2_bin NOT NULL, 
+UNIQUE KEY key1 USING HASH (col1, col2)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES('A', 'A'), ('B', 'B'), ('C', 'C');
+INSERT INTO t1 VALUES('A ', 'A ');
+ERROR 23000: Duplicate entry '' for key 1
+DROP TABLE t1;
+End of 5.0 tests

mysql-test/r/heap_hash.result

@@ -366,3 +366,19 @@ explain select a from t1 where a in (1,3);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	range	a	a	5	NULL	4	Using where
 drop table t1;
+End of 4.1 tests
+CREATE TABLE t1(col1 VARCHAR(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, 
+col2 VARCHAR(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, 
+UNIQUE KEY key1 USING HASH (col1, col2)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES('A', 'A');
+INSERT INTO t1 VALUES('A ', 'A ');
+ERROR 23000: Duplicate entry 'A -A ' for key 1
+DROP TABLE t1;
+CREATE TABLE t1(col1 VARCHAR(32) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, 
+col2 VARCHAR(32) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, 
+UNIQUE KEY key1 USING HASH (col1, col2)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES('A', 'A');
+INSERT INTO t1 VALUES('A ', 'A ');
+ERROR 23000: Duplicate entry 'A -A ' for key 1
+DROP TABLE t1;
+End of 5.0 tests

mysql-test/t/ctype_ucs2_def.test

@@ -14,3 +14,19 @@ DROP TABLE IF EXISTS t1;
 --enable_warnings
 create table t1 (a int);
 drop table t1;
+
+#
+# Bug #27643: query failed : 1114 (The table '' is full)
+#
+# Check that HASH indexes ignore trailing spaces when comparing 
+# strings with the ucs2_bin collation
+
+CREATE TABLE t1(col1 VARCHAR(32) CHARACTER SET ucs2 COLLATE ucs2_bin NOT NULL, 
+                col2 VARCHAR(32) CHARACTER SET ucs2 COLLATE ucs2_bin NOT NULL, 
+                UNIQUE KEY key1 USING HASH (col1, col2)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES('A', 'A'), ('B', 'B'), ('C', 'C');
+--error ER_DUP_ENTRY
+INSERT INTO t1 VALUES('A ', 'A ');
+DROP TABLE t1;
+
+--echo End of 5.0 tests

mysql-test/t/heap_hash.test

@@ -260,4 +260,27 @@ select a from t1 where a in (1,3);
 explain select a from t1 where a in (1,3);
 drop table t1;
 
-# End of 4.1 tests
+--echo End of 4.1 tests
+
+#
+# Bug #27643: query failed : 1114 (The table '' is full)
+#
+# Check that HASH indexes disregard trailing spaces when comparing 
+# strings with binary collations
+
+CREATE TABLE t1(col1 VARCHAR(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, 
+                col2 VARCHAR(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, 
+                UNIQUE KEY key1 USING HASH (col1, col2)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES('A', 'A');
+--error ER_DUP_ENTRY
+INSERT INTO t1 VALUES('A ', 'A ');
+DROP TABLE t1;
+CREATE TABLE t1(col1 VARCHAR(32) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, 
+                col2 VARCHAR(32) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, 
+                UNIQUE KEY key1 USING HASH (col1, col2)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES('A', 'A');
+--error ER_DUP_ENTRY
+INSERT INTO t1 VALUES('A ', 'A ');
+DROP TABLE t1;
+
+--echo End of 5.0 tests

sql/sql_select.cc

@@ -10112,7 +10112,7 @@ bool create_myisam_from_heap(THD *thd, TABLE *table, TMP_TABLE_PARAM *param,
 
  err:
   DBUG_PRINT("error",("Got error: %d",write_err));
-  table->file->print_error(error,MYF(0));	// Give table is full error
+  table->file->print_error(write_err, MYF(0));
   (void) table->file->ha_rnd_end();
   (void) new_table.file->close();
  err1:

strings/ctype-bin.c

@@ -271,6 +271,29 @@ static int my_wc_mb_bin(CHARSET_INFO *cs __attribute__((unused)),
 }
 
 
+void my_hash_sort_8bit_bin(CHARSET_INFO *cs __attribute__((unused)),
+		      const uchar *key, uint len,ulong *nr1, ulong *nr2)
+{
+  const uchar *pos = key;
+  
+  key+= len;
+  
+  /*
+     Remove trailing spaces. We have to do this to be able to compare
+    'A ' and 'A' as identical
+  */
+  while (key > pos && key[-1] == ' ')
+    key--;
+
+  for (; pos < (uchar*) key ; pos++)
+  {
+    nr1[0]^=(ulong) ((((uint) nr1[0] & 63)+nr2[0]) * 
+	     ((uint)*pos)) + (nr1[0] << 8);
+    nr2[0]+=3;
+  }
+}
+
+
 void my_hash_sort_bin(CHARSET_INFO *cs __attribute__((unused)),
 		      const uchar *key, uint len,ulong *nr1, ulong *nr2)
 {
@@ -471,7 +494,7 @@ MY_COLLATION_HANDLER my_collation_8bit_bin_handler =
     my_wildcmp_bin,
     my_strcasecmp_bin,
     my_instr_bin,
-    my_hash_sort_bin,
+    my_hash_sort_8bit_bin,
     my_propagate_simple
 };
 

strings/ctype-mb.c

@@ -467,6 +467,13 @@ static void my_hash_sort_mb_bin(CHARSET_INFO *cs __attribute__((unused)),
   
   key+= len;
   
+  /*
+     Remove trailing spaces. We have to do this to be able to compare
+    'A ' and 'A' as identical
+  */
+  while (key > pos && key[-1] == ' ')
+    key--;
+  
   for (; pos < (uchar*) key ; pos++)
   {
     nr1[0]^=(ulong) ((((uint) nr1[0] & 63)+nr2[0]) * 

strings/ctype-ucs2.c

@@ -1484,7 +1484,10 @@ void my_hash_sort_ucs2_bin(CHARSET_INFO *cs __attribute__((unused)),
   const uchar *pos = key;
   
   key+= len;
-  
+
+  while (key > pos+1 && key[-1] == ' ' && key[-2] == '\0')
+    key-= 2;
+
   for (; pos < (uchar*) key ; pos++)
   {
     nr1[0]^=(ulong) ((((uint) nr1[0] & 63)+nr2[0]) *