Bug#12565 - ERROR 1034 when running simple UPDATE or DELETE on large MyISAM table

Changed end-space comparison so that the key is not used past
its end. This is due to the new end-space behaviour in 4.1.
See also bug 6151 and 9188.

myisam/mi_search.c

@@ -316,19 +316,21 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
   get_key_pack_length(kseg_len,length_pack,kseg);
   key_len_skip=length_pack+kseg_len;
   key_len_left=(int) key_len- (int) key_len_skip;
+  /* If key_len is 0, then lenght_pack is 1, then key_len_left is -1. */
   cmplen=(key_len_left>=0) ? kseg_len : key_len-length_pack;
   DBUG_PRINT("info",("key: '%.*s'",kseg_len,kseg));
 
   /*
     Keys are compressed the following way:
 
-    If the max length of first key segment <= 127 characters the prefix is
+    If the max length of first key segment <= 127 bytes the prefix is
     1 byte else it's 2 byte
 
-    prefix         The high bit is set if this is a prefix for the prev key
-    length         Packed length if the previous was a prefix byte
-    [length]       Length character of data
-    next-key-seg   Next key segments
+    (prefix) length  The high bit is set if this is a prefix for the prev key.
+    [suffix length]  Packed length of suffix if the previous was a prefix.
+    (suffix) data    Key data bytes (past the common prefix or whole segment).
+    [next-key-seg]   Next key segments (([packed length], data), ...)
+    pointer          Reference to the data file (last_keyseg->length).
   */
 
   matched=0;  /* how many char's from prefix were alredy matched */
@@ -349,16 +351,23 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 
     if (packed)
     {
-      if (suffix_len == 0)                      /* Same key */
+      if (suffix_len == 0)
+      {
+        /* == 0x80 or 0x8000, same key, prefix length == old key length. */
         prefix_len=len;
+      }
       else
       {
+        /* > 0x80 or 0x8000, this is prefix lgt, packed suffix lgt follows. */
         prefix_len=suffix_len;
         get_key_length(suffix_len,vseg);
       }
     }
     else
+    {
+      /* Not packed. No prefix used from last key. */
       prefix_len=0;
+    }
 
     len=prefix_len+suffix_len;
     seg_len_pack=get_pack_length(len);
@@ -414,7 +423,12 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
       uint  left;
       uchar *k=kseg+prefix_len;
 
-      left=(len>cmplen) ? cmplen-prefix_len : suffix_len;
+      /*
+        If prefix_len > cmplen then we are in the end-space comparison
+        phase. Do not try to acces the key any more ==> left= 0.
+      */
+      left= ((len <= cmplen) ? suffix_len :
+             ((prefix_len < cmplen) ? cmplen - prefix_len : 0));
 
       matched=prefix_len+left;
 
@@ -451,7 +465,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 	    my_flag= -1;
 	  else
 	  {
-	    /* We have to compare k and vseg as if they where space extended */
+	    /* We have to compare k and vseg as if they were space extended */
 	    uchar *end= k+ (cmplen - len);
 	    for ( ; k < end && *k == ' '; k++) ;
 	    if (k == end)
@@ -470,7 +484,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 	  if ((nextflag & SEARCH_PREFIX) && key_len_left == 0)
 	    goto fix_flag;
 
-	  /* We have to compare k and vseg as if they where space extended */
+	  /* We have to compare k and vseg as if they were space extended */
 	  for (end=vseg + (len-cmplen) ;
 	       vseg < end && *vseg == (uchar) ' ';
 	       vseg++, matched++) ;

mysql-test/r/key.result

@@ -325,3 +325,19 @@ ERROR 42S21: Duplicate column name 'c1'
 alter table t1 add key (c1,c1,c2);
 ERROR 42S21: Duplicate column name 'c1'
 drop table t1;
+create table t1 (
+c1 int,
+c2 varchar(20) not null,
+primary key (c1),
+key (c2(10))
+) engine=myisam;
+insert into t1 values (1,'');
+insert into t1 values (2,' \t\tTest String');
+insert into t1 values (3,' \n\tTest String');
+update t1 set c2 = 'New Test String' where c1 = 1;
+select * from t1;
+c1	c2
+1	New Test String
+2	 		Test String
+3	 
+	Test String

mysql-test/t/key.test

@@ -321,4 +321,20 @@ alter table t1 add key (c1,c2,c1);
 alter table t1 add key (c1,c1,c2);
 drop table t1;
 
+#
+# Bug#12565 - ERROR 1034 when running simple UPDATE or DELETE 
+#             on large MyISAM table
+#
+create table t1 (
+  c1 int,
+  c2 varchar(20) not null,
+  primary key (c1),
+  key (c2(10))
+) engine=myisam;
+insert into t1 values (1,'');
+insert into t1 values (2,' \t\tTest String');
+insert into t1 values (3,' \n\tTest String');
+update t1 set c2 = 'New Test String' where c1 = 1;
+select * from t1;
+
 # End of 4.1 tests