BUG#23175 - MYISAM crash/repair failed during repair

Repair table could crash a server if there is not sufficient
memory (myisam_sort_buffer_size) to operate. Affects not only
repair, but also all statements that use create index by sort:
repair by sort, parallel repair, bulk insert.

Return an error if there is not sufficient memory to store at
least one key per BUFFPEK.

Also fixed memory leak if thr_find_all_keys returns an error.

myisam/sort.c

@@ -148,7 +148,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
 	skr=maxbuffer;
 	if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
 	    (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
-	     (sort_length+sizeof(char*))) <= 1)
+             (sort_length+sizeof(char*))) <= 1 ||
+            keys < (uint) maxbuffer)
 	{
 	  mi_check_print_error(info->sort_info->param,
 			       "sort_buffer_size is to small");
@@ -363,7 +364,8 @@ pthread_handler_decl(thr_find_all_keys,arg)
         skr=maxbuffer;
         if (memavl < sizeof(BUFFPEK)*maxbuffer ||
             (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
-             (sort_length+sizeof(char*))) <= 1)
+             (sort_length+sizeof(char*))) <= 1 ||
+            keys < (uint) maxbuffer)
         {
           mi_check_print_error(sort_param->sort_info->param,
                                "sort_buffer_size is to small");
@@ -497,6 +499,8 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
     if (!sinfo->sort_keys)
     {
       got_error=1;
+      my_free(mi_get_rec_buff_ptr(info, sinfo->rec_buff),
+              MYF(MY_ALLOW_ZERO_PTR));
       continue;
     }
     if (!got_error)

mysql-test/r/repair.result

@@ -48,3 +48,31 @@ Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_par
 t1	1	a	1	a	A	5	NULL	NULL	YES	BTREE	
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 DROP TABLE t1;
+CREATE TABLE t1(a CHAR(255), KEY(a));
+SET myisam_sort_buffer_size=4096;
+INSERT INTO t1 VALUES
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0');
+SET myisam_repair_threads=2;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	error	sort_buffer_size is to small
+test.t1	repair	warning	Number of rows changed from 0 to 157
+test.t1	repair	status	OK
+SET myisam_repair_threads=@@global.myisam_repair_threads;
+SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
+DROP TABLE t1;

mysql-test/t/repair.test

@@ -45,4 +45,32 @@ SHOW INDEX FROM t1;
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 DROP TABLE t1;
 
+#
+# BUG#23175 - MYISAM crash/repair failed during repair
+#
+CREATE TABLE t1(a CHAR(255), KEY(a));
+SET myisam_sort_buffer_size=4096;
+INSERT INTO t1 VALUES
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0');
+SET myisam_repair_threads=2;
+REPAIR TABLE t1;
+SET myisam_repair_threads=@@global.myisam_repair_threads;
+SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
+DROP TABLE t1;
+
 # End of 4.1 tests