Added some fixes that should make MyISAM & Aria REPAIR work with more than 4G records

- If one specifies --force twice to myisamchk and aria_chk, then we will try to finnish the repair even if sort_buffer would be too small.
  This was done by dynamically allocate buffer handler objects as long as memory lasts.
- New option for myisamchk and aria_chk: create-missing-keys
- Changed default size of myisam_sort_buffer_size from 8M to 128M.
- Changed default size of sort_buffer_size in aria_chk from 128M to 256M.
- Increased information in error message about 'sort_buffer_size' beeing to small.
- Print also to 'show warnings' if repair was retried.
- Increased size of internal sort-buffer-readers from 16K to 128K
- Changed printing of 'number of records' to use %ll instead of casting to long
- Changed buffer sizes for myisam and aria to use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines.





include/my_global.h:
  Added MY_ALIGN_DOWN() to get previous alignment (for big memory areas)
include/myisam.h:
  Increased size of types to be able to handle more records
include/myisamchk.h:
  Increased size of types to be able to handle more records
  Added T_FORCE_SORT_MEMORY to force repair to work even if sort_buffer would not be big enough
mysql-test/r/myisam.result:
  Updated result
mysql-test/r/mysqld--help.result:
  Updated result
mysql-test/r/repair.result:
  Updated result
mysql-test/suite/maria/maria.result:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria.test:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria3.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test:
  Updated result after sort buffer size increase
mysql-test/t/myisam.test:
  Fixed error messages to not print system specific data
mysql-test/t/repair.test:
  Fixed error messages to not print system specific data
storage/maria/ha_maria.cc:
  Print also to 'show warnings' if repair was retried
  Changed default size of sort_buffer_size from 128M to 256M (same as in mysqld)
storage/maria/ma_check.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/ma_sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug where maria_update_key_parts() was called too early.
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
storage/maria/maria_chk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
storage/maria/maria_def.h:
  Increased size of types to be able to handle more records
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_ftdump.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_read_log.c:
  Use PAGE_BUFFER_INIT for page cache
storage/myisam/ha_myisam.cc:
  Changed default size of myisam_sort_buffer_size from 8M to 128M
storage/myisam/mi_check.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisam_ftdump.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamchk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamdef.h:
  Increased SORT_BUFFER_INIT to 64M
  (speeds up repair a lot and most machines have nowadays a lot of memory)
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.

include/my_global.h

@@ -848,6 +848,7 @@ typedef long long	my_ptrdiff_t;
 #endif
 
 #define MY_ALIGN(A,L)	   (((A) + (L) - 1) & ~((L) - 1))
+#define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1))
 #define ALIGN_SIZE(A)	MY_ALIGN((A),sizeof(double))
 #define ALIGN_MAX_UNIT  (sizeof(double))
 /* Size to make adressable obj. */

include/myisam.h

@@ -356,8 +356,10 @@ typedef struct st_mi_sort_param
   ulonglong notnull[HA_MAX_KEY_SEG+1];
 
   my_off_t pos,max_pos,filepos,start_recpos;
-  uint key, key_length,real_key_length,sortbuff_size;
-  uint maxbuffers, keys, find_length, sort_keys_length;
+  uint key, key_length,real_key_length;
+  uint maxbuffers, find_length;
+  ulonglong sortbuff_size;
+  ha_rows keys;
   my_bool fix_datafile, master;
   my_bool calc_checksum;                /* calculate table checksum */
 
@@ -366,10 +368,10 @@ typedef struct st_mi_sort_param
   int (*key_write)(struct st_mi_sort_param *, const void *);
   void (*lock_in_memory)(HA_CHECK *);
   int (*write_keys)(struct st_mi_sort_param *, register uchar **,
-                    uint , struct st_buffpek *, IO_CACHE *);
-  uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
+                    ulonglong , struct st_buffpek *, IO_CACHE *);
+  my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
   int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *,
-                   uint, uint);
+                   uint, ulonglong);
 } MI_SORT_PARAM;
 
 /* functions in mi_check */

include/myisamchk.h

@@ -62,9 +62,10 @@
 #define T_ZEROFILL              ((ulonglong) 1L << 32)
 #define T_ZEROFILL_KEEP_LSN     ((ulonglong) 1L << 33)
 /** If repair should not bump create_rename_lsn */
-#define T_NO_CREATE_RENAME_LSN  ((ulonglong) 1L << 33)
-#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 34)
-#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 35)
+#define T_NO_CREATE_RENAME_LSN  ((ulonglong) 1L << 34)
+#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 35)
+#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 36)
+#define T_FORCE_SORT_MEMORY     ((ulonglong) 1L << 37)
 
 #define T_REP_ANY               (T_REP | T_REP_BY_SORT | T_REP_PARALLEL)
 
@@ -180,8 +181,8 @@ typedef struct st_buffpek {
   my_off_t file_pos;                    /* Where we are in the sort file */
   uchar *base, *key;                    /* Key pointers */
   ha_rows count;                        /* Number of rows in table */
-  ulong mem_count;                      /* numbers of keys in memory */
-  ulong max_keys;                       /* Max keys in buffert */
+  ha_rows mem_count;                    /* Numbers of keys in memory */
+  ha_rows max_keys;                     /* Max keys in buffert */
 } BUFFPEK;
 
 #endif /* _myisamchk_h */

mysql-test/r/myisam.result

@@ -2355,7 +2355,7 @@ Warnings:
 Warning	1292	Truncated incorrect myisam_sort_buffer_size value: '4'
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	repair	error	myisam_sort_buffer_size is too small
+test.t1	repair	error	myisam_sort_buffer_size is too small. X
 test.t1	repair	warning	Number of rows changed from 0 to 7168
 test.t1	repair	status	OK
 SET myisam_repair_threads=2;
@@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
 Warnings:
-Error	1034	myisam_sort_buffer_size is too small
+Error	1034	myisam_sort_buffer_size is too small. X
 Error	1034	Number of rows changed from 0 to 157
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
 INSERT INTO t1 VALUES('1');

mysql-test/r/mysqld--help.result

@@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232
 myisam-mmap-size 18446744073709551615
 myisam-recover-options DEFAULT
 myisam-repair-threads 1
-myisam-sort-buffer-size 8388608
+myisam-sort-buffer-size 134216704
 myisam-stats-method nulls_unequal
 myisam-use-mmap FALSE
 net-buffer-length 16384

mysql-test/r/repair.result

@@ -77,12 +77,12 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
 Warnings:
-Error	1034	myisam_sort_buffer_size is too small
+Error	1034	myisam_sort_buffer_size is too small. X
 Error	1034	Number of rows changed from 0 to 157
 SET myisam_repair_threads=2;
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	repair	error	myisam_sort_buffer_size is too small
+test.t1	repair	error	myisam_sort_buffer_size is too small. X
 test.t1	repair	warning	Number of rows changed from 0 to 157
 test.t1	repair	status	OK
 SET myisam_repair_threads=@@global.myisam_repair_threads;

mysql-test/suite/maria/maria.result

@@ -2679,3 +2679,65 @@ select count(*) from t1;
 count(*)
 13
 drop table t1;
+#
+# BUG#47444 - --myisam_repair_threads > 1 can result in all index
+#             cardinalities=1
+#
+SET aria_repair_threads=2;
+SET aria_sort_buffer_size=8192;
+CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
+INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	status	OK
+SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
+CARDINALITY
+14
+14
+14
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+SET aria_repair_threads=@@global.aria_repair_threads;
+#
+# BUG#47073 - valgrind errs, corruption,failed repair of partition,
+#             low myisam_sort_buffer_size
+#
+CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
+INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
+(6,'0'),(7,'0');
+INSERT INTO t1 SELECT a+10,b FROM t1;
+INSERT INTO t1 SELECT a+20,b FROM t1;
+INSERT INTO t1 SELECT a+40,b FROM t1;
+INSERT INTO t1 SELECT a+80,b FROM t1;
+INSERT INTO t1 SELECT a+160,b FROM t1;
+INSERT INTO t1 SELECT a+320,b FROM t1;
+INSERT INTO t1 SELECT a+640,b FROM t1;
+INSERT INTO t1 SELECT a+1280,b FROM t1;
+INSERT INTO t1 SELECT a+2560,b FROM t1;
+INSERT INTO t1 SELECT a+5120,b FROM t1;
+SET aria_sort_buffer_size=4096;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	error	aria_sort_buffer_size is too small. X
+test.t1	repair	error	Create index by sort failed
+test.t1	repair	info	Retrying repair with keycache
+test.t1	repair	status	OK
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SET aria_repair_threads=2;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	error	aria_sort_buffer_size is too small. X
+test.t1	repair	error	Create index by sort failed
+test.t1	repair	info	Retrying repair with keycache
+test.t1	repair	status	OK
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SET aria_repair_threads=@@global.aria_repair_threads;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+DROP TABLE t1;

mysql-test/suite/maria/maria.test

@@ -1963,6 +1963,52 @@ unlock tables;
 select count(*) from t1;
 drop table t1;
 
+--echo #
+--echo # BUG#47444 - --myisam_repair_threads > 1 can result in all index
+--echo #             cardinalities=1
+--echo #
+SET aria_repair_threads=2;
+SET aria_sort_buffer_size=8192;
+CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
+INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
+--replace_regex /Current aria_sort_buffer_size.*/X/
+REPAIR TABLE t1;
+SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
+CHECK TABLE t1;
+DROP TABLE t1;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+SET aria_repair_threads=@@global.aria_repair_threads;
+
+--echo #
+--echo # BUG#47073 - valgrind errs, corruption,failed repair of partition,
+--echo #             low myisam_sort_buffer_size
+--echo #
+CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
+INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
+                     (6,'0'),(7,'0');
+INSERT INTO t1 SELECT a+10,b FROM t1;
+INSERT INTO t1 SELECT a+20,b FROM t1;
+INSERT INTO t1 SELECT a+40,b FROM t1;
+INSERT INTO t1 SELECT a+80,b FROM t1;
+INSERT INTO t1 SELECT a+160,b FROM t1;
+INSERT INTO t1 SELECT a+320,b FROM t1;
+INSERT INTO t1 SELECT a+640,b FROM t1;
+INSERT INTO t1 SELECT a+1280,b FROM t1;
+INSERT INTO t1 SELECT a+2560,b FROM t1;
+INSERT INTO t1 SELECT a+5120,b FROM t1;
+SET aria_sort_buffer_size=4096;
+--replace_regex /Current aria_sort_buffer_size.*/X/
+REPAIR TABLE t1;
+CHECK TABLE t1;
+SET aria_repair_threads=2;
+# May report different values depending on threads activity.
+--replace_regex /Current aria_sort_buffer_size.*/X/
+REPAIR TABLE t1;
+CHECK TABLE t1;
+SET aria_repair_threads=@@global.aria_repair_threads;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+DROP TABLE t1;
+
 #
 # End of test
 #

mysql-test/suite/maria/maria3.result

@@ -317,7 +317,7 @@ aria_pagecache_division_limit	100
 aria_page_checksum	OFF
 aria_recover	NORMAL
 aria_repair_threads	1
-aria_sort_buffer_size	134217728
+aria_sort_buffer_size	268434432
 aria_stats_method	nulls_unequal
 aria_sync_log_dir	NEWFILE
 show status like 'aria%';

mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result

 SET @start_global_value = @@global.aria_sort_buffer_size;
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-134217728
+268434432
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-134217728
+268434432
 show global variables like 'aria_sort_buffer_size';
 Variable_name	Value
-aria_sort_buffer_size	134217728
+aria_sort_buffer_size	268434432
 show session variables like 'aria_sort_buffer_size';
 Variable_name	Value
-aria_sort_buffer_size	134217728
+aria_sort_buffer_size	268434432
 select * from information_schema.global_variables where variable_name='aria_sort_buffer_size';
 VARIABLE_NAME	VARIABLE_VALUE
-ARIA_SORT_BUFFER_SIZE	134217728
+ARIA_SORT_BUFFER_SIZE	268434432
 select * from information_schema.session_variables where variable_name='aria_sort_buffer_size';
 VARIABLE_NAME	VARIABLE_VALUE
-ARIA_SORT_BUFFER_SIZE	134217728
+ARIA_SORT_BUFFER_SIZE	268434432
 set global aria_sort_buffer_size=10;
+Warnings:
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '10'
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-10
+4096
 set session aria_sort_buffer_size=10;
+Warnings:
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '10'
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-10
+4096
 set global aria_sort_buffer_size=1.1;
 ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size'
 set session aria_sort_buffer_size=1e1;
@@ -36,7 +40,7 @@ Warnings:
 Warning	1292	Truncated incorrect aria_sort_buffer_size value: '0'
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-4
+4096
 set session aria_sort_buffer_size=cast(-1 as unsigned int);
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size

mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result

 SET @start_global_value = @@global.myisam_sort_buffer_size ;
 SELECT @start_global_value;
 @start_global_value
-8388608
+134216704
 SET @start_session_value = @@session.myisam_sort_buffer_size ;
 SELECT @start_session_value;
 @start_session_value
-8388608
+134216704
 '#--------------------FN_DYNVARS_005_01-------------------------#'
 SET @@global.myisam_sort_buffer_size   = 100;
 Warnings:
@@ -13,22 +13,22 @@ Warning	1292	Truncated incorrect myisam_sort_buffer_size value: '100'
 SET @@global.myisam_sort_buffer_size   = DEFAULT;
 SELECT @@global.myisam_sort_buffer_size  ;
 @@global.myisam_sort_buffer_size
-8388608
+134216704
 SET @@session.myisam_sort_buffer_size   = 200;
 Warnings:
 Warning	1292	Truncated incorrect myisam_sort_buffer_size value: '200'
 SET @@session.myisam_sort_buffer_size   = DEFAULT;
 SELECT @@session.myisam_sort_buffer_size  ;
 @@session.myisam_sort_buffer_size
-8388608
+134216704
 '#--------------------FN_DYNVARS_005_02-------------------------#'
 SET @@global.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@global.myisam_sort_buffer_size   = 8388608;
-@@global.myisam_sort_buffer_size   = 8388608
+SELECT @@global.myisam_sort_buffer_size   = 134216704;
+@@global.myisam_sort_buffer_size   = 134216704
 1
 SET @@session.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@session.myisam_sort_buffer_size   = 8388608;
-@@session.myisam_sort_buffer_size   = 8388608
+SELECT @@session.myisam_sort_buffer_size   = 134216704;
+@@session.myisam_sort_buffer_size   = 134216704
 1
 '#--------------------FN_DYNVARS_005_03-------------------------#'
 SET @@global.myisam_sort_buffer_size   = 4;
@@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list'
 SET @@global.myisam_sort_buffer_size   = @start_global_value;
 SELECT @@global.myisam_sort_buffer_size  ;
 @@global.myisam_sort_buffer_size
-8388608
+134216704
 SET @@session.myisam_sort_buffer_size   = @start_session_value;
 SELECT @@session.myisam_sort_buffer_size  ;
 @@session.myisam_sort_buffer_size
-8388608
+134216704

mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test

@@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size  ;
 ########################################################################
 
 SET @@global.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@global.myisam_sort_buffer_size   = 8388608;
+SELECT @@global.myisam_sort_buffer_size   = 134216704;
 
 SET @@session.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@session.myisam_sort_buffer_size   = 8388608;
+SELECT @@session.myisam_sort_buffer_size   = 134216704;
 
 
 --echo '#--------------------FN_DYNVARS_005_03-------------------------#'

mysql-test/t/myisam.test

@@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1;
 INSERT INTO t1 SELECT a+2560,b FROM t1;
 INSERT INTO t1 SELECT a+5120,b FROM t1;
 SET myisam_sort_buffer_size=4;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 REPAIR TABLE t1;
 
 SET myisam_repair_threads=2;
@@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3;
 CREATE TABLE t1(a CHAR(255), KEY(a));
 SELECT * FROM t1, t1 AS a1;
 SET myisam_sort_buffer_size=4;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),

mysql-test/t/repair.test

@@ -61,6 +61,7 @@ DROP TABLE t1;
 #
 CREATE TABLE t1(a CHAR(255), KEY(a));
 SET myisam_sort_buffer_size=4096;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
@@ -79,6 +80,7 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
 SET myisam_repair_threads=2;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 REPAIR TABLE t1;
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;

storage/maria/ha_maria.cc

@@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
        "disables parallel repair.",
        0, 0, 1, 1, 128, 1);
 
-static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
+static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
        "The buffer that is allocated when sorting the index when doing a "
-       "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.",
-       0, 0, 128L*1024L*1024L, 4, UINT_MAX32, 1);
+       "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
+       SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
 
 static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
        "Specifies how Aria index statistics collection code should treat "
@@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt)
     if ((param.testflag & T_REP_BY_SORT))
     {
       param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP;
+      if (thd->vio_ok())
+        _ma_check_print_info(&param, "Retrying repair with keycache");
       sql_print_information("Retrying repair of: '%s' with keycache",
                             table->s->path.str);
       continue;

storage/maria/ma_check.c

@@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param)
   param->keys_in_use= ~(ulonglong) 0;
   param->search_after_block=HA_OFFSET_ERROR;
   param->auto_increment_value= 0;
-  param->use_buffers=USE_BUFFER_INIT;
+  param->use_buffers= PAGE_BUFFER_INIT;
   param->read_buffer_length=READ_BUFFER_INIT;
   param->write_buffer_length=READ_BUFFER_INIT;
   param->sort_buffer_length=SORT_BUFFER_INIT;

storage/maria/maria_chk.c

@@ -191,7 +191,7 @@ end:
 
 enum options_mc {
   OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
-  OPT_CORRECT_CHECKSUM, OPT_PAGE_BUFFER_SIZE,
+  OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_PAGE_BUFFER_SIZE,
   OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE,
   OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
   OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@@ -229,6 +229,11 @@ static struct my_option my_long_options[] =
   {"correct-checksum", OPT_CORRECT_CHECKSUM,
    "Correct checksum information for table.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"create-missing-keys", OPT_CREATE_MISSING_KEYS,
+   "Create missing keys. This assumes that the data file is correct and that "
+   "the the number of rows stored in the index file is correct. Enables "
+   "--quick",
+   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DBUG_OFF
   {"debug", '#',
    "Output debug log. Often this is 'd:t:o,filename'.",
@@ -362,8 +367,8 @@ static struct my_option my_long_options[] =
   { "page_buffer_size", OPT_PAGE_BUFFER_SIZE,
     "Size of page buffer. Used by --safe-repair",
     &check_param.use_buffers, &check_param.use_buffers, 0,
-    GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L,
-    (long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
+    GET_ULONG, REQUIRED_ARG, PAGE_BUFFER_INIT, 1024L*1024L,
+    SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
   { "read_buffer_size", OPT_READ_BUFFER_SIZE,
     "Read buffer size for sequential reads during scanning",
     &check_param.read_buffer_length,
@@ -379,9 +384,8 @@ static struct my_option my_long_options[] =
   { "sort_buffer_size", OPT_SORT_BUFFER_SIZE,
     "Size of sort buffer. Used by --recover",
     &check_param.sort_buffer_length,
-    &check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
-    (long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD),
-    (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
+    &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG,
+    SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0},
   { "sort_key_blocks", OPT_SORT_KEY_BLOCKS,
     "Internal buffer for sorting keys; Don't touch :)",
     &check_param.sort_key_blocks,
@@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
   --correct-checksum  Correct checksum information for table.\n\
   -D, --data-file-length=#  Max length of data file (when recreating data\n\
                       file when it's full).\n\
+ --create-missing-keys\n\
+                      Create missing keys. This assumes that the data\n\
+                      file is correct and that the the number of rows stored\n\
+                      in the index file is correct. Enables --quick.\n\
   -e, --extend-check  Try to recover every possible row from the data file\n\
 		      Normally this will also find a lot of garbage rows;\n\
 		      Don't use this option if you are not totally desperate.\n\
-  -f, --force         Overwrite old temporary files.\n\
+  -f, --force         Overwrite old temporary files. Add another --force to\n\
+                      avoid 'sort_buffer_size is too small' errors.\n\
+                      In this case we will attempt to do the repair with the\n\
+                      given sort_buffer_size and dynamically allocate\n\
+                      as many management buffers as needed.\n\
   -k, --keys-used=#   Tell Aria to update only some specific keys. # is a\n\
 	              bit mask of which keys to use. This can be used to\n\
 		      get faster inserts.\n\
@@ -664,10 +676,13 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
     {
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
-      check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
+     check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
+                              T_FORCE_SORT_MEMORY);
     }
     else
     {
+     if (check_param.testflag & T_FORCE_CREATE)
+       check_param.testflag= T_FORCE_SORT_MEMORY;
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
       check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
     }
@@ -720,8 +735,26 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
       check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
     else
+    {
+      /*
+        If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
+        then add T_FORCE_UNIQUENESS.
+      */
       check_param.testflag|=
-        (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
+        ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
+         T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
+    }
+    break;
+  case OPT_CREATE_MISSING_KEYS:
+    if (argument == disabled_my_option)
+      check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
+    else
+    {
+      check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
+      /* Use repair by sort by default */
+      if (!(check_param.testflag & T_REP_ANY))
+        check_param.testflag|= T_REP_BY_SORT;
+    }
     break;
   case 'u':
     if (argument == disabled_my_option)

storage/maria/maria_def.h

@@ -94,9 +94,10 @@ typedef struct st_maria_sort_param
   */
   ulonglong unique[HA_MAX_KEY_SEG+1];
   ulonglong notnull[HA_MAX_KEY_SEG+1];
+  ulonglong sortbuff_size;
 
   MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos;
-  uint key, key_length,real_key_length,sortbuff_size;
+  uint key, key_length,real_key_length;
   uint maxbuffers, keys, find_length, sort_keys_length;
   my_bool fix_datafile, master;
   my_bool calc_checksum;                /* calculate table checksum */
@@ -107,10 +108,10 @@ typedef struct st_maria_sort_param
   int (*key_write)(struct st_maria_sort_param *, const uchar *);
   void (*lock_in_memory)(HA_CHECK *);
   int (*write_keys)(struct st_maria_sort_param *, register uchar **,
-                         uint , struct st_buffpek *, IO_CACHE *);
-  uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
+                         ulonglong , struct st_buffpek *, IO_CACHE *);
+  my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
   int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *,
-                   uint, uint);
+                   uint, ulonglong);
 } MARIA_SORT_PARAM;
 
 int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile);
@@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info
 #define UPDATE_AUTO_INC		8
 #define UPDATE_OPEN_COUNT	16
 
-#define USE_BUFFER_INIT		(((1024L*1024L*128-MALLOC_OVERHEAD)/8192)*8192)
-#define READ_BUFFER_INIT	(1024L*256L-MALLOC_OVERHEAD)
-#define SORT_BUFFER_INIT	(1024L*1024L*256-MALLOC_OVERHEAD)
-#define MIN_SORT_BUFFER		(4096-MALLOC_OVERHEAD)
+/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
+#define PAGE_BUFFER_INIT	MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 8192)
+#define READ_BUFFER_INIT	MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
+#define SORT_BUFFER_INIT	MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 1024)
+#define MIN_SORT_BUFFER		4096
 
 #define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0)
 #define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1)

storage/maria/maria_ftdump.c

@@ -84,7 +84,7 @@ int main(int argc,char *argv[])
       usage();
   }
 
-  init_pagecache(maria_pagecache, USE_BUFFER_INIT, 0, 0,
+  init_pagecache(maria_pagecache, PAGE_BUFFER_INIT, 0, 0,
                  MARIA_KEY_BLOCK_LENGTH, MY_WME);
 
   if (!(info=maria_open(argv[0], O_RDONLY,

storage/maria/maria_read_log.c

@@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace";
 static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent;
 static my_bool opt_check;
 static const char *opt_tmpdir;
-static ulong opt_page_buffer_size, opt_translog_buffer_size;
+static ulong opt_translog_buffer_size;
+static ulonglong opt_page_buffer_size;
 static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint;
 static MY_TMPDIR maria_chk_tmpdir;
 
@@ -204,9 +205,8 @@ static struct my_option my_long_options[] =
   { "page-buffer-size", 'P',
     "The size of the buffer used for index blocks for Aria tables",
     &opt_page_buffer_size, &opt_page_buffer_size, 0,
-    GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT,
-    1024L*1024L, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD,
-    (long) IO_SIZE, 0},
+    GET_ULL, REQUIRED_ARG, PAGE_BUFFER_INIT,
+    PAGE_BUFFER_INIT, SIZE_T_MAX, MALLOC_OVERHEAD, (long) IO_SIZE, 0},
   { "start-from-lsn", 'o', "Start reading log from this lsn",
     &opt_start_from_lsn, &opt_start_from_lsn,
     0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },

storage/myisam/ha_myisam.cc

@@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
 static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
   "The buffer that is allocated when sorting the index when doing "
   "a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL,
-  8192 * 1024, MIN_SORT_BUFFER + MALLOC_OVERHEAD, SIZE_T_MAX, 1);
+  SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
 
 static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG,
   "Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE);

storage/myisam/mi_check.c

@@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param)
   param->opt_follow_links=1;
   param->keys_in_use= ~(ulonglong) 0;
   param->search_after_block=HA_OFFSET_ERROR;
-  param->use_buffers=USE_BUFFER_INIT;
+  param->use_buffers= KEY_BUFFER_INIT;
   param->read_buffer_length=READ_BUFFER_INIT;
   param->write_buffer_length=READ_BUFFER_INIT;
   param->sort_buffer_length=SORT_BUFFER_INIT;

storage/myisam/myisam_ftdump.c

@@ -84,7 +84,7 @@ int main(int argc,char *argv[])
       usage();
   }
 
-  init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0);
+  init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0);
 
   if (!(info=mi_open(argv[0], O_RDONLY,
                      HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER)))

storage/myisam/myisamchk.c

@@ -134,7 +134,7 @@ int main(int argc, char **argv)
 
 enum options_mc {
   OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
-  OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE,
+  OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE,
   OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE,
   OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
   OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@@ -165,6 +165,11 @@ static struct my_option my_long_options[] =
   {"correct-checksum", OPT_CORRECT_CHECKSUM,
    "Correct checksum information for table.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"create-missing-keys", OPT_CREATE_MISSING_KEYS,
+   "Create missing keys. This assumes that the data file is correct and that "
+   "the the number of rows stored in the index file is correct. Enables "
+   "--quick",
+   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DBUG_OFF
   {"debug", '#',
    "Output debug log. Often this is 'd:t:o,filename'.",
@@ -270,7 +275,7 @@ static struct my_option my_long_options[] =
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   { "key_buffer_size", OPT_KEY_BUFFER_SIZE, "",
     &check_param.use_buffers, &check_param.use_buffers, 0,
-    GET_ULL, REQUIRED_ARG, USE_BUFFER_INIT, MALLOC_OVERHEAD,
+    GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD,
     SIZE_T_MAX, MALLOC_OVERHEAD,  IO_SIZE, 0},
   { "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE,  "",
     &opt_key_cache_block_size,
@@ -395,10 +400,18 @@ static void usage(void)
   -e, --extend-check  Try to recover every possible row from the data file\n\
 		      Normally this will also find a lot of garbage rows;\n\
 		      Don't use this option if you are not totally desperate.\n\
-  -f, --force         Overwrite old temporary files.\n\
+  -f, --force         Overwrite old temporary files. Add another --force to\n\
+                      avoid 'myisam_sort_buffer_size is too small' errors.\n\
+                      In this case we will attempt to do the repair with the\n\
+                      given myisam_sort_buffer_size and dynamically allocate\n\
+                      as many management buffers as needed.\n\
   -k, --keys-used=#   Tell MyISAM to update only some specific keys. # is a\n\
 	              bit mask of which keys to use. This can be used to\n\
 		      get faster inserts.\n\
+  --create-missing-keys\n\
+                      Create missing keys. This assumes that the data\n\
+                      file is correct and that the the number of rows stored\n\
+                      in the index file is correct. Enables --quick\n\
   --max-record-length=#\n\
                       Skip rows bigger than this if myisamchk can't allocate\n\
 		      memory to hold it.\n\
@@ -541,10 +554,13 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
     {
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
-      check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
+      check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
+                               T_FORCE_SORT_MEMORY);
     }
     else
     {
+      if (check_param.testflag & T_FORCE_CREATE)
+        check_param.testflag= T_FORCE_SORT_MEMORY;
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
       check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
     }
@@ -597,8 +613,26 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
       check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
     else
+    {
+      /*
+        If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
+        then add T_FORCE_UNIQUENESS.
+      */
       check_param.testflag|=
-        (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
+        ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
+         T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
+    }
+    break;
+  case OPT_CREATE_MISSING_KEYS:
+    if (argument == disabled_my_option)
+      check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
+    else
+    {
+      check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
+      /* Use repair by sort by default */
+      if (!(check_param.testflag & T_REP_ANY))
+        check_param.testflag|= T_REP_BY_SORT;
+    }
     break;
   case 'u':
     if (argument == disabled_my_option)

storage/myisam/myisamdef.h

@@ -629,10 +629,11 @@ typedef struct st_mi_block_info         /* Parameter to _mi_get_block_info */
 #define UPDATE_AUTO_INC         8
 #define UPDATE_OPEN_COUNT       16
 
-#define USE_BUFFER_INIT         (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE)
-#define READ_BUFFER_INIT        (1024L*256L-MALLOC_OVERHEAD)
-#define SORT_BUFFER_INIT        (2048L*1024L-MALLOC_OVERHEAD)
-#define MIN_SORT_BUFFER         (4096-MALLOC_OVERHEAD)
+/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
+#define KEY_BUFFER_INIT	        MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE)
+#define READ_BUFFER_INIT	MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
+#define SORT_BUFFER_INIT	MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024)
+#define MIN_SORT_BUFFER		4096
 
 enum myisam_log_commands
 {