Merge with mysql-5.1-bugteam latest changes

include/my_global.h

@@ -868,7 +868,7 @@ typedef SOCKET_SIZE_TYPE size_socket;
 #define FLT_MAX		((float)3.40282346638528860e+38)
 #endif
 #ifndef SIZE_T_MAX
-#define SIZE_T_MAX ~((size_t) 0)
+#define SIZE_T_MAX      (~((size_t) 0))
 #endif
 
 #ifndef isfinite

include/myisam.h

@@ -259,6 +259,8 @@ extern ulong myisam_bulk_insert_tree_size, myisam_data_pointer_size;
 /* usually used to check if a symlink points into the mysql data home */
 /* which is normally forbidden                                        */
 extern int (*myisam_test_invalid_symlink)(const char *filename);
+extern ulonglong myisam_mmap_size, myisam_mmap_used;
+extern pthread_mutex_t THR_LOCK_myisam_mmap;
 
 	/* Prototypes for myisam-functions */
 
@@ -304,6 +306,7 @@ extern int mi_delete_all_rows(struct st_myisam_info *info);
 extern ulong _mi_calc_blob_length(uint length , const uchar *pos);
 extern uint mi_get_pointer_length(ulonglong file_length, uint def);
 
+#define MEMMAP_EXTRA_MARGIN     7       /* Write this as a suffix for mmap file */
 /* this is used to pass to mysql_myisamchk_table */
 
 #define   MYISAMCHK_REPAIR 1  /* equivalent to myisamchk -r */

mysql-test/r/join_outer.result

@@ -1254,3 +1254,38 @@ SELECT * FROM t1 LEFT JOIN t2 ON e<>0 WHERE c=1 AND d<=>NULL;
 c	e	d
 1	0	NULL
 DROP TABLE t1,t2;
+#
+# Bug#47650: using group by with rollup without indexes returns incorrect 
+# results with where
+#
+CREATE TABLE t1 ( a INT );
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 ( a INT, b INT );
+INSERT INTO t2 VALUES (1, 1),(1, 2),(1, 3),(2, 4),(2, 5);
+EXPLAIN
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 LEFT JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	system	NULL	NULL	NULL	NULL	1	Using temporary; Using filesort
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	5	
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 LEFT JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+a	COUNT( t2.b )	SUM( t2.b )	MAX( t2.b )
+1	3	6	3
+NULL	3	6	3
+EXPLAIN
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	system	NULL	NULL	NULL	NULL	1	Using filesort
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	5	Using where
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+a	COUNT( t2.b )	SUM( t2.b )	MAX( t2.b )
+1	3	6	3
+NULL	3	6	3
+DROP TABLE t1, t2;

mysql-test/r/myisam.result

@@ -1853,6 +1853,21 @@ CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 DROP TABLE t1;
+#
+# Bug #49465: valgrind warnings and incorrect live checksum...
+#
+CREATE TABLE t1(
+a VARCHAR(1), b VARCHAR(1), c VARCHAR(1),
+f VARCHAR(1), g VARCHAR(1), h VARCHAR(1),
+i VARCHAR(1), j VARCHAR(1), k VARCHAR(1)) CHECKSUM=1;
+INSERT INTO t1 VALUES('', '', '', '', '', '', '', '', '');
+CHECKSUM TABLE t1 QUICK;
+Table	Checksum
+test.t1	467455460
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	467455460
+DROP TABLE t1;
 End of 5.0 tests
 create table t1 (a int not null, key `a` (a) key_block_size=1024);
 show create table t1;

mysql-test/r/variables.result

@@ -1018,6 +1018,12 @@ ERROR HY000: Variable 'hostname' is a read only variable
 show variables like 'hostname';
 Variable_name	Value
 hostname	#
+#
+# BUG#37408 - Compressed MyISAM files should not require/use mmap()
+#
+# Test 'myisam_mmap_size' option is not dynamic
+SET @@myisam_mmap_size= 500M;
+ERROR HY000: Variable 'myisam_mmap_size' is a read only variable
 End of 5.0 tests
 set join_buffer_size=1;
 Warnings:

mysql-test/suite/binlog/r/binlog_index.result

@@ -38,7 +38,7 @@ purge binary logs TO 'master-bin.000002';
 ERROR HY000: Fatal error during log purge
 show warnings;
 Level	Code	Message
-Error	1377	a problem with deleting master-bin.000001; consider examining correspondence of your binlog index file to the actual binlog files
+Warning	1377	a problem with deleting master-bin.000001; consider examining correspondence of your binlog index file to the actual binlog files
 Error	1377	Fatal error during log purge
 reset master;
 # crash_purge_before_update_index

mysql-test/suite/rpl/r/rpl_temporary.result

@@ -36,8 +36,10 @@ ERROR 42000: Access denied; you need the SUPER privilege for this operation
 SELECT @@session.sql_select_limit = @save_select_limit;
 @@session.sql_select_limit = @save_select_limit
 1
+SET @save_conn_id= connection_id();
 SET @@session.pseudo_thread_id=100;
 SET @@session.pseudo_thread_id=connection_id();
+SET @@session.pseudo_thread_id=@save_conn_id;
 SET @@session.sql_log_bin=0;
 SET @@session.sql_log_bin=1;
 drop table if exists t1,t2;

mysql-test/suite/rpl/t/rpl_temporary.test

@@ -116,8 +116,10 @@ SET @@session.sql_select_limit=10, @@session.sql_log_bin=0;
 SELECT @@session.sql_select_limit = @save_select_limit; #shouldn't have changed
 # Now as root, to be sure it works
 connection con2;
+SET @save_conn_id= connection_id();
 SET @@session.pseudo_thread_id=100;
 SET @@session.pseudo_thread_id=connection_id();
+SET @@session.pseudo_thread_id=@save_conn_id;
 SET @@session.sql_log_bin=0;
 SET @@session.sql_log_bin=1;
 

mysql-test/t/join_outer.test

@@ -867,3 +867,32 @@ SELECT * FROM t1 LEFT JOIN t2 ON e<>0 WHERE c=1 AND d<=>NULL;
 
 DROP TABLE t1,t2;
 
+--echo #
+--echo # Bug#47650: using group by with rollup without indexes returns incorrect 
+--echo # results with where
+--echo #
+CREATE TABLE t1 ( a INT );
+INSERT INTO t1 VALUES (1);
+
+CREATE TABLE t2 ( a INT, b INT );
+INSERT INTO t2 VALUES (1, 1),(1, 2),(1, 3),(2, 4),(2, 5);
+
+EXPLAIN
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 LEFT JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 LEFT JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+
+EXPLAIN
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+
+SELECT t1.a, COUNT( t2.b ), SUM( t2.b ), MAX( t2.b )
+FROM t1 JOIN t2 USING( a )
+GROUP BY t1.a WITH ROLLUP;
+
+DROP TABLE t1, t2;

mysql-test/t/myisam.test

@@ -1184,6 +1184,20 @@ SELECT a FROM t1;
 CHECK TABLE t1;
 DROP TABLE t1; 
 
+
+--echo #
+--echo # Bug #49465: valgrind warnings and incorrect live checksum...
+--echo #
+CREATE TABLE t1(
+a VARCHAR(1), b VARCHAR(1), c VARCHAR(1),
+f VARCHAR(1), g VARCHAR(1), h VARCHAR(1),
+i VARCHAR(1), j VARCHAR(1), k VARCHAR(1)) CHECKSUM=1;
+INSERT INTO t1 VALUES('', '', '', '', '', '', '', '', '');
+CHECKSUM TABLE t1 QUICK;
+CHECKSUM TABLE t1 EXTENDED;
+DROP TABLE t1;
+
+
 --echo End of 5.0 tests
 
 

mysql-test/t/variables.test

@@ -779,6 +779,12 @@ set @@hostname= "anothername";
 --replace_column 2 #
 show variables like 'hostname';
 
+--echo #
+--echo # BUG#37408 - Compressed MyISAM files should not require/use mmap()
+--echo #
+--echo # Test 'myisam_mmap_size' option is not dynamic
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET @@myisam_mmap_size= 500M;
 --echo End of 5.0 tests
 
 #

mysys/my_thr_init.c

@@ -30,7 +30,9 @@ pthread_key(struct st_my_thread_var, THR_KEY_mysys);
 #endif /* USE_TLS */
 pthread_mutex_t THR_LOCK_malloc,THR_LOCK_open,
 	        THR_LOCK_lock,THR_LOCK_isam,THR_LOCK_myisam,THR_LOCK_heap,
-                THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads, THR_LOCK_time;
+                THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads, THR_LOCK_time,
+                THR_LOCK_myisam_mmap;
+
 pthread_cond_t  THR_COND_threads;
 uint            THR_thread_count= 0;
 uint 		my_thread_end_wait_time= 5;
@@ -144,6 +146,7 @@ my_bool my_thread_global_init(void)
   pthread_mutex_init(&THR_LOCK_lock,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_isam,MY_MUTEX_INIT_SLOW);
   pthread_mutex_init(&THR_LOCK_myisam,MY_MUTEX_INIT_SLOW);
+  pthread_mutex_init(&THR_LOCK_myisam_mmap,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_heap,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_net,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_charset,MY_MUTEX_INIT_FAST);
@@ -210,6 +213,7 @@ void my_thread_global_end(void)
   pthread_mutex_destroy(&THR_LOCK_lock);
   pthread_mutex_destroy(&THR_LOCK_isam);
   pthread_mutex_destroy(&THR_LOCK_myisam);
+  pthread_mutex_destroy(&THR_LOCK_myisam_mmap);
   pthread_mutex_destroy(&THR_LOCK_heap);
   pthread_mutex_destroy(&THR_LOCK_net);
   pthread_mutex_destroy(&THR_LOCK_time);

sql/log.cc

@@ -3014,7 +3014,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd)
       }
       else
       {
-        push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+        push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                             ER_BINLOG_PURGE_FATAL_ERR,
                             "a problem with deleting %s; "
                             "consider examining correspondence "
@@ -3045,7 +3045,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD* thd)
     }
     else
     {
-      push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+      push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                           ER_BINLOG_PURGE_FATAL_ERR,
                           "a problem with deleting %s; "
                           "consider examining correspondence "
@@ -3481,7 +3481,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *decrease_log_space,
         */
         if (thd)
         {
-          push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+          push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                               ER_BINLOG_PURGE_FATAL_ERR,
                               "a problem with getting info on being purged %s; "
                               "consider examining correspondence "
@@ -3509,7 +3509,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *decrease_log_space,
         {
           if (thd)
           {
-            push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+            push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                                 ER_BINLOG_PURGE_FATAL_ERR,
                                 "a problem with deleting %s and "
                                 "reading the binlog index file",
@@ -3557,7 +3557,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *decrease_log_space,
           {
             if (thd)
             {
-              push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+              push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                                   ER_BINLOG_PURGE_FATAL_ERR,
                                   "a problem with deleting %s; "
                                   "consider examining correspondence "
@@ -3647,7 +3647,7 @@ int MYSQL_BIN_LOG::purge_logs_before_date(time_t purge_time)
         */
         if (thd)
         {
-          push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+          push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                               ER_BINLOG_PURGE_FATAL_ERR,
                               "a problem with getting info on being purged %s; "
                               "consider examining correspondence "

sql/mysqld.cc

@@ -5651,6 +5651,7 @@ enum options_mysqld
   OPT_MYISAM_BLOCK_SIZE, OPT_MYISAM_MAX_EXTRA_SORT_FILE_SIZE,
   OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_SORT_BUFFER_SIZE,
   OPT_MYISAM_USE_MMAP, OPT_MYISAM_REPAIR_THREADS,
+  OPT_MYISAM_MMAP_SIZE,
   OPT_MYISAM_STATS_METHOD,
   OPT_NET_BUFFER_LENGTH, OPT_NET_RETRY_COUNT,
   OPT_NET_READ_TIMEOUT, OPT_NET_WRITE_TIMEOUT,
@@ -6802,6 +6803,10 @@ The minimum value for this variable is 4096.",
    (uchar**) &max_system_variables.myisam_max_sort_file_size, 0,
    GET_ULL, REQUIRED_ARG, (longlong) LONG_MAX, 0, (ulonglong) MAX_FILE_SIZE,
    0, 1024*1024, 0},
+  {"myisam_mmap_size", OPT_MYISAM_MMAP_SIZE,
+   "Can be used to restrict the total memory used for memory mmaping of myisam files",
+   (uchar**) &myisam_mmap_size, (uchar**) &myisam_mmap_size, 0,
+   GET_ULL, REQUIRED_ARG, SIZE_T_MAX, MEMMAP_EXTRA_MARGIN, SIZE_T_MAX, 0, 1, 0},
   {"myisam_repair_threads", OPT_MYISAM_REPAIR_THREADS,
    "Number of threads to use when repairing MyISAM tables. The value of 1 disables parallel repair.",
    (uchar**) &global_system_variables.myisam_repair_threads,

sql/set_var.cc

@@ -147,6 +147,7 @@ static bool sys_update_general_log_path(THD *thd, set_var * var);
 static void sys_default_general_log_path(THD *thd, enum_var_type type);
 static bool sys_update_slow_log_path(THD *thd, set_var * var);
 static void sys_default_slow_log_path(THD *thd, enum_var_type type);
+static uchar *get_myisam_mmap_size(THD *thd);
 
 /*
   Variable definition list
@@ -898,6 +899,10 @@ sys_var_str sys_var_slow_log_path(&vars, "slow_query_log_file", sys_check_log_pa
 				  opt_slow_logname);
 static sys_var_log_output sys_var_log_output_state(&vars, "log_output", &log_output_options,
 					    &log_output_typelib, 0);
+static sys_var_readonly         sys_myisam_mmap_size(&vars, "myisam_mmap_size",
+                                                     OPT_GLOBAL,
+                                                     SHOW_LONGLONG,
+                                                     get_myisam_mmap_size);
 
 
 bool sys_var::check(THD *thd, set_var *var)
@@ -3175,6 +3180,12 @@ static uchar *get_tmpdir(THD *thd)
   return (uchar*)mysql_tmpdir;
 }
 
+static uchar *get_myisam_mmap_size(THD *thd)
+{
+  return (uchar *)&myisam_mmap_size;
+}
+
+
 /****************************************************************************
   Main handling of variables:
   - Initialisation

sql/sql_insert.cc

@@ -782,12 +782,21 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,
 	restore_record(table,s->default_values);	// Get empty record
       else
       {
+        TABLE_SHARE *share= table->s;
+
         /*
           Fix delete marker. No need to restore rest of record since it will
           be overwritten by fill_record() anyway (and fill_record() does not
           use default values in this case).
         */
-	table->record[0][0]= table->s->default_values[0];
+        table->record[0][0]= share->default_values[0];
+
+        /* Fix undefined null_bits. */
+        if (share->null_bytes > 1 && share->last_null_bit_pos)
+        {
+          table->record[0][share->null_bytes - 1]= 
+            share->default_values[share->null_bytes - 1];
+        }
       }
       if (fill_record_n_invoke_before_triggers(thd, table->field, *values, 0,
                                                table->triggers,

sql/sql_select.cc

@@ -7124,7 +7124,19 @@ remove_const(JOIN *join,ORDER *first_order, COND *cond,
   for (order=first_order; order ; order=order->next)
   {
     table_map order_tables=order->item[0]->used_tables();
-    if (order->item[0]->with_sum_func)
+    if (order->item[0]->with_sum_func ||
+        /*
+          If the outer table of an outer join is const (either by itself or
+          after applying WHERE condition), grouping on a field from such a
+          table will be optimized away and filesort without temporary table
+          will be used unless we prevent that now. Filesort is not fit to
+          handle joins and the join condition is not applied. We can't detect
+          the case without an expensive test, however, so we force temporary
+          table for all queries containing more than one table, ROLLUP, and an
+          outer join.
+         */
+        (join->tables > 1 && join->rollup.state == ROLLUP::STATE_INITED &&
+        join->outer_join))
       *simple_order=0;				// Must do a temp table to sort
     else if (!(order_tables & not_const_tables))
     {

storage/myisam/mi_packrec.c

@@ -1492,21 +1492,55 @@ static int _mi_read_rnd_mempack_record(MI_INFO*, uchar *,my_off_t, my_bool);
 my_bool _mi_memmap_file(MI_INFO *info)
 {
   MYISAM_SHARE *share=info->s;
+  my_bool eom;
+
   DBUG_ENTER("mi_memmap_file");
 
   if (!info->s->file_map)
   {
+    my_off_t data_file_length= share->state.state.data_file_length;
+
+    if (myisam_mmap_size != SIZE_T_MAX)
+    {
+      pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+      eom= data_file_length > myisam_mmap_size - myisam_mmap_used - MEMMAP_EXTRA_MARGIN;
+      if (!eom)
+        myisam_mmap_used+= data_file_length + MEMMAP_EXTRA_MARGIN;
+      pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+    }
+    else
+      eom= data_file_length > myisam_mmap_size - MEMMAP_EXTRA_MARGIN;
+
+    if (eom)
+    {
+      DBUG_PRINT("warning", ("File is too large for mmap"));
+      DBUG_RETURN(0);
+    }
     if (my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)) <
         share->state.state.data_file_length+MEMMAP_EXTRA_MARGIN)
     {
       DBUG_PRINT("warning",("File isn't extended for memmap"));
+      if (myisam_mmap_size != SIZE_T_MAX)
+      {
+        pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+        myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+        pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+      }
       DBUG_RETURN(0);
     }
     if (mi_dynmap_file(info,
                        share->state.state.data_file_length + 
                          MEMMAP_EXTRA_MARGIN))
+    {
+      if (myisam_mmap_size != SIZE_T_MAX)
+      {
+        pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+        myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+        pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+      }
       DBUG_RETURN(0);
     }
+  }
   info->opt_flag|= MEMMAP_USED;
   info->read_record= share->read_record= _mi_read_mempack_record;
   share->read_rnd= _mi_read_rnd_mempack_record;
@@ -1518,6 +1552,13 @@ void _mi_unmap_file(MI_INFO *info)
 {
   VOID(my_munmap((char*) info->s->file_map, 
                  (size_t) info->s->mmaped_length + MEMMAP_EXTRA_MARGIN));
+
+  if (myisam_mmap_size != SIZE_T_MAX)
+  {
+    pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+    myisam_mmap_used-= info->s->mmaped_length + MEMMAP_EXTRA_MARGIN;
+    pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+  }
 }
 
 

storage/myisam/mi_static.c

@@ -40,7 +40,8 @@ ulong myisam_concurrent_insert= 0;
 my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
 ulong    myisam_bulk_insert_tree_size=8192*1024;
 ulong    myisam_data_pointer_size=4;
-
+ulonglong    myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0;
+pthread_mutex_t  THR_LOCK_myisam_mmap;
 
 static int always_valid(const char *filename __attribute__((unused)))
 {

storage/myisam/myisamdef.h

@@ -437,7 +437,6 @@ typedef struct st_mi_sort_param
 #define MI_MAX_BLOCK_LENGTH	((((ulong) 1 << 24)-1) & (~ (ulong) (MI_DYN_ALIGN_SIZE-1)))
 #define MI_REC_BUFF_OFFSET      ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER+sizeof(uint32))
 
-#define MEMMAP_EXTRA_MARGIN	7	/* Write this as a suffix for file */
 
 #define PACK_TYPE_SELECTED	1	/* Bits in field->pack_type */
 #define PACK_TYPE_SPACE_FIELDS	2