Many files:

   Added the MAX_LENGTH_FOR_SORT_DATA option
filesort.cc:
  Added the MAX_LENGTH_FOR_SORT_DATA option


sql/filesort.cc:
  Added the MAX_LENGTH_FOR_SORT_DATA option
sql/mysqld.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/opt_range.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/records.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/set_var.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_base.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_class.h:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_delete.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_select.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_sort.h:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_table.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/sql_update.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/structs.h:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/table.h:
   Added the MAX_LENGTH_FOR_SORT_DATA option
sql/uniques.cc:
   Added the MAX_LENGTH_FOR_SORT_DATA option

sql/mysqld.cc

@@ -3457,7 +3457,7 @@ enum options
   OPT_MAX_BINLOG_CACHE_SIZE, OPT_MAX_BINLOG_SIZE,
   OPT_MAX_CONNECTIONS, OPT_MAX_CONNECT_ERRORS,
   OPT_MAX_DELAYED_THREADS, OPT_MAX_HEP_TABLE_SIZE,
-  OPT_MAX_JOIN_SIZE, OPT_MAX_SORT_LENGTH,
+  OPT_MAX_JOIN_SIZE, OPT_MAX_LENGTH_FOR_SORT_DATA, OPT_MAX_SORT_LENGTH,
   OPT_MAX_TMP_TABLES, OPT_MAX_USER_CONNECTIONS,
   OPT_MAX_WRITE_LOCK_COUNT, OPT_BULK_INSERT_BUFFER_SIZE,
   OPT_MAX_ERROR_COUNT, OPT_MAX_PREP_STMT,
@@ -4159,6 +4159,11 @@ struct my_option my_long_options[] =
    (gptr*) &global_system_variables.max_join_size,
    (gptr*) &max_system_variables.max_join_size, 0, GET_HA_ROWS, REQUIRED_ARG,
    ~0L, 1, ~0L, 0, 1, 0},
+   {"max_length_for_sort_data", OPT_MAX_LENGTH_FOR_SORT_DATA,
+    "Max number of bytes in sorted records",
+    (gptr*) &global_system_variables.max_length_for_sort_data,
+    (gptr*) &max_system_variables.max_length_for_sort_data, 0, GET_ULONG,
+    REQUIRED_ARG, 1024, 4, 8192*1024L, 0, 1, 0},
   {"max_prepared_statements", OPT_MAX_PREP_STMT,
    "Max number of prepared_statements for a thread",
    (gptr*) &global_system_variables.max_prep_stmt_count,

sql/opt_range.cc

@@ -349,13 +349,13 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables,
   select->head=head;
   select->cond=conds;
 
-  if (head->io_cache)
+  if (head->sort.io_cache)
   {
-    select->file= *head->io_cache;
+    select->file= *head->sort.io_cache;
     select->records=(ha_rows) (select->file.end_of_file/
 			       head->file->ref_length);
-    my_free((gptr) (head->io_cache),MYF(0));
-    head->io_cache=0;
+    my_free((gptr) (head->sort.io_cache),MYF(0));
+    head->sort.io_cache=0;
   }
   DBUG_RETURN(select);
 }

sql/records.cc

+
 /* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
 
    This program is free software; you can redistribute it and/or modify
@@ -22,6 +23,8 @@
 static int rr_quick(READ_RECORD *info);
 static int rr_sequential(READ_RECORD *info);
 static int rr_from_tempfile(READ_RECORD *info);
+static int rr_unpack_from_tempfile(READ_RECORD *info);
+static int rr_unpack_from_buffer(READ_RECORD *info);
 static int rr_from_pointers(READ_RECORD *info);
 static int rr_from_cache(READ_RECORD *info);
 static int init_rr_cache(READ_RECORD *info);
@@ -41,8 +44,16 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
   info->table=table;
   info->file= table->file;
   info->forms= &info->table;		/* Only one table */
-  info->record=table->record[0];
-  info->ref_length=table->file->ref_length;
+  if (table->sort.addon_field)
+  {
+    info->rec_buf= table->sort.addon_buf;
+    info->ref_length= table->sort.addon_length;
+  }
+  else
+  {
+    info->record= table->record[0];
+    info->ref_length= table->file->ref_length;
+  }
   info->select=select;
   info->print_error=print_error;
   info->ignore_not_found_rows= 0;
@@ -51,11 +62,12 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
   if (select && my_b_inited(&select->file))
     tempfile= &select->file;
   else
-    tempfile= table->io_cache;
+    tempfile= table->sort.io_cache;
   if (tempfile && my_b_inited(tempfile)) // Test if ref-records was used
   {
     DBUG_PRINT("info",("using rr_from_tempfile"));
-    info->read_record=rr_from_tempfile;
+    info->read_record= (table->sort.addon_field ?
+                        rr_unpack_from_tempfile : rr_from_tempfile);
     info->io_cache=tempfile;
     reinit_io_cache(info->io_cache,READ_CACHE,0L,0,0);
     info->ref_pos=table->file->ref;
@@ -85,13 +97,15 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
     DBUG_PRINT("info",("using rr_quick"));
     info->read_record=rr_quick;
   }
-  else if (table->record_pointers)
+  else if (table->sort.record_pointers)
   {
     DBUG_PRINT("info",("using record_pointers"));
     table->file->rnd_init(0);
-    info->cache_pos=table->record_pointers;
-    info->cache_end=info->cache_pos+ table->found_records*info->ref_length;
-    info->read_record= rr_from_pointers;
+    info->cache_pos=table->sort.record_pointers;
+    info->cache_end=info->cache_pos+ 
+                    table->sort.found_records*info->ref_length;
+    info->read_record= (table->sort.addon_field ?
+                        rr_unpack_from_buffer : rr_from_pointers);
   }
   else
   {
@@ -120,6 +134,19 @@ void end_read_record(READ_RECORD *info)
   }
   if (info->table)
   {
+    TABLE *table= info->table;
+    if (table->sort.record_pointers)
+    {
+      my_free((gptr) table->sort.record_pointers,MYF(0));
+      table->sort.record_pointers=0;
+    }
+    if (table->sort.addon_buf)
+    {
+      my_free((char *) table->sort.addon_buf, MYF(0));
+      my_free((char *) table->sort.addon_field, MYF(MY_ALLOW_ZERO_PTR));
+      table->sort.addon_buf=0;
+      table->sort.addon_field=0;
+    }
     (void) info->file->extra(HA_EXTRA_NO_CACHE);
     (void) info->file->rnd_end();
     info->table=0;
@@ -200,6 +227,34 @@ tryNext:
 } /* rr_from_tempfile */
 
 
+/*
+  Read a result set record from a temporary file after sorting
+
+  SYNOPSIS
+    rr_unpack_from_tempfile()
+    info                Reference to the context including record descriptors
+
+  DESCRIPTION
+    The function first reads the next sorted record from the temporary file.
+    into a buffer. If a success it calls a callback function that unpacks 
+    the fields values use in the result set from this buffer into their
+    positions in the regular record buffer.
+
+  RETURN
+     0 - Record successfully read.
+    -1 - There is no record to be read anymore. 
+*/
+
+static int rr_unpack_from_tempfile(READ_RECORD *info)
+{
+  if (my_b_read(info->io_cache, info->rec_buf, info->ref_length))
+    return -1;
+  TABLE *table= info->table;
+  (*table->sort.unpack)(table->sort.addon_field, info->rec_buf);
+
+  return 0;
+}
+
 static int rr_from_pointers(READ_RECORD *info)
 {
   int tmp;
@@ -228,6 +283,34 @@ tryNext:
   return tmp;
 }
 
+/*
+  Read a result set record from a buffer after sorting
+
+  SYNOPSIS
+    rr_unpack_from_buffer()
+    info                   Reference to the context including record descriptors
+
+  DESCRIPTION
+    The function first reads the next sorted record from the sort buffer.
+    If a success it calls a callback function that unpacks 
+    the fields values use in the result set from this buffer into their
+    positions in the regular record buffer.
+
+  RETURN
+     0 - Record successfully read.
+    -1 - There is no record to be read anymore. 
+*/
+
+static int rr_unpack_from_buffer(READ_RECORD *info)
+{
+  if (info->cache_pos == info->cache_end)
+    return -1;                      /* End of buffer */
+  TABLE *table= info->table;
+  (*table->sort.unpack)(table->sort.addon_field, info->cache_pos);
+  info->cache_pos+= info->ref_length;
+
+  return 0;
+}
 	/* cacheing of records from a database */
 
 static int init_rr_cache(READ_RECORD *info)

sql/set_var.cc

@@ -166,6 +166,8 @@ sys_var_thd_ulong       sys_pseudo_thread_id("pseudo_thread_id",
 sys_var_thd_ha_rows	sys_max_join_size("max_join_size",
 					  &SV::max_join_size,
 					  fix_max_join_size);
+sys_var_thd_ulong   sys_max_length_for_sort_data("max_length_for_sort_data",
+                        &SV::max_length_for_sort_data);
 #ifndef TO_BE_DELETED	/* Alias for max_join_size */
 sys_var_thd_ha_rows	sys_sql_max_join_size("sql_max_join_size",
 					      &SV::max_join_size,
@@ -380,6 +382,7 @@ sys_var *sys_variables[]=
   &sys_max_error_count,
   &sys_max_heap_table_size,
   &sys_max_join_size,
+  &sys_max_length_for_sort_data,
   &sys_max_prep_stmt_count,
   &sys_max_sort_length,
   &sys_max_tmp_tables,
@@ -533,6 +536,9 @@ struct show_var_st init_vars[]= {
   {sys_max_delayed_threads.name,(char*) &sys_max_delayed_threads,   SHOW_SYS},
   {sys_max_heap_table_size.name,(char*) &sys_max_heap_table_size,   SHOW_SYS},
   {sys_max_join_size.name,	(char*) &sys_max_join_size,	    SHOW_SYS},
+  {sys_max_length_for_sort_data.name,
+   (char*) &sys_max_length_for_sort_data,
+   SHOW_SYS},
   {sys_max_prep_stmt_count.name,(char*) &sys_max_prep_stmt_count,   SHOW_SYS},
   {sys_max_sort_length.name,	(char*) &sys_max_sort_length,	    SHOW_SYS},
   {sys_max_user_connections.name,(char*) &sys_max_user_connections, SHOW_SYS},

sql/sql_base.cc

@@ -245,16 +245,11 @@ static void free_cache_entry(TABLE *table)
 void free_io_cache(TABLE *table)
 {
   DBUG_ENTER("free_io_cache");
-  if (table->io_cache)
+  if (table->sort.io_cache)
   {
-    close_cached_file(table->io_cache);
-    my_free((gptr) table->io_cache,MYF(0));
-    table->io_cache=0;
-  }
-  if (table->record_pointers)
-  {
-    my_free((gptr) table->record_pointers,MYF(0));
-    table->record_pointers=0;
+    close_cached_file(table->sort.io_cache);
+    my_free((gptr) table->sort.io_cache,MYF(0));
+    table->sort.io_cache=0;
   }
   DBUG_VOID_RETURN;
 }

sql/sql_class.h

@@ -352,6 +352,7 @@ struct system_variables
   ulong max_allowed_packet;
   ulong max_error_count;
   ulong max_heap_table_size;
+  ulong max_length_for_sort_data;
   ulong max_prep_stmt_count;
   ulong max_sort_length;
   ulong max_tmp_tables;

sql/sql_delete.cc

@@ -124,11 +124,11 @@ int mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, ORDER *order,
     bzero((char*) &tables,sizeof(tables));
     tables.table = table;
 
-    table->io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
+    table->sort.io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
                                                   MYF(MY_FAE | MY_ZEROFILL));
     if (setup_order(thd, 0, &tables, fields, all_fields, order) ||
         !(sortorder=make_unireg_sortorder(order, &length)) ||
-        (table->found_records = filesort(thd, table, sortorder, length,
+        (table->sort.found_records = filesort(thd, table, sortorder, length,
                                               (SQL_SELECT *) 0, HA_POS_ERROR,
                                               &examined_rows))
         == HA_POS_ERROR)

sql/sql_select.cc

@@ -4009,7 +4009,7 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type,
     case Item_sum::AVG_FUNC:			/* Place for sum & count */
       if (group)
 	return new Field_string(sizeof(double)+sizeof(longlong),
-				maybe_null, item->name,table,&my_charset_bin);
+				0, item->name,table,&my_charset_bin);
       else
 	return new Field_double(item_sum->max_length,maybe_null,
 				item->name, table, item_sum->decimals);
@@ -4017,7 +4017,7 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type,
     case Item_sum::STD_FUNC:	
       if (group)
 	return	new Field_string(sizeof(double)*2+sizeof(longlong),
-				 maybe_null, item->name,table,&my_charset_bin);
+				 0, item->name,table,&my_charset_bin);
       else
 	return new Field_double(item_sum->max_length, maybe_null,
 				item->name,table,item_sum->decimals);				
@@ -5621,11 +5621,11 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
 	  TABLE *table=jt->table;
 
 	  join->select_options ^= OPTION_FOUND_ROWS;
-	  if (table->record_pointers ||
-	      (table->io_cache && my_b_inited(table->io_cache)))
+	  if (table->sort.record_pointers ||
+	      (table->sort.io_cache && my_b_inited(table->sort.io_cache)))
 	  {
 	    /* Using filesort */
-	    join->send_records= table->found_records;
+	    join->send_records= table->sort.found_records;
 	  }
 	  else
 	  {
@@ -6460,7 +6460,7 @@ create_sort_index(THD *thd, JOIN_TAB *tab, ORDER *order,
   if (!(sortorder=make_unireg_sortorder(order,&length)))
     goto err;				/* purecov: inspected */
   /* It's not fatal if the following alloc fails */
-  table->io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
+  table->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
                                              MYF(MY_WME | MY_ZEROFILL));
   table->status=0;				// May be wrong if quick_select
 
@@ -6490,9 +6490,9 @@ create_sort_index(THD *thd, JOIN_TAB *tab, ORDER *order,
   }
   if (table->tmp_table)
     table->file->info(HA_STATUS_VARIABLE);	// Get record count
-  table->found_records=filesort(thd, table,sortorder, length,
+  table->sort.found_records=filesort(thd, table,sortorder, length,
                                      select, filesort_limit, &examined_rows);
-  tab->records=table->found_records;		// For SQL_CALC_ROWS
+  tab->records=table->sort.found_records;		// For SQL_CALC_ROWS
   delete select;				// filesort did select
   tab->select=0;
   tab->select_cond=0;
@@ -6504,7 +6504,7 @@ create_sort_index(THD *thd, JOIN_TAB *tab, ORDER *order,
     table->key_read=0;
     table->file->extra(HA_EXTRA_NO_KEYREAD);
   }
-  DBUG_RETURN(table->found_records == HA_POS_ERROR);
+  DBUG_RETURN(table->sort.found_records == HA_POS_ERROR);
 err:
   DBUG_RETURN(-1);
 }

sql/sql_sort.h

@@ -19,6 +19,30 @@
 #define MERGEBUFF		7
 #define MERGEBUFF2		15
 
+/*
+   The structure SORT_ADDON_FIELD describes a fixed layout
+   for field values appended to sorted values in records to be sorted
+   in the sort buffer.
+   Only fixed layout is supported now.
+   Null bit maps for the appended values is placed before the values 
+   themselves. Offsets are from the last sorted field, that is from the
+   record referefence, which is still last component of sorted records.
+   It is preserved for backward compatiblility.
+   The structure is used tp store values of the additional fields 
+   in the sort buffer. It is used also when these values are read
+   from a temporary file/buffer. As the reading procedures are beyond the
+   scope of the 'filesort' code the values have to be retrieved via
+   the callback function 'unpack_addon_fields'.
+*/
+
+typedef struct st_sort_addon_field {  /* Sort addon packed field */
+  Field *field;          /* Original field */
+  uint   offset;         /* Offset from the last sorted field */
+  uint   null_offset;    /* Offset to to null bit from the last sorted field */
+  uint   length;         /* Length in the sort buffer */
+  uint8  null_bit;       /* Null bit mask for the field */
+} SORT_ADDON_FIELD;
+
 typedef struct st_buffpek {		/* Struktur om sorteringsbuffrarna */
   my_off_t file_pos;			/* Where we are in the sort file */
   uchar *base,*key;			/* key pointers */
@@ -27,15 +51,18 @@ typedef struct st_buffpek {		/* Struktur om sorteringsbuffrarna */
   ulong max_keys;			/* Max keys in buffert */
 } BUFFPEK;
 
-
 typedef struct st_sort_param {
-  uint sort_length;			/* Length of sort columns */
-  uint keys;				/* Max keys / buffert */
+  uint rec_length;          /* Length of sorted records */
+  uint sort_length;			/* Length of sorted columns */
   uint ref_length;			/* Length of record ref. */
+  uint addon_length;        /* Length of added packed fields */
+  uint res_length;          /* Length of records in final sorted file/buffer */
+  uint keys;				/* Max keys / buffer */
   ha_rows max_rows,examined_rows;
   TABLE *sort_form;			/* For quicker make_sortkey */
   SORT_FIELD *local_sortorder;
   SORT_FIELD *end;
+  SORT_ADDON_FIELD *addon_field; /* Descriptors for companion fields */
   uchar *unique_buff;
   bool not_killable;
   char* tmp_buffer;

sql/sql_table.cc

@@ -2342,7 +2342,7 @@ copy_data_between_tables(TABLE *from,TABLE *to,
 
   if (order)
   {
-    from->io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
+    from->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
                                               MYF(MY_FAE | MY_ZEROFILL));
     bzero((char*) &tables,sizeof(tables));
     tables.table = from;
@@ -2355,7 +2355,7 @@ copy_data_between_tables(TABLE *from,TABLE *to,
 	setup_order(thd, thd->lex.select_lex.ref_pointer_array,
 		    &tables, fields, all_fields, order) ||
         !(sortorder=make_unireg_sortorder(order, &length)) ||
-        (from->found_records = filesort(thd, from, sortorder, length, 
+        (from->sort.found_records = filesort(thd, from, sortorder, length, 
                                              (SQL_SELECT *) 0, HA_POS_ERROR,
                                              &examined_rows))
         == HA_POS_ERROR)

sql/sql_update.cc

@@ -201,14 +201,14 @@ int mysql_update(THD *thd,
       bzero((char*) &tables,sizeof(tables));
       tables.table = table;
 
-      table->io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
+      table->sort.io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
                                                MYF(MY_FAE | MY_ZEROFILL));
       if (setup_ref_array(thd, &thd->lex.select_lex.ref_pointer_array,
 			order_num)||
 	  setup_order(thd, thd->lex.select_lex.ref_pointer_array,
 		      &tables, fields, all_fields, order) ||
           !(sortorder=make_unireg_sortorder(order, &length)) ||
-          (table->found_records = filesort(thd, table, sortorder, length,
+          (table->sort.found_records = filesort(thd, table, sortorder, length,
                                            (SQL_SELECT *) 0,
 					   HA_POS_ERROR, &examined_rows))
           == HA_POS_ERROR)

sql/structs.h

@@ -104,6 +104,7 @@ typedef struct st_read_record {			/* Parameter to read_record */
   uint index;
   byte *ref_pos;				/* pointer to form->refpos */
   byte *record;
+  byte *rec_buf;                /* to read field values  after filesort */
   byte	*cache,*cache_pos,*cache_end,*read_positions;
   IO_CACHE *io_cache;
   bool print_error, ignore_not_found_rows;

sql/table.h

@@ -44,6 +44,17 @@ typedef struct st_grant_info
 
 enum tmp_table_type {NO_TMP_TABLE=0, TMP_TABLE=1, TRANSACTIONAL_TMP_TABLE=2};
 
+typedef struct st_filesort_info
+{
+  IO_CACHE *io_cache;           /* If sorted through filebyte                */
+  byte     *addon_buf;          /* Pointer to a buffer if sorted with fields */
+  uint      addon_length;       /* Length of the buffer                      */
+  struct st_sort_addon_field *addon_field;     /* Pointer to the fields info */
+  void    (*unpack)(struct st_sort_addon_field *, byte *); /* To unpack back */
+  byte     *record_pointers;    /* If sorted in memory                       */
+  ha_rows   found_records;      /* How many records in sort                  */
+} FILESORT_INFO;
+
 /* Table cache entry struct */
 
 class Field_timestamp;
@@ -120,9 +131,7 @@ struct st_table {
   table_map	map;                    /* ID bit of table (1,2,4,8,16...) */
   ulong		version,flush_version;
   uchar		*null_flags;
-  IO_CACHE	*io_cache;		/* If sorted trough filebyte */
-  byte		*record_pointers;	/* If sorted in memory */
-  ha_rows	found_records;		/* How many records in sort */
+  FILESORT_INFO sort;
   ORDER		*group;
   ha_rows	quick_rows[MAX_KEY];
   uint		quick_key_parts[MAX_KEY];

sql/uniques.cc

@@ -95,12 +95,12 @@ bool Unique::flush()
 bool Unique::get(TABLE *table)
 {
   SORTPARAM sort_param;
-  table->found_records=elements+tree.elements_in_tree;
+  table->sort.found_records=elements+tree.elements_in_tree;
 
   if (my_b_tell(&file) == 0)
   {
     /* Whole tree is in memory;  Don't use disk if you don't need to */
-    if ((record_pointers=table->record_pointers= (byte*)
+    if ((record_pointers=table->sort.record_pointers= (byte*)
 	 my_malloc(tree.size_of_element * tree.elements_in_tree, MYF(0))))
     {
       (void) tree_walk(&tree, (tree_walk_action) unique_write_to_ptrs,
@@ -112,7 +112,7 @@ bool Unique::get(TABLE *table)
   if (flush())
     return 1;
 
-  IO_CACHE *outfile=table->io_cache;
+  IO_CACHE *outfile=table->sort.io_cache;
   BUFFPEK *file_ptr= (BUFFPEK*) file_ptrs.buffer;
   uint maxbuffer= file_ptrs.elements - 1;
   uchar *sort_buffer;
@@ -120,7 +120,7 @@ bool Unique::get(TABLE *table)
   bool error=1;
 
       /* Open cached file if it isn't open */
-  outfile=table->io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
+  outfile=table->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE), 
                                 MYF(MY_ZEROFILL));
 
   if (!outfile || ! my_b_inited(outfile) &&