limit HEAP table size with max_heap_table_size, better estimation for mem_per_row

heap/hp_create.c:
  limit HEAP table size with max_heap_table_size
heap/hp_write.c:
  limit HEAP table size with max_heap_table_size
include/heap.h:
  limit HEAP table size with max_heap_table_size
sql/ha_heap.cc:
  limit HEAP table size with max_heap_table_size
  better estimation for mem_per_row

heap/hp_create.c

@@ -123,15 +123,15 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
 	keyseg->flag=     0;
 	keyseg->null_bit= 0;
 	keyseg++;
-	
-	init_tree(&keyinfo->rb_tree, 0, 0, sizeof(byte*), 
+
+	init_tree(&keyinfo->rb_tree, 0, 0, sizeof(byte*),
 		  (qsort_cmp2)keys_compare, 1, NULL, NULL);
 	keyinfo->delete_key= hp_rb_delete_key;
 	keyinfo->write_key= hp_rb_write_key;
       }
       else
       {
-	init_block(&keyinfo->block, sizeof(HASH_INFO), min_records, 
+	init_block(&keyinfo->block, sizeof(HASH_INFO), min_records,
 		   max_records);
 	keyinfo->delete_key= hp_delete_key;
 	keyinfo->write_key= hp_write_key;
@@ -140,6 +140,7 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
     }
     share->min_records= min_records;
     share->max_records= max_records;
+    share->max_table_size= create_info->max_table_size;
     share->data_length= share->index_length= 0;
     share->reclength= reclength;
     share->blength= 1;

heap/hp_write.c

@@ -143,7 +143,8 @@ static byte *next_free_record_pos(HP_SHARE *info)
   }
   if (!(block_pos=(info->records % info->block.records_in_block)))
   {
-    if (info->records > info->max_records && info->max_records)
+    if ((info->records > info->max_records && info->max_records) ||
+        (info->data_length + info->index_length >= info->max_table_size))
     {
       my_errno=HA_ERR_RECORD_FILE_FULL;
       DBUG_RETURN(NULL);

include/heap.h

@@ -125,8 +125,8 @@ typedef struct st_hp_keydef		/* Key definition with open */
   TREE rb_tree;
   int (*write_key)(struct st_heap_info *info, struct st_hp_keydef *keyinfo,
 		   const byte *record, byte *recpos);
-  int (*delete_key)(struct st_heap_info *info, struct st_hp_keydef *keyinfo, 
-  		   const byte *record, byte *recpos, int flag);
+  int (*delete_key)(struct st_heap_info *info, struct st_hp_keydef *keyinfo,
+		   const byte *record, byte *recpos, int flag);
   uint (*get_key_length)(struct st_hp_keydef *keydef, const byte *key);
 } HP_KEYDEF;
 
@@ -135,7 +135,7 @@ typedef struct st_heap_share
   HP_BLOCK block;
   HP_KEYDEF  *keydef;
   ulong min_records,max_records;	/* Params to open */
-  ulong data_length,index_length;
+  ulong data_length,index_length,max_table_size;
   uint records;				/* records */
   uint blength;				/* records rounded up to 2^n */
   uint deleted;				/* Deleted records in database */
@@ -185,6 +185,7 @@ typedef struct st_heap_create_info
 {
   uint auto_key;
   uint auto_key_type;
+  ulong max_table_size;
   ulonglong auto_increment;
 } HP_CREATE_INFO;
 

sql/ha_heap.cc

@@ -460,12 +460,24 @@ int ha_heap::create(const char *name, TABLE *table_arg,
     KEY_PART_INFO *key_part=     pos->key_part;
     KEY_PART_INFO *key_part_end= key_part + pos->key_parts;
 
-    mem_per_row+= (pos->key_length + (sizeof(char*) * 2));
-
     keydef[key].keysegs=   (uint) pos->key_parts;
     keydef[key].flag=      (pos->flags & (HA_NOSAME | HA_NULL_ARE_EQUAL));
     keydef[key].seg=       seg;
-    keydef[key].algorithm= ((pos->algorithm == HA_KEY_ALG_UNDEF) ? 
+
+    switch (pos->algorithm) {
+    case HA_KEY_ALG_UNDEF:
+    case HA_KEY_ALG_HASH:
+      keydef[key].algorithm= HA_KEY_ALG_HASH;
+      mem_per_row+= sizeof(char*) * 2; // = sizeof(HASH_INFO)
+      break;
+    case HA_KEY_ALG_BTREE:
+      keydef[key].algorithm= HA_KEY_ALG_BTREE;
+      mem_per_row+=sizeof(TREE_ELEMENT)+pos->key_length+sizeof(char*);
+      break;
+    default:
+      DBUG_ASSERT(0); // cannot happen
+    }
+    keydef[key].algorithm= ((pos->algorithm == HA_KEY_ALG_UNDEF) ?
 			    HA_KEY_ALG_HASH : pos->algorithm);
 
     for (; key_part != key_part_end; key_part++, seg++)
@@ -501,17 +513,17 @@ int ha_heap::create(const char *name, TABLE *table_arg,
     }
   }
   mem_per_row+= MY_ALIGN(table_arg->reclength + 1, sizeof(char*));
-  max_rows = (ha_rows) (current_thd->variables.max_heap_table_size /
-			mem_per_row);
   HP_CREATE_INFO hp_create_info;
   hp_create_info.auto_key= auto_key;
   hp_create_info.auto_key_type= auto_key_type;
   hp_create_info.auto_increment= (create_info->auto_increment_value ?
 				  create_info->auto_increment_value - 1 : 0);
+  hp_create_info.max_table_size=current_thd->variables.max_heap_table_size;
+  max_rows = (ha_rows) (hp_create_info.max_table_size / mem_per_row);
   error= heap_create(fn_format(buff,name,"","",4+2),
 		     table_arg->keys,keydef, table_arg->reclength,
 		     (ulong) ((table_arg->max_rows < max_rows &&
-			       table_arg->max_rows) ? 
+			       table_arg->max_rows) ?
 			      table_arg->max_rows : max_rows),
 		     (ulong) table_arg->min_rows, &hp_create_info);
   my_free((gptr) keydef, MYF(0));