BUG#22594514: HANDLE_FATAL_SIGNAL (SIG=11) IN

              UNIQUE::~UNIQUE | SQL/UNIQUES.CC:355

Analysis
========

Enabling the sort_buffer_size with a large value
can cause operations utilizing the sort buffer
like DELETE as mentioned in the bug report to
fail. 5.5 and 5.6 versions reports OOM error
while in 5.7+, the server crashes.

While initializing the mem_root for the sort buffer
tree, the block size for the mem_root is determined
from the 'sort_buffer_size' value. This unsigned
long value is typecasted to unsigned int, hence
it becomes zero. Further block_size computation
while initializing the mem_root results in a very
large block_size value. Hence while trying to
allocate a block during the DELETE operation,
an OOM error is reported. In case of 5.7+, the PFS
instrumentation for memory allocation, overshoots
the unsigned value and allocates a block of just
one byte. While trying to free the block of the
mem_root, the original block_size is used. This
triggers the crash since the server tries to free
unallocated memory.

Fix:
====
In order to restrict usage of such unreasonable
sort_buffer_size, the typecast of block size
to 'unsigned int' is removed and hence reports
OOM error across all versions for sizes
exceeding unsigned int range.

include/my_tree.h

-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -62,7 +62,7 @@ typedef struct st_tree {
 } TREE;
 
 	/* Functions on whole tree */
-void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,
+void init_tree(TREE *tree, size_t default_alloc_size, ulong memory_limit,
                int size, qsort_cmp2 compare, my_bool with_delete,
 	       tree_element_free free_element, void *custom_arg);
 void delete_tree(TREE*);

mysys/tree.c

-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -83,7 +83,7 @@ static void rb_delete_fixup(TREE *tree,TREE_ELEMENT ***parent);
 static int test_rb_tree(TREE_ELEMENT *element);
 #endif
 
-void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,
+void init_tree(TREE *tree, size_t default_alloc_size, ulong memory_limit,
                int size, qsort_cmp2 compare, my_bool with_delete,
 	       tree_element_free free_element, void *custom_arg)
 {
@@ -127,7 +127,7 @@ void init_tree(TREE *tree, ulong default_alloc_size, ulong memory_limit,
   }
   if (!(tree->with_delete=with_delete))
   {
-    init_alloc_root(&tree->mem_root, (uint) default_alloc_size, 0);
+    init_alloc_root(&tree->mem_root, default_alloc_size, 0);
     tree->mem_root.min_malloc=(sizeof(TREE_ELEMENT)+tree->size_of_element);
   }
   DBUG_VOID_RETURN;