Bug #46080: group_concat(... order by) crashes server when

  sort_buffer_size cannot allocate

The NULL return from tree_insert() (on low memory) was not
checked for in Item_func_group_concat::add(). As a result
on low memory conditions a crash happens.

Fixed by properly checking the return code.

mysql-test/r/bug46080.result

+#
+# Bug #46080: group_concat(... order by) crashes server when
+#  sort_buffer_size cannot allocate
+#
+CREATE TABLE t1(a CHAR(255));
+INSERT INTO t1 VALUES ('a');
+SET @@SESSION.sort_buffer_size=5*16*1000000;
+SET @@SESSION.max_heap_table_size=5*1000000;
+# Must not crash.
+SELECT GROUP_CONCAT(a ORDER BY a) FROM t1 GROUP BY a
+DROP TABLE t1;
+SET @@SESSION.sort_buffer_size=default;
+SET @@SESSION.max_heap_table_size=default;
+End of 5.0 tests

mysql-test/t/bug46080-master.opt

+--skip-grant-tables --skip-name-resolve --safemalloc-mem-limit=4000000

mysql-test/t/bug46080.test

+--echo #
+--echo # Bug #46080: group_concat(... order by) crashes server when
+--echo #  sort_buffer_size cannot allocate
+--echo #
+
+CREATE TABLE t1(a CHAR(255));
+INSERT INTO t1 VALUES ('a');
+
+SET @@SESSION.sort_buffer_size=5*16*1000000;
+SET @@SESSION.max_heap_table_size=5*1000000;
+
+echo # Must not crash.
+SELECT GROUP_CONCAT(a ORDER BY a) FROM t1 GROUP BY a;
+
+DROP TABLE t1;
+SET @@SESSION.sort_buffer_size=default;
+SET @@SESSION.max_heap_table_size=default;
+
+--echo End of 5.0 tests

sql/item_sum.cc

@@ -3291,8 +3291,13 @@ bool Item_func_group_concat::add()
 
   TREE_ELEMENT *el= 0;                          // Only for safety
   if (row_eligible && tree)
+  {
     el= tree_insert(tree, table->record[0] + table->s->null_bytes, 0,
                     tree->custom_arg);
+    /* check if there was enough memory to insert the row */
+    if (!el)
+      return 1;
+  }
   /*
     If the row is not a duplicate (el->count == 1)
     we can dump the row here in case of GROUP_CONCAT(DISTINCT...)