Bug#19660891 HANDLE_FATAL_SIGNAL (SIG=11) IN QUEUE_INSERT

Backport from 5.6 to 5.5
This makes filesort robust to misc variants of order by / group by
on columns/expressions with zero length.

mysys/ptr_cmp.c

-/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -41,6 +41,16 @@ static int native_compare(size_t *length, unsigned char **a, unsigned char **b)
 
 #else	/* __sun */
 
+/**
+  Special case for ORDER BY / GROUP BY CHAR(0) NOT NULL
+ */
+static
+int ptr_compare_zero_length(size_t *compare_length __attribute__((unused)),
+                            uchar **a __attribute__((unused)),
+                            uchar **b __attribute__((unused)))
+{
+  return 0;
+}
 static int ptr_compare(size_t *compare_length, uchar **a, uchar **b);
 static int ptr_compare_0(size_t *compare_length, uchar **a, uchar **b);
 static int ptr_compare_1(size_t *compare_length, uchar **a, uchar **b);
@@ -58,6 +68,8 @@ qsort2_cmp get_ptr_compare (size_t size __attribute__((unused)))
 #else
 qsort2_cmp get_ptr_compare (size_t size)
 {
+  if (size == 0)
+    return (qsort2_cmp) ptr_compare_zero_length;
   if (size < 4)
     return (qsort2_cmp) ptr_compare;
   switch (size & 3) {
@@ -85,6 +97,7 @@ static int ptr_compare(size_t *compare_length, uchar **a, uchar **b)
   reg3 int length= *compare_length;
   reg1 uchar *first,*last;
 
+  DBUG_ASSERT(length > 0);
   first= *a; last= *b;
   while (--length)
   {

sql/filesort.cc

@@ -265,9 +265,6 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   }
   else
   {
-    /* filesort cannot handle zero-length records during merge. */
-    DBUG_ASSERT(param.sort_length != 0);
-
     if (table_sort.buffpek && table_sort.buffpek_len < maxbuffer)
     {
       my_free(table_sort.buffpek);