MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion

This assert is hit when we do filesort using the priority queue and try to insert elements in
the queue. The compare function used for the priority queue should handle the case for zerolength
sortkey.

mysql-test/r/order_by_zerolength-4285.result

@@ -24,3 +24,23 @@ Warning	1292	Truncated incorrect CHAR(0) value: '8'
 Warning	1292	Truncated incorrect CHAR(0) value: '9'
 Warning	1292	Truncated incorrect CHAR(0) value: '10'
 drop table t1;
+#
+# MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion
+#
+set @save_sql_mode= @@sql_mode;
+SET @@sql_mode= '';
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+explain
+SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	PRIMARY	4	NULL	2	Using index; Using filesort
+SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2;
+pk
+1
+2
+Warnings:
+Warning	1292	Truncated incorrect CHAR(0) value: '1'
+Warning	1292	Truncated incorrect CHAR(0) value: '2'
+set @@sql_mode= @save_sql_mode;
+drop table t1;

mysql-test/t/order_by_zerolength-4285.test

@@ -6,3 +6,16 @@ insert into t1 values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 select * from t1 order by now(), cast(pk as char(0));
 drop table t1;
 
+--echo #
+--echo # MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion
+--echo #
+
+set @save_sql_mode= @@sql_mode;
+SET @@sql_mode= '';
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+explain
+SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2;
+SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2;
+set @@sql_mode= @save_sql_mode;
+drop table t1;

mysys/ptr_cmp.c

@@ -52,6 +52,11 @@ static int ptr_compare_0(size_t *compare_length, uchar **a, uchar **b);
 static int ptr_compare_1(size_t *compare_length, uchar **a, uchar **b);
 static int ptr_compare_2(size_t *compare_length, uchar **a, uchar **b);
 static int ptr_compare_3(size_t *compare_length, uchar **a, uchar **b);
+static int degenerate_compare_func(size_t *compare_length, uchar **a, uchar **b)
+{
+  DBUG_ASSERT(*compare_length == 0);
+  return 0;
+}
 #endif	/* __sun */
 
 	/* Get a pointer to a optimal byte-compare function for a given size */
@@ -64,6 +69,8 @@ qsort2_cmp get_ptr_compare (size_t size __attribute__((unused)))
 #else
 qsort2_cmp get_ptr_compare (size_t size)
 {
+  if (size == 0)
+    return (qsort2_cmp) degenerate_compare_func;
   if (size < 4)
     return (qsort2_cmp) ptr_compare;
   switch (size & 3) {