Fix for bug MDEV-3992

Analysis:
  The crash is a result of incorrect analysis of whether a secondary key
  can be extended with a primary in order to compute ORDER BY. The analysis
  is done in test_if_order_by_key(). This function doesn't take into account
  that the primary key may in fact index the same columns as the secondary
  key. For the test query test_if_order_by_key says that there is an extended
  key with total 2 keyparts.
  At the same time, the condition
    if (pkinfo->key_part[i].field->key_start.is_set(nr))
  in test_if_cheaper_oredring() becomes true for (i == 0), which results in
  an invalid access to rec_per_key[-1].
  
Solution:
  The best solution would be to reuse KEY::ext_key_parts that is already computed
  by open_binary_frm(), however after detailed analysis the conclusion is that
  the change would be too intrusive for a GA release.
  The solution for 5.5 is to add a guard for the case when the 0-th key part is
  considered, and to assume that all keys will be scanned in this case.

mysql-test/r/group_by_innodb.result

+#
+# MDEV-3992 Server crash or valgrind errors in test_if_skip_sort_order/test_if_cheaper_ordering
+# on GROUP BY with indexes on InnoDB table
+#
+CREATE TABLE t1 (
+pk INT PRIMARY KEY,
+a VARCHAR(1) NOT NULL,
+KEY (pk)
+) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,'a'),(2,'b');
+EXPLAIN
+SELECT COUNT(*), pk field1, pk AS field2
+FROM t1 WHERE a = 'r' OR pk = 183 
+GROUP BY field1, field2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	PRIMARY,pk	pk	4	NULL	2	Using where
+SELECT COUNT(*), pk field1, pk AS field2
+FROM t1 WHERE a = 'r' OR pk = 183 
+GROUP BY field1, field2;
+COUNT(*)	field1	field2
+EXPLAIN
+SELECT COUNT(*), pk field1 FROM t1
+WHERE a = 'r' OR pk = 183 GROUP BY field1, field1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	PRIMARY,pk	pk	4	NULL	2	Using where
+SELECT COUNT(*), pk field1 FROM t1
+WHERE a = 'r' OR pk = 183 GROUP BY field1, field1;
+COUNT(*)	field1
+drop table t1;
+End of 5.5 tests

mysql-test/t/group_by_innodb.test

+#
+# Test GROUP BY queries that utilize InnoDB extended keys
+#
+
+--source include/have_innodb.inc
+
+--echo #
+--echo # MDEV-3992 Server crash or valgrind errors in test_if_skip_sort_order/test_if_cheaper_ordering
+--echo # on GROUP BY with indexes on InnoDB table
+--echo #
+
+CREATE TABLE t1 (
+  pk INT PRIMARY KEY,
+  a VARCHAR(1) NOT NULL,
+  KEY (pk)
+) ENGINE=InnoDB;
+
+INSERT INTO t1 VALUES (1,'a'),(2,'b');
+
+EXPLAIN
+SELECT COUNT(*), pk field1, pk AS field2
+FROM t1 WHERE a = 'r' OR pk = 183 
+GROUP BY field1, field2;
+
+SELECT COUNT(*), pk field1, pk AS field2
+FROM t1 WHERE a = 'r' OR pk = 183 
+GROUP BY field1, field2;
+
+EXPLAIN
+SELECT COUNT(*), pk field1 FROM t1
+WHERE a = 'r' OR pk = 183 GROUP BY field1, field1;
+
+SELECT COUNT(*), pk field1 FROM t1
+WHERE a = 'r' OR pk = 183 GROUP BY field1, field1;
+
+drop table t1;
+
+--echo End of 5.5 tests

sql/sql_select.cc

@@ -18144,7 +18144,15 @@ static int test_if_order_by_key(ORDER *order, TABLE *table, uint idx,
         key_part_end=key_part+table->key_info[table->s->primary_key].key_parts;
         const_key_parts=table->const_key_parts[table->s->primary_key];
 
-        for (; const_key_parts & 1 ; const_key_parts>>= 1)
+        /*
+          Check for constness only those keyparts of the PK suffix, that will
+          be used to extend the secondary key 'idx'. This handles the case when
+          some columns of the PK are used in the secondary index.
+        */
+        for (uint pk_part_idx= 0;
+             ((const_key_parts & 1) &&
+              (table->key_info[idx].ext_key_part_map & (1 << pk_part_idx)));
+             const_key_parts>>= 1, pk_part_idx++)
           key_part++; 
         /*
          The primary and secondary key parts were all const (i.e. there's
@@ -23009,7 +23017,8 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table,
                     if rec_per_key[0] != 0.
 	          */
                   DBUG_ASSERT(pkinfo->rec_per_key[i]);
-                  rec_per_key*= pkinfo->rec_per_key[i-1];
+                  rec_per_key*= (i == 0) ? table_records :
+                                           pkinfo->rec_per_key[i-1];
                   rec_per_key/= pkinfo->rec_per_key[i];
                 }
 	      }