MDEV-5963: InnoDB: Assertion failure in file row0sel.cc line 2503 ...

- Backport the fix for MySQL Bug#13947868
- Add our testcase (they don't publish theirs)

mysql-test/r/partition_innodb.result

@@ -693,4 +693,36 @@ count(*)
 802
 drop table t3;
 drop table t1,t2;
+#
+# MDEV-5963: InnoDB: Assertion failure in file row0sel.cc line 2503, 
+#    Failing assertion: 0 with "key ptr now exceeds key end by 762 bytes"
+# (independent testcase for Oracle Bug#13947868)
+#
+CREATE TABLE t1 (f1 VARCHAR(512) CHARACTER SET utf8) ENGINE=InnoDB;
+INSERT INTO t1 VALUES ('j');
+CREATE TABLE t2 (
+f2 VARCHAR(5) CHARACTER SET latin1,
+f3 VARCHAR(5) CHARACTER SET utf8,
+f4 INT,
+f5 VARCHAR(512) CHARACTER SET utf8,
+f6 VARCHAR(256) CHARACTER SET utf8,
+key (f2),
+key (f3),
+key (f5)
+) ENGINE=InnoDB PARTITION BY LIST COLUMNS (f4)
+SUBPARTITION BY KEY(f6) SUBPARTITIONS 4 (
+PARTITION p0 VALUES IN (1,3,9,null),
+PARTITION p1 VALUES IN (2,4,0)
+);
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+INSERT INTO t2 VALUES  
+('k','s',3,'b','j'),('a','b',NULL,'v','j'),('c','m',9,'t',NULL),
+('b','l',9,'b',NULL),('i','y',3,'o','w'),('c','m',NULL,'a','m'),  
+('f','o',9,'m','w'),('f','q',NULL,'o','a');
+CREATE TABLE t3 LIKE t2;
+SELECT * FROM t1 INNER JOIN t2 ON ( f5 = f1 );
+f1	f2	f3	f4	f5	f6
+INSERT INTO t3 SELECT * FROM t2 WHERE f3 = 'm' AND f2 ='c';
+DROP TABLE t1,t2,t3;
 set global default_storage_engine=default;

mysql-test/t/partition_innodb.test

@@ -776,5 +776,39 @@ drop table t3;
 
 drop table t1,t2;
 
+--echo #
+--echo # MDEV-5963: InnoDB: Assertion failure in file row0sel.cc line 2503, 
+--echo #    Failing assertion: 0 with "key ptr now exceeds key end by 762 bytes"
+--echo # (independent testcase for Oracle Bug#13947868)
+--echo #
+CREATE TABLE t1 (f1 VARCHAR(512) CHARACTER SET utf8) ENGINE=InnoDB;
+INSERT INTO t1 VALUES ('j');
+
+CREATE TABLE t2 (
+  f2 VARCHAR(5) CHARACTER SET latin1,
+  f3 VARCHAR(5) CHARACTER SET utf8,
+  f4 INT,
+  f5 VARCHAR(512) CHARACTER SET utf8,
+  f6 VARCHAR(256) CHARACTER SET utf8,
+  key (f2),
+  key (f3),
+  key (f5)
+) ENGINE=InnoDB PARTITION BY LIST COLUMNS (f4)
+  SUBPARTITION BY KEY(f6) SUBPARTITIONS 4 (
+    PARTITION p0 VALUES IN (1,3,9,null),
+    PARTITION p1 VALUES IN (2,4,0)
+);
+
+INSERT INTO t2 VALUES  
+  ('k','s',3,'b','j'),('a','b',NULL,'v','j'),('c','m',9,'t',NULL),
+  ('b','l',9,'b',NULL),('i','y',3,'o','w'),('c','m',NULL,'a','m'),  
+  ('f','o',9,'m','w'),('f','q',NULL,'o','a');
+
+CREATE TABLE t3 LIKE t2;
+
+SELECT * FROM t1 INNER JOIN t2 ON ( f5 = f1 );
+INSERT INTO t3 SELECT * FROM t2 WHERE f3 = 'm' AND f2 ='c';
+
+DROP TABLE t1,t2,t3;
 
 set global default_storage_engine=default;

sql/opt_range.cc

@@ -2181,7 +2181,7 @@ int QUICK_ROR_INTERSECT_SELECT::init_ror_merged_scan(bool reuse_handler,
     quick->record= head->record[0];
   }
 
-  if (need_to_fetch_row && head->file->ha_rnd_init_with_error(1))
+  if (need_to_fetch_row && head->file->ha_rnd_init_with_error(false))
   {
     DBUG_PRINT("error", ("ROR index_merge rnd_init call failed"));
     DBUG_RETURN(1);
@@ -2363,8 +2363,13 @@ int QUICK_ROR_UNION_SELECT::reset()
     quick->save_last_pos();
     queue_insert(&queue, (uchar*)quick);
   }
-
-  if ((error= head->file->ha_rnd_init(1)))
+  /* Prepare for ha_rnd_pos calls. */
+  if (head->file->inited && (error= head->file->ha_rnd_end()))
+  {
+    DBUG_PRINT("error", ("ROR index_merge rnd_end call failed"));
+    DBUG_RETURN(error);
+  }
+  if ((error= head->file->ha_rnd_init(false)))
   {
     DBUG_PRINT("error", ("ROR index_merge rnd_init call failed"));
     DBUG_RETURN(error);