Fix LP BUG#719198, LP BUG#730604

Analysis (BUG#719198):
The assert failed because the execution code for
partial matching is designed with the assumption that
NULLs on the left side are detected as early as possible,
and a NULL result is returned before any lookups are
performed at all.

However, in the case of an Item_cache object on the left
side, null was not detected properly, because detection
was done via Item::is_null(), which is not implemented at
all for Item_cache, and resolved to the default Item::is_null()
which always returns FALSE.

Solution:
Imlpement Item::is_null().

******

Analysis (BUG#730604):
The method Item_field::is_null() determines if an item is NULL from its
Item_field::field object. However, for Item_fields that represent internal
temporary tables, Item_field::field represents the field of the original
table that was the source for the temporary table (in this case t1.f3).
Both in the committed test case, and in the original bug report the current
value of t1.f3 is not NULL. This results in an incorrect count of NULLs
for this column. As a consequence, all related Ordered_key buffers are
allocated with incorrect sizes. Depending on the exact query and data,
these incorrect sizes result in various crashes or failed asserts.

Solution:
The correct value of the current field of the internal temp table is
in Item_field::result_field. This value is determined by
Item::is_null_result().

mysql-test/r/subselect_mat.result

@@ -1371,3 +1371,64 @@ SELECT pk FROM t1 WHERE (b,c,d) IN (SELECT b,c,d FROM t2 WHERE pk > 0);
 pk
 2
 DROP TABLE t1, t2;
+#
+# LPBUG#719198 Ordered_key::cmp_key_with_search_key(rownum_t): Assertion `!compare_pred[i]->null_value'
+# failed with subquery on both sides of NOT IN and materialization
+#
+CREATE TABLE t1 (f1a int, f1b int) ;
+INSERT IGNORE INTO t1 VALUES (1,1),(2,2);
+CREATE TABLE t2 ( f2 int);
+INSERT IGNORE INTO t2 VALUES (3),(4);
+CREATE TABLE t3 (f3a int, f3b int);
+set session optimizer_switch='materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=off';
+EXPLAIN
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3) NOT IN (SELECT f1a FROM t1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	t3	system	NULL	NULL	NULL	NULL	0	const row not found
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3) NOT IN (SELECT f1a FROM t1);
+f2
+EXPLAIN
+SELECT * FROM t2 WHERE (SELECT f3a, f3b FROM t3) NOT IN (SELECT f1a, f1b FROM t1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	t3	system	NULL	NULL	NULL	NULL	0	const row not found
+SELECT * FROM t2 WHERE (SELECT f3a, f3b FROM t3) NOT IN (SELECT f1a, f1b FROM t1);
+f2
+insert into t3 values (1,1),(2,2);
+EXPLAIN
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3 where f3a > 3) NOT IN (SELECT f1a FROM t1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	t3	ALL	NULL	NULL	NULL	NULL	2	Using where
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3 where f3a > 3) NOT IN (SELECT f1a FROM t1);
+f2
+3
+4
+drop table t1, t2, t3;
+#
+# LPBUG#730604 Assertion `bit < (map)->n_bits' failed in maria-5.3 with
+#  partial_match_rowid_merge
+#
+CREATE TABLE t1 (f1 int NOT NULL, f2 int, f3 int) ;
+CREATE TABLE t2 (f1 int NOT NULL, f2 int, f3 int) ;
+INSERT INTO t1 VALUES (60, 3, null), (61, null, 77);
+INSERT INTO t2 VALUES (1000,6,2);
+set @@optimizer_switch='materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=off';
+EXPLAIN
+SELECT (f1, f2, f3) NOT IN
+(SELECT COUNT(DISTINCT f2), f1, f3 FROM t1 GROUP BY f1, f3)
+FROM t2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	system	NULL	NULL	NULL	NULL	1	
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	Using filesort
+SELECT (f1, f2, f3) NOT IN
+(SELECT COUNT(DISTINCT f2), f1, f3 FROM t1 GROUP BY f1, f3)
+FROM t2;
+(f1, f2, f3) NOT IN
+(SELECT COUNT(DISTINCT f2), f1, f3 FROM t1 GROUP BY f1, f3)
+1
+drop table t1, t2;

mysql-test/t/subselect_mat.test

@@ -1011,3 +1011,55 @@ SELECT pk FROM t1 WHERE (a) IN (SELECT a FROM t2 WHERE pk > 0);
 SELECT pk FROM t1 WHERE (b,c,d) IN (SELECT b,c,d FROM t2 WHERE pk > 0);
 DROP TABLE t1, t2;
 
+--echo #
+--echo # LPBUG#719198 Ordered_key::cmp_key_with_search_key(rownum_t): Assertion `!compare_pred[i]->null_value'
+--echo # failed with subquery on both sides of NOT IN and materialization
+--echo #
+
+CREATE TABLE t1 (f1a int, f1b int) ;
+INSERT IGNORE INTO t1 VALUES (1,1),(2,2);
+CREATE TABLE t2 ( f2 int);
+INSERT IGNORE INTO t2 VALUES (3),(4);
+CREATE TABLE t3 (f3a int, f3b int);
+
+set session optimizer_switch='materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=off';
+
+EXPLAIN
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3) NOT IN (SELECT f1a FROM t1);
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3) NOT IN (SELECT f1a FROM t1);
+
+EXPLAIN
+SELECT * FROM t2 WHERE (SELECT f3a, f3b FROM t3) NOT IN (SELECT f1a, f1b FROM t1);
+SELECT * FROM t2 WHERE (SELECT f3a, f3b FROM t3) NOT IN (SELECT f1a, f1b FROM t1);
+
+insert into t3 values (1,1),(2,2);
+
+EXPLAIN
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3 where f3a > 3) NOT IN (SELECT f1a FROM t1);
+SELECT * FROM t2 WHERE (SELECT f3a FROM t3 where f3a > 3) NOT IN (SELECT f1a FROM t1);
+
+drop table t1, t2, t3;
+
+--echo #
+--echo # LPBUG#730604 Assertion `bit < (map)->n_bits' failed in maria-5.3 with
+--echo #  partial_match_rowid_merge
+--echo #
+
+CREATE TABLE t1 (f1 int NOT NULL, f2 int, f3 int) ;
+CREATE TABLE t2 (f1 int NOT NULL, f2 int, f3 int) ;
+
+INSERT INTO t1 VALUES (60, 3, null), (61, null, 77);
+INSERT INTO t2 VALUES (1000,6,2);
+
+set @@optimizer_switch='materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=off';
+
+EXPLAIN
+SELECT (f1, f2, f3) NOT IN
+       (SELECT COUNT(DISTINCT f2), f1, f3 FROM t1 GROUP BY f1, f3)
+FROM t2;
+
+SELECT (f1, f2, f3) NOT IN
+       (SELECT COUNT(DISTINCT f2), f1, f3 FROM t1 GROUP BY f1, f3)
+FROM t2;
+
+drop table t1, t2;

sql/item.h

@@ -3417,6 +3417,7 @@ public:
 
   virtual void store(Item *item);
   virtual bool cache_value()= 0;
+  bool is_null() { return null_value; }
 };
 
 

sql/sql_class.cc

@@ -3102,7 +3102,7 @@ bool select_materialize_with_stats::send_data(List<Item> &items)
 
   while ((cur_item= item_it++))
   {
-    if (cur_item->is_null())
+    if (cur_item->is_null_result())
     {
       ++cur_col_stat->null_count;
       cur_col_stat->max_null_row= count_rows;