Bug#34760 Character set autodetection appears to fail

the problem is the same as reported in bug#20835,
so the fix is backport of bug#20835 patch.


mysql-test/r/subselect.result:
  test result
mysql-test/t/subselect.test:
  test case

mysql-test/r/subselect.result

@@ -4407,4 +4407,19 @@ pk	a
 3	30
 2	20
 DROP TABLE t1,t2;
+CREATE TABLE t1 (s1 char(1));
+INSERT INTO t1 VALUES ('a');
+SELECT * FROM t1 WHERE _utf8'a' = ANY (SELECT s1 FROM t1);
+s1
+a
+DROP TABLE t1;
+CREATE TABLE t1(id BIGINT);
+CREATE TABLE t2(id1 BIGINT, id2 BIGINT);
+INSERT INTO t1 VALUES (1),(2),(3);
+INSERT INTO t2 VALUES (2,1),(3,1);
+SELECT * FROM t1 i WHERE 1 IN (SELECT l.id2 FROM t2 l WHERE i.id=l.id1);
+id
+2
+3
+DROP TABLE t1, t2;
 End of 5.0 tests.

mysql-test/t/subselect.test

@@ -3307,5 +3307,22 @@ SELECT * FROM t1
    WHERE EXISTS (SELECT DISTINCT a FROM t2 WHERE t1.a < t2.a ORDER BY b);
 DROP TABLE t1,t2;
 
---echo End of 5.0 tests.
+#
+# Bug#20835 (literal string with =any values)
+#
+CREATE TABLE t1 (s1 char(1));
+INSERT INTO t1 VALUES ('a');
+SELECT * FROM t1 WHERE _utf8'a' = ANY (SELECT s1 FROM t1);
+DROP TABLE t1;
 
+#
+# Bug#40519 Subselect query using bigint fails
+#
+CREATE TABLE t1(id BIGINT);
+CREATE TABLE t2(id1 BIGINT, id2 BIGINT);
+INSERT INTO t1 VALUES (1),(2),(3);
+INSERT INTO t2 VALUES (2,1),(3,1);
+SELECT * FROM t1 i WHERE 1 IN (SELECT l.id2 FROM t2 l WHERE i.id=l.id1);
+DROP TABLE t1, t2;
+
+--echo End of 5.0 tests.

sql/item_cmpfunc.cc

@@ -1434,7 +1434,8 @@ bool Item_in_optimizer::fix_left(THD *thd, Item **ref)
   }
   not_null_tables_cache= args[0]->not_null_tables();
   with_sum_func= args[0]->with_sum_func;
-  const_item_cache= args[0]->const_item();
+  if ((const_item_cache= args[0]->const_item()))
+    cache->store(args[0]);
   return 0;
 }