Fix for BUG##5714 "Insert into MyISAM table and select ... for update]":

the fact that the transaction log is empty does not mean we're not in a transaction
(it could be BEGIN; SELECT * FOR UPDATE FROM ibtable: then we don't want to commit now, even if
the statement is a MyISAM update).
With a testcase.

mysql-test/r/mix_innodb_myisam_binlog.result

@@ -177,4 +177,25 @@ master-bin.001	79	Query	1	79	use `test`; BEGIN
 master-bin.001	119	Query	1	79	use `test`; insert into t1 values(16)
 master-bin.001	179	Query	1	79	use `test`; insert into t1 values(18)
 master-bin.001	239	Query	1	239	use `test`; COMMIT
+delete from t1;
+delete from t2;
+alter table t2 type=MyISAM;
+insert into t1 values (1);
+begin;
+select * from t1 for update;
+a
+1
+select (@before:=unix_timestamp())*0;
+(@before:=unix_timestamp())*0
+0
+begin;
+ select * from t1 for update;
+insert into t2 values (20);
+Lock wait timeout exceeded; Try restarting transaction
+select (@after:=unix_timestamp())*0;
+(@after:=unix_timestamp())*0
+0
+select (@after-@before) >= 2;
+(@after-@before) >= 2
+1
 drop table t1,t2;

mysql-test/t/mix_innodb_myisam_binlog-master.opt

+--loose-innodb_lock_wait_timeout=2

mysql-test/t/mix_innodb_myisam_binlog.test

@@ -175,4 +175,36 @@ select a from t1 order by a; # check that savepoints work :)
 
 show binlog events from 79;
 
+# Test for BUG#5714, where a MyISAM update in the transaction used to
+# release row-level locks in InnoDB
+
+connect (con3,localhost,root,,);
+
+connection con3;
+delete from t1;
+delete from t2;
+--disable_warnings
+alter table t2 type=MyISAM;
+--enable_warnings
+insert into t1 values (1);
+begin;
+select * from t1 for update;
+
+connection con2;
+select (@before:=unix_timestamp())*0; # always give repeatable output
+begin;
+send select * from t1 for update;
+
+connection con3;
+insert into t2 values (20);
+
+connection con2;
+--error 1205
+reap;
+select (@after:=unix_timestamp())*0; # always give repeatable output
+# verify that innodb_lock_wait_timeout was exceeded. When there was
+# the bug, the reap would return immediately after the insert into t2.
+select (@after-@before) >= 2;
+
+# cleanup
 drop table t1,t2;

sql/log.cc

@@ -1254,7 +1254,8 @@ bool MYSQL_LOG::write(Log_event* event_info)
       if (flush_io_cache(file))
 	goto err;
  
-      if (opt_using_transactions && !my_b_tell(&thd->transaction.trans_log))
+      if (opt_using_transactions &&
+          !(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))
       {
         /*
           LOAD DATA INFILE in AUTOCOMMIT=1 mode writes to the binlog