Add missing check for thd->killed in mysql_binlog_send().

The slave dump thread running on the master only checked thd->killed whenever
it reached the end of a binlog file, not between events. This could
unnecessarily delay server shutdown.

This was found by code inspection while tracking down some occasional "forcing
close of thread..." errors in Buildbot. Hopefully this will fix the failures,
but the fix is correct in any case.

Also increase the wait during server shutdown, 2 seconds is a bit tight in
case of heavy I/O stall, and it seems better to delay shutdown a bit than
force-kill threads unnecessarily.

Also fix some races in test cases that restart the mysqld server. The .expect
file should be changed with --append_file, --remove_file + --write_file
creates a short window where mysqld can error out due to .expect file missing.

mysql-test/suite/binlog/t/binlog_mdev342.test

@@ -44,8 +44,7 @@ connection con1;
 --error 2006,2013
 reap;
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-binlog_mdev342.test
 EOF
 

mysql-test/suite/binlog/t/binlog_xa_recover.test

@@ -117,8 +117,7 @@ connection con4;
 --error 2006,2013
 reap;
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-group_commit_binlog_pos.test
 EOF
 
@@ -212,8 +211,7 @@ SET SESSION debug_dbug="+d,crash_commit_after_log";
 --error 2006,2013
 INSERT INTO t1 VALUES (14, NULL);
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-group_commit_binlog_pos.test
 EOF
 
@@ -257,8 +255,7 @@ SET SESSION debug_dbug="+d,crash_before_write_checkpoint_event";
 --error 2006,2013
 INSERT INTO t1 VALUES (24, REPEAT("x", 4100));
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-group_commit_binlog_pos.test
 EOF
 

mysql-test/suite/rpl/t/rpl_gtid_crash.test

@@ -52,8 +52,7 @@ SET SESSION debug_dbug="+d,crash_dispatch_command_before";
 SELECT 1;
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-rpl_gtid_crash.test
 EOF
 
@@ -112,8 +111,7 @@ SET SESSION debug_dbug="+d,crash_dispatch_command_before";
 SELECT 1;
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart
 EOF
 
@@ -147,8 +145,7 @@ INSERT INTO t1 VALUES (4);
 --connection server_2
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart: --skip-slave-start=0
 EOF
 
@@ -172,8 +169,7 @@ INSERT INTO t1 VALUES (5);
 --connection server_2
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart: --skip-slave-start=0
 EOF
 
@@ -197,8 +193,7 @@ INSERT INTO t1 VALUES (6);
 --connection server_2
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart: --skip-slave-start=0
 EOF
 
@@ -222,8 +217,7 @@ INSERT INTO t1 VALUES (7);
 --connection server_2
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart: --skip-slave-start=0
 EOF
 
@@ -247,8 +241,7 @@ INSERT INTO t1 VALUES (8);
 --connection server_2
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart: --skip-slave-start=0
 EOF
 

mysql-test/suite/rpl/t/rpl_gtid_stop_start.test

@@ -41,8 +41,7 @@ INSERT INTO t1 VALUES (3);
 --source include/show_binary_logs.inc
 
 # Let the slave mysqld server start again.
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart: --skip-slave-start=0
 EOF
 
@@ -72,8 +71,7 @@ EOF
 --shutdown_server 30
 --source include/wait_until_disconnected.inc
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart
 EOF
 

mysql-test/suite/rpl/t/rpl_mdev382.test

@@ -210,8 +210,7 @@ EOF
 
 --shutdown_server 30
 
---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-rpl_mdev382.test
 EOF
 

mysql-test/suite/storage_engine/alter_tablespace.test

@@ -53,8 +53,7 @@ EOF
   --replace_result $datadir <DATADIR>
   --copy_file $datadir/test/t1.ibd $datadir/test/t1.ibd.save
 
-  --remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
-  --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+  --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart
 EOF
 

sql/mysqld.cc

@@ -1494,8 +1494,21 @@ static void close_connections(void)
   Events::deinit();
   end_slave();
 
-  /* Give threads time to die. */
-  for (int i= 0; *(volatile int32*) &thread_count && i < 100; i++)
+  /*
+    Give threads time to die.
+
+    In 5.5, this was waiting 100 rounds @ 20 milliseconds/round, so as little
+    as 2 seconds, depending on thread scheduling.
+
+    From 10.0, we increase this to 1000 rounds / 20 seconds. The rationale is
+    that on a server with heavy I/O load, it is quite possible for eg. an
+    fsync() of the binlog or whatever to cause something like LOCK_log to be
+    held for more than 2 seconds. We do not want to force kill threads in
+    such cases, if it can be avoided. Note that normally, the wait will be
+    much smaller than even 2 seconds, this is only a safety fallback against
+    stuck threads so server shutdown is not held up forever.
+  */
+  for (int i= 0; *(volatile int32*) &thread_count && i < 1000; i++)
     my_sleep(20000);
 
   /*

sql/sql_repl.cc

@@ -1727,13 +1727,15 @@ impossible position";
   while (!net->error && net->vio != 0 && !thd->killed)
   {
     Log_event_type event_type= UNKNOWN_EVENT;
+    killed_state killed;
 
     /* reset the transmit packet for the event read from binary log
        file */
     if (reset_transmit_packet(thd, flags, &ev_offset, &errmsg))
       goto err;
 
-    while (!(error = Log_event::read_log_event(&log, packet, log_lock,
+    while (!(killed= thd->killed) &&
+           !(error = Log_event::read_log_event(&log, packet, log_lock,
                                                current_checksum_alg)))
     {
 #ifndef DBUG_OFF
@@ -1818,6 +1820,8 @@ impossible position";
       if (reset_transmit_packet(thd, flags, &ev_offset, &errmsg))
         goto err;
     }
+    if (killed)
+      goto end;
 
     /*
       TODO: now that we are logging the offset, check to make sure