Commit 35d0b90f authored by Mitchell Hashimoto's avatar Mitchell Hashimoto

communicator/ssh: heartbeat the SSH connection to detect drops [GH-200]

parent 747f2606
...@@ -28,6 +28,8 @@ BUG FIXES: ...@@ -28,6 +28,8 @@ BUG FIXES:
* core: Fixed a couple cases where a double ctrl-C could panic. * core: Fixed a couple cases where a double ctrl-C could panic.
* core: Template validation fails if an override is specified for a * core: Template validation fails if an override is specified for a
non-existent builder. [GH-336] non-existent builder. [GH-336]
* core: The SSH connection is heartbeated so that drops can be
detected. [GH-200]
* builder/amazon/instance: Remove check for ec2-ami-tools because it * builder/amazon/instance: Remove check for ec2-ami-tools because it
didn't allow absolute paths to work properly. [GH-330] didn't allow absolute paths to work properly. [GH-330]
* builder/digitalocean: Send a soft shutdown request so that files * builder/digitalocean: Send a soft shutdown request so that files
......
...@@ -12,6 +12,8 @@ import ( ...@@ -12,6 +12,8 @@ import (
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"time"
) )
type comm struct { type comm struct {
...@@ -80,10 +82,16 @@ func (c *comm) Start(cmd *packer.RemoteCmd) (err error) { ...@@ -80,10 +82,16 @@ func (c *comm) Start(cmd *packer.RemoteCmd) (err error) {
return return
} }
// A channel to keep track of our done state
doneCh := make(chan struct{})
sessionLock := new(sync.Mutex)
timedOut := false
// Start a goroutine to wait for the session to end and set the // Start a goroutine to wait for the session to end and set the
// exit boolean and status. // exit boolean and status.
go func() { go func() {
defer session.Close() defer session.Close()
err := session.Wait() err := session.Wait()
exitStatus := 0 exitStatus := 0
if err != nil { if err != nil {
...@@ -93,8 +101,54 @@ func (c *comm) Start(cmd *packer.RemoteCmd) (err error) { ...@@ -93,8 +101,54 @@ func (c *comm) Start(cmd *packer.RemoteCmd) (err error) {
} }
} }
sessionLock.Lock()
defer sessionLock.Unlock()
if timedOut {
// We timed out, so set the exit status to -1
exitStatus = -1
}
log.Printf("remote command exited with '%d': %s", exitStatus, cmd.Command) log.Printf("remote command exited with '%d': %s", exitStatus, cmd.Command)
cmd.SetExited(exitStatus) cmd.SetExited(exitStatus)
close(doneCh)
}()
go func() {
failures := 0
for {
dummy, err := c.config.Connection()
if err == nil {
dummy.Close()
}
select {
case <-doneCh:
return
default:
}
if err != nil {
log.Printf("background SSH connection checker failure: %s", err)
failures += 1
}
if failures < 5 {
time.Sleep(5 * time.Second)
continue
}
// Acquire a lock in order to modify session state
sessionLock.Lock()
defer sessionLock.Unlock()
// Kill the connection and mark that we timed out.
log.Printf("Too many SSH connection failures. Killing it!")
c.conn.Close()
timedOut = true
return
}
}() }()
return return
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment