Commit 6fb2586f authored by Matt Page's avatar Matt Page

fix: Ignore transient network errors when fetching AWS state.

Given that state fetching is an idempotent operation, a transient
network error should not cause the entire build to fail. Instead,
retry when such errors are encountered.
parent 2aa5a198
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"github.com/mitchellh/goamz/ec2" "github.com/mitchellh/goamz/ec2"
"github.com/mitchellh/multistep" "github.com/mitchellh/multistep"
"log" "log"
"net"
"time" "time"
) )
...@@ -29,6 +30,14 @@ type StateChangeConf struct { ...@@ -29,6 +30,14 @@ type StateChangeConf struct {
Target string Target string
} }
func isTransientNetworkError(err error) bool {
ret := false
if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
ret = true
}
return ret
}
// AMIStateRefreshFunc returns a StateRefreshFunc that is used to watch // AMIStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an AMI for state changes. // an AMI for state changes.
func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc { func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc {
...@@ -38,6 +47,9 @@ func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc { ...@@ -38,6 +47,9 @@ func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc {
if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidAMIID.NotFound" { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidAMIID.NotFound" {
// Set this to nil as if we didn't find anything. // Set this to nil as if we didn't find anything.
resp = nil resp = nil
} else if isTransientNetworkError(err) {
// Transient network error, treat it as if we didn't find anything
resp = nil
} else { } else {
log.Printf("Error on AMIStateRefresh: %s", err) log.Printf("Error on AMIStateRefresh: %s", err)
return nil, "", err return nil, "", err
...@@ -64,6 +76,9 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc { ...@@ -64,6 +76,9 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidInstanceID.NotFound" { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidInstanceID.NotFound" {
// Set this to nil as if we didn't find anything. // Set this to nil as if we didn't find anything.
resp = nil resp = nil
} else if isTransientNetworkError(err) {
// Transient network error, treat it as if we didn't find anything
resp = nil
} else { } else {
log.Printf("Error on InstanceStateRefresh: %s", err) log.Printf("Error on InstanceStateRefresh: %s", err)
return nil, "", err return nil, "", err
...@@ -90,6 +105,9 @@ func SpotRequestStateRefreshFunc(conn *ec2.EC2, spotRequestId string) StateRefre ...@@ -90,6 +105,9 @@ func SpotRequestStateRefreshFunc(conn *ec2.EC2, spotRequestId string) StateRefre
if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidSpotInstanceRequestID.NotFound" { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidSpotInstanceRequestID.NotFound" {
// Set this to nil as if we didn't find anything. // Set this to nil as if we didn't find anything.
resp = nil resp = nil
} else if isTransientNetworkError(err) {
// Transient network error, treat it as if we didn't find anything
resp = nil
} else { } else {
log.Printf("Error on SpotRequestStateRefresh: %s", err) log.Printf("Error on SpotRequestStateRefresh: %s", err)
return nil, "", err return nil, "", err
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment