Merge branch 'henrysher-add-spot-instance-support'

builder/amazon/common/run_config.go

@@ -17,6 +17,7 @@ type RunConfig struct {
 	InstanceType             string            `mapstructure:"instance_type"`
 	RunTags                  map[string]string `mapstructure:"run_tags"`
 	SourceAmi                string            `mapstructure:"source_ami"`
+	SpotPrice                string            `mapstructure:"spot_price"`
 	RawSSHTimeout            string            `mapstructure:"ssh_timeout"`
 	SSHUsername              string            `mapstructure:"ssh_username"`
 	SSHPrivateKeyFile        string            `mapstructure:"ssh_private_key_file"`
@@ -46,6 +47,7 @@ func (c *RunConfig) Prepare(t *packer.ConfigTemplate) []error {
 	templates := map[string]*string{
 		"iam_instance_profile":    &c.IamInstanceProfile,
 		"instance_type":           &c.InstanceType,
+		"spot_price":              &c.SpotPrice,
 		"ssh_timeout":             &c.RawSSHTimeout,
 		"ssh_username":            &c.SSHUsername,
 		"ssh_private_key_file":    &c.SSHPrivateKeyFile,

builder/amazon/common/state.go

@@ -81,6 +81,32 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc {
 	}
 }
 
+// SpotRequestStateRefreshFunc returns a StateRefreshFunc that is used to watch
+// a spot request for state changes.
+func SpotRequestStateRefreshFunc(conn *ec2.EC2, spotRequestId string) StateRefreshFunc {
+	return func() (interface{}, string, error) {
+		resp, err := conn.DescribeSpotRequests([]string{spotRequestId}, ec2.NewFilter())
+		if err != nil {
+			if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidSpotInstanceRequestID.NotFound" {
+				// Set this to nil as if we didn't find anything.
+				resp = nil
+			} else {
+				log.Printf("Error on SpotRequestStateRefresh: %s", err)
+				return nil, "", err
+			}
+		}
+
+		if resp == nil || len(resp.SpotRequestResults) == 0 {
+			// Sometimes AWS has consistency issues and doesn't see the
+			// SpotRequest. Return an empty state.
+			return nil, "", nil
+		}
+
+		i := resp.SpotRequestResults[0]
+		return i, i.State, nil
+	}
+}
+
 // WaitForState watches an object and waits for it to achieve a certain
 // state.
 func WaitForState(conf *StateChangeConf) (i interface{}, err error) {
@@ -125,8 +151,8 @@ func WaitForState(conf *StateChangeConf) (i interface{}, err error) {
 			}
 
 			if !found {
-				fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target)
-				return
+				err := fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target)
+				return nil, err
 			}
 		}
 

builder/amazon/common/step_run_source_instance.go

@@ -10,6 +10,7 @@ import (
 
 type StepRunSourceInstance struct {
 	AssociatePublicIpAddress bool
+	SpotPrice                string
 	AvailabilityZone         string
 	BlockDevices             BlockDevices
 	Debug                    bool
@@ -21,8 +22,8 @@ type StepRunSourceInstance struct {
 	Tags                     map[string]string
 	UserData                 string
 	UserDataFile             string
-
-	instance *ec2.Instance
+	spotRequest              *ec2.SpotRequestResult
+	instance                 *ec2.Instance
 }
 
 func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepAction {
@@ -47,21 +48,6 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi
 		securityGroups[n] = ec2.SecurityGroup{Id: securityGroupId}
 	}
 
-	runOpts := &ec2.RunInstances{
-		KeyName:                  keyName,
-		ImageId:                  s.SourceAMI,
-		InstanceType:             s.InstanceType,
-		UserData:                 []byte(userData),
-		MinCount:                 0,
-		MaxCount:                 0,
-		SecurityGroups:           securityGroups,
-		IamInstanceProfile:       s.IamInstanceProfile,
-		SubnetId:                 s.SubnetId,
-		AssociatePublicIpAddress: s.AssociatePublicIpAddress,
-		BlockDevices:             s.BlockDevices.BuildLaunchDevices(),
-		AvailZone:                s.AvailabilityZone,
-	}
-
 	ui.Say("Launching a source AWS instance...")
 	imageResp, err := ec2conn.Images([]string{s.SourceAMI}, ec2.NewFilter())
 	if err != nil {
@@ -82,15 +68,85 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi
 		return multistep.ActionHalt
 	}
 
-	runResp, err := ec2conn.RunInstances(runOpts)
+	var instanceId []string
+	if s.SpotPrice == "" {
+		runOpts := &ec2.RunInstances{
+			KeyName:                  keyName,
+			ImageId:                  s.SourceAMI,
+			InstanceType:             s.InstanceType,
+			UserData:                 []byte(userData),
+			MinCount:                 0,
+			MaxCount:                 0,
+			SecurityGroups:           securityGroups,
+			IamInstanceProfile:       s.IamInstanceProfile,
+			SubnetId:                 s.SubnetId,
+			AssociatePublicIpAddress: s.AssociatePublicIpAddress,
+			BlockDevices:             s.BlockDevices.BuildLaunchDevices(),
+			AvailZone:                s.AvailabilityZone,
+		}
+		runResp, err := ec2conn.RunInstances(runOpts)
+		if err != nil {
+			err := fmt.Errorf("Error launching source instance: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+		instanceId = []string{runResp.Instances[0].InstanceId}
+	} else {
+		runOpts := &ec2.RequestSpotInstances{
+			SpotPrice:                s.SpotPrice,
+			KeyName:                  keyName,
+			ImageId:                  s.SourceAMI,
+			InstanceType:             s.InstanceType,
+			UserData:                 []byte(userData),
+			SecurityGroups:           securityGroups,
+			IamInstanceProfile:       s.IamInstanceProfile,
+			SubnetId:                 s.SubnetId,
+			AssociatePublicIpAddress: s.AssociatePublicIpAddress,
+			BlockDevices:             s.BlockDevices.BuildLaunchDevices(),
+			AvailZone:                s.AvailabilityZone,
+		}
+		runSpotResp, err := ec2conn.RequestSpotInstances(runOpts)
+		if err != nil {
+			err := fmt.Errorf("Error launching source spot instance: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+		s.spotRequest = &runSpotResp.SpotRequestResults[0]
+		spotRequestId := s.spotRequest.SpotRequestId
+		ui.Say(fmt.Sprintf("Waiting for spot request (%s) to become ready...", spotRequestId))
+		stateChange := StateChangeConf{
+			Pending:   []string{"open"},
+			Target:    "active",
+			Refresh:   SpotRequestStateRefreshFunc(ec2conn, spotRequestId),
+			StepState: state,
+		}
+		_, err = WaitForState(&stateChange)
+		if err != nil {
+			err := fmt.Errorf("Error waiting for spot request (%s) to become ready: %s", spotRequestId, err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+		spotResp, err := ec2conn.DescribeSpotRequests([]string{spotRequestId}, nil)
+		if err != nil {
+			err := fmt.Errorf("Error finding spot request (%s): %s", spotRequestId, err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+		instanceId = []string{spotResp.SpotRequestResults[0].InstanceId}
+	}
+
+	instanceResp, err := ec2conn.Instances(instanceId, nil)
 	if err != nil {
-		err := fmt.Errorf("Error launching source instance: %s", err)
+		err := fmt.Errorf("Error finding source instance (%s): %s", instanceId, err)
 		state.Put("error", err)
 		ui.Error(err.Error())
 		return multistep.ActionHalt
 	}
-
-	s.instance = &runResp.Instances[0]
+	s.instance = &instanceResp.Reservations[0].Instances[0]
 	ui.Message(fmt.Sprintf("Instance ID: %s", s.instance.InstanceId))
 
 	ui.Say(fmt.Sprintf("Waiting for instance (%s) to become ready...", s.instance.InstanceId))
@@ -142,24 +198,41 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi
 }
 
 func (s *StepRunSourceInstance) Cleanup(state multistep.StateBag) {
-	if s.instance == nil {
-		return
-	}
 
 	ec2conn := state.Get("ec2").(*ec2.EC2)
 	ui := state.Get("ui").(packer.Ui)
 
-	ui.Say("Terminating the source AWS instance...")
-	if _, err := ec2conn.TerminateInstances([]string{s.instance.InstanceId}); err != nil {
-		ui.Error(fmt.Sprintf("Error terminating instance, may still be around: %s", err))
-		return
-	}
+	// Cancel the spot request if it exists
+	if s.spotRequest != nil {
+		ui.Say("Cancelling the spot request...")
+		if _, err := ec2conn.CancelSpotRequests([]string{s.spotRequest.SpotRequestId}); err != nil {
+			ui.Error(fmt.Sprintf("Error cancelling the spot request, may still be around: %s", err))
+			return
+		}
+		stateChange := StateChangeConf{
+			Pending: []string{"active", "open"},
+			Refresh: SpotRequestStateRefreshFunc(ec2conn, s.spotRequest.SpotRequestId),
+			Target:  "cancelled",
+		}
+
+		WaitForState(&stateChange)
 
-	stateChange := StateChangeConf{
-		Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"},
-		Refresh: InstanceStateRefreshFunc(ec2conn, s.instance),
-		Target:  "terminated",
 	}
 
-	WaitForState(&stateChange)
+	// Terminate the source instance if it exists
+	if s.instance != nil {
+
+		ui.Say("Terminating the source AWS instance...")
+		if _, err := ec2conn.TerminateInstances([]string{s.instance.InstanceId}); err != nil {
+			ui.Error(fmt.Sprintf("Error terminating instance, may still be around: %s", err))
+			return
+		}
+		stateChange := StateChangeConf{
+			Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"},
+			Refresh: InstanceStateRefreshFunc(ec2conn, s.instance),
+			Target:  "terminated",
+		}
+
+		WaitForState(&stateChange)
+	}
 }

builder/amazon/ebs/builder.go

@@ -102,6 +102,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		&awscommon.StepRunSourceInstance{
 			Debug:                    b.config.PackerDebug,
 			ExpectedRootDevice:       "ebs",
+			SpotPrice:                b.config.SpotPrice,
 			InstanceType:             b.config.InstanceType,
 			UserData:                 b.config.UserData,
 			UserDataFile:             b.config.UserDataFile,
@@ -120,7 +121,8 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			SSHWaitTimeout: b.config.SSHTimeout(),
 		},
 		&common.StepProvision{},
-		&stepStopInstance{},
+		&stepStopInstance{SpotPrice: b.config.SpotPrice},
+		// TODO(mitchellh): verify works with spots
 		&stepModifyInstance{},
 		&stepCreateAMI{},
 		&awscommon.StepAMIRegionCopy{

builder/amazon/ebs/step_stop_instance.go

@@ -8,13 +8,21 @@ import (
 	"github.com/mitchellh/packer/packer"
 )
 
-type stepStopInstance struct{}
+type stepStopInstance struct {
+	SpotPrice string
+}
 
 func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction {
 	ec2conn := state.Get("ec2").(*ec2.EC2)
 	instance := state.Get("instance").(*ec2.Instance)
 	ui := state.Get("ui").(packer.Ui)
 
+	// Skip when it is a spot instance
+	if s.SpotPrice != "" {
+		ui.Say(fmt.Sprintf("This is a spot instance, no need to stop for the AMI"))
+		return multistep.ActionContinue
+	}
+
 	// Stop the instance so we can create an AMI from it
 	ui.Say("Stopping the source instance...")
 	_, err := ec2conn.StopInstances(instance.InstanceId)

builder/amazon/instance/builder.go

@@ -206,6 +206,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		},
 		&awscommon.StepRunSourceInstance{
 			Debug:                    b.config.PackerDebug,
+			SpotPrice:                b.config.SpotPrice,
 			InstanceType:             b.config.InstanceType,
 			IamInstanceProfile:       b.config.IamInstanceProfile,
 			UserData:                 b.config.UserData,

website/source/docs/builders/amazon-ebs.html.markdown

@@ -120,6 +120,13 @@ each category, the available configuration keys are alphabetized.
   described above. Note that if this is specified, you must omit the
   `security_group_id`.
 
+* `spot_price` (string) - The maximum hourly price to launch a spot instance
+  to create the AMI. It is a type of instances that EC2 starts when the maximum
+  price that you specify exceeds the current spot price. Spot price will be
+  updated based on available spot instance capacity and current spot Instance
+  requests. It may save you some costs. For example, it takes only "0.001" to
+  launch a spot "m3.medium" instance while "0.07" needed for on-demand.
+
 * `ssh_port` (integer) - The port that SSH will be available on. This defaults
   to port 22.
 

website/source/docs/builders/amazon-instance.html.markdown

@@ -158,6 +158,13 @@ each category, the available configuration keys are alphabetized.
   described above. Note that if this is specified, you must omit the
   `security_group_id`.
 
+* `spot_price` (string) - The maximum hourly price to launch a spot instance
+  to create the AMI. It is a type of instances that EC2 starts when the maximum
+  price that you specify exceeds the current spot price. Spot price will be
+  updated based on available spot instance capacity and current spot Instance
+  requests. It may save you some costs. For example, it takes only "0.001" to
+  launch a spot "m3.medium" instance while "0.07" needed for on-demand.
+
 * `ssh_port` (integer) - The port that SSH will be available on. This defaults
   to port 22.