Services for creating an EKS cluster via GitLab

There are several steps to this process:

* GitLab assumes the role provided by the user and stores
  a set of temporary credentials on the provider record. By default
  these credentials are valid for one hour.

* A CloudFormation stack is created, based on the template in
  vendor/aws/cloudformation/eks_cluster.yaml. This triggers creation
  of all resources required for an EKS cluster.

* GitLab polls the status of the stack until all resources are ready,
  which takes somewhere between 10 and 15 minutes in most cases.

* When the cluster is ready, GitLab stores the cluster details and
  fetches another set of temporary credentials, this time to allow
  connecting to the cluster via Kubeclient. These credentials
  are valid for one minute.

* GitLab configures the worker nodes so that they are able to
  authenticate to the cluster, and creates a service account for
  itself for future operations.

* Finally, all details and credentials that are no longer required
  are removed.

app/models/clusters/providers/aws.rb

@@ -3,6 +3,7 @@
 module Clusters
   module Providers
     class Aws < ApplicationRecord
+      include Gitlab::Utils::StrongMemoize
       include Clusters::Concerns::ProviderStatus
 
       self.table_name = 'cluster_providers_aws'
@@ -42,6 +43,18 @@ module Clusters
           session_token: nil
         )
       end
+
+      def api_client
+        strong_memoize(:api_client) do
+          ::Aws::CloudFormation::Client.new(credentials: credentials, region: region)
+        end
+      end
+
+      def credentials
+        strong_memoize(:credentials) do
+          ::Aws::Credentials.new(access_key_id, secret_access_key, session_token)
+        end
+      end
     end
   end
 end

app/services/clusters/aws/fetch_credentials_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class FetchCredentialsService
+      attr_reader :provider
+
+      MissingRoleError = Class.new(StandardError)
+
+      def initialize(provider)
+        @provider = provider
+      end
+
+      def execute
+        raise MissingRoleError.new('AWS provisioning role not configured') unless provision_role.present?
+
+        ::Aws::AssumeRoleCredentials.new(
+          client: client,
+          role_arn: provision_role.role_arn,
+          role_session_name: session_name,
+          external_id: provision_role.role_external_id
+        ).credentials
+      end
+
+      private
+
+      def provision_role
+        provider.created_by_user.aws_role
+      end
+
+      def client
+        ::Aws::STS::Client.new(credentials: gitlab_credentials, region: provider.region)
+      end
+
+      def gitlab_credentials
+        ::Aws::Credentials.new(access_key_id, secret_access_key)
+      end
+
+      ##
+      # This setting is not yet configurable or documented as these
+      # services are not currently used. This will be addressed in
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
+      def access_key_id
+        Gitlab.config.kubernetes.provisioners.aws.access_key_id
+      end
+
+      ##
+      # This setting is not yet configurable or documented as these
+      # services are not currently used. This will be addressed in
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
+      def secret_access_key
+        Gitlab.config.kubernetes.provisioners.aws.secret_access_key
+      end
+
+      def session_name
+        "gitlab-eks-cluster-#{provider.cluster_id}-user-#{provider.created_by_user_id}"
+      end
+    end
+  end
+end

app/services/clusters/aws/finalize_creation_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class FinalizeCreationService
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :provider
+
+      delegate :cluster, to: :provider
+
+      def execute(provider)
+        @provider = provider
+
+        configure_provider
+        create_gitlab_service_account!
+        configure_platform_kubernetes
+        configure_node_authentication!
+
+        cluster.save!
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to fetch CloudFormation stack: %{message}') % { message: e.message })
+      rescue Kubeclient::HttpError => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to run Kubeclient: %{message}') % { message: e.message })
+      rescue ActiveRecord::RecordInvalid => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to configure EKS provider: %{message}') % { message: e.message })
+      end
+
+      private
+
+      def create_gitlab_service_account!
+        Clusters::Kubernetes::CreateOrUpdateServiceAccountService.gitlab_creator(
+          kube_client,
+          rbac: true
+        ).execute
+      end
+
+      def configure_provider
+        provider.status_event = :make_created
+      end
+
+      def configure_platform_kubernetes
+        cluster.build_platform_kubernetes(
+          api_url: cluster_endpoint,
+          ca_cert: cluster_certificate,
+          token: request_kubernetes_token)
+      end
+
+      def request_kubernetes_token
+        Clusters::Kubernetes::FetchKubernetesTokenService.new(
+          kube_client,
+          Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME,
+          Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE
+        ).execute
+      end
+
+      def kube_client
+        @kube_client ||= build_kube_client!(
+          cluster_endpoint,
+          cluster_certificate
+        )
+      end
+
+      def build_kube_client!(api_url, ca_pem)
+        raise "Incomplete settings" unless api_url
+
+        Gitlab::Kubernetes::KubeClient.new(
+          api_url,
+          auth_options: kubeclient_auth_options,
+          ssl_options: kubeclient_ssl_options(ca_pem),
+          http_proxy_uri: ENV['http_proxy']
+        )
+      end
+
+      def kubeclient_auth_options
+        { bearer_token: Kubeclient::AmazonEksCredentials.token(provider.credentials, cluster.name) }
+      end
+
+      def kubeclient_ssl_options(ca_pem)
+        opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
+
+        if ca_pem.present?
+          opts[:cert_store] = OpenSSL::X509::Store.new
+          opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
+        end
+
+        opts
+      end
+
+      def cluster_stack
+        @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
+      end
+
+      def stack_output_value(key)
+        cluster_stack.outputs.detect { |output| output.output_key == key }.output_value
+      end
+
+      def node_instance_role_arn
+        stack_output_value('NodeInstanceRole')
+      end
+
+      def cluster_endpoint
+        strong_memoize(:cluster_endpoint) do
+          stack_output_value('ClusterEndpoint')
+        end
+      end
+
+      def cluster_certificate
+        strong_memoize(:cluster_certificate) do
+          Base64.decode64(stack_output_value('ClusterCertificate'))
+        end
+      end
+
+      def configure_node_authentication!
+        kube_client.create_config_map(node_authentication_config)
+      end
+
+      def node_authentication_config
+        Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth.new(node_instance_role_arn).generate
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
+      end
+
+      def log_service_error(exception, provider_id, message)
+        logger.error(
+          exception: exception.class.name,
+          service: self.class.name,
+          provider_id: provider_id,
+          message: message
+        )
+      end
+    end
+  end
+end

app/services/clusters/aws/provision_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class ProvisionService
+      attr_reader :provider
+
+      def execute(provider)
+        @provider = provider
+
+        configure_provider_credentials
+        provision_cluster
+
+        if provider.make_creating
+          WaitForClusterCreationWorker.perform_in(
+            Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL,
+            provider.cluster_id
+          )
+        else
+          provider.make_errored!("Failed to update provider record; #{provider.errors.full_messages}")
+        end
+      rescue Clusters::Aws::FetchCredentialsService::MissingRoleError
+        provider.make_errored!('Amazon role is not configured')
+      rescue ::Aws::Errors::MissingCredentialsError, Settingslogic::MissingSetting
+        provider.make_errored!('Amazon credentials are not configured')
+      rescue ::Aws::STS::Errors::ServiceError => e
+        provider.make_errored!("Amazon authentication failed; #{e.message}")
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
+      end
+
+      private
+
+      def credentials
+        @credentials ||= Clusters::Aws::FetchCredentialsService.new(provider).execute
+      end
+
+      def configure_provider_credentials
+        provider.update!(
+          access_key_id: credentials.access_key_id,
+          secret_access_key: credentials.secret_access_key,
+          session_token: credentials.session_token
+        )
+      end
+
+      def provision_cluster
+        provider.api_client.create_stack(
+          stack_name: provider.cluster.name,
+          template_body: stack_template,
+          parameters: parameters,
+          capabilities: ["CAPABILITY_IAM"]
+        )
+      end
+
+      def parameters
+        [
+          parameter('ClusterName', provider.cluster.name),
+          parameter('ClusterRole', provider.role_arn),
+          parameter('ClusterControlPlaneSecurityGroup', provider.security_group_id),
+          parameter('VpcId', provider.vpc_id),
+          parameter('Subnets', provider.subnet_ids.join(',')),
+          parameter('NodeAutoScalingGroupDesiredCapacity', provider.num_nodes.to_s),
+          parameter('NodeInstanceType', provider.instance_type),
+          parameter('KeyName', provider.key_name)
+        ]
+      end
+
+      def parameter(key, value)
+        { parameter_key: key, parameter_value: value }
+      end
+
+      def stack_template
+        File.read(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml'))
+      end
+    end
+  end
+end

app/services/clusters/aws/verify_provision_status_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class VerifyProvisionStatusService
+      attr_reader :provider
+
+      INITIAL_INTERVAL = 5.minutes
+      POLL_INTERVAL = 1.minute
+      TIMEOUT = 30.minutes
+
+      def execute(provider)
+        @provider = provider
+
+        case cluster_stack.stack_status
+        when 'CREATE_IN_PROGRESS'
+          continue_creation
+        when 'CREATE_COMPLETE'
+          finalize_creation
+        else
+          provider.make_errored!("Unexpected status; #{cluster_stack.stack_status}")
+        end
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
+      end
+
+      private
+
+      def cluster_stack
+        @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
+      end
+
+      def continue_creation
+        if timeout_threshold.future?
+          WaitForClusterCreationWorker.perform_in(POLL_INTERVAL, provider.cluster_id)
+        else
+          provider.make_errored!(_('Kubernetes cluster creation time exceeds timeout; %{timeout}') % { timeout: TIMEOUT })
+        end
+      end
+
+      def timeout_threshold
+        cluster_stack.creation_time + TIMEOUT
+      end
+
+      def finalize_creation
+        Clusters::Aws::FinalizeCreationService.new.execute(provider)
+      end
+    end
+  end
+end

app/workers/cluster_provision_worker.rb

@@ -9,7 +9,11 @@ class ClusterProvisionWorker
   def perform(cluster_id)
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       cluster.provider.try do |provider|
-        Clusters::Gcp::ProvisionService.new.execute(provider) if cluster.gcp?
+        if cluster.gcp?
+          Clusters::Gcp::ProvisionService.new.execute(provider)
+        elsif cluster.aws?
+          Clusters::Aws::ProvisionService.new.execute(provider)
+        end
       end
     end
   end

app/workers/wait_for_cluster_creation_worker.rb

@@ -9,7 +9,11 @@ class WaitForClusterCreationWorker
   def perform(cluster_id)
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       cluster.provider.try do |provider|
-        Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider) if cluster.gcp?
+        if cluster.gcp?
+          Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider)
+        elsif cluster.aws?
+          Clusters::Aws::VerifyProvisionStatusService.new.execute(provider)
+        end
       end
     end
   end

lib/gitlab/kubernetes/config_maps/aws_node_auth.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module ConfigMaps
+      class AwsNodeAuth
+        attr_reader :node_role
+
+        def initialize(node_role)
+          @node_role = node_role
+        end
+
+        def generate
+          Kubeclient::Resource.new(
+            metadata: metadata,
+            data: data
+          )
+        end
+
+        private
+
+        def metadata
+          {
+            'name' => 'aws-auth',
+            'namespace' => 'kube-system'
+          }
+        end
+
+        def data
+          { 'mapRoles' => instance_role_config(node_role) }
+        end
+
+        def instance_role_config(role)
+          [{
+            'rolearn' => role,
+            'username' => 'system:node:{{EC2PrivateDNSName}}',
+            'groups' => [
+              'system:bootstrappers',
+              'system:nodes'
+            ]
+          }].to_yaml
+        end
+      end
+    end
+  end
+end

locale/gitlab.pot

@@ -3585,9 +3585,15 @@ msgstr ""
 msgid "ClusterIntegration|Every new Google Cloud Platform (GCP) account receives $300 in credit upon %{sign_up_link}. In partnership with Google, GitLab is able to offer an additional $200 for both new and existing GCP accounts to get started with GitLab's Google Kubernetes Engine Integration."
 msgstr ""
 
+msgid "ClusterIntegration|Failed to configure EKS provider: %{message}"
+msgstr ""
+
 msgid "ClusterIntegration|Failed to configure Google Kubernetes Engine Cluster: %{message}"
 msgstr ""
 
+msgid "ClusterIntegration|Failed to fetch CloudFormation stack: %{message}"
+msgstr ""
+
 msgid "ClusterIntegration|Failed to request to Google Cloud Platform: %{message}"
 msgstr ""
 

spec/lib/gitlab/kubernetes/config_maps/aws_node_auth_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth do
+  describe '#generate' do
+    let(:role) { 'arn:aws:iam::123456789012:role/node-instance-role' }
+
+    let(:name) { 'aws-auth' }
+    let(:namespace) { 'kube-system' }
+    let(:role_config) do
+      [{
+        'rolearn' => role,
+        'username' => 'system:node:{{EC2PrivateDNSName}}',
+        'groups' => [
+          'system:bootstrappers',
+          'system:nodes'
+        ]
+      }]
+    end
+
+    subject { described_class.new(role).generate }
+
+    it 'builds a Kubeclient Resource' do
+      expect(subject).to be_a(Kubeclient::Resource)
+
+      expect(subject.metadata.name).to eq(name)
+      expect(subject.metadata.namespace).to eq(namespace)
+
+      expect(YAML.safe_load(subject.data.mapRoles)).to eq(role_config)
+    end
+  end
+end

spec/models/clusters/providers/aws_spec.rb

@@ -64,13 +64,48 @@ describe Clusters::Providers::Aws do
     before do
       expect(provider.access_key_id).to be_present
       expect(provider.secret_access_key).to be_present
+      expect(provider.session_token).to be_present
     end
 
-    it 'removes access_key_id and secret_access_key' do
+    it 'removes access_key_id, secret_access_key and session_token' do
       subject
 
       expect(provider.access_key_id).to be_nil
       expect(provider.secret_access_key).to be_nil
+      expect(provider.session_token).to be_nil
     end
   end
+
+  describe '#api_client' do
+    let(:provider) { create(:cluster_provider_aws) }
+    let(:credentials) { double }
+    let(:client) { double }
+
+    subject { provider.api_client }
+
+    before do
+      allow(provider).to receive(:credentials).and_return(credentials)
+
+      expect(Aws::CloudFormation::Client).to receive(:new)
+        .with(credentials: credentials, region: provider.region)
+        .and_return(client)
+    end
+
+    it { is_expected.to eq client }
+  end
+
+  describe '#credentials' do
+    let(:provider) { create(:cluster_provider_aws) }
+    let(:credentials) { double }
+
+    subject { provider.credentials }
+
+    before do
+      expect(Aws::Credentials).to receive(:new)
+        .with(provider.access_key_id, provider.secret_access_key, provider.session_token)
+        .and_return(credentials)
+    end
+
+    it { is_expected.to eq credentials }
+  end
 end

spec/services/clusters/aws/fetch_credentials_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::FetchCredentialsService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws) }
+
+    let(:gitlab_access_key_id) { 'gitlab-access-key-id' }
+    let(:gitlab_secret_access_key) { 'gitlab-secret-access-key' }
+
+    let(:gitlab_credentials) { Aws::Credentials.new(gitlab_access_key_id, gitlab_secret_access_key) }
+    let(:sts_client) { Aws::STS::Client.new(credentials: gitlab_credentials, region: provider.region) }
+    let(:assumed_role) { instance_double(Aws::AssumeRoleCredentials, credentials: assumed_role_credentials) }
+
+    let(:kubernetes_provisioner_settings) do
+      {
+        aws: {
+          access_key_id: gitlab_access_key_id,
+          secret_access_key: gitlab_secret_access_key
+        }
+      }
+    end
+
+    let(:assumed_role_credentials) { double }
+
+    subject { described_class.new(provider).execute }
+
+    context 'provision role is configured' do
+      let(:provision_role) { create(:aws_role, user: provider.created_by_user) }
+
+      before do
+        stub_config(kubernetes: { provisioners: kubernetes_provisioner_settings })
+
+        expect(Aws::Credentials).to receive(:new)
+          .with(gitlab_access_key_id, gitlab_secret_access_key)
+          .and_return(gitlab_credentials)
+
+        expect(Aws::STS::Client).to receive(:new)
+          .with(credentials: gitlab_credentials, region: provider.region)
+          .and_return(sts_client)
+
+        expect(Aws::AssumeRoleCredentials).to receive(:new)
+          .with(
+            client: sts_client,
+            role_arn: provision_role.role_arn,
+            role_session_name: "gitlab-eks-cluster-#{provider.cluster_id}-user-#{provider.created_by_user_id}",
+            external_id: provision_role.role_external_id
+          ).and_return(assumed_role)
+      end
+
+      it { is_expected.to eq assumed_role_credentials }
+    end
+
+    context 'provision role is not configured' do
+      before do
+        expect(provider.created_by_user.aws_role).to be_nil
+      end
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(described_class::MissingRoleError, 'AWS provisioning role not configured')
+      end
+    end
+  end
+end

spec/services/clusters/aws/finalize_creation_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::FinalizeCreationService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws, :creating) }
+    let(:platform) { provider.cluster.platform_kubernetes }
+
+    let(:create_service_account_service) { double(execute: true) }
+    let(:fetch_token_service) { double(execute: gitlab_token) }
+    let(:kube_client) { double(create_config_map: true) }
+    let(:cluster_stack) { double(outputs: [endpoint_output, cert_output, node_role_output]) }
+    let(:node_auth_config_map) { double }
+
+    let(:endpoint_output) { double(output_key: 'ClusterEndpoint', output_value: api_url) }
+    let(:cert_output) { double(output_key: 'ClusterCertificate', output_value: Base64.encode64(ca_pem)) }
+    let(:node_role_output) { double(output_key: 'NodeInstanceRole', output_value: node_role) }
+
+    let(:api_url) { 'https://kubernetes.example.com' }
+    let(:ca_pem) { File.read(Rails.root.join('spec/fixtures/clusters/sample_cert.pem')) }
+    let(:gitlab_token) { 'gitlab-token' }
+    let(:iam_token) { 'iam-token' }
+    let(:node_role) { 'arn::aws::iam::123456789012:role/node-role' }
+
+    subject { described_class.new.execute(provider) }
+
+    before do
+      allow(Clusters::Kubernetes::CreateOrUpdateServiceAccountService).to receive(:gitlab_creator)
+        .with(kube_client, rbac: true)
+        .and_return(create_service_account_service)
+
+      allow(Clusters::Kubernetes::FetchKubernetesTokenService).to receive(:new)
+        .with(
+          kube_client,
+          Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME,
+          Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE)
+        .and_return(fetch_token_service)
+
+      allow(Gitlab::Kubernetes::KubeClient).to receive(:new)
+        .with(
+          api_url,
+          auth_options: { bearer_token: iam_token },
+          ssl_options: {
+            verify_ssl: OpenSSL::SSL::VERIFY_PEER,
+            cert_store: instance_of(OpenSSL::X509::Store)
+          },
+          http_proxy_uri: nil
+        )
+        .and_return(kube_client)
+
+      allow(provider.api_client).to receive(:describe_stacks)
+        .with(stack_name: provider.cluster.name)
+        .and_return(double(stacks: [cluster_stack]))
+
+      allow(Kubeclient::AmazonEksCredentials).to receive(:token)
+        .with(provider.credentials, provider.cluster.name)
+        .and_return(iam_token)
+
+      allow(Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth).to receive(:new)
+        .with(node_role).and_return(double(generate: node_auth_config_map))
+    end
+
+    it 'configures the provider and platform' do
+      subject
+
+      expect(provider).to be_created
+      expect(platform.api_url).to eq(api_url)
+      expect(platform.ca_pem).to eq(ca_pem)
+      expect(platform.token).to eq(gitlab_token)
+      expect(platform).to be_rbac
+    end
+
+    it 'calls the create_service_account_service' do
+      expect(create_service_account_service).to receive(:execute).once
+
+      subject
+    end
+
+    it 'configures cluster node authentication' do
+      expect(kube_client).to receive(:create_config_map).with(node_auth_config_map).once
+
+      subject
+    end
+
+    describe 'error handling' do
+      shared_examples 'provision error' do |message|
+        it "sets the status to :errored with an appropriate error message" do
+          subject
+
+          expect(provider).to be_errored
+          expect(provider.status_reason).to include(message)
+        end
+      end
+
+      context 'failed to request stack details from AWS' do
+        before do
+          allow(provider.api_client).to receive(:describe_stacks)
+            .and_raise(Aws::CloudFormation::Errors::ServiceError.new(double, "Error message"))
+        end
+
+        include_examples 'provision error', 'Failed to fetch CloudFormation stack'
+      end
+
+      context 'failed to create auth config map' do
+        before do
+          allow(kube_client).to receive(:create_config_map)
+            .and_raise(Kubeclient::HttpError.new(500, 'Error', nil))
+        end
+
+        include_examples 'provision error', 'Failed to run Kubeclient'
+      end
+
+      context 'failed to save records' do
+        before do
+          allow(provider.cluster).to receive(:save!)
+            .and_raise(ActiveRecord::RecordInvalid)
+        end
+
+        include_examples 'provision error', 'Failed to configure EKS provider'
+      end
+    end
+  end
+end

spec/services/clusters/aws/provision_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::ProvisionService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws) }
+
+    let(:client) { instance_double(Aws::CloudFormation::Client, create_stack: true) }
+    let(:cloudformation_template) { double }
+    let(:credentials) do
+      instance_double(
+        Aws::Credentials,
+        access_key_id: 'key',
+        secret_access_key: 'secret',
+        session_token: 'token'
+      )
+    end
+
+    let(:parameters) do
+      [
+        { parameter_key: 'ClusterName', parameter_value: provider.cluster.name },
+        { parameter_key: 'ClusterRole', parameter_value: provider.role_arn },
+        { parameter_key: 'ClusterControlPlaneSecurityGroup', parameter_value: provider.security_group_id },
+        { parameter_key: 'VpcId', parameter_value: provider.vpc_id },
+        { parameter_key: 'Subnets', parameter_value: provider.subnet_ids.join(',') },
+        { parameter_key: 'NodeAutoScalingGroupDesiredCapacity', parameter_value: provider.num_nodes.to_s },
+        { parameter_key: 'NodeInstanceType', parameter_value: provider.instance_type },
+        { parameter_key: 'KeyName', parameter_value: provider.key_name }
+      ]
+    end
+
+    subject { described_class.new.execute(provider) }
+
+    before do
+      allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+        .with(provider).and_return(double(execute: credentials))
+
+      allow(provider).to receive(:api_client)
+        .and_return(client)
+
+      allow(File).to receive(:read)
+        .with(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml'))
+        .and_return(cloudformation_template)
+    end
+
+    it 'updates the provider status to :creating and configures the provider with credentials' do
+      subject
+
+      expect(provider).to be_creating
+      expect(provider.access_key_id).to eq 'key'
+      expect(provider.secret_access_key).to eq 'secret'
+      expect(provider.session_token).to eq 'token'
+    end
+
+    it 'creates a CloudFormation stack' do
+      expect(client).to receive(:create_stack).with(
+        stack_name: provider.cluster.name,
+        template_body: cloudformation_template,
+        parameters: parameters,
+        capabilities: ["CAPABILITY_IAM"]
+      )
+
+      subject
+    end
+
+    it 'schedules a worker to monitor creation status' do
+      expect(WaitForClusterCreationWorker).to receive(:perform_in)
+        .with(Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL, provider.cluster_id)
+
+      subject
+    end
+
+    describe 'error handling' do
+      shared_examples 'provision error' do |message|
+        it "sets the status to :errored with an appropriate error message" do
+          subject
+
+          expect(provider).to be_errored
+          expect(provider.status_reason).to include(message)
+        end
+      end
+
+      context 'invalid state transition' do
+        before do
+          allow(provider).to receive(:make_creating).and_return(false)
+        end
+
+        include_examples 'provision error', 'Failed to update provider record'
+      end
+
+      context 'AWS role is not configured' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Clusters::Aws::FetchCredentialsService::MissingRoleError)
+        end
+
+        include_examples 'provision error', 'Amazon role is not configured'
+      end
+
+      context 'AWS credentials are not configured' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Aws::Errors::MissingCredentialsError)
+        end
+
+        include_examples 'provision error', 'Amazon credentials are not configured'
+      end
+
+      context 'AWS credentials are not configured' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Settingslogic::MissingSetting)
+        end
+
+        include_examples 'provision error', 'Amazon credentials are not configured'
+      end
+
+      context 'Authentication failure' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Aws::STS::Errors::ServiceError.new(double, 'Error message'))
+        end
+
+        include_examples 'provision error', 'Amazon authentication failed'
+      end
+
+      context 'CloudFormation failure' do
+        before do
+          allow(client).to receive(:create_stack)
+            .and_raise(Aws::CloudFormation::Errors::ServiceError.new(double, 'Error message'))
+        end
+
+        include_examples 'provision error', 'Amazon CloudFormation request failed'
+      end
+    end
+  end
+end

spec/services/clusters/aws/verify_provision_status_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::VerifyProvisionStatusService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws) }
+
+    let(:stack) { double(stack_status: stack_status, creation_time: creation_time) }
+    let(:creation_time) { 1.minute.ago }
+
+    subject { described_class.new.execute(provider) }
+
+    before do
+      allow(provider.api_client).to receive(:describe_stacks)
+        .with(stack_name: provider.cluster.name)
+        .and_return(double(stacks: [stack]))
+    end
+
+    shared_examples 'provision error' do |message|
+      it "sets the status to :errored with an appropriate error message" do
+        subject
+
+        expect(provider).to be_errored
+        expect(provider.status_reason).to include(message)
+      end
+    end
+
+    context 'stack creation is still in progress' do
+      let(:stack_status) { 'CREATE_IN_PROGRESS' }
+      let(:verify_service) { double(execute: true) }
+
+      it 'schedules a worker to check again later' do
+        expect(WaitForClusterCreationWorker).to receive(:perform_in)
+          .with(described_class::POLL_INTERVAL, provider.cluster_id)
+
+        subject
+      end
+
+      context 'stack creation is taking too long' do
+        let(:creation_time) { 1.hour.ago }
+
+        include_examples 'provision error', 'Kubernetes cluster creation time exceeds timeout'
+      end
+    end
+
+    context 'stack creation is complete' do
+      let(:stack_status) { 'CREATE_COMPLETE' }
+      let(:finalize_service) { double(execute: true) }
+
+      it 'finalizes creation' do
+        expect(Clusters::Aws::FinalizeCreationService).to receive(:new).and_return(finalize_service)
+        expect(finalize_service).to receive(:execute).with(provider).once
+
+        subject
+      end
+    end
+
+    context 'stack creation failed' do
+      let(:stack_status) { 'CREATE_FAILED' }
+
+      include_examples 'provision error', 'Unexpected status'
+    end
+
+    context 'error communicating with CloudFormation API' do
+      let(:stack_status) { 'CREATE_IN_PROGRESS' }
+
+      before do
+        allow(provider.api_client).to receive(:describe_stacks)
+          .and_raise(Aws::CloudFormation::Errors::ServiceError.new(double, 'Error message'))
+      end
+
+      include_examples 'provision error', 'Amazon CloudFormation request failed'
+    end
+  end
+end

spec/workers/cluster_provision_worker_spec.rb

@@ -9,7 +9,18 @@ describe ClusterProvisionWorker do
       let(:provider) { create(:cluster_provider_gcp, :scheduled) }
 
       it 'provision a cluster' do
-        expect_any_instance_of(Clusters::Gcp::ProvisionService).to receive(:execute)
+        expect_any_instance_of(Clusters::Gcp::ProvisionService).to receive(:execute).with(provider)
+
+        described_class.new.perform(cluster.id)
+      end
+    end
+
+    context 'when provider type is aws' do
+      let(:cluster) { create(:cluster, provider_type: :aws, provider_aws: provider) }
+      let(:provider) { create(:cluster_provider_aws, :scheduled) }
+
+      it 'provision a cluster' do
+        expect_any_instance_of(Clusters::Aws::ProvisionService).to receive(:execute).with(provider)
 
         described_class.new.perform(cluster.id)
       end

spec/workers/wait_for_cluster_creation_worker_spec.rb

@@ -8,8 +8,19 @@ describe WaitForClusterCreationWorker do
       let(:cluster) { create(:cluster, provider_type: :gcp, provider_gcp: provider) }
       let(:provider) { create(:cluster_provider_gcp, :creating) }
 
-      it 'provision a cluster' do
-        expect_any_instance_of(Clusters::Gcp::VerifyProvisionStatusService).to receive(:execute)
+      it 'provisions a cluster' do
+        expect_any_instance_of(Clusters::Gcp::VerifyProvisionStatusService).to receive(:execute).with(provider)
+
+        described_class.new.perform(cluster.id)
+      end
+    end
+
+    context 'when provider type is aws' do
+      let(:cluster) { create(:cluster, provider_type: :aws, provider_aws: provider) }
+      let(:provider) { create(:cluster_provider_aws, :creating) }
+
+      it 'provisions a cluster' do
+        expect_any_instance_of(Clusters::Aws::VerifyProvisionStatusService).to receive(:execute).with(provider)
 
         described_class.new.perform(cluster.id)
       end