Merge branch 'aws_credentials_chain' into 'master'

AWS credentials chain

Closes #2169

See merge request !2020

Gemfile.lock

@@ -67,13 +67,13 @@ GEM
       execjs
       json
     awesome_print (1.2.0)
-    aws-sdk (2.7.8)
-      aws-sdk-resources (= 2.7.8)
-    aws-sdk-core (2.7.8)
+    aws-sdk (2.9.32)
+      aws-sdk-resources (= 2.9.32)
+    aws-sdk-core (2.9.32)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.7.8)
-      aws-sdk-core (= 2.7.8)
+    aws-sdk-resources (2.9.32)
+      aws-sdk-core (= 2.9.32)
     aws-sigv4 (1.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
@@ -216,8 +216,8 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.11.0.1)
       faraday (>= 0.7.4, < 1.0)
-    faraday_middleware-aws-signers-v4 (0.1.5)
-      aws-sdk (~> 2.1)
+    faraday_middleware-aws-signers-v4 (0.1.7)
+      aws-sdk-resources (~> 2)
       faraday (~> 0.9)
     faraday_middleware-multi_json (0.0.6)
       faraday_middleware

changelogs/unreleased-ee/aws_credentials_chain.yml

+---
+title: Adding support for AWS ec2 instance profile credentials with elasticsearch
+merge_request:
+author: Matt Gresko

doc/integration/elasticsearch.md

@@ -58,7 +58,7 @@ The following Elasticsearch settings are available:
 | `Use experimental repository indexer` | Perform repository indexing using [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). |
 | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
 | `URL`                              | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). |
-| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization][aws-iam]. The access key must be allowed to perform `es:*` actions. |
+| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization][aws-iam] or [AWS EC2 Instance Profile Credentials][aws-instance-profile]. The policies must be configured to allow `es:*` actions. |
 | `AWS Region` | The AWS region your Elasticsearch service is located in. |
 | `AWS Access Key` | The AWS access key. |
 | `AWS Secret Access Key` | The AWS secret access key. |
@@ -325,6 +325,7 @@ Make sure you indexed all the database data as stated above (`sudo gitlab-rake g
 [ee-1305]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1305
 [aws-elasticsearch]: http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html
 [aws-iam]: http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
+[aws-instance-profile]: http://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli
 [ee-109]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/109 "Elasticsearch Merge Request"
 [elasticsearch]: https://www.elastic.co/products/elasticsearch "Elasticsearch website"
 [install]: https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html "Elasticsearch installation documentation"

lib/gitlab/elastic/client.rb

@@ -13,7 +13,7 @@ module Gitlab
         base_config = { urls: config[:url] }
 
         if config[:aws]
-          creds = Aws::Credentials.new(config[:aws_access_key], config[:aws_secret_access_key])
+          creds = resolve_aws_credentials(config)
           region = config[:aws_region]
 
           ::Elasticsearch::Client.new(base_config) do |fmid|
@@ -23,6 +23,19 @@ module Gitlab
           ::Elasticsearch::Client.new(base_config)
         end
       end
+
+      def self.resolve_aws_credentials(config)
+        # Resolve credentials in order
+        # 1.  Static config
+        # 2.  ec2 instance profile
+        credentials = [
+          Aws::Credentials.new(config[:aws_access_key], config[:aws_secret_access_key]),
+          Aws::InstanceProfileCredentials.new
+        ]
+        credentials.find do |creds|
+          creds&.set?
+        end
+      end
     end
   end
 end

spec/lib/gitlab/elastic/client_spec.rb

 require 'spec_helper'
 
 describe Gitlab::Elastic::Client do
+  let(:creds_valid_response) do
+    '{
+      "Code": "Success",
+      "Type": "AWS-HMAC",
+      "AccessKeyId": "0",
+      "SecretAccessKey": "0",
+      "Token": "token",
+      "Expiration": "2018-12-16T01:51:37Z",
+      "LastUpdated": "2009-11-23T0:00:00Z"
+    }'
+  end
+
+  let(:creds_fail_response) do
+    '{
+      "Code": "ErrorCode",
+      "Message": "ErrorMsg",
+      "LastUpdated": "2009-11-23T0:00:00Z"
+    }'
+  end
+
+  def stub_instance_credentials(creds_response)
+    stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/security-credentials/")
+      .to_return(status: 200, body: "RoleName", headers: {})
+    stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/security-credentials/RoleName")
+      .to_return(status: 200, body: creds_response, headers: {})
+  end
+
   describe 'build' do
     let(:client) { described_class.build(params) }
 
@@ -16,7 +43,7 @@ describe Gitlab::Elastic::Client do
       end
     end
 
-    context 'with AWS IAM credentials' do
+    context 'with AWS IAM static credentials' do
       let(:params) do
         {
           url: 'http://example-elastic:9200',
@@ -27,21 +54,78 @@ describe Gitlab::Elastic::Client do
         }
       end
 
-      it 'signs requests' do
+      it 'signs_requests' do
+        stub_instance_credentials(creds_fail_response)
         travel_to(Time.parse('20170303T133952Z')) do
           stub_request(:get, 'http://example-elastic:9200/foo/_all/1')
             .with(
               headers: {
-               'Authorization'        => 'AWS4-HMAC-SHA256 Credential=0/20170303/us-east-1/es/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=4ba2aae19a476152dacf5a2191da67b0cf81b9d7152dab5c42b1bba701da19f1',
-               'Content-Type'         => 'application/json',
-               'X-Amz-Content-Sha256' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
-               'X-Amz-Date'           => '20170303T133952Z'
-            })
-            .to_return(status: 200, body: [:fake_response])
+                'Authorization'        => 'AWS4-HMAC-SHA256 Credential=0/20170303/us-east-1/es/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=4ba2aae19a476152dacf5a2191da67b0cf81b9d7152dab5c42b1bba701da19f1',
+                'Content-Type'         => 'application/json',
+                'X-Amz-Content-Sha256' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+                'X-Amz-Date'           => '20170303T133952Z'
+              })
+              .to_return(status: 200, body: [:fake_response])
 
           expect(client.get(index: 'foo', id: 1)).to eq([:fake_response])
         end
       end
     end
   end
+
+  describe 'resolve_aws_credentials' do
+    let(:creds) { described_class.resolve_aws_credentials(params) }
+
+    context 'with AWS IAM static credentials' do
+      let(:params) do
+        {
+          url: 'http://example-elastic:9200',
+          aws: true,
+          aws_region: 'us-east-1',
+          aws_access_key: '0',
+          aws_secret_access_key: '0'
+        }
+      end
+
+      it 'returns credentials from static credentials' do
+        stub_instance_credentials(creds_fail_response)
+
+        expect(creds.credentials.access_key_id).to eq '0'
+        expect(creds.credentials.secret_access_key).to eq '0'
+      end
+    end
+
+    context 'with AWS ec2 instance profile' do
+      let(:params) do
+        {
+          url: 'http://example-elastic:9200',
+          aws: true,
+          aws_region: 'us-east-1'
+        }
+      end
+
+      it 'returns credentials from ec2 instance profile' do
+        stub_instance_credentials(creds_valid_response)
+
+        expect(creds.credentials.access_key_id).to eq '0'
+        expect(creds.credentials.secret_access_key).to eq '0'
+      end
+    end
+
+    context 'with AWS no credentials' do
+      let(:params) do
+        {
+          url: 'http://example-elastic:9200',
+          aws: true,
+          aws_region: 'us-east-1'
+        }
+      end
+
+      it 'returns nil' do
+        stub_instance_credentials(creds_fail_response)
+
+        expect(creds).to be_nil
+      end
+    end
+  end
 end