From a9ff3e3a9bdcba91b6bbac651a093f95d43b4d4e Mon Sep 17 00:00:00 2001
From: Matt Gresko <mgresko@veracode.com>
Date: Wed, 31 May 2017 17:23:18 -0400
Subject: [PATCH] AWS credentials chain
---
Gemfile.lock | 14 +--
.../unreleased-ee/aws_credentials_chain.yml | 4 +
doc/integration/elasticsearch.md | 3 +-
lib/gitlab/elastic/client.rb | 15 ++-
spec/lib/gitlab/elastic/client_spec.rb | 100 ++++++++++++++++--
5 files changed, 119 insertions(+), 17 deletions(-)
create mode 100644 changelogs/unreleased-ee/aws_credentials_chain.yml
diff --git a/Gemfile.lock b/Gemfile.lock
index 0e94e279a6c4c..ea427a585912b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -67,13 +67,13 @@ GEM
execjs
json
awesome_print (1.2.0)
- aws-sdk (2.7.8)
- aws-sdk-resources (= 2.7.8)
- aws-sdk-core (2.7.8)
+ aws-sdk (2.9.32)
+ aws-sdk-resources (= 2.9.32)
+ aws-sdk-core (2.9.32)
aws-sigv4 (~> 1.0)
jmespath (~> 1.0)
- aws-sdk-resources (2.7.8)
- aws-sdk-core (= 2.7.8)
+ aws-sdk-resources (2.9.32)
+ aws-sdk-core (= 2.9.32)
aws-sigv4 (1.0.0)
axiom-types (0.1.1)
descendants_tracker (~> 0.0.4)
@@ -216,8 +216,8 @@ GEM
multipart-post (>= 1.2, < 3)
faraday_middleware (0.11.0.1)
faraday (>= 0.7.4, < 1.0)
- faraday_middleware-aws-signers-v4 (0.1.5)
- aws-sdk (~> 2.1)
+ faraday_middleware-aws-signers-v4 (0.1.7)
+ aws-sdk-resources (~> 2)
faraday (~> 0.9)
faraday_middleware-multi_json (0.0.6)
faraday_middleware
diff --git a/changelogs/unreleased-ee/aws_credentials_chain.yml b/changelogs/unreleased-ee/aws_credentials_chain.yml
new file mode 100644
index 0000000000000..f8181c7fe9a10
--- /dev/null
+++ b/changelogs/unreleased-ee/aws_credentials_chain.yml
@@ -0,0 +1,4 @@
+---
+title: Adding support for AWS ec2 instance profile credentials with elasticsearch
+merge_request:
+author: Matt Gresko
diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md
index e381f3f18260f..b7688f30adff7 100644
--- a/doc/integration/elasticsearch.md
+++ b/doc/integration/elasticsearch.md
@@ -58,7 +58,7 @@ The following Elasticsearch settings are available:
| `Use experimental repository indexer` | Perform repository indexing using [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). |
| `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
| `URL` | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). |
-| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization][aws-iam]. The access key must be allowed to perform `es:*` actions. |
+| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization][aws-iam] or [AWS EC2 Instance Profile Credentials][aws-instance-profile]. The policies must be configured to allow `es:*` actions. |
| `AWS Region` | The AWS region your Elasticsearch service is located in. |
| `AWS Access Key` | The AWS access key. |
| `AWS Secret Access Key` | The AWS secret access key. |
@@ -325,6 +325,7 @@ Make sure you indexed all the database data as stated above (`sudo gitlab-rake g
[ee-1305]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1305
[aws-elasticsearch]: http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html
[aws-iam]: http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
+[aws-instance-profile]: http://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli
[ee-109]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/109 "Elasticsearch Merge Request"
[elasticsearch]: https://www.elastic.co/products/elasticsearch "Elasticsearch website"
[install]: https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html "Elasticsearch installation documentation"
diff --git a/lib/gitlab/elastic/client.rb b/lib/gitlab/elastic/client.rb
index f51672361d918..d3eb2735c78b9 100644
--- a/lib/gitlab/elastic/client.rb
+++ b/lib/gitlab/elastic/client.rb
@@ -13,7 +13,7 @@ def self.build(config)
base_config = { urls: config[:url] }
if config[:aws]
- creds = Aws::Credentials.new(config[:aws_access_key], config[:aws_secret_access_key])
+ creds = resolve_aws_credentials(config)
region = config[:aws_region]
::Elasticsearch::Client.new(base_config) do |fmid|
@@ -23,6 +23,19 @@ def self.build(config)
::Elasticsearch::Client.new(base_config)
end
end
+
+ def self.resolve_aws_credentials(config)
+ # Resolve credentials in order
+ # 1. Static config
+ # 2. ec2 instance profile
+ credentials = [
+ Aws::Credentials.new(config[:aws_access_key], config[:aws_secret_access_key]),
+ Aws::InstanceProfileCredentials.new
+ ]
+ credentials.find do |creds|
+ creds&.set?
+ end
+ end
end
end
end
diff --git a/spec/lib/gitlab/elastic/client_spec.rb b/spec/lib/gitlab/elastic/client_spec.rb
index 6b3be78661851..dd609d0a720d9 100644
--- a/spec/lib/gitlab/elastic/client_spec.rb
+++ b/spec/lib/gitlab/elastic/client_spec.rb
@@ -1,6 +1,33 @@
require 'spec_helper'
describe Gitlab::Elastic::Client do
+ let(:creds_valid_response) do
+ '{
+ "Code": "Success",
+ "Type": "AWS-HMAC",
+ "AccessKeyId": "0",
+ "SecretAccessKey": "0",
+ "Token": "token",
+ "Expiration": "2018-12-16T01:51:37Z",
+ "LastUpdated": "2009-11-23T0:00:00Z"
+ }'
+ end
+
+ let(:creds_fail_response) do
+ '{
+ "Code": "ErrorCode",
+ "Message": "ErrorMsg",
+ "LastUpdated": "2009-11-23T0:00:00Z"
+ }'
+ end
+
+ def stub_instance_credentials(creds_response)
+ stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/security-credentials/")
+ .to_return(status: 200, body: "RoleName", headers: {})
+ stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/security-credentials/RoleName")
+ .to_return(status: 200, body: creds_response, headers: {})
+ end
+
describe 'build' do
let(:client) { described_class.build(params) }
@@ -16,7 +43,7 @@
end
end
- context 'with AWS IAM credentials' do
+ context 'with AWS IAM static credentials' do
let(:params) do
{
url: 'http://example-elastic:9200',
@@ -27,21 +54,78 @@
}
end
- it 'signs requests' do
+ it 'signs_requests' do
+ stub_instance_credentials(creds_fail_response)
travel_to(Time.parse('20170303T133952Z')) do
stub_request(:get, 'http://example-elastic:9200/foo/_all/1')
.with(
headers: {
- 'Authorization' => 'AWS4-HMAC-SHA256 Credential=0/20170303/us-east-1/es/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=4ba2aae19a476152dacf5a2191da67b0cf81b9d7152dab5c42b1bba701da19f1',
- 'Content-Type' => 'application/json',
- 'X-Amz-Content-Sha256' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
- 'X-Amz-Date' => '20170303T133952Z'
- })
- .to_return(status: 200, body: [:fake_response])
+ 'Authorization' => 'AWS4-HMAC-SHA256 Credential=0/20170303/us-east-1/es/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=4ba2aae19a476152dacf5a2191da67b0cf81b9d7152dab5c42b1bba701da19f1',
+ 'Content-Type' => 'application/json',
+ 'X-Amz-Content-Sha256' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+ 'X-Amz-Date' => '20170303T133952Z'
+ })
+ .to_return(status: 200, body: [:fake_response])
expect(client.get(index: 'foo', id: 1)).to eq([:fake_response])
end
end
end
end
+
+ describe 'resolve_aws_credentials' do
+ let(:creds) { described_class.resolve_aws_credentials(params) }
+
+ context 'with AWS IAM static credentials' do
+ let(:params) do
+ {
+ url: 'http://example-elastic:9200',
+ aws: true,
+ aws_region: 'us-east-1',
+ aws_access_key: '0',
+ aws_secret_access_key: '0'
+ }
+ end
+
+ it 'returns credentials from static credentials' do
+ stub_instance_credentials(creds_fail_response)
+
+ expect(creds.credentials.access_key_id).to eq '0'
+ expect(creds.credentials.secret_access_key).to eq '0'
+ end
+ end
+
+ context 'with AWS ec2 instance profile' do
+ let(:params) do
+ {
+ url: 'http://example-elastic:9200',
+ aws: true,
+ aws_region: 'us-east-1'
+ }
+ end
+
+ it 'returns credentials from ec2 instance profile' do
+ stub_instance_credentials(creds_valid_response)
+
+ expect(creds.credentials.access_key_id).to eq '0'
+ expect(creds.credentials.secret_access_key).to eq '0'
+ end
+ end
+
+ context 'with AWS no credentials' do
+ let(:params) do
+ {
+ url: 'http://example-elastic:9200',
+ aws: true,
+ aws_region: 'us-east-1'
+ }
+ end
+
+ it 'returns nil' do
+ stub_instance_credentials(creds_fail_response)
+
+ expect(creds).to be_nil
+ end
+ end
+ end
end
--
GitLab