From 1cb01f407f19888da2b5c6f7ad2c05ac4630ff94 Mon Sep 17 00:00:00 2001
From: Matt Gresko <mgresko@veracode.com>
Date: Thu, 16 Feb 2017 13:28:42 -0500
Subject: [PATCH] [1181] Expose low level elasticsearch client params
* replace elasticsearch_host and elasticsearch_port with elasticsearch_url
* Add support for AWS Elasticsearch Service
* created universal gitlab elasticsearch client
* add ability to sign requests with aws_signers_v4
* expose elasticsearch_aws_region param
* expose elasticsearch_aws_access_key param
* expose elasticsearch_aws_secret_access_key param
* If using AWS instance credentials they will automatically be picked up by client
---
.gitlab-ci.yml | 2 +-
Gemfile | 2 +
Gemfile.lock | 14 +++++++
.../admin/application_settings_controller.rb | 7 +++-
app/models/application_setting.rb | 17 ++++----
.../application_settings/_form.html.haml | 35 +++++++++++++---
bin/elastic_repo_indexer | 21 +++++++---
.../1181-elasticsearch-client-params.yml | 4 ++
config/initializers/1_settings.rb | 3 +-
config/initializers/elastic_client_setup.rb | 24 ++++++-----
.../20170215151539_add_aws_elasticsearch.rb | 13 ++++++
...1540_migrate_old_elasticsearch_settings.rb | 38 +++++++++++++++++
...51541_remove_old_elasticsearch_settings.rb | 12 ++++++
db/schema.rb | 9 ++--
doc/api/settings.md | 7 +++-
lib/api/settings.rb | 9 +++-
lib/api/v3/settings.rb | 3 +-
lib/gitlab/elastic/client.rb | 42 +++++++++++++++++++
lib/gitlab/elastic/indexer.rb | 7 +++-
lib/tasks/gitlab/check.rake | 12 +++++-
spec/lib/gitlab/elastic/indexer_spec.rb | 5 +--
spec/workers/elastic_indexer_worker_spec.rb | 3 +-
22 files changed, 237 insertions(+), 52 deletions(-)
create mode 100644 changelogs/unreleased-ee/1181-elasticsearch-client-params.yml
create mode 100644 db/migrate/20170215151539_add_aws_elasticsearch.rb
create mode 100644 db/migrate/20170215151540_migrate_old_elasticsearch_settings.rb
create mode 100644 db/migrate/20170215151541_remove_old_elasticsearch_settings.rb
create mode 100644 lib/gitlab/elastic/client.rb
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cc304f62eb134..3ad3c77af0d81 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,7 +9,7 @@ variables:
MYSQL_ALLOW_EMPTY_PASSWORD: "1"
# retry tests only in CI environment
RSPEC_RETRY_RETRY_COUNT: "3"
- ELASTIC_HOST: "elasticsearch"
+ ELASTIC_URL: "http://elasticsearch:9200"
RAILS_ENV: "test"
SIMPLECOV: "true"
SETUP_DB: "true"
diff --git a/Gemfile b/Gemfile
index bfae11191eb52..c746d167b3a1d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -108,6 +108,8 @@ gem 'elasticsearch-model', '~> 0.1.9'
gem 'elasticsearch-rails', '~> 0.1.9'
gem 'elasticsearch-api', '5.0.3'
gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
+gem 'aws-sdk'
+gem 'faraday_middleware-aws-signers-v4'
# Markdown and HTML processing
gem 'html-pipeline', '~> 1.11.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 357e4ca620393..7679e0888c73b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -67,6 +67,14 @@ GEM
execjs
json
awesome_print (1.2.0)
+ aws-sdk (2.7.8)
+ aws-sdk-resources (= 2.7.8)
+ aws-sdk-core (2.7.8)
+ aws-sigv4 (~> 1.0)
+ jmespath (~> 1.0)
+ aws-sdk-resources (2.7.8)
+ aws-sdk-core (= 2.7.8)
+ aws-sigv4 (1.0.0)
axiom-types (0.1.1)
descendants_tracker (~> 0.0.4)
ice_nine (~> 0.11.0)
@@ -203,6 +211,9 @@ GEM
multipart-post (>= 1.2, < 3)
faraday_middleware (0.10.0)
faraday (>= 0.7.4, < 0.10)
+ faraday_middleware-aws-signers-v4 (0.1.5)
+ aws-sdk (~> 2.1)
+ faraday (~> 0.9)
faraday_middleware-multi_json (0.0.6)
faraday_middleware
multi_json
@@ -393,6 +404,7 @@ GEM
jira-ruby (1.1.2)
activesupport
oauth (~> 0.5, >= 0.5.0)
+ jmespath (1.3.1)
jquery-atwho-rails (1.3.2)
jquery-rails (4.1.1)
rails-dom-testing (>= 1, < 3)
@@ -862,6 +874,7 @@ DEPENDENCIES
asciidoctor-plantuml (= 0.0.7)
attr_encrypted (~> 3.0.0)
awesome_print (~> 1.2.0)
+ aws-sdk
babosa (~> 1.0.2)
base32 (~> 0.3.0)
benchmark-ips (~> 2.3.0)
@@ -896,6 +909,7 @@ DEPENDENCIES
email_reply_trimmer (~> 0.1)
email_spec (~> 1.6.0)
factory_girl_rails (~> 4.7.0)
+ faraday_middleware-aws-signers-v4
ffaker (~> 2.4)
flay (~> 2.6.1)
fog-aws (~> 0.9)
diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb
index 78b199c4c9172..22f6d2258c761 100644
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@@ -161,9 +161,12 @@ def application_setting_params_ce
def application_setting_params_ee
[
:help_text,
- :elasticsearch_host,
+ :elasticsearch_url,
:elasticsearch_indexing,
- :elasticsearch_port,
+ :elasticsearch_aws,
+ :elasticsearch_aws_access_key,
+ :elasticsearch_aws_secret_access_key,
+ :elasticsearch_aws_region,
:elasticsearch_search,
:repository_size_limit,
:shared_runners_minutes,
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 57881d1dbf81a..d5eddf9d53f2d 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -91,13 +91,13 @@ class ApplicationSetting < ActiveRecord::Base
presence: true,
numericality: { only_integer: true, greater_than: 0 }
- validates :elasticsearch_host,
+ validates :elasticsearch_url,
presence: { message: "can't be blank when indexing is enabled" },
if: :elasticsearch_indexing?
- validates :elasticsearch_port,
- presence: { message: "can't be blank when indexing is enabled" },
- if: :elasticsearch_indexing?
+ validates :elasticsearch_aws_region,
+ presence: { message: "can't be blank when using aws hosted elasticsearch" },
+ if: ->(setting) { setting.elasticsearch_indexing? && setting.elasticsearch_aws? }
validates :repository_storages, presence: true
validate :check_repository_storages
@@ -236,8 +236,9 @@ def self.defaults_ce
def self.defaults_ee
{
- elasticsearch_host: ENV['ELASTIC_HOST'] || 'localhost',
- elasticsearch_port: ENV['ELASTIC_PORT'] || '9200',
+ elasticsearch_url: ENV['ELASTIC_URL'] || 'http://localhost:9200',
+ elasticsearch_aws: false,
+ elasticsearch_aws_region: ENV['ELASTIC_REGION'] || 'us-east-1',
usage_ping_enabled: true,
minimum_mirror_sync_time: Gitlab::Mirror::FIFTEEN
}
@@ -268,8 +269,8 @@ def update_mirror_cron_jobs
Gitlab::Mirror.configure_cron_jobs!
end
- def elasticsearch_host
- read_attribute(:elasticsearch_host).split(',').map(&:strip)
+ def elasticsearch_url
+ read_attribute(:elasticsearch_url).split(',').map(&:strip)
end
def home_page_url_column_exist
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index f65956d28cd75..dff7a1e28d0e0 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -454,16 +454,41 @@
Search with Elasticsearch enabled
.form-group
- = f.label :elasticsearch_host, 'Host', class: 'control-label col-sm-2'
+ = f.label :elasticsearch_url, 'URL', class: 'control-label col-sm-2'
.col-sm-10
- = f.text_field :elasticsearch_host, value: @application_setting.elasticsearch_host.join(', '), class: 'form-control', placeholder: 'localhost'
+ = f.text_field :elasticsearch_url, value: @application_setting.elasticsearch_url.join(', '), class: 'form-control', placeholder: 'http://localhost:9200'
.help-block
- The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2").
+ The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201").
+
+ %fieldset
+ %legend Elasticsearch AWS
+ .form-group
+ .col-sm-offset-2.col-sm-10
+ .checkbox
+ = f.label :elasticsearch_aws do
+ = f.check_box :elasticsearch_aws
+ Using AWS hosted Elasticsearch
+
+ .form-group
+ = f.label :elasticsearch_aws_region, 'AWS region', class: 'control-label col-sm-2'
+ .col-sm-10
+ = f.text_field :elasticsearch_aws_region, value: @application_setting.elasticsearch_aws_region, class: 'form-control'
+ .help-block
+ Region that elasticsearch is configured
.form-group
- = f.label :elasticsearch_port, 'Port', class: 'control-label col-sm-2'
+ = f.label :elasticsearch_aws_access_key, 'AWS Access Key', class: 'control-label col-sm-2'
.col-sm-10
- = f.text_field :elasticsearch_port, class: 'form-control', placeholder: ApplicationSetting.current.elasticsearch_port
+ = f.text_field :elasticsearch_aws_access_key, value: @application_setting.elasticsearch_aws_access_key, class: 'form-control'
+ .help-block
+ AWS Access Key. Only required if not using role instance credentials
+
+ .form-group
+ = f.label :elasticsearch_aws_secret_access_key, 'AWS Secret Access Key', class: 'control-label col-sm-2'
+ .col-sm-10
+ = f.password_field :elasticsearch_aws_secret_access_key, value: @application_setting.elasticsearch_aws_secret_access_key, class: 'form-control'
+ .help-block
+ AWS Secret Access Key. Only required if not using role instance credentials
%fieldset
%legend Koding
diff --git a/bin/elastic_repo_indexer b/bin/elastic_repo_indexer
index 7d968495ca589..bde5bb99174bb 100755
--- a/bin/elastic_repo_indexer
+++ b/bin/elastic_repo_indexer
@@ -8,6 +8,8 @@ require 'active_support'
require 'active_support/core_ext'
require 'benchmark'
+load File.join(File.dirname(__FILE__), '..', '/lib/gitlab/elastic/client.rb')
+
path_to_log_file = File.expand_path('../../log/es-indexer.log', __FILE__)
LOGGER = Logger.new(path_to_log_file)
@@ -20,18 +22,25 @@ RAILS_ENV = ENV['RAILS_ENV']
LOGGER.info("Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
elastic_connection_info = JSON.parse ENV['ELASTIC_CONNECTION_INFO']
-ELASTIC_HOST = elastic_connection_info['host']
-ELASTIC_PORT = elastic_connection_info['port']
+ELASTIC_URL = elastic_connection_info['url']
+ELASTIC_AWS = elastic_connection_info['aws']
+ELASTIC_AWS_ACCESS_KEY = elastic_connection_info['aws_access_key']
+ELASTIC_AWS_SECRET_ACCESS_KEY = elastic_connection_info['aws_secret_access_key']
+ELASTIC_AWS_REGION = elastic_connection_info['aws_region']
class Repository
include Elasticsearch::Git::Repository
+ include Gitlab::Elastic
index_name ['gitlab', RAILS_ENV].compact.join('-')
- self.__elasticsearch__.client = Elasticsearch::Client.new(
- host: ELASTIC_HOST,
- port: ELASTIC_PORT
- )
+ def initialize
+ if ELASTIC_AWS
+ self.__elasticsearch__.client = AWSClient(ELASTIC_URL, ELASTIC_REGION, ELASTIC_AWS_ACCESS_KEY, ELASTIC_AWS_SECRET_ACCESS_KEY).client
+ else
+ self.__elasticsearch__.client = BaseClient.new(ELASTIC_URL).client
+ end
+ end
def client_for_indexing
self.__elasticsearch__.client
diff --git a/changelogs/unreleased-ee/1181-elasticsearch-client-params.yml b/changelogs/unreleased-ee/1181-elasticsearch-client-params.yml
new file mode 100644
index 0000000000000..af9a0407c4642
--- /dev/null
+++ b/changelogs/unreleased-ee/1181-elasticsearch-client-params.yml
@@ -0,0 +1,4 @@
+---
+title: Expose elasticsearch client params for AWS signing and HTTPS
+merge_request: 1281
+author:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 2e15a602c9ee8..9451f74b30b4f 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -277,8 +277,7 @@ def cron_random_weekly_time
#
Settings['elasticsearch'] ||= Settingslogic.new({})
Settings.elasticsearch['enabled'] = false if Settings.elasticsearch['enabled'].nil?
-Settings.elasticsearch['host'] ||= ENV['ELASTIC_HOST'] || "localhost"
-Settings.elasticsearch['port'] ||= ENV['ELASTIC_PORT'] || 9200
+Settings.elasticsearch['url'] = ENV['ELASTIC_URL'] || "http://localhost:9200"
#
# CI
diff --git a/config/initializers/elastic_client_setup.rb b/config/initializers/elastic_client_setup.rb
index c6063499da7eb..5d220d69816de 100644
--- a/config/initializers/elastic_client_setup.rb
+++ b/config/initializers/elastic_client_setup.rb
@@ -7,24 +7,28 @@ module Model
module Client
module ClassMethods
include Gitlab::CurrentSettings
-
+ include Gitlab::Elastic
def client(client = nil)
if @client.nil? || es_configuration_changed?
- @es_host = current_application_settings.elasticsearch_host
- @es_port = current_application_settings.elasticsearch_port
+ @es_url = current_application_settings.elasticsearch_url
+ @es_aws = current_application_settings.elasticsearch_aws
- @client = Elasticsearch::Client.new(
- host: current_application_settings.elasticsearch_host,
- port: current_application_settings.elasticsearch_port
- )
+ if @es_aws
+ # AWS specific handling
+ @es_region = current_application_settings.elasticsearch_aws_region
+ @es_access_key = current_application_settings.elasticsearch_aws_access_key
+ @es_secret_access_key = current_application_settings.elasticsearch_aws_secret_access_key
+ @client = AWSClient.new(@es_url, @es_region, @es_access_key, @es_secret_access_key).client
+ else
+ @client = BaseClient.new(@es_url).client
+ end
end
-
@client
end
def es_configuration_changed?
- @es_host != current_application_settings.elasticsearch_host ||
- @es_port != current_application_settings.elasticsearch_port
+ @es_url != current_application_settings.elasticsearch_url ||
+ @es_aws != current_application_settings.elasticsearch_aws
end
end
end
diff --git a/db/migrate/20170215151539_add_aws_elasticsearch.rb b/db/migrate/20170215151539_add_aws_elasticsearch.rb
new file mode 100644
index 0000000000000..623e6625f1d44
--- /dev/null
+++ b/db/migrate/20170215151539_add_aws_elasticsearch.rb
@@ -0,0 +1,13 @@
+class AddAwsElasticsearch < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ def change
+ add_column :application_settings, :elasticsearch_url, :string, default: 'http://localhost:9200'
+ add_column :application_settings, :elasticsearch_aws, :boolean, default: false, null: false
+ add_column :application_settings, :elasticsearch_aws_region, :string, default: 'us-east-1'
+ add_column :application_settings, :elasticsearch_aws_access_key, :string
+ add_column :application_settings, :elasticsearch_aws_secret_access_key, :string
+ end
+end
diff --git a/db/migrate/20170215151540_migrate_old_elasticsearch_settings.rb b/db/migrate/20170215151540_migrate_old_elasticsearch_settings.rb
new file mode 100644
index 0000000000000..639df1f3e571f
--- /dev/null
+++ b/db/migrate/20170215151540_migrate_old_elasticsearch_settings.rb
@@ -0,0 +1,38 @@
+class MigrateOldElasticsearchSettings < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ def up
+ settings = Arel::Table.new(:application_settings)
+
+ finder =
+ settings
+ .where(settings[:elasticsearch_host].not_eq(nil))
+ .project(:id, :elasticsearch_host, :elasticsearch_port)
+
+ result = connection.exec_query(finder.to_sql)
+
+ # There are only a few rows in the `application_settings` table
+ result.rows.each do |id, hosts, port|
+ # elasticsearch_host may look like "1.example.com,2.example.com, 3.example.com"
+ urls = hosts.split(',').map do |host|
+ url = URI.parse('http://' + host.strip)
+ url.port = port
+ url.to_s
+ end
+
+ updater =
+ Arel::UpdateManager.new(ActiveRecord::Base)
+ .table(settings)
+ .set(settings[:elasticsearch_url] => urls.join(','))
+ .where(settings[:id].eq(id))
+
+ connection.exec_update(updater.to_sql)
+ end
+ end
+
+ def down
+ # no-op
+ end
+end
diff --git a/db/migrate/20170215151541_remove_old_elasticsearch_settings.rb b/db/migrate/20170215151541_remove_old_elasticsearch_settings.rb
new file mode 100644
index 0000000000000..88482a4790a57
--- /dev/null
+++ b/db/migrate/20170215151541_remove_old_elasticsearch_settings.rb
@@ -0,0 +1,12 @@
+class RemoveOldElasticsearchSettings < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = true
+
+ DOWNTIME_REASON = 'Removing two columns from application_settings'
+
+ def change
+ remove_column :application_settings, :elasticsearch_host, :string, default: 'localhost'
+ remove_column :application_settings, :elasticsearch_port, :string, default: '9200'
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 238b7fc5100d1..365b738a7af6c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -93,8 +93,6 @@
t.boolean "user_default_external", default: false, null: false
t.boolean "elasticsearch_indexing", default: false, null: false
t.boolean "elasticsearch_search", default: false, null: false
- t.string "elasticsearch_host", default: "localhost"
- t.string "elasticsearch_port", default: "9200"
t.string "repository_storages", default: "default"
t.string "enabled_git_access_protocol"
t.boolean "domain_blacklist_enabled", default: false
@@ -121,7 +119,12 @@
t.integer "repository_size_limit", limit: 8, default: 0
t.integer "terminal_max_session_time", default: 0, null: false
t.integer "minimum_mirror_sync_time", default: 15, null: false
- t.string "default_artifacts_expire_in", default: '0', null: false
+ t.string "default_artifacts_expire_in", default: "0", null: false
+ t.string "elasticsearch_url", default: "http://localhost:9200"
+ t.boolean "elasticsearch_aws", default: false, null: false
+ t.string "elasticsearch_aws_region", default: "us-east-1"
+ t.string "elasticsearch_aws_access_key"
+ t.string "elasticsearch_aws_secret_access_key"
end
create_table "approvals", force: :cascade do |t|
diff --git a/doc/api/settings.md b/doc/api/settings.md
index ed5dc985a9de0..5b48526bc203f 100644
--- a/doc/api/settings.md
+++ b/doc/api/settings.md
@@ -86,8 +86,11 @@ PUT /application/settings
| `help_text` | string | no | GitLab server administrator information |
| `elasticsearch_indexing` | boolean | no | Enable Elasticsearch indexing |
| `elasticsearch_search` | boolean | no | Enable Elasticsearch search |
-| `elasticsearch_host` | string | no | The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2") |
-| `elasticsearch_port` | integer | no | The TCP/IP port that Elasticsearch listens to. The default value is 9200 |
+| `elasticsearch_url` | string | no | The url to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (e.g., "http://localhost:9200, http://localhost:9201") |
+| `elasticsearch_aws` | boolean | no | Enable the use of AWS hosted Elasticsearch |
+| `elasticsearch_aws_region` | string | no | The AWS region the elasticsearch domain is configured |
+| `elasticsearch_aws_access_key` | string | no | AWS IAM access key |
+| `elasticsearch_aws_secret_access_key` | string | no | AWS IAM secret access key |
| `usage_ping_enabled` | boolean | no | Every week GitLab will report license usage back to GitLab, Inc.|
| `repository_size_limit` | integer | no | Size limit per repository (MB) |
| `plantuml_enabled` | boolean | no | Enable PlantUML integration. Default is `false`. |
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 9f5768abfd461..054f2e6e0aa07 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -115,8 +115,13 @@ def current_settings
optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
given elasticsearch_indexing: ->(val) { val } do
optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search'
- requires :elasticsearch_host, type: String, desc: 'The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2")'
- requires :elasticsearch_port, type: Integer, desc: 'The TCP/IP port that Elasticsearch listens to. The default value is 9200'
+ requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
+ end
+ optional :elasticsearch_aws, type: Boolean, desc: 'Enable support for AWS hosted elasticsearch'
+ given elasticsearch_aws: ->(val) { val } do
+ requires :elasticsearch_aws_region, type: String, desc: 'The AWS region the elasticsearch domain is configured'
+ optional :elasticsearch_aws_access_key, type: String, desc: 'AWS IAM access key'
+ optional :elasticsearch_aws_secret_access_key, type: String, desc: 'AWS IAM secret access key'
end
optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
optional :repository_storages, type: Array[String], desc: 'A list of names of enabled storage paths, taken from `gitlab.yml`. New projects will be created in one of these stores, chosen at random.'
diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb
index 7a05bc9577b2d..d40ce29e88913 100644
--- a/lib/api/v3/settings.rb
+++ b/lib/api/v3/settings.rb
@@ -115,8 +115,7 @@ def current_settings
optional :elasticsearch_indexing, type: Boolean, desc: 'Enable Elasticsearch indexing'
given elasticsearch_indexing: ->(val) { val } do
optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search'
- requires :elasticsearch_host, type: String, desc: 'The TCP/IP host to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "host1, host2")'
- requires :elasticsearch_port, type: Integer, desc: 'The TCP/IP port that Elasticsearch listens to. The default value is 9200'
+ requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
end
optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
optional :repository_storage, type: String, desc: 'The first entry in `repository_storages`. Deprecated, but retained for compatibility reasons'
diff --git a/lib/gitlab/elastic/client.rb b/lib/gitlab/elastic/client.rb
new file mode 100644
index 0000000000000..4571b04762f6f
--- /dev/null
+++ b/lib/gitlab/elastic/client.rb
@@ -0,0 +1,42 @@
+require 'elasticsearch'
+require 'aws-sdk'
+require 'faraday_middleware/aws_signers_v4'
+
+module Gitlab
+ module Elastic
+ class BaseClient
+ attr_accessor :client
+
+ def initialize(urls)
+ @urls = urls
+ @client = Elasticsearch::Client.new urls: @urls
+ end
+ end
+ end
+end
+
+module Gitlab
+ module Elastic
+ class AWSClient < BaseClient
+ def initialize(urls, region, access_key = nil, secret_access_key = nil)
+ @urls = urls
+ @region = region
+ @access_key = access_key
+ @secret_access_key = secret_access_key
+
+ if @access_key.nil? || @secret_access_key.nil?
+ @credentials = Aws::Credentials.new()
+ else
+ @credentials = Aws::Credentials.new(@access_key, @secret_access_key)
+ end
+
+ @client = Elasticsearch::Client.new urls: @urls do |fmid|
+ fmid.request :aws_signers_v4,
+ credentials: @credentials,
+ service_name: 'es',
+ region: @region
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/elastic/indexer.rb b/lib/gitlab/elastic/indexer.rb
index 75ddc70536a7d..787e54ca7722e 100644
--- a/lib/gitlab/elastic/indexer.rb
+++ b/lib/gitlab/elastic/indexer.rb
@@ -14,8 +14,11 @@ def initialize(project)
@project = project
connection_info = {
- host: current_application_settings.elasticsearch_host,
- port: current_application_settings.elasticsearch_port
+ url: current_application_settings.elasticsearch_url,
+ aws: current_application_settings.elasticsearch_aws,
+ aws_access_key: current_application_settings.elasticsearch_aws_access_key,
+ aws_secret_access_key: current_application_settings.elasticsearch_aws_secret_access_key,
+ aws_region: current_application_settings.elasticsearch_aws_region
}.to_json
# We accept any form of settings, including string and array
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index c7a8293572057..4aa7f9241f159 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -1072,8 +1072,16 @@ namespace :gitlab do
end
def check_elasticsearch
- client = Elasticsearch::Client.new(host: ApplicationSetting.current.elasticsearch_host,
- port: ApplicationSetting.current.elasticsearch_port)
+ if ApplicationSetting.current.elasticsearch_aws
+ client = Gitlab::Elastic::AWSClient.new(
+ ApplicationSetting.current.elasticsearch_url,
+ ApplicationSetting.current.elasticsearch_aws_region,
+ ApplicationSetting.current.elasticsearch_aws_access_key,
+ ApplicationSetting.current.elasticsearch_aws_secret_access_key
+ ).client
+ else
+ client = Gitlab::Elastic::BaseClient.new(ApplicationSetting.current.elasticsearch_url).client
+ end
print "Elasticsearch version 5.1.x? ... "
diff --git a/spec/lib/gitlab/elastic/indexer_spec.rb b/spec/lib/gitlab/elastic/indexer_spec.rb
index 973b0d3dfb4be..141f3953628f8 100644
--- a/spec/lib/gitlab/elastic/indexer_spec.rb
+++ b/spec/lib/gitlab/elastic/indexer_spec.rb
@@ -5,7 +5,7 @@
before do
stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'true')
- stub_application_setting(es_host: ['elastic-host1', 'elastic-host2'])
+ stub_application_setting(es_url: ['http://localhost:9200', 'http://localhost:9201'])
end
let(:project) { create(:project) }
@@ -18,8 +18,7 @@
let(:elastic_connection_info) do
{
- host: current_application_settings.elasticsearch_host,
- port: current_application_settings.elasticsearch_port,
+ url: current_application_settings.elasticsearch_url
}
end
diff --git a/spec/workers/elastic_indexer_worker_spec.rb b/spec/workers/elastic_indexer_worker_spec.rb
index 8820b386cb1d8..e5fff9219f7da 100644
--- a/spec/workers/elastic_indexer_worker_spec.rb
+++ b/spec/workers/elastic_indexer_worker_spec.rb
@@ -6,8 +6,7 @@
before do
Elasticsearch::Model.client = Elasticsearch::Client.new(
- host: current_application_settings.elasticsearch_host,
- port: current_application_settings.elasticsearch_port
+ url: current_application_settings.elasticsearch_url
)
Gitlab::Elastic::Helper.create_empty_index
--
GitLab