diff --git a/Gemfile.checksum b/Gemfile.checksum index 66244e03c0fccfadf6b75230037a78c4f66ba819..296e894ac27d8efb03f3ee24de446a7017a8e7e9 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -459,6 +459,7 @@ {"name":"omniauth_openid_connect","version":"0.8.0","platform":"ruby","checksum":"1f2f3890386e2a742221cee0d2e903b78d874e6fab9ea3bfa31c1462f4793d25"}, {"name":"open4","version":"1.3.4","platform":"ruby","checksum":"a1df037310624ecc1ea1d81264b11c83e96d0c3c1c6043108d37d396dcd0f4b1"}, {"name":"openid_connect","version":"2.3.0","platform":"ruby","checksum":"0dbb9cefeb11e0a65e706349266355bbbb060382ae138fc9e199ab1aa622744c"}, +{"name":"opensearch-ruby","version":"3.4.0","platform":"ruby","checksum":"0a8621686bed3c59b4c23e08cbaef873685a3fe4568e9d2703155ca92b8ca05d"}, {"name":"openssl","version":"3.2.0","platform":"java","checksum":"9a1c870b4175ee90bcd233b5041a5ca8072f5f5f06d404ab3c786aa31daffa02"}, {"name":"openssl","version":"3.2.0","platform":"ruby","checksum":"3c4bb8760977b4becd2819c6c2569bcf5c6f48b32b9f7a4ce1fd37f996378d14"}, {"name":"openssl-signature_algorithm","version":"1.3.0","platform":"ruby","checksum":"a3b40b5e8276162d4a6e50c7c97cdaf1446f9b2c3946a6fa2c14628e0c957e80"}, diff --git a/Gemfile.lock b/Gemfile.lock index 279872d49f14c12abfe4eb75ec251073b8b854bd..69f9459bd89a17b67297cf23c124dc49a0e1b774 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -30,6 +30,7 @@ PATH activesupport connection_pool elasticsearch + opensearch-ruby pg zeitwerk @@ -1293,6 +1294,9 @@ GEM tzinfo validate_url webfinger (~> 2.0) + opensearch-ruby (3.4.0) + faraday (>= 1.0, < 3) + multi_json (>= 1.0) openssl (3.2.0) openssl-signature_algorithm (1.3.0) openssl (> 2.0) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 058a7234154643d336a75b2a22cf72bd03a81c9c..9188744ec88b07cc8254f9128d89e4fe7fc932c2 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -464,6 +464,7 @@ {"name":"omniauth_openid_connect","version":"0.8.0","platform":"ruby","checksum":"1f2f3890386e2a742221cee0d2e903b78d874e6fab9ea3bfa31c1462f4793d25"}, {"name":"open4","version":"1.3.4","platform":"ruby","checksum":"a1df037310624ecc1ea1d81264b11c83e96d0c3c1c6043108d37d396dcd0f4b1"}, {"name":"openid_connect","version":"2.3.0","platform":"ruby","checksum":"0dbb9cefeb11e0a65e706349266355bbbb060382ae138fc9e199ab1aa622744c"}, +{"name":"opensearch-ruby","version":"3.4.0","platform":"ruby","checksum":"0a8621686bed3c59b4c23e08cbaef873685a3fe4568e9d2703155ca92b8ca05d"}, {"name":"openssl","version":"3.2.0","platform":"java","checksum":"9a1c870b4175ee90bcd233b5041a5ca8072f5f5f06d404ab3c786aa31daffa02"}, {"name":"openssl","version":"3.2.0","platform":"ruby","checksum":"3c4bb8760977b4becd2819c6c2569bcf5c6f48b32b9f7a4ce1fd37f996378d14"}, {"name":"openssl-signature_algorithm","version":"1.3.0","platform":"ruby","checksum":"a3b40b5e8276162d4a6e50c7c97cdaf1446f9b2c3946a6fa2c14628e0c957e80"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 30c5dc5063f2bdad405512b7ba9f5b21a8b49271..44ab9f4a5166b9ffdf088fdd8eb227e11cf27bab 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -30,6 +30,7 @@ PATH activesupport connection_pool elasticsearch + opensearch-ruby pg zeitwerk @@ -1311,6 +1312,9 @@ GEM tzinfo validate_url webfinger (~> 2.0) + opensearch-ruby (3.4.0) + faraday (>= 1.0, < 3) + multi_json (>= 1.0) openssl (3.2.0) openssl-signature_algorithm (1.3.0) openssl (> 2.0) diff --git a/gems/gitlab-active-context/Gemfile.lock b/gems/gitlab-active-context/Gemfile.lock index 277f3b38fd6e1c34ca2bec6e8bd18d3102093364..65542b80a33e68415182dcff14a4bf759db4f6bf 100644 --- a/gems/gitlab-active-context/Gemfile.lock +++ b/gems/gitlab-active-context/Gemfile.lock @@ -5,6 +5,7 @@ PATH activesupport connection_pool elasticsearch + opensearch-ruby pg zeitwerk @@ -43,6 +44,15 @@ GEM addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) + aws-eventstream (1.3.0) + aws-partitions (1.1001.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sigv4 (1.9.1) + aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) benchmark (0.4.0) bigdecimal (3.1.8) @@ -73,6 +83,9 @@ GEM logger faraday-net_http (3.4.0) net-http (>= 0.5.0) + faraday_middleware-aws-sigv4 (1.0.1) + aws-sigv4 (~> 1.0) + faraday (>= 2.0, < 3) gitlab-styles (13.0.2) rubocop (~> 1.68.0) rubocop-capybara (~> 2.21.0) @@ -89,6 +102,7 @@ GEM irb (1.14.1) rdoc (>= 4.0.0) reline (>= 0.4.2) + jmespath (1.6.2) json (2.9.0) language_server-protocol (3.17.0.3) logger (1.6.2) @@ -105,6 +119,9 @@ GEM racc (~> 1.4) nokogiri (1.17.1-arm64-darwin) racc (~> 1.4) + opensearch-ruby (3.4.0) + faraday (>= 1.0, < 3) + multi_json (>= 1.0) parallel (1.26.3) parser (3.3.6.0) ast (~> 2.4.1) @@ -218,7 +235,9 @@ PLATFORMS DEPENDENCIES activesupport + aws-sdk-core byebug + faraday_middleware-aws-sigv4 gitlab-active-context! gitlab-styles rake (~> 13.0) diff --git a/gems/gitlab-active-context/gitlab-active-context.gemspec b/gems/gitlab-active-context/gitlab-active-context.gemspec index 1fcd37891df7af0ecdc4e2a304aa3789756b7279..85dcb86381caa6697d97c540a351f0ff2523437d 100644 --- a/gems/gitlab-active-context/gitlab-active-context.gemspec +++ b/gems/gitlab-active-context/gitlab-active-context.gemspec @@ -22,9 +22,12 @@ Gem::Specification.new do |spec| spec.add_dependency 'activesupport' spec.add_dependency 'connection_pool' spec.add_dependency 'elasticsearch' + spec.add_dependency 'opensearch-ruby' spec.add_dependency 'pg' spec.add_dependency 'zeitwerk' + spec.add_development_dependency 'aws-sdk-core' + spec.add_development_dependency 'faraday_middleware-aws-sigv4' spec.add_development_dependency 'gitlab-styles' spec.add_development_dependency 'rspec-rails' spec.add_development_dependency 'rubocop-rspec' diff --git a/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb b/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb new file mode 100644 index 0000000000000000000000000000000000000000..7c685b7bba0d796f8a6d352f2270c1c80951aefb --- /dev/null +++ b/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module ActiveContext + module Databases + module Opensearch + class Adapter + include ActiveContext::Databases::Concerns::Adapter + + def client_klass + ActiveContext::Databases::Opensearch::Client + end + end + end + end +end diff --git a/gems/gitlab-active-context/lib/active_context/databases/opensearch/client.rb b/gems/gitlab-active-context/lib/active_context/databases/opensearch/client.rb new file mode 100644 index 0000000000000000000000000000000000000000..310a724499d28db35f1694f7b96126236d185153 --- /dev/null +++ b/gems/gitlab-active-context/lib/active_context/databases/opensearch/client.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'opensearch' +require 'faraday_middleware/aws_sigv4' + +module ActiveContext + module Databases + module Opensearch + class Client + include ActiveContext::Databases::Concerns::Client + + OPEN_TIMEOUT = 5 + NO_RETRY = 0 + + def initialize(options) + @options = options + end + + def search(_query) + res = client.search + QueryResult.new(res) + end + + def client + ::OpenSearch::Client.new(opensearch_config) do |fmid| + next unless options[:aws] + + fmid.request( + :aws_sigv4, + credentials_provider: aws_credentials, + service: 'es', + region: options[:aws_region] + ) + end + end + + def aws_credentials + static_credentials = ::Aws::Credentials.new(options[:aws_access_key], options[:aws_secret_access_key]) + + return static_credentials if static_credentials&.set? + + aws_credential_provider = ::Aws::CredentialProviderChain.new.resolve + aws_credential_provider if aws_credential_provider&.set? + end + + private + + def opensearch_config + { + adapter: :net_http, + urls: options[:url], + transport_options: { + request: { + timeout: options[:client_request_timeout], + open_timeout: OPEN_TIMEOUT + } + }, + randomize_hosts: true, + retry_on_failure: options[:retry_on_failure] || NO_RETRY, + log: options[:debug], + debug: options[:debug] + }.compact + end + end + end + end +end diff --git a/gems/gitlab-active-context/lib/active_context/databases/opensearch/query_result.rb b/gems/gitlab-active-context/lib/active_context/databases/opensearch/query_result.rb new file mode 100644 index 0000000000000000000000000000000000000000..1d2170e7da5f6aeedb647deb51260f3bf24e744f --- /dev/null +++ b/gems/gitlab-active-context/lib/active_context/databases/opensearch/query_result.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module ActiveContext + module Databases + module Opensearch + class QueryResult + include ActiveContext::Databases::Concerns::QueryResult + + def initialize(result) + @result = result + end + + def count + result['hits']['total']['value'] + end + + def each + return enum_for(:each) unless block_given? + + result['hits']['hits'].each do |hit| + yield hit['_source'] + end + end + + private + + attr_reader :result + end + end + end +end diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1afba9a992b1e5954085d9c732c8b08c27f0dbbe --- /dev/null +++ b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +RSpec.describe ActiveContext::Databases::Opensearch::Adapter do + let(:options) { { url: 'http://localhost:9200' } } + + subject(:adapter) { described_class.new(options) } + + it 'delegates search to client' do + query = ActiveContext::Query.filter(foo: :bar) + expect(adapter.client).to receive(:search).with(query) + + adapter.search(query) + end +end diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..40cba13c9333ef039646d6343cecc6b203223e5d --- /dev/null +++ b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +RSpec.describe ActiveContext::Databases::Opensearch::Client do + let(:options) { { url: 'http://localhost:9200' } } + + subject(:client) { described_class.new(options) } + + describe '#search' do + let(:opensearch_client) { instance_double(OpenSearch::Client) } + let(:search_response) { { 'hits' => { 'total' => 5, 'hits' => [] } } } + + before do + allow(client).to receive(:client).and_return(opensearch_client) + allow(opensearch_client).to receive(:search).and_return(search_response) + end + + it 'calls search on the Opensearch client' do + expect(opensearch_client).to receive(:search) + client.search('query') + end + + it 'returns a QueryResult object' do + result = client.search('query') + expect(result).to be_a(ActiveContext::Databases::Opensearch::QueryResult) + end + end + + describe '#client' do + it 'returns an instance of OpenSearch::Client' do + expect(OpenSearch::Client).to receive(:new).with(client.send(:opensearch_config)) + client.client + end + end + + describe '#aws_credentials' do + context 'when static credentials are provided' do + let(:options) do + { + url: 'http://localhost:9200', + aws: true, + aws_access_key: 'access_key', + aws_secret_access_key: 'secret_key' + } + end + + it 'returns static credentials' do + credentials = client.aws_credentials + expect(credentials).to be_a(Aws::Credentials) + expect(credentials.access_key_id).to eq('access_key') + expect(credentials.secret_access_key).to eq('secret_key') + end + end + + context 'when static credentials are not provided' do + let(:options) { { url: 'http://localhost:9200', aws: true } } + let(:mock_provider) { instance_double(Aws::Credentials, set?: true) } + let(:mock_chain) { instance_double(Aws::CredentialProviderChain, resolve: mock_provider) } + + before do + allow(Aws::CredentialProviderChain).to receive(:new).and_return(mock_chain) + end + + it 'uses the AWS credential provider chain' do + expect(client.aws_credentials).to eq(mock_provider) + end + end + + context 'when no valid credentials are found' do + let(:options) { { url: 'http://localhost:9200', aws: true } } + let(:mock_chain) { instance_double(Aws::CredentialProviderChain, resolve: nil) } + + before do + allow(Aws::CredentialProviderChain).to receive(:new).and_return(mock_chain) + end + + it 'returns nil' do + expect(client.aws_credentials).to be_nil + end + end + end + + describe '#prefix' do + it 'returns default prefix when not specified' do + expect(client.prefix).to eq('gitlab_active_context') + end + + it 'returns configured prefix' do + client = described_class.new(options.merge(prefix: 'custom')) + expect(client.prefix).to eq('custom') + end + end +end diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/query_result_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/query_result_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..250f8385fe890c2080382a866412f694a0e04a3e --- /dev/null +++ b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/query_result_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +RSpec.describe ActiveContext::Databases::Opensearch::QueryResult do + let(:elasticsearch_result) do + { + 'hits' => { + 'total' => { 'value' => 2 }, + 'hits' => [ + { '_source' => { 'id' => 1, 'name' => 'test1' } }, + { '_source' => { 'id' => 2, 'name' => 'test2' } } + ] + } + } + end + + subject(:query_result) { described_class.new(elasticsearch_result) } + + describe '#count' do + it 'returns the total number of hits' do + expect(query_result.count).to eq(2) + end + end + + describe '#each' do + it 'yields each hit source' do + expected_sources = [ + { 'id' => 1, 'name' => 'test1' }, + { 'id' => 2, 'name' => 'test2' } + ] + + expect { |b| query_result.each(&b) }.to yield_successive_args(*expected_sources) + end + + it 'returns an enumerator when no block is given' do + expect(query_result.each).to be_a(Enumerator) + end + end + + describe 'enumerable behavior' do + it 'implements Enumerable methods' do + expect(query_result.map { |hit| hit['id'] }).to eq([1, 2]) # rubocop: disable Rails/Pluck -- pluck not implemented + expect(query_result.select { |hit| hit['id'] == 1 }).to eq([{ 'id' => 1, 'name' => 'test1' }]) + end + end +end diff --git a/gems/gitlab-active-context/spec/spec_helper.rb b/gems/gitlab-active-context/spec/spec_helper.rb index 24c11cbbb417668d6e76bd9b03271d48a4e2005c..dcfec747cbbb1e1e1b94a35487cd5e2cb08a7608 100644 --- a/gems/gitlab-active-context/spec/spec_helper.rb +++ b/gems/gitlab-active-context/spec/spec_helper.rb @@ -3,6 +3,8 @@ require "active_context" require 'logger' require 'elasticsearch' +require 'opensearch' +require 'aws-sdk-core' RSpec.configure do |config| # Enable flags like --only-failures and --next-failure