diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 3a55d0f6747a212a5363ae731a7bf23a3559ad0f..e20f211bd3300f7435376f6527b4599fa53c382c 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -264,7 +264,7 @@ .zoekt-services: services: - - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1 + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.2 alias: zoekt-ci-image .use-pg12: diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 31b058e9e8529602ff305254a19271d0e9928cd4..bc5a2ec1d246ebfb158f9c06fb1eeeb7b8923b19 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -565,6 +565,8 @@ - 1 - - search_wiki_elastic_delete_group_wiki - 1 +- - search_zoekt_delete_project + - 1 - - search_zoekt_namespace_indexer - 1 - - security_auto_fix diff --git a/ee/app/models/concerns/elastic/projects_search.rb b/ee/app/models/concerns/elastic/projects_search.rb index 2c238c0bd69a3b30ec627b19b72cabf012ce6e8a..c3b3a7bffc7ff434e1cfeb63d6f2c68ae591b3dd 100644 --- a/ee/app/models/concerns/elastic/projects_search.rb +++ b/ee/app/models/concerns/elastic/projects_search.rb @@ -38,6 +38,7 @@ def maintain_elasticsearch_update(updated_attributes: previous_changes.keys) override :maintain_elasticsearch_destroy def maintain_elasticsearch_destroy ElasticDeleteProjectWorker.perform_async(self.id, self.es_id) + Search::Zoekt::DeleteProjectWorker.perform_async(self.root_namespace&.id, self.id) end def invalidate_elasticsearch_indexes_cache! diff --git a/ee/app/models/zoekt/shard.rb b/ee/app/models/zoekt/shard.rb index 39109a3cae9b500641cbe09d699e3243b5c0cb18..9a88adf52e7cfc1b81e79660df3d7ed543b5c1be 100644 --- a/ee/app/models/zoekt/shard.rb +++ b/ee/app/models/zoekt/shard.rb @@ -7,5 +7,11 @@ def self.table_name_prefix end has_many :indexed_namespaces, foreign_key: :zoekt_shard_id, inverse_of: :shard + + def self.for_namespace(root_namespace_id:) + ::Zoekt::Shard.find_by( + id: ::Zoekt::IndexedNamespace.where(namespace_id: root_namespace_id).select(:zoekt_shard_id) + ) + end end end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index 0513aa73705a8e321d1ede33e502b2eb738845f3..0ee38d71a787a8f852c2108a08a7fc1e622e4492 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -1794,6 +1794,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: search_zoekt_delete_project + :worker_name: Search::Zoekt::DeleteProjectWorker + :feature_category: :global_search + :has_external_dependencies: false + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: search_zoekt_namespace_indexer :worker_name: Search::Zoekt::NamespaceIndexerWorker :feature_category: :global_search diff --git a/ee/app/workers/search/zoekt/delete_project_worker.rb b/ee/app/workers/search/zoekt/delete_project_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..81869e85467327a069df38b4e0d4c0716a2f7b45 --- /dev/null +++ b/ee/app/workers/search/zoekt/delete_project_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Search + module Zoekt + class DeleteProjectWorker + include ApplicationWorker + include Gitlab::ExclusiveLeaseHelpers + + TIMEOUT = 1.minute + + data_consistency :delayed + + feature_category :global_search + urgency :throttled + idempotent! + + def perform(root_namespace_id, project_id) + return unless ::Feature.enabled?(:index_code_with_zoekt) + return unless ::License.feature_available?(:zoekt_code_search) + + in_lock("#{self.class.name}/#{project_id}", ttl: TIMEOUT, retries: 0) do + ::Gitlab::Search::Zoekt::Client.delete(root_namespace_id: root_namespace_id, project_id: project_id) + end + end + end + end +end diff --git a/ee/lib/gitlab/search/zoekt/client.rb b/ee/lib/gitlab/search/zoekt/client.rb index b9758dce1148e42cf0ea5f3888653aca2b8d83f1..149c7792217c9ed664942ff1b7136c21483a009c 100644 --- a/ee/lib/gitlab/search/zoekt/client.rb +++ b/ee/lib/gitlab/search/zoekt/client.rb @@ -11,7 +11,7 @@ def instance @instance ||= new end - delegate :search, :index, :truncate, to: :instance + delegate :search, :index, :delete, :truncate, to: :instance end def search(query, num:, project_ids:) @@ -52,6 +52,21 @@ def index(project) use_new_zoekt_indexer? ? index_with_new_indexer(project) : index_with_legacy_indexer(project) end + def delete(root_namespace_id:, project_id:) + return false unless use_new_zoekt_indexer? + + shard = ::Zoekt::Shard.for_namespace(root_namespace_id: root_namespace_id) + + return false unless shard + + response = delete_request(URI.join(shard.index_base_url, "/indexer/index/#{project_id}")) + + raise "Request failed with: #{response.inspect}" unless response.success? + raise response['Error'] if response['Error'] + + response + end + def truncate post(URI.join(index_base_url, zoekt_indexer_truncate_path)) end @@ -71,6 +86,17 @@ def post(url, payload = {}, **options) ) end + def delete_request(url, **options) + defaults = { + allow_local_requests: true, + basic_auth: basic_auth_params + } + ::Gitlab::HTTP.delete( + url, + defaults.merge(options) + ) + end + def zoekt_indexer_post(path, payload) post( URI.join(index_base_url, path), diff --git a/ee/spec/lib/gitlab/search/zoekt/client_spec.rb b/ee/spec/lib/gitlab/search/zoekt/client_spec.rb index 7b7fe01135a9e5e6a975b9b031942cd13aeac903..f9a189b7185c26c25677d6c1d080fbd13bd9d65d 100644 --- a/ee/spec/lib/gitlab/search/zoekt/client_spec.rb +++ b/ee/spec/lib/gitlab/search/zoekt/client_spec.rb @@ -148,6 +148,50 @@ end end + describe '#delete' do + subject { described_class.delete(root_namespace_id: project_1.root_namespace.id, project_id: project_1.id) } + + context 'when project is indexed' do + before do + zoekt_ensure_project_indexed!(project_1) + end + + it 'removes project data from the Zoekt shard' do + search_results = described_class.new.search('use.*egex', num: 10, project_ids: [project_1.id]) + expect(search_results[:Result][:Files].to_a.size).to eq(2) + + subject + + search_results = described_class.new.search('use.*egex', num: 10, project_ids: [project_1.id]) + expect(search_results[:Result][:Files].to_a).to be_empty + end + end + + context 'when use_new_zoekt_indexer is disabled' do + before do + stub_feature_flags(use_new_zoekt_indexer: false) + end + + it 'returns false' do + expect(subject).to eq(false) + end + end + + context 'when request fails' do + let(:response) { {} } + + before do + zoekt_ensure_project_indexed!(project_1) + allow(response).to receive(:success?).and_return(false) + allow(::Gitlab::HTTP).to receive(:delete).and_return(response) + end + + it 'raises and exception' do + expect { subject }.to raise_error(StandardError, /Request failed/) + end + end + end + describe '#truncate' do it 'removes all data from the Zoekt shard' do client.index(project_1) diff --git a/ee/spec/models/concerns/elastic/projects_search_spec.rb b/ee/spec/models/concerns/elastic/projects_search_spec.rb index c5419502498526137dafafdf20b8de13cadd013a..8c084cddaa045be9bf6248cd9f1774fb421c24f6 100644 --- a/ee/spec/models/concerns/elastic/projects_search_spec.rb +++ b/ee/spec/models/concerns/elastic/projects_search_spec.rb @@ -22,6 +22,10 @@ def pending_delete? def project_feature ProjectFeature.new end + + def root_namespace + Namespace.new + end end.new end @@ -45,6 +49,7 @@ def project_feature describe '#maintain_elasticsearch_destroy' do it 'calls delete worker' do expect(ElasticDeleteProjectWorker).to receive(:perform_async) + expect(Search::Zoekt::DeleteProjectWorker).to receive(:perform_async) subject.maintain_elasticsearch_destroy end diff --git a/ee/spec/models/zoekt/shard_spec.rb b/ee/spec/models/zoekt/shard_spec.rb index c8b2245b226b0a8d1f0b1640d9947e1186070c97..5ddb2d25ef04ce5108bd62122297884e8453e066 100644 --- a/ee/spec/models/zoekt/shard_spec.rb +++ b/ee/spec/models/zoekt/shard_spec.rb @@ -17,4 +17,14 @@ expect(shard.indexed_namespaces.count).to eq(2) expect(shard.indexed_namespaces.map(&:namespace)).to contain_exactly(indexed_namespace1, indexed_namespace2) end + + describe '.for_namespace' do + it 'returns associated shard' do + expect(described_class.for_namespace(root_namespace_id: indexed_namespace1.id)).to eq(shard) + end + + it 'returns nil when no shard is associated' do + expect(described_class.for_namespace(root_namespace_id: unindexed_namespace.id)).to be_nil + end + end end diff --git a/ee/spec/workers/search/zoekt/delete_project_worker_spec.rb b/ee/spec/workers/search/zoekt/delete_project_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..8b6349b836d83379215592b4b2a119e0c09a7e8c --- /dev/null +++ b/ee/spec/workers/search/zoekt/delete_project_worker_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Search::Zoekt::DeleteProjectWorker, feature_category: :global_search do + let(:root_namespace_id) { 10 } + let(:project_id) { 128 } + + describe '#perform' do + subject { described_class.new.perform(root_namespace_id, project_id) } + + it 'executes delete_zoekt_index!' do + expect(::Gitlab::Search::Zoekt::Client).to receive(:delete) + .with(root_namespace_id: root_namespace_id, project_id: project_id) + + subject + end + + context 'when zoekt indexing is disabled' do + before do + stub_feature_flags(index_code_with_zoekt: false) + end + + it 'does nothing' do + expect(::Gitlab::Search::Zoekt::Client).not_to receive(:delete) + + subject + end + end + end +end