diff --git a/ee/elastic/docs/20230720010000_delete_wikis_from_original_index.yml b/ee/elastic/docs/20230720010000_delete_wikis_from_original_index.yml deleted file mode 100644 index 560c1cd2e08e4948414fb44eb1018454af79faf2..0000000000000000000000000000000000000000 --- a/ee/elastic/docs/20230720010000_delete_wikis_from_original_index.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: DeleteWikisFromOriginalIndex -version: '20230720010000' -description: Delete the wikis from the main index -group: group::global search -milestone: 16.3 -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/126538 -obsolete: false -marked_obsolete_by_url: -marked_obsolete_in_milestone: diff --git a/ee/elastic/migrate/20230720010000_delete_wikis_from_original_index.rb b/ee/elastic/migrate/20230720010000_delete_wikis_from_original_index.rb deleted file mode 100644 index e774f8e3889cd449e99860f6bdd990ef0a912490..0000000000000000000000000000000000000000 --- a/ee/elastic/migrate/20230720010000_delete_wikis_from_original_index.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -class DeleteWikisFromOriginalIndex < Elastic::Migration - include Elastic::MigrationHelper - - batched! - retry_on_failure - - def migrate - task_id = migration_state[:task_id] - if task_id - task_status = helper.task_status(task_id: task_id) - log_raise 'Failed to delete wikis', task_id: task_id, error: task_status['error'] if task_status['error'].present? - - if task_status['completed'] - log 'Removing wikis from the original index is completed for a specific task', task_id: task_id - set_migration_state(task_id: nil, documents_remaining: 0) - else - log 'Removing wikis from the original index is still in progress for a specific task', task_id: task_id - end - - return - end - - if completed? - log 'There are no wikis to remove from original index' - return - end - - log 'Launching delete by query' - response = client.delete_by_query(index: helper.target_name, conflicts: 'proceed', wait_for_completion: false, - body: { query: { bool: { filter: { term: { type: 'wiki_blob' } } } } }, - slices: get_number_of_shards(index_name: helper.target_name) - ) - task_id = response['task'] - log 'Removing wikis from the original index is started for a task', task_id: task_id - set_migration_state(task_id: task_id, documents_remaining: original_documents_count) - rescue StandardError => e - set_migration_state(task_id: nil, documents_remaining: original_documents_count) - raise e - end - - def completed? - helper.refresh_index - total_remaining = original_documents_count - log 'Checking if migration is completed based on documents counts remaining', remaining: total_remaining - total_remaining == 0 - end - - private - - def document_type - :wiki_blob - end -end diff --git a/ee/spec/elastic/migrate/20230720010000_delete_wikis_from_original_index_spec.rb b/ee/spec/elastic/migrate/20230720010000_delete_wikis_from_original_index_spec.rb deleted file mode 100644 index 7dac477a6b0681fbcd322e7394c4b6df2f91ff33..0000000000000000000000000000000000000000 --- a/ee/spec/elastic/migrate/20230720010000_delete_wikis_from_original_index_spec.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require File.expand_path('ee/elastic/migrate/20230720010000_delete_wikis_from_original_index.rb') - -RSpec.describe DeleteWikisFromOriginalIndex, feature_category: :global_search do - let(:version) { 20230720010000 } - let(:migration) { described_class.new(version) } - let(:helper) { Gitlab::Elastic::Helper.new } - - before do - stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true) - allow(migration).to receive(:helper).and_return(helper) - end - - describe 'migration_options' do - it 'has migration options set', :aggregate_failures do - expect(migration.batched?).to be_truthy - expect(migration.batch_size).to eq(1000) - expect(migration).to be_retry_on_failure - end - end - - describe '.migrate', :elastic, :sidekiq_inline do - let_it_be(:project) { create(:project, :public, :repository, :wiki_repo) } - let(:client) { ::Gitlab::Search::Client.new } - - before do - allow(migration).to receive(:client).and_return(client) - set_elasticsearch_migration_to :migrate_wikis_to_separate_index, including: false - 5.times do |i| - project.wiki.create_page("test#{i}", 'test') - project.wiki.index_wiki_blobs - end - ensure_elasticsearch_index! # ensure objects are indexed - end - - context 'when wikis are still present in the index' do - it 'removes wikis from the index' do - expect(migration.completed?).to be_falsey - migration.migrate - expect(migration.migration_state).to match(documents_remaining: anything, task_id: anything) - # the migration might not complete after the initial task is created - # so make sure it actually completes - 10.times do - migration.migrate - break if migration.completed? - end - - expect(migration.completed?).to be_truthy - expect(migration.migration_state).to match(task_id: nil, documents_remaining: 0) - end - - context 'and task in progress' do - before do - allow(migration).to receive(:completed?).and_return(false) - allow(migration).to receive(:client).and_return(client) - allow(helper).to receive(:task_status).and_return('completed' => false) - migration.set_migration_state(task_id: 'task_1') - end - - it 'does nothing if task is not completed' do - migration.migrate - expect(client).not_to receive(:delete_by_query) - end - end - end - - context 'when migration fails' do - context 'with exception is raised' do - before do - allow(client).to receive(:delete_by_query).and_raise(StandardError) - end - - it 'resets task_id' do - migration.set_migration_state(task_id: 'task_1') - expect { migration.migrate }.to raise_error(StandardError) - expect(migration.migration_state).to match(task_id: nil, documents_remaining: anything) - end - end - - context 'with task throws an error' do - before do - allow(client).to receive(:delete_by_query).and_return('task' => 'task_1') - allow(migration).to receive(:get_number_of_shards).and_return(1) - allow(helper).to receive(:task_status).and_return('error' => ['failed']) - migration.migrate - end - - it 'resets task_id' do - expect { migration.migrate }.to raise_error(/Failed to delete wikis/) - expect(migration.migration_state).to match(task_id: nil, documents_remaining: anything) - end - end - end - - context 'when wikis are already deleted' do - before do - client.delete_by_query(index: helper.target_name, - body: { query: { bool: { filter: { term: { type: 'wiki_blob' } } } } }) - end - - it 'does not execute delete_by_query' do - expect(migration.completed?).to be_truthy - expect(helper.client).not_to receive(:delete_by_query) - migration.migrate - end - end - end - - describe '.completed?' do - context 'when original_documents_count is zero' do - before do - allow(migration).to receive(:original_documents_count).and_return 0 - end - - it 'returns true' do - expect(migration.completed?).to eq true - end - end - - context 'when original_documents_count is non zero' do - before do - allow(migration).to receive(:original_documents_count).and_return 1 - end - - it 'returns false' do - expect(migration.completed?).to eq false - end - end - end -end