From a7d8c59ab67b030b982418f7d764c736e724406d Mon Sep 17 00:00:00 2001
From: Terri Chu <tchu@gitlab.com>
Date: Fri, 17 May 2024 23:01:22 +0000
Subject: [PATCH] Mark 20230428500000 as obsolete
This migration marks the 20230428500000 AddSuffixProjectInWikiRid Advanced search migration as obsolete.
This MR will still need changes to remove references to the migration in the code.
At the moment the `gitlab-housekeeper` is not always capable of removing all references so you must check the
diff and pipeline failures to confirm if there are any issues.
It is the responsibility of the assignee (picked from ~"group::global search") to push those changes to this branch.
You can read more about the process for marking Advanced search migrations as obsolete in
https://docs.gitlab.com/ee/development/search/advanced_search_migration_styleguide.html#deleting-advanced-search-migrations-in-a-major-version-upgrade.
As part of our process we want to ensure all Advanced search migrations have had at least one
[required stop](https://docs.gitlab.com/ee/development/database/required_stops.html)
to process the migration. Therefore we can mark any Advanced search migrations added before the
last required stop as obsolete.
This change was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
using the Keeps::MarkOldAdvancedSearchMigrationsAsObsolete keep.
To provide feedback on your experience with `gitlab-housekeeper` please comment in
<https://gitlab.com/gitlab-org/gitlab/-/issues/442003>.
Changelog: other
EE: true
---
.rubocop_todo/rspec/named_subject.yml | 1 -
...8500000_add_suffix_project_in_wiki_rid.yml | 6 +-
...28500000_add_suffix_project_in_wiki_rid.rb | 2 +
ee/lib/elastic/latest/git_instance_proxy.rb | 5 -
ee/lib/elastic/latest/wiki_instance_proxy.rb | 6 +-
...000_add_suffix_project_in_wiki_rid_spec.rb | 238 +-----------------
...ct_in_wiki_rid_running_wiki_search_spec.rb | 51 ----
.../latest/wiki_instance_proxy_spec.rb | 2 +-
8 files changed, 8 insertions(+), 303 deletions(-)
delete mode 100644 ee/spec/lib/elastic/latest/git_class_proxy_add_suffix_project_in_wiki_rid_running_wiki_search_spec.rb
diff --git a/.rubocop_todo/rspec/named_subject.yml b/.rubocop_todo/rspec/named_subject.yml
index efbb4c5d9d0bf..2c545b19767bc 100644
--- a/.rubocop_todo/rspec/named_subject.yml
+++ b/.rubocop_todo/rspec/named_subject.yml
@@ -329,7 +329,6 @@ RSpec/NamedSubject:
- 'ee/spec/lib/elastic/latest/application_instance_proxy_spec.rb'
- 'ee/spec/lib/elastic/latest/epic_class_proxy_spec.rb'
- 'ee/spec/lib/elastic/latest/epic_instance_proxy_spec.rb'
- - 'ee/spec/lib/elastic/latest/git_class_proxy_add_suffix_project_in_wiki_rid_running_wiki_search_spec.rb'
- 'ee/spec/lib/elastic/latest/git_class_proxy_spec.rb'
- 'ee/spec/lib/elastic/latest/git_instance_proxy_spec.rb'
- 'ee/spec/lib/elastic/latest/issue_class_proxy_spec.rb'
diff --git a/ee/elastic/docs/20230428500000_add_suffix_project_in_wiki_rid.yml b/ee/elastic/docs/20230428500000_add_suffix_project_in_wiki_rid.yml
index 8c96ed09b4854..7a9fd5b1402b2 100644
--- a/ee/elastic/docs/20230428500000_add_suffix_project_in_wiki_rid.yml
+++ b/ee/elastic/docs/20230428500000_add_suffix_project_in_wiki_rid.yml
@@ -5,6 +5,6 @@ description: AddSuffixProjectInWikiRid
group: group::global search
milestone: '16.0'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119261
-obsolete: false
-marked_obsolete_by_url:
-marked_obsolete_in_milestone:
+obsolete: true
+marked_obsolete_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/152979
+marked_obsolete_in_milestone: '17.1'
diff --git a/ee/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid.rb b/ee/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid.rb
index bf3b3d2260063..c74af4441037d 100644
--- a/ee/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid.rb
+++ b/ee/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid.rb
@@ -113,3 +113,5 @@ def process_already_started_task(task_id, slice)
end
end
end
+
+AddSuffixProjectInWikiRid.prepend ::Elastic::MigrationObsolete
diff --git a/ee/lib/elastic/latest/git_instance_proxy.rb b/ee/lib/elastic/latest/git_instance_proxy.rb
index d710df8a94256..a45b214865313 100644
--- a/ee/lib/elastic/latest/git_instance_proxy.rb
+++ b/ee/lib/elastic/latest/git_instance_proxy.rb
@@ -34,7 +34,6 @@ def blob_aggregations(query, options)
# If is_wiki is true then set index as (#{env}-wikis)
# rid as (wiki_project_#{id}) for ProjectWiki and (wiki_group_#{id}) for GroupWiki
- # If add_suffix_project_in_wiki_rid has not finished then rid might not have prefix(project/group) then
# run delete_query_by_rid with sending rid as 'wiki_#{project_id}'
def delete_index_for_commits_and_blobs(is_wiki: false)
types = is_wiki ? %w[wiki_blob] : %w[commit blob]
@@ -47,10 +46,6 @@ def delete_index_for_commits_and_blobs(is_wiki: false)
end
response = delete_query_by_rid(index, rid, is_wiki)
- # Consider to delete wikis by older rid(without suffix _project) as well
- if is_wiki && project_id && !::Elastic::DataMigrationService.migration_has_finished?(:add_suffix_project_in_wiki_rid)
- response = delete_query_by_rid(index, "wiki_#{project_id}", is_wiki)
- end
return response if is_wiki # if condition can be removed once the blob gets migrated to the separate index
end
diff --git a/ee/lib/elastic/latest/wiki_instance_proxy.rb b/ee/lib/elastic/latest/wiki_instance_proxy.rb
index 7ba2a58950590..4f411b8f4707a 100644
--- a/ee/lib/elastic/latest/wiki_instance_proxy.rb
+++ b/ee/lib/elastic/latest/wiki_instance_proxy.rb
@@ -18,11 +18,7 @@ def elastic_search_as_wiki_page(query, page: 1, per: 20, options: {})
private
def repository_id
- if ::Elastic::DataMigrationService.migration_has_finished?(:add_suffix_project_in_wiki_rid)
- "wiki_project_#{project.id}"
- else
- "wiki_#{project.id}"
- end
+ "wiki_project_#{project.id}"
end
end
end
diff --git a/ee/spec/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid_spec.rb b/ee/spec/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid_spec.rb
index 7de27037acf26..66bdd747e828b 100644
--- a/ee/spec/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid_spec.rb
+++ b/ee/spec/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid_spec.rb
@@ -4,241 +4,5 @@
require File.expand_path('ee/elastic/migrate/20230428500000_add_suffix_project_in_wiki_rid.rb')
RSpec.describe AddSuffixProjectInWikiRid, :elastic_clean, :sidekiq_inline, feature_category: :global_search do
- let(:version) { 20230428500000 }
- let(:migration) { described_class.new(version) }
- let(:helper) { Gitlab::Elastic::Helper.new }
- let(:client) { ::Gitlab::Search::Client.new }
-
- before do
- stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
- allow(migration).to receive(:helper).and_return(helper)
- set_elasticsearch_migration_to :add_suffix_project_in_wiki_rid, including: false
- allow(migration).to receive(:client).and_return(client)
- end
-
- describe 'migration_options' do
- it 'has migration options set', :aggregate_failures do
- expect(migration).to be_batched
- expect(migration.throttle_delay).to eq(1.minute)
- expect(migration).to be_pause_indexing
- expect(migration).to be_space_requirements
- end
- end
-
- describe '.migrate' do
- context 'for batch run' do
- it 'sets migration_state task_id' do
- migration.migrate
-
- expect(migration.migration_state).to include(slice: 0, max_slices: 5)
- expect(migration.migration_state['task_id']).not_to be nil
- end
-
- it 'sets correct number of slices for 1 shard' do
- allow(migration).to receive(:get_number_of_shards).and_return(1)
-
- migration.migrate
-
- expect(migration.migration_state).to include(slice: 0, max_slices: 2)
- end
-
- it 'sets next slice and clears task_id after task check' do
- allow(migration).to receive(:reindexing_completed?).and_return(true)
-
- migration.set_migration_state(slice: 0, max_slices: 5, retry_attempt: 0, task_id: 'task_id')
-
- migration.migrate
-
- expect(migration.migration_state).to include(slice: 1, max_slices: 5, task_id: nil)
- end
-
- it 'resets retry_attempt clears task_id for the next slice' do
- allow(migration).to receive(:reindexing_completed?).and_return(true)
-
- migration.set_migration_state(slice: 0, max_slices: 5, retry_attempt: 5, task_id: 'task_id')
-
- migration.migrate
-
- expect(migration.migration_state).to match(slice: 1, max_slices: 5, retry_attempt: 0, task_id: nil)
- end
-
- context 'when reindexing is still in progress' do
- before do
- allow(migration).to receive(:reindexing_completed?).and_return(false)
- end
-
- it 'does nothing' do
- migration.set_migration_state(slice: 0, max_slices: 5, retry_attempt: 0, task_id: 'task_id')
-
- migration.migrate
-
- expect(client).not_to receive(:update_by_query)
- end
- end
-
- context 'with wikis in elastic' do
- # Create wikis on different projects to ensure they are spread across
- # all shards. If they all end up in 1 ES shard then they'll be migrated
- # in a single slice.
- let_it_be(:projects) { create_list(:project, 3, :wiki_repo, visibility_level: 0, wiki_access_level: 0) }
-
- before do
- projects.each do |project|
- project.wiki.create_page('index_page', 'Bla bla term')
- project.wiki.create_page('home_page', 'Bla bla term2')
- project.wiki.index_wiki_blobs
- end
- ensure_elasticsearch_index! # ensure objects are indexed
- end
-
- it 'migrates all wikis' do
- slices = 2
- migration.set_migration_state(slice: 0, max_slices: slices, retry_attempt: 0)
- migration.migrate
-
- 10.times do
- break if migration.completed?
-
- migration.migrate
- end
- expect(migration.completed?).to be_truthy
- expect(client.search(index: "#{es_helper.target_name}-wikis")['hits']['hits'].map do |hit|
- hit['_source']['rid'].match(/wiki_project_[0-9].*/)
- end.all?).to be true
- end
- end
- end
-
- context 'for failed run' do
- context 'if exception is raised' do
- before do
- allow(migration).to receive(:client).and_return(client)
- allow(client).to receive(:update_by_query).and_raise(StandardError)
- end
-
- it 'increases retry_attempt and clears task_id' do
- migration.set_migration_state(slice: 0, max_slices: 2, retry_attempt: 1)
-
- expect { migration.migrate }.to raise_error(StandardError)
- expect(migration.migration_state).to match(slice: 0, max_slices: 2, retry_attempt: 2, task_id: nil)
- end
-
- it 'fails the migration after too many attempts' do
- migration.set_migration_state(slice: 0, max_slices: 2, retry_attempt: 30)
-
- migration.migrate
-
- expect(migration.migration_state).to match(
- slice: 0,
- max_slices: 2,
- retry_attempt: 30,
- halted: true,
- failed: true,
- halted_indexing_unpaused: false
- )
- expect(client).not_to receive(:update_by_query)
- end
- end
-
- context 'when elasticsearch failures' do
- context 'if total is not equal' do
- before do
- allow(helper).to receive(:task_status).and_return(
- {
- "completed" => true,
- "response" => {
- "total" => 60, "updated" => 0, "created" => 45, "deleted" => 0, "failures" => []
- }
- }
- )
- end
-
- it 'raises an error and clears task_id' do
- migration.set_migration_state(slice: 0, max_slices: 2, retry_attempt: 0, task_id: 'task_id')
-
- expect { migration.migrate }.to raise_error(/total is not equal/)
- expect(migration.migration_state[:task_id]).to be_nil
- end
- end
-
- context 'when reindexing fails' do
- before do
- allow(helper).to receive(:task_status).with(task_id: 'task_id').and_return(
- {
- "completed" => true,
- "response" => {
- "total" => 60,
- "updated" => 0,
- "created" => 0,
- "deleted" => 0,
- "failures" => [
- { type: "es_rejected_execution_exception" }
- ]
- }
- }
- )
- end
-
- it 'raises an error and clears task_id' do
- migration.set_migration_state(slice: 0, max_slices: 2, retry_attempt: 0, task_id: 'task_id')
-
- expect { migration.migrate }.to raise_error(/failed with/)
- expect(migration.migration_state[:task_id]).to be_nil
- end
- end
- end
- end
- end
-
- describe '.completed?' do
- subject { migration.completed? }
-
- let_it_be(:project) { create(:project, :wiki_repo, visibility_level: 0, wiki_access_level: 0) }
-
- before do
- project.wiki.create_page('index_page', 'Bla bla term')
- project.wiki.index_wiki_blobs
- ensure_elasticsearch_index! # ensure objects are indexed
- end
-
- context 'when there are no items which are missing project prefix in rid' do
- before do
- client.update_by_query(index: Elastic::Latest::WikiConfig.index_name,
- body: {
- script: { lang: 'painless',
- source: "ctx._source.rid = ctx._source.rid.replace('wiki', 'wiki_project')" }
- }
- )
- end
-
- it 'returns true' do
- is_expected.to be_truthy
- end
- end
-
- context 'when some items are missing project prefix in rid' do
- before do
- client.update_by_query(index: Elastic::Latest::WikiConfig.index_name,
- body: {
- script: { lang: 'painless',
- source: "ctx._source.rid = ctx._source.rid.replace('wiki_project', 'wiki')" }
- }
- )
- end
-
- it 'returns false' do
- is_expected.to be_falsey
- end
- end
- end
-
- describe 'space_required_bytes' do
- subject { migration.space_required_bytes }
-
- before do
- allow(helper).to receive(:index_size_bytes).and_return(300)
- end
-
- it { is_expected.to eq(3) }
- end
+ it_behaves_like 'a deprecated Advanced Search migration', 20230428500000
end
diff --git a/ee/spec/lib/elastic/latest/git_class_proxy_add_suffix_project_in_wiki_rid_running_wiki_search_spec.rb b/ee/spec/lib/elastic/latest/git_class_proxy_add_suffix_project_in_wiki_rid_running_wiki_search_spec.rb
deleted file mode 100644
index eac9bcb98e57a..0000000000000
--- a/ee/spec/lib/elastic/latest/git_class_proxy_add_suffix_project_in_wiki_rid_running_wiki_search_spec.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-# This spec file can be removed once the migration add_suffix_project_in_wiki_rid is deprecated
-require 'spec_helper'
-
-RSpec.describe Elastic::Latest::GitClassProxy, :elastic, :sidekiq_inline, feature_category: :global_search do
- let_it_be(:wiki_project) { create(:project, :wiki_repo) }
- let(:included_class) { Elastic::Latest::WikiClassProxy }
-
- subject { included_class.new(wiki_project.wiki.class, use_separate_indices: ProjectWiki.use_separate_indices?) }
-
- it 'fetches the results considering new and old format of rid' do
- stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
- set_elasticsearch_migration_to(:add_suffix_project_in_wiki_rid, including: false)
-
- wiki_project.wiki.create_page('home_page', 'Bla bla term')
- wiki_project.wiki.create_page('home_page2', 'Bla bla term')
- wiki_project.wiki.index_wiki_blobs
-
- ensure_elasticsearch_index!
-
- # Remove the project from the rid in one document
- # It simulates a situtation where rid of some wiki blobs are already updated
- remove_project_from_wiki_blob_rid
-
- options = { repository_id: "wiki_project_#{wiki_project.id}" }
-
- results = subject.elastic_search('Bla', type: 'wiki_blob', options: options)[:wiki_blobs][:results]
-
- expect(results.total).to eq(2)
- result_rids = results.as_json.map { |r| r['_source']['rid'] }
- expect(result_rids).to include("wiki_project_#{wiki_project.id}", "wiki_#{wiki_project.id}")
- end
-
- def remove_project_from_wiki_blob_rid
- Project.__elasticsearch__.client.update_by_query({
- index: Elastic::Latest::WikiConfig.index_name, refresh: true, max_docs: 1,
- body: {
- script: {
- lang: 'painless',
- source: "ctx._source.rid = ctx._source.rid.replace('wiki_project', 'wiki')"
- },
- query: {
- regexp: {
- rid: "wiki_project_[0-9].*"
- }
- }
- }
- })
- end
-end
diff --git a/ee/spec/lib/elastic/latest/wiki_instance_proxy_spec.rb b/ee/spec/lib/elastic/latest/wiki_instance_proxy_spec.rb
index be9ca49fef62e..b606668a2cbd0 100644
--- a/ee/spec/lib/elastic/latest/wiki_instance_proxy_spec.rb
+++ b/ee/spec/lib/elastic/latest/wiki_instance_proxy_spec.rb
@@ -18,7 +18,7 @@
it 'provides repository_id if not provided' do
expected_params = params.deep_dup
- expected_params[:options][:repository_id] = "wiki_#{project.id}"
+ expected_params[:options][:repository_id] = "wiki_project_#{project.id}"
expect(subject.class).to receive(:elastic_search_as_wiki_page).with('foo', expected_params)
--
GitLab