diff --git a/ee/elastic/docs/20230518064300_backfill_project_permissions_in_blobs.yml b/ee/elastic/docs/20230518064300_backfill_project_permissions_in_blobs.yml
index e61da5a151ba7264f73222c389e53090af7f760f..61b5a8f8c9d0c176f4a77864c97fca02961032e5 100644
--- a/ee/elastic/docs/20230518064300_backfill_project_permissions_in_blobs.yml
+++ b/ee/elastic/docs/20230518064300_backfill_project_permissions_in_blobs.yml
@@ -5,6 +5,6 @@ description: BackfillProjectPermissionsInBlobs
group: group::global search
milestone: '16.1'
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121077
-obsolete: false
-marked_obsolete_by_url:
-marked_obsolete_in_milestone:
+obsolete: true
+marked_obsolete_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/152982
+marked_obsolete_in_milestone: '17.0'
diff --git a/ee/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs.rb b/ee/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs.rb
index 310a3ed1525d6fbfe9b2988ad1d117732db81a58..951549531b60f92174dfa73a9c3cb6bfcfdd2c19 100644
--- a/ee/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs.rb
+++ b/ee/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs.rb
@@ -189,3 +189,5 @@ def determine_project_limit
[get_number_of_shards(index_name: helper.target_name), max_projects_to_process].min
end
end
+
+BackfillProjectPermissionsInBlobs.prepend ::Elastic::MigrationObsolete
diff --git a/ee/lib/elastic/latest/git_class_proxy.rb b/ee/lib/elastic/latest/git_class_proxy.rb
index b9a994b0650f7d34b5124f664522ecf69d60e022..0b11132e956db2ced7abf3ec48d33d0e291ca2b4 100644
--- a/ee/lib/elastic/latest/git_class_proxy.rb
+++ b/ee/lib/elastic/latest/git_class_proxy.rb
@@ -382,7 +382,7 @@ def archived_filter_applicable_for_blob_search?(options)
end
def disable_project_joins_for_blob?
- Elastic::DataMigrationService.migration_has_finished?(:backfill_project_permissions_in_blobs)
+ true
end
end
end
diff --git a/ee/spec/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs_spec.rb b/ee/spec/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs_spec.rb
index b699565d0f2e511096e9f653420424178231a60f..968c5a3c3b2814379eadee6997a8e76fcace6229 100644
--- a/ee/spec/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs_spec.rb
+++ b/ee/spec/elastic/migrate/20230518064300_backfill_project_permissions_in_blobs_spec.rb
@@ -5,226 +5,5 @@
RSpec.describe BackfillProjectPermissionsInBlobs, :elastic_clean, :sidekiq_inline,
feature_category: :global_search do
- let(:version) { 20230518064300 }
- let(:helper) { Gitlab::Elastic::Helper.new }
- let(:migration) { described_class.new(version) }
-
- let_it_be_with_reload(:projects) { create_list(:project, 3, :repository) }
-
- before do
- stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
- set_elasticsearch_migration_to(version, including: false)
-
- allow(migration).to receive(:helper).and_return(helper)
- end
-
- describe 'migration_options' do
- it 'has migration options set', :aggregate_failures do
- expect(migration).to be_batched
- expect(migration).to be_retry_on_failure
- expect(migration.throttle_delay).to eq(5.seconds)
- expect(migration.batch_size).to eq(10_000)
- end
- end
-
- describe '.migrate' do
- context 'with project permissions in all projects' do
- it 'does not execute update_by_query' do
- projects = create_list(:project, 2, :repository)
- projects.each { |p| p.repository.index_commits_and_blobs }
-
- ensure_elasticsearch_index!
-
- migration.migrate
-
- expect(migration).to be_completed
- expect(helper.client).not_to receive(:update_by_query)
- end
- end
-
- context 'when task in progress' do
- let(:project_in_progress) { Project.first }
-
- before do
- projects.each do |project|
- project.repository.index_commits_and_blobs # ensure objects are indexed
- end
- ensure_elasticsearch_index!
- projects.each { |project| remove_permissions_for_blob(project) }
- allow(migration).to receive(:completed?).and_return(false)
- allow(helper).to receive(:task_status).with(task_id: 'task_1').and_return('completed' => false)
- migration.set_migration_state(
- projects_in_progress: [{ task_id: 'task_1',
- project_id: project_in_progress.id }], remaining_count: 3)
- end
-
- it 'does not send update_by_query to the project in progress' do
- expect(migration).to receive(:update_by_query).with(Project.second).and_call_original
- expect(migration).to receive(:update_by_query).with(Project.last).and_call_original
- expect(migration).not_to receive(:update_by_query).with(project_in_progress)
- migration.migrate
- end
- end
-
- context 'with project not found exception' do
- let(:client) { ::Gitlab::Search::Client.new }
-
- before do
- allow(migration).to receive(:client).and_return(client)
- allow(migration).to receive(:projects_with_missing_project_permissions).and_return([0])
- allow(migration).to receive(:completed?).and_return(false)
- end
-
- it 'schedules ElasticDeleteProjectWorker when project is not found' do
- expect(ElasticDeleteProjectWorker).to receive(:perform_async).with(0, "project_0")
- migration.migrate
- end
- end
-
- context 'when migration fails' do
- let(:client) { ::Gitlab::Search::Client.new }
-
- before do
- allow(client).to receive(:update_by_query).and_return(update_by_query_response)
- allow(helper).to receive(:task_status).with(task_id: 'task_1').and_return(task_status_response)
-
- allow(migration).to receive(:projects_with_missing_project_permissions).and_return(projects.map(&:id))
- allow(migration).to receive(:completed?).and_return(false)
- allow(migration).to receive(:client).and_return(client)
- end
-
- context 'when Elasticsearch responds with errors' do
- context 'when a task throws an error' do
- let(:task_status_response) { { 'error' => ['failed'] } }
- let(:update_by_query_response) { { 'task' => 'task_1' } }
-
- it 'removes entry from projects_in_progress in migration_state' do
- migration_state = projects.map { |p| { task_id: 'task_1', project_id: p.id } }
- migration.set_migration_state(projects_in_progress: migration_state)
- expect(migration).to receive(:set_migration_state).with(projects_in_progress: [], remaining_count: 0)
- expect(migration).to receive(:set_migration_state).with(projects_in_progress: migration_state,
- remaining_count: 0)
-
- expect { migration.migrate }.not_to raise_error
-
- expect(migration.migration_state[:projects_in_progress]).to match_array(migration_state)
- end
- end
-
- context 'when update_by_query throws an error' do
- let(:task_status_response) { {} }
- let(:update_by_query_response) { { 'failures' => ['failed'] } }
-
- it 'removes entry from projects_in_progress in migration_state' do
- migration.set_migration_state({}) # simulate first run
-
- expect { migration.migrate }.not_to raise_error
- expect(migration.migration_state).to match({ projects_in_progress: [], remaining_count: 0 })
- end
- end
- end
- end
- end
-
- describe 'when Elasticsearch gives 404', :elastic_clean do
- context 'when Elasticsearch responds with NotFoundException' do
- let(:client) { ::Gitlab::Search::Client.new }
- let(:update_by_query_response) { { 'failures' => ['failed'] } }
-
- before do
- allow(client).to receive(:update_by_query).and_return(update_by_query_response)
-
- allow(migration).to receive(:projects_with_missing_project_permissions).and_return(projects.map(&:id))
- allow(migration).to receive(:completed?).and_return(false)
- allow(migration).to receive(:client).and_return(client)
- end
-
- context 'when a task_status throws a NotFound Exception' do
- it 'removes entry from projects_in_progress in migration_state' do
- migration_state = projects.map { |p| { task_id: 'oTUltX4IQMOUUVeiohTt8A:124', project_id: p.id } }
- migration.set_migration_state(projects_in_progress: migration_state, remaining_count: 1)
- expect(migration).to receive(:set_migration_state).with(projects_in_progress: [], remaining_count: 0).twice
-
- expect { migration.migrate }.not_to raise_error
-
- expect(migration.migration_state[:projects_in_progress]).to match_array(migration_state)
- end
- end
- end
- end
-
- describe 'integration test', :elastic_clean do
- before do
- set_elasticsearch_migration_to(version, including: false)
-
- projects.each do |project|
- project.repository.index_commits_and_blobs # ensure objects are indexed
- end
- ensure_elasticsearch_index!
- projects.each { |project| remove_permissions_for_blob(project) }
- set_elasticsearch_migration_to(version, including: false)
- end
-
- it 'updates documents in batches' do
- # calculate how many files are in each project in the index
- query = { bool: { must: [{ term: { project_id: projects.first.id } }, { terms: { type: %w[blob] } }] } }
- file_count = helper.client.count(index: helper.target_name, body: { query: query })['count']
- allow(migration).to receive(:batch_size).and_return((file_count / 2.to_f).ceil)
- allow(migration).to receive(:max_projects_to_process).and_return(2)
- expect(migration).not_to be_completed
- expect(migration).to receive(:update_by_query).exactly(6).times.and_call_original
-
- expected_migration_project_ids = projects.pluck(:id)
-
- migrate_batch_of_projects(migration)
-
- counter = 0
- while counter < 10 && migration.migration_state[:projects_in_progress].present?
- actual_project_ids = migration.migration_state[:projects_in_progress].pluck(:project_id)
- expect(expected_migration_project_ids).to include(*actual_project_ids)
-
- migrate_batch_of_projects(migration)
- counter += 1
- end
-
- expect(migration).to be_completed
- end
- end
-
- def migrate_batch_of_projects(migration)
- old_migration_state = migration.migration_state[:projects_in_progress]
- 10.times do |_| # Max 0.1s waiting
- migration.migrate
- break if old_migration_state != migration.migration_state[:projects_in_progress]
-
- sleep 0.01
- end
- end
-
- def remove_permissions_for_blob(project)
- source = "ctx._source.remove('visibility_level');ctx._source.remove('repository_access_level')"
- update_by_query project, source
- end
-
- def update_by_query(project, source)
- Repository.__elasticsearch__.client.update_by_query({
- index: Repository.__elasticsearch__.index_name,
- wait_for_completion: true,
- refresh: true,
- body: {
- query: {
- bool: {
- filter: [
- { term: { project_id: project.id } },
- { term: { type: 'blob' } }
- ]
- }
- },
- script: {
- lang: "painless",
- source: source
- }
- }
- })
- end
+ it_behaves_like 'a deprecated Advanced Search migration', 20230518064300
end
diff --git a/ee/spec/lib/elastic/latest/git_class_proxy_spec.rb b/ee/spec/lib/elastic/latest/git_class_proxy_spec.rb
index 2b18b346742f3c5c6804d05260eee3558f743c82..bec7c76392cab37ba0b6a555ba5d08918d2ab59e 100644
--- a/ee/spec/lib/elastic/latest/git_class_proxy_spec.rb
+++ b/ee/spec/lib/elastic/latest/git_class_proxy_spec.rb
@@ -36,17 +36,6 @@
subject.elastic_search('*', type: 'blob', options: search_options)
assert_named_queries('doc:is_a:blob', 'blob:authorized:project', 'blob:match:search_terms')
end
-
- context 'when backfill_project_permissions_in_blobs migration is not finished' do
- before do
- set_elasticsearch_migration_to(:backfill_project_permissions_in_blobs, including: false)
- end
-
- it 'uses the correct elasticsearch query' do
- subject.elastic_search('*', type: 'blob', options: search_options)
- assert_named_queries('doc:is_a:blob', 'blob:authorized:project:parent', 'blob:match:search_terms')
- end
- end
end
context 'when performing a group search' do
@@ -394,20 +383,6 @@
'blob:match:search_terms')
end
- it 'assert names queries for global blob search when migration is not complete' do
- search_options = {
- current_user: user,
- public_and_internal_projects: true,
- order_by: nil,
- sort: nil
- }
- set_elasticsearch_migration_to(:backfill_project_permissions_in_blobs, including: false)
-
- subject.blob_aggregations('*', search_options)
- assert_named_queries('doc:is_a:blob', 'blob:authorized:project:parent',
- 'blob:match:search_terms')
- end
-
it 'assert names queries for group blob search' do
group_search_options = {
current_user: user,
diff --git a/ee/spec/lib/peek/views/elasticsearch_spec.rb b/ee/spec/lib/peek/views/elasticsearch_spec.rb
index 3fdd9689ad924c57b6fc693d36117db25acf4a45..efe5c4d8b692531269868b330a29e7636957d367 100644
--- a/ee/spec/lib/peek/views/elasticsearch_spec.rb
+++ b/ee/spec/lib/peek/views/elasticsearch_spec.rb
@@ -10,8 +10,6 @@
::Gitlab::Instrumentation::ElasticsearchTransport.detail_store # Create store in redis
allow(::Gitlab::PerformanceBar).to receive(:enabled_for_request?).and_return(true)
ensure_elasticsearch_index!
-
- set_elasticsearch_migration_to(:backfill_project_permissions_in_blobs)
end
describe '#results' do
@@ -25,11 +23,12 @@
expect(results[:calls]).to be > 0
expect(results[:duration]).to be_kind_of(String)
- expect(results[:details].last[:method]).to eq('POST')
- expect(results[:details].last[:path]).to eq('gitlab-test/_search')
- expect(results[:details].last[:params]).to eq({ routing: "project_#{project.id}", timeout: timeout })
-
- expect(results[:details].last[:request]).to eq("POST gitlab-test/_search?routing=project_#{project.id}&timeout=#{timeout}")
+ expect(results[:details]).to include(hash_including({
+ method: 'POST',
+ path: 'gitlab-test/_search',
+ params: { routing: "project_#{project.id}", timeout: timeout },
+ request: "POST gitlab-test/_search?routing=project_#{project.id}&timeout=#{timeout}"
+ }))
end
end
end