From ae48298c3089d74cd26cdabaeb2a51937b60f69a Mon Sep 17 00:00:00 2001
From: Terri Chu <tchu@gitlab.com>
Date: Wed, 24 Jul 2024 18:58:33 +0000
Subject: [PATCH] Revert "Merge branch '470138-reindex-issues' into 'master'"
This reverts merge request !158039
---
.../migration_database_backfill_helper.rb | 22 +++++--------
...9110316_index_all_issues_from_database.yml | 10 ------
...09110316_index_all_issues_from_database.rb | 26 ----------------
ee/lib/elastic/latest/issue_instance_proxy.rb | 2 +-
...316_index_all_issues_from_database_spec.rb | 31 -------------------
.../elastic/migration_shared_examples.rb | 1 +
6 files changed, 10 insertions(+), 82 deletions(-)
delete mode 100644 ee/elastic/docs/20240609110316_index_all_issues_from_database.yml
delete mode 100644 ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb
delete mode 100644 ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb
diff --git a/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb b/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb
index 9ccc1d85858c3..c0830e211e9e6 100644
--- a/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb
+++ b/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb
@@ -16,19 +16,13 @@ def migrate
end
def completed?
- completed = documents_after_current_id.empty?
+ maximum_id = documents_after_current_id.maximum(:id).to_i
+ documents_remaining_approximate = maximum_id - current_id
+ set_migration_state(maximum_id: maximum_id, documents_remaining_approximate: documents_remaining_approximate)
+ log 'Checking if migration is finished', maximum_id: maximum_id, current_id: current_id,
+ documents_remaining_approximate: documents_remaining_approximate
- unless completed
- maximum_id = documents_after_current_id.maximum(:id).to_i
- documents_remaining_approximate = maximum_id - current_id
-
- set_migration_state(maximum_id: maximum_id, documents_remaining_approximate: documents_remaining_approximate)
-
- log 'Migration is not finished', maximum_id: maximum_id, current_id: current_id,
- documents_remaining_approximate: documents_remaining_approximate
- end
-
- completed
+ documents_after_current_id.empty?
end
def document_type
@@ -55,7 +49,7 @@ def limit_per_iteration
DEFAULT_LIMIT_PER_ITERATION
end
- def number_of_iterations_per_run
+ def number_of_iteration_per_run
(batch_size / limit_per_iteration.to_f).ceil
end
@@ -69,7 +63,7 @@ def documents_after_current_id
def backfill_documents
[].tap do |documents|
- number_of_iterations_per_run.times do
+ number_of_iteration_per_run.times do
documents = documents_after_current_id.limit(limit_per_iteration)
if limit_indexing?
diff --git a/ee/elastic/docs/20240609110316_index_all_issues_from_database.yml b/ee/elastic/docs/20240609110316_index_all_issues_from_database.yml
deleted file mode 100644
index 5bb64e510f09d..0000000000000
--- a/ee/elastic/docs/20240609110316_index_all_issues_from_database.yml
+++ /dev/null
@@ -1,10 +0,0 @@
----
-name: IndexAllIssuesFromDatabase
-version: '20240609110316'
-description: Indexes every issue from the database to ensure the correct state
-group: group::global search
-milestone: '17.3'
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/158039
-obsolete: false
-marked_obsolete_by_url:
-marked_obsolete_in_milestone:
diff --git a/ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb b/ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb
deleted file mode 100644
index 4533a9cd6b97f..0000000000000
--- a/ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class IndexAllIssuesFromDatabase < Elastic::Migration
- include ::Search::Elastic::MigrationDatabaseBackfillHelper
-
- batch_size 50_000
- batched!
- throttle_delay 1.minute
- retry_on_failure
-
- DOCUMENT_TYPE = Issue
-
- def respect_limited_indexing?
- true
- end
-
- def item_to_preload
- :project
- end
-
- # rubocop:disable CodeReuse/ActiveRecord -- we need to select only unprocessed ids
- def documents_after_current_id
- document_type.without_issue_type(:epic).where("issues.id > ?", current_id).order(:id)
- end
- # rubocop:enable CodeReuse/ActiveRecord
-end
diff --git a/ee/lib/elastic/latest/issue_instance_proxy.rb b/ee/lib/elastic/latest/issue_instance_proxy.rb
index a0d7badec2d09..3be726395d2d1 100644
--- a/ee/lib/elastic/latest/issue_instance_proxy.rb
+++ b/ee/lib/elastic/latest/issue_instance_proxy.rb
@@ -3,7 +3,7 @@
module Elastic
module Latest
class IssueInstanceProxy < ApplicationInstanceProxy
- SCHEMA_VERSION = 24_07
+ SCHEMA_VERSION = 24_05
OPTIONAL_FIELDS = %w[embedding embedding_version].freeze
def as_indexed_json(options = {})
diff --git a/ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb b/ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb
deleted file mode 100644
index b165b2013e3a5..0000000000000
--- a/ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require File.expand_path('ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb')
-
-RSpec.describe IndexAllIssuesFromDatabase, feature_category: :global_search do
- let(:version) { 20240609110316 }
-
- describe 'migration', :elastic do
- it_behaves_like 'migration reindexes all data' do
- let(:objects) { create_list(:issue, 3) }
- let(:expected_throttle_delay) { 1.minute }
- let(:expected_batch_size) { 50_000 }
- end
- end
-
- describe '.documents_after_current_id' do
- let(:migration) { described_class.new(version) }
- let_it_be(:project) { create(:project, :public) }
- let_it_be(:issue) { create(:issue, project: project) }
- let_it_be(:issue_epic_type) { create(:issue, :epic) }
- let_it_be(:issue_task_type) { create(:issue, :task) }
- let_it_be(:work_item) { create(:work_item, :epic_with_legacy_epic, :group_level) }
- let_it_be(:non_group_work_item) { create(:work_item) }
-
- it 'only indexes project-level work_item_type issues' do
- expected_ids = [issue.id, issue_task_type.id, non_group_work_item.id]
- expect(migration.documents_after_current_id.pluck(:id)).to match_array(expected_ids)
- end
- end
-end
diff --git a/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb b/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb
index 187c985a93d4b..f8293a73c7492 100644
--- a/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb
+++ b/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb
@@ -730,6 +730,7 @@ def update_by_query(objects, script)
migration.migrate # To set a pristine state
expect(migration.completed?).to be_truthy
+ expect(migration.migration_state).to match(task_id: nil, documents_remaining: 0)
end
context 'and task in progress' do
--
GitLab