diff --git a/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb b/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb
index c0830e211e9e64ca46941ce81c66f6d6e33ed6f4..9ccc1d85858c3934bed6e956f13f141d0f2b8454 100644
--- a/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb
+++ b/ee/app/workers/concerns/search/elastic/migration_database_backfill_helper.rb
@@ -16,13 +16,19 @@ def migrate
end
def completed?
- maximum_id = documents_after_current_id.maximum(:id).to_i
- documents_remaining_approximate = maximum_id - current_id
- set_migration_state(maximum_id: maximum_id, documents_remaining_approximate: documents_remaining_approximate)
- log 'Checking if migration is finished', maximum_id: maximum_id, current_id: current_id,
- documents_remaining_approximate: documents_remaining_approximate
+ completed = documents_after_current_id.empty?
- documents_after_current_id.empty?
+ unless completed
+ maximum_id = documents_after_current_id.maximum(:id).to_i
+ documents_remaining_approximate = maximum_id - current_id
+
+ set_migration_state(maximum_id: maximum_id, documents_remaining_approximate: documents_remaining_approximate)
+
+ log 'Migration is not finished', maximum_id: maximum_id, current_id: current_id,
+ documents_remaining_approximate: documents_remaining_approximate
+ end
+
+ completed
end
def document_type
@@ -49,7 +55,7 @@ def limit_per_iteration
DEFAULT_LIMIT_PER_ITERATION
end
- def number_of_iteration_per_run
+ def number_of_iterations_per_run
(batch_size / limit_per_iteration.to_f).ceil
end
@@ -63,7 +69,7 @@ def documents_after_current_id
def backfill_documents
[].tap do |documents|
- number_of_iteration_per_run.times do
+ number_of_iterations_per_run.times do
documents = documents_after_current_id.limit(limit_per_iteration)
if limit_indexing?
diff --git a/ee/elastic/docs/20240609110316_index_all_issues_from_database.yml b/ee/elastic/docs/20240609110316_index_all_issues_from_database.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5bb64e510f09d1d9318c7a63ac59c83b84792c2a
--- /dev/null
+++ b/ee/elastic/docs/20240609110316_index_all_issues_from_database.yml
@@ -0,0 +1,10 @@
+---
+name: IndexAllIssuesFromDatabase
+version: '20240609110316'
+description: Indexes every issue from the database to ensure the correct state
+group: group::global search
+milestone: '17.3'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/158039
+obsolete: false
+marked_obsolete_by_url:
+marked_obsolete_in_milestone:
diff --git a/ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb b/ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4533a9cd6b97f41eb2330d5d6b8d6eb2006989e8
--- /dev/null
+++ b/ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class IndexAllIssuesFromDatabase < Elastic::Migration
+ include ::Search::Elastic::MigrationDatabaseBackfillHelper
+
+ batch_size 50_000
+ batched!
+ throttle_delay 1.minute
+ retry_on_failure
+
+ DOCUMENT_TYPE = Issue
+
+ def respect_limited_indexing?
+ true
+ end
+
+ def item_to_preload
+ :project
+ end
+
+ # rubocop:disable CodeReuse/ActiveRecord -- we need to select only unprocessed ids
+ def documents_after_current_id
+ document_type.without_issue_type(:epic).where("issues.id > ?", current_id).order(:id)
+ end
+ # rubocop:enable CodeReuse/ActiveRecord
+end
diff --git a/ee/lib/elastic/latest/issue_instance_proxy.rb b/ee/lib/elastic/latest/issue_instance_proxy.rb
index 3be726395d2d1115ba37a08c5081df8f109aa4d6..a0d7badec2d09ff3cdec125836e857f74d2a6231 100644
--- a/ee/lib/elastic/latest/issue_instance_proxy.rb
+++ b/ee/lib/elastic/latest/issue_instance_proxy.rb
@@ -3,7 +3,7 @@
module Elastic
module Latest
class IssueInstanceProxy < ApplicationInstanceProxy
- SCHEMA_VERSION = 24_05
+ SCHEMA_VERSION = 24_07
OPTIONAL_FIELDS = %w[embedding embedding_version].freeze
def as_indexed_json(options = {})
diff --git a/ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb b/ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b165b2013e3a5b7242f7d28ed8beb816c6d2b292
--- /dev/null
+++ b/ee/spec/elastic/migrate/20240609110316_index_all_issues_from_database_spec.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require File.expand_path('ee/elastic/migrate/20240609110316_index_all_issues_from_database.rb')
+
+RSpec.describe IndexAllIssuesFromDatabase, feature_category: :global_search do
+ let(:version) { 20240609110316 }
+
+ describe 'migration', :elastic do
+ it_behaves_like 'migration reindexes all data' do
+ let(:objects) { create_list(:issue, 3) }
+ let(:expected_throttle_delay) { 1.minute }
+ let(:expected_batch_size) { 50_000 }
+ end
+ end
+
+ describe '.documents_after_current_id' do
+ let(:migration) { described_class.new(version) }
+ let_it_be(:project) { create(:project, :public) }
+ let_it_be(:issue) { create(:issue, project: project) }
+ let_it_be(:issue_epic_type) { create(:issue, :epic) }
+ let_it_be(:issue_task_type) { create(:issue, :task) }
+ let_it_be(:work_item) { create(:work_item, :epic_with_legacy_epic, :group_level) }
+ let_it_be(:non_group_work_item) { create(:work_item) }
+
+ it 'only indexes project-level work_item_type issues' do
+ expected_ids = [issue.id, issue_task_type.id, non_group_work_item.id]
+ expect(migration.documents_after_current_id.pluck(:id)).to match_array(expected_ids)
+ end
+ end
+end
diff --git a/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb b/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb
index f8293a73c7492ea091473abed45ddd852b8448da..187c985a93d4bcce1bca8d715fd3bf7810317379 100644
--- a/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb
+++ b/ee/spec/support/shared_examples/elastic/migration_shared_examples.rb
@@ -730,7 +730,6 @@ def update_by_query(objects, script)
migration.migrate # To set a pristine state
expect(migration.completed?).to be_truthy
- expect(migration.migration_state).to match(task_id: nil, documents_remaining: 0)
end
context 'and task in progress' do