diff --git a/GITLAB_ELASTICSEARCH_INDEXER_VERSION b/GITLAB_ELASTICSEARCH_INDEXER_VERSION
index 43270543f7786f5b0a89f03caa73843633aecf22..7e7f33c2e387472f8ead1e579e675455d6b52b6d 100644
--- a/GITLAB_ELASTICSEARCH_INDEXER_VERSION
+++ b/GITLAB_ELASTICSEARCH_INDEXER_VERSION
@@ -1 +1 @@
-4.3.6
+4.3.7
diff --git a/ee/app/services/ee/search/project_service.rb b/ee/app/services/ee/search/project_service.rb
index b9ce0d01d7f8a731cb408348271ac4b6750d666b..c80f3bd12e2c8a1bcde883442e460c7798d14c3d 100644
--- a/ee/app/services/ee/search/project_service.rb
+++ b/ee/app/services/ee/search/project_service.rb
@@ -20,11 +20,14 @@ def execute
sort = params[:sort]
if project.is_a?(Array)
- project_ids = Array(project).map(&:id)
+ project_id_root_ancestor_id_hash = project.to_h { |p| [p.id, p.root_ancestor.id] }
+ project_ids = project_id_root_ancestor_id_hash.keys
+ root_ancestor_ids = project_id_root_ancestor_id_hash.values
::Gitlab::Elastic::SearchResults.new(
current_user,
search,
project_ids,
+ root_ancestor_ids: root_ancestor_ids,
public_and_internal_projects: false,
order_by: order_by,
sort: sort,
@@ -35,6 +38,7 @@ def execute
current_user,
search,
project: project,
+ root_ancestor_ids: [project.root_ancestor.id],
repository_ref: repository_ref,
order_by: order_by,
sort: sort,
diff --git a/ee/app/workers/elastic_delete_project_worker.rb b/ee/app/workers/elastic_delete_project_worker.rb
index 58318eddb380a307dbfe11a4708af831ebe8fd13..89972e8d726042054aa5c91534a39f3bf8c295f7 100644
--- a/ee/app/workers/elastic_delete_project_worker.rb
+++ b/ee/app/workers/elastic_delete_project_worker.rb
@@ -13,14 +13,13 @@ class ElasticDeleteProjectWorker
def perform(project_id, es_id)
remove_project_and_children_documents(project_id, es_id)
+ helper.remove_wikis_from_the_standalone_index(project_id, 'Project') # Wikis have different routing that's why one more query is needed.
IndexStatus.for_project(project_id).delete_all
end
private
def indices
- helper = Gitlab::Elastic::Helper.default
-
# some standalone indices may not be created yet if pending advanced search migrations exist
standalone_indices = helper.standalone_indices_proxies.select do |klass|
alias_name = helper.klass_to_alias_name(klass: klass)
@@ -31,7 +30,7 @@ def indices
end
def remove_project_and_children_documents(project_id, es_id)
- Gitlab::Elastic::Helper.default.client.delete_by_query({
+ helper.client.delete_by_query({
index: indices,
routing: es_id,
body: {
@@ -70,4 +69,8 @@ def remove_project_and_children_documents(project_id, es_id)
}
})
end
+
+ def helper
+ Gitlab::Elastic::Helper.default
+ end
end
diff --git a/ee/app/workers/search/wiki/elastic_delete_group_wiki_worker.rb b/ee/app/workers/search/wiki/elastic_delete_group_wiki_worker.rb
index 3bb45601eb8a96fdaa5eacf8fd6c92f273eb7573..0cf29b901528d5e38bb71c92da78f180b209c076 100644
--- a/ee/app/workers/search/wiki/elastic_delete_group_wiki_worker.rb
+++ b/ee/app/workers/search/wiki/elastic_delete_group_wiki_worker.rb
@@ -16,30 +16,7 @@ class ElasticDeleteGroupWikiWorker
idempotent!
def perform(group_id)
- remove_group_wiki_documents(group_id)
- end
-
- private
-
- def remove_group_wiki_documents(group_id)
- Gitlab::Elastic::Helper.default.client.delete_by_query(
- {
- index: Elastic::Latest::WikiConfig.index_name,
- routing: "group_#{group_id}",
- conflicts: 'proceed',
- body: {
- query: {
- bool: {
- filter: {
- term: {
- rid: "wiki_group_#{group_id}"
- }
- }
- }
- }
- }
- }
- )
+ Gitlab::Elastic::Helper.default.remove_wikis_from_the_standalone_index(group_id, 'Group')
end
end
end
diff --git a/ee/elastic/docs/20230720000000_reindex_wikis_to_fix_routing.yml b/ee/elastic/docs/20230720000000_reindex_wikis_to_fix_routing.yml
new file mode 100644
index 0000000000000000000000000000000000000000..28f9669c802a2249bf895eeb06ca96778afc948c
--- /dev/null
+++ b/ee/elastic/docs/20230720000000_reindex_wikis_to_fix_routing.yml
@@ -0,0 +1,10 @@
+---
+name: ReindexWikisToFixRouting
+version: '20230720000000'
+description: Change the wiki docs routing from project_{id} or group_{id} to n_{id}
+group: group::global search
+milestone: 16.3
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/125137
+obsolete: false
+marked_obsolete_by_url:
+marked_obsolete_in_milestone:
diff --git a/ee/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing.rb b/ee/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e205a4e5606861575ae32086eef91736289f4fde
--- /dev/null
+++ b/ee/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class ReindexWikisToFixRouting < Elastic::Migration
+ include Elastic::MigrationHelper
+
+ batched!
+ throttle_delay 5.minutes
+ batch_size 200
+ retry_on_failure
+
+ ELASTIC_TIMEOUT = '5m'
+ SCHEMA_VERSION = 23_07
+
+ def migrate
+ if completed?
+ log 'Migration Completed', total_remaining: remaining_documents_count
+ return
+ end
+
+ remaining_rids_to_reindex.each do |rid|
+ match = rid.match(/wiki_(project|group)_(\d+)/)
+ ElasticWikiIndexerWorker.perform_in(rand(throttle_delay).seconds, match[2], match[1].capitalize, force: true)
+ end
+ end
+
+ def completed?
+ total_remaining = remaining_documents_count
+ set_migration_state(documents_remaining: total_remaining)
+ log('Checking if migration is finished', total_remaining: total_remaining)
+ total_remaining == 0
+ end
+
+ private
+
+ def remaining_rids_to_reindex
+ results = client.search(
+ index: index_name,
+ body: {
+ size: 0, query: query_with_old_schema_version, aggs: { rids: { terms: { size: batch_size, field: 'rid' } } }
+ }
+ )
+ rids_hist = results.dig('aggregations', 'rids', 'buckets') || []
+ rids_hist.pluck('key') # rubocop: disable CodeReuse/ActiveRecord
+ end
+
+ def remaining_documents_count
+ helper.refresh_index(index_name: index_name)
+ client.count(index: index_name, body: { query: query_with_old_schema_version })['count']
+ end
+
+ def query_with_old_schema_version
+ { range: { schema_version: { lt: SCHEMA_VERSION } } }
+ end
+
+ def index_name
+ Elastic::Latest::WikiConfig.index_name
+ end
+end
diff --git a/ee/lib/elastic/latest/git_instance_proxy.rb b/ee/lib/elastic/latest/git_instance_proxy.rb
index 5f01a8af06aa06e86334a42bc9fbe2afe0aac843..d850705a04a5547dbcfe767055f4b6559b7f85bf 100644
--- a/ee/lib/elastic/latest/git_instance_proxy.rb
+++ b/ee/lib/elastic/latest/git_instance_proxy.rb
@@ -11,8 +11,11 @@ def methods_for_all_write_targets
end
end
- def es_parent
- project_id ? "project_#{project_id}" : "group_#{group_id}"
+ def es_parent(is_wiki = false)
+ return "project_#{project_id}" unless is_wiki
+ return unless ::Elastic::DataMigrationService.migration_has_finished?(:reindex_wikis_to_fix_routing)
+
+ "n_#{project ? project.root_ancestor.id : group.root_ancestor.id}"
end
def elastic_search(query, type: 'all', page: 1, per: 20, options: {})
@@ -32,28 +35,28 @@ def blob_aggregations(query, options)
self.class.blob_aggregations(query, repository_specific_options(options))
end
- # If wiki is true and migrate_wikis_to_separate_index is finished then set
+ # If is_wiki is true and migrate_wikis_to_separate_index is finished then set
# index as (#{env}-wikis)
# rid as (wiki_project_#{id}) for ProjectWiki and (wiki_group_#{id}) for GroupWiki
# If add_suffix_project_in_wiki_rid has not finished then rid might not have prefix(project/group) then
# run delete_query_by_rid with sending rid as 'wiki_#{project_id}'
- def delete_index_for_commits_and_blobs(wiki: false)
- types = wiki ? %w[wiki_blob] : %w[commit blob]
+ def delete_index_for_commits_and_blobs(is_wiki: false)
+ types = is_wiki ? %w[wiki_blob] : %w[commit blob]
- if (wiki && ::Elastic::DataMigrationService.migration_has_finished?(:migrate_wikis_to_separate_index)) || types.include?('commit')
- index, rid = if wiki
- [::Elastic::Latest::WikiConfig.index_name, "wiki_#{es_parent}"]
+ if (is_wiki && ::Elastic::DataMigrationService.migration_has_finished?(:migrate_wikis_to_separate_index)) || types.include?('commit')
+ index, rid = if is_wiki
+ [::Elastic::Latest::WikiConfig.index_name, wiki_rid]
else
[::Elastic::Latest::CommitConfig.index_name, project_id]
end
- response = delete_query_by_rid(index, rid)
+ response = delete_query_by_rid(index, rid, is_wiki)
# Consider to delete wikis by older rid(without suffix _project) as well
- if wiki && project_id && !::Elastic::DataMigrationService.migration_has_finished?(:add_suffix_project_in_wiki_rid)
- response = delete_query_by_rid(index, "wiki_#{project_id}")
+ if is_wiki && project_id && !::Elastic::DataMigrationService.migration_has_finished?(:add_suffix_project_in_wiki_rid)
+ response = delete_query_by_rid(index, "wiki_#{project_id}", is_wiki)
end
- return response if wiki # if condition can be removed once the blob gets migrated to the separate index
+ return response if is_wiki # if condition can be removed once the blob gets migrated to the separate index
end
client.delete_by_query(
@@ -115,6 +118,10 @@ def delete_index_for_commits_and_blobs(wiki: false)
private
+ def wiki_rid
+ project ? "wiki_project_#{project_id}" : "wiki_group_#{group_id}"
+ end
+
def repository_id
raise NotImplementedError
end
@@ -127,24 +134,26 @@ def repository_specific_options(options)
options
end
- def delete_query_by_rid(index, rid)
+ def delete_query_by_rid(index, rid, is_wiki)
client.delete_by_query(
- index: index,
- routing: es_parent,
- conflicts: 'proceed',
- body: {
- query: {
- bool: {
- filter: [
- {
- term: {
- rid: rid
+ {
+ index: index,
+ routing: es_parent(is_wiki),
+ conflicts: 'proceed',
+ body: {
+ query: {
+ bool: {
+ filter: [
+ {
+ term: {
+ rid: rid
+ }
}
- }
- ]
+ ]
+ }
}
}
- }
+ }.compact
)
end
end
diff --git a/ee/lib/elastic/latest/routing.rb b/ee/lib/elastic/latest/routing.rb
index 65c8af5c5b14f2b709cc0968362f7ba11d40cdb9..74d9b4ce7fade7ec95df25489bf744ef31b7591a 100644
--- a/ee/lib/elastic/latest/routing.rb
+++ b/ee/lib/elastic/latest/routing.rb
@@ -31,10 +31,10 @@ def routing_options(options)
private
- def build_routing(ids)
+ def build_routing(ids, prefix: 'project')
return [] if ids.count > ES_ROUTING_MAX_COUNT
- ids.map { |id| "project_#{id}" }.join(',')
+ ids.map { |id| "#{prefix}_#{id}" }.join(',')
end
def routing_disabled?(options)
diff --git a/ee/lib/elastic/latest/wiki_class_proxy.rb b/ee/lib/elastic/latest/wiki_class_proxy.rb
index abe94e41c4c5af0b0394525e61aa11556e82bcc6..ff48ba956ed4246be95edea5cb62f38ecdacd9d7 100644
--- a/ee/lib/elastic/latest/wiki_class_proxy.rb
+++ b/ee/lib/elastic/latest/wiki_class_proxy.rb
@@ -3,8 +3,8 @@
module Elastic
module Latest
class WikiClassProxy < ApplicationClassProxy
- include GitClassProxy
include Routing
+ include GitClassProxy
def es_type
'wiki_blob'
@@ -16,12 +16,12 @@ def elastic_search_as_wiki_page(*args, **kwargs)
end
end
- # Disable the routing for group level search
- # Will be enabled from MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/125137
def routing_options(options)
- return {} if options[:group_ids].present?
+ return {} if routing_disabled?(options)
- super
+ ids = options[:root_ancestor_ids].presence || []
+ routing = build_routing(ids, prefix: 'n')
+ { routing: routing.presence }.compact
end
end
end
diff --git a/ee/lib/gitlab/elastic/helper.rb b/ee/lib/gitlab/elastic/helper.rb
index 0a8847369c1ae6bba6d75190ef36dbae81e6d4dd..456b536e0a39d921034fb874fd46f514b7407f03 100644
--- a/ee/lib/gitlab/elastic/helper.rb
+++ b/ee/lib/gitlab/elastic/helper.rb
@@ -449,6 +449,21 @@ def create_index(index_name:, alias_name:, with_alias:, settings:, mappings:, op
def get_alias_info(pattern)
client.indices.get_alias(index: pattern)
end
+
+ def remove_wikis_from_the_standalone_index(container_id, container_type)
+ return unless %w[Group Project].include?(container_type) && Wiki.use_separate_indices?
+
+ container = container_type.constantize.find_by_id(container_id)
+ route = if container && ::Elastic::DataMigrationService.migration_has_finished?(:reindex_wikis_to_fix_routing)
+ "n_#{container.root_ancestor.id}"
+ end
+
+ client.delete_by_query({
+ index: ::Elastic::Latest::WikiConfig.index_name,
+ routing: route,
+ body: { query: { bool: { filter: { term: { rid: "wiki_#{container_type.downcase}_#{container_id}" } } } } }
+ }.compact)
+ end
end
end
end
diff --git a/ee/lib/gitlab/elastic/indexer.rb b/ee/lib/gitlab/elastic/indexer.rb
index e97b6be55269c16c6517ac00bb27e701db42e46d..de502c827ae9477a32a033092c80cc66027326b9 100644
--- a/ee/lib/gitlab/elastic/indexer.rb
+++ b/ee/lib/gitlab/elastic/indexer.rb
@@ -132,7 +132,7 @@ def run_indexer!(base_sha, to_sha, target)
#
# @return: whether the index has been purged
def purge_unreachable_commits_from_index!(target)
- target.delete_index_for_commits_and_blobs(wiki: index_wiki?)
+ target.delete_index_for_commits_and_blobs(is_wiki: index_wiki?)
rescue ::Elasticsearch::Transport::Transport::Errors::BadRequest => e
Gitlab::ErrorTracking.track_exception(e, group_id: group&.id, project_id: project&.id)
end
diff --git a/ee/lib/gitlab/elastic/project_search_results.rb b/ee/lib/gitlab/elastic/project_search_results.rb
index 75b5d94a8fd5a3062d527852e1ea1e3ebe23e198..c20b085d289a4d23af0ef6e79b6db82c57188c5c 100644
--- a/ee/lib/gitlab/elastic/project_search_results.rb
+++ b/ee/lib/gitlab/elastic/project_search_results.rb
@@ -10,11 +10,11 @@ class ProjectSearchResults < Gitlab::Elastic::SearchResults
attr_reader :project, :repository_ref, :filters
- def initialize(current_user, query, project:, repository_ref: nil, order_by: nil, sort: nil, filters: {})
+ def initialize(current_user, query, project:, root_ancestor_ids: nil, repository_ref: nil, order_by: nil, sort: nil, filters: {})
@project = project
@repository_ref = repository_ref.presence || project.default_branch
- super(current_user, query, [project.id], public_and_internal_projects: false, order_by: order_by, sort: sort, filters: filters)
+ super(current_user, query, [project.id], root_ancestor_ids: root_ancestor_ids, public_and_internal_projects: false, order_by: order_by, sort: sort, filters: filters)
end
private
@@ -43,7 +43,7 @@ def wiki_blobs(page: 1, per_page: DEFAULT_PER_PAGE, count_only: false)
query,
page: (page || 1).to_i,
per: per_page,
- options: base_options.merge(count_only: count_only)
+ options: scope_options(:wiki_blobs).merge(count_only: count_only)
)
end
end
diff --git a/ee/lib/gitlab/elastic/search_results.rb b/ee/lib/gitlab/elastic/search_results.rb
index 4f60266d599448e796e318b4cbcc0f3b0f82f3ee..64b037a01c4de96a919cd4a6958abdce1b514056 100644
--- a/ee/lib/gitlab/elastic/search_results.rb
+++ b/ee/lib/gitlab/elastic/search_results.rb
@@ -9,16 +9,17 @@ class SearchResults
ELASTIC_COUNT_LIMIT = 10000
DEFAULT_PER_PAGE = Gitlab::SearchResults::DEFAULT_PER_PAGE
- attr_reader :current_user, :query, :public_and_internal_projects, :order_by, :sort, :filters
+ attr_reader :current_user, :query, :public_and_internal_projects, :order_by, :sort, :filters, :root_ancestor_ids
# Limit search results by passed projects
# It allows us to search only for projects user has access to
attr_reader :limit_project_ids
- def initialize(current_user, query, limit_project_ids = nil, public_and_internal_projects: true, order_by: nil, sort: nil, filters: {})
+ def initialize(current_user, query, limit_project_ids = nil, root_ancestor_ids: nil, public_and_internal_projects: true, order_by: nil, sort: nil, filters: {})
@current_user = current_user
@query = query
@limit_project_ids = limit_project_ids
+ @root_ancestor_ids = root_ancestor_ids
@public_and_internal_projects = public_and_internal_projects
@order_by = order_by
@sort = sort
@@ -299,6 +300,8 @@ def scope_options(scope)
base_options.merge(admin: current_user&.admin?, routing_disabled: true) # rubocop:disable Cop/UserAdmin
when :blobs
base_options.merge(filters.slice(:language))
+ when :wiki_blobs
+ base_options.merge(root_ancestor_ids: root_ancestor_ids, routing_disabled: !reindex_wikis_to_fix_routing_done?)
else
base_options
end
@@ -446,6 +449,10 @@ def self.get_highlight_content(result)
content_key = use_separate_wiki_index?(result['_source']['type']) ? 'content' : 'blob.content'
result.dig('highlight', content_key)&.first || ''
end
+
+ def reindex_wikis_to_fix_routing_done?
+ ::Elastic::DataMigrationService.migration_has_finished?(:reindex_wikis_to_fix_routing)
+ end
end
end
end
diff --git a/ee/spec/elastic/migrate/20230702000000_backfill_existing_group_wiki_spec.rb b/ee/spec/elastic/migrate/20230702000000_backfill_existing_group_wiki_spec.rb
index 1eee3bc8c8e78186078f76b7db5135702583adad..8c1ad31da202f26a14597f0967f5511d90c2c915 100644
--- a/ee/spec/elastic/migrate/20230702000000_backfill_existing_group_wiki_spec.rb
+++ b/ee/spec/elastic/migrate/20230702000000_backfill_existing_group_wiki_spec.rb
@@ -98,7 +98,7 @@
def remove_all_group_wikis
helper.client.delete_by_query(
index: Elastic::Latest::WikiConfig.index_name,
- routing: "group_#{group.id},group_#{group2.id}",
+ routing: "n_#{group.id},n_#{group2.id}",
conflicts: 'proceed',
refresh: true,
body: { query: { regexp: { rid: "wiki_group_[0-9].*" } } }
diff --git a/ee/spec/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing_spec.rb b/ee/spec/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f48f9c6606b1710e173af18cf218960380a30074
--- /dev/null
+++ b/ee/spec/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing_spec.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require File.expand_path('ee/elastic/migrate/20230720000000_reindex_wikis_to_fix_routing.rb')
+
+RSpec.describe ReindexWikisToFixRouting, :elastic_clean, :sidekiq_inline, feature_category: :global_search do
+ let(:migration) { described_class.new(20230720000000) }
+ let(:helper) { Gitlab::Elastic::Helper.new }
+ let(:client) { ::Gitlab::Search::Client.new }
+ let_it_be(:project) { create(:project, :wiki_repo) }
+ let_it_be(:group) { create(:group) }
+ let_it_be(:group_wiki) { create(:group_wiki, group: group) }
+ let_it_be(:project_wiki) { create(:project_wiki, project: project) }
+
+ before do
+ stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
+ allow(migration).to receive(:helper).and_return(helper)
+ set_elasticsearch_migration_to :reindex_wikis_to_fix_routing, including: false
+ allow(migration).to receive(:client).and_return(client)
+ [project_wiki, group_wiki].each do |wiki|
+ wiki.create_page('index_page', 'Bla bla term')
+ wiki.create_page('index_page2', 'Bla bla term')
+ wiki.index_wiki_blobs
+ end
+ ensure_elasticsearch_index! # ensure objects are indexed
+ end
+
+ describe 'migration_options' do
+ it 'has migration options set', :aggregate_failures do
+ expect(migration).to be_batched
+ expect(migration.batch_size).to eq 200
+ expect(migration.throttle_delay).to eq(5.minutes)
+ expect(migration).to be_retry_on_failure
+ end
+ end
+
+ describe '.migrate' do
+ context 'if migration is completed' do
+ it 'performs logging and does not call ElasticWikiIndexerWorker' do
+ expect(migration).to receive(:log).with("Setting migration_state to #{{ documents_remaining: 0 }.to_json}").once
+ expect(migration).to receive(:log).with('Checking if migration is finished', { total_remaining: 0 }).once
+ expect(migration).to receive(:log).with('Migration Completed', { total_remaining: 0 }).once
+ expect(ElasticWikiIndexerWorker).not_to receive(:perform_in)
+ migration.migrate
+ end
+ end
+
+ context 'if migration is not completed' do
+ before do
+ set_old_schema_version_in_three_documents!
+ end
+
+ it 'performs logging and calls ElasticWikiIndexerWorker' do
+ expect(migration).to receive(:log).with("Setting migration_state to #{{ documents_remaining: 3 }.to_json}").once
+ expect(migration).to receive(:log).with('Checking if migration is finished', { total_remaining: 3 }).once
+ delay = a_value_between(0, migration.throttle_delay.seconds)
+ expect(ElasticWikiIndexerWorker).to receive(:perform_in).with(delay, project.id.to_s, project.class.name,
+ force: true)
+ expect(ElasticWikiIndexerWorker).to receive(:perform_in).with(delay, group.id.to_s, group.class.name,
+ force: true)
+ migration.migrate
+ end
+ end
+ end
+
+ describe '.completed?' do
+ subject { migration.completed? }
+
+ context 'when all the documents have the new schema_version(2307)' do
+ # With the 4.3.7 GITLAB_ELASTICSEARCH_INDEXER_VERSION all the new wikis will have schema_version 2307
+ it 'returns true' do
+ is_expected.to be true
+ end
+ end
+
+ context 'when some items are missing new schema_version' do
+ before do
+ set_old_schema_version_in_three_documents!
+ end
+
+ it 'returns false' do
+ is_expected.to be false
+ end
+ end
+ end
+
+ def set_old_schema_version_in_three_documents!
+ client.update_by_query(index: Elastic::Latest::WikiConfig.index_name, max_docs: 3, refresh: true,
+ body: { script: { lang: 'painless', source: 'ctx._source.schema_version = 2305' } }
+ )
+ end
+end
diff --git a/ee/spec/lib/ee/gitlab/elastic/helper_spec.rb b/ee/spec/lib/ee/gitlab/elastic/helper_spec.rb
index 698e438db2b8821e234d7310cfe10590f9172983..50b569d665dc497a46d716ace7a910b6a8054571 100644
--- a/ee/spec/lib/ee/gitlab/elastic/helper_spec.rb
+++ b/ee/spec/lib/ee/gitlab/elastic/helper_spec.rb
@@ -737,4 +737,90 @@
it { is_expected.to eq(false) }
end
end
+
+ describe '#remove_wikis_from_the_standalone_index' do
+ context 'container_type is other than Group or Project' do
+ it 'not calls delete_by_query' do
+ expect(helper.client).not_to receive(:delete_by_query)
+ helper.remove_wikis_from_the_standalone_index(1, 'Random')
+ end
+ end
+
+ context 'Wiki does not use separate indices' do
+ let_it_be(:project) { create(:project) }
+
+ it 'not calls delete_by_query' do
+ expect(helper.client).not_to receive(:delete_by_query)
+ helper.remove_wikis_from_the_standalone_index(project.id, project.class.name)
+ end
+ end
+
+ context 'Wiki uses separate indices' do
+ include ElasticsearchHelpers
+ let(:body) do
+ { query: { bool: { filter: { term: { rid: rid } } } } }
+ end
+
+ let(:index) { Elastic::Latest::WikiConfig.index_name }
+
+ before do
+ allow(Wiki).to receive(:use_separate_indices?).and_return true
+ end
+
+ context 'container not found' do
+ let(:rid) { "wiki_project_#{non_existing_record_id}" }
+
+ it 'calls delete_by_query without routing' do
+ expect(helper.client).to receive(:delete_by_query).with({ body: body, index: index })
+ helper.remove_wikis_from_the_standalone_index(non_existing_record_id, 'Project')
+ end
+ end
+
+ context 'container found is Group' do
+ let_it_be(:group) { create :group }
+ let(:rid) { "wiki_group_#{group.id}" }
+
+ context 'migration reindex_wikis_to_fix_routing is finished' do
+ before do
+ set_elasticsearch_migration_to :reindex_wikis_to_fix_routing, including: true
+ end
+
+ it 'calls delete_by_query with routing' do
+ expect(helper.client).to receive(:delete_by_query).with({ body: body, index: index, routing: "n_#{group.root_ancestor.id}" })
+ helper.remove_wikis_from_the_standalone_index(group.id, group.class.name)
+ end
+ end
+
+ context 'migration reindex_wikis_to_fix_routing is not finished' do
+ it 'calls delete_by_query without routing' do
+ expect(helper.client).to receive(:delete_by_query).with({ body: body, index: index })
+ helper.remove_wikis_from_the_standalone_index(group.id, group.class.name)
+ end
+ end
+ end
+
+ context 'container found is Project' do
+ let_it_be(:project) { create :project }
+ let(:rid) { "wiki_project_#{project.id}" }
+
+ context 'migration reindex_wikis_to_fix_routing is finished' do
+ before do
+ set_elasticsearch_migration_to :reindex_wikis_to_fix_routing, including: true
+ end
+
+ it 'calls delete_by_query with routing' do
+ expect(helper.client).to receive(:delete_by_query).with({ body: body, index: index, routing: "n_#{project.root_ancestor.id}" })
+ helper.remove_wikis_from_the_standalone_index(project.id, project.class.name)
+ end
+ end
+
+ context 'migration reindex_wikis_to_fix_routing is not finished' do
+ it 'calls delete_by_query without routing' do
+ expect(helper.client).to receive(:delete_by_query).with({ body: body, index: index })
+ helper.remove_wikis_from_the_standalone_index(project.id, project.class.name)
+ end
+ end
+ end
+ end
+ end
end
diff --git a/ee/spec/lib/elastic/latest/git_instance_proxy_spec.rb b/ee/spec/lib/elastic/latest/git_instance_proxy_spec.rb
index 461abd3e8e20ead73fe2d797abe4fedb8d25a9f7..c7eb7cc64d18358010cd4dd91562a4b94f856f89 100644
--- a/ee/spec/lib/elastic/latest/git_instance_proxy_spec.rb
+++ b/ee/spec/lib/elastic/latest/git_instance_proxy_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
-RSpec.describe Elastic::Latest::GitInstanceProxy do
+RSpec.describe Elastic::Latest::GitInstanceProxy, feature_category: :global_search do
let(:project) { create(:project, :repository) }
let(:included_class) { Elastic::Latest::RepositoryInstanceProxy }
@@ -18,13 +18,54 @@
end
describe '#es_parent' do
- it 'contains project id for ProjectWiki repository' do
- expect(included_class.new(project.wiki.repository).es_parent).to eq("project_#{project.id}")
+ context 'for wiki is false' do
+ it 'contains project id' do
+ expect(included_class.new(project.repository).es_parent).to eq("project_#{project.id}")
+ end
end
- it 'contains group id for GroupWiki repository' do
- group = create(:group)
- expect(included_class.new(group.wiki.repository).es_parent).to eq("group_#{group.id}")
+ context 'for wiki is true' do
+ include ElasticsearchHelpers
+ context 'when migration reindex_wikis_to_fix_routing is finished' do
+ before do
+ set_elasticsearch_migration_to(:reindex_wikis_to_fix_routing, including: true)
+ end
+
+ context 'for ProjectWiki repository' do
+ it "contains project's root ancestor id" do
+ repository = project.wiki.repository
+ expect(included_class.new(repository).es_parent(true)).to eq "n_#{project.root_ancestor.id}"
+ end
+ end
+
+ context 'for GroupWiki repository' do
+ let_it_be(:group) { create :group }
+
+ it "contains group's root ancestor id" do
+ expect(included_class.new(group.wiki.repository).es_parent(true)).to eq "n_#{group.root_ancestor.id}"
+ end
+ end
+ end
+
+ context 'when migration reindex_wikis_to_fix_routing is not finished' do
+ before do
+ set_elasticsearch_migration_to(:reindex_wikis_to_fix_routing, including: false)
+ end
+
+ context 'for ProjectWiki repository' do
+ it 'returns nil' do
+ expect(included_class.new(project.wiki.repository).es_parent(true)).to be nil
+ end
+ end
+
+ context 'for GroupWiki repository' do
+ let_it_be(:group) { create :group }
+
+ it 'returns nil' do
+ expect(included_class.new(project.wiki.repository).es_parent(true)).to be nil
+ end
+ end
+ end
end
end
diff --git a/ee/spec/lib/elastic/latest/routing_spec.rb b/ee/spec/lib/elastic/latest/routing_spec.rb
index 31156fa43d7b4064d3d6227c86c9414deeaa787f..1b94b88b1fe5faae72941f355072b98843b6ba0b 100644
--- a/ee/spec/lib/elastic/latest/routing_spec.rb
+++ b/ee/spec/lib/elastic/latest/routing_spec.rb
@@ -5,25 +5,12 @@
RSpec.describe Elastic::Latest::Routing, feature_category: :global_search do
let(:proxified_class) { Issue }
let(:included_class) { Elastic::Latest::ApplicationClassProxy }
- let(:project_ids) { [1, 2, 3] }
+ let(:ids) { [1, 2, 3] }
let(:project_routing) { 'project_1,project_2,project_3' }
+ let(:n_routing) { 'n_1,n_2,n_3' }
subject { included_class.new(proxified_class) }
- describe '#search' do
- it 'calls routing_options with empty hash' do
- expect(subject).to receive(:routing_options).and_return({})
-
- subject.search('q')
- end
-
- it 'calls routing_options with correct routing' do
- expect(subject).to receive(:routing_options).and_return({ routing: project_routing })
-
- subject.search('q', project_ids: project_ids)
- end
- end
-
describe '#routing_options' do
it 'returns correct options for project_id' do
expect(subject.routing_options({ project_id: 1 })).to eq({ routing: 'project_1' })
@@ -34,7 +21,7 @@
end
it 'returns correct options for project_ids' do
- expect(subject.routing_options({ project_ids: project_ids })).to eq({ routing: project_routing })
+ expect(subject.routing_options({ project_ids: ids })).to eq({ routing: project_routing })
end
it 'returns empty hash when provided an empty array' do
@@ -46,7 +33,7 @@
end
it 'returns empty hash when public projects flag is passed' do
- expect(subject.routing_options({ project_ids: project_ids, public_and_internal_projects: true })).to eq({})
+ expect(subject.routing_options({ project_ids: ids, public_and_internal_projects: true })).to eq({})
end
it 'returns empty hash when routing_disabled flag is passed' do
@@ -54,7 +41,7 @@
end
it 'uses project_ids rather than repository_id when both are supplied' do
- options = { project_ids: project_ids, repository_id: 'wiki_5' }
+ options = { project_ids: ids, repository_id: 'wiki_5' }
expect(subject.routing_options(options)).to eq({ routing: project_routing })
end
diff --git a/ee/spec/lib/elastic/latest/wiki_class_proxy_spec.rb b/ee/spec/lib/elastic/latest/wiki_class_proxy_spec.rb
index 37ea42eac07a2afd773f98334655acad62b0924c..05e9e5adeeba433c98b18684e6e19b4f55ab87d0 100644
--- a/ee/spec/lib/elastic/latest/wiki_class_proxy_spec.rb
+++ b/ee/spec/lib/elastic/latest/wiki_class_proxy_spec.rb
@@ -2,12 +2,12 @@
require 'spec_helper'
-RSpec.describe Elastic::Latest::WikiClassProxy, :elastic, feature_category: :global_search do
+RSpec.describe Elastic::Latest::WikiClassProxy, feature_category: :global_search do
let_it_be(:project) { create(:project, :wiki_repo) }
subject { described_class.new(project.wiki.class, use_separate_indices: ProjectWiki.use_separate_indices?) }
- describe '#elastic_search_as_wiki_page' do
+ describe '#elastic_search_as_wiki_page', :elastic do
let!(:page) { create(:wiki_page, wiki: project.wiki) }
before do
@@ -32,9 +32,40 @@
end
end
- it 'names elasticsearch queries' do
+ it 'names elasticsearch queries', :elastic do
subject.elastic_search_as_wiki_page('*')
assert_named_queries('doc:is_a:wiki_blob', 'blob:match:search_terms')
end
+
+ describe '#routing_options' do
+ let(:n_routing) { 'n_1,n_2,n_3' }
+ let(:ids) { [1, 2, 3] }
+ let(:default_ops) { { root_ancestor_ids: ids, scope: 'wiki_blob' } }
+
+ context 'when routing is disabled' do
+ context 'and option routing_disabled is set' do
+ it 'returns empty hash' do
+ expect(subject.routing_options(default_ops.merge(routing_disabled: true))).to be_empty
+ end
+ end
+
+ context 'and option public_and_internal_projects is set' do
+ it 'returns empty hash' do
+ expect(subject.routing_options(default_ops.merge(public_and_internal_projects: true))).to be_empty
+ end
+ end
+ end
+
+ context 'when ids count are more than 128' do
+ it 'returns empty hash' do
+ max_count = Elastic::Latest::Routing::ES_ROUTING_MAX_COUNT
+ expect(subject.routing_options(default_ops.merge(root_ancestor_ids: 1.upto(max_count + 1).to_a))).to be_empty
+ end
+ end
+
+ it 'returns routing hash' do
+ expect(subject.routing_options(default_ops)).to eq({ routing: n_routing })
+ end
+ end
end
diff --git a/ee/spec/lib/gitlab/elastic/project_search_results_spec.rb b/ee/spec/lib/gitlab/elastic/project_search_results_spec.rb
index 56eaf276fa5dc28b17f499f14a96d59051da5306..604c0a71c5466f8b2e07a906439bf7ef65ca426f 100644
--- a/ee/spec/lib/gitlab/elastic/project_search_results_spec.rb
+++ b/ee/spec/lib/gitlab/elastic/project_search_results_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
-RSpec.describe Gitlab::Elastic::ProjectSearchResults, :elastic do
+RSpec.describe Gitlab::Elastic::ProjectSearchResults, :elastic, feature_category: :global_search do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :public, :repository) }
diff --git a/ee/spec/workers/search/wiki/elastic_delete_group_wiki_worker_spec.rb b/ee/spec/workers/search/wiki/elastic_delete_group_wiki_worker_spec.rb
index 896e3977c1e3a24050f257b49d40c34b4288995e..ac449e40fa6d4937fbc2e25c884c7c8fe392c73d 100644
--- a/ee/spec/workers/search/wiki/elastic_delete_group_wiki_worker_spec.rb
+++ b/ee/spec/workers/search/wiki/elastic_delete_group_wiki_worker_spec.rb
@@ -21,25 +21,25 @@
end
it 'removes all the wikis in the Elastic of the passed group' do
- expect(get_wiki_documents_count(group.id, group.class.name)).to be 1
+ expect(get_wiki_documents_count(group)).to be 1
described_class.new.perform(group.id)
refresh_index!
- expect(get_wiki_documents_count(group.id, group.class.name)).to be 0
- expect(get_wiki_documents_count(group2.id, group2.class.name)).to be 1
- expect(get_wiki_documents_count(project.id, project.class.name)).to be 1
+ expect(get_wiki_documents_count(group)).to be 0
+ expect(get_wiki_documents_count(group2)).to be 1
+ expect(get_wiki_documents_count(project)).to be 1
end
- def get_wiki_documents_count(container_id, container_type)
+ def get_wiki_documents_count(container)
Gitlab::Elastic::Helper.default.client.search(
{
index: Elastic::Latest::WikiConfig.index_name,
- routing: "#{container_type.downcase}_#{container_id}",
+ routing: "n_#{container.root_ancestor.id}",
body: {
query: {
bool: {
filter: {
term: {
- rid: "wiki_#{container_type.downcase}_#{container_id}"
+ rid: "wiki_#{container.class.name.downcase}_#{container.id}"
}
}
}