diff --git a/.rubocop_todo/gitlab/bounded_contexts.yml b/.rubocop_todo/gitlab/bounded_contexts.yml index 3944932aba26f4c78b066a03712c346668009fed..e1da7df79e3c4deff3808f2034bb1ebd20ca8b77 100644 --- a/.rubocop_todo/gitlab/bounded_contexts.yml +++ b/.rubocop_todo/gitlab/bounded_contexts.yml @@ -3393,7 +3393,6 @@ Gitlab/BoundedContexts: - 'ee/app/services/elastic/data_migration_service.rb' - 'ee/app/services/elastic/index_projects_by_id_service.rb' - 'ee/app/services/elastic/index_projects_by_range_service.rb' - - 'ee/app/services/elastic/index_projects_service.rb' - 'ee/app/services/elastic/indexing_control_service.rb' - 'ee/app/services/elastic/process_bookkeeping_service.rb' - 'ee/app/services/elastic/process_initial_bookkeeping_service.rb' diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index 555c8613b1a0b66fcf0935ec29f35de71e39de3d..a07dab05627ac764c1a1ee75d6dabd9f685fd1f1 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -635,7 +635,6 @@ Layout/LineLength: - 'danger/feature_flag/Dangerfile' - 'danger/roulette/Dangerfile' - 'danger/vue_shared_documentation/Dangerfile' - - 'ee/app/controllers/admin/elasticsearch_controller.rb' - 'ee/app/controllers/admin/licenses_controller.rb' - 'ee/app/controllers/concerns/audit_events/date_range.rb' - 'ee/app/controllers/concerns/credentials_inventory_actions.rb' diff --git a/.rubocop_todo/rspec/named_subject.yml b/.rubocop_todo/rspec/named_subject.yml index dd412c78b8e3f858de87005428b5ee202554af7f..27c930542a64fd3c77fb9fb8ff83e149ce70ad9a 100644 --- a/.rubocop_todo/rspec/named_subject.yml +++ b/.rubocop_todo/rspec/named_subject.yml @@ -954,7 +954,6 @@ RSpec/NamedSubject: - 'ee/spec/services/ee/users/build_service_spec.rb' - 'ee/spec/services/ee/work_items/import_csv_service_spec.rb' - 'ee/spec/services/elastic/data_migration_service_spec.rb' - - 'ee/spec/services/elastic/index_projects_service_spec.rb' - 'ee/spec/services/elastic/indexing_control_service_spec.rb' - 'ee/spec/services/elastic/metrics_update_service_spec.rb' - 'ee/spec/services/epic_issues/create_service_spec.rb' diff --git a/.rubocop_todo/search/namespaced_class.yml b/.rubocop_todo/search/namespaced_class.yml index e60b1c9fdeb13a0f2975705d3dbaef92d5e15fa4..78c537bf70482e0690b763c5158be32d782a9361 100644 --- a/.rubocop_todo/search/namespaced_class.yml +++ b/.rubocop_todo/search/namespaced_class.yml @@ -43,7 +43,6 @@ Search/NamespacedClass: - 'ee/app/services/elastic/data_migration_service.rb' - 'ee/app/services/elastic/index_projects_by_id_service.rb' - 'ee/app/services/elastic/index_projects_by_range_service.rb' - - 'ee/app/services/elastic/index_projects_service.rb' - 'ee/app/services/elastic/indexing_control_service.rb' - 'ee/app/services/elastic/process_bookkeeping_service.rb' - 'ee/app/services/elastic/process_initial_bookkeeping_service.rb' diff --git a/.rubocop_todo/style/format_string.yml b/.rubocop_todo/style/format_string.yml index dcaf49d4b34d9379e388f4430bcecca2671efc6a..422f8c1951d00685b175d4f1f45c0da4d51cba58 100644 --- a/.rubocop_todo/style/format_string.yml +++ b/.rubocop_todo/style/format_string.yml @@ -140,7 +140,6 @@ Style/FormatString: - 'ee/app/components/namespaces/free_user_cap/enforcement_alert_component.rb' - 'ee/app/components/namespaces/free_user_cap/usage_quota_alert_component.rb' - 'ee/app/components/namespaces/free_user_cap/usage_quota_trial_alert_component.rb' - - 'ee/app/controllers/admin/elasticsearch_controller.rb' - 'ee/app/controllers/admin/licenses_controller.rb' - 'ee/app/controllers/concerns/audit_events/date_range.rb' - 'ee/app/controllers/ee/projects/issues_controller.rb' diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 0b65ae641d55912c263bac9409d30b7f61c56db6..bc772d6ee0e7d6db67efe4676f9ec478fa306c53 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -361,7 +361,7 @@ Elasticsearch cluster. For GitLab instances with more than 50 GB of repository data, see [Index large instances efficiently](#index-large-instances-efficiently). -### Index all projects +### Index the instance DETAILS: **Offering:** Self-managed @@ -370,28 +370,34 @@ Prerequisites: - You must have administrator access to the instance. -You can only use the **Index all projects** setting to perform -initial indexing, not to re-create an index from scratch. -To enable advanced search with **Index all projects**: +You can use **Index the instance** to perform initial indexing +or re-create an index from scratch. + +To enable advanced search with this setting: 1. On the left sidebar, at the bottom, select **Admin area**. 1. Select **Settings > Search**. 1. Select the **Elasticsearch indexing** checkbox, then select **Save changes**. -1. Select **Index all projects**. -1. Optional. Select **Check progress** to see the status of background jobs. +1. Select **Index the instance**. -To index epics, group wikis, personal snippets, and users, you must use Rake tasks: +To index specific data, you can use the following Rake tasks: ```shell # Omnibus installations sudo gitlab-rake gitlab:elastic:index_epics +sudo gitlab-rake gitlab:elastic:index_work_items sudo gitlab-rake gitlab:elastic:index_group_wikis +sudo gitlab-rake gitlab:elastic:index_namespaces +sudo gitlab-rake gitlab:elastic:index_projects sudo gitlab-rake gitlab:elastic:index_snippets sudo gitlab-rake gitlab:elastic:index_users # Installations from source bundle exec rake gitlab:elastic:index_epics RAILS_ENV=production +bundle exec rake gitlab:elastic:index_work_items RAILS_ENV=production bundle exec rake gitlab:elastic:index_group_wikis RAILS_ENV=production +bundle exec rake gitlab:elastic:index_namespaces RAILS_ENV=production +bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production bundle exec rake gitlab:elastic:index_users RAILS_ENV=production ``` diff --git a/ee/app/controllers/admin/elasticsearch_controller.rb b/ee/app/controllers/admin/elasticsearch_controller.rb index 864963d1a0001515c0569e8d7bbcd13f847dce44..987465c88d2c45fda392ce0aaa5e99e153d661e4 100644 --- a/ee/app/controllers/admin/elasticsearch_controller.rb +++ b/ee/app/controllers/admin/elasticsearch_controller.rb @@ -7,15 +7,11 @@ class Admin::ElasticsearchController < Admin::ApplicationController # POST # Scheduling indexing jobs def enqueue_index - if Gitlab::Elastic::Helper.default.index_exists? - ::Elastic::IndexProjectsService.new.execute + ::Search::Elastic::ReindexingService.execute - notice = _('Elasticsearch indexing started') - queue_link = helpers.link_to(_('(check progress)'), sidekiq_path + '/queues/elastic_commit_indexer') - flash[:notice] = "#{notice} #{queue_link}".html_safe - else - flash[:warning] = _('Please create an index before enabling indexing') - end + flash[:notice] = + _('Advanced search indexing in progress. It might take a few minutes to create indices and initiate indexing.' \ + 'Please use gitlab:elastic:info rake task to check progress.') redirect_to redirect_path end @@ -31,7 +27,7 @@ def trigger_reindexing flash[:notice] = _('Elasticsearch reindexing triggered') else errors = @elasticsearch_reindexing_task.errors.full_messages.join(', ') - flash[:alert] = _("Elasticsearch reindexing was not started: %{errors}") % { errors: errors } + flash[:alert] = format(_("Elasticsearch reindexing was not started: %{errors}"), errors: errors) end end @@ -69,10 +65,18 @@ def redirect_path(anchor: 'js-elasticsearch-settings') end def trigger_reindexing_params - permitted_params = params.require(:elastic_reindexing_task).permit(:elasticsearch_max_slices_running, :elasticsearch_slice_multiplier) + permitted_params = params.require(:elastic_reindexing_task).permit(:elasticsearch_max_slices_running, + :elasticsearch_slice_multiplier) trigger_reindexing_params = {} - trigger_reindexing_params[:max_slices_running] = permitted_params[:elasticsearch_max_slices_running] if permitted_params.has_key?(:elasticsearch_max_slices_running) - trigger_reindexing_params[:slice_multiplier] = permitted_params[:elasticsearch_slice_multiplier] if permitted_params.has_key?(:elasticsearch_slice_multiplier) + if permitted_params.has_key?(:elasticsearch_max_slices_running) + trigger_reindexing_params[:max_slices_running] = + permitted_params[:elasticsearch_max_slices_running] + end + + if permitted_params.has_key?(:elasticsearch_slice_multiplier) + trigger_reindexing_params[:slice_multiplier] = + permitted_params[:elasticsearch_slice_multiplier] + end trigger_reindexing_params end diff --git a/ee/app/services/elastic/index_projects_service.rb b/ee/app/services/elastic/index_projects_service.rb deleted file mode 100644 index f1d2bc55f4d069d2d1d756cfda8acfdbf8617c73..0000000000000000000000000000000000000000 --- a/ee/app/services/elastic/index_projects_service.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Elastic - class IndexProjectsService - def execute - if Gitlab::CurrentSettings.elasticsearch_limit_indexing? - IndexProjectsByIdService.new.execute( - project_ids: ElasticsearchIndexedProject.target_ids, - namespace_ids: ElasticsearchIndexedNamespace.target_ids - ) - else - IndexProjectsByRangeService.new.execute - end - end - end -end diff --git a/ee/app/services/search/elastic/reindexing_service.rb b/ee/app/services/search/elastic/reindexing_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..b38d46692b70250390b52d5a42ff5cefc1a79e7e --- /dev/null +++ b/ee/app/services/search/elastic/reindexing_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Search + module Elastic + class ReindexingService + # skip projects, all namespace and project data is handled by `namespaces` task + OPTIONS = { 'skip' => 'projects' }.freeze + + attr_reader :delay + + def self.execute(...) + new(...).execute + end + + def initialize(delay: 0) + @delay = delay + end + + def execute + initial_task = Search::Elastic::TriggerIndexingWorker::INITIAL_TASK + Search::Elastic::TriggerIndexingWorker.perform_in(delay, initial_task, OPTIONS) + end + end + end +end diff --git a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml index a78b14401ea4963ade76839aef2b31212295f10c..b965b451a58d2966074e252968b2c38e5dd8983f 100644 --- a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml +++ b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml @@ -38,7 +38,7 @@ - if Gitlab::CurrentSettings.elasticsearch_indexing? = f.gitlab_ui_checkbox_component :elasticsearch_indexing, s_('AdminSettings|Elasticsearch indexing'), checkbox_options: { data: { testid: 'indexing-checkbox' } } = render Pajamas::ButtonComponent.new(variant: :confirm, href: admin_elasticsearch_enqueue_index_path, method: :post) do - = _('Index all projects') + = _('Index the instance') .form-group - first_pending_migration = Elastic::DataMigrationService.pending_migrations.first if elasticsearch_available diff --git a/ee/lib/tasks/gitlab/elastic.rake b/ee/lib/tasks/gitlab/elastic.rake index 26c4f2ca561d13c4c2df28dbd8122e6922620adb..c924af7bff711816f4e33543b947b7ae221e4730 100644 --- a/ee/lib/tasks/gitlab/elastic.rake +++ b/ee/lib/tasks/gitlab/elastic.rake @@ -25,9 +25,7 @@ namespace :gitlab do TEXT stdout_logger.info(worker_info_msg) - # skip projects, all namespace and project data is handled by `namespaces` task - Search::Elastic::TriggerIndexingWorker.perform_in(1.minute, - Search::Elastic::TriggerIndexingWorker::INITIAL_TASK, { 'skip' => 'projects' }) + Search::Elastic::ReindexingService.execute(delay: 1.minute) stdout_logger.info("Scheduling indexing with TriggerIndexingWorker... #{Rainbow('done').green}") end diff --git a/ee/spec/controllers/admin/elasticsearch_controller_spec.rb b/ee/spec/controllers/admin/elasticsearch_controller_spec.rb index 572fa4389f48d1036f1f3641189451c76af473dd..c068b497e0b10e1dda388d2f7ebb0c66a5e6219e 100644 --- a/ee/spec/controllers/admin/elasticsearch_controller_spec.rb +++ b/ee/spec/controllers/admin/elasticsearch_controller_spec.rb @@ -9,35 +9,15 @@ describe 'POST #enqueue_index' do before do sign_in(admin) - allow(Gitlab::Elastic::Helper).to receive(:default).and_return(helper) end it 'starts indexing' do - expect(helper).to(receive(:index_exists?)).and_return(true) - expect_next_instance_of(::Elastic::IndexProjectsService) do |service| - expect(service).to receive(:execute) - end + expect(::Search::Elastic::ReindexingService).to receive(:execute) post :enqueue_index - expect(controller).to set_flash[:notice].to include('/admin/sidekiq/queues/elastic_commit_indexer') expect(response).to redirect_to advanced_search_admin_application_settings_path(anchor: 'js-elasticsearch-settings') end - - context 'without an index' do - before do - allow(helper).to(receive(:index_exists?)).and_return(false) - end - - it 'does nothing and returns 404' do - expect(::Elastic::IndexProjectsService).not_to receive(:new) - - post :enqueue_index - - expect(controller).to set_flash[:warning].to include('create an index before enabling indexing') - expect(response).to redirect_to advanced_search_admin_application_settings_path(anchor: 'js-elasticsearch-settings') - end - end end describe 'POST #trigger_reindexing' do diff --git a/ee/spec/services/elastic/index_projects_service_spec.rb b/ee/spec/services/elastic/index_projects_service_spec.rb deleted file mode 100644 index 9305f28410720262b59c2bdcd3d4ca97a3e44550..0000000000000000000000000000000000000000 --- a/ee/spec/services/elastic/index_projects_service_spec.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Elastic::IndexProjectsService, feature_category: :global_search do - describe '#execute' do - context 'when elasticsearch_limit_indexing? is true' do - before do - stub_ee_application_setting(elasticsearch_limit_indexing: true) - create(:elasticsearch_indexed_project) - create(:elasticsearch_indexed_namespace) - end - - it 'schedules indexing for selected projects and namespaces' do - expect_next_instance_of(::Elastic::IndexProjectsByIdService) do |service| - expect(service).to receive(:execute).with( - project_ids: ElasticsearchIndexedProject.target_ids, - namespace_ids: ElasticsearchIndexedNamespace.target_ids - ) - end - - subject.execute - end - end - - context 'when elasticsearch_limit_indexing? is false' do - before do - stub_ee_application_setting(elasticsearch_limit_indexing: false) - end - - it 'schedules indexing for all projects' do - expect_next_instance_of(::Elastic::IndexProjectsByRangeService) do |service| - expect(service).to receive(:execute) - end - - subject.execute - end - end - end -end diff --git a/ee/spec/services/search/elastic/reindexing_service_spec.rb b/ee/spec/services/search/elastic/reindexing_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..63013a835f7f38837f4ae4320ca17a2df1f5e553 --- /dev/null +++ b/ee/spec/services/search/elastic/reindexing_service_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Search::Elastic::ReindexingService, feature_category: :global_search do + describe '.execute' do + subject(:service) { described_class.new(delay: delay) } + + let(:delay) { 1.minute } + + it 'instanciates the service and executes it' do + expect(described_class).to receive(:new).with(delay: delay).and_return(service) + expect(service).to receive(:execute) + + described_class.execute(delay: delay) + end + end + + describe '#execute' do + context 'when delay is not set' do + subject(:service) { described_class.new } + + it 'schedules indexing for the instance without delay' do + expect(Search::Elastic::TriggerIndexingWorker).to receive(:perform_in).with( + 0, Search::Elastic::TriggerIndexingWorker::INITIAL_TASK, described_class::OPTIONS + ) + + service.execute + end + end + + context 'when delay is set' do + subject(:service) { described_class.new(delay: delay) } + + let(:delay) { 1.minute } + + it 'schedules indexing for the instance without delay' do + expect(Search::Elastic::TriggerIndexingWorker).to receive(:perform_in).with( + delay, Search::Elastic::TriggerIndexingWorker::INITIAL_TASK, described_class::OPTIONS + ) + + service.execute + end + end + end +end diff --git a/ee/spec/views/admin/application_settings/_elasticsearch_form.html.haml_spec.rb b/ee/spec/views/admin/application_settings/_elasticsearch_form.html.haml_spec.rb index c57a2b36352adaeb7d6a8b23cb7a444f23e929a4..27c1089b0ad57b974d4f2b584a6995609c43f9e1 100644 --- a/ee/spec/views/admin/application_settings/_elasticsearch_form.html.haml_spec.rb +++ b/ee/spec/views/admin/application_settings/_elasticsearch_form.html.haml_spec.rb @@ -23,7 +23,7 @@ context 'es indexing' do let(:application_setting) { build(:application_setting) } - let(:button_text) { 'Index all projects' } + let(:button_text) { 'Index the instance' } before do allow(Gitlab::CurrentSettings).to(receive(:elasticsearch_indexing?)).and_return(es_indexing) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 5d20a4955b5ccadf771f6a34c3f4f8ac4cfc043a..6aac19a426c1db9b0591bb764b7224dd660f6d37 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -1556,9 +1556,6 @@ msgstr "" msgid "(Unlimited compute minutes)" msgstr "" -msgid "(check progress)" -msgstr "" - msgid "(deleted)" msgstr "" @@ -4771,6 +4768,9 @@ msgstr "" msgid "Advanced export options" msgstr "" +msgid "Advanced search indexing in progress. It might take a few minutes to create indices and initiate indexing.Please use gitlab:elastic:info rake task to check progress." +msgstr "" + msgid "AdvancedSearch|Could not fetch index entities. Please try again later." msgstr "" @@ -19838,9 +19838,6 @@ msgstr "" msgid "Elasticsearch indexing restrictions" msgstr "" -msgid "Elasticsearch indexing started" -msgstr "" - msgid "Elasticsearch migration halted" msgstr "" @@ -27814,15 +27811,15 @@ msgstr "" msgid "Index" msgstr "" -msgid "Index all projects" -msgstr "" - msgid "Index deletion is canceled" msgstr "" msgid "Index root namespaces automatically" msgstr "" +msgid "Index the instance" +msgstr "" + msgid "Indexing status" msgstr "" @@ -39639,9 +39636,6 @@ msgstr "" msgid "Please create a username with only alphanumeric characters." msgstr "" -msgid "Please create an index before enabling indexing" -msgstr "" - msgid "Please enter a URL for the custom emoji." msgstr "" diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml index c84f14a7637c0392ea51bb5aaa42758126b7752e..e3cf9d033f9d486ffeb0699a37be6e31de9620b8 100644 --- a/spec/support/rspec_order_todo.yml +++ b/spec/support/rspec_order_todo.yml @@ -2408,7 +2408,6 @@ - './ee/spec/services/elastic/indexing_control_service_spec.rb' - './ee/spec/services/elastic/index_projects_by_id_service_spec.rb' - './ee/spec/services/elastic/index_projects_by_range_service_spec.rb' -- './ee/spec/services/elastic/index_projects_service_spec.rb' - './ee/spec/services/elastic/metrics_update_service_spec.rb' - './ee/spec/services/elastic/process_bookkeeping_service_spec.rb' - './ee/spec/services/elastic/process_initial_bookkeeping_service_spec.rb'