Merge branch 'jm-send-zoekt-stop-indexing' into 'master'

Send stop indexing signal to zoekt nodes when critical watermark is exceeded

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173167



Merged-by: Dmitry Gruzd <dgruzd@gitlab.com>
Approved-by: Arturo Herrero <arturo.herrero@gmail.com>
Approved-by: Dmitry Gruzd <dgruzd@gitlab.com>
Reviewed-by: GitLab Duo <gitlab-duo@gitlab.com>
Co-authored-by: John Mason <jmason@gitlab.com>
Co-authored-by: Arturo Herrero <arturo.herrero@gmail.com>

ee/config/feature_flags/ops/zoekt_critical_watermark_stop_indexing.yml

+---
+name: zoekt_critical_watermark_stop_indexing
+feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/504945
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173167
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/505334
+milestone: '17.7'
+group: group::global search
+type: ops
+default_enabled: false

ee/lib/api/internal/search/zoekt.rb

@@ -39,6 +39,9 @@ def logger
                     { id: node.id, truncate: new_node }.tap do |resp|
                       resp[:tasks] = ::Search::Zoekt::TaskPresenterService.execute(node)
                       resp[:pull_frequency] = node.task_pull_frequency
+                      if Feature.enabled?(:zoekt_critical_watermark_stop_indexing)
+                        resp[:stop_indexing] = node.watermark_exceeded_critical?
+                      end
                     end
                   else
                     unprocessable_entity!

ee/spec/requests/api/internal/search/zoekt_spec.rb

@@ -34,11 +34,16 @@
     context 'with valid auth' do
       subject(:request) { get api(endpoint), params: valid_params, headers: gitlab_shell_internal_api_request_header }
 
+      let(:node) { build(:zoekt_node) }
+
+      before do
+        allow(::Search::Zoekt::Node).to receive(:find_or_initialize_by_task_request)
+                                           .with(valid_params).and_return(node)
+      end
+
       context 'with feature flag disabled' do
         before do
           stub_feature_flags(zoekt_internal_api_register_nodes: false)
-          allow(::Search::Zoekt::Node).to receive(:find_or_initialize_by_task_request)
-            .with(valid_params).and_return(node)
         end
 
         context 'when node does not exist' do
@@ -52,7 +57,7 @@
             expect(response).to have_gitlab_http_status(:ok)
             expect(json_response).to eq(
               { 'id' => nil, 'tasks' => [], 'pull_frequency' => Search::Zoekt::Node::TASK_PULL_FREQUENCY_DEFAULT,
-                'truncate' => true }
+                'truncate' => true, 'stop_indexing' => false }
             )
           end
         end
@@ -68,7 +73,7 @@
             expect(response).to have_gitlab_http_status(:ok)
             expect(json_response).to eq(
               { 'id' => node.id, 'tasks' => [], 'pull_frequency' => Search::Zoekt::Node::TASK_PULL_FREQUENCY_DEFAULT,
-                'truncate' => false }
+                'truncate' => false, 'stop_indexing' => false }
             )
           end
         end
@@ -83,8 +88,6 @@
         end
 
         it 'returns node ID and tasks for task request' do
-          expect(::Search::Zoekt::Node).to receive(:find_or_initialize_by_task_request)
-            .with(valid_params).and_return(node)
           expect(node).to receive(:save).and_return(true)
 
           get api(endpoint), params: valid_params, headers: gitlab_shell_internal_api_request_header
@@ -92,7 +95,7 @@
           expect(response).to have_gitlab_http_status(:ok)
           expect(json_response).to eq(
             { 'id' => node.id, 'tasks' => tasks, 'pull_frequency' => Search::Zoekt::Node::TASK_PULL_FREQUENCY_DEFAULT,
-              'truncate' => true }
+              'truncate' => true, 'stop_indexing' => false }
           )
         end
 
@@ -102,8 +105,6 @@
           end
 
           it 'does not adds pull_frequency in the response' do
-            expect(::Search::Zoekt::Node).to receive(:find_or_initialize_by_task_request)
-                                               .with(valid_params).and_return(node)
             expect(node).to receive(:save).and_return(true)
 
             get api(endpoint), params: valid_params, headers: gitlab_shell_internal_api_request_header
@@ -111,16 +112,41 @@
             expect(::Search::Zoekt::TaskPresenterService).not_to receive(:execute)
             expect(response).to have_gitlab_http_status(:ok)
             expect(json_response).to eq({ 'id' => node.id, 'tasks' => tasks, 'truncate' => true,
-                                          'pull_frequency' =>  Search::Zoekt::Node::TASK_PULL_FREQUENCY_DEFAULT })
+                                          'pull_frequency' =>  Search::Zoekt::Node::TASK_PULL_FREQUENCY_DEFAULT,
+                                          'stop_indexing' => false })
+          end
+        end
+
+        context 'when node is over critical watermark' do
+          before do
+            allow(::Search::Zoekt::TaskPresenterService).to receive(:execute).with(node).and_return([])
+            allow(node).to receive_messages(save_debouce: true, watermark_exceeded_critical?: true)
+          end
+
+          it 'sets stop_indexing attribute in response to true' do
+            get api(endpoint), params: valid_params, headers: gitlab_shell_internal_api_request_header
+            expect(response).to have_gitlab_http_status(:ok)
+            expect(json_response).to include('stop_indexing' => true)
+          end
+
+          context 'when zoekt_critical_watermark_stop_indexing is disabled' do
+            before do
+              stub_feature_flags(zoekt_critical_watermark_stop_indexing: false)
+            end
+
+            it 'does not add stop_indexing in the response' do
+              get api(endpoint), params: valid_params, headers: gitlab_shell_internal_api_request_header
+              expect(response).to have_gitlab_http_status(:ok)
+              expect(json_response).not_to have_key('stop_indexing')
+            end
           end
         end
       end
 
       context 'when a heartbeat has valid params but a node validation error occurs' do
+        let(:node) { build(:zoekt_node, id: 123, search_base_url: nil) }
+
         it 'returns 422' do
-          node = ::Search::Zoekt::Node.new(search_base_url: nil) # null attributes makes this invalid
-          expect(::Search::Zoekt::Node).to receive(:find_or_initialize_by_task_request)
-            .with(valid_params).and_return(node)
           get api(endpoint), params: valid_params, headers: gitlab_shell_internal_api_request_header
           expect(response).to have_gitlab_http_status(:unprocessable_entity)
         end