Merge branch...

Merge branch '433408-spike-prepare-dashboards-to-observe-performance-of-security-policies-workers' into 'master' 

Add execution duration metric for ProcessScanResultPolicyWorker

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139263



Merged-by: Andy Soiron <asoiron@gitlab.com>
Approved-by: Lorena Ciutacu <lciutacu@gitlab.com>
Approved-by: Sashi Kumar Kumaresan <skumar@gitlab.com>
Approved-by: Andy Soiron <asoiron@gitlab.com>
Reviewed-by: Lorena Ciutacu <lciutacu@gitlab.com>
Co-authored-by: Dominic Bauer <dbauer@gitlab.com>

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -179,6 +179,7 @@ The following metrics are available:
 | `gitlab_ci_queue_active_runners_total` | Histogram | 16.3 | The amount of active runners that can process queue in a project |
 | `gitlab_connection_pool_size` | Gauge | 16.7 | Size of connection pool |
 | `gitlab_connection_pool_available_count` | Gauge | 16.7 | Number of available connections in the pool |
+| `gitlab_security_policies_scan_result_process_duration_seconds` | Histogram | 16.7 | The amount of time to process scan result policies |
 
 ## Metrics controlled by a feature flag
 

ee/app/workers/security/process_scan_result_policy_worker.rb

@@ -11,16 +11,20 @@ class ProcessScanResultPolicyWorker
     sidekiq_options retry: true
     feature_category :security_policy_management
 
+    HISTOGRAM_BUCKETS = [120, 240, 360, 480, 600, 720, 840, 960].freeze
+
     def perform(project_id, configuration_id)
-      project = Project.find_by_id(project_id)
-      configuration = Security::OrchestrationPolicyConfiguration.find_by_id(configuration_id)
-      return unless project && configuration
+      measure(project_id, configuration_id) do
+        project = Project.find_by_id(project_id)
+        configuration = Security::OrchestrationPolicyConfiguration.find_by_id(configuration_id)
+        break unless project && configuration
 
-      sync_policies(project, configuration, applicable_active_policies(configuration, project))
+        sync_policies(project, configuration, applicable_active_policies(configuration, project))
 
-      Security::SecurityOrchestrationPolicies::SyncOpenedMergeRequestsService
-        .new(project: project, policy_configuration: configuration)
-        .execute
+        Security::SecurityOrchestrationPolicies::SyncOpenedMergeRequestsService
+          .new(project: project, policy_configuration: configuration)
+          .execute
+      end
     end
 
     private
@@ -46,5 +50,21 @@ def sync_policies(project, configuration, active_scan_result_policies)
           .execute
       end
     end
+
+    def measure(project_id, configuration_id)
+      lo = ::Gitlab::Metrics::System.monotonic_time
+      yield
+      hi = ::Gitlab::Metrics::System.monotonic_time
+
+      histogram.observe({ project_id: project_id, configuration_id: configuration_id }, hi - lo)
+    end
+
+    def histogram
+      Gitlab::Metrics.histogram(
+        :gitlab_security_policies_scan_result_process_duration_seconds,
+        'The amount of time to process scan result policies',
+        {},
+        HISTOGRAM_BUCKETS)
+    end
   end
 end

ee/spec/workers/security/process_scan_result_policy_worker_spec.rb

@@ -58,6 +58,29 @@
   describe '#perform' do
     subject(:worker) { described_class.new }
 
+    describe 'metrics' do
+      specify do
+        expect(Gitlab::Metrics).to receive(:histogram).with(
+          :gitlab_security_policies_scan_result_process_duration_seconds,
+          'The amount of time to process scan result policies',
+          {},
+          described_class::HISTOGRAM_BUCKETS
+        ).and_call_original
+
+        worker.perform(configuration.project_id, configuration.id)
+      end
+
+      specify do
+        histogram = instance_double('Prometheus::Client::Histogram')
+        expect(histogram).to receive(:observe).with(
+          { project_id: configuration.project_id,
+            configuration_id: configuration.id }, an_instance_of(Float))
+
+        allow(worker).to receive(:histogram).and_return(histogram)
+        worker.perform(configuration.project_id, configuration.id)
+      end
+    end
+
     it_behaves_like 'when no policy is applicable due to the policy scope' do
       it 'does not call ProcessScanResultPolicyService to create approval rules' do
         expect(Security::SecurityOrchestrationPolicies::ProcessScanResultPolicyService).not_to receive(:new)