Introduce histogram for ApplicationRateLimiter

This MR removes the emit_application_rate_limiter_histogram feature flag
and starts tracking application rate limit throttles using a histogram.

Changelog: other
MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169655

config/feature_flags/gitlab_com_derisk/emit_application_rate_limiter_histogram.yml

----
-name: emit_application_rate_limiter_histogram
-feature_issue_url: https://gitlab.com/gitlab-com/gl-infra/production-engineering/-/work_items/25767
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169655
-rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/499992
-milestone: '17.6'
-group: group::scalability
-type: gitlab_com_derisk
-default_enabled: false

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -197,7 +197,7 @@ The following metrics are available:
 | `gitlab_rack_attack_events_total` | Counter | 17.6 | Counts the total number of events handled by Rack Attack. | `event_type`, `event_name` |
 | `gitlab_rack_attack_throttle_limit` | Gauge | 17.6 | Reports the maximum number of requests that a client can make before Rack Attack throttles them. | `event_name` |
 | `gitlab_rack_attack_throttle_period_seconds` | Gauge | 17.6 | Reports the duration over which requests for a client are counted before Rack Attack throttles them. | `event_name` |
-| `gitlab_application_rate_limiter_throttles_total` | Histogram | 17.6 | Utilization ratio of a throttle in GitLab Application Rate Limiter. | `throttle_key`, `peek`, `feature_category` |
+| `gitlab_application_rate_limiter_throttle_utilization_ratio` | Histogram | 17.6 | Utilization ratio of a throttle in GitLab Application Rate Limiter. | `throttle_key`, `peek`, `feature_category` |
 
 ## Metrics controlled by a feature flag
 

lib/gitlab/application_rate_limiter.rb

@@ -199,7 +199,6 @@ def peek(key, scope:, threshold: nil, interval: nil, users_allowlist: nil)
       end
 
       def report_metrics(key, value, threshold, peek)
-        return if Feature.disabled?(:emit_application_rate_limiter_histogram, Feature.current_request)
         return if threshold == 0 # guard against div-by-zero
 
         label = {

spec/lib/gitlab/application_rate_limiter_spec.rb

@@ -77,6 +77,12 @@
       let(:project1) { instance_double(Project, id: '1') }
       let(:project2) { instance_double(Project, id: '2') }
 
+      before do
+        if described_class.instance_variable_defined?(:@application_rate_limiter_histogram)
+          described_class.remove_instance_variable(:@application_rate_limiter_histogram)
+        end
+      end
+
       it 'returns true when unique actioned resources count exceeds threshold' do
         travel_to(start_time) do
           expect(subject.throttled?(:test_action, scope: scope, resource: project1)).to eq(false)
@@ -108,15 +114,21 @@
       end
     end
 
-    context 'when the emit_application_rate_limiter_histogram feature flag is enabled' do
-      before do
+    describe 'emitting metrics for throttling utilization' do
+      let(:histogram_double) { instance_double(Prometheus::Client::Histogram) }
+
+      around do |example|
+        # check if defined
         if described_class.instance_variable_defined?(:@application_rate_limiter_histogram)
           described_class.remove_instance_variable(:@application_rate_limiter_histogram)
         end
+
+        example.run
+
+        described_class.remove_instance_variable(:@application_rate_limiter_histogram)
       end
 
       it 'observe histogram metrics using a memoized histogram instance' do
-        histogram_double = instance_double(Prometheus::Client::Histogram)
         expect(Gitlab::Metrics).to receive(:histogram)
           .once
           .with(
@@ -128,24 +140,8 @@
           .and_return(histogram_double)
         expect(histogram_double).to receive(:observe).twice
 
-        subject.throttled?(:test_action, scope: [])
-        subject.throttled?(:test_action, scope: [])
-      end
-    end
-
-    context 'when the emit_application_rate_limiter_histogram feature flag is disabled' do
-      before do
-        if described_class.instance_variable_defined?(:@application_rate_limiter_histogram)
-          described_class.remove_instance_variable(:@application_rate_limiter_histogram)
-        end
-
-        stub_feature_flags(emit_application_rate_limiter_histogram: false)
-      end
-
-      it 'does not emit metrics' do
-        expect(Gitlab::Metrics).not_to receive(:histogram)
-
-        subject.throttled?(:test_action, scope: [])
+        subject.throttled?(:test_action, scope: [], threshold: 1)
+        subject.throttled?(:test_action, scope: [], threshold: 1)
       end
     end