Merge branch 'bypass-min-max-queries' into 'master'

Bypass min/max queries in usage data queries

See merge request gitlab-org/gitlab!58006

ee/lib/ee/gitlab/usage_data.rb

@@ -198,7 +198,7 @@ def system_usage_data
                 geo_nodes: count(::GeoNode),
                 geo_event_log_max_id: alt_usage_data { Geo::EventLog.maximum(:id) || 0 },
                 ldap_group_links: count(::LdapGroupLink),
-                issues_with_health_status: count(::Issue.with_health_status, start: issue_minimum_id, finish: issue_maximum_id),
+                issues_with_health_status: count(::Issue.with_health_status, start: minimum_id(::Issue), finish: maximum_id(::Issue)),
                 ldap_keys: count(::LDAPKey),
                 ldap_users: count(::User.ldap, 'users.id'),
                 pod_logs_usages_total: redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] },
@@ -208,7 +208,7 @@ def system_usage_data
                 projects_mirrored_with_pipelines_enabled: count(::Project.mirrored_with_enabled_pipelines),
                 projects_reporting_ci_cd_back_to_github: count(::GithubService.active),
                 status_page_projects: count(::StatusPage::ProjectSetting.enabled),
-                status_page_issues: count(::Issue.on_status_page, start: issue_minimum_id, finish: issue_maximum_id),
+                status_page_issues: count(::Issue.on_status_page, start: minimum_id(::Issue), finish: maximum_id(::Issue)),
                 template_repositories: add(count(::Project.with_repos_templates), count(::Project.with_groups_level_repos_templates))
               },
               requirements_counts,
@@ -260,16 +260,16 @@ def usage_activity_by_stage_create(time_period)
             projects_imported_from_github: distinct_count(::Project.github_imported.where(time_period), :creator_id),
             projects_with_repositories_enabled: distinct_count(::Project.with_repositories_enabled.where(time_period),
                                                                :creator_id,
-                                                               start: user_minimum_id,
-                                                               finish: user_maximum_id),
+                                                               start: minimum_id(::User),
+                                                               finish: maximum_id(::User)),
             protected_branches: distinct_count(::Project.with_protected_branches.where(time_period),
                                                :creator_id,
-                                               start: user_minimum_id,
-                                               finish: user_maximum_id),
+                                               start: minimum_id(::User),
+                                               finish: maximum_id(::User)),
             suggestions: distinct_count(::Note.with_suggestions.where(time_period),
                                         :author_id,
-                                        start: user_minimum_id,
-                                        finish: user_maximum_id),
+                                        start: minimum_id(::User),
+                                        finish: maximum_id(::User)),
             users_using_path_locks: distinct_count(PathLock.where(time_period), :user_id),
             users_using_lfs_locks: distinct_count(LfsFileLock.where(time_period), :user_id),
             total_number_of_path_locks: count(::PathLock.where(time_period)),
@@ -377,8 +377,8 @@ def usage_activity_by_stage_secure(time_period)
           SECURE_PRODUCT_TYPES.each do |secure_type, attribs|
             results["#{prefix}#{attribs[:name]}".to_sym] = distinct_count(::Ci::Build.where(name: secure_type).where(time_period),
                                                                           :user_id,
-                                                                          start: user_minimum_id,
-                                                                          finish: user_maximum_id)
+                                                                          start: minimum_id(::User),
+                                                                          finish: maximum_id(::User))
           end
 
           results.merge!(count_secure_user_scans(time_period))
@@ -613,8 +613,8 @@ def projects_with_sectional_code_owner_rules(time_period)
               .where.not(section: ::Gitlab::CodeOwners::Entry::DEFAULT_SECTION)
               .where(time_period),
             'merge_requests.target_project_id',
-            start: project_minimum_id,
-            finish: project_maximum_id
+            start: minimum_id(::Project),
+            finish: maximum_id(::Project)
           )
         end
         # rubocop:enable CodeReuse/ActiveRecord

lib/gitlab/usage_data.rb

@@ -87,7 +87,7 @@ def recorded_at
       # rubocop: disable Metrics/AbcSize
       # rubocop: disable CodeReuse/ActiveRecord
       def system_usage_data
-        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
+        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue))
 
         {
           counts: {
@@ -138,7 +138,7 @@ def system_usage_data
             in_review_folder: count(::Environment.in_review_folder),
             grafana_integrated_projects: count(GrafanaIntegration.enabled),
             groups: count(Group),
-            issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id),
+            issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)),
             issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
             issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
             issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
@@ -146,9 +146,9 @@ def system_usage_data
             issues_created_from_alerts: total_alert_issues,
             issues_created_gitlab_alerts: issues_created_manually_from_alerts,
             issues_created_manually_from_alerts: issues_created_manually_from_alerts,
-            incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id),
-            alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id),
-            incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
+            incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)),
+            alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)),
+            incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: minimum_id(Issue), finish: maximum_id(Issue)),
             keys: count(Key),
             label_lists: count(List.label),
             lfs_objects: count(LfsObject),
@@ -389,8 +389,8 @@ def ingress_modsecurity_usage
       # rubocop: disable CodeReuse/ActiveRecord
       def container_expiration_policies_usage
         results = {}
-        start = ::Project.minimum(:id)
-        finish = ::Project.maximum(:id)
+        start = minimum_id(Project)
+        finish = maximum_id(Project)
 
         results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
         # rubocop: disable UsageData/LargeTable
@@ -591,7 +591,7 @@ def usage_activity_by_stage_manage(time_period)
         {
           events: distinct_count(::Event.where(time_period), :author_id),
           groups: distinct_count(::GroupMember.where(time_period), :user_id),
-          users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
+          users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
           omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
           user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
           unique_users_all_imports: unique_users_all_imports(time_period),
@@ -636,8 +636,8 @@ def usage_activity_by_stage_monitor(time_period)
           clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
           clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
           operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
-                                                        start: user_minimum_id,
-                                                        finish: user_maximum_id),
+                                                        start: minimum_id(User),
+                                                        finish: maximum_id(User)),
           projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id),
           projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
           projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id),
@@ -691,12 +691,12 @@ def usage_activity_by_stage_release(time_period)
       def usage_activity_by_stage_verify(time_period)
         {
           ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
-          ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
-          ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
-          ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
-          ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+          ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+          ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+          ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
           ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
-          ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
           ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
           clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
         }
@@ -801,8 +801,8 @@ def report_snowplow_events?
       end
 
       def distinct_count_service_desk_enabled_projects(time_period)
-        project_creator_id_start = user_minimum_id
-        project_creator_id_finish = user_maximum_id
+        project_creator_id_start = minimum_id(User)
+        project_creator_id_finish = maximum_id(User)
 
         distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord
       end
@@ -834,57 +834,9 @@ def unique_visit_service
       def total_alert_issues
         # Remove prometheus table queries once they are deprecated
         # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
-        add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id),
-          count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id),
-          count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id)
-      end
-
-      def user_minimum_id
-        strong_memoize(:user_minimum_id) do
-          ::User.minimum(:id)
-        end
-      end
-
-      def user_maximum_id
-        strong_memoize(:user_maximum_id) do
-          ::User.maximum(:id)
-        end
-      end
-
-      def issue_minimum_id
-        strong_memoize(:issue_minimum_id) do
-          ::Issue.minimum(:id)
-        end
-      end
-
-      def issue_maximum_id
-        strong_memoize(:issue_maximum_id) do
-          ::Issue.maximum(:id)
-        end
-      end
-
-      def deployment_minimum_id
-        strong_memoize(:deployment_minimum_id) do
-          ::Deployment.minimum(:id)
-        end
-      end
-
-      def deployment_maximum_id
-        strong_memoize(:deployment_maximum_id) do
-          ::Deployment.maximum(:id)
-        end
-      end
-
-      def project_minimum_id
-        strong_memoize(:project_minimum_id) do
-          ::Project.minimum(:id)
-        end
-      end
-
-      def project_maximum_id
-        strong_memoize(:project_maximum_id) do
-          ::Project.maximum(:id)
-        end
+        add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)),
+          count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)),
+          count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue))
       end
 
       def self_monitoring_project
@@ -918,7 +870,7 @@ def filtered_omniauth_provider_names
       end
 
       def deployment_count(relation)
-        count relation, start: deployment_minimum_id, finish: deployment_maximum_id
+        count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment)
       end
 
       def project_imports(time_period)

lib/gitlab/usage_data_queries.rb

@@ -36,6 +36,12 @@ def add(*args)
         'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ')
       end
 
+      def maximum_id(model)
+      end
+
+      def minimum_id(model)
+      end
+
       private
 
       def raw_sql(relation, column, distinct = nil)

lib/gitlab/utils/usage_data.rb

@@ -36,6 +36,7 @@
 module Gitlab
   module Utils
     module UsageData
+      include Gitlab::Utils::StrongMemoize
       extend self
 
       FALLBACK = -1
@@ -209,6 +210,20 @@ def track_usage_event(event_name, values)
         Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values)
       end
 
+      def maximum_id(model)
+        key = :"#{model.name.downcase}_maximum_id"
+        strong_memoize(key) do
+          model.maximum(:id)
+        end
+      end
+
+      def minimum_id(model)
+        key = :"#{model.name.downcase}_minimum_id"
+        strong_memoize(key) do
+          model.minimum(:id)
+        end
+      end
+
       private
 
       def prometheus_client(verify:)

spec/lib/gitlab/usage_data_queries_spec.rb

@@ -58,4 +58,24 @@
         .to eq('SELECT (SELECT COUNT("users"."id") FROM "users") + (SELECT COUNT("issues"."id") FROM "issues")')
     end
   end
+
+  describe 'min/max methods' do
+    it 'returns nil' do
+      # user min/max
+      expect(described_class.minimum_id(User)).to eq(nil)
+      expect(described_class.maximum_id(User)).to eq(nil)
+
+      # issue min/max
+      expect(described_class.minimum_id(Issue)).to eq(nil)
+      expect(described_class.maximum_id(Issue)).to eq(nil)
+
+      # deployment min/max
+      expect(described_class.minimum_id(Deployment)).to eq(nil)
+      expect(described_class.maximum_id(Deployment)).to eq(nil)
+
+      # project min/max
+      expect(described_class.minimum_id(Project)).to eq(nil)
+      expect(described_class.maximum_id(Project)).to eq(nil)
+    end
+  end
 end

spec/lib/gitlab/utils/usage_data_spec.rb

@@ -479,4 +479,22 @@ def expect_error(exception, message, &block)
       expect { described_class.track_usage_event(unknown_event, value) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
     end
   end
+
+  describe 'min/max' do
+    let(:model) { double(:relation) }
+
+    it 'returns min from the model' do
+      allow(model).to receive(:minimum).and_return(2)
+      allow(model).to receive(:name).and_return('sample_min_model')
+
+      expect(described_class.minimum_id(model)).to eq(2)
+    end
+
+    it 'returns max from the model' do
+      allow(model).to receive(:maximum).and_return(100)
+      allow(model).to receive(:name).and_return('sample_max_model')
+
+      expect(described_class.maximum_id(model)).to eq(100)
+    end
+  end
 end