Schedule DORA DF score recalculation

Old data has DF score as high even with
1 deployment per month. This data migration
recalculates all previous DF scores to
correct ones.

Changelog: added
EE: true

db/docs/batched_background_migrations/recalculate_dora_deployment_frequency_score.yml

+---
+migration_job_name: RecalculateDoraDeploymentFrequencyScore
+description: Recalculates DORA DF scores for all projects with a score
+feature_category: dora_metrics
+introduced_by_url: 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/160259'
+milestone: '17.3'
+queued_migration_version: 20240722104700
+finalize_after: '2024-08-20'
+finalized_by: # version of the migration that finalized this BBM

db/post_migrate/20240722104700_queue_recalculate_dora_deployment_frequency_score.rb

+# frozen_string_literal: true
+
+class QueueRecalculateDoraDeploymentFrequencyScore < Gitlab::Database::Migration[2.2]
+  milestone '17.3'
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  MIGRATION = "RecalculateDoraDeploymentFrequencyScore"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 500
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :dora_performance_scores,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :dora_performance_scores, :id, [])
+  end
+end

db/schema_migrations/20240722104700

+2bb3736f72c9cb6b00510f629d0c924447d5389263420bd51dbbf4220bb7a7e8
\ No newline at end of file

lib/gitlab/background_migration/recalculate_dora_deployment_frequency_score.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class RecalculateDoraDeploymentFrequencyScore < BatchedMigrationJob
+      operation_name :recalculate_dora_df_score
+      feature_category :dora_metrics
+
+      SCORE_BOUNDARIES = { low: 0.033, high: 1.0 }.freeze
+
+      class DailyMetric < ApplicationRecord
+        self.table_name = 'dora_daily_metrics'
+      end
+
+      class Environment < ApplicationRecord
+        self.table_name = 'environments'
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch_score_groups(sub_batch).each do |score, group|
+            sub_batch.where(id: group.map(&:last)).update_all(deployment_frequency: score)
+          end
+        end
+      end
+
+      private
+
+      def sub_batch_score_groups(sub_batch)
+        sub_batch.map { |row| [recalculate_df_score(row), row.id] }.group_by(&:first)
+      end
+
+      def recalculate_df_score(row)
+        from = row.date.beginning_of_month
+        to = row.date.end_of_month
+        prods = Environment.where(project_id: row.project_id).where(tier: 0)
+
+        data = DailyMetric.where(environment_id: prods)
+                          .where(date: from..to)
+                          .select('SUM(deployment_frequency) as deployments_count').take
+
+        deployment_frequency = (data['deployments_count'] || 0) / (to - from + 1).to_f
+
+        frequency_score(deployment_frequency)
+      end
+
+      def frequency_score(value)
+        return 10 if value < SCORE_BOUNDARIES[:low]
+        return 30 if value > SCORE_BOUNDARIES[:high]
+
+        20
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/recalculate_dora_deployment_frequency_score_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::RecalculateDoraDeploymentFrequencyScore, feature_category: :dora_metrics do
+  describe '#perform', :freeze_time do
+    let(:environments) { table(:environments) }
+    let(:namespaces) { table(:namespaces) }
+    let(:projects) { table(:projects) }
+    let(:performance_scores) { table(:dora_performance_scores) }
+    let(:daily_metrics) { table(:dora_daily_metrics) }
+
+    let!(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+    let!(:project) do
+      projects.create!(namespace_id: namespace.id, name: 'foo', project_namespace_id: namespace.id)
+    end
+
+    let!(:production) { environments.create!(project_id: project.id, tier: 0, name: 'prod', slug: 'prod') }
+    let!(:staging) { environments.create!(project_id: project.id, tier: 1, name: 'stg', slug: 'stg') }
+
+    let!(:high_performance_score) do
+      performance_scores.create!(project_id: project.id, date: 30.days.ago.beginning_of_month,
+        deployment_frequency: 30).tap do |s|
+        daily_metrics.create!(project_id: project.id, environment_id: production.id, deployment_frequency: 100,
+          date: s.date)
+        daily_metrics.create!(project_id: project.id, environment_id: staging.id, deployment_frequency: 0, date: s.date)
+      end
+    end
+
+    let!(:mid_performance_score) do
+      performance_scores.create!(project_id: project.id, date: 90.days.ago.beginning_of_month,
+        deployment_frequency: 30).tap do |s|
+        daily_metrics.create!(project_id: project.id, environment_id: production.id, deployment_frequency: 15,
+          date: s.date)
+        daily_metrics.create!(project_id: project.id, environment_id: production.id, deployment_frequency: 100,
+          date: s.date - 1)
+        daily_metrics.create!(project_id: project.id, environment_id: staging.id, deployment_frequency: 100,
+          date: s.date)
+      end
+    end
+
+    let!(:low_performance_score) do
+      performance_scores.create!(project_id: project.id, date: 180.days.ago.beginning_of_month,
+        deployment_frequency: 30).tap do |s|
+        daily_metrics.create!(project_id: project.id, environment_id: production.id, deployment_frequency: 0,
+          date: s.date)
+        daily_metrics.create!(project_id: project.id, environment_id: production.id, deployment_frequency: 100,
+          date: s.date - 1)
+        daily_metrics.create!(project_id: project.id, environment_id: staging.id, deployment_frequency: 100,
+          date: s.date)
+      end
+    end
+
+    let!(:low_performance_score_2) do
+      # score with no daily metrics
+      performance_scores.create!(project_id: project.id, date: 240.days.ago.beginning_of_month,
+        deployment_frequency: 30)
+    end
+
+    subject(:migration) do
+      described_class.new(
+        start_id: performance_scores.order(id: :asc).first.id,
+        end_id: performance_scores.order(id: :asc).last.id,
+        batch_table: :dora_performance_scores,
+        batch_column: :id,
+        sub_batch_size: 200,
+        pause_ms: 2.minutes,
+        connection: ApplicationRecord.connection
+      )
+    end
+
+    it 'updates deployment frequency score with correct value' do
+      migration.perform
+
+      expect(high_performance_score.reload.deployment_frequency).to eq(30)
+      expect(mid_performance_score.reload.deployment_frequency).to eq(20)
+      expect(low_performance_score.reload.deployment_frequency).to eq(10)
+      expect(low_performance_score_2.reload.deployment_frequency).to eq(10)
+    end
+  end
+end

spec/migrations/20240722104700_queue_recalculate_dora_deployment_frequency_score_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueRecalculateDoraDeploymentFrequencyScore, feature_category: :dora_metrics do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :dora_performance_scores,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      }
+    end
+  end
+end