Merge branch '419178-add-duration-column-to-vsa-tables' into 'master'

Store event durations in VSA tables

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137529



Merged-by: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com>
Approved-by: Igor Drozdov <idrozdov@gitlab.com>
Reviewed-by: Igor Drozdov <idrozdov@gitlab.com>
Co-authored-by: Adam Hegyi <ahegyi@gitlab.com>

app/models/concerns/analytics/cycle_analytics/stage_event_model.rb

@@ -111,7 +111,8 @@ def column_list
             :author_id,
             :state_id,
             :start_event_timestamp,
-            :end_event_timestamp
+            :end_event_timestamp,
+            :duration_in_milliseconds
           ]
         end
 
@@ -125,7 +126,8 @@ def insert_column_list
             :author_id,
             :state_id,
             :start_event_timestamp,
-            :end_event_timestamp
+            :end_event_timestamp,
+            :duration_in_milliseconds
           ]
         end
 

db/migrate/20231116065541_add_duration_to_issue_stage_events.rb

+# frozen_string_literal: true
+
+class AddDurationToIssueStageEvents < Gitlab::Database::Migration[2.2]
+  milestone '16.7'
+
+  def change
+    add_column :analytics_cycle_analytics_issue_stage_events, :duration_in_milliseconds, :bigint
+  end
+end

db/migrate/20231116071027_add_duration_to_mr_stage_events.rb

+# frozen_string_literal: true
+
+class AddDurationToMrStageEvents < Gitlab::Database::Migration[2.2]
+  milestone '16.7'
+
+  def change
+    add_column :analytics_cycle_analytics_merge_request_stage_events, :duration_in_milliseconds, :bigint
+  end
+end

db/schema_migrations/20231116065541

+6d5b44a7961e78b76c10453005e4fab6de12a106d5437a8ccc52312cf02d3d3d
\ No newline at end of file

db/schema_migrations/20231116071027

+7a7b73d20d7048dae1b63651b5874702b474820325ecbcd22dd0af1c8529ba59
\ No newline at end of file

ee/app/services/analytics/cycle_analytics/data_loader_service.rb

@@ -115,6 +115,8 @@ def upsert_data(records)
             # Avoid negative duration values
             next if end_event && start_event > end_event
 
+            duration_in_milliseconds = end_event && (end_event - start_event).in_milliseconds
+
             data << {
               stage_event_hash_id: stage.stage_event_hash_id,
               issuable_id: record['id'],
@@ -125,6 +127,7 @@ def upsert_data(records)
               state_id: record['state_id'],
               start_event_timestamp: start_event,
               end_event_timestamp: end_event,
+              duration_in_milliseconds: duration_in_milliseconds,
               weight: record['weight'],
               sprint_id: record['sprint_id']
             }

ee/spec/services/analytics/cycle_analytics/data_loader_service_spec.rb

@@ -93,9 +93,18 @@
     end
 
     context 'when MergeRequest data is present' do
-      let_it_be(:mr1) { create(:merge_request, :unique_branches, :with_merged_metrics, updated_at: 2.days.ago, source_project: project1) }
-      let_it_be(:mr2) { create(:merge_request, :unique_branches, :with_merged_metrics, updated_at: 5.days.ago, source_project: project1) }
-      let_it_be(:mr3) { create(:merge_request, :unique_branches, :with_merged_metrics, updated_at: 10.days.ago, source_project: project2) }
+      let_it_be(:current_time) { Time.current }
+      let_it_be(:mr1) { create(:merge_request, :unique_branches, :with_merged_metrics, created_at: current_time, updated_at: current_time + 2.days, source_project: project1) }
+      let_it_be(:mr2) { create(:merge_request, :unique_branches, :with_merged_metrics, created_at: current_time, updated_at: current_time + 5.days, source_project: project1) }
+      let_it_be(:mr3) { create(:merge_request, :unique_branches, :with_merged_metrics, created_at: current_time, updated_at: current_time + 10.days, source_project: project2) }
+
+      let(:durations) do
+        {
+          mr1 => 2.days.to_i * 1000,
+          mr2 => 5.days.to_i * 1000,
+          mr3 => 10.days.to_i * 1000
+        }
+      end
 
       it 'inserts stage records' do
         expected_data = [mr1, mr2, mr3].map do |mr|
@@ -196,11 +205,20 @@
 
     context 'when Issue data is present' do
       let_it_be(:iteration) { create(:iteration, group: top_level_group) }
-      let_it_be(:issue1) { create(:issue, project: project1, closed_at: 5.minutes.from_now, weight: 5) }
-      let_it_be(:issue2) { create(:issue, project: project1, closed_at: 5.minutes.from_now) }
-      let_it_be(:issue3) { create(:issue, project: project2, closed_at: 5.minutes.from_now, weight: 2, iteration: iteration) }
+      let_it_be(:creation_time) { Time.current }
+      let_it_be(:issue1) { create(:issue, project: project1, created_at: creation_time, closed_at: creation_time + 5.minutes, weight: 5) }
+      let_it_be(:issue2) { create(:issue, project: project1, created_at: creation_time, closed_at: creation_time + 10.minutes) }
+      let_it_be(:issue3) { create(:issue, project: project2, created_at: creation_time, closed_at: creation_time + 15.minutes, weight: 2, iteration: iteration) }
       # invalid the creation time would be later than closed_at, this should not be aggregated
-      let_it_be(:issue4) { create(:issue, project: project2, closed_at: 5.minutes.ago) }
+      let_it_be(:issue4) { create(:issue, project: project2, created_at: creation_time, closed_at: creation_time - 5.minutes) }
+
+      let(:durations) do
+        {
+          issue1 => 5.minutes.to_i * 1000,
+          issue2 => 10.minutes.to_i * 1000,
+          issue3 => 15.minutes.to_i * 1000
+        }
+      end
 
       it 'inserts stage records' do
         expected_data = [issue1, issue2, issue3].map do |issue|
@@ -213,7 +231,8 @@
             issue.closed_at,
             issue.state_id,
             issue.weight,
-            issue.sprint_id
+            issue.sprint_id,
+            durations.fetch(issue)
           ]
         end
 
@@ -229,7 +248,8 @@
             event.end_event_timestamp,
             Analytics::CycleAnalytics::IssueStageEvent.states[event.state_id],
             event.weight,
-            event.sprint_id
+            event.sprint_id,
+            event.duration_in_milliseconds
           ]
         end
 

spec/support/shared_examples/models/concerns/analytics/cycle_analytics/stage_event_model_examples.rb

@@ -14,7 +14,8 @@
           milestone_id: 6,
           state_id: 1,
           start_event_timestamp: time,
-          end_event_timestamp: time
+          end_event_timestamp: time,
+          duration_in_milliseconds: 3
         },
         {
           stage_event_hash_id: 7,
@@ -25,7 +26,8 @@
           milestone_id: 13,
           state_id: 1,
           start_event_timestamp: time,
-          end_event_timestamp: time
+          end_event_timestamp: time,
+          duration_in_milliseconds: 5
         }
       ]
     end
@@ -40,7 +42,8 @@
         :milestone_id,
         :state_id,
         :start_event_timestamp,
-        :end_event_timestamp
+        :end_event_timestamp,
+        :duration_in_milliseconds
       ]
     end