diff --git a/app/models/analytics/cycle_analytics/issue_stage_event.rb b/app/models/analytics/cycle_analytics/issue_stage_event.rb index 837eb35c8391cbbda179d9c7a334c88687e557fa..4930788c8b5cc35a794b9e4db2a13b7bc21d083c 100644 --- a/app/models/analytics/cycle_analytics/issue_stage_event.rb +++ b/app/models/analytics/cycle_analytics/issue_stage_event.rb @@ -17,12 +17,42 @@ class IssueStageEvent < ApplicationRecord where(condition.arel.exists) end - def self.issuable_id_column - :issue_id - end + class << self + def project_column + :project_id + end + + def issuable_id_column + :issue_id + end + + def issuable_model + ::Issue + end + + def select_columns + [ + *super, + issuable_model.arel_table[:weight], + issuable_model.arel_table[:sprint_id] + ] + end + + def column_list + [ + *super, + :weight, + :sprint_id + ] + end - def self.issuable_model - ::Issue + def insert_column_list + [ + *super, + :weight, + :sprint_id + ] + end end end end diff --git a/app/models/analytics/cycle_analytics/merge_request_stage_event.rb b/app/models/analytics/cycle_analytics/merge_request_stage_event.rb index 0dfa322b2c3c39ea9d58228812679a295d14e911..7f85d2840347c4541ea8a8b7d539a45496c61db7 100644 --- a/app/models/analytics/cycle_analytics/merge_request_stage_event.rb +++ b/app/models/analytics/cycle_analytics/merge_request_stage_event.rb @@ -17,6 +17,10 @@ class MergeRequestStageEvent < ApplicationRecord where(condition.arel.exists) end + def self.project_column + :target_project_id + end + def self.issuable_id_column :merge_request_id end diff --git a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb index d268c32c08859f8359113700718c51678320e215..8f20e3880b35ee767b26a3355f45dae26985a92a 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb @@ -57,45 +57,19 @@ def total_time class_methods do def upsert_data(data) - upsert_values = data.map do |row| - row.values_at( - :stage_event_hash_id, - :issuable_id, - :group_id, - :project_id, - :milestone_id, - :author_id, - :state_id, - :start_event_timestamp, - :end_event_timestamp - ) - end + upsert_values = data.map { |row| row.values_at(*column_list) } value_list = Arel::Nodes::ValuesList.new(upsert_values).to_sql query = <<~SQL INSERT INTO #{quoted_table_name} ( - stage_event_hash_id, - #{connection.quote_column_name(issuable_id_column)}, - group_id, - project_id, - milestone_id, - author_id, - state_id, - start_event_timestamp, - end_event_timestamp + #{insert_column_list.join(",\n")} ) #{value_list} ON CONFLICT(stage_event_hash_id, #{issuable_id_column}) DO UPDATE SET - group_id = excluded.group_id, - project_id = excluded.project_id, - milestone_id = excluded.milestone_id, - author_id = excluded.author_id, - state_id = excluded.state_id, - start_event_timestamp = excluded.start_event_timestamp, - end_event_timestamp = excluded.end_event_timestamp + #{column_updates.join(",\n")} SQL result = connection.execute(query) @@ -113,6 +87,51 @@ def keyset_order(column_definition_options) def arel_order(arel_node, direction) direction.to_sym == :desc ? arel_node.desc : arel_node.asc end + + def select_columns + [ + issuable_model.arel_table[:id], + issuable_model.arel_table[project_column].as('project_id'), + issuable_model.arel_table[:milestone_id], + issuable_model.arel_table[:author_id], + issuable_model.arel_table[:state_id], + Project.arel_table[:parent_id].as('group_id') + ] + end + + def column_list + [ + :stage_event_hash_id, + :issuable_id, + :group_id, + :project_id, + :milestone_id, + :author_id, + :state_id, + :start_event_timestamp, + :end_event_timestamp + ] + end + + def insert_column_list + [ + :stage_event_hash_id, + connection.quote_column_name(issuable_id_column), + :group_id, + :project_id, + :milestone_id, + :author_id, + :state_id, + :start_event_timestamp, + :end_event_timestamp + ] + end + + def column_updates + insert_column_list.map do |column| + "#{column} = excluded.#{column}" + end + end end end end diff --git a/db/migrate/20230918123357_add_sprint_id_and_weight_to_vsa_issues.rb b/db/migrate/20230918123357_add_sprint_id_and_weight_to_vsa_issues.rb new file mode 100644 index 0000000000000000000000000000000000000000..0c3b7b3086a30a7c269a83bdfd4f9e176a9761bc --- /dev/null +++ b/db/migrate/20230918123357_add_sprint_id_and_weight_to_vsa_issues.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddSprintIdAndWeightToVsaIssues < Gitlab::Database::Migration[2.1] + enable_lock_retries! + + def up + add_column :analytics_cycle_analytics_issue_stage_events, :weight, :integer + add_column :analytics_cycle_analytics_issue_stage_events, :sprint_id, :bigint + end + + def down + remove_column :analytics_cycle_analytics_issue_stage_events, :sprint_id + remove_column :analytics_cycle_analytics_issue_stage_events, :weight + end +end diff --git a/db/schema_migrations/20230918123357 b/db/schema_migrations/20230918123357 new file mode 100644 index 0000000000000000000000000000000000000000..5fc1d62a96366a8019f781daa7f2dd97d1efdd6a --- /dev/null +++ b/db/schema_migrations/20230918123357 @@ -0,0 +1 @@ +99e55170557dcda361f441d1333f4dc9d99133a469f1d17805478f3407d2a093 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9283a0c4800cdc1d9a449428379fd9f41136c063..469692bb884cffe32d2bbf139cd75470e7ce2f32 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -720,7 +720,9 @@ CREATE TABLE analytics_cycle_analytics_issue_stage_events ( author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ) PARTITION BY HASH (stage_event_hash_id); @@ -733,7 +735,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_00 FOR VALUES WITH (modulus 32, remainder 0); @@ -746,7 +750,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_01 FOR VALUES WITH (modulus 32, remainder 1); @@ -759,7 +765,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_02 FOR VALUES WITH (modulus 32, remainder 2); @@ -772,7 +780,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_03 FOR VALUES WITH (modulus 32, remainder 3); @@ -785,7 +795,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_04 FOR VALUES WITH (modulus 32, remainder 4); @@ -798,7 +810,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_05 FOR VALUES WITH (modulus 32, remainder 5); @@ -811,7 +825,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_06 FOR VALUES WITH (modulus 32, remainder 6); @@ -824,7 +840,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_07 FOR VALUES WITH (modulus 32, remainder 7); @@ -837,7 +855,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_08 FOR VALUES WITH (modulus 32, remainder 8); @@ -850,7 +870,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_09 FOR VALUES WITH (modulus 32, remainder 9); @@ -863,7 +885,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_10 FOR VALUES WITH (modulus 32, remainder 10); @@ -876,7 +900,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_11 FOR VALUES WITH (modulus 32, remainder 11); @@ -889,7 +915,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_12 FOR VALUES WITH (modulus 32, remainder 12); @@ -902,7 +930,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_13 FOR VALUES WITH (modulus 32, remainder 13); @@ -915,7 +945,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_14 FOR VALUES WITH (modulus 32, remainder 14); @@ -928,7 +960,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_15 FOR VALUES WITH (modulus 32, remainder 15); @@ -941,7 +975,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_16 FOR VALUES WITH (modulus 32, remainder 16); @@ -954,7 +990,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_17 FOR VALUES WITH (modulus 32, remainder 17); @@ -967,7 +1005,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_18 FOR VALUES WITH (modulus 32, remainder 18); @@ -980,7 +1020,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_19 FOR VALUES WITH (modulus 32, remainder 19); @@ -993,7 +1035,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_20 FOR VALUES WITH (modulus 32, remainder 20); @@ -1006,7 +1050,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_21 FOR VALUES WITH (modulus 32, remainder 21); @@ -1019,7 +1065,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_22 FOR VALUES WITH (modulus 32, remainder 22); @@ -1032,7 +1080,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_23 FOR VALUES WITH (modulus 32, remainder 23); @@ -1045,7 +1095,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_24 FOR VALUES WITH (modulus 32, remainder 24); @@ -1058,7 +1110,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_25 FOR VALUES WITH (modulus 32, remainder 25); @@ -1071,7 +1125,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_26 FOR VALUES WITH (modulus 32, remainder 26); @@ -1084,7 +1140,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_27 FOR VALUES WITH (modulus 32, remainder 27); @@ -1097,7 +1155,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_28 FOR VALUES WITH (modulus 32, remainder 28); @@ -1110,7 +1170,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_29 FOR VALUES WITH (modulus 32, remainder 29); @@ -1123,7 +1185,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_30 FOR VALUES WITH (modulus 32, remainder 30); @@ -1136,7 +1200,9 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_issue_stage_even author_id bigint, start_event_timestamp timestamp with time zone NOT NULL, end_event_timestamp timestamp with time zone, - state_id smallint DEFAULT 1 NOT NULL + state_id smallint DEFAULT 1 NOT NULL, + weight integer, + sprint_id bigint ); ALTER TABLE ONLY analytics_cycle_analytics_issue_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_issue_stage_events_31 FOR VALUES WITH (modulus 32, remainder 31); diff --git a/ee/app/services/analytics/cycle_analytics/data_loader_service.rb b/ee/app/services/analytics/cycle_analytics/data_loader_service.rb index 061ee7edc239ba7ce86af9f75051498ae45bc4da..d792205ff5e7dd81995c3d06566bcf8fff124b32 100644 --- a/ee/app/services/analytics/cycle_analytics/data_loader_service.rb +++ b/ee/app/services/analytics/cycle_analytics/data_loader_service.rb @@ -11,8 +11,8 @@ class DataLoaderService EVENTS_LIMIT = 25 CONFIG_MAPPING = { - Issue => { event_model: IssueStageEvent, project_column: :project_id }.freeze, - MergeRequest => { event_model: MergeRequestStageEvent, project_column: :target_project_id }.freeze + Issue => { event_model: IssueStageEvent }.freeze, + MergeRequest => { event_model: MergeRequestStageEvent }.freeze }.freeze def initialize(group:, model:, context: Analytics::CycleAnalytics::AggregationContext.new) @@ -67,7 +67,9 @@ def iterator opts = { in_operator_optimization_options: { array_scope: group.all_projects.select(:id), - array_mapping_scope: -> (id_expression) { model.where(model.arel_table[project_column].eq(id_expression)) } + array_mapping_scope: -> (id_expression) { + model.where(model.arel_table[event_model.project_column].eq(id_expression)) + } } } @@ -82,7 +84,7 @@ def load_timestamp_data_into_value_stream_analytics(loaded_records) events.each_slice(EVENTS_LIMIT) do |event_slice| scope = model.join_project.id_in(loaded_records.pluck(:id)) - current_select_columns = select_columns # default SELECT columns + current_select_columns = event_model.select_columns # default SELECT columns # Add the stage timestamp columns to the SELECT event_slice.each do |event| scope = event.include_in(scope) @@ -122,7 +124,9 @@ def upsert_data(records) milestone_id: record['milestone_id'], state_id: record['state_id'], start_event_timestamp: start_event, - end_event_timestamp: end_event + end_event_timestamp: end_event, + weight: record['weight'], + sprint_id: record['sprint_id'] } if data.size == UPSERT_LIMIT @@ -135,17 +139,6 @@ def upsert_data(records) @upsert_count += event_model.upsert_data(data) if data.any? end - def select_columns - [ - model.arel_table[:id], - model.arel_table[project_column].as('project_id'), - model.arel_table[:milestone_id], - model.arel_table[:author_id], - model.arel_table[:state_id], - Project.arel_table[:parent_id].as('group_id') - ] - end - # rubocop: disable CodeReuse/ActiveRecord def cursor_for_node(record) scope, _ = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(iterator_base_scope) @@ -154,10 +147,6 @@ def cursor_for_node(record) end # rubocop: enable CodeReuse/ActiveRecord - def project_column - CONFIG_MAPPING.fetch(model).fetch(:project_column) - end - def event_model CONFIG_MAPPING.fetch(model).fetch(:event_model) end diff --git a/ee/spec/services/analytics/cycle_analytics/data_loader_service_spec.rb b/ee/spec/services/analytics/cycle_analytics/data_loader_service_spec.rb index 42e6917afb91d2634cef8aa88e043ebb5e10d46d..43fd7f20cbcb91bed448e4b91dcc7692a0f80690 100644 --- a/ee/spec/services/analytics/cycle_analytics/data_loader_service_spec.rb +++ b/ee/spec/services/analytics/cycle_analytics/data_loader_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Analytics::CycleAnalytics::DataLoaderService do +RSpec.describe Analytics::CycleAnalytics::DataLoaderService, feature_category: :value_stream_management do let_it_be_with_refind(:top_level_group) { create(:group) } describe 'validations' do @@ -195,9 +195,10 @@ end context 'when Issue data is present' do - let_it_be(:issue1) { create(:issue, project: project1, closed_at: 5.minutes.from_now) } + let_it_be(:iteration) { create(:iteration, group: top_level_group) } + let_it_be(:issue1) { create(:issue, project: project1, closed_at: 5.minutes.from_now, weight: 5) } let_it_be(:issue2) { create(:issue, project: project1, closed_at: 5.minutes.from_now) } - let_it_be(:issue3) { create(:issue, project: project2, closed_at: 5.minutes.from_now) } + let_it_be(:issue3) { create(:issue, project: project2, closed_at: 5.minutes.from_now, weight: 2, iteration: iteration) } # invalid the creation time would be later than closed_at, this should not be aggregated let_it_be(:issue4) { create(:issue, project: project2, closed_at: 5.minutes.ago) } @@ -210,7 +211,9 @@ issue.project_id, issue.created_at, issue.closed_at, - issue.state_id + issue.state_id, + issue.weight, + issue.sprint_id ] end @@ -224,7 +227,9 @@ event.project_id, event.start_event_timestamp, event.end_event_timestamp, - Analytics::CycleAnalytics::IssueStageEvent.states[event.state_id] + Analytics::CycleAnalytics::IssueStageEvent.states[event.state_id], + event.weight, + event.sprint_id ] end diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index cfd6bbf3094df466daa83a085a9f8c221cd785ec..e4e1772f08e61acd55d498c8f44e392106bce085 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -36,7 +36,7 @@ approvers: %w[target_id user_id], analytics_cycle_analytics_aggregations: %w[last_full_issues_id last_full_merge_requests_id last_incremental_issues_id last_full_run_issues_id last_full_run_merge_requests_id last_incremental_merge_requests_id last_consistency_check_issues_stage_event_hash_id last_consistency_check_issues_issuable_id last_consistency_check_merge_requests_stage_event_hash_id last_consistency_check_merge_requests_issuable_id], analytics_cycle_analytics_merge_request_stage_events: %w[author_id group_id merge_request_id milestone_id project_id stage_event_hash_id state_id], - analytics_cycle_analytics_issue_stage_events: %w[author_id group_id issue_id milestone_id project_id stage_event_hash_id state_id], + analytics_cycle_analytics_issue_stage_events: %w[author_id group_id issue_id milestone_id project_id stage_event_hash_id state_id sprint_id], audit_events: %w[author_id entity_id target_id], award_emoji: %w[awardable_id user_id], aws_roles: %w[role_external_id],