diff --git a/config/feature_flags/development/cache_issue_sums.yml b/config/feature_flags/development/cache_issue_sums.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9d045eca2c94a909990a0b2d8779f24b3f1000cd
--- /dev/null
+++ b/config/feature_flags/development/cache_issue_sums.yml
@@ -0,0 +1,8 @@
+---
+name: cache_issue_sums
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95048
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/365940
+milestone: '15.3'
+type: development
+group: group::product planning
+default_enabled: false
diff --git a/ee/app/models/ee/epic.rb b/ee/app/models/ee/epic.rb
index 7d4d6bd3536fc5c1e7645e19c49f21ee8819f1f1..b83635544c22532c7c059f5e9f333fdae7a5e408 100644
--- a/ee/app/models/ee/epic.rb
+++ b/ee/app/models/ee/epic.rb
@@ -176,6 +176,7 @@ def etag_caching_enabled?
before_save :set_fixed_start_date, if: :start_date_is_fixed?
before_save :set_fixed_due_date, if: :due_date_is_fixed?
after_create_commit :usage_ping_record_epic_creation
+ after_commit :update_cached_metadata
def epic_tree_root?
parent_id.nil?
@@ -613,5 +614,75 @@ def blocked_by_epics_for(user)
unfiltered_epics = self.class.where(id: blocking_epics_ids)
self.class.epics_readable_by_user(unfiltered_epics, user)
end
+
+ def total_issue_weight_and_count
+ subepic_sums = subepics_weight_and_count
+ issue_sums = issues_weight_and_count
+
+ {
+ total_opened_issue_weight: subepic_sums[:opened_issue_weight] + issue_sums[:opened_issue_weight],
+ total_closed_issue_weight: subepic_sums[:closed_issue_weight] + issue_sums[:closed_issue_weight],
+ total_opened_issue_count: subepic_sums[:opened_issue_count] + issue_sums[:opened_issue_count],
+ total_closed_issue_count: subepic_sums[:closed_issue_count] + issue_sums[:closed_issue_count]
+ }
+ end
+
+ def subepics_weight_and_count
+ sum = children.select(
+ 'SUM(total_opened_issue_weight) AS opened_issue_weight',
+ 'SUM(total_closed_issue_weight) AS closed_issue_weight',
+ 'SUM(total_opened_issue_count) AS opened_issue_count',
+ 'SUM(total_closed_issue_count) AS closed_issue_count'
+ )[0]
+
+ {
+ opened_issue_weight: sum.opened_issue_weight.to_i,
+ closed_issue_weight: sum.closed_issue_weight.to_i,
+ opened_issue_count: sum.opened_issue_count.to_i,
+ closed_issue_count: sum.closed_issue_count.to_i
+ }
+ end
+
+ def issues_weight_and_count
+ state_sums = issues
+ .select('issues.state_id AS issues_state_id',
+ 'SUM(COALESCE(issues.weight, 0)) AS issues_weight_sum',
+ 'COUNT(issues.id) AS issues_count')
+ .reorder(nil)
+ .group("issues.state_id")
+
+ by_state = state_sums.each_with_object({}) do |state_sum, result|
+ key = ::Issue.available_states.key(state_sum.issues_state_id)
+ result[key] = state_sum
+ end
+
+ {
+ opened_issue_weight: by_state['opened']&.issues_weight_sum.to_i,
+ closed_issue_weight: by_state['closed']&.issues_weight_sum.to_i,
+ opened_issue_count: by_state['opened']&.issues_count.to_i,
+ closed_issue_count: by_state['closed']&.issues_count.to_i
+ }
+ end
+
+ def propagate_issue_metadata_change?
+ return false unless parent_id
+ return true if destroyed?
+
+ attrs = %w[total_opened_issue_weight total_closed_issue_weight
+ total_opened_issue_count total_closed_issue_count
+ parent_id]
+
+ (previous_changes.keys & attrs).any?
+ end
+
+ def update_cached_metadata
+ return unless ::Feature.enabled?(:cache_issue_sums)
+
+ ::Epics::UpdateCachedMetadataWorker.perform_async([parent_id]) if propagate_issue_metadata_change?
+
+ if parent_id_previously_changed? && parent_id_previously_was
+ ::Epics::UpdateCachedMetadataWorker.perform_async([parent_id_previously_was])
+ end
+ end
end
end
diff --git a/ee/app/models/ee/issue.rb b/ee/app/models/ee/issue.rb
index 8de14ed9fc535d3704f7adfddaff71de0a3bd272..8e1bd31ca3434e42184f19881d7c9218ce461f33 100644
--- a/ee/app/models/ee/issue.rb
+++ b/ee/app/models/ee/issue.rb
@@ -138,6 +138,8 @@ module Issue
::Dora::DailyMetrics::RefreshWorker.perform_async(related_production_env.id, issue.created_at.to_date.to_s)
end
+
+ after_commit :update_cached_metadata
end
class_methods do
@@ -337,5 +339,13 @@ def confidentiality_error
_('This issue cannot be assigned to a confidential epic because it is public.')
end
+
+ def update_cached_metadata
+ return unless ::Feature.enabled?(:cache_issue_sums)
+ return unless epic_issue
+ return unless weight_previously_changed? || state_id_previously_changed?
+
+ ::Epics::UpdateCachedMetadataWorker.perform_async([epic_issue.epic_id])
+ end
end
end
diff --git a/ee/app/models/epic_issue.rb b/ee/app/models/epic_issue.rb
index 6f97629a48c929f0c6c2d1e8b687f88011c7f1f0..20eaa0ebeb3a575632d1bdd1ecaadfc47024f854 100644
--- a/ee/app/models/epic_issue.rb
+++ b/ee/app/models/epic_issue.rb
@@ -18,6 +18,7 @@ class EpicIssue < ApplicationRecord
validate :validate_confidential_epic
validate :validate_group_structure
+ after_commit :update_cached_metadata
def epic_tree_root?
false
@@ -48,4 +49,14 @@ def validate_confidential_epic
errors.add :issue, _('Cannot assign a confidential epic to a non-confidential issue. Make the issue confidential and try again')
end
end
+
+ def update_cached_metadata
+ return unless ::Feature.enabled?(:cache_issue_sums)
+
+ ::Epics::UpdateCachedMetadataWorker.perform_async([epic_id])
+
+ if epic_id_previously_changed? && epic_id_previously_was
+ ::Epics::UpdateCachedMetadataWorker.perform_async([epic_id_previously_was])
+ end
+ end
end
diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml
index 77eee3611c683c5a2e7307b1478eb5d25af82a69..f994d0206779b2f38040901ee0a1360c904ee22a 100644
--- a/ee/app/workers/all_queues.yml
+++ b/ee/app/workers/all_queues.yml
@@ -507,6 +507,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: epics:epics_update_cached_metadata
+ :worker_name: Epics::UpdateCachedMetadataWorker
+ :feature_category: :portfolio_management
+ :has_external_dependencies: false
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 2
+ :idempotent: true
+ :tags: []
- :name: epics:epics_update_epics_dates
:worker_name: Epics::UpdateEpicsDatesWorker
:feature_category: :portfolio_management
diff --git a/ee/app/workers/epics/update_cached_metadata_worker.rb b/ee/app/workers/epics/update_cached_metadata_worker.rb
new file mode 100644
index 0000000000000000000000000000000000000000..ac9e9458c7c474765949e27427420ec48c5921c2
--- /dev/null
+++ b/ee/app/workers/epics/update_cached_metadata_worker.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Epics
+ class UpdateCachedMetadataWorker
+ include ApplicationWorker
+
+ BATCH_SIZE = 100
+
+ data_consistency :delayed
+ idempotent!
+ queue_namespace :epics
+ feature_category :portfolio_management
+
+ def perform(ids)
+ return unless Feature.enabled?(:cache_issue_sums)
+
+ ::Epic.id_in(ids).find_each do |epic|
+ update_epic(epic)
+ end
+ end
+
+ private
+
+ def update_epic(epic)
+ total_sums = epic.total_issue_weight_and_count
+ epic.assign_attributes(total_sums)
+ epic.save!(touch: false)
+ end
+ end
+end
diff --git a/ee/spec/models/epic_issue_spec.rb b/ee/spec/models/epic_issue_spec.rb
index e80af54d3e4e44526d1b35341e6a997617f445a0..28804c2468452c8011d604cc03e0c33f95622017 100644
--- a/ee/spec/models/epic_issue_spec.rb
+++ b/ee/spec/models/epic_issue_spec.rb
@@ -120,4 +120,43 @@ def as_item(item)
end
end
end
+
+ describe '#update_cached_metadata' do
+ it 'schedules cache update for epic when new issue is added' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([epic.id]).once
+
+ create(:epic_issue, epic: epic, issue: issue)
+ end
+
+ context 'when epic issue already exists' do
+ let_it_be_with_reload(:epic_issue) { create(:epic_issue, epic: epic, issue: issue) }
+
+ it 'schedules cache update for epic when epic issue is updated' do
+ new_epic = create(:epic, group: group)
+
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([epic.id]).once
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([new_epic.id]).once
+
+ epic_issue.update!(epic: new_epic)
+ end
+
+ it 'schedules cache update for epic when epic issue is destroyed' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([epic.id]).once
+
+ epic_issue.destroy!
+ end
+
+ context 'when cache_issue_sums flag is disabled' do
+ before do
+ stub_feature_flags(cache_issue_sums: false)
+ end
+
+ it 'does nothing' do
+ expect(::Epics::UpdateCachedMetadataWorker).not_to receive(:perform_async)
+
+ epic_issue.destroy!
+ end
+ end
+ end
+ end
end
diff --git a/ee/spec/models/epic_spec.rb b/ee/spec/models/epic_spec.rb
index aad1a4507e6198da3fee05e11cfa2d0160820a19..c30e84718c3d24ec9c1989f36dff18501b6ec9ce 100644
--- a/ee/spec/models/epic_spec.rb
+++ b/ee/spec/models/epic_spec.rb
@@ -1143,4 +1143,87 @@ def as_item(item)
let(:params) { { noteable: issuable } }
end
end
+
+ describe '#total_issue_weight_and_count' do
+ let_it_be(:group) { create(:group) }
+ let_it_be(:project) { create(:project, group: group) }
+ let_it_be(:epic) { create(:epic, group: group) }
+ let_it_be(:issue) { create(:issue, weight: 20, project: project) }
+ let_it_be(:issue2) { create(:issue, :closed, weight: 30, project: project) }
+ let_it_be(:epic_issue) { create(:epic_issue, epic: epic, issue: issue) }
+ let_it_be(:epic_issue2) { create(:epic_issue, epic: epic, issue: issue2) }
+
+ let_it_be(:subepic) do
+ create(:epic, parent: epic, group: group, total_opened_issue_weight: 10,
+ total_closed_issue_weight: 20, total_opened_issue_count: 2,
+ total_closed_issue_count: 3)
+ end
+
+ it 'returns hash of total issue weight and count including its subepics' do
+ expect(epic.total_issue_weight_and_count).to eq(
+ {
+ total_opened_issue_weight: 30,
+ total_closed_issue_weight: 50,
+ total_opened_issue_count: 3,
+ total_closed_issue_count: 4
+ }
+ )
+ end
+ end
+
+ describe '#update_cached_metadata' do
+ let_it_be(:parent_epic) { create(:epic, group: group) }
+
+ it 'schedules cache update for parent epic when new subepic is created' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([parent_epic.id]).once
+
+ create(:epic, parent: parent_epic, group: group)
+ end
+
+ context 'when adding existing subepic' do
+ let_it_be_with_reload(:subepic) { create(:epic, group: group) }
+
+ it 'schedules cache update for parent epic' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([parent_epic.id]).once
+
+ subepic.update!(parent: parent_epic)
+ end
+ end
+
+ context 'when epic is already assigned to other epic' do
+ let_it_be(:old_parent) { create(:epic, group: group) }
+ let_it_be_with_reload(:subepic) { create(:epic, group: group, parent: old_parent) }
+
+ it 'schedules cache update for old parent and new parent epics' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([parent_epic.id]).once
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([old_parent.id]).once
+
+ subepic.update!(parent: parent_epic)
+ end
+
+ it 'schedules cache update for parent epic when removing subepic parent' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([old_parent.id]).once
+
+ subepic.update!(parent: nil)
+ end
+
+ it 'schedules cache update for parent epic when subepic is destroyed' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([old_parent.id]).once
+
+ subepic.destroy!
+ end
+
+ context 'when cache_issue_sums flag is disabled' do
+ before do
+ stub_feature_flags(cache_issue_sums: false)
+ end
+
+ it 'does nothing' do
+ expect(::Epics::UpdateCachedMetadataWorker).not_to receive(:perform_async)
+
+ subepic.destroy!
+ end
+ end
+ end
+ end
end
diff --git a/ee/spec/models/issue_spec.rb b/ee/spec/models/issue_spec.rb
index bddeea6f0e3f88a01c6349075b33d25731d19a7a..71ada60898a91574a333d402240b45f47b07c4b8 100644
--- a/ee/spec/models/issue_spec.rb
+++ b/ee/spec/models/issue_spec.rb
@@ -1188,4 +1188,74 @@
.to contain_exactly(issue, test_case)
end
end
+
+ describe '#update_cached_metadata' do
+ let_it_be(:group) { create(:group) }
+ let_it_be(:project) { create(:project, group: group) }
+ let_it_be(:epic) { create(:epic, group: group) }
+
+ before do
+ stub_licensed_features(epics: true)
+ end
+
+ context 'when epic is not assigned' do
+ let(:issue) { build(:issue, project: project) }
+
+ it 'does nothing' do
+ expect(::Epics::UpdateCachedMetadataWorker).not_to receive(:perform_async)
+
+ issue.save!
+ end
+ end
+
+ context 'when creating new issue' do
+ let(:issue) { build(:issue, project: project, epic: epic) }
+
+ it 'schedules cache update for epic' do
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([epic.id]).once
+
+ issue.save!
+ end
+
+ context 'when cache_issue_sums flag is disabled' do
+ before do
+ stub_feature_flags(cache_issue_sums: false)
+ end
+
+ it 'does nothing' do
+ expect(::Epics::UpdateCachedMetadataWorker).not_to receive(:perform_async)
+
+ issue.save!
+ end
+ end
+ end
+
+ context 'when updating an existing issue' do
+ let_it_be(:issue) { create(:issue, project: project, epic: epic) }
+
+ it 'schedules cache update for epic if state is changed' do
+ issue.state = :closed
+
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([epic.id]).once
+
+ issue.save!
+ end
+
+ it 'schedules cache update for epic if weight is changed' do
+ issue.weight = 2
+
+ expect(::Epics::UpdateCachedMetadataWorker).to receive(:perform_async).with([epic.id]).once
+
+ issue.save!
+ end
+
+ it 'does nothing when unrelated attributes are changed' do
+ issue.title = 'new title'
+
+ expect(::Epics::UpdateCachedMetadataWorker).not_to receive(:perform_async)
+
+ issue.save!
+ end
+ end
+ end
end
diff --git a/ee/spec/services/ee/issues/update_service_spec.rb b/ee/spec/services/ee/issues/update_service_spec.rb
index c5eff9d146e26b477f0b146c98c66c66d002abb1..7ea85a8d642108714980a744724e17816fcfadc2 100644
--- a/ee/spec/services/ee/issues/update_service_spec.rb
+++ b/ee/spec/services/ee/issues/update_service_spec.rb
@@ -317,6 +317,10 @@ def update_issue(opts)
let(:property) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_ADDED_TO_EPIC }
end
end
+
+ it 'updates epic cache counts', :sidekiq_inline do
+ expect { subject }.to change { epic.reload.total_opened_issue_count }.by(1)
+ end
end
context 'when issue belongs to another epic' do
@@ -352,6 +356,11 @@ def update_issue(opts)
let(:property) { Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CHANGED_EPIC }
end
end
+
+ it 'updates epic cache counts', :sidekiq_inline do
+ expect { subject }.to change { epic.reload.total_opened_issue_count }.by(1)
+ .and(change { epic2.reload.total_opened_issue_count }.by(-1))
+ end
end
context 'when updating issue epic and milestone and assignee attributes' do
diff --git a/ee/spec/workers/epics/update_cached_metadata_worker_spec.rb b/ee/spec/workers/epics/update_cached_metadata_worker_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..bad53318d30cc0b2892e0aea5216b748e652e6ce
--- /dev/null
+++ b/ee/spec/workers/epics/update_cached_metadata_worker_spec.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Epics::UpdateCachedMetadataWorker do
+ describe '#perform', :sidekiq_inline do
+ let_it_be(:group) { create(:group) }
+ let_it_be(:project) { create(:project, group: group) }
+ let_it_be_with_reload(:parent_epic) { create(:epic, group: group) }
+ let_it_be_with_reload(:epic) { create(:epic, parent: parent_epic, group: group) }
+ let_it_be_with_reload(:other_epic) { create(:epic) }
+ let_it_be(:issue1) { create(:issue, weight: 20, project: project) }
+ let_it_be(:issue2) { create(:issue, weight: 30, project: project) }
+ let_it_be(:issue3) { create(:issue, :closed, weight: 10, project: project) }
+ let_it_be(:epic_issue1) { create(:epic_issue, epic: epic, issue: issue1) }
+ let_it_be(:epic_issue2) { create(:epic_issue, epic: parent_epic, issue: issue2) }
+ let_it_be(:epic_issue3) { create(:epic_issue, epic: epic, issue: issue3) }
+
+ let(:epic_ids) { [epic.id] }
+ let(:worker) { described_class.new }
+
+ subject(:perform) { worker.perform(epic_ids) }
+
+ shared_examples_for 'successful metadata update' do
+ it 'updates epic issue cached metadata and changes are propagated to ancestors', :aggregate_failures do
+ expect(epic.total_opened_issue_weight).to eq(0)
+ expect(epic.total_closed_issue_weight).to eq(0)
+ expect(epic.total_opened_issue_count).to eq(0)
+ expect(epic.total_closed_issue_count).to eq(0)
+ expect(parent_epic.total_opened_issue_weight).to eq(0)
+ expect(parent_epic.total_closed_issue_weight).to eq(0)
+ expect(parent_epic.total_opened_issue_count).to eq(0)
+ expect(parent_epic.total_closed_issue_count).to eq(0)
+
+ subject
+
+ epic.reload
+ parent_epic.reload
+ expect(epic.total_opened_issue_weight).to eq(20)
+ expect(epic.total_closed_issue_weight).to eq(10)
+ expect(epic.total_opened_issue_count).to eq(1)
+ expect(epic.total_closed_issue_count).to eq(1)
+ expect(parent_epic.total_opened_issue_weight).to eq(50)
+ expect(parent_epic.total_closed_issue_weight).to eq(10)
+ expect(parent_epic.total_opened_issue_count).to eq(2)
+ expect(parent_epic.total_closed_issue_count).to eq(1)
+ end
+ end
+
+ it_behaves_like 'successful metadata update'
+
+ include_examples 'an idempotent worker' do
+ let(:job_args) { epic_ids }
+
+ it_behaves_like 'successful metadata update'
+ end
+
+ context 'when cache_issue_sums flag is disabled' do
+ before do
+ stub_feature_flags(cache_issue_sums: false)
+ end
+
+ it 'does nothing' do
+ expect(worker).not_to receive(:update_epic)
+
+ perform
+ end
+ end
+
+ context 'when epic id not found' do
+ let(:epic_ids) { [non_existing_record_id] }
+
+ it 'does nothing' do
+ expect(worker).not_to receive(:update_epic)
+
+ perform
+ end
+ end
+
+ context 'when multiple epic ids are passed' do
+ let(:epic_ids) { [epic.id, other_epic.id] }
+
+ it 'updates epic issue cached metadata for each epic' do
+ expect(worker).to receive(:update_epic).with(epic)
+ expect(worker).to receive(:update_epic).with(other_epic)
+
+ perform
+ end
+ end
+ end
+end