Merge branch 'sk/515866-resync-links' into 'master'

Add background migration to resync policy project links

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181164



Merged-by: Marcos Rocha <mrocha@gitlab.com>
Approved-by: Martin Čavoj <mcavoj@gitlab.com>
Approved-by: Dominic Bauer <dbauer@gitlab.com>
Approved-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: Marcos Rocha <mrocha@gitlab.com>
Reviewed-by: Dominic Bauer <dbauer@gitlab.com>
Reviewed-by: Martin Čavoj <mcavoj@gitlab.com>
Co-authored-by: Sashi Kumar Kumaresan <skumar@gitlab.com>

db/docs/batched_background_migrations/sync_unlinked_security_policy_project_links.yml

+---
+migration_job_name: SyncUnlinkedSecurityPolicyProjectLinks
+description: Creates the missing SecurityPolicyProjectLink records for existing security policies without project links.
+feature_category: security_policy_management
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181164
+milestone: '17.10'
+queued_migration_version: 20250211095858
+finalized_by: # version of the migration that finalized this BBM

db/post_migrate/20250211095858_queue_sync_unlinked_security_policy_project_links.rb

+# frozen_string_literal: true
+
+class QueueSyncUnlinkedSecurityPolicyProjectLinks < Gitlab::Database::Migration[2.2]
+  milestone '17.10'
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  MIGRATION = "SyncUnlinkedSecurityPolicyProjectLinks"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :security_policies,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :security_policies, :id, [])
+  end
+end

db/schema_migrations/20250211095858

+fdb0f72e49d467d354b2c1f51e24df8740aee8c53cf6ead1c38fbf88d95fa9aa
\ No newline at end of file

ee/lib/ee/gitlab/background_migration/sync_unlinked_security_policy_project_links.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module SyncUnlinkedSecurityPolicyProjectLinks
+        extend ActiveSupport::Concern
+        extend ::Gitlab::Utils::Override
+
+        prepended do
+          operation_name :sync_unlinked_security_policy_project_links
+        end
+
+        class Namespace < ::ApplicationRecord
+          self.table_name = 'namespaces'
+          self.inheritance_column = :_type_disabled
+
+          belongs_to :parent,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Namespace'
+        end
+
+        class ProjectNamespace < ::ApplicationRecord
+          self.table_name = 'namespaces'
+          self.inheritance_column = :_type_disabled
+
+          PROJECT_STI_NAME = 'Project'
+        end
+
+        class Group < ::ApplicationRecord
+          self.table_name = 'namespaces'
+          self.inheritance_column = :_type_disabled
+
+          include ::Namespaces::Traversal::Linear
+          include ::Namespaces::Traversal::Cached
+
+          GROUP_STI_NAME = 'Group'
+        end
+
+        class Project < ::ApplicationRecord
+          self.table_name = 'projects'
+
+          belongs_to :parent,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Namespace'
+
+          has_many :compliance_framework_settings, class_name:
+            '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::ComplianceFrameworkSetting',
+            inverse_of: :project
+          belongs_to :namespace,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Namespace'
+          belongs_to :group, -> { where(type: Group::GROUP_STI_NAME) },
+            foreign_key: 'namespace_id',
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Group'
+
+          # rubocop:disable Database/AvoidUsingPluckWithoutLimit -- this is also used in the original class:
+          # https://gitlab.com/gitlab-org/gitlab/-/blob/f4b8d8936ffe111af417b1e2f36d9aba4462a53b/ee/app/models/ee/project.rb?page=2#L1404
+          def compliance_framework_ids
+            compliance_framework_settings.pluck(:framework_id)
+          end
+          # rubocop:enable Database/AvoidUsingPluckWithoutLimit
+        end
+
+        class ComplianceFrameworkSetting < ::ApplicationRecord
+          self.table_name = 'project_compliance_framework_settings'
+
+          belongs_to :project,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Project'
+        end
+
+        class SecurityOrchestrationPolicyConfiguration < ::ApplicationRecord
+          include ::Gitlab::Utils::StrongMemoize
+
+          self.table_name = 'security_orchestration_policy_configurations'
+
+          belongs_to :project,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Project',
+            optional: true
+          belongs_to :namespace,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::Group',
+            optional: true
+          has_many :security_policies,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::SecurityPolicy'
+
+          def all_projects
+            if namespace_id.present?
+              projects = []
+              cursor = { current_id: namespace_id, depth: [namespace_id] }
+              iterator = ::Gitlab::Database::NamespaceEachBatch.new(namespace_class: ::Namespace, cursor: cursor)
+
+              iterator.each_batch(of: 1000) do |ids|
+                namespace_ids = ProjectNamespace.where(id: ids, type: ProjectNamespace::PROJECT_STI_NAME)
+                projects.concat(Project.where(project_namespace_id: namespace_ids))
+              end
+              projects
+            else
+              Array.wrap(Project.find(project_id))
+            end
+          end
+          strong_memoize_attr :all_projects
+        end
+
+        class SecurityPolicy < ::ApplicationRecord
+          self.table_name = 'security_policies'
+          self.inheritance_column = :_type_disabled
+
+          enum type: {
+            approval_policy: 0,
+            scan_execution_policy: 1,
+            pipeline_execution_policy: 2,
+            vulnerability_management_policy: 3
+          }, _prefix: true
+
+          # rubocop:disable Layout/LineLength -- the name is long
+          belongs_to :security_orchestration_policy_configuration, class_name:
+            '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::SecurityOrchestrationPolicyConfiguration'
+          # rubocop:enable Layout/LineLength
+          has_many :approval_policy_rules,
+            class_name: '::EE::Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks::ApprovalPolicyRule'
+        end
+
+        class SecurityPolicyProjectLink < ::ApplicationRecord
+          self.table_name = 'security_policy_project_links'
+        end
+
+        class ApprovalPolicyRule < ::ApplicationRecord
+          self.table_name = 'approval_policy_rules'
+          self.inheritance_column = :_type_disabled
+
+          enum type: { scan_finding: 0, license_finding: 1, any_merge_request: 2 }, _prefix: true
+        end
+
+        class ApprovalPolicyRuleProjectLink < ::ApplicationRecord
+          self.table_name = 'approval_policy_rule_project_links'
+        end
+
+        # This is copied from Security::SecurityOrchestrationPolicies::PolicyScopeChecker
+        class PolicyScopeChecker
+          def initialize(project:)
+            @project = project
+          end
+
+          def security_policy_applicable?(security_policy)
+            return false if security_policy.blank?
+            return true if security_policy.scope.blank?
+
+            scope_applicable?(security_policy.scope.deep_symbolize_keys)
+          end
+
+          private
+
+          attr_accessor :project
+
+          def scope_applicable?(policy_scope)
+            applicable_for_compliance_framework?(policy_scope) &&
+              applicable_for_project?(policy_scope) &&
+              applicable_for_group?(policy_scope)
+          end
+
+          def applicable_for_compliance_framework?(policy_scope)
+            policy_scope_compliance_frameworks = policy_scope[:compliance_frameworks].to_a
+            return true if policy_scope_compliance_frameworks.blank?
+
+            compliance_framework_ids = project.compliance_framework_ids
+            return false if compliance_framework_ids.blank?
+
+            policy_scope_compliance_frameworks.any? { |framework| framework[:id].in?(compliance_framework_ids) }
+          end
+
+          def applicable_for_project?(policy_scope)
+            policy_scope_included_projects = policy_scope.dig(:projects, :including).to_a
+            policy_scope_excluded_projects = policy_scope.dig(:projects, :excluding).to_a
+
+            return false if policy_scope_excluded_projects.any? do |policy_project|
+              policy_project[:id] == project.id
+            end
+
+            return true if policy_scope_included_projects.blank?
+
+            policy_scope_included_projects.any? { |policy_project| policy_project[:id] == project.id }
+          end
+
+          def applicable_for_group?(policy_scope)
+            policy_scope_included_groups = policy_scope.dig(:groups, :including).to_a
+            policy_scope_excluded_groups = policy_scope.dig(:groups, :excluding).to_a
+
+            return true if policy_scope_included_groups.blank? && policy_scope_excluded_groups.blank?
+
+            ancestor_group_ids = project.group&.self_and_ancestor_ids.to_a
+
+            return false if policy_scope_excluded_groups.any? do |policy_group|
+              policy_group[:id].in?(ancestor_group_ids)
+            end
+
+            return true if policy_scope_included_groups.blank?
+
+            policy_scope_included_groups.any? { |policy_group| policy_group[:id].in?(ancestor_group_ids) }
+          end
+        end
+
+        override :perform
+        def perform
+          each_sub_batch do |sub_batch|
+            SecurityPolicy.id_in(sub_batch.where(enabled: true)).each do |security_policy|
+              process_security_policy(security_policy)
+            end
+          end
+        end
+
+        private
+
+        def process_security_policy(security_policy)
+          policy_configuration = security_policy.security_orchestration_policy_configuration
+          applicable_project_ids = find_applicable_project_ids(policy_configuration, security_policy)
+
+          applicable_project_ids.each_slice(1000) do |project_ids|
+            create_policy_links(security_policy, project_ids)
+            create_approval_policy_links(security_policy, project_ids) if security_policy.type_approval_policy?
+          end
+        end
+
+        def find_applicable_project_ids(policy_configuration, security_policy)
+          policy_configuration.all_projects.select do |project|
+            PolicyScopeChecker.new(project: project).security_policy_applicable?(security_policy)
+          end.map(&:id)
+        end
+
+        def create_policy_links(security_policy, project_ids)
+          records = project_ids.map do |project_id|
+            { security_policy_id: security_policy.id, project_id: project_id }
+          end
+
+          SecurityPolicyProjectLink.insert_all(records, unique_by: [:security_policy_id, :project_id])
+        end
+
+        def create_approval_policy_links(security_policy, project_ids)
+          policy_rules_records = security_policy.approval_policy_rules.flat_map do |rule|
+            project_ids.map do |project_id|
+              { approval_policy_rule_id: rule.id, project_id: project_id }
+            end
+          end
+
+          ApprovalPolicyRuleProjectLink.insert_all(policy_rules_records,
+            unique_by: [:approval_policy_rule_id, :project_id])
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/sync_unlinked_security_policy_project_links_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks, feature_category: :security_policy_management do
+  let(:security_orchestration_policy_configurations) { table(:security_orchestration_policy_configurations) }
+  let(:namespaces) { table(:namespaces) }
+  let(:security_policies) { table(:security_policies) }
+  let(:approval_policy_rules) { table(:approval_policy_rules) }
+  let(:security_policy_project_links) { table(:security_policy_project_links) }
+  let(:approval_policy_rule_project_links) { table(:approval_policy_rule_project_links) }
+  let(:compliance_management_frameworks) { table(:compliance_management_frameworks) }
+  let(:compliance_framework_project_settings) { table(:project_compliance_framework_settings) }
+
+  let(:args) do
+    min, max = security_policies.pick('MIN(id)', 'MAX(id)')
+
+    {
+      start_id: min,
+      end_id: max,
+      batch_table: 'security_policies',
+      batch_column: 'id',
+      sub_batch_size: 100,
+      pause_ms: 0,
+      connection: ApplicationRecord.connection
+    }
+  end
+
+  let!(:organization) { table(:organizations).create!(name: 'organization', path: 'organization') }
+
+  let!(:group_namespace) do
+    namespaces.create!(
+      organization_id: organization.id,
+      name: 'gitlab-org',
+      path: 'gitlab-org',
+      type: 'Group'
+    ).tap { |namespace| namespace.update!(traversal_ids: [namespace.id]) }
+  end
+
+  let(:policy_scope) { {} }
+
+  let!(:project) { create_project('project', group_namespace) }
+  let!(:policy_project) { create_project('policy_project', group_namespace) }
+
+  let!(:security_policy_config) do
+    security_orchestration_policy_configurations.create!(
+      security_policy_management_project_id: policy_project.id,
+      namespace_id: group_namespace.id
+    )
+  end
+
+  subject(:perform_migration) { described_class.new(**args).perform }
+
+  shared_examples_for 'creates only policy project links' do
+    it 'creates only policy project links' do
+      perform_migration
+
+      expect(security_policy_project_links.where(security_policy_id: policy.id, project_id: project.id)).to exist
+      expect(approval_policy_rule_project_links.count).to eq(0)
+    end
+  end
+
+  describe '#perform' do
+    let(:policy_hash) do
+      {
+        name: 'Test Policy',
+        description: 'Test Description',
+        enabled: true,
+        metadata: {},
+        policy_scope: policy_scope
+      }
+    end
+
+    context 'with project level config' do
+      let!(:security_policy_config) do
+        security_orchestration_policy_configurations.create!(
+          security_policy_management_project_id: policy_project.id,
+          project_id: project.id
+        )
+      end
+
+      let!(:policy) { create_policy(:scan_execution_policy, policy_hash, 0) }
+
+      it_behaves_like 'creates only policy project links'
+    end
+
+    context 'with approval_policy_rules' do
+      let!(:project_2) { create_project('project_2', group_namespace) }
+      let!(:policy) { create_policy(:approval_policy, policy_hash, 0) }
+      let!(:approval_policy_rule_1) do
+        approval_policy_rules.create!(
+          security_policy_id: policy.id,
+          type: 0,
+          rule_index: 0,
+          security_policy_management_project_id: policy_project.id
+        )
+      end
+
+      let!(:approval_policy_rule_2) do
+        approval_policy_rules.create!(
+          security_policy_id: policy.id,
+          type: 0,
+          rule_index: 1,
+          security_policy_management_project_id: policy_project.id
+        )
+      end
+
+      before do
+        security_policy_project_links.create!(security_policy_id: policy.id, project_id: policy_project.id)
+        approval_policy_rule_project_links.create!(approval_policy_rule_id: approval_policy_rule_1.id,
+          project_id: policy_project.id)
+        approval_policy_rule_project_links.create!(approval_policy_rule_id: approval_policy_rule_2.id,
+          project_id: policy_project.id)
+      end
+
+      it 'creates policy and rule project links' do
+        expect { perform_migration }.to change { security_policy_project_links.count }.by(2)
+          .and change { approval_policy_rule_project_links.count }.by(4)
+
+        expect(security_policy_project_links.where(security_policy_id: policy.id, project_id: project.id)).to exist
+        expect(security_policy_project_links.where(security_policy_id: policy.id, project_id: project_2.id)).to exist
+
+        expect(approval_policy_rule_project_links.where(approval_policy_rule_id: approval_policy_rule_1.id,
+          project_id: project.id)).to exist
+        expect(approval_policy_rule_project_links.where(approval_policy_rule_id: approval_policy_rule_1.id,
+          project_id: project_2.id)).to exist
+
+        expect(approval_policy_rule_project_links.where(approval_policy_rule_id: approval_policy_rule_2.id,
+          project_id: project.id)).to exist
+        expect(approval_policy_rule_project_links.where(approval_policy_rule_id: approval_policy_rule_2.id,
+          project_id: project_2.id)).to exist
+      end
+
+      context 'when links already exist' do
+        before do
+          security_policy_project_links.create!(security_policy_id: policy.id, project_id: project.id)
+          approval_policy_rule_project_links.create!(approval_policy_rule_id: approval_policy_rule_1.id,
+            project_id: project.id)
+          approval_policy_rule_project_links.create!(approval_policy_rule_id: approval_policy_rule_2.id,
+            project_id: project.id)
+        end
+
+        it 'creates only the missing links' do
+          expect { perform_migration }.to change { security_policy_project_links.count }.by(1)
+          .and change { approval_policy_rule_project_links.count }.by(2)
+
+          expect(security_policy_project_links.where(security_policy_id: policy.id, project_id: project.id)).to exist
+
+          expect(approval_policy_rule_project_links.where(approval_policy_rule_id: approval_policy_rule_1.id,
+            project_id: project_2.id)).to exist
+          expect(approval_policy_rule_project_links.where(approval_policy_rule_id: approval_policy_rule_2.id,
+            project_id: project_2.id)).to exist
+        end
+      end
+    end
+
+    context 'without approval_policy_rules' do
+      let!(:policy) { create_policy(:scan_execution_policy, policy_hash, 0) }
+
+      it_behaves_like 'creates only policy project links'
+    end
+
+    context 'with policy scopes' do
+      let!(:another_project) { create_project('another_project', group_namespace) }
+      let!(:policy) { create_policy(:scan_execution_policy, policy_hash, 0) }
+
+      let!(:sub_group_namespace) do
+        namespaces.create!(
+          organization_id: organization.id,
+          name: 'gitlab-com',
+          path: 'gitlab-com',
+          type: 'Group',
+          parent_id: group_namespace.id
+        ).tap { |namespace| namespace.update!(traversal_ids: [group_namespace.id, namespace.id]) }
+      end
+
+      let!(:project_in_sub_group) { create_project('project in subgroup', sub_group_namespace) }
+
+      context 'with project scope' do
+        let(:policy_scope) do
+          {
+            projects: {
+              including: [{ id: project.id }],
+              excluding: [{ id: another_project.id }]
+            }
+          }
+        end
+
+        it 'creates links only for included projects' do
+          perform_migration
+
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: project.id)).to exist
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: another_project.id)).not_to exist
+        end
+      end
+
+      context 'when policy is scoped to compliance framework' do
+        let!(:compliance_management_framework) do
+          compliance_management_frameworks.create!(
+            name: 'name',
+            color: '#000000',
+            description: 'description',
+            namespace_id: group_namespace.id
+          )
+        end
+
+        let!(:compliance_framework_project_setting) do
+          compliance_framework_project_settings.create!(
+            project_id: project.id,
+            framework_id: compliance_management_framework.id
+          )
+        end
+
+        let(:policy_scope) do
+          {
+            compliance_frameworks: [
+              { id: compliance_management_framework.id }
+            ]
+          }
+        end
+
+        it 'creates links only for the project in scope', :aggregate_failures do
+          perform_migration
+
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: project.id)).to exist
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: another_project.id)).not_to exist
+        end
+      end
+
+      context 'when policy is scoped to a group' do
+        let(:policy_scope) do
+          {
+            groups: {
+              including: [
+                { id: sub_group_namespace.id }
+              ]
+            }
+          }
+        end
+
+        it 'creates links only for the project in scope', :aggregate_failures do
+          perform_migration
+
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: project_in_sub_group.id)).to exist
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: another_project.id)).not_to exist
+        end
+      end
+
+      context 'when policy is unscoped to a group' do
+        let(:policy_scope) do
+          {
+            groups: {
+              excluding: [
+                { id: sub_group_namespace.id }
+              ]
+            }
+          }
+        end
+
+        it 'creates links only for the project in scope', :aggregate_failures do
+          perform_migration
+
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: project_in_sub_group.id)).not_to exist
+          expect(security_policy_project_links.where(security_policy_id: policy.id,
+            project_id: another_project.id)).to exist
+        end
+      end
+    end
+
+    context 'when policy is disabled' do
+      let!(:policy) { create_policy(:scan_execution_policy, policy_hash.merge(enabled: false), 0) }
+
+      it 'does not create any links' do
+        expect { perform_migration }.not_to change { security_policy_project_links.count }
+      end
+    end
+  end
+
+  def create_project(name, group)
+    project_namespace = namespaces.create!(
+      name: name,
+      path: name,
+      type: 'Project',
+      parent_id: group.id,
+      organization_id: group.organization_id
+    ).tap { |namespace| namespace.update!(traversal_ids: [*group.traversal_ids, namespace.id]) }
+
+    table(:projects).create!(
+      organization_id: group.organization_id,
+      namespace_id: group.id,
+      project_namespace_id: project_namespace.id,
+      name: name,
+      path: name
+    )
+  end
+
+  def create_policy(policy_type, policy_hash, policy_index)
+    security_policies.create!(
+      {
+        type: described_class::SecurityPolicy.types[policy_type],
+        policy_index: policy_index,
+        name: policy_hash[:name],
+        description: policy_hash[:description],
+        enabled: policy_hash[:enabled],
+        metadata: policy_hash.fetch(:metadata, {}),
+        scope: policy_hash.fetch(:policy_scope, {}),
+        content: policy_hash.fetch(:content, {}),
+        checksum: Digest::SHA256.hexdigest(policy_hash.to_json),
+        security_orchestration_policy_configuration_id: security_policy_config.id,
+        security_policy_management_project_id: policy_project.id
+      }
+    )
+  end
+end

lib/gitlab/background_migration/sync_unlinked_security_policy_project_links.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class SyncUnlinkedSecurityPolicyProjectLinks < BatchedMigrationJob
+      feature_category :security_policy_management
+
+      def perform; end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::SyncUnlinkedSecurityPolicyProjectLinks.prepend_mod

spec/migrations/20250211095858_queue_sync_unlinked_security_policy_project_links_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueSyncUnlinkedSecurityPolicyProjectLinks, migration: :gitlab_main_cell, feature_category: :security_policy_management do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          gitlab_schema: :gitlab_main,
+          table_name: :security_policies,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      }
+    end
+  end
+end