Merge branch '416261-migration-sync-policies-to-scan_result_policy_read' into 'master'

Add migration to sync ScanResultPolicyRead

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/128378



Merged-by: Mayra Cabrera <mcabrera@gitlab.com>
Approved-by: Rajendra Kadam <rkadam@gitlab.com>
Approved-by: Mohamed Hamda <mhamda@gitlab.com>
Approved-by: Dominic Bauer <dbauer@gitlab.com>
Approved-by: Mayra Cabrera <mcabrera@gitlab.com>
Reviewed-by: Mayra Cabrera <mcabrera@gitlab.com>
Reviewed-by: Rajendra Kadam <rkadam@gitlab.com>
Reviewed-by: Mohamed Hamda <mhamda@gitlab.com>
Reviewed-by: Martin Čavoj <mcavoj@gitlab.com>
Co-authored-by: Martin Čavoj <mcavoj@gitlab.com>
Co-authored-by: Andy Soiron <asoiron@gitlab.com>

db/docs/batched_background_migrations/sync_scan_result_policies.yml

+---
+migration_job_name: SyncScanResultPolicies
+description: Security policies are stored as YAML files in the security policy project. This migration
+  syncs existing policies to a DB table scan_result_policies by kicking off Sidekiq sync job
+  for all security policy configurations.
+feature_category: security_policy_management
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/128378
+milestone: 16.4

db/post_migrate/20230831084632_queue_sync_scan_result_policies.rb

+# frozen_string_literal: true
+
+class QueueSyncScanResultPolicies < Gitlab::Database::Migration[2.1]
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  MIGRATION = "SyncScanResultPolicies"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :security_orchestration_policy_configurations,
+      :id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :security_orchestration_policy_configurations, :id, [])
+  end
+end

db/schema_migrations/20230831084632

+f523e00eeac359503976360c824582e8c9bd0e82cbf28ca02df17d0adacc90d8
\ No newline at end of file

ee/lib/ee/gitlab/background_migration/sync_scan_result_policies.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module SyncScanResultPolicies
+        extend ActiveSupport::Concern
+        extend ::Gitlab::Utils::Override
+
+        prepended do
+          operation_name :sync_scan_result_policies
+        end
+
+        override :perform
+        def perform
+          each_sub_batch do |sub_batch|
+            sub_batch.pluck(:id).each do |config_id|
+              ::Security::SyncScanPoliciesWorker.perform_async(config_id)
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/sync_scan_result_policies_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::SyncScanResultPolicies, feature_category: :security_policy_management do
+  describe '#perform' do
+    let(:batch_table) { :security_orchestration_policy_configurations }
+    let(:batch_column) { :id }
+    let(:sub_batch_size) { 1 }
+    let(:pause_ms) { 0 }
+    let(:connection) { ApplicationRecord.connection }
+
+    let(:projects) { table(:projects) }
+    let(:namespaces) { table(:namespaces) }
+    let(:security_orchestration_policy_configurations) { table(:security_orchestration_policy_configurations) }
+
+    let(:group_namespace) do
+      namespaces.create!(name: 'group_1', path: 'group_1', type: 'Group').tap do |group|
+        group.update!(traversal_ids: [group.id])
+      end
+    end
+
+    let(:project_namespace_1) { namespaces.create!(name: '1', path: '1', type: 'Project', parent_id: group_namespace) }
+    let(:project_namespace_2) { namespaces.create!(name: '2', path: '2', type: 'Project', parent_id: group_namespace) }
+    let(:project_namespace_3) { namespaces.create!(name: '3', path: '3', type: 'Project', parent_id: group_namespace) }
+
+    let(:policy_project_namespace) { namespaces.create!(name: '4', path: '4', type: 'Project') }
+    let(:policy_project) do
+      projects.create!(
+        name: 'Policy Project',
+        namespace_id: policy_project_namespace.id,
+        project_namespace_id: policy_project_namespace.id
+      )
+    end
+
+    let(:project_1) { projects.create!(namespace_id: group_namespace.id, project_namespace_id: project_namespace_1.id) }
+    let(:project_2) { projects.create!(namespace_id: group_namespace.id, project_namespace_id: project_namespace_2.id) }
+    let(:project_3) { projects.create!(namespace_id: group_namespace.id, project_namespace_id: project_namespace_3.id) }
+
+    let(:project_policy_configuration) { create_policy_configuration(project_id: project_1.id) }
+    let(:project_policy_configuration_2) { create_policy_configuration(project_id: project_2.id) }
+    let(:namespace_policy_configuration) { create_policy_configuration(namespace_id: group_namespace.id) }
+
+    subject(:perform) do
+      described_class.new(
+        start_id: project_policy_configuration.id,
+        end_id: namespace_policy_configuration.id,
+        batch_table: batch_table,
+        batch_column: batch_column,
+        sub_batch_size: sub_batch_size,
+        pause_ms: pause_ms,
+        connection: connection
+      ).perform
+    end
+
+    it 'enqueues Security::SyncScanPoliciesWorker for each project of policy configuration' do
+      expect(Security::SyncScanPoliciesWorker).to receive(:perform_async).with(project_policy_configuration.id)
+      expect(Security::SyncScanPoliciesWorker).to receive(:perform_async).with(project_policy_configuration_2.id)
+      expect(Security::SyncScanPoliciesWorker).to receive(:perform_async).with(namespace_policy_configuration.id)
+
+      perform
+    end
+
+    def create_policy_configuration(project_id: nil, namespace_id: nil)
+      security_orchestration_policy_configurations.create!(
+        project_id: project_id,
+        namespace_id: namespace_id,
+        security_policy_management_project_id: policy_project.id
+      )
+    end
+  end
+end

lib/gitlab/background_migration/sync_scan_result_policies.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Background migration to sync scan result policies from YAML to DB table by kicking off sync Sidekiq jobs
+    class SyncScanResultPolicies < BatchedMigrationJob
+      feature_category :security_policy_management
+
+      def perform; end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::SyncScanResultPolicies.prepend_mod

spec/migrations/20230831084632_queue_sync_scan_result_policies_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueSyncScanResultPolicies, feature_category: :security_policy_management do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :security_orchestration_policy_configurations,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      }
+    end
+  end
+end